Improvements

Signed-off-by: Juan Cruz Viotti <jv@jviotti.com>

Author: jviotti

src/core/yaml/CMakeLists.txt

@@ -7,5 +7,6 @@ if(SOURCEMETA_CORE_INSTALL)
 endif()
 
 target_link_libraries(sourcemeta_core_yaml PUBLIC sourcemeta::core::json)
+target_link_libraries(sourcemeta_core_yaml PUBLIC sourcemeta::core::jsonpointer)
 target_link_libraries(sourcemeta_core_yaml PRIVATE sourcemeta::core::io)
 target_link_libraries(sourcemeta_core_yaml PRIVATE sourcemeta::core::unicode)

src/core/yaml/include/sourcemeta/core/yaml_roundtrip.h

@@ -5,14 +5,62 @@
 #include <sourcemeta/core/yaml_export.h>
 #endif
 
+#include <sourcemeta/core/json.h>
+#include <sourcemeta/core/jsonpointer.h>
+
+#include <cstdint>  // std::uint8_t, std::size_t
+#include <map>      // std::map
+#include <optional> // std::optional
+#include <string>   // std::string
+#include <vector>   // std::vector
+
 namespace sourcemeta::core {
 
+/// @ingroup yaml
+enum class YAMLScalarStyle : std::uint8_t {
+  Plain,
+  SingleQuoted,
+  DoubleQuoted,
+  Literal,
+  Folded
+};
+
+/// @ingroup yaml
+enum class YAMLCollectionStyle : std::uint8_t { Block, Flow };
+
+/// @ingroup yaml
+enum class YAMLChomping : std::uint8_t { Clip, Strip, Keep };
+
+/// @ingroup yaml
+struct SOURCEMETA_CORE_YAML_EXPORT YAMLNodeStyle {
+  std::optional<YAMLScalarStyle> scalar;
+  std::optional<YAMLCollectionStyle> collection;
+  std::optional<YAMLChomping> chomping;
+  std::optional<std::string> block_content;
+  std::optional<std::string> anchor;
+  std::vector<std::string> comments_before;
+  std::optional<std::string> comment_inline;
+  std::optional<std::string> comment_on_indicator;
+};
+
 /// @ingroup yaml
 /// Holds per-node metadata collected during YAML parsing to reproduce the
 /// original formatting
 class SOURCEMETA_CORE_YAML_EXPORT YAMLRoundTrip {
 public:
   auto clear() -> void;
+  std::map<Pointer, YAMLNodeStyle> styles;
+  std::map<Pointer, std::string> aliases;
+  std::map<Pointer, YAMLScalarStyle> key_styles;
+  bool explicit_document_start{false};
+  bool explicit_document_end{false};
+  std::optional<std::string> document_start_comment;
+  std::optional<std::string> document_end_comment;
+  std::vector<std::string> leading_comments;
+  std::vector<std::string> post_start_comments;
+  std::vector<std::string> pre_end_comments;
+  std::vector<std::string> trailing_comments;
+  std::size_t indent_width{2};
 };
 
 } // namespace sourcemeta::core

src/core/yaml/lexer.h

@@ -9,6 +9,7 @@
 #include <optional>    // std::optional
 #include <string>      // std::string
 #include <string_view> // std::string_view
+#include <vector>      // std::vector
 
 namespace sourcemeta::core::yaml {
 
@@ -42,22 +43,33 @@ enum class ScalarStyle : std::uint8_t {
   Folded
 };
 
+enum class BlockChomping : std::uint8_t { Clip, Strip, Keep };
+
 struct Token {
   TokenType type;
   std::string_view value;
   std::uint64_t line;
   std::uint64_t column;
   std::size_t position{0};
   ScalarStyle scalar_style{ScalarStyle::Plain};
+  BlockChomping chomping{BlockChomping::Clip};
   bool multiline{false};
+  std::string_view block_original{};
 };
 
 class Lexer {
 public:
-  Lexer(const std::string_view input) : input_{input} {}
+  Lexer(const std::string_view input, const bool roundtrip_mode = false)
+      : input_{input}, roundtrip_{roundtrip_mode} {}
 
   auto next() -> std::optional<Token> {
+    if (this->roundtrip_) {
+      this->inline_comment_buffer_.reset();
+    }
     this->skip_whitespace_and_comments();
+    if (this->roundtrip_) {
+      this->comment_reference_line_ = this->line_;
+    }
 
     if (this->position_ >= this->input_.size()) {
       if (!this->stream_started_) {
@@ -270,6 +282,24 @@ class Lexer {
     return this->position_;
   }
 
+  auto take_inline_comment() -> std::optional<std::string> {
+    auto result{std::move(this->inline_comment_buffer_)};
+    this->inline_comment_buffer_.reset();
+    return result;
+  }
+
+  auto take_preceding_comments() -> std::vector<std::string> {
+    auto result{std::move(this->preceding_comments_buffer_)};
+    this->preceding_comments_buffer_.clear();
+    return result;
+  }
+
+  auto take_block_scalar_comment() -> std::optional<std::string> {
+    auto result{std::move(this->block_scalar_comment_)};
+    this->block_scalar_comment_.reset();
+    return result;
+  }
+
 private:
   [[nodiscard]] static auto is_whitespace(const char character) noexcept
       -> bool {
@@ -312,6 +342,7 @@ class Lexer {
         (this->position_ > 0 &&
          is_whitespace(this->input_[this->position_ - 1]))};
     bool at_line_start{this->column_ == 1};
+    bool blank_line{at_line_start};
     this->tab_at_line_start_ = false;
     while (this->position_ < this->input_.size()) {
       const char current{this->peek()};
@@ -332,19 +363,43 @@ class Lexer {
       }
 
       if (current == '\n' || current == '\r') {
+        if (this->roundtrip_ && blank_line) {
+          this->preceding_comments_buffer_.emplace_back();
+        }
         this->advance(1);
         if (current == '\r' && this->peek() == '\n') {
           this->advance(1);
         }
         preceded_by_whitespace = true;
         at_line_start = true;
+        blank_line = true;
         this->tab_at_line_start_ = false;
         continue;
       }
 
       if (current == '#' && preceded_by_whitespace) {
-        while (this->position_ < this->input_.size() && this->peek() != '\n') {
-          this->advance(1);
+        blank_line = false;
+        if (this->roundtrip_) {
+          const auto comment_line{this->line_};
+          const auto comment_start{this->position_};
+          while (this->position_ < this->input_.size() &&
+                 this->peek() != '\n') {
+            this->advance(1);
+          }
+          std::string text{this->input_.substr(
+              comment_start, this->position_ - comment_start)};
+          if (comment_line == this->comment_reference_line_ &&
+              this->comment_reference_line_ > 0 &&
+              !this->inline_comment_buffer_.has_value()) {
+            this->inline_comment_buffer_ = std::move(text);
+          } else {
+            this->preceding_comments_buffer_.push_back(std::move(text));
+          }
+        } else {
+          while (this->position_ < this->input_.size() &&
+                 this->peek() != '\n') {
+            this->advance(1);
+          }
         }
         continue;
       }
@@ -942,8 +997,19 @@ class Lexer {
         seen_header_whitespace = true;
         this->advance(1);
       } else if (current == '#' && seen_header_whitespace) {
-        while (this->position_ < this->input_.size() && this->peek() != '\n') {
-          this->advance(1);
+        if (this->roundtrip_) {
+          const auto comment_start{this->position_};
+          while (this->position_ < this->input_.size() &&
+                 this->peek() != '\n') {
+            this->advance(1);
+          }
+          this->block_scalar_comment_ = std::string{this->input_.substr(
+              comment_start, this->position_ - comment_start)};
+        } else {
+          while (this->position_ < this->input_.size() &&
+                 this->peek() != '\n') {
+            this->advance(1);
+          }
         }
       } else if (current == '\n' || current == '\r') {
         break;
@@ -961,6 +1027,16 @@ class Lexer {
     }
 
     auto &buffer{this->get_buffer()};
+
+    // For folded scalars in roundtrip mode, build a parallel buffer that
+    // preserves original line breaks (literal-style) for round-trip output
+    const bool build_original{style == ScalarStyle::Folded && this->roundtrip_};
+    std::string *original{nullptr};
+    std::string original_trailing;
+    if (build_original) {
+      original = &this->get_buffer();
+    }
+
     const auto content_indent{this->detect_block_scalar_indent(
         explicit_indent, indicator_position, start_line, start_column)};
 
@@ -990,6 +1066,17 @@ class Lexer {
           trailing_newlines += '\n';
         } else {
           blank_line_count++;
+          if (original) {
+            if (line_indent > content_indent) {
+              *original += original_trailing;
+              original_trailing.clear();
+              for (std::size_t index = content_indent; index < line_indent;
+                   ++index) {
+                *original += ' ';
+              }
+            }
+            original_trailing += '\n';
+          }
         }
         this->advance(1);
         if (this->input_[this->position_ - 1] == '\r' && this->peek() == '\n') {
@@ -1042,15 +1129,27 @@ class Lexer {
         blank_line_count = 0;
         had_line_break = false;
         previous_started_with_whitespace = starts_with_whitespace;
+
+        if (original) {
+          *original += original_trailing;
+          original_trailing.clear();
+        }
       }
 
       for (std::size_t index = content_indent; index < line_indent; ++index) {
         buffer += ' ';
+        if (original) {
+          *original += ' ';
+        }
       }
 
       while (this->position_ < this->input_.size() && this->peek() != '\n' &&
              this->peek() != '\r') {
-        buffer += this->peek();
+        const auto character{this->peek()};
+        buffer += character;
+        if (original) {
+          *original += character;
+        }
         this->advance(1);
       }
 
@@ -1063,6 +1162,9 @@ class Lexer {
           trailing_newlines += '\n';
         } else {
           had_line_break = true;
+          if (original) {
+            original_trailing += '\n';
+          }
         }
         this->advance(1);
         if (this->input_[this->position_ - 1] == '\r' && this->peek() == '\n') {
@@ -1081,6 +1183,9 @@ class Lexer {
         for (std::size_t count = 0; count < blank_line_count; ++count) {
           buffer += '\n';
         }
+        if (original) {
+          *original += original_trailing;
+        }
       }
     } else if (chomping == 'c' && !buffer.empty()) {
       if (style == ScalarStyle::Literal) {
@@ -1089,14 +1194,27 @@ class Lexer {
         }
       } else if (had_line_break || blank_line_count > 0) {
         buffer += '\n';
+        if (original && !original_trailing.empty()) {
+          *original += '\n';
+        }
       }
     }
 
+    BlockChomping block_chomping{BlockChomping::Clip};
+    if (chomping == '-') {
+      block_chomping = BlockChomping::Strip;
+    } else if (chomping == '+') {
+      block_chomping = BlockChomping::Keep;
+    }
+
     return Token{.type = TokenType::Scalar,
                  .value = buffer,
                  .line = start_line,
                  .column = start_column,
-                 .scalar_style = style};
+                 .scalar_style = style,
+                 .chomping = block_chomping,
+                 .block_original = original ? std::string_view{*original}
+                                            : std::string_view{}};
   }
 
   auto scan_plain_scalar() -> Token {
@@ -1336,6 +1454,11 @@ class Lexer {
   bool stream_ended_{false};
   bool last_was_quoted_scalar_{false};
   bool tab_at_line_start_{false};
+  bool roundtrip_{false};
+  std::uint64_t comment_reference_line_{0};
+  std::optional<std::string> inline_comment_buffer_;
+  std::optional<std::string> block_scalar_comment_;
+  std::vector<std::string> preceding_comments_buffer_;
   // SIZE_MAX means "not set" (top-level), 0 means parent at indent 0
   std::size_t block_indent_{SIZE_MAX};
   std::deque<std::string> scalar_buffers_;