feat: carry serializer metadata beside the mapping (#331)

CHANGELOG.md

@@ -7,7 +7,9 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 
 ## \[Unreleased\]
 
-- Nothing yet.
+### Added
+
+- `SerializationOptions.metadata_sidecar`, which carries `__is_block__`, `__comments__` and `__inline_comments__` beside the mapping rather than among its keys. HCL reserves none of those names, so a document may declare an attribute called any of them -- and in-band one of the two has to lose: on read the marker overwrites the attribute, on write the deserializer drops it, and by then the dict holds a single value with no way to tell which happened. With the option set, `loads` returns an `HclDict`, a `dict` subclass whose `hcl_meta` holds the three, so the mapping contains attributes and nothing else. `dumps` accepts either form, including a hand-built dict using the old keys. Off by default: the keys are a documented part of the output shape, and JSON cannot carry a sidecar. `HclDict`, `HclMeta` and `meta_of` are exported from `hcl2`. Copying, merging with `|` and pickling carry the metadata; `dict(d)` and `{**d}` deliberately do not, since asking for a `dict` gives the mapping and nothing else. ([#331](https://github.com/amplify-education/python-hcl2/issues/331))
 
 ## \[8.1.3\] - 2026-08-26
 

hcl2/__init__.py

@@ -24,27 +24,31 @@
 from .builder import Builder
 from .deserializer import DeserializerOptions
 from .formatter import FormatterOptions
+from .meta import HclDict, HclMeta, meta_of
 from .rules.base import StartRule
 from .utils import SerializationOptions
 
 __all__ = [
+    "Builder",
+    "DeserializerOptions",
     "dump",
     "dumps",
+    "FormatterOptions",
     "from_dict",
     "from_json",
+    "HclDict",
+    "HclMeta",
     "load",
     "loads",
+    "meta_of",
     "parse",
     "parse_to_tree",
     "parses",
     "parses_to_tree",
     "query",
     "reconstruct",
+    "SerializationOptions",
     "serialize",
-    "transform",
-    "Builder",
-    "DeserializerOptions",
-    "FormatterOptions",
     "StartRule",
-    "SerializationOptions",
+    "transform",
 ]

hcl2/deserializer.py

@@ -10,6 +10,7 @@
 from regex import regex
 
 from hcl2.const import COMMENTS_KEY, INLINE_COMMENTS_KEY, IS_BLOCK
+from hcl2.meta import meta_of
 from hcl2.parser import parser as _get_parser
 from hcl2.rules.abstract import LarkElement, LarkRule
 from hcl2.rules.base import (
@@ -144,7 +145,7 @@ def _deserialize_block_elements(self, value: dict) -> List[LarkElement]:
 
             else:
                 # otherwise it's just an attribute
-                if not self._is_reserved_key(key):
+                if not self._is_reserved_key(key, value):
                     children.append(self._deserialize_attribute(key, val))
 
         return children
@@ -294,8 +295,8 @@ def _deserialize_block(self, first_label: str, value: dict) -> BlockRule:
         body = value
 
         # Keep peeling off single-key layers until we hit the body (dict with IS_BLOCK)
-        while isinstance(body, dict) and not body.get(IS_BLOCK):
-            non_block_keys = [k for k in body.keys() if not self._is_reserved_key(k)]
+        while isinstance(body, dict) and not self._is_marked_block(body):
+            non_block_keys = [k for k in body.keys() if not self._is_reserved_key(k, body)]
             if len(non_block_keys) == 1:
                 # This is another label level
                 label = non_block_keys[0]
@@ -367,10 +368,23 @@ def _deserialize_object_elem(self, key: Any, value: Any) -> ObjectElemRule:
 
         return ObjectElemRule(result)
 
-    def _is_reserved_key(self, key: str) -> bool:
-        """Check if a key is a reserved metadata key that should be skipped during deserialization."""
+    def _is_reserved_key(self, key: str, container: Optional[dict] = None) -> bool:
+        """Whether *key* in *container* is metadata rather than an attribute.
+
+        A container carrying its metadata beside the mapping reserves nothing:
+        every key in it is an attribute the document declared, including one
+        spelled `__is_block__`. Only the in-band form has to reserve the names,
+        and only there can it lose an attribute to one.
+        """
+        if container is not None and meta_of(container) is not None:
+            return False
         return key in (IS_BLOCK, COMMENTS_KEY, INLINE_COMMENTS_KEY)
 
+    def _is_marked_block(self, body: dict) -> bool:
+        """Whether *body* is itself a block, in whichever form marks it."""
+        meta = meta_of(body)
+        return meta.is_block if meta is not None else bool(body.get(IS_BLOCK))
+
     def _is_expression(self, value: Any) -> bool:
         return isinstance(value, str) and value.startswith("${") and value.endswith("}")
 
@@ -387,8 +401,12 @@ def _is_block(self, value: Any) -> bool:
         return False
 
     def _contains_block_marker(self, obj: dict) -> bool:
-        """Recursively check if a dict contains IS_BLOCK marker anywhere"""
-        if obj.get(IS_BLOCK):
+        """Recursively check whether a dict is marked as a block, in either form"""
+        meta = meta_of(obj)
+        if meta is not None:
+            if meta.is_block:
+                return True
+        elif obj.get(IS_BLOCK):
             return True
         for value in obj.values():
             if isinstance(value, dict) and self._contains_block_marker(value):

hcl2/meta.py

@@ -0,0 +1,137 @@
+"""Out-of-band metadata for serialized bodies.
+
+The serializer has three things to say about a body that are not attributes of
+it: that it is a block, what comments surround it, and which of those were
+inline. They have always travelled as `__is_block__`, `__comments__` and
+`__inline_comments__` keys in the same dict as the attributes, which works only
+while no document declares an attribute by those names. HCL puts no such name
+out of reach, so one that does loses either the attribute or the metadata,
+silently and in both directions.
+
+`HclDict` carries them beside the mapping instead. It is a `dict`, so every
+consumer that reads attributes keeps working unchanged, and `hcl_meta` holds
+what used to sit among them.
+"""
+
+import copy as copy_module
+from dataclasses import dataclass, field
+from typing import Any, Dict, List, Optional, Tuple
+
+
+@dataclass
+class HclMeta:
+    """What the serializer knows about a body that is not one of its attributes."""
+
+    is_block: bool = False
+    comments: List[dict] = field(default_factory=list)
+    inline_comments: List[dict] = field(default_factory=list)
+
+    def is_empty(self) -> bool:
+        """Whether there is nothing here worth carrying."""
+        return not (self.is_block or self.comments or self.inline_comments)
+
+
+class HclDict(Dict[str, Any]):
+    """A dict whose HCL metadata lives on the object rather than among the keys.
+
+    Equality, iteration, `json.dumps` and every other mapping operation behave
+    exactly as `dict` does -- the metadata is deliberately not part of the
+    mapping, so a document declaring an attribute called `__is_block__` gets
+    that attribute back and nothing else.
+
+    JSON cannot carry the sidecar. Serializing an `HclDict` yields the
+    attributes alone, which is why the in-band keys remain the default.
+    """
+
+    __slots__ = ("hcl_meta",)
+
+    def __init__(self, *args: Any, meta: Optional[HclMeta] = None) -> None:
+        """Build from a mapping, with the metadata passed separately.
+
+        No `**kwargs`: this is the one class whose whole point is that no key
+        name is reserved, and taking keyword items would reserve `meta` --
+        `HclDict(**{"meta": "prod"})` would swallow the attribute and store a
+        string where the metadata goes. `meta` is a real name in real configs.
+        Pass the mapping positionally, as `dict` also allows.
+        """
+        super().__init__(*args)
+        if meta is not None and not isinstance(meta, HclMeta):
+            raise TypeError(
+                "HclDict(meta=...) takes an HclMeta; to store a key called "
+                f"'meta', pass the mapping positionally: HclDict({{'meta': {meta!r}}})"
+            )
+        self.hcl_meta = meta if meta is not None else HclMeta()
+
+    def __repr__(self) -> str:
+        """Show the metadata, so a debugging session does not have to guess."""
+        if self.hcl_meta.is_empty():
+            return super().__repr__()
+        return f"{super().__repr__()} + {self.hcl_meta!r}"
+
+    def copy(self) -> "HclDict":
+        """Copy the mapping and the metadata together.
+
+        `dict.copy` returns a plain `dict`, which would drop the sidecar --
+        and `document = document.copy()` is ordinary enough that losing block
+        metadata to it would be a trap. The in-band form survives a copy
+        because its metadata is among the keys; this has to say so explicitly.
+        """
+        return HclDict(self, meta=copy_module.copy(self.hcl_meta))
+
+    def __copy__(self) -> "HclDict":
+        """Same for `copy.copy`."""
+        return self.copy()
+
+    def __deepcopy__(self, memo: dict) -> "HclDict":
+        """Same for `copy.deepcopy`, metadata included.
+
+        The duplicate is recorded in *memo* before anything inside it is
+        copied. A mapping may hold a reference back to itself, and copying
+        the children first means the recursion reaches this dict again with
+        nothing recorded, which does not terminate. `dict` registers its own
+        copy first for that reason; a subclass that did not would make a
+        cyclic document worse than the plain mapping it replaces.
+        """
+        duplicate = HclDict()
+        memo[id(self)] = duplicate
+        duplicate.hcl_meta = copy_module.deepcopy(self.hcl_meta, memo)
+        for key, value in self.items():
+            duplicate[copy_module.deepcopy(key, memo)] = copy_module.deepcopy(value, memo)
+        return duplicate
+
+    def __reduce__(self) -> Tuple[Any, ...]:
+        """Carry the metadata through pickling, which `dict` would not."""
+        return (_rebuild, (dict(self), self.hcl_meta))
+
+    def __or__(self, other: Any) -> "HclDict":  # type: ignore[override]
+        """Merge, keeping this side's metadata.
+
+        Narrower than `dict.__or__`, which is declared to return `dict` for any
+        mapping: this always returns an `HclDict`, so the ignore records a
+        deliberate narrowing rather than a mismatch.
+
+        `dict.__or__` returns a plain `dict`, so `body | {"size": ...}` -- the
+        idiomatic non-mutating edit -- would drop the sidecar and the block
+        would then be written as an object. `{**body, ...}` cannot be helped:
+        unpacking always builds a plain `dict`, and there is no hook for it.
+        """
+        merged = HclDict(self, meta=copy_module.copy(self.hcl_meta))
+        merged.update(other)
+        return merged
+
+    def __ror__(self, other: Any) -> "HclDict":  # type: ignore[override]
+        """Same from the left, keeping this side's metadata."""
+        merged = HclDict(other, meta=copy_module.copy(self.hcl_meta))
+        merged.update(self)
+        return merged
+
+
+def meta_of(value: Any) -> Optional[HclMeta]:
+    """Return the metadata carried beside *value*, or None if it carries none."""
+    meta = getattr(value, "hcl_meta", None)
+    return meta if isinstance(meta, HclMeta) else None
+
+
+def _rebuild(items: Dict[str, Any], meta: HclMeta) -> HclDict:
+    """Reconstruct an `HclDict` from its pickled parts."""
+    return HclDict(items, meta=meta)

hcl2/query/blocks.py

@@ -3,6 +3,7 @@
 from typing import Any, List, Optional
 
 from hcl2.const import COMMENTS_KEY
+from hcl2.meta import meta_of
 from hcl2.query._base import NodeView, register_view
 from hcl2.rules.abstract import LarkElement
 from hcl2.rules.base import BlockRule
@@ -72,8 +73,18 @@ def to_dict(self, options: Optional[SerializationOptions] = None) -> Any:
         ):
             # Place adjacent comments at the outer level of the block dict,
             # alongside the label keys — not drilled into the body dict.
-            existing = result.get(COMMENTS_KEY, [])
-            result[COMMENTS_KEY] = self._adjacent_comments + existing
+            #
+            # Whichever form the serializer used: writing the in-band key onto
+            # a dict carrying a sidecar would put it back among the attributes,
+            # where nothing reserves it any more, and `dumps` would emit it as
+            # real HCL. Reading `result.get(COMMENTS_KEY)` there would also
+            # find nothing, because the block's own comments are in the meta.
+            meta = meta_of(result)
+            if meta is not None:
+                meta.comments = self._adjacent_comments + meta.comments
+            else:
+                existing = result.get(COMMENTS_KEY, [])
+                result[COMMENTS_KEY] = self._adjacent_comments + existing
         return result
 
     def blocks(self, block_type: Optional[str] = None, *labels: str) -> List["NodeView"]:

hcl2/rules/base.py

@@ -5,7 +5,8 @@
 
 from lark.tree import Meta
 
-from hcl2.const import INLINE_COMMENTS_KEY, IS_BLOCK
+from hcl2.const import COMMENTS_KEY, INLINE_COMMENTS_KEY, IS_BLOCK
+from hcl2.meta import HclDict, HclMeta, meta_of
 from hcl2.rules.abstract import LarkRule, LarkToken
 from hcl2.rules.expressions import ExprTermRule
 from hcl2.rules.literal_rules import IdentifierRule
@@ -87,9 +88,16 @@ def serialize(self, options=SerializationOptions(), context=SerializationContext
                 if child_comments:
                     comments.extend(child_comments)
 
+        if options.metadata_sidecar:
+            meta = HclMeta()
+            if options.with_comments:
+                meta.comments = comments
+                meta.inline_comments = inline_comments
+            return HclDict(result.items(), meta=meta)
+
         if options.with_comments:
             if comments:
-                result["__comments__"] = comments
+                result[COMMENTS_KEY] = comments
             if inline_comments:
                 result[INLINE_COMMENTS_KEY] = inline_comments
 
@@ -151,7 +159,11 @@ def serialize(self, options=SerializationOptions(), context=SerializationContext
         """Serialize to a nested dict with labels as keys."""
         result = self._body.serialize(options)
         if options.explicit_blocks:
-            result.update({IS_BLOCK: True})
+            meta = meta_of(result)
+            if meta is not None:
+                meta.is_block = True
+            else:
+                result.update({IS_BLOCK: True})
 
         labels = self._labels
         for label in reversed(labels[1:]):

hcl2/rules/containers.py

@@ -2,6 +2,7 @@
 
 from typing import Any, List, Optional, Tuple, Union
 
+from hcl2.meta import HclDict
 from hcl2.rules.abstract import LarkRule
 from hcl2.rules.expressions import ExpressionRule
 from hcl2.rules.literal_rules import (
@@ -192,6 +193,13 @@ def serialize(self, options=SerializationOptions(), context=SerializationContext
             dict_result: dict = {}
             for element in self.elements:
                 dict_result.update(element.serialize(options, context))
+            if options.metadata_sidecar:
+                # An object literal has no metadata of its own, but it has to
+                # say so in the same form a body does. Left a plain dict, a key
+                # the document wrote as `__is_block__` reads back as the marker
+                # and the object is emitted as a block -- which is the very
+                # collision the option exists to remove.
+                return HclDict(dict_result)
             return dict_result
 
         with context.modify(inside_dollar_string=True):

hcl2/utils.py

@@ -43,6 +43,16 @@ class SerializationOptions:
     # producing backwards-compatible output (e.g. "hello" instead of '"hello"').
     # Note: round-trip through from_dict/dumps is NOT supported WITH this option.
     strip_string_quotes: bool = False
+    # Appended rather than grouped with the other block options on purpose:
+    # this dataclass is not `kw_only`, so inserting a field anywhere else
+    # silently changes what every positional argument after it means.
+    #
+    # Carry the metadata keys beside the mapping instead of among its keys, as
+    # `HclDict.hcl_meta`. The in-band keys collide with any attribute a document
+    # happens to name `__is_block__`, `__comments__` or `__inline_comments__`;
+    # the sidecar cannot. Off by default because the keys are a documented part
+    # of the output shape, and because JSON cannot carry the sidecar.
+    metadata_sidecar: bool = False
 
 
 _SIMPLE_ESCAPES = {