Fix metadata xs:dateTime serialization and GZip compression handshake for C# subscriber interop

src/sttp/data/dataset.py

@@ -30,7 +30,7 @@
 from .datatype import DataType, parse_xsddatatype
 from typing import Dict, Iterator, List, Tuple
 from decimal import Decimal
-from datetime import datetime
+from datetime import datetime, timezone
 from uuid import UUID
 from io import BytesIO, StringIO
 from xml.etree import ElementTree
@@ -51,10 +51,28 @@
 
 def xsdformat(value: datetime) -> str:
     """
-    Converts date/time value to a string in XSD XML schema format.
+    Converts a date/time value to a canonical XSD `xs:dateTime` lexical string.
+
+    The result is formatted as `yyyy-MM-ddTHH:mm:ss[.ffffff]` with no timezone
+    designator and no UTC offset, matching the output of the .NET `DataSet` XML
+    serializer (default `DataSetDateTime.UnspecifiedLocal`) used by the STTP C#
+    implementation. Emitting a bare value ensures the C# subscriber accepts it and
+    round-trips it without a timezone shift.
+
+    Timezone-aware values are first normalized to their UTC wall-clock time and made
+    naive; naive values are treated as-is. Trailing fractional-second zeros are trimmed
+    without leaving a dangling decimal point.
     """
 
-    return value.isoformat(timespec="milliseconds")[:-1]  # 2 digit fractional second
+    if value.tzinfo is not None:
+        value = value.astimezone(timezone.utc).replace(tzinfo=None)
+
+    text = value.isoformat()  # naive -> no offset; omits fraction when microsecond is 0
+
+    if "." in text:
+        text = text.rstrip("0").rstrip(".")
+
+    return text
 
 
 class DataSet:
@@ -353,13 +371,8 @@ def _value_to_xml_text(value, datatype: DataType) -> str:
         if datatype == DataType.BOOLEAN:
             return 'true' if (value if isinstance(value, bool) else str(value).lower() in ("true", "1", "yes")) else 'false'
         elif datatype == DataType.DATETIME:
-            # Format as ISO 8601 with milliseconds and Z suffix
-            dt_str = value.isoformat(timespec='milliseconds')
-            if '.' in dt_str:
-                dt_str = dt_str.rstrip('0')
-            if not dt_str.endswith('Z'):
-                dt_str += 'Z'
-            return dt_str
+            # Emit a bare xs:dateTime (no Z, no offset) for C# DataSet interop
+            return xsdformat(value)
         elif datatype == DataType.GUID:
             return str(value)
         elif datatype == DataType.DECIMAL:

src/sttp/transport/subscriberconnection.py

@@ -35,7 +35,7 @@
 from .compactmeasurement import CompactMeasurement
 from .signalindexcache import SignalIndexCache
 from .constants import OperationalModes, OperationalEncoding, ServerCommand, ServerResponse
-from .constants import DataPacketFlags, BufferBlockFlags, Defaults
+from .constants import DataPacketFlags, BufferBlockFlags, Defaults, CompressionModes
 from .tssc.encoder import Encoder as TSSCEncoder
 from ..ticks import Ticks
 from typing import List, Set, TYPE_CHECKING, Tuple
@@ -59,6 +59,24 @@
 DEFAULT_PUBLISHINTERVAL = 1.0
 
 
+def should_gzip_compress(operational_modes, content_flag: OperationalModes) -> bool:
+    """
+    Determines whether GZip compression should be applied to a metadata or signal index cache
+    payload for the given negotiated `operational_modes`.
+
+    Content is compressed only when both the content's compression flag (e.g.,
+    `COMPRESSMETADATA` or `COMPRESSSIGNALINDEXCACHE`) and the `GZIP` compression mode are
+    negotiated. This matches the STTP C# reference (`DataPublisher.SerializeMetadata` /
+    `SerializeSignalIndexCache`), which uses `compress<Content> && compressionModes.HasFlag(GZip)`.
+    Honoring the GZip mode bit is required for interop: a subscriber (such as the .NET
+    `DataSubscriber` default) may request `COMPRESSMETADATA` without advertising `GZIP`, in which
+    case it expects uncompressed content.
+    """
+
+    modes = int(operational_modes)
+    return bool(modes & int(content_flag)) and bool(modes & int(CompressionModes.GZIP))
+
+
 class SubscriberConnection:
     """
     Represents a connection to a data subscriber.
@@ -595,9 +613,9 @@ def _handle_metadata_refresh(self, data: bytearray):
             # with open("C:\\temp\\publisher_metadata.xml", "w", encoding="utf-8") as f:
             #     f.write(metadata_xml)
 
-            # Compress if requested
-            compress_metadata = bool(int(self._operational_modes) & int(OperationalModes.COMPRESSMETADATA))
-            
+            # Compress only when the subscriber negotiated both COMPRESSMETADATA and GZip
+            compress_metadata = should_gzip_compress(self._operational_modes, OperationalModes.COMPRESSMETADATA)
+
             if compress_metadata:
                 uncompressed_size = len(metadata_bytes)
                 metadata_bytes = gzip.compress(metadata_bytes)
@@ -1094,9 +1112,9 @@ def _serialize_signal_index_cache(self, cache: SignalIndexCache) -> bytes:
         binary_length = len(buffer) - 4
         buffer[binary_length_offset:binary_length_offset+4] = BigEndian.from_uint32(binary_length)
 
-        # Compress if requested
-        compress_cache = self._operational_modes & OperationalModes.COMPRESSSIGNALINDEXCACHE
-        
+        # Compress only when the subscriber negotiated both COMPRESSSIGNALINDEXCACHE and GZip
+        compress_cache = should_gzip_compress(self._operational_modes, OperationalModes.COMPRESSSIGNALINDEXCACHE)
+
         if compress_cache:
             compressed = bytearray(gzip.compress(bytes(buffer)))
 

test/interop/README.md

@@ -0,0 +1,81 @@
+# Cross-language metadata interop harness
+
+Verifies interop between the STTP **Python publisher** and the STTP **C# subscriber** (the `gsfapi`
+reference implementation) for XML metadata exchange. It spins up a real Python publisher in-process
+and runs the real C# subscriber against it. It covers two fixes:
+
+1. **`xs:dateTime` serialization** — the Python publisher previously serialized timezone-aware UTC
+   datetimes as a lexically invalid string carrying both a `+00` offset and a `Z` designator (e.g.
+   `2033-03-03T03:33:33.123+00:Z`). Python's lenient parser accepts it; the C# subscriber's `.NET
+   DataSet.ReadXml` rejects it. The fix in [`src/sttp/data/dataset.py`](../../src/sttp/data/dataset.py)
+   (`xsdformat` / `_value_to_xml_text`) emits a **bare** `xs:dateTime`, matching the C# reference.
+
+2. **Metadata compression handshake** — the Python publisher previously GZip-compressed metadata
+   whenever `CompressMetadata` was set, ignoring the negotiated GZip mode. The `.NET DataSubscriber`
+   default requests `CompressMetadata` **without** GZip and so receives gzip bytes it will not
+   inflate, failing on the `0x1F` magic byte. The fix in
+   [`src/sttp/transport/subscriberconnection.py`](../../src/sttp/transport/subscriberconnection.py)
+   (`should_gzip_compress`) compresses only when both the content flag **and** GZip are negotiated,
+   matching the C# reference (`DataPublisher.SerializeMetadata` / `SerializeSignalIndexCache`).
+
+The harness seeds the served metadata with a fixed **timezone-aware UTC** `UpdatedOn` sentinel
+(`2033-03-03T03:33:33.123Z`) — the exact input that used to fail — and, by default, runs the
+exchange in **both** compression negotiations (uncompressed and GZip), asserting the sentinel
+round-trips in each.
+
+## Prerequisites
+
+- **Python**: the repo's virtual environment (`numpy`, `python-dateutil`). The script imports the
+  worktree's `src`, so the fix under test is exercised.
+- **C# build**: Roslyn `csc.exe` (Visual Studio 2022 or Build Tools) and the prebuilt STTP library
+  assemblies under `<gsfapi>\build\output\{Release,Debug}\lib\` (must contain `sttp.gsf.dll`). The
+  script compiles the C# sample [`InteropTest-gsf/Program.cs`](file:///C:/Projects/sttp/gsfapi/src/samples/InteropTest-gsf/Program.cs)
+  against that self-consistent assembly set.
+
+  > Build note: a full `msbuild sttp.gsf.sln` from source also works in a normal Visual Studio
+  > environment. This harness compiles only `Program.cs` with `csc` because the bundled
+  > `depends\GSF` DLLs can be out of sync with the library source (a `[Label]` attribute-target
+  > mismatch breaks the from-source library build in some environments).
+
+## Running
+
+```
+# From the pyapi worktree root, using the project venv:
+python test/interop/run_metadata_interop.py                      # sentinel, BOTH compression modes
+python test/interop/run_metadata_interop.py --compression plain  # only the uncompressed path
+python test/interop/run_metadata_interop.py --compression gzip   # only the compressed path
+python test/interop/run_metadata_interop.py --baseline           # stock (naive) metadata
+python test/interop/run_metadata_interop.py --keep               # keep _artifacts/metadata-received.xml
+python test/interop/run_metadata_interop.py --port 7180          # override the base publisher port
+```
+
+By default both compression negotiations run (each on its own port); both must succeed. Exit code
+`0` = every selected exchange succeeded (C# parsed the metadata; sentinel datetime found). Non-zero
+= failure (build error, C# deserialization error, timeout, or value mismatch).
+
+### Environment overrides
+
+- `STTP_GSFAPI_ROOT` — path to the `gsfapi` repo (default `C:\Projects\sttp\gsfapi`).
+- `STTP_CSC` — path to `csc.exe` (default: discovered via `vswhere`).
+
+## How it detects a regression
+
+The C# `InteropTest HOSTNAME PORT --metadata [--gzip]` mode (added to `Program.cs`) requests
+metadata on connect, then:
+
+- on **`MetaDataReceived`** — writes `metadata-received.xml`, prints each `DateTime` column value,
+  and exits `0`;
+- on **`ProcessException`** — a malformed datetime or undecodable (unexpectedly gzip-compressed)
+  payload makes `DataSet.ReadXml` throw, surfaced as "Failed to process publisher response packet …";
+  the process exits non-zero;
+- on **timeout** (15 s) — exits non-zero.
+
+Without `--gzip` the subscriber uses the `.NET`-default operational modes (`CompressMetadata`
+without GZip); with `--gzip` it advertises GZip. The orchestrator drives both.
+
+Both fixes have teeth — revert one and re-run:
+
+- **date fix** (`_value_to_xml_text`) → C# reports
+  `The string '2033-03-03T03:33:33.123+00:Z' is not a valid AllXsd value`;
+- **compression fix** (`should_gzip_compress`) → with `--compression plain` the publisher compresses
+  anyway and C# fails with `hexadecimal value 0x1F, is an invalid character`.