diff --git a/src/sttp/data/dataset.py b/src/sttp/data/dataset.py index 797e720..d7d7e4a 100644 --- a/src/sttp/data/dataset.py +++ b/src/sttp/data/dataset.py @@ -30,7 +30,7 @@ from .datatype import DataType, parse_xsddatatype from typing import Dict, Iterator, List, Tuple from decimal import Decimal -from datetime import datetime +from datetime import datetime, timezone from uuid import UUID from io import BytesIO, StringIO from xml.etree import ElementTree @@ -51,10 +51,28 @@ def xsdformat(value: datetime) -> str: """ - Converts date/time value to a string in XSD XML schema format. + Converts a date/time value to a canonical XSD `xs:dateTime` lexical string. + + The result is formatted as `yyyy-MM-ddTHH:mm:ss[.ffffff]` with no timezone + designator and no UTC offset, matching the output of the .NET `DataSet` XML + serializer (default `DataSetDateTime.UnspecifiedLocal`) used by the STTP C# + implementation. Emitting a bare value ensures the C# subscriber accepts it and + round-trips it without a timezone shift. + + Timezone-aware values are first normalized to their UTC wall-clock time and made + naive; naive values are treated as-is. Trailing fractional-second zeros are trimmed + without leaving a dangling decimal point. """ - return value.isoformat(timespec="milliseconds")[:-1] # 2 digit fractional second + if value.tzinfo is not None: + value = value.astimezone(timezone.utc).replace(tzinfo=None) + + text = value.isoformat() # naive -> no offset; omits fraction when microsecond is 0 + + if "." in text: + text = text.rstrip("0").rstrip(".") + + return text class DataSet: @@ -353,13 +371,8 @@ def _value_to_xml_text(value, datatype: DataType) -> str: if datatype == DataType.BOOLEAN: return 'true' if (value if isinstance(value, bool) else str(value).lower() in ("true", "1", "yes")) else 'false' elif datatype == DataType.DATETIME: - # Format as ISO 8601 with milliseconds and Z suffix - dt_str = value.isoformat(timespec='milliseconds') - if '.' in dt_str: - dt_str = dt_str.rstrip('0') - if not dt_str.endswith('Z'): - dt_str += 'Z' - return dt_str + # Emit a bare xs:dateTime (no Z, no offset) for C# DataSet interop + return xsdformat(value) elif datatype == DataType.GUID: return str(value) elif datatype == DataType.DECIMAL: diff --git a/src/sttp/transport/subscriberconnection.py b/src/sttp/transport/subscriberconnection.py index 8b4509d..7e1fa4d 100644 --- a/src/sttp/transport/subscriberconnection.py +++ b/src/sttp/transport/subscriberconnection.py @@ -35,7 +35,7 @@ from .compactmeasurement import CompactMeasurement from .signalindexcache import SignalIndexCache from .constants import OperationalModes, OperationalEncoding, ServerCommand, ServerResponse -from .constants import DataPacketFlags, BufferBlockFlags, Defaults +from .constants import DataPacketFlags, BufferBlockFlags, Defaults, CompressionModes from .tssc.encoder import Encoder as TSSCEncoder from ..ticks import Ticks from typing import List, Set, TYPE_CHECKING, Tuple @@ -59,6 +59,24 @@ DEFAULT_PUBLISHINTERVAL = 1.0 +def should_gzip_compress(operational_modes, content_flag: OperationalModes) -> bool: + """ + Determines whether GZip compression should be applied to a metadata or signal index cache + payload for the given negotiated `operational_modes`. + + Content is compressed only when both the content's compression flag (e.g., + `COMPRESSMETADATA` or `COMPRESSSIGNALINDEXCACHE`) and the `GZIP` compression mode are + negotiated. This matches the STTP C# reference (`DataPublisher.SerializeMetadata` / + `SerializeSignalIndexCache`), which uses `compress && compressionModes.HasFlag(GZip)`. + Honoring the GZip mode bit is required for interop: a subscriber (such as the .NET + `DataSubscriber` default) may request `COMPRESSMETADATA` without advertising `GZIP`, in which + case it expects uncompressed content. + """ + + modes = int(operational_modes) + return bool(modes & int(content_flag)) and bool(modes & int(CompressionModes.GZIP)) + + class SubscriberConnection: """ Represents a connection to a data subscriber. @@ -595,9 +613,9 @@ def _handle_metadata_refresh(self, data: bytearray): # with open("C:\\temp\\publisher_metadata.xml", "w", encoding="utf-8") as f: # f.write(metadata_xml) - # Compress if requested - compress_metadata = bool(int(self._operational_modes) & int(OperationalModes.COMPRESSMETADATA)) - + # Compress only when the subscriber negotiated both COMPRESSMETADATA and GZip + compress_metadata = should_gzip_compress(self._operational_modes, OperationalModes.COMPRESSMETADATA) + if compress_metadata: uncompressed_size = len(metadata_bytes) metadata_bytes = gzip.compress(metadata_bytes) @@ -1094,9 +1112,9 @@ def _serialize_signal_index_cache(self, cache: SignalIndexCache) -> bytes: binary_length = len(buffer) - 4 buffer[binary_length_offset:binary_length_offset+4] = BigEndian.from_uint32(binary_length) - # Compress if requested - compress_cache = self._operational_modes & OperationalModes.COMPRESSSIGNALINDEXCACHE - + # Compress only when the subscriber negotiated both COMPRESSSIGNALINDEXCACHE and GZip + compress_cache = should_gzip_compress(self._operational_modes, OperationalModes.COMPRESSSIGNALINDEXCACHE) + if compress_cache: compressed = bytearray(gzip.compress(bytes(buffer))) diff --git a/test/interop/README.md b/test/interop/README.md new file mode 100644 index 0000000..37966f8 --- /dev/null +++ b/test/interop/README.md @@ -0,0 +1,81 @@ +# Cross-language metadata interop harness + +Verifies interop between the STTP **Python publisher** and the STTP **C# subscriber** (the `gsfapi` +reference implementation) for XML metadata exchange. It spins up a real Python publisher in-process +and runs the real C# subscriber against it. It covers two fixes: + +1. **`xs:dateTime` serialization** — the Python publisher previously serialized timezone-aware UTC + datetimes as a lexically invalid string carrying both a `+00` offset and a `Z` designator (e.g. + `2033-03-03T03:33:33.123+00:Z`). Python's lenient parser accepts it; the C# subscriber's `.NET + DataSet.ReadXml` rejects it. The fix in [`src/sttp/data/dataset.py`](../../src/sttp/data/dataset.py) + (`xsdformat` / `_value_to_xml_text`) emits a **bare** `xs:dateTime`, matching the C# reference. + +2. **Metadata compression handshake** — the Python publisher previously GZip-compressed metadata + whenever `CompressMetadata` was set, ignoring the negotiated GZip mode. The `.NET DataSubscriber` + default requests `CompressMetadata` **without** GZip and so receives gzip bytes it will not + inflate, failing on the `0x1F` magic byte. The fix in + [`src/sttp/transport/subscriberconnection.py`](../../src/sttp/transport/subscriberconnection.py) + (`should_gzip_compress`) compresses only when both the content flag **and** GZip are negotiated, + matching the C# reference (`DataPublisher.SerializeMetadata` / `SerializeSignalIndexCache`). + +The harness seeds the served metadata with a fixed **timezone-aware UTC** `UpdatedOn` sentinel +(`2033-03-03T03:33:33.123Z`) — the exact input that used to fail — and, by default, runs the +exchange in **both** compression negotiations (uncompressed and GZip), asserting the sentinel +round-trips in each. + +## Prerequisites + +- **Python**: the repo's virtual environment (`numpy`, `python-dateutil`). The script imports the + worktree's `src`, so the fix under test is exercised. +- **C# build**: Roslyn `csc.exe` (Visual Studio 2022 or Build Tools) and the prebuilt STTP library + assemblies under `\build\output\{Release,Debug}\lib\` (must contain `sttp.gsf.dll`). The + script compiles the C# sample [`InteropTest-gsf/Program.cs`](file:///C:/Projects/sttp/gsfapi/src/samples/InteropTest-gsf/Program.cs) + against that self-consistent assembly set. + + > Build note: a full `msbuild sttp.gsf.sln` from source also works in a normal Visual Studio + > environment. This harness compiles only `Program.cs` with `csc` because the bundled + > `depends\GSF` DLLs can be out of sync with the library source (a `[Label]` attribute-target + > mismatch breaks the from-source library build in some environments). + +## Running + +``` +# From the pyapi worktree root, using the project venv: +python test/interop/run_metadata_interop.py # sentinel, BOTH compression modes +python test/interop/run_metadata_interop.py --compression plain # only the uncompressed path +python test/interop/run_metadata_interop.py --compression gzip # only the compressed path +python test/interop/run_metadata_interop.py --baseline # stock (naive) metadata +python test/interop/run_metadata_interop.py --keep # keep _artifacts/metadata-received.xml +python test/interop/run_metadata_interop.py --port 7180 # override the base publisher port +``` + +By default both compression negotiations run (each on its own port); both must succeed. Exit code +`0` = every selected exchange succeeded (C# parsed the metadata; sentinel datetime found). Non-zero += failure (build error, C# deserialization error, timeout, or value mismatch). + +### Environment overrides + +- `STTP_GSFAPI_ROOT` — path to the `gsfapi` repo (default `C:\Projects\sttp\gsfapi`). +- `STTP_CSC` — path to `csc.exe` (default: discovered via `vswhere`). + +## How it detects a regression + +The C# `InteropTest HOSTNAME PORT --metadata [--gzip]` mode (added to `Program.cs`) requests +metadata on connect, then: + +- on **`MetaDataReceived`** — writes `metadata-received.xml`, prints each `DateTime` column value, + and exits `0`; +- on **`ProcessException`** — a malformed datetime or undecodable (unexpectedly gzip-compressed) + payload makes `DataSet.ReadXml` throw, surfaced as "Failed to process publisher response packet …"; + the process exits non-zero; +- on **timeout** (15 s) — exits non-zero. + +Without `--gzip` the subscriber uses the `.NET`-default operational modes (`CompressMetadata` +without GZip); with `--gzip` it advertises GZip. The orchestrator drives both. + +Both fixes have teeth — revert one and re-run: + +- **date fix** (`_value_to_xml_text`) → C# reports + `The string '2033-03-03T03:33:33.123+00:Z' is not a valid AllXsd value`; +- **compression fix** (`should_gzip_compress`) → with `--compression plain` the publisher compresses + anyway and C# fails with `hexadecimal value 0x1F, is an invalid character`. diff --git a/test/interop/run_metadata_interop.py b/test/interop/run_metadata_interop.py new file mode 100644 index 0000000..2365693 --- /dev/null +++ b/test/interop/run_metadata_interop.py @@ -0,0 +1,270 @@ +#!/usr/bin/env python3 +# ****************************************************************************************************** +# run_metadata_interop.py - Cross-language metadata date/time interop harness +# +# Copyright © 2026, Grid Protection Alliance. All Rights Reserved. +# +# Licensed to the Grid Protection Alliance (GPA) under one or more contributor license agreements. +# ****************************************************************************************************** +# +# Spins up a real STTP Python publisher and a real STTP C# subscriber (the gsfapi "InteropTest" +# sample, run in --metadata mode) and exercises XML metadata exchange that includes date/time +# values. It verifies two fixes end-to-end: +# 1. xs:dateTime serialization - the Python publisher emits a bare lexical value the .NET +# DataSet parser accepts (previously "...+00:Z", which C# rejects). +# 2. metadata compression handshake - the Python publisher GZip-compresses metadata only when +# the subscriber negotiates GZip. The .NET-default subscriber requests CompressMetadata +# WITHOUT GZip, so it must receive uncompressed metadata; previously the publisher always +# compressed and C# failed on the 0x1F gzip magic byte. +# +# By default it runs the exchange in BOTH compression negotiations (uncompressed and GZip) with a +# seeded *timezone-aware UTC* UpdatedOn sentinel, and both must succeed. +# +# Exit code: 0 = all selected exchanges succeeded, non-zero = failure (build error, C# +# deserialization error, timeout, or value mismatch). +# +# Usage: +# python test/interop/run_metadata_interop.py [--baseline] [--compression {both,plain,gzip}] +# [--keep] [--port N] +# +# --baseline Serve stock (naive) metadata without injecting a tz-aware value. +# --compression Which negotiation(s) to exercise (default: both). "plain" = .NET default +# (uncompressed, the path the compression fix repairs), "gzip" = compressed. +# --keep Do not delete the C# output directory (metadata-received.xml) on exit. +# --port N Base publisher TCP port (default 7169; each mode uses a distinct port). +# +# Environment overrides: +# STTP_GSFAPI_ROOT Path to the gsfapi repo (default: C:\\Projects\\sttp\\gsfapi) +# STTP_CSC Path to csc.exe (default: discovered via vswhere) +# +# Build note: this harness compiles the C# Program.cs with csc against the prebuilt assemblies in +# \build\output\{Release,Debug}\lib. That folder is a self-consistent assembly set. A full +# "msbuild sttp.gsf.sln" from source also works in a normal Visual Studio environment; it is not +# used here because the bundled depends\GSF DLLs can be out of sync with the library source. +# +# ****************************************************************************************************** + +import argparse +import os +import subprocess +import sys +import time +from datetime import datetime, timezone + +HERE = os.path.dirname(os.path.abspath(__file__)) +PYAPI_ROOT = os.path.abspath(os.path.join(HERE, '..', '..')) +SRC_DIR = os.path.join(PYAPI_ROOT, 'src') + +# Ensure the worktree's src (with the fix) is imported, not any installed sttpapi package. +sys.path.insert(0, SRC_DIR) + +GSFAPI_ROOT = os.environ.get('STTP_GSFAPI_ROOT', r'C:\Projects\sttp\gsfapi') +PROGRAM_CS = os.path.join(GSFAPI_ROOT, 'src', 'samples', 'InteropTest-gsf', 'Program.cs') +METADATA_XML = os.path.join(PYAPI_ROOT, 'examples', 'simplepublish', 'Metadata.xml') + +# A distinctive, fixed timezone-aware UTC value so we can assert its round-trip in the C# output. +# Sub-second precision is included to exercise fractional-second handling. +SENTINEL = datetime(2033, 3, 3, 3, 33, 33, 123000, tzinfo=timezone.utc) +SENTINEL_SECONDS = '2033-03-03T03:33:33' # what the C# dump prints (to-seconds prefix) + + +def _log(msg): + print(f'[HARNESS] {msg}', flush=True) + + +def _find_csc(): + """Locate the Roslyn csc.exe (needed for the C# 9+ syntax in Program.cs).""" + override = os.environ.get('STTP_CSC') + if override and os.path.isfile(override): + return override + + vswhere = r'C:\Program Files (x86)\Microsoft Visual Studio\Installer\vswhere.exe' + if os.path.isfile(vswhere): + try: + out = subprocess.run( + [vswhere, '-latest', '-products', '*', '-requires', 'Microsoft.Component.MSBuild', + '-find', r'MSBuild\**\Bin\Roslyn\csc.exe'], + capture_output=True, text=True, check=False).stdout.strip() + first = out.splitlines()[0].strip() if out else '' + if first and os.path.isfile(first): + return first + except Exception as ex: + _log(f'vswhere probe failed ({ex}); continuing with fallback discovery') + + raise FileNotFoundError( + 'Could not locate Roslyn csc.exe. Set STTP_CSC to its path, or build the C# sample with ' + 'Visual Studio / msbuild first.') + + +def _prebuilt_lib_dir(): + for config in ('Release', 'Debug'): + lib = os.path.join(GSFAPI_ROOT, 'build', 'output', config, 'lib') + if os.path.isfile(os.path.join(lib, 'sttp.gsf.dll')): + return lib + raise FileNotFoundError( + f'Could not find prebuilt sttp.gsf.dll under {GSFAPI_ROOT}\\build\\output\\*\\lib. ' + 'Build the gsfapi library once (Visual Studio / msbuild) to produce it.') + + +def build_csharp_exe(): + """Compile Program.cs into /InteropTest.exe against the prebuilt assembly set.""" + lib = _prebuilt_lib_dir() + csc = _find_csc() + exe = os.path.join(lib, 'InteropTest.exe') + + if not os.path.isfile(PROGRAM_CS): + raise FileNotFoundError(f'C# sample source not found: {PROGRAM_CS}') + + _log(f'Compiling C# subscriber with csc -> {exe}') + cmd = [ + csc, '-nologo', '-target:exe', '-langversion:latest', f'-out:{exe}', + '-r:' + os.path.join(lib, 'sttp.gsf.dll'), + '-r:' + os.path.join(lib, 'GSF.Core.dll'), + '-r:' + os.path.join(lib, 'GSF.TimeSeries.dll'), + '-r:' + os.path.join(lib, 'GSF.Communication.dll'), + PROGRAM_CS, + ] + result = subprocess.run(cmd, capture_output=True, text=True, check=False) + if result.returncode != 0 or not os.path.isfile(exe): + _log('csc output:\n' + (result.stdout or '') + (result.stderr or '')) + raise RuntimeError('C# compilation failed.') + return exe, lib + + +def start_publisher(port, inject_sentinel): + """Start an in-process Python publisher serving metadata (optionally seeded with the sentinel).""" + from sttp.publisher import Publisher + from sttp.data.dataset import DataSet + + with open(METADATA_XML, 'r') as f: + metadata_xml = f.read() + + metadata, err = DataSet.from_xml(metadata_xml) + if err: + raise RuntimeError(f'Failed to load metadata: {err}') + + if inject_sentinel: + table = metadata.table('DeviceDetail') + if table is None or table.rowcount == 0: + raise RuntimeError('DeviceDetail table with rows is required to inject the sentinel.') + set_err = table.row(0).set_value_byname('UpdatedOn', SENTINEL) + if set_err is not None: + raise RuntimeError(f'Failed to set sentinel UpdatedOn: {set_err}') + _log(f'Injected timezone-aware UTC UpdatedOn = {SENTINEL.isoformat()} into DeviceDetail[0]') + + publisher = Publisher() + publisher.statusmessage_logger = lambda m: print(f'[PUB] {m}', flush=True) + publisher.errormessage_logger = lambda m: print(f'[PUB ERROR] {m}', file=sys.stderr, flush=True) + publisher.clientconnected_receiver = lambda c: _log(f'C# client connected: {c.connection_id}') + publisher.start(port) + publisher.define_metadata(metadata) + _log(f'Publisher listening on port {port}') + return publisher + + +def run_once(exe, port, inject_sentinel, use_gzip, out_dir): + """Run one publisher/subscriber metadata exchange in the given compression mode. + + Returns True on success (C# exited 0 and, unless baseline, the sentinel round-tripped). + """ + label = 'gzip (compressed)' if use_gzip else 'default (uncompressed)' + _log(f'=== Metadata exchange: {label} negotiation on port {port} ===') + + publisher = None + try: + publisher = start_publisher(port, inject_sentinel=inject_sentinel) + time.sleep(0.5) # let the listener settle + + cmd = [exe, 'localhost', str(port), '--metadata'] + if use_gzip: + cmd.append('--gzip') + _log('Launching C# subscriber: ' + ' '.join(os.path.basename(cmd[0]) if i == 0 else c + for i, c in enumerate(cmd))) + proc = subprocess.run(cmd, cwd=out_dir, capture_output=True, text=True, timeout=45) + + stdout = proc.stdout or '' + stderr = proc.stderr or '' + print('----- C# stdout -----') + print(stdout, end='') + if stderr.strip(): + print('----- C# stderr -----') + print(stderr, end='') + print('---------------------') + _log(f'C# subscriber exit code: {proc.returncode}') + + if proc.returncode != 0: + _log(f'FAIL [{label}]: C# subscriber exited non-zero (deserialization error or timeout).') + return False + + if inject_sentinel and SENTINEL_SECONDS not in stdout: + _log(f'FAIL [{label}]: sentinel datetime {SENTINEL_SECONDS} not found in C# output.') + return False + + detail = (f'sentinel {SENTINEL_SECONDS} round-tripped' if inject_sentinel + else 'metadata received and parsed') + _log(f'PASS [{label}]: {detail}.') + return True + finally: + if publisher is not None: + try: + publisher.stop() + except Exception as ex: + _log(f'Error stopping publisher: {ex}') + time.sleep(0.3) # allow the listening socket to release before the next run + + +def run(args): + exe, _ = build_csharp_exe() + + # C# writes metadata-received.xml into its working directory. + out_dir = os.path.join(HERE, '_artifacts') + os.makedirs(out_dir, exist_ok=True) + + # Which compression negotiation(s) to exercise. "plain" is the .NET-default path the + # compression fix repairs; "gzip" is the compressed path. + modes = {'both': [False, True], 'plain': [False], 'gzip': [True]}[args.compression] + + try: + results = [] + for index, use_gzip in enumerate(modes): + ok = run_once(exe, args.port + index, inject_sentinel=not args.baseline, + use_gzip=use_gzip, out_dir=out_dir) + results.append(ok) + return 0 if all(results) else 1 + finally: + if not args.keep: + try: + received = os.path.join(out_dir, 'metadata-received.xml') + if os.path.isfile(received): + os.remove(received) + if os.path.isdir(out_dir) and not os.listdir(out_dir): + os.rmdir(out_dir) + except OSError as ex: + _log(f'Cleanup warning (ignored): {ex}') + else: + _log(f'Artifacts kept in {out_dir}') + + +def main(): + parser = argparse.ArgumentParser(description='STTP Python-publisher / C#-subscriber metadata interop harness') + parser.add_argument('--baseline', action='store_true', + help='Serve stock (naive) metadata without the tz-aware sentinel') + parser.add_argument('--compression', choices=['both', 'plain', 'gzip'], default='both', + help='Compression negotiation(s) to exercise (default: both). ' + '"plain" = .NET default (uncompressed), "gzip" = compressed.') + parser.add_argument('--keep', action='store_true', help='Keep the C# output directory') + parser.add_argument('--port', type=int, default=7169, help='Base publisher TCP port (default 7169)') + args = parser.parse_args() + + try: + rc = run(args) + except Exception as ex: + _log(f'ERROR: {ex}') + rc = 2 + + _log('RESULT: ' + ('SUCCESS' if rc == 0 else f'FAILURE (exit {rc})')) + sys.exit(rc) + + +if __name__ == '__main__': + main() diff --git a/test/test_metadata_compression.py b/test/test_metadata_compression.py new file mode 100644 index 0000000..e40ce6b --- /dev/null +++ b/test/test_metadata_compression.py @@ -0,0 +1,77 @@ +#!/usr/bin/env python3 +# ****************************************************************************************************** +# test_metadata_compression.py - Pytest tests for publisher metadata/SIC compression negotiation +# +# Copyright © 2026, Grid Protection Alliance. All Rights Reserved. +# +# Licensed to the Grid Protection Alliance (GPA) under one or more contributor license agreements. +# ****************************************************************************************************** +# +# Regression tests for the publisher compression handshake. The publisher must GZip-compress +# metadata (and the signal index cache) only when the subscriber negotiates BOTH the content flag +# (COMPRESSMETADATA / COMPRESSSIGNALINDEXCACHE) AND the GZIP compression mode -- matching the STTP +# C# reference (DataPublisher.SerializeMetadata / SerializeSignalIndexCache). +# +# Previously the publisher compressed whenever the content flag was set, ignoring the GZip bit. +# A .NET DataSubscriber requests COMPRESSMETADATA without advertising GZIP by default, so it then +# received GZip bytes it would not inflate and failed to parse the metadata (0x1F "invalid +# character"). See src/sttp/transport/subscriberconnection.py (should_gzip_compress). +# +# ****************************************************************************************************** + +import os +import sys + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', 'src')) + +import pytest + +from sttp.transport.subscriberconnection import should_gzip_compress +from sttp.transport.constants import OperationalModes, CompressionModes + +META = int(OperationalModes.COMPRESSMETADATA) +SIC = int(OperationalModes.COMPRESSSIGNALINDEXCACHE) +GZIP = int(CompressionModes.GZIP) +NOISE = 0x00000002 | 0x00010000 # version + an arbitrary unrelated bit + + +@pytest.mark.parametrize("content_flag", [ + OperationalModes.COMPRESSMETADATA, + OperationalModes.COMPRESSSIGNALINDEXCACHE, +]) +def test_compress_requires_flag_and_gzip(content_flag): + flag = int(content_flag) + + # Both the content flag and GZip negotiated -> compress. + assert should_gzip_compress(flag | GZIP, content_flag) is True + + # Content flag set but GZip NOT negotiated -> do NOT compress (the .NET-default case). + assert should_gzip_compress(flag, content_flag) is False + + # GZip negotiated but the content flag not set -> do NOT compress. + assert should_gzip_compress(GZIP, content_flag) is False + + # Neither negotiated -> do NOT compress. + assert should_gzip_compress(0, content_flag) is False + + +def test_flags_are_independent(): + # GZip + only the metadata flag compresses metadata but not the signal index cache. + modes = META | GZIP + assert should_gzip_compress(modes, OperationalModes.COMPRESSMETADATA) is True + assert should_gzip_compress(modes, OperationalModes.COMPRESSSIGNALINDEXCACHE) is False + + # GZip + only the SIC flag compresses the signal index cache but not metadata. + modes = SIC | GZIP + assert should_gzip_compress(modes, OperationalModes.COMPRESSSIGNALINDEXCACHE) is True + assert should_gzip_compress(modes, OperationalModes.COMPRESSMETADATA) is False + + +def test_unrelated_bits_do_not_affect_decision(): + # Version/encoding and other unrelated bits must not trigger or suppress compression. + assert should_gzip_compress(META | GZIP | NOISE, OperationalModes.COMPRESSMETADATA) is True + assert should_gzip_compress(META | NOISE, OperationalModes.COMPRESSMETADATA) is False + + +if __name__ == "__main__": + sys.exit(pytest.main([__file__, "-v"])) diff --git a/test/test_metadata_datetime_xml.py b/test/test_metadata_datetime_xml.py new file mode 100644 index 0000000..6404379 --- /dev/null +++ b/test/test_metadata_datetime_xml.py @@ -0,0 +1,122 @@ +#!/usr/bin/env python3 +# ****************************************************************************************************** +# test_metadata_datetime_xml.py - Pytest tests for xs:dateTime metadata serialization +# +# Copyright © 2026, Grid Protection Alliance. All Rights Reserved. +# +# Licensed to the Grid Protection Alliance (GPA) under one or more contributor license agreements. +# ****************************************************************************************************** +# +# Regression tests for the metadata date/time serialization fix. The Python publisher must emit a +# bare xs:dateTime lexical value (no "Z" designator and no UTC offset) so the STTP C# subscriber, +# which parses metadata via .NET DataSet.ReadXml, accepts it and round-trips it without a timezone +# shift. The prior implementation produced strings such as "...+00:Z" for timezone-aware UTC values, +# which C# rejects. See src/sttp/data/dataset.py (xsdformat / _value_to_xml_text). +# +# ****************************************************************************************************** + +import os +import sys +from datetime import datetime, timezone, timedelta + +# Add src to path (parent directory) +sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', 'src')) + +import pytest + +from sttp.data.dataset import DataSet, xsdformat +from sttp.data.datatype import DataType +from sttp.ticks import Ticks + +METADATA_PATH = os.path.join(os.path.dirname(__file__), + '..', 'examples', 'simplepublish', 'Metadata.xml') + + +def _assert_bare(text: str): + """A canonical xs:dateTime carries no timezone designator or offset.""" + assert 'Z' not in text, f"unexpected 'Z' designator in {text!r}" + assert '+' not in text, f"unexpected offset in {text!r}" + assert '+00' not in text, f"unexpected '+00' offset in {text!r}" + # Never leave a dangling decimal point from fractional-second trimming + assert not text.endswith('.'), f"dangling decimal point in {text!r}" + + +@pytest.mark.parametrize("value, expected", [ + # timezone-aware UTC, whole second -> bare, no fractional part + (datetime(2024, 1, 1, 0, 0, 0, tzinfo=timezone.utc), '2024-01-01T00:00:00'), + # timezone-aware UTC with milliseconds + (datetime(2024, 1, 1, 12, 34, 56, 789000, tzinfo=timezone.utc), '2024-01-01T12:34:56.789'), + # timezone-aware non-UTC (-06:00) -> normalized to UTC wall-clock + (datetime(2026, 2, 7, 21, 27, 24, 282000, tzinfo=timezone(timedelta(hours=-6))), + '2026-02-08T03:27:24.282'), + # naive with trailing zeros -> trimmed, no dangling dot + (datetime(2024, 1, 1, 12, 34, 56, 100000), '2024-01-01T12:34:56.1'), + # naive whole second -> no fractional part, no dangling dot + (datetime(2024, 1, 1, 12, 34, 56), '2024-01-01T12:34:56'), + # naive with full microsecond precision -> preserved + (datetime(2024, 1, 1, 12, 34, 56, 123456), '2024-01-01T12:34:56.123456'), +]) +def test_xsdformat_bare_output(value, expected): + """xsdformat emits a bare xs:dateTime for both tz-aware and naive inputs.""" + text = xsdformat(value) + assert text == expected + _assert_bare(text) + + +def test_value_to_xml_text_datetime_is_bare(): + """The metadata serializer path emits the same bare form for tz-aware UTC values.""" + value = datetime(2024, 1, 1, 0, 0, 0, tzinfo=timezone.utc) + text = DataSet._value_to_xml_text(value, DataType.DATETIME) + assert text == '2024-01-01T00:00:00' + _assert_bare(text) + + +def test_ticks_utcnow_serializes_bare(): + """A programmatic (tz-aware UTC) datetime from Ticks must serialize bare. + + Ticks.to_datetime()/Ticks.utcnow() produce timezone-aware UTC datetimes; this is the + exact input that previously produced the malformed '...+00:Z' string. + """ + value = Ticks.to_datetime(Ticks.utcnow()) + assert value.tzinfo is not None, "precondition: Ticks datetime is timezone-aware" + _assert_bare(xsdformat(value)) + _assert_bare(DataSet._value_to_xml_text(value, DataType.DATETIME)) + + +def test_to_xml_injected_tzaware_utc_roundtrips(): + """End-to-end: inject a tz-aware UTC UpdatedOn, serialize, and confirm the XML is bare + and round-trips through DataSet.from_xml without a shift.""" + with open(METADATA_PATH, 'r') as f: + metadata_xml = f.read() + + metadata, err = DataSet.from_xml(metadata_xml) + assert not err, f"failed to load metadata: {err}" + + # Inject a timezone-aware UTC value (the previously-failing path) with sub-millisecond + # precision to also exercise fidelity of the fractional component. + injected = datetime(2024, 1, 1, 12, 34, 56, 123456, tzinfo=timezone.utc) + table = metadata.table("DeviceDetail") + assert table is not None and table.rowcount > 0, "DeviceDetail rows required for test" + row = table.row(0) + set_err = row.set_value_byname("UpdatedOn", injected) + assert set_err is None, f"failed to set UpdatedOn: {set_err}" + + xml = metadata.to_xml() + + # The serialized field must be bare, and the document must contain neither the malformed + # '+00:Z' string nor any offset/designator on the injected value. + assert '2024-01-01T12:34:56.123456' in xml, "injected datetime not serialized as bare form" + assert '+00:Z' not in xml + assert '+00' not in xml + assert '123456Z' not in xml and 'T12:34:56.123456+' not in xml + + # Round-trip: reload and confirm the value survives (as naive UTC wall-clock). + reloaded, err = DataSet.from_xml(xml) + assert not err, f"failed to reload metadata: {err}" + value, verr = reloaded.table("DeviceDetail").row(0).value_byname("UpdatedOn") + assert verr is None, f"failed to read UpdatedOn: {verr}" + assert value == injected.astimezone(timezone.utc).replace(tzinfo=None) + + +if __name__ == "__main__": + sys.exit(pytest.main([__file__, "-v"]))