Expanded VLA and Optional support in photon-serde

.github/workflows/python.yml

@@ -78,6 +78,43 @@ jobs:
       - name: Run mypy type checking
         run: mypy --show-column-numbers --config-file photon-lib/py/pyproject.toml photon-lib
 
+  test-photonserde-py:
+    needs: build-py
+    runs-on: ubuntu-24.04
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v6
+        with:
+          fetch-depth: 0
+
+      - name: Set up Python
+        uses: actions/setup-python@v6
+        with:
+          python-version: 3.14
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install pytest mypy
+          pip install -r photon-serde-tests/requirements.txt
+
+      - name: Download artifacts
+        uses: actions/download-artifact@v8
+        with:
+          name: dist
+          path: ./photon-lib/py/dist/
+
+      - name: Install PhotonLibPy package
+        working-directory: ./photon-lib/py
+        shell: bash
+        run: |
+            pip install --no-cache-dir dist/*.whl
+
+      - name: Run PhotonSerde tests
+        shell: bash
+        run: python -m pytest --import-mode=importlib photon-serde-tests/py/AutoSerdeTest.py
+
   build-python-examples:
     needs: build-py
     strategy:

.wpiformat

@@ -6,6 +6,8 @@ generatedFileExclude {
   photon-lib/py/photonlibpy/generated/
   photon-targeting/src/generated/
   photon-targeting/src/main/native/cpp/photon/constrained_solvepnp/generate/
+  photon-serde-tests/src
+  photon-serde-tests/py
 }
 
 licenseUpdateExclude {

build.gradle

@@ -65,7 +65,7 @@ spotless {
     java {
         target fileTree('.') {
             include '**/*.java'
-            exclude '**/build/**', '**/build-*/**', '**/src/generated/**', "**/bin/generated-sources/**"
+            exclude '**/build/**', '**/build-*/**', '**/src/generated/**', '**/src/generated-test/**', "**/bin/generated-sources/**", "photon-serde-tests/src/**"
         }
         toggleOffOn()
         googleJavaFormat()

photon-lib/py/photonlibpy/generated/MultiTargetPNPResultSerde.py

@@ -29,14 +29,19 @@
 
 from typing import TYPE_CHECKING
 
+
 from ..packet import Packet
 from ..targeting import *  # noqa
 
+
+
 if TYPE_CHECKING:
     from ..targeting import MultiTargetPNPResult  # noqa
+
     from ..targeting import PnpResult  # noqa
 
 
+
 class MultiTargetPNPResultSerde:
     # Message definition md5sum. See photon_packet.adoc for details
     MESSAGE_VERSION = "541096947e9f3ca2d3f425ff7b04aa7b"
@@ -50,7 +55,7 @@ def pack(value: "MultiTargetPNPResult") -> "Packet":
         ret.encodeBytes(PnpResult.photonStruct.pack(value.estimatedPose).getData())
 
         # fiducialIDsUsed is a custom VLA!
-        ret.encodeShortList(value.fiducialIDsUsed)
+        ret.encodeListShimmed(value.fiducialIDsUsed, ret.encode16)
         return ret
 
     @staticmethod
@@ -60,8 +65,8 @@ def unpack(packet: "Packet") -> "MultiTargetPNPResult":
         # estimatedPose is of non-intrinsic type PnpResult
         ret.estimatedPose = PnpResult.photonStruct.unpack(packet)
 
-        # fiducialIDsUsed is a custom VLA!
-        ret.fiducialIDsUsed = packet.decodeShortList()
+        # fiducialIDsUsed is an intrinsic VLA!
+        ret.fiducialIDsUsed = packet.decodeListShimmed(packet.decode16)
 
         return ret
 

photon-lib/py/photonlibpy/generated/PhotonPipelineMetadataSerde.py

@@ -29,13 +29,17 @@
 
 from typing import TYPE_CHECKING
 
+
 from ..packet import Packet
 from ..targeting import *  # noqa
 
+
+
 if TYPE_CHECKING:
     from ..targeting import PhotonPipelineMetadata  # noqa
 
 
+
 class PhotonPipelineMetadataSerde:
     # Message definition md5sum. See photon_packet.adoc for details
     MESSAGE_VERSION = "ac0a45f686457856fb30af77699ea356"

photon-lib/py/photonlibpy/generated/PhotonPipelineResultSerde.py

@@ -29,16 +29,23 @@
 
 from typing import TYPE_CHECKING
 
+
 from ..packet import Packet
 from ..targeting import *  # noqa
 
+
+
 if TYPE_CHECKING:
     from ..targeting import MultiTargetPNPResult  # noqa
+
     from ..targeting import PhotonPipelineMetadata  # noqa
+
     from ..targeting import PhotonPipelineResult  # noqa
+
     from ..targeting import PhotonTrackedTarget  # noqa
 
 
+
 class PhotonPipelineResultSerde:
     # Message definition md5sum. See photon_packet.adoc for details
     MESSAGE_VERSION = "4b2ff16a964b5e2bf04be0c1454d91c4"
@@ -49,9 +56,7 @@ def pack(value: "PhotonPipelineResult") -> "Packet":
         ret = Packet()
 
         # metadata is of non-intrinsic type PhotonPipelineMetadata
-        ret.encodeBytes(
-            PhotonPipelineMetadata.photonStruct.pack(value.metadata).getData()
-        )
+        ret.encodeBytes(PhotonPipelineMetadata.photonStruct.pack(value.metadata).getData())
 
         # targets is a custom VLA!
         ret.encodeList(value.targets, PhotonTrackedTarget.photonStruct)

photon-lib/py/photonlibpy/generated/PhotonTrackedTargetSerde.py

@@ -29,14 +29,19 @@
 
 from typing import TYPE_CHECKING
 
+
 from ..packet import Packet
 from ..targeting import *  # noqa
 
+
+
 if TYPE_CHECKING:
     from ..targeting import PhotonTrackedTarget  # noqa
+
     from ..targeting import TargetCorner  # noqa
 
 
+
 class PhotonTrackedTargetSerde:
     # Message definition md5sum. See photon_packet.adoc for details
     MESSAGE_VERSION = "cc6dbb5c5c1e0fa808108019b20863f1"
@@ -67,8 +72,10 @@ def pack(value: "PhotonTrackedTarget") -> "Packet":
         # objDetectConf is of intrinsic type float32
         ret.encodeFloat(value.objDetectConf)
 
+        # bestCameraToTarget is of shimmed type Transform3d
         ret.encodeTransform(value.bestCameraToTarget)
 
+        # altCameraToTarget is of shimmed type Transform3d
         ret.encodeTransform(value.altCameraToTarget)
 
         # poseAmbiguity is of intrinsic type float64

photon-lib/py/photonlibpy/generated/PnpResultSerde.py

@@ -29,13 +29,17 @@
 
 from typing import TYPE_CHECKING
 
+
 from ..packet import Packet
 from ..targeting import *  # noqa
 
+
+
 if TYPE_CHECKING:
     from ..targeting import PnpResult  # noqa
 
 
+
 class PnpResultSerde:
     # Message definition md5sum. See photon_packet.adoc for details
     MESSAGE_VERSION = "ae4d655c0a3104d88df4f5db144c1e86"
@@ -45,8 +49,10 @@ class PnpResultSerde:
     def pack(value: "PnpResult") -> "Packet":
         ret = Packet()
 
+        # best is of shimmed type Transform3d
         ret.encodeTransform(value.best)
 
+        # alt is of shimmed type Transform3d
         ret.encodeTransform(value.alt)
 
         # bestReprojErr is of intrinsic type float64

photon-lib/py/photonlibpy/generated/TargetCornerSerde.py

@@ -29,13 +29,17 @@
 
 from typing import TYPE_CHECKING
 
+
 from ..packet import Packet
 from ..targeting import *  # noqa
 
+
+
 if TYPE_CHECKING:
     from ..targeting import TargetCorner  # noqa
 
 
+
 class TargetCornerSerde:
     # Message definition md5sum. See photon_packet.adoc for details
     MESSAGE_VERSION = "16f6ac0dedc8eaccb951f4895d9e18b6"

photon-lib/py/photonlibpy/packet.py

@@ -16,7 +16,7 @@
 ###############################################################################
 
 import struct
-from typing import Generic, Optional, Protocol, TypeVar
+from typing import Callable, Generic, Optional, Protocol, TypeVar
 
 import wpilib
 from wpimath import Quaternion, Rotation3d, Transform3d, Translation3d
@@ -200,12 +200,25 @@ def decodeList(self, serde: Serde[T]) -> list[T]:
             retList.append(serde.unpack(self))
         return retList
 
+    def decodeListShimmed(self, shim: Callable[[], T]) -> list[T]:
+        retList = []
+        arr_len = self.decode8()
+        for _ in range(arr_len):
+            retList.append(shim())
+        return retList
+
     def decodeOptional(self, serde: Serde[T]) -> Optional[T]:
         if self.decodeBoolean():
             return serde.unpack(self)
         else:
             return None
 
+    def decodeOptionalShimmed(self, shim: Callable[[], T]) -> Optional[T]:
+        if self.decodeBoolean():
+            return shim()
+        else:
+            return None
+
     def _encodeGeneric(self, packFormat, value):
         """
         Append bytes to the packet data buffer.
@@ -255,22 +268,6 @@ def encodeBoolean(self, value: bool):
         """
         self.encode8(1 if value else 0)
 
-    def encodeDoubleArray(self, values: list[float]):
-        """
-        Encodes an array of doubles and appends it to the packet.
-        """
-        self.encode8(len(values))
-        for value in values:
-            self.encodeDouble(value)
-
-    def encodeShortList(self, values: list[int]):
-        """
-        Encodes a list of shorts, with length prefixed as a single byte.
-        """
-        self.encode8(len(values))
-        for value in values:
-            self.encode16(value)
-
     def encodeTransform(self, transform: Transform3d):
         """
         Encodes a Transform3d (translation and rotation) and appends it to the packet.
@@ -297,6 +294,14 @@ def encodeList(self, values: list[T], serde: Serde[T]):
             self.packetData = self.packetData + packed.getData()
             self.size = len(self.packetData)
 
+    def encodeListShimmed(self, values: list[T], shim: Callable[[T], None]):
+        """
+        Encodes a list of items using a specific serializer and appends it to the packet.
+        """
+        self.encode8(len(values))
+        for item in values:
+            shim(item)
+
     def encodeOptional(self, value: Optional[T], serde: Serde[T]):
         """
         Encodes an optional value using a specific serializer.
@@ -309,6 +314,16 @@ def encodeOptional(self, value: Optional[T], serde: Serde[T]):
             self.packetData = self.packetData + packed.getData()
             self.size = len(self.packetData)
 
+    def encodeOptionalShimmed(self, value: Optional[T], shim: Callable[[T], None]):
+        """
+        Encodes an optional value using a specific shimmed serializer.
+        """
+        if value is None:
+            self.encodeBoolean(False)
+        else:
+            self.encodeBoolean(True)
+            shim(value)
+
     def encodeBytes(self, value: bytes):
         self.packetData = self.packetData + value
         self.size = len(self.packetData)

photon-serde-tests/README.md

@@ -0,0 +1,5 @@
+These are autogenerated tests for photon-serde. ALL CODE IN THIS DIRECTORY (except for build.gradle and requirements.txt) IS AUTOMATICALLY GENERATED
+
+requirements.txt is only for version-controlling WPILib. Photonlibpy itself is installed separately, and SHOULD NOT be in the requirements.txt
+
+These tests are separated from the main code as we don't use the autogenerated photon-serde test messages anywhere except in the tests, and we don't want to expose them to end-users in photon-lib