Add ROS1 build CI and simulation regression report

.github/scripts/eval_report.py

@@ -0,0 +1,89 @@
+#!/usr/bin/env python3
+"""Render a master-vs-PR markdown block from mins_eval run_comparison output.
+
+Parses the LaTeX ATE/NEES/Time table that `run_comparison` (viz_type:=0) prints
+and emits a collapsible markdown section comparing the MINS_master and MINS_PR
+"algorithms" for one sensor config. Informational only -- never fails the build.
+
+Usage: eval_report.py <config_name> <run_comparison_stdout.txt>
+"""
+import re
+import sys
+
+FLOAT = re.compile(r"-?\d+\.?\d*(?:[eE][-+]?\d+)?")
+
+
+def parse(path):
+    """Return {'master': {...}, 'pr': {...}} with rmse/nees/time number lists."""
+    rows = {}
+    with open(path, "r", errors="replace") as fh:
+        for line in fh:
+            line = line.strip()
+            m = re.match(r"^MINS\\?_(\w+)\s*&(.*)", line)
+            if not m:
+                continue
+            alg = m.group(1).lower()                 # master | pr
+            rest = m.group(2).split("\\\\")[0]       # drop everything after the row terminator
+            rest = (rest.replace("\\textbf{", "")
+                        .replace("\\colorbox{orange}{", "")
+                        .replace("}", ""))
+            cells = rest.split("&")
+            nums = lambda s: [float(x) for x in FLOAT.findall(s)]
+            rows[alg] = {
+                "rmse": nums(cells[0]) if len(cells) > 0 else [],   # [ori_mean, ori_std, pos_mean, pos_std]
+                "nees": nums(cells[1]) if len(cells) > 1 else [],   # [ori_mean, ori_std, pos_mean, pos_std]
+                "time": nums(cells[2]) if len(cells) > 2 else [],   # [mean, std]
+            }
+    return rows
+
+
+def cell(mean, std):
+    return f"{mean:.3f} +/- {std:.3f}"
+
+
+def delta(pr_mean, ma_mean):
+    if ma_mean == 0:
+        return "n/a"
+    pct = (pr_mean - ma_mean) / abs(ma_mean) * 100.0
+    sign = "+" if pct >= 0 else ""
+    return f"{sign}{pct:.1f}%"
+
+
+def main():
+    config = sys.argv[1]
+    rows = parse(sys.argv[2])
+    out = [f"<details><summary><b>{config}</b></summary>\n"]
+
+    if "master" not in rows or "pr" not in rows:
+        out.append(f"> Note: could not parse results for `{config}` "
+                   "(a master or PR run may have failed - see job logs).\n")
+        out.append("</details>")
+        print("\n".join(out))
+        return
+
+    out.append("| Metric | master | PR | Delta (mean) |")
+    out.append("|---|---|---|---|")
+
+    def row(label, key, mi, si):
+        ma, pr = rows["master"][key], rows["pr"][key]
+        if len(ma) <= mi or len(pr) <= mi:
+            return
+        out.append(f"| {label} | {cell(ma[mi], ma[si])} | {cell(pr[mi], pr[si])} "
+                   f"| {delta(pr[mi], ma[mi])} |")
+
+    row("RMSE ori (deg)", "rmse", 0, 1)
+    row("RMSE pos (m)",  "rmse", 2, 3)
+    row("NEES ori",      "nees", 0, 1)
+    row("NEES pos",      "nees", 2, 3)
+    row("Time (s)",      "time", 0, 1)
+    out.append("\n</details>")
+    print("\n".join(out))
+
+
+if __name__ == "__main__":
+    try:
+        main()
+    except Exception as exc:  # never break the report job
+        cfg = sys.argv[1] if len(sys.argv) > 1 else "?"
+        print(f"<details><summary><b>{cfg}</b></summary>\n\n"
+              f"> Note: report parsing error: `{exc}`\n\n</details>")

.github/workflows/build_ros1.yml

@@ -0,0 +1,37 @@
+name: ROS 1 Workflow
+
+on:
+  push:
+    branches: [ "master" ]
+  pull_request:
+    branches: [ "master" ]
+
+jobs:
+
+  build_1804:
+    name: "ROS1 Ubuntu 18.04"
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Build the Docker image (Melodic 18.04)
+        run: docker build . --file Dockerfile_melodic_18_04 --tag mins:melodic
+      - name: Run MINS simulation (smoke test)
+        run: docker run --rm --entrypoint /bin/bash mins:melodic -c "source /opt/ros/melodic/setup.bash && source /catkin_ws/devel/setup.bash && export OMP_NUM_THREADS=1 && roslaunch mins simulation.launch sys_verbosity:=3"
+
+  build_2004:
+    name: "ROS1 Ubuntu 20.04"
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Build the Docker image (Noetic 20.04)
+        run: docker build . --file Dockerfile_noetic_20_04 --tag mins:noetic
+      - name: Run MINS simulation (smoke test)
+        run: docker run --rm --entrypoint /bin/bash mins:noetic -c "source /opt/ros/noetic/setup.bash && source /catkin_ws/devel/setup.bash && export OMP_NUM_THREADS=1 && roslaunch mins simulation.launch sys_verbosity:=3"
+
+  # build_1604 ("ROS1 Ubuntu 16.04", Kinetic): PARKED - revisit later.
+  # MINS's LiDAR dependencies (libpointmatcher / libnabo require Eigen >= 3.3, while
+  # Ubuntu 16.04/Xenial ships Eigen 3.2) plus the modern PCL/Ceres usage make a clean
+  # 16.04 build unlikely without dedicated dependency work. To re-enable:
+  #   1. Add Dockerfile_kinetic_16_04 (FROM osrf/ros:kinetic-perception, python-catkin-tools,
+  #      Eigen upgraded to 3.3.7 from source), and
+  #   2. Add a build_1604 job here mirroring the two above.

.github/workflows/evaluation.yml

@@ -0,0 +1,186 @@
+name: Evaluation Report
+
+# Simulation-based regression REPORT (master vs PR). Informational only:
+# it posts a per-sensor accuracy / consistency / timing comparison to the PR so
+# the contributor is aware of any change. It is NOT a required check and never
+# blocks merge. Keep it off the branch-protection required-checks list.
+
+on:
+  pull_request:
+    branches: [ "master" ]
+  workflow_dispatch:
+
+permissions:
+  contents: read
+  pull-requests: write   # to post / update the PR comment
+
+env:
+  SEEDS: 8               # simulations per algorithm per sensor config
+  DATASET: UD_Warehouse  # bundled B-spline sim trajectory
+
+jobs:
+
+  # 1) Build the master-baseline and PR Noetic images once each (parallel).
+  build:
+    name: "Build image (${{ matrix.ref }})"
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - ref: pr
+            checkout: ""                       # PR merge ref (default)
+          - ref: master
+            checkout: ${{ github.base_ref }}   # master baseline
+    steps:
+      - name: Checkout (${{ matrix.ref }})
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ matrix.checkout }}
+          submodules: recursive                # GroundTruths needed for eval
+      - name: Free disk space
+        run: |
+          sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /opt/hostedtoolcache/CodeQL /usr/local/share/boost /opt/microsoft || true
+          df -h /
+      - name: Build Docker image
+        run: docker build . --file Dockerfile_noetic_20_04 --tag mins:${{ matrix.ref }}
+      - name: Export image
+        run: docker save mins:${{ matrix.ref }} | gzip > image_${{ matrix.ref }}.tar.gz
+      - name: Upload image artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: image-${{ matrix.ref }}
+          path: image_${{ matrix.ref }}.tar.gz
+          retention-days: 1
+
+  # 2) Per sensor config (parallel): run SEEDS sims for master & PR, then compare.
+  evaluate:
+    name: "Eval ${{ matrix.config }}"
+    needs: build
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - config: vio_stereo
+            args: "cam_enabled:=true cam_use_stereo:=true cam_max_n:=2 lidar_enabled:=false gps_enabled:=false wheel_enabled:=false vicon_enabled:=false"
+          - config: stereo_lidar
+            args: "cam_enabled:=true cam_use_stereo:=true cam_max_n:=2 lidar_enabled:=true gps_enabled:=false wheel_enabled:=false vicon_enabled:=false"
+          - config: stereo_gps
+            args: "cam_enabled:=true cam_use_stereo:=true cam_max_n:=2 lidar_enabled:=false gps_enabled:=true wheel_enabled:=false vicon_enabled:=false"
+          - config: stereo_wheel
+            args: "cam_enabled:=true cam_use_stereo:=true cam_max_n:=2 lidar_enabled:=false gps_enabled:=false wheel_enabled:=true vicon_enabled:=false"
+          - config: full
+            args: "cam_enabled:=true cam_use_stereo:=true cam_max_n:=2 lidar_enabled:=true gps_enabled:=true wheel_enabled:=true vicon_enabled:=false"
+    steps:
+      - name: Checkout (for report script)
+        uses: actions/checkout@v4
+      - name: Free disk space
+        run: |
+          sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /opt/hostedtoolcache/CodeQL /usr/local/share/boost /opt/microsoft || true
+          df -h /
+      - name: Download images
+        uses: actions/download-artifact@v4
+        with:
+          pattern: image-*
+          path: images
+      - name: Load images
+        run: |
+          docker load < images/image-master/image_master.tar.gz
+          docker load < images/image-pr/image_pr.tar.gz
+      - name: Run ${{ env.SEEDS }} simulations per algorithm
+        run: |
+          set -e
+          last=$((SEEDS-1))
+          for ref in master pr; do
+            mkdir -p out_$ref
+            docker run --rm --entrypoint /bin/bash -v "$PWD/out_$ref:/outputs" "mins:$ref" -c '
+              source /opt/ros/noetic/setup.bash && source /catkin_ws/devel/setup.bash
+              export OMP_NUM_THREADS=1
+              for s in $(seq 0 '"$last"'); do
+                rm -f /outputs/tmp/$s.txt /outputs/tmp/$s.time
+                roslaunch mins simulation.launch sim_seed:=$s sys_verbosity:=3 \
+                  sys_save_state:=false '"${{ matrix.args }}"'
+              done'
+          done
+      - name: Arrange results for run_comparison
+        run: |
+          for ref in master pr; do
+            mkdir -p "alg/MINS_$ref/${DATASET}"
+            cp out_$ref/tmp/*.txt out_$ref/tmp/*.time "alg/MINS_$ref/${DATASET}/" 2>/dev/null || true
+          done
+          echo "Staged result tree:"; ls -R alg || true
+      - name: Run comparison (master vs PR)
+        run: |
+          docker run --rm --entrypoint /bin/bash -v "$PWD/alg:/alg" mins:pr -c '
+            source /opt/ros/noetic/setup.bash && source /catkin_ws/devel/setup.bash
+            roslaunch mins_eval comparison.launch viz_type:=0 align_mode:=se3 \
+              path_alg:=/alg \
+              path_gts:=/catkin_ws/src/MINS/mins_data/GroundTruths/holonomic' \
+            2>/dev/null | tee "comparison_${{ matrix.config }}.txt"
+      - name: Parse to markdown
+        run: |
+          python3 .github/scripts/eval_report.py "${{ matrix.config }}" \
+            "comparison_${{ matrix.config }}.txt" > "report_${{ matrix.config }}.md"
+          cat "report_${{ matrix.config }}.md"
+      - name: Upload report snippet
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: report-${{ matrix.config }}
+          path: report_${{ matrix.config }}.md
+          retention-days: 1
+
+  # 3) Assemble all per-config snippets into one PR comment + job summary.
+  report:
+    name: "Post report"
+    needs: evaluate
+    if: always()
+    runs-on: ubuntu-latest
+    steps:
+      - name: Download report snippets
+        uses: actions/download-artifact@v4
+        with:
+          pattern: report-*
+          path: reports
+          merge-multiple: true
+      - name: Assemble report
+        run: |
+          {
+            echo "<!-- mins-eval-report -->"
+            echo "## Simulation regression report - master vs PR"
+            echo ""
+            echo "_${SEEDS:-8} seeds/config, sim \`${DATASET:-UD_Warehouse}\`, informational only (does not block merge)._"
+            echo "Delta is PR relative to master (mean). Lower is better for RMSE / Time; NEES closer to ~1-3 is well-calibrated."
+            echo ""
+            if ls reports/report_*.md >/dev/null 2>&1; then
+              for f in $(ls reports/report_*.md | sort); do cat "$f"; echo ""; done
+            else
+              echo "> Note: no report snippets were produced - check the \`evaluate\` jobs."
+            fi
+          } > final_report.md
+          cat final_report.md >> "$GITHUB_STEP_SUMMARY"
+      - name: Post / update PR comment
+        if: github.event_name == 'pull_request'
+        uses: actions/github-script@v7
+        with:
+          script: |
+            const fs = require('fs');
+            const body = fs.readFileSync('final_report.md', 'utf8');
+            const marker = '<!-- mins-eval-report -->';
+            const { data: comments } = await github.rest.issues.listComments({
+              owner: context.repo.owner, repo: context.repo.repo,
+              issue_number: context.issue.number,
+            });
+            const existing = comments.find(c => c.body.includes(marker));
+            if (existing) {
+              await github.rest.issues.updateComment({
+                owner: context.repo.owner, repo: context.repo.repo,
+                comment_id: existing.id, body,
+              });
+            } else {
+              await github.rest.issues.createComment({
+                owner: context.repo.owner, repo: context.repo.repo,
+                issue_number: context.issue.number, body,
+              });
+            }

ReadMe.md

@@ -1,5 +1,5 @@
 # MINS
-[![Docker Image CI](https://github.com/rpng/MINS/actions/workflows/docker-image.yml/badge.svg)](https://github.com/rpng/MINS/actions/workflows/docker-image.yml)
+[![ROS 1 Workflow](https://github.com/rpng/MINS/actions/workflows/build_ros1.yml/badge.svg)](https://github.com/rpng/MINS/actions/workflows/build_ros1.yml)
 
 An efficient, robust, and tightly-coupled **Multisensor-aided Inertial Navigation System (MINS)** which is capable of 
 flexibly fusing all five sensing modalities (**IMU**, **wheel** **encoders**, **camera**, **GNSS**, and **LiDAR**) in a filtering