diff --git a/.github/scripts/__pycache__/eval_report.cpython-38.pyc b/.github/scripts/__pycache__/eval_report.cpython-38.pyc new file mode 100644 index 0000000..d9d5e27 Binary files /dev/null and b/.github/scripts/__pycache__/eval_report.cpython-38.pyc differ diff --git a/.github/scripts/eval_report.py b/.github/scripts/eval_report.py new file mode 100644 index 0000000..c0cfa98 --- /dev/null +++ b/.github/scripts/eval_report.py @@ -0,0 +1,89 @@ +#!/usr/bin/env python3 +"""Render a master-vs-PR markdown block from mins_eval run_comparison output. + +Parses the LaTeX ATE/NEES/Time table that `run_comparison` (viz_type:=0) prints +and emits a collapsible markdown section comparing the MINS_master and MINS_PR +"algorithms" for one sensor config. Informational only -- never fails the build. + +Usage: eval_report.py +""" +import re +import sys + +FLOAT = re.compile(r"-?\d+\.?\d*(?:[eE][-+]?\d+)?") + + +def parse(path): + """Return {'master': {...}, 'pr': {...}} with rmse/nees/time number lists.""" + rows = {} + with open(path, "r", errors="replace") as fh: + for line in fh: + line = line.strip() + m = re.match(r"^MINS\\?_(\w+)\s*&(.*)", line) + if not m: + continue + alg = m.group(1).lower() # master | pr + rest = m.group(2).split("\\\\")[0] # drop everything after the row terminator + rest = (rest.replace("\\textbf{", "") + .replace("\\colorbox{orange}{", "") + .replace("}", "")) + cells = rest.split("&") + nums = lambda s: [float(x) for x in FLOAT.findall(s)] + rows[alg] = { + "rmse": nums(cells[0]) if len(cells) > 0 else [], # [ori_mean, ori_std, pos_mean, pos_std] + "nees": nums(cells[1]) if len(cells) > 1 else [], # [ori_mean, ori_std, pos_mean, pos_std] + "time": nums(cells[2]) if len(cells) > 2 else [], # [mean, std] + } + return rows + + +def cell(mean, std): + return f"{mean:.3f} +/- {std:.3f}" + + +def delta(pr_mean, ma_mean): + if ma_mean == 0: + return "n/a" + pct = (pr_mean - ma_mean) / abs(ma_mean) * 100.0 + sign = "+" if pct >= 0 else "" + return f"{sign}{pct:.1f}%" + + +def main(): + config = sys.argv[1] + rows = parse(sys.argv[2]) + out = [f"
{config}\n"] + + if "master" not in rows or "pr" not in rows: + out.append(f"> Note: could not parse results for `{config}` " + "(a master or PR run may have failed - see job logs).\n") + out.append("
") + print("\n".join(out)) + return + + out.append("| Metric | master | PR | Delta (mean) |") + out.append("|---|---|---|---|") + + def row(label, key, mi, si): + ma, pr = rows["master"][key], rows["pr"][key] + if len(ma) <= mi or len(pr) <= mi: + return + out.append(f"| {label} | {cell(ma[mi], ma[si])} | {cell(pr[mi], pr[si])} " + f"| {delta(pr[mi], ma[mi])} |") + + row("RMSE ori (deg)", "rmse", 0, 1) + row("RMSE pos (m)", "rmse", 2, 3) + row("NEES ori", "nees", 0, 1) + row("NEES pos", "nees", 2, 3) + row("Time (s)", "time", 0, 1) + out.append("\n") + print("\n".join(out)) + + +if __name__ == "__main__": + try: + main() + except Exception as exc: # never break the report job + cfg = sys.argv[1] if len(sys.argv) > 1 else "?" + print(f"
{cfg}\n\n" + f"> Note: report parsing error: `{exc}`\n\n
") diff --git a/.github/workflows/build_ros1.yml b/.github/workflows/build_ros1.yml new file mode 100644 index 0000000..da94ea2 --- /dev/null +++ b/.github/workflows/build_ros1.yml @@ -0,0 +1,37 @@ +name: ROS 1 Workflow + +on: + push: + branches: [ "master" ] + pull_request: + branches: [ "master" ] + +jobs: + + build_1804: + name: "ROS1 Ubuntu 18.04" + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Build the Docker image (Melodic 18.04) + run: docker build . --file Dockerfile_melodic_18_04 --tag mins:melodic + - name: Run MINS simulation (smoke test) + run: docker run --rm --entrypoint /bin/bash mins:melodic -c "source /opt/ros/melodic/setup.bash && source /catkin_ws/devel/setup.bash && export OMP_NUM_THREADS=1 && roslaunch mins simulation.launch sys_verbosity:=3" + + build_2004: + name: "ROS1 Ubuntu 20.04" + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Build the Docker image (Noetic 20.04) + run: docker build . --file Dockerfile_noetic_20_04 --tag mins:noetic + - name: Run MINS simulation (smoke test) + run: docker run --rm --entrypoint /bin/bash mins:noetic -c "source /opt/ros/noetic/setup.bash && source /catkin_ws/devel/setup.bash && export OMP_NUM_THREADS=1 && roslaunch mins simulation.launch sys_verbosity:=3" + + # build_1604 ("ROS1 Ubuntu 16.04", Kinetic): PARKED - revisit later. + # MINS's LiDAR dependencies (libpointmatcher / libnabo require Eigen >= 3.3, while + # Ubuntu 16.04/Xenial ships Eigen 3.2) plus the modern PCL/Ceres usage make a clean + # 16.04 build unlikely without dedicated dependency work. To re-enable: + # 1. Add Dockerfile_kinetic_16_04 (FROM osrf/ros:kinetic-perception, python-catkin-tools, + # Eigen upgraded to 3.3.7 from source), and + # 2. Add a build_1604 job here mirroring the two above. diff --git a/.github/workflows/docker-image.yml b/.github/workflows/docker-image.yml deleted file mode 100644 index c18a82c..0000000 --- a/.github/workflows/docker-image.yml +++ /dev/null @@ -1,27 +0,0 @@ -name: Docker Image CI - -on: - push: - branches: [ "master" ] - pull_request: - branches: [ "master" ] - -jobs: - - build-melodic: - - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v3 - - name: Build the Docker image - run: docker build . --file Dockerfile_melodic_18_04 --tag mins:melodic - - build-noetic: - - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v3 - - name: Build the Docker image - run: docker build . --file Dockerfile_noetic_20_04 --tag mins:noetic diff --git a/.github/workflows/evaluation.yml b/.github/workflows/evaluation.yml new file mode 100644 index 0000000..e9d2346 --- /dev/null +++ b/.github/workflows/evaluation.yml @@ -0,0 +1,186 @@ +name: Evaluation Report + +# Simulation-based regression REPORT (master vs PR). Informational only: +# it posts a per-sensor accuracy / consistency / timing comparison to the PR so +# the contributor is aware of any change. It is NOT a required check and never +# blocks merge. Keep it off the branch-protection required-checks list. + +on: + pull_request: + branches: [ "master" ] + workflow_dispatch: + +permissions: + contents: read + pull-requests: write # to post / update the PR comment + +env: + SEEDS: 8 # simulations per algorithm per sensor config + DATASET: UD_Warehouse # bundled B-spline sim trajectory + +jobs: + + # 1) Build the master-baseline and PR Noetic images once each (parallel). + build: + name: "Build image (${{ matrix.ref }})" + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + include: + - ref: pr + checkout: "" # PR merge ref (default) + - ref: master + checkout: ${{ github.base_ref }} # master baseline + steps: + - name: Checkout (${{ matrix.ref }}) + uses: actions/checkout@v4 + with: + ref: ${{ matrix.checkout }} + submodules: recursive # GroundTruths needed for eval + - name: Free disk space + run: | + sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /opt/hostedtoolcache/CodeQL /usr/local/share/boost /opt/microsoft || true + df -h / + - name: Build Docker image + run: docker build . --file Dockerfile_noetic_20_04 --tag mins:${{ matrix.ref }} + - name: Export image + run: docker save mins:${{ matrix.ref }} | gzip > image_${{ matrix.ref }}.tar.gz + - name: Upload image artifact + uses: actions/upload-artifact@v4 + with: + name: image-${{ matrix.ref }} + path: image_${{ matrix.ref }}.tar.gz + retention-days: 1 + + # 2) Per sensor config (parallel): run SEEDS sims for master & PR, then compare. + evaluate: + name: "Eval ${{ matrix.config }}" + needs: build + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + include: + - config: vio_stereo + args: "cam_enabled:=true cam_use_stereo:=true cam_max_n:=2 lidar_enabled:=false gps_enabled:=false wheel_enabled:=false vicon_enabled:=false" + - config: stereo_lidar + args: "cam_enabled:=true cam_use_stereo:=true cam_max_n:=2 lidar_enabled:=true gps_enabled:=false wheel_enabled:=false vicon_enabled:=false" + - config: stereo_gps + args: "cam_enabled:=true cam_use_stereo:=true cam_max_n:=2 lidar_enabled:=false gps_enabled:=true wheel_enabled:=false vicon_enabled:=false" + - config: stereo_wheel + args: "cam_enabled:=true cam_use_stereo:=true cam_max_n:=2 lidar_enabled:=false gps_enabled:=false wheel_enabled:=true vicon_enabled:=false" + - config: full + args: "cam_enabled:=true cam_use_stereo:=true cam_max_n:=2 lidar_enabled:=true gps_enabled:=true wheel_enabled:=true vicon_enabled:=false" + steps: + - name: Checkout (for report script) + uses: actions/checkout@v4 + - name: Free disk space + run: | + sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /opt/hostedtoolcache/CodeQL /usr/local/share/boost /opt/microsoft || true + df -h / + - name: Download images + uses: actions/download-artifact@v4 + with: + pattern: image-* + path: images + - name: Load images + run: | + docker load < images/image-master/image_master.tar.gz + docker load < images/image-pr/image_pr.tar.gz + - name: Run ${{ env.SEEDS }} simulations per algorithm + run: | + set -e + last=$((SEEDS-1)) + for ref in master pr; do + mkdir -p out_$ref + docker run --rm --entrypoint /bin/bash -v "$PWD/out_$ref:/outputs" "mins:$ref" -c ' + source /opt/ros/noetic/setup.bash && source /catkin_ws/devel/setup.bash + export OMP_NUM_THREADS=1 + for s in $(seq 0 '"$last"'); do + rm -f /outputs/tmp/$s.txt /outputs/tmp/$s.time + roslaunch mins simulation.launch sim_seed:=$s sys_verbosity:=3 \ + sys_save_state:=false '"${{ matrix.args }}"' + done' + done + - name: Arrange results for run_comparison + run: | + for ref in master pr; do + mkdir -p "alg/MINS_$ref/${DATASET}" + cp out_$ref/tmp/*.txt out_$ref/tmp/*.time "alg/MINS_$ref/${DATASET}/" 2>/dev/null || true + done + echo "Staged result tree:"; ls -R alg || true + - name: Run comparison (master vs PR) + run: | + docker run --rm --entrypoint /bin/bash -v "$PWD/alg:/alg" mins:pr -c ' + source /opt/ros/noetic/setup.bash && source /catkin_ws/devel/setup.bash + roslaunch mins_eval comparison.launch viz_type:=0 align_mode:=se3 \ + path_alg:=/alg \ + path_gts:=/catkin_ws/src/MINS/mins_data/GroundTruths/holonomic' \ + 2>/dev/null | tee "comparison_${{ matrix.config }}.txt" + - name: Parse to markdown + run: | + python3 .github/scripts/eval_report.py "${{ matrix.config }}" \ + "comparison_${{ matrix.config }}.txt" > "report_${{ matrix.config }}.md" + cat "report_${{ matrix.config }}.md" + - name: Upload report snippet + if: always() + uses: actions/upload-artifact@v4 + with: + name: report-${{ matrix.config }} + path: report_${{ matrix.config }}.md + retention-days: 1 + + # 3) Assemble all per-config snippets into one PR comment + job summary. + report: + name: "Post report" + needs: evaluate + if: always() + runs-on: ubuntu-latest + steps: + - name: Download report snippets + uses: actions/download-artifact@v4 + with: + pattern: report-* + path: reports + merge-multiple: true + - name: Assemble report + run: | + { + echo "" + echo "## Simulation regression report - master vs PR" + echo "" + echo "_${SEEDS:-8} seeds/config, sim \`${DATASET:-UD_Warehouse}\`, informational only (does not block merge)._" + echo "Delta is PR relative to master (mean). Lower is better for RMSE / Time; NEES closer to ~1-3 is well-calibrated." + echo "" + if ls reports/report_*.md >/dev/null 2>&1; then + for f in $(ls reports/report_*.md | sort); do cat "$f"; echo ""; done + else + echo "> Note: no report snippets were produced - check the \`evaluate\` jobs." + fi + } > final_report.md + cat final_report.md >> "$GITHUB_STEP_SUMMARY" + - name: Post / update PR comment + if: github.event_name == 'pull_request' + uses: actions/github-script@v7 + with: + script: | + const fs = require('fs'); + const body = fs.readFileSync('final_report.md', 'utf8'); + const marker = ''; + const { data: comments } = await github.rest.issues.listComments({ + owner: context.repo.owner, repo: context.repo.repo, + issue_number: context.issue.number, + }); + const existing = comments.find(c => c.body.includes(marker)); + if (existing) { + await github.rest.issues.updateComment({ + owner: context.repo.owner, repo: context.repo.repo, + comment_id: existing.id, body, + }); + } else { + await github.rest.issues.createComment({ + owner: context.repo.owner, repo: context.repo.repo, + issue_number: context.issue.number, body, + }); + } diff --git a/ReadMe.md b/ReadMe.md index 7df51dd..1747d1a 100644 --- a/ReadMe.md +++ b/ReadMe.md @@ -1,5 +1,5 @@ # MINS -[![Docker Image CI](https://github.com/rpng/MINS/actions/workflows/docker-image.yml/badge.svg)](https://github.com/rpng/MINS/actions/workflows/docker-image.yml) +[![ROS 1 Workflow](https://github.com/rpng/MINS/actions/workflows/build_ros1.yml/badge.svg)](https://github.com/rpng/MINS/actions/workflows/build_ros1.yml) An efficient, robust, and tightly-coupled **Multisensor-aided Inertial Navigation System (MINS)** which is capable of flexibly fusing all five sensing modalities (**IMU**, **wheel** **encoders**, **camera**, **GNSS**, and **LiDAR**) in a filtering