mfdz · derhuerst · May 1, 2026 · May 1, 2026 · May 1, 2026 · May 2, 2026
diff --git a/generate_gtfs_index.sh b/generate_gtfs_index.sh
@@ -9,45 +9,66 @@ cat << EOF
 <meta charset='utf-8'/>
 <meta name='viewport' content='width=device-width, initial-scale=1.0, user-scalable=no'/>
 <style>
+table { border-collapse: collapse; }
+table thead th, table tbody td { padding: .1em .2em; }
+table thead tr:last-child th { border-bottom: 1px solid #222; }
+table tbody tr:nth-child(odd) { background-color: #f0f0f0; }
+.border-left { border-left: 1px solid #222; padding-left: .2em; }
 .errors { text-align: right; color: rgb(255, 0, 0); }
 .warnings { text-align: right; color: rgb(255, 120, 0) }
 </style>
 <title>GTFS-Publikationen</title></head>
 <body><h1>GTFS-Publikationen</h1>
-<p>Nachfolgend sind f&uuml;r die uns derzeit bekannten GTFS-Ver&ouml;ffentlichungen deutscher Verkehrsunternehmen und- verb&uuml;nde die
-Ergebnisse der GTFSVTOR-Pr&uuml;fung mit dem <a href="https://github.com/mecatran/gtfsvtor">Mecatran GTFSVTOR</a> Validator von Laurent Grégoire aufgelistet.</p>
+<p>Nachfolgend sind f&uuml;r die uns derzeit bekannten GTFS-Ver&ouml;ffentlichungen deutscher Verkehrsunternehmen und- verb&uuml;nde die Ergebnisse der GTFSVTOR-Prüfung mittels <a href="https://github.com/mecatran/gtfsvtor">GTFSVTOR</a> und <a href="https://github.com/MobilityData/gtfs-validator">GTFS Validator</a> aufgelistet.</p>
 <p><b>HINWEIS</b>: Einige Verkehrsverb&uuml;nde ver&ouml;ffentlichen Datens&auml;tze derzeit unter einer versionsbezogenen URL. VBB und HVV rufen wir nicht automatisiert ab,
 da Last-Modified/If-Modified-Since derzeit nicht unterst&uuml;tzt werden bzw. der Datensatz nicht unter eine permanten URL bereitgestellt wird.
 F&uuml;r diese k&ouml;nnen wir nicht automatisch die aktuellste Version pr&uuml;fen und hier listen. Wir freuen uns &uuml;ber einen Hinweis, sollte es aktuellere Daten oder auch
 weitere Datenquellen geben.</p>
 <p>Feedback bitte an "hb at mfdz de"</p>
-<table><tr>
+<table>
+<thead>
+<tr>
+  <th colspan="5"></th>
+  <th colspan="3" class="border-left">Validierung GTFSVTOR</th>
+  <th colspan="3" class="border-left">Validierung GTFS Validator</th>
+</tr>
+<tr>
   <th>Verbund</th>
   <th>Datum</th>
   <th>Lizenz</th>
   <th>Namensnennung</th>
   <th>Download</th>
-  <th>Validierung</th>
+  <th class="border-left">Report</th>
+  <th>Fehler</th>
+  <th>Warnungen</th>
+  <th class="border-left">Report</th>
   <th>Fehler</th>
   <th>Warnungen</th>
 </tr>
+</thead>
+<tbody>
 EOF
 
 while IFS=';' read -r name lizenz nammensnennung permanent downloadurl infourl email addshapes
 do
   if [ "$name" == "shortname" ]; then continue; fi
 
-  ERRORS=""
-  WARNINGS=""
-  ERROR_REGEX='^.* ([1-9][0-9]*) ERROR.*$'
-  WARNING_REGEX='^.* ([0-9]*) WARNING.*$'
-  if [[ `cat $GTFS_DIR/$name.gtfsvtor.log` =~ $ERROR_REGEX ]]; then
-    ERRORS=${BASH_REMATCH[1]}
+  GTFSVTOR_ERRORS=""
+  GTFSVTOR_WARNINGS=""
+  GTFSVTOR_ERROR_REGEX='^.* ([1-9][0-9]*) ERROR.*$'
+  GTFSVTOR_WARNING_REGEX='^.* ([0-9]*) WARNING.*$'
+  if [[ `cat $GTFS_DIR/$name.gtfsvtor.log` =~ $GTFSVTOR_ERROR_REGEX ]]; then
+    GTFSVTOR_ERRORS=${BASH_REMATCH[1]}
   fi
-  if [[ `cat $GTFS_DIR/$name.gtfsvtor.log` =~ $WARNING_REGEX ]]; then
-    WARNINGS=${BASH_REMATCH[1]}
+  if [[ `cat $GTFS_DIR/$name.gtfsvtor.log` =~ $GTFSVTOR_WARNING_REGEX ]]; then
+    GTFSVTOR_WARNINGS=${BASH_REMATCH[1]}
   fi
-  1>&2 echo "$name: $ERRORS errors, $WARNINGS warnings"
+
+  gtfs_validator_report="$GTFS_DIR/$name.raw.gtfs.zip.gtfs-validator-result/report.json"
+  GTFS_VALIDATOR_ERRORS="$(jq -rc 'add(.notices[] | select(.severity == "ERROR") | .totalNotices) // 0' "$gtfs_validator_report")"
+  GTFS_VALIDATOR_WARNINGS="$(jq -rc 'add(.notices[] | select(.severity == "WARNING") | .totalNotices) // 0' "$gtfs_validator_report")"
+
+  1>&2 echo "$name: $GTFSVTOR_ERRORS/$GTFS_VALIDATOR_ERRORS errors, $GTFSVTOR_WARNINGS/$GTFS_VALIDATOR_WARNINGS warnings"
 
   cat << EOF
   <tr>
@@ -56,14 +77,18 @@ do
           <td>$lizenz</td>
           <td>$nammensnennung</td>
           <td><a href="$downloadurl">Download</a></td>
-          <td><a href="gtfsvtor_$name.html">Report</a></td>
-          <td class='errors'>$ERRORS</td>
-          <td class='warnings'>$WARNINGS</td>
+          <td class="border-left"><a href="gtfsvtor_$name.html">Report</a></td>
+          <td class='errors'>$GTFSVTOR_ERRORS</td>
+          <td class='warnings'>$GTFSVTOR_WARNINGS</td>
+          <td class="border-left"><a href="gtfs_validator_$name.html">Report</a></td>
+          <td class='errors'>$GTFS_VALIDATOR_ERRORS</td>
+          <td class='warnings'>$GTFS_VALIDATOR_WARNINGS</td>
         </tr>
 EOF
 done
 
 cat << EOF
+</tbody>
 </table>
 
 <p>Unter <a href='https://www.github.com/mfdz/GTFS-Issues'>github/mfdz/GTFS-Issues</a> sind weitere Probleme oder Erweiterungswünsche

diff --git a/makefile b/makefile
@@ -1,3 +1,5 @@
+THREADS:=$(shell nproc)
+
 HOST_MOUNT = $(shell set +e +u; if [ -n "$$HOST_MOUNT" ]; then echo "$$HOST_MOUNT"; else echo "$$PWD"; fi)
 TOOL_CFG = /cfg
 TOOL_DATA = /data
@@ -7,11 +9,13 @@ TAIL = $(shell set +e; if [ -x "$$(which gtail)" ]; then echo gtail; else echo t
 GTFS_FEEDS = $(shell cat config/gtfs-feeds.csv | $(TAIL) -n +2 | awk -F';' '{print $$1}' | tr '\n' ' ')
 RAW_GTFS_FEEDS = $(GTFS_FEEDS:%=data/gtfs/%.raw.gtfs.zip)
 GTFS_FEEDS_WITH_SHAPES = $(GTFS_FEEDS:%=data/gtfs/%.with_shapes.gtfs)
-GTFS_VALIDATION_RESULTS = $(GTFS_FEEDS:%=data/www/gtfsvtor_%.html)
+GTFSVTOR_RESULTS = $(GTFS_FEEDS:%=data/www/gtfsvtor_%.html)
+GTFS_VALIDATOR_RESULTS = $(GTFS_FEEDS:%=data/www/gtfs_validator_%.html)
 
 # Docker image definitions
 PFAEDLE_IMAGE=ghcr.io/ad-freiburg/pfaedle:2024-10-31t12-25
 GTFSVTOR_IMAGE=mfdz/gtfsvtor
+GTFS_VALIDATOR_IMAGE=ghcr.io/mobilitydata/gtfs-validator:7.1.0
 OBA_TRANSFORMER_IMAGE=mfdz/onebusaway-gtfs-transformer-cli:4.0.1-SNAPSHOT
 OBA_MERGE_IMAGE=mfdz/onebusaway-gtfs-merge-cli:4.0.1-SNAPSHOT
 GTFSTIDY_IMAGE=derhuerst/gtfstidy
@@ -44,7 +48,9 @@ TRANSFORM = docker run -i --rm -v $(HOST_MOUNT)/config/gtfs-rules:$(TOOL_CFG) -v
 PFAEDLE = docker run -i --rm -v $(HOST_MOUNT)/config:$(TOOL_CFG) -v $(HOST_MOUNT)/data/osm:$(TOOL_DATA)/osm -v $(HOST_MOUNT)/data/gtfs:$(TOOL_DATA)/gtfs $(PFAEDLE_IMAGE)
 MERGE = docker run -v $(HOST_MOUNT)/data/gtfs:$(TOOL_DATA)/gtfs --rm  -e JAVA_TOOL_OPTIONS="-Xmx18g" $(OBA_MERGE_IMAGE) --file=stops.txt --duplicateDetection=identity 
 GTFSVTOR = docker run -i --rm -v $(HOST_MOUNT)/data/gtfs:$(TOOL_DATA)/gtfs -v $(HOST_MOUNT)/data/www:$(TOOL_DATA)/www -e GTFSVTOR_OPTS=-Xmx8G $(GTFSVTOR_IMAGE)
+GTFS_VALIDATOR = docker run -i --rm --memory 8g --platform linux/amd64 -v $(HOST_MOUNT)/data/gtfs:$(TOOL_DATA)/gtfs -v $(HOST_MOUNT)/data/www:$(TOOL_DATA)/www $(GTFS_VALIDATOR_IMAGE)
 GTFSTIDY = docker run -i --rm -v $(HOST_MOUNT)/data/gtfs:$(TOOL_DATA)/gtfs $(GTFSTIDY_IMAGE)
+GTFS_VALIDATOR_REPORT_AS_METRICS = docker run -i --rm -v $(HOST_MOUNT)/data/gtfs:$(TOOL_DATA)/gtfs ghcr.io/mobidata-bw/gtfs-validator-metrics-service:1.3.0 /app/format-metrics.js
 
 
 # Download/Update OSM extracts from Geofabrik
@@ -221,14 +227,31 @@ data/gtfs/%.gtfs.zip: data/gtfs/%.with_shapes.gtfs.zip
 	$(info symlinking $(@F) -> $(<F))
 	ln -f "$<" "$@"
 
+data/gtfs/%.gtfs.zip.gtfs-validator-result: data/gtfs/%.gtfs.zip data/gtfs/%.gtfs
+	rm -rf "$@"
+	$(info running canonical GTFS Validator on the $* GTFS feed)
+	# todo: pass `--country_code` with value defined in config/gtfs-feeds.csv
+	$(GTFS_VALIDATOR) \
+		--date "$$(unzip -v -qq "$<" routes.txt | awk '{print $$5}' | awk -F- '{print $$3 "-" $$1 "-" $$2}')" \
+		--input "$<" \
+		--output_base "$@" \
+		--pretty \
+		--skip_validator_update \
+		--threads $(THREADS)
+data/www/gtfs_validator_%.html: data/gtfs/%.raw.gtfs.zip.gtfs-validator-result
+	cp "$</report.html" "$@"
+grafana_gtfs_validator_metrics_%: data/gtfs/%.raw.gtfs.zip.gtfs-validator-result
+	$(GTFS_VALIDATOR_REPORT_AS_METRICS) "$(TOOL_DATA)/gtfs/$(<F)/report.json" \
+		| ./scripts/push-to-prometheus-pushgateway.sh "$(<F)"
+
 data/www/gtfsvtor_%.html: data/gtfs/%.raw.gtfs
 	$(info running GTFSVTOR on the $* GTFS feed)
 	2>/dev/null $(GTFSVTOR) -o $(TOOL_DATA)/www/$(@F) -p -l 1000 $(TOOL_DATA)/gtfs/$(<F) | $(TAIL) -1 >data/gtfs/$*.gtfsvtor.log
 
 download: $(RAW_GTFS_FEEDS)
 	$(info Downloaded feeds)
 
-data/www/index.html: $(RAW_GTFS_FEEDS) $(GTFS_VALIDATION_RESULTS)
+data/www/index.html: $(RAW_GTFS_FEEDS) $(GTFSVTOR_RESULTS) $(GTFS_VALIDATOR_RESULTS)
 	$(info generating GTFS feed index from $(^F))
 	./generate_gtfs_index.sh <config/gtfs-feeds.csv >data/www/index.html
 

diff --git a/metrics/.gitignore b/metrics/.gitignore
@@ -0,0 +1,2 @@
+grafana-data
+prometheus-data
diff --git a/metrics/docker-compose.yml b/metrics/docker-compose.yml
@@ -0,0 +1,25 @@
+name: metrics
+
+services:
+  prometheus:
+    image: docker.io/prom/prometheus:v3-distroless
+    ports:
+      - '127.0.0.1:9090:9090'
+      - '[::1]:9090:9090'
+    volumes:
+      - ./prometheus.yml:/etc/prometheus/prometheus.yml
+      - ./prometheus-data:/var/lib/prometheus
+
+  prometheus_pushgateway:
+    image: docker.io/prom/pushgateway # todo: pin?
+    ports:
+      - '127.0.0.1:9091:9091'
+      - '[::1]:9091:9091'
+
+  grafana:
+    image: docker.io/grafana/grafana:main # todo: pin?
+    ports:
+      - '127.0.0.1:3000:3000'
+      - '[::1]:3000:3000'
+    volumes:
+      - ./grafana-data:/var/lib/grafana
diff --git a/metrics/prometheus.yml b/metrics/prometheus.yml
@@ -0,0 +1,13 @@
+storage:
+  tsdb:
+    retention:
+      size: 1GB
+
+scrape_configs:
+  - job_name: pushgateway
+    scrape_interval: 10s
+    scrape_timeout: 5s
+    honor_labels: true
+    static_configs:
+      - targets:
+          - 'prometheus_pushgateway:9091'
diff --git a/scripts/push-to-prometheus-pushgateway.sh b/scripts/push-to-prometheus-pushgateway.sh
@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+curl -fsS --data-binary '@-' "http://localhost:9091/metrics/job/${1:?missing job name (1st arguement)}"