From 20f0f90fbe65738a7848908c3033409d981234f0 Mon Sep 17 00:00:00 2001 From: Kevin Duret Date: Mon, 13 Jul 2026 08:11:21 +0200 Subject: [PATCH 01/49] test(ci): deliver plugins to Pulp unstable only TEMP test branch (do not merge). In plugins.yml: - disable the Artifactory deliver-packages job (if: false) - force the Pulp deliver to stability: unstable, drop the stability gates (a PR runs with canary stability) and decouple the twin from test-plugins - add a marker file in the os::linux::snmp plugin packaging so get-plugins packages something to deliver All changes tagged TEMP(test-pulp-unstable). (cherry picked from commit 9174df176acb2032243f3208c5c85b4279bc7fb6) --- .github/workflows/plugins.yml | 18 +++++++----------- .../TEMP-test-pulp-unstable.md | 2 ++ 2 files changed, 9 insertions(+), 11 deletions(-) create mode 100644 packaging/centreon-plugin-Operatingsystems-Linux-Snmp/TEMP-test-pulp-unstable.md diff --git a/.github/workflows/plugins.yml b/.github/workflows/plugins.yml index 9c622b7850..55c7e46143 100644 --- a/.github/workflows/plugins.yml +++ b/.github/workflows/plugins.yml @@ -434,13 +434,8 @@ jobs: deliver-packages: needs: [get-environment, get-plugins, test-plugins] - if: | - needs.get-environment.outputs.skip_workflow == 'false' && - needs.get-plugins.outputs.package_plugins == 'True' && - (contains(fromJson('["testing", "unstable"]'), needs.get-environment.outputs.stability) || (needs.get-environment.outputs.stability == 'stable' && github.event_name != 'workflow_dispatch')) && - ! cancelled() && - ! contains(needs.*.result, 'failure') && - ! contains(needs.*.result, 'cancelled') + # TEMP(test-pulp-unstable): Pulp-only test, Artifactory delivery disabled + if: false runs-on: ubuntu-24.04 strategy: fail-fast: false @@ -479,11 +474,12 @@ jobs: artifactory_token: ${{ secrets.ARTIFACTORY_ACCESS_TOKEN }} deliver-packages-pulp: - needs: [get-environment, get-plugins, test-plugins] + # TEMP(test-pulp-unstable): decouple from test-plugins so the Pulp delivery + # runs regardless of the PR stability (canary) + needs: [get-environment, get-plugins, package] if: | needs.get-environment.outputs.skip_workflow == 'false' && needs.get-plugins.outputs.package_plugins == 'True' && - (contains(fromJson('["testing", "unstable"]'), needs.get-environment.outputs.stability) || (needs.get-environment.outputs.stability == 'stable' && github.event_name != 'workflow_dispatch')) && ! cancelled() && ! contains(needs.*.result, 'failure') && ! contains(needs.*.result, 'cancelled') @@ -566,7 +562,7 @@ jobs: core.setOutput('release_type', releaseType); - name: Deliver packages on Pulp - if: ${{ contains(fromJson('["testing", "unstable"]'), needs.get-environment.outputs.stability) }} + # TEMP(test-pulp-unstable): run on the PR (canary) stability too id: deliver_pulp continue-on-error: true uses: ./.github/actions/package-delivery-pulp @@ -576,7 +572,7 @@ jobs: distrib: ${{ matrix.distrib }} version: "" cache_key: ${{ github.sha }}-${{ github.run_id }}-${{ matrix.package_extension }}-${{ matrix.distrib }} - stability: ${{ needs.get-environment.outputs.stability }} + stability: unstable # TEMP(test-pulp-unstable): force unstable Pulp delivery release_type: ${{ needs.get-environment.outputs.release_type }} is_cloud: "false" pulp_url: ${{ vars.PULP_API_URL }} diff --git a/packaging/centreon-plugin-Operatingsystems-Linux-Snmp/TEMP-test-pulp-unstable.md b/packaging/centreon-plugin-Operatingsystems-Linux-Snmp/TEMP-test-pulp-unstable.md new file mode 100644 index 0000000000..6ea9934bf2 --- /dev/null +++ b/packaging/centreon-plugin-Operatingsystems-Linux-Snmp/TEMP-test-pulp-unstable.md @@ -0,0 +1,2 @@ +TEMP(test-pulp-unstable): marker file so get-plugins packages this plugin for +the Pulp delivery test; delete before merge. From 89f9673ce321ddd0438ef2d51cf78a21902e396e Mon Sep 17 00:00:00 2001 From: Kevin Duret Date: Mon, 13 Jul 2026 13:39:23 +0200 Subject: [PATCH 02/49] test(ci): mark 30 more plugins to exercise the batched Pulp delivery TEMP(test-pulp-unstable): reproduce the volume of the failed develop run (~30 packages per distrib) so the batched task wait is validated under a realistic load. --- .../TEMP-test-pulp-unstable.md | 2 ++ .../TEMP-test-pulp-unstable.md | 2 ++ .../TEMP-test-pulp-unstable.md | 2 ++ .../TEMP-test-pulp-unstable.md | 2 ++ .../TEMP-test-pulp-unstable.md | 2 ++ .../TEMP-test-pulp-unstable.md | 2 ++ .../TEMP-test-pulp-unstable.md | 2 ++ .../TEMP-test-pulp-unstable.md | 2 ++ .../TEMP-test-pulp-unstable.md | 2 ++ .../TEMP-test-pulp-unstable.md | 2 ++ .../TEMP-test-pulp-unstable.md | 2 ++ .../TEMP-test-pulp-unstable.md | 2 ++ .../TEMP-test-pulp-unstable.md | 2 ++ .../TEMP-test-pulp-unstable.md | 2 ++ .../TEMP-test-pulp-unstable.md | 2 ++ .../TEMP-test-pulp-unstable.md | 2 ++ .../TEMP-test-pulp-unstable.md | 2 ++ .../TEMP-test-pulp-unstable.md | 2 ++ .../TEMP-test-pulp-unstable.md | 2 ++ .../TEMP-test-pulp-unstable.md | 2 ++ .../TEMP-test-pulp-unstable.md | 2 ++ .../TEMP-test-pulp-unstable.md | 2 ++ .../TEMP-test-pulp-unstable.md | 2 ++ .../TEMP-test-pulp-unstable.md | 2 ++ .../TEMP-test-pulp-unstable.md | 2 ++ .../TEMP-test-pulp-unstable.md | 2 ++ .../TEMP-test-pulp-unstable.md | 2 ++ .../TEMP-test-pulp-unstable.md | 2 ++ .../TEMP-test-pulp-unstable.md | 2 ++ .../TEMP-test-pulp-unstable.md | 2 ++ 30 files changed, 60 insertions(+) create mode 100644 packaging/centreon-plugin-Applications-ActiveDirectory-Wsman/TEMP-test-pulp-unstable.md create mode 100644 packaging/centreon-plugin-Applications-ActiveMQ-Jmx/TEMP-test-pulp-unstable.md create mode 100644 packaging/centreon-plugin-Applications-Ansible-Cli/TEMP-test-pulp-unstable.md create mode 100644 packaging/centreon-plugin-Applications-Ansible-Tower/TEMP-test-pulp-unstable.md create mode 100644 packaging/centreon-plugin-Applications-Antivirus-Kaspersky-Snmp/TEMP-test-pulp-unstable.md create mode 100644 packaging/centreon-plugin-Applications-Antivirus-Mcafee-Webgateway-Snmp/TEMP-test-pulp-unstable.md create mode 100644 packaging/centreon-plugin-Applications-Antivirus-Skyhigh-Webgateway-Snmp/TEMP-test-pulp-unstable.md create mode 100644 packaging/centreon-plugin-Applications-Apache-Cxf-Jmx/TEMP-test-pulp-unstable.md create mode 100644 packaging/centreon-plugin-Applications-Awa-Jmx/TEMP-test-pulp-unstable.md create mode 100644 packaging/centreon-plugin-Applications-Backup-Emc-Recoverypoint-Ssh/TEMP-test-pulp-unstable.md create mode 100644 packaging/centreon-plugin-Applications-Backup-Veeam-Wsman/TEMP-test-pulp-unstable.md create mode 100644 packaging/centreon-plugin-Applications-Bind9-Web/TEMP-test-pulp-unstable.md create mode 100644 packaging/centreon-plugin-Applications-Biztalk/TEMP-test-pulp-unstable.md create mode 100644 packaging/centreon-plugin-Applications-Bluemind-Ssh/TEMP-test-pulp-unstable.md create mode 100644 packaging/centreon-plugin-Applications-Ceph-Restapi/TEMP-test-pulp-unstable.md create mode 100644 packaging/centreon-plugin-Applications-Cisco-Cms-Restapi/TEMP-test-pulp-unstable.md create mode 100644 packaging/centreon-plugin-Applications-Cisco-Dnac-Restapi/TEMP-test-pulp-unstable.md create mode 100644 packaging/centreon-plugin-Applications-Cisco-Ise-Restapi/TEMP-test-pulp-unstable.md create mode 100644 packaging/centreon-plugin-Applications-Cisco-Ssms-Restapi/TEMP-test-pulp-unstable.md create mode 100644 packaging/centreon-plugin-Applications-Clamav-Ssh/TEMP-test-pulp-unstable.md create mode 100644 packaging/centreon-plugin-Applications-Commvault-Commserve-Restapi/TEMP-test-pulp-unstable.md create mode 100644 packaging/centreon-plugin-Applications-Controlm-Restapi/TEMP-test-pulp-unstable.md create mode 100644 packaging/centreon-plugin-Applications-Database-Cassandra-Jmx/TEMP-test-pulp-unstable.md create mode 100644 packaging/centreon-plugin-Applications-Database-Couchdb-Restapi/TEMP-test-pulp-unstable.md create mode 100644 packaging/centreon-plugin-Applications-Databases-Db2/TEMP-test-pulp-unstable.md create mode 100644 packaging/centreon-plugin-Applications-Databases-Elasticsearch/TEMP-test-pulp-unstable.md create mode 100644 packaging/centreon-plugin-Applications-Databases-Firebird/TEMP-test-pulp-unstable.md create mode 100644 packaging/centreon-plugin-Applications-Databases-Influxdb/TEMP-test-pulp-unstable.md create mode 100644 packaging/centreon-plugin-Applications-Databases-Informix-Snmp/TEMP-test-pulp-unstable.md create mode 100644 packaging/centreon-plugin-Applications-Databases-Informix/TEMP-test-pulp-unstable.md diff --git a/packaging/centreon-plugin-Applications-ActiveDirectory-Wsman/TEMP-test-pulp-unstable.md b/packaging/centreon-plugin-Applications-ActiveDirectory-Wsman/TEMP-test-pulp-unstable.md new file mode 100644 index 0000000000..ff49431e12 --- /dev/null +++ b/packaging/centreon-plugin-Applications-ActiveDirectory-Wsman/TEMP-test-pulp-unstable.md @@ -0,0 +1,2 @@ +TEMP(test-pulp-unstable): marker file so get-plugins packages this plugin for +the Pulp batch-delivery test; delete before merge. diff --git a/packaging/centreon-plugin-Applications-ActiveMQ-Jmx/TEMP-test-pulp-unstable.md b/packaging/centreon-plugin-Applications-ActiveMQ-Jmx/TEMP-test-pulp-unstable.md new file mode 100644 index 0000000000..ff49431e12 --- /dev/null +++ b/packaging/centreon-plugin-Applications-ActiveMQ-Jmx/TEMP-test-pulp-unstable.md @@ -0,0 +1,2 @@ +TEMP(test-pulp-unstable): marker file so get-plugins packages this plugin for +the Pulp batch-delivery test; delete before merge. diff --git a/packaging/centreon-plugin-Applications-Ansible-Cli/TEMP-test-pulp-unstable.md b/packaging/centreon-plugin-Applications-Ansible-Cli/TEMP-test-pulp-unstable.md new file mode 100644 index 0000000000..ff49431e12 --- /dev/null +++ b/packaging/centreon-plugin-Applications-Ansible-Cli/TEMP-test-pulp-unstable.md @@ -0,0 +1,2 @@ +TEMP(test-pulp-unstable): marker file so get-plugins packages this plugin for +the Pulp batch-delivery test; delete before merge. diff --git a/packaging/centreon-plugin-Applications-Ansible-Tower/TEMP-test-pulp-unstable.md b/packaging/centreon-plugin-Applications-Ansible-Tower/TEMP-test-pulp-unstable.md new file mode 100644 index 0000000000..ff49431e12 --- /dev/null +++ b/packaging/centreon-plugin-Applications-Ansible-Tower/TEMP-test-pulp-unstable.md @@ -0,0 +1,2 @@ +TEMP(test-pulp-unstable): marker file so get-plugins packages this plugin for +the Pulp batch-delivery test; delete before merge. diff --git a/packaging/centreon-plugin-Applications-Antivirus-Kaspersky-Snmp/TEMP-test-pulp-unstable.md b/packaging/centreon-plugin-Applications-Antivirus-Kaspersky-Snmp/TEMP-test-pulp-unstable.md new file mode 100644 index 0000000000..ff49431e12 --- /dev/null +++ b/packaging/centreon-plugin-Applications-Antivirus-Kaspersky-Snmp/TEMP-test-pulp-unstable.md @@ -0,0 +1,2 @@ +TEMP(test-pulp-unstable): marker file so get-plugins packages this plugin for +the Pulp batch-delivery test; delete before merge. diff --git a/packaging/centreon-plugin-Applications-Antivirus-Mcafee-Webgateway-Snmp/TEMP-test-pulp-unstable.md b/packaging/centreon-plugin-Applications-Antivirus-Mcafee-Webgateway-Snmp/TEMP-test-pulp-unstable.md new file mode 100644 index 0000000000..ff49431e12 --- /dev/null +++ b/packaging/centreon-plugin-Applications-Antivirus-Mcafee-Webgateway-Snmp/TEMP-test-pulp-unstable.md @@ -0,0 +1,2 @@ +TEMP(test-pulp-unstable): marker file so get-plugins packages this plugin for +the Pulp batch-delivery test; delete before merge. diff --git a/packaging/centreon-plugin-Applications-Antivirus-Skyhigh-Webgateway-Snmp/TEMP-test-pulp-unstable.md b/packaging/centreon-plugin-Applications-Antivirus-Skyhigh-Webgateway-Snmp/TEMP-test-pulp-unstable.md new file mode 100644 index 0000000000..ff49431e12 --- /dev/null +++ b/packaging/centreon-plugin-Applications-Antivirus-Skyhigh-Webgateway-Snmp/TEMP-test-pulp-unstable.md @@ -0,0 +1,2 @@ +TEMP(test-pulp-unstable): marker file so get-plugins packages this plugin for +the Pulp batch-delivery test; delete before merge. diff --git a/packaging/centreon-plugin-Applications-Apache-Cxf-Jmx/TEMP-test-pulp-unstable.md b/packaging/centreon-plugin-Applications-Apache-Cxf-Jmx/TEMP-test-pulp-unstable.md new file mode 100644 index 0000000000..ff49431e12 --- /dev/null +++ b/packaging/centreon-plugin-Applications-Apache-Cxf-Jmx/TEMP-test-pulp-unstable.md @@ -0,0 +1,2 @@ +TEMP(test-pulp-unstable): marker file so get-plugins packages this plugin for +the Pulp batch-delivery test; delete before merge. diff --git a/packaging/centreon-plugin-Applications-Awa-Jmx/TEMP-test-pulp-unstable.md b/packaging/centreon-plugin-Applications-Awa-Jmx/TEMP-test-pulp-unstable.md new file mode 100644 index 0000000000..ff49431e12 --- /dev/null +++ b/packaging/centreon-plugin-Applications-Awa-Jmx/TEMP-test-pulp-unstable.md @@ -0,0 +1,2 @@ +TEMP(test-pulp-unstable): marker file so get-plugins packages this plugin for +the Pulp batch-delivery test; delete before merge. diff --git a/packaging/centreon-plugin-Applications-Backup-Emc-Recoverypoint-Ssh/TEMP-test-pulp-unstable.md b/packaging/centreon-plugin-Applications-Backup-Emc-Recoverypoint-Ssh/TEMP-test-pulp-unstable.md new file mode 100644 index 0000000000..ff49431e12 --- /dev/null +++ b/packaging/centreon-plugin-Applications-Backup-Emc-Recoverypoint-Ssh/TEMP-test-pulp-unstable.md @@ -0,0 +1,2 @@ +TEMP(test-pulp-unstable): marker file so get-plugins packages this plugin for +the Pulp batch-delivery test; delete before merge. diff --git a/packaging/centreon-plugin-Applications-Backup-Veeam-Wsman/TEMP-test-pulp-unstable.md b/packaging/centreon-plugin-Applications-Backup-Veeam-Wsman/TEMP-test-pulp-unstable.md new file mode 100644 index 0000000000..ff49431e12 --- /dev/null +++ b/packaging/centreon-plugin-Applications-Backup-Veeam-Wsman/TEMP-test-pulp-unstable.md @@ -0,0 +1,2 @@ +TEMP(test-pulp-unstable): marker file so get-plugins packages this plugin for +the Pulp batch-delivery test; delete before merge. diff --git a/packaging/centreon-plugin-Applications-Bind9-Web/TEMP-test-pulp-unstable.md b/packaging/centreon-plugin-Applications-Bind9-Web/TEMP-test-pulp-unstable.md new file mode 100644 index 0000000000..ff49431e12 --- /dev/null +++ b/packaging/centreon-plugin-Applications-Bind9-Web/TEMP-test-pulp-unstable.md @@ -0,0 +1,2 @@ +TEMP(test-pulp-unstable): marker file so get-plugins packages this plugin for +the Pulp batch-delivery test; delete before merge. diff --git a/packaging/centreon-plugin-Applications-Biztalk/TEMP-test-pulp-unstable.md b/packaging/centreon-plugin-Applications-Biztalk/TEMP-test-pulp-unstable.md new file mode 100644 index 0000000000..ff49431e12 --- /dev/null +++ b/packaging/centreon-plugin-Applications-Biztalk/TEMP-test-pulp-unstable.md @@ -0,0 +1,2 @@ +TEMP(test-pulp-unstable): marker file so get-plugins packages this plugin for +the Pulp batch-delivery test; delete before merge. diff --git a/packaging/centreon-plugin-Applications-Bluemind-Ssh/TEMP-test-pulp-unstable.md b/packaging/centreon-plugin-Applications-Bluemind-Ssh/TEMP-test-pulp-unstable.md new file mode 100644 index 0000000000..ff49431e12 --- /dev/null +++ b/packaging/centreon-plugin-Applications-Bluemind-Ssh/TEMP-test-pulp-unstable.md @@ -0,0 +1,2 @@ +TEMP(test-pulp-unstable): marker file so get-plugins packages this plugin for +the Pulp batch-delivery test; delete before merge. diff --git a/packaging/centreon-plugin-Applications-Ceph-Restapi/TEMP-test-pulp-unstable.md b/packaging/centreon-plugin-Applications-Ceph-Restapi/TEMP-test-pulp-unstable.md new file mode 100644 index 0000000000..ff49431e12 --- /dev/null +++ b/packaging/centreon-plugin-Applications-Ceph-Restapi/TEMP-test-pulp-unstable.md @@ -0,0 +1,2 @@ +TEMP(test-pulp-unstable): marker file so get-plugins packages this plugin for +the Pulp batch-delivery test; delete before merge. diff --git a/packaging/centreon-plugin-Applications-Cisco-Cms-Restapi/TEMP-test-pulp-unstable.md b/packaging/centreon-plugin-Applications-Cisco-Cms-Restapi/TEMP-test-pulp-unstable.md new file mode 100644 index 0000000000..ff49431e12 --- /dev/null +++ b/packaging/centreon-plugin-Applications-Cisco-Cms-Restapi/TEMP-test-pulp-unstable.md @@ -0,0 +1,2 @@ +TEMP(test-pulp-unstable): marker file so get-plugins packages this plugin for +the Pulp batch-delivery test; delete before merge. diff --git a/packaging/centreon-plugin-Applications-Cisco-Dnac-Restapi/TEMP-test-pulp-unstable.md b/packaging/centreon-plugin-Applications-Cisco-Dnac-Restapi/TEMP-test-pulp-unstable.md new file mode 100644 index 0000000000..ff49431e12 --- /dev/null +++ b/packaging/centreon-plugin-Applications-Cisco-Dnac-Restapi/TEMP-test-pulp-unstable.md @@ -0,0 +1,2 @@ +TEMP(test-pulp-unstable): marker file so get-plugins packages this plugin for +the Pulp batch-delivery test; delete before merge. diff --git a/packaging/centreon-plugin-Applications-Cisco-Ise-Restapi/TEMP-test-pulp-unstable.md b/packaging/centreon-plugin-Applications-Cisco-Ise-Restapi/TEMP-test-pulp-unstable.md new file mode 100644 index 0000000000..ff49431e12 --- /dev/null +++ b/packaging/centreon-plugin-Applications-Cisco-Ise-Restapi/TEMP-test-pulp-unstable.md @@ -0,0 +1,2 @@ +TEMP(test-pulp-unstable): marker file so get-plugins packages this plugin for +the Pulp batch-delivery test; delete before merge. diff --git a/packaging/centreon-plugin-Applications-Cisco-Ssms-Restapi/TEMP-test-pulp-unstable.md b/packaging/centreon-plugin-Applications-Cisco-Ssms-Restapi/TEMP-test-pulp-unstable.md new file mode 100644 index 0000000000..ff49431e12 --- /dev/null +++ b/packaging/centreon-plugin-Applications-Cisco-Ssms-Restapi/TEMP-test-pulp-unstable.md @@ -0,0 +1,2 @@ +TEMP(test-pulp-unstable): marker file so get-plugins packages this plugin for +the Pulp batch-delivery test; delete before merge. diff --git a/packaging/centreon-plugin-Applications-Clamav-Ssh/TEMP-test-pulp-unstable.md b/packaging/centreon-plugin-Applications-Clamav-Ssh/TEMP-test-pulp-unstable.md new file mode 100644 index 0000000000..ff49431e12 --- /dev/null +++ b/packaging/centreon-plugin-Applications-Clamav-Ssh/TEMP-test-pulp-unstable.md @@ -0,0 +1,2 @@ +TEMP(test-pulp-unstable): marker file so get-plugins packages this plugin for +the Pulp batch-delivery test; delete before merge. diff --git a/packaging/centreon-plugin-Applications-Commvault-Commserve-Restapi/TEMP-test-pulp-unstable.md b/packaging/centreon-plugin-Applications-Commvault-Commserve-Restapi/TEMP-test-pulp-unstable.md new file mode 100644 index 0000000000..ff49431e12 --- /dev/null +++ b/packaging/centreon-plugin-Applications-Commvault-Commserve-Restapi/TEMP-test-pulp-unstable.md @@ -0,0 +1,2 @@ +TEMP(test-pulp-unstable): marker file so get-plugins packages this plugin for +the Pulp batch-delivery test; delete before merge. diff --git a/packaging/centreon-plugin-Applications-Controlm-Restapi/TEMP-test-pulp-unstable.md b/packaging/centreon-plugin-Applications-Controlm-Restapi/TEMP-test-pulp-unstable.md new file mode 100644 index 0000000000..ff49431e12 --- /dev/null +++ b/packaging/centreon-plugin-Applications-Controlm-Restapi/TEMP-test-pulp-unstable.md @@ -0,0 +1,2 @@ +TEMP(test-pulp-unstable): marker file so get-plugins packages this plugin for +the Pulp batch-delivery test; delete before merge. diff --git a/packaging/centreon-plugin-Applications-Database-Cassandra-Jmx/TEMP-test-pulp-unstable.md b/packaging/centreon-plugin-Applications-Database-Cassandra-Jmx/TEMP-test-pulp-unstable.md new file mode 100644 index 0000000000..ff49431e12 --- /dev/null +++ b/packaging/centreon-plugin-Applications-Database-Cassandra-Jmx/TEMP-test-pulp-unstable.md @@ -0,0 +1,2 @@ +TEMP(test-pulp-unstable): marker file so get-plugins packages this plugin for +the Pulp batch-delivery test; delete before merge. diff --git a/packaging/centreon-plugin-Applications-Database-Couchdb-Restapi/TEMP-test-pulp-unstable.md b/packaging/centreon-plugin-Applications-Database-Couchdb-Restapi/TEMP-test-pulp-unstable.md new file mode 100644 index 0000000000..ff49431e12 --- /dev/null +++ b/packaging/centreon-plugin-Applications-Database-Couchdb-Restapi/TEMP-test-pulp-unstable.md @@ -0,0 +1,2 @@ +TEMP(test-pulp-unstable): marker file so get-plugins packages this plugin for +the Pulp batch-delivery test; delete before merge. diff --git a/packaging/centreon-plugin-Applications-Databases-Db2/TEMP-test-pulp-unstable.md b/packaging/centreon-plugin-Applications-Databases-Db2/TEMP-test-pulp-unstable.md new file mode 100644 index 0000000000..ff49431e12 --- /dev/null +++ b/packaging/centreon-plugin-Applications-Databases-Db2/TEMP-test-pulp-unstable.md @@ -0,0 +1,2 @@ +TEMP(test-pulp-unstable): marker file so get-plugins packages this plugin for +the Pulp batch-delivery test; delete before merge. diff --git a/packaging/centreon-plugin-Applications-Databases-Elasticsearch/TEMP-test-pulp-unstable.md b/packaging/centreon-plugin-Applications-Databases-Elasticsearch/TEMP-test-pulp-unstable.md new file mode 100644 index 0000000000..ff49431e12 --- /dev/null +++ b/packaging/centreon-plugin-Applications-Databases-Elasticsearch/TEMP-test-pulp-unstable.md @@ -0,0 +1,2 @@ +TEMP(test-pulp-unstable): marker file so get-plugins packages this plugin for +the Pulp batch-delivery test; delete before merge. diff --git a/packaging/centreon-plugin-Applications-Databases-Firebird/TEMP-test-pulp-unstable.md b/packaging/centreon-plugin-Applications-Databases-Firebird/TEMP-test-pulp-unstable.md new file mode 100644 index 0000000000..ff49431e12 --- /dev/null +++ b/packaging/centreon-plugin-Applications-Databases-Firebird/TEMP-test-pulp-unstable.md @@ -0,0 +1,2 @@ +TEMP(test-pulp-unstable): marker file so get-plugins packages this plugin for +the Pulp batch-delivery test; delete before merge. diff --git a/packaging/centreon-plugin-Applications-Databases-Influxdb/TEMP-test-pulp-unstable.md b/packaging/centreon-plugin-Applications-Databases-Influxdb/TEMP-test-pulp-unstable.md new file mode 100644 index 0000000000..ff49431e12 --- /dev/null +++ b/packaging/centreon-plugin-Applications-Databases-Influxdb/TEMP-test-pulp-unstable.md @@ -0,0 +1,2 @@ +TEMP(test-pulp-unstable): marker file so get-plugins packages this plugin for +the Pulp batch-delivery test; delete before merge. diff --git a/packaging/centreon-plugin-Applications-Databases-Informix-Snmp/TEMP-test-pulp-unstable.md b/packaging/centreon-plugin-Applications-Databases-Informix-Snmp/TEMP-test-pulp-unstable.md new file mode 100644 index 0000000000..ff49431e12 --- /dev/null +++ b/packaging/centreon-plugin-Applications-Databases-Informix-Snmp/TEMP-test-pulp-unstable.md @@ -0,0 +1,2 @@ +TEMP(test-pulp-unstable): marker file so get-plugins packages this plugin for +the Pulp batch-delivery test; delete before merge. diff --git a/packaging/centreon-plugin-Applications-Databases-Informix/TEMP-test-pulp-unstable.md b/packaging/centreon-plugin-Applications-Databases-Informix/TEMP-test-pulp-unstable.md new file mode 100644 index 0000000000..ff49431e12 --- /dev/null +++ b/packaging/centreon-plugin-Applications-Databases-Informix/TEMP-test-pulp-unstable.md @@ -0,0 +1,2 @@ +TEMP(test-pulp-unstable): marker file so get-plugins packages this plugin for +the Pulp batch-delivery test; delete before merge. From cd24137cdc47c998bdcae264ed5cb9f9c37f4834 Mon Sep 17 00:00:00 2001 From: Kevin Duret Date: Mon, 13 Jul 2026 13:43:19 +0200 Subject: [PATCH 03/49] test(ci): skip the test jobs on the Pulp delivery validation TEMP(test-pulp-unstable): unit-tests and test-plugins bring nothing to the batched-delivery validation and would consume up to 25 runners for the 31 marked plugins; fatpacker/package tolerate the skipped dependency. --- .github/workflows/plugins.yml | 19 ++++--------------- 1 file changed, 4 insertions(+), 15 deletions(-) diff --git a/.github/workflows/plugins.yml b/.github/workflows/plugins.yml index 55c7e46143..3ed68362d0 100644 --- a/.github/workflows/plugins.yml +++ b/.github/workflows/plugins.yml @@ -173,12 +173,8 @@ jobs: unit-tests: needs: [get-environment, get-plugins] - if: | - needs.get-environment.outputs.skip_workflow == 'false' && - needs.get-environment.outputs.stability != 'stable' && - ! cancelled() && - ! contains(needs.*.result, 'failure') && - ! contains(needs.*.result, 'cancelled') + # TEMP(test-pulp-unstable): skip the test jobs, this PR only validates the Pulp delivery + if: false strategy: fail-fast: false max-parallel: 3 @@ -363,15 +359,8 @@ jobs: test-plugins: needs: [get-environment, get-plugins, package] - if: | - always() && - needs.get-environment.outputs.skip_workflow == 'false' && - needs.get-plugins.outputs.test_plugins == 'True' && - needs.get-environment.outputs.stability != 'stable' && - (needs.package.result == 'success' || needs.package.result == 'skipped') && - ! cancelled() && - ! contains(needs.*.result, 'failure') && - ! contains(needs.*.result, 'cancelled') + # TEMP(test-pulp-unstable): skip the test jobs, this PR only validates the Pulp delivery + if: false strategy: fail-fast: false matrix: From 760d26b696fab3c17a01d6c2094f50373f33c27a Mon Sep 17 00:00:00 2001 From: Kevin Duret Date: Mon, 13 Jul 2026 13:53:05 +0200 Subject: [PATCH 04/49] test(ci): package every plugin for the Pulp delivery load test TEMP(test-pulp-unstable): replace the 31 packaging markers with a touch of the common packaging template (changes_common=true), so get-plugins lists all 675 plugins and the batched Pulp delivery is exercised at full nightly scale (~675 packages per distrib across the 8 parallel deliver jobs). --- .github/packaging/centreon-plugin.yaml.template | 1 + .../TEMP-test-pulp-unstable.md | 2 -- .../TEMP-test-pulp-unstable.md | 2 -- .../TEMP-test-pulp-unstable.md | 2 -- .../TEMP-test-pulp-unstable.md | 2 -- .../TEMP-test-pulp-unstable.md | 2 -- .../TEMP-test-pulp-unstable.md | 2 -- .../TEMP-test-pulp-unstable.md | 2 -- .../TEMP-test-pulp-unstable.md | 2 -- .../TEMP-test-pulp-unstable.md | 2 -- .../TEMP-test-pulp-unstable.md | 2 -- .../TEMP-test-pulp-unstable.md | 2 -- .../TEMP-test-pulp-unstable.md | 2 -- .../TEMP-test-pulp-unstable.md | 2 -- .../TEMP-test-pulp-unstable.md | 2 -- .../TEMP-test-pulp-unstable.md | 2 -- .../TEMP-test-pulp-unstable.md | 2 -- .../TEMP-test-pulp-unstable.md | 2 -- .../TEMP-test-pulp-unstable.md | 2 -- .../TEMP-test-pulp-unstable.md | 2 -- .../TEMP-test-pulp-unstable.md | 2 -- .../TEMP-test-pulp-unstable.md | 2 -- .../TEMP-test-pulp-unstable.md | 2 -- .../TEMP-test-pulp-unstable.md | 2 -- .../TEMP-test-pulp-unstable.md | 2 -- .../TEMP-test-pulp-unstable.md | 2 -- .../TEMP-test-pulp-unstable.md | 2 -- .../TEMP-test-pulp-unstable.md | 2 -- .../TEMP-test-pulp-unstable.md | 2 -- .../TEMP-test-pulp-unstable.md | 2 -- .../TEMP-test-pulp-unstable.md | 2 -- .../TEMP-test-pulp-unstable.md | 2 -- 32 files changed, 1 insertion(+), 62 deletions(-) delete mode 100644 packaging/centreon-plugin-Applications-ActiveDirectory-Wsman/TEMP-test-pulp-unstable.md delete mode 100644 packaging/centreon-plugin-Applications-ActiveMQ-Jmx/TEMP-test-pulp-unstable.md delete mode 100644 packaging/centreon-plugin-Applications-Ansible-Cli/TEMP-test-pulp-unstable.md delete mode 100644 packaging/centreon-plugin-Applications-Ansible-Tower/TEMP-test-pulp-unstable.md delete mode 100644 packaging/centreon-plugin-Applications-Antivirus-Kaspersky-Snmp/TEMP-test-pulp-unstable.md delete mode 100644 packaging/centreon-plugin-Applications-Antivirus-Mcafee-Webgateway-Snmp/TEMP-test-pulp-unstable.md delete mode 100644 packaging/centreon-plugin-Applications-Antivirus-Skyhigh-Webgateway-Snmp/TEMP-test-pulp-unstable.md delete mode 100644 packaging/centreon-plugin-Applications-Apache-Cxf-Jmx/TEMP-test-pulp-unstable.md delete mode 100644 packaging/centreon-plugin-Applications-Awa-Jmx/TEMP-test-pulp-unstable.md delete mode 100644 packaging/centreon-plugin-Applications-Backup-Emc-Recoverypoint-Ssh/TEMP-test-pulp-unstable.md delete mode 100644 packaging/centreon-plugin-Applications-Backup-Veeam-Wsman/TEMP-test-pulp-unstable.md delete mode 100644 packaging/centreon-plugin-Applications-Bind9-Web/TEMP-test-pulp-unstable.md delete mode 100644 packaging/centreon-plugin-Applications-Biztalk/TEMP-test-pulp-unstable.md delete mode 100644 packaging/centreon-plugin-Applications-Bluemind-Ssh/TEMP-test-pulp-unstable.md delete mode 100644 packaging/centreon-plugin-Applications-Ceph-Restapi/TEMP-test-pulp-unstable.md delete mode 100644 packaging/centreon-plugin-Applications-Cisco-Cms-Restapi/TEMP-test-pulp-unstable.md delete mode 100644 packaging/centreon-plugin-Applications-Cisco-Dnac-Restapi/TEMP-test-pulp-unstable.md delete mode 100644 packaging/centreon-plugin-Applications-Cisco-Ise-Restapi/TEMP-test-pulp-unstable.md delete mode 100644 packaging/centreon-plugin-Applications-Cisco-Ssms-Restapi/TEMP-test-pulp-unstable.md delete mode 100644 packaging/centreon-plugin-Applications-Clamav-Ssh/TEMP-test-pulp-unstable.md delete mode 100644 packaging/centreon-plugin-Applications-Commvault-Commserve-Restapi/TEMP-test-pulp-unstable.md delete mode 100644 packaging/centreon-plugin-Applications-Controlm-Restapi/TEMP-test-pulp-unstable.md delete mode 100644 packaging/centreon-plugin-Applications-Database-Cassandra-Jmx/TEMP-test-pulp-unstable.md delete mode 100644 packaging/centreon-plugin-Applications-Database-Couchdb-Restapi/TEMP-test-pulp-unstable.md delete mode 100644 packaging/centreon-plugin-Applications-Databases-Db2/TEMP-test-pulp-unstable.md delete mode 100644 packaging/centreon-plugin-Applications-Databases-Elasticsearch/TEMP-test-pulp-unstable.md delete mode 100644 packaging/centreon-plugin-Applications-Databases-Firebird/TEMP-test-pulp-unstable.md delete mode 100644 packaging/centreon-plugin-Applications-Databases-Influxdb/TEMP-test-pulp-unstable.md delete mode 100644 packaging/centreon-plugin-Applications-Databases-Informix-Snmp/TEMP-test-pulp-unstable.md delete mode 100644 packaging/centreon-plugin-Applications-Databases-Informix/TEMP-test-pulp-unstable.md delete mode 100644 packaging/centreon-plugin-Operatingsystems-Linux-Snmp/TEMP-test-pulp-unstable.md diff --git a/.github/packaging/centreon-plugin.yaml.template b/.github/packaging/centreon-plugin.yaml.template index 3acc9262b5..bdbd8a03ff 100644 --- a/.github/packaging/centreon-plugin.yaml.template +++ b/.github/packaging/centreon-plugin.yaml.template @@ -106,3 +106,4 @@ rpm: signature: key_file: ${RPM_SIGNING_KEY_FILE} key_id: ${RPM_SIGNING_KEY_ID} +# TEMP(test-pulp-unstable): touch the common template so every plugin is packaged for the Pulp delivery load test; revert before merge diff --git a/packaging/centreon-plugin-Applications-ActiveDirectory-Wsman/TEMP-test-pulp-unstable.md b/packaging/centreon-plugin-Applications-ActiveDirectory-Wsman/TEMP-test-pulp-unstable.md deleted file mode 100644 index ff49431e12..0000000000 --- a/packaging/centreon-plugin-Applications-ActiveDirectory-Wsman/TEMP-test-pulp-unstable.md +++ /dev/null @@ -1,2 +0,0 @@ -TEMP(test-pulp-unstable): marker file so get-plugins packages this plugin for -the Pulp batch-delivery test; delete before merge. diff --git a/packaging/centreon-plugin-Applications-ActiveMQ-Jmx/TEMP-test-pulp-unstable.md b/packaging/centreon-plugin-Applications-ActiveMQ-Jmx/TEMP-test-pulp-unstable.md deleted file mode 100644 index ff49431e12..0000000000 --- a/packaging/centreon-plugin-Applications-ActiveMQ-Jmx/TEMP-test-pulp-unstable.md +++ /dev/null @@ -1,2 +0,0 @@ -TEMP(test-pulp-unstable): marker file so get-plugins packages this plugin for -the Pulp batch-delivery test; delete before merge. diff --git a/packaging/centreon-plugin-Applications-Ansible-Cli/TEMP-test-pulp-unstable.md b/packaging/centreon-plugin-Applications-Ansible-Cli/TEMP-test-pulp-unstable.md deleted file mode 100644 index ff49431e12..0000000000 --- a/packaging/centreon-plugin-Applications-Ansible-Cli/TEMP-test-pulp-unstable.md +++ /dev/null @@ -1,2 +0,0 @@ -TEMP(test-pulp-unstable): marker file so get-plugins packages this plugin for -the Pulp batch-delivery test; delete before merge. diff --git a/packaging/centreon-plugin-Applications-Ansible-Tower/TEMP-test-pulp-unstable.md b/packaging/centreon-plugin-Applications-Ansible-Tower/TEMP-test-pulp-unstable.md deleted file mode 100644 index ff49431e12..0000000000 --- a/packaging/centreon-plugin-Applications-Ansible-Tower/TEMP-test-pulp-unstable.md +++ /dev/null @@ -1,2 +0,0 @@ -TEMP(test-pulp-unstable): marker file so get-plugins packages this plugin for -the Pulp batch-delivery test; delete before merge. diff --git a/packaging/centreon-plugin-Applications-Antivirus-Kaspersky-Snmp/TEMP-test-pulp-unstable.md b/packaging/centreon-plugin-Applications-Antivirus-Kaspersky-Snmp/TEMP-test-pulp-unstable.md deleted file mode 100644 index ff49431e12..0000000000 --- a/packaging/centreon-plugin-Applications-Antivirus-Kaspersky-Snmp/TEMP-test-pulp-unstable.md +++ /dev/null @@ -1,2 +0,0 @@ -TEMP(test-pulp-unstable): marker file so get-plugins packages this plugin for -the Pulp batch-delivery test; delete before merge. diff --git a/packaging/centreon-plugin-Applications-Antivirus-Mcafee-Webgateway-Snmp/TEMP-test-pulp-unstable.md b/packaging/centreon-plugin-Applications-Antivirus-Mcafee-Webgateway-Snmp/TEMP-test-pulp-unstable.md deleted file mode 100644 index ff49431e12..0000000000 --- a/packaging/centreon-plugin-Applications-Antivirus-Mcafee-Webgateway-Snmp/TEMP-test-pulp-unstable.md +++ /dev/null @@ -1,2 +0,0 @@ -TEMP(test-pulp-unstable): marker file so get-plugins packages this plugin for -the Pulp batch-delivery test; delete before merge. diff --git a/packaging/centreon-plugin-Applications-Antivirus-Skyhigh-Webgateway-Snmp/TEMP-test-pulp-unstable.md b/packaging/centreon-plugin-Applications-Antivirus-Skyhigh-Webgateway-Snmp/TEMP-test-pulp-unstable.md deleted file mode 100644 index ff49431e12..0000000000 --- a/packaging/centreon-plugin-Applications-Antivirus-Skyhigh-Webgateway-Snmp/TEMP-test-pulp-unstable.md +++ /dev/null @@ -1,2 +0,0 @@ -TEMP(test-pulp-unstable): marker file so get-plugins packages this plugin for -the Pulp batch-delivery test; delete before merge. diff --git a/packaging/centreon-plugin-Applications-Apache-Cxf-Jmx/TEMP-test-pulp-unstable.md b/packaging/centreon-plugin-Applications-Apache-Cxf-Jmx/TEMP-test-pulp-unstable.md deleted file mode 100644 index ff49431e12..0000000000 --- a/packaging/centreon-plugin-Applications-Apache-Cxf-Jmx/TEMP-test-pulp-unstable.md +++ /dev/null @@ -1,2 +0,0 @@ -TEMP(test-pulp-unstable): marker file so get-plugins packages this plugin for -the Pulp batch-delivery test; delete before merge. diff --git a/packaging/centreon-plugin-Applications-Awa-Jmx/TEMP-test-pulp-unstable.md b/packaging/centreon-plugin-Applications-Awa-Jmx/TEMP-test-pulp-unstable.md deleted file mode 100644 index ff49431e12..0000000000 --- a/packaging/centreon-plugin-Applications-Awa-Jmx/TEMP-test-pulp-unstable.md +++ /dev/null @@ -1,2 +0,0 @@ -TEMP(test-pulp-unstable): marker file so get-plugins packages this plugin for -the Pulp batch-delivery test; delete before merge. diff --git a/packaging/centreon-plugin-Applications-Backup-Emc-Recoverypoint-Ssh/TEMP-test-pulp-unstable.md b/packaging/centreon-plugin-Applications-Backup-Emc-Recoverypoint-Ssh/TEMP-test-pulp-unstable.md deleted file mode 100644 index ff49431e12..0000000000 --- a/packaging/centreon-plugin-Applications-Backup-Emc-Recoverypoint-Ssh/TEMP-test-pulp-unstable.md +++ /dev/null @@ -1,2 +0,0 @@ -TEMP(test-pulp-unstable): marker file so get-plugins packages this plugin for -the Pulp batch-delivery test; delete before merge. diff --git a/packaging/centreon-plugin-Applications-Backup-Veeam-Wsman/TEMP-test-pulp-unstable.md b/packaging/centreon-plugin-Applications-Backup-Veeam-Wsman/TEMP-test-pulp-unstable.md deleted file mode 100644 index ff49431e12..0000000000 --- a/packaging/centreon-plugin-Applications-Backup-Veeam-Wsman/TEMP-test-pulp-unstable.md +++ /dev/null @@ -1,2 +0,0 @@ -TEMP(test-pulp-unstable): marker file so get-plugins packages this plugin for -the Pulp batch-delivery test; delete before merge. diff --git a/packaging/centreon-plugin-Applications-Bind9-Web/TEMP-test-pulp-unstable.md b/packaging/centreon-plugin-Applications-Bind9-Web/TEMP-test-pulp-unstable.md deleted file mode 100644 index ff49431e12..0000000000 --- a/packaging/centreon-plugin-Applications-Bind9-Web/TEMP-test-pulp-unstable.md +++ /dev/null @@ -1,2 +0,0 @@ -TEMP(test-pulp-unstable): marker file so get-plugins packages this plugin for -the Pulp batch-delivery test; delete before merge. diff --git a/packaging/centreon-plugin-Applications-Biztalk/TEMP-test-pulp-unstable.md b/packaging/centreon-plugin-Applications-Biztalk/TEMP-test-pulp-unstable.md deleted file mode 100644 index ff49431e12..0000000000 --- a/packaging/centreon-plugin-Applications-Biztalk/TEMP-test-pulp-unstable.md +++ /dev/null @@ -1,2 +0,0 @@ -TEMP(test-pulp-unstable): marker file so get-plugins packages this plugin for -the Pulp batch-delivery test; delete before merge. diff --git a/packaging/centreon-plugin-Applications-Bluemind-Ssh/TEMP-test-pulp-unstable.md b/packaging/centreon-plugin-Applications-Bluemind-Ssh/TEMP-test-pulp-unstable.md deleted file mode 100644 index ff49431e12..0000000000 --- a/packaging/centreon-plugin-Applications-Bluemind-Ssh/TEMP-test-pulp-unstable.md +++ /dev/null @@ -1,2 +0,0 @@ -TEMP(test-pulp-unstable): marker file so get-plugins packages this plugin for -the Pulp batch-delivery test; delete before merge. diff --git a/packaging/centreon-plugin-Applications-Ceph-Restapi/TEMP-test-pulp-unstable.md b/packaging/centreon-plugin-Applications-Ceph-Restapi/TEMP-test-pulp-unstable.md deleted file mode 100644 index ff49431e12..0000000000 --- a/packaging/centreon-plugin-Applications-Ceph-Restapi/TEMP-test-pulp-unstable.md +++ /dev/null @@ -1,2 +0,0 @@ -TEMP(test-pulp-unstable): marker file so get-plugins packages this plugin for -the Pulp batch-delivery test; delete before merge. diff --git a/packaging/centreon-plugin-Applications-Cisco-Cms-Restapi/TEMP-test-pulp-unstable.md b/packaging/centreon-plugin-Applications-Cisco-Cms-Restapi/TEMP-test-pulp-unstable.md deleted file mode 100644 index ff49431e12..0000000000 --- a/packaging/centreon-plugin-Applications-Cisco-Cms-Restapi/TEMP-test-pulp-unstable.md +++ /dev/null @@ -1,2 +0,0 @@ -TEMP(test-pulp-unstable): marker file so get-plugins packages this plugin for -the Pulp batch-delivery test; delete before merge. diff --git a/packaging/centreon-plugin-Applications-Cisco-Dnac-Restapi/TEMP-test-pulp-unstable.md b/packaging/centreon-plugin-Applications-Cisco-Dnac-Restapi/TEMP-test-pulp-unstable.md deleted file mode 100644 index ff49431e12..0000000000 --- a/packaging/centreon-plugin-Applications-Cisco-Dnac-Restapi/TEMP-test-pulp-unstable.md +++ /dev/null @@ -1,2 +0,0 @@ -TEMP(test-pulp-unstable): marker file so get-plugins packages this plugin for -the Pulp batch-delivery test; delete before merge. diff --git a/packaging/centreon-plugin-Applications-Cisco-Ise-Restapi/TEMP-test-pulp-unstable.md b/packaging/centreon-plugin-Applications-Cisco-Ise-Restapi/TEMP-test-pulp-unstable.md deleted file mode 100644 index ff49431e12..0000000000 --- a/packaging/centreon-plugin-Applications-Cisco-Ise-Restapi/TEMP-test-pulp-unstable.md +++ /dev/null @@ -1,2 +0,0 @@ -TEMP(test-pulp-unstable): marker file so get-plugins packages this plugin for -the Pulp batch-delivery test; delete before merge. diff --git a/packaging/centreon-plugin-Applications-Cisco-Ssms-Restapi/TEMP-test-pulp-unstable.md b/packaging/centreon-plugin-Applications-Cisco-Ssms-Restapi/TEMP-test-pulp-unstable.md deleted file mode 100644 index ff49431e12..0000000000 --- a/packaging/centreon-plugin-Applications-Cisco-Ssms-Restapi/TEMP-test-pulp-unstable.md +++ /dev/null @@ -1,2 +0,0 @@ -TEMP(test-pulp-unstable): marker file so get-plugins packages this plugin for -the Pulp batch-delivery test; delete before merge. diff --git a/packaging/centreon-plugin-Applications-Clamav-Ssh/TEMP-test-pulp-unstable.md b/packaging/centreon-plugin-Applications-Clamav-Ssh/TEMP-test-pulp-unstable.md deleted file mode 100644 index ff49431e12..0000000000 --- a/packaging/centreon-plugin-Applications-Clamav-Ssh/TEMP-test-pulp-unstable.md +++ /dev/null @@ -1,2 +0,0 @@ -TEMP(test-pulp-unstable): marker file so get-plugins packages this plugin for -the Pulp batch-delivery test; delete before merge. diff --git a/packaging/centreon-plugin-Applications-Commvault-Commserve-Restapi/TEMP-test-pulp-unstable.md b/packaging/centreon-plugin-Applications-Commvault-Commserve-Restapi/TEMP-test-pulp-unstable.md deleted file mode 100644 index ff49431e12..0000000000 --- a/packaging/centreon-plugin-Applications-Commvault-Commserve-Restapi/TEMP-test-pulp-unstable.md +++ /dev/null @@ -1,2 +0,0 @@ -TEMP(test-pulp-unstable): marker file so get-plugins packages this plugin for -the Pulp batch-delivery test; delete before merge. diff --git a/packaging/centreon-plugin-Applications-Controlm-Restapi/TEMP-test-pulp-unstable.md b/packaging/centreon-plugin-Applications-Controlm-Restapi/TEMP-test-pulp-unstable.md deleted file mode 100644 index ff49431e12..0000000000 --- a/packaging/centreon-plugin-Applications-Controlm-Restapi/TEMP-test-pulp-unstable.md +++ /dev/null @@ -1,2 +0,0 @@ -TEMP(test-pulp-unstable): marker file so get-plugins packages this plugin for -the Pulp batch-delivery test; delete before merge. diff --git a/packaging/centreon-plugin-Applications-Database-Cassandra-Jmx/TEMP-test-pulp-unstable.md b/packaging/centreon-plugin-Applications-Database-Cassandra-Jmx/TEMP-test-pulp-unstable.md deleted file mode 100644 index ff49431e12..0000000000 --- a/packaging/centreon-plugin-Applications-Database-Cassandra-Jmx/TEMP-test-pulp-unstable.md +++ /dev/null @@ -1,2 +0,0 @@ -TEMP(test-pulp-unstable): marker file so get-plugins packages this plugin for -the Pulp batch-delivery test; delete before merge. diff --git a/packaging/centreon-plugin-Applications-Database-Couchdb-Restapi/TEMP-test-pulp-unstable.md b/packaging/centreon-plugin-Applications-Database-Couchdb-Restapi/TEMP-test-pulp-unstable.md deleted file mode 100644 index ff49431e12..0000000000 --- a/packaging/centreon-plugin-Applications-Database-Couchdb-Restapi/TEMP-test-pulp-unstable.md +++ /dev/null @@ -1,2 +0,0 @@ -TEMP(test-pulp-unstable): marker file so get-plugins packages this plugin for -the Pulp batch-delivery test; delete before merge. diff --git a/packaging/centreon-plugin-Applications-Databases-Db2/TEMP-test-pulp-unstable.md b/packaging/centreon-plugin-Applications-Databases-Db2/TEMP-test-pulp-unstable.md deleted file mode 100644 index ff49431e12..0000000000 --- a/packaging/centreon-plugin-Applications-Databases-Db2/TEMP-test-pulp-unstable.md +++ /dev/null @@ -1,2 +0,0 @@ -TEMP(test-pulp-unstable): marker file so get-plugins packages this plugin for -the Pulp batch-delivery test; delete before merge. diff --git a/packaging/centreon-plugin-Applications-Databases-Elasticsearch/TEMP-test-pulp-unstable.md b/packaging/centreon-plugin-Applications-Databases-Elasticsearch/TEMP-test-pulp-unstable.md deleted file mode 100644 index ff49431e12..0000000000 --- a/packaging/centreon-plugin-Applications-Databases-Elasticsearch/TEMP-test-pulp-unstable.md +++ /dev/null @@ -1,2 +0,0 @@ -TEMP(test-pulp-unstable): marker file so get-plugins packages this plugin for -the Pulp batch-delivery test; delete before merge. diff --git a/packaging/centreon-plugin-Applications-Databases-Firebird/TEMP-test-pulp-unstable.md b/packaging/centreon-plugin-Applications-Databases-Firebird/TEMP-test-pulp-unstable.md deleted file mode 100644 index ff49431e12..0000000000 --- a/packaging/centreon-plugin-Applications-Databases-Firebird/TEMP-test-pulp-unstable.md +++ /dev/null @@ -1,2 +0,0 @@ -TEMP(test-pulp-unstable): marker file so get-plugins packages this plugin for -the Pulp batch-delivery test; delete before merge. diff --git a/packaging/centreon-plugin-Applications-Databases-Influxdb/TEMP-test-pulp-unstable.md b/packaging/centreon-plugin-Applications-Databases-Influxdb/TEMP-test-pulp-unstable.md deleted file mode 100644 index ff49431e12..0000000000 --- a/packaging/centreon-plugin-Applications-Databases-Influxdb/TEMP-test-pulp-unstable.md +++ /dev/null @@ -1,2 +0,0 @@ -TEMP(test-pulp-unstable): marker file so get-plugins packages this plugin for -the Pulp batch-delivery test; delete before merge. diff --git a/packaging/centreon-plugin-Applications-Databases-Informix-Snmp/TEMP-test-pulp-unstable.md b/packaging/centreon-plugin-Applications-Databases-Informix-Snmp/TEMP-test-pulp-unstable.md deleted file mode 100644 index ff49431e12..0000000000 --- a/packaging/centreon-plugin-Applications-Databases-Informix-Snmp/TEMP-test-pulp-unstable.md +++ /dev/null @@ -1,2 +0,0 @@ -TEMP(test-pulp-unstable): marker file so get-plugins packages this plugin for -the Pulp batch-delivery test; delete before merge. diff --git a/packaging/centreon-plugin-Applications-Databases-Informix/TEMP-test-pulp-unstable.md b/packaging/centreon-plugin-Applications-Databases-Informix/TEMP-test-pulp-unstable.md deleted file mode 100644 index ff49431e12..0000000000 --- a/packaging/centreon-plugin-Applications-Databases-Informix/TEMP-test-pulp-unstable.md +++ /dev/null @@ -1,2 +0,0 @@ -TEMP(test-pulp-unstable): marker file so get-plugins packages this plugin for -the Pulp batch-delivery test; delete before merge. diff --git a/packaging/centreon-plugin-Operatingsystems-Linux-Snmp/TEMP-test-pulp-unstable.md b/packaging/centreon-plugin-Operatingsystems-Linux-Snmp/TEMP-test-pulp-unstable.md deleted file mode 100644 index 6ea9934bf2..0000000000 --- a/packaging/centreon-plugin-Operatingsystems-Linux-Snmp/TEMP-test-pulp-unstable.md +++ /dev/null @@ -1,2 +0,0 @@ -TEMP(test-pulp-unstable): marker file so get-plugins packages this plugin for -the Pulp delivery test; delete before merge. From f5037353d84ead20ab1fd79da3cc5e6191c4a44c Mon Sep 17 00:00:00 2001 From: Kevin Duret Date: Mon, 13 Jul 2026 14:21:57 +0200 Subject: [PATCH 05/49] chore(ci): retrigger the full-scale Pulp delivery validation (skip label removed) From 57ca42ebfa4375a0896c94fa07c3040d4f75e84d Mon Sep 17 00:00:00 2001 From: Kevin Duret Date: Mon, 13 Jul 2026 14:31:27 +0200 Subject: [PATCH 06/49] test(ci): batched rpm delivery through a single add_content_units modify TEMP(test-pulp-unstable) experiment: upload the rpm packages as unassociated content (repository-less create tasks parallelize across the pulp workers instead of serializing on the repository lock), resolve the created content hrefs (sha256 fallback for job re-runs), then add the whole batch to the repository with a single modify task. Two repository-serialized tasks per delivery (modify + publication) instead of one per package. Requires the reconciled rpm/packages access policy (delivery-tooling#209). --- .../package-delivery-pulp/deliver-rpm.sh | 81 +++++++++++++++---- 1 file changed, 67 insertions(+), 14 deletions(-) diff --git a/.github/actions/package-delivery-pulp/deliver-rpm.sh b/.github/actions/package-delivery-pulp/deliver-rpm.sh index c4736dbbcd..50610bd17c 100755 --- a/.github/actions/package-delivery-pulp/deliver-rpm.sh +++ b/.github/actions/package-delivery-pulp/deliver-rpm.sh @@ -49,6 +49,47 @@ assert_not_in_stable() { fi } +# TEMP(test-pulp-unstable): wait for a repository-less upload task and emit +# the created content href. A task that did not produce one (content already +# existing on a job re-run) is resolved by the package sha256 instead of +# failing the delivery. +resolve_uploaded_content() { + local task_href=$1 sha256=$2 + local body state content attempt + for ((attempt = 0; attempt < 200; attempt++)); do + refresh_pulp_token + body=$(curl -fsSL -H "Authorization: Github $PULP_TOKEN" "$PULP_URL$task_href" 2>/dev/null) || body="" + state=$(echo "$body" | jq -r '.state' 2>/dev/null) || state="" + case "$state" in + completed) + content=$(echo "$body" | jq -r '.created_resources[0] // empty') + if [[ -n "$content" ]]; then + echo "$content" + return 0 + fi + break + ;; + failed | canceled) + break + ;; + *) + sleep 3 + ;; + esac + done + content=$( + curl -fsSL -H "Authorization: Github $PULP_TOKEN" -G \ + --data-urlencode "sha256=$sha256" \ + --data-urlencode "limit=1" \ + "$PULP_URL/api/v3/content/rpm/packages/" | jq -r '.results[0].pulp_href // empty' + ) + if [[ -z "$content" ]]; then + echo "::error::Cannot resolve the uploaded content for task $task_href (sha256 $sha256)" >&2 + return 1 + fi + echo "$content" +} + FILES=(*.rpm) if [[ ${#FILES[@]} -eq 0 ]]; then echo "::error::No rpm package found to deliver" @@ -95,23 +136,23 @@ for ARCH in noarch x86_64; do REPOSITORY_HREF=$(pulp rpm repository show --name "$REPOSITORY_NAME" | jq -r '.pulp_href') - # Packages are uploaded straight into the repository: pulpcore requires - # non-admin accounts to provide the destination repository on content upload, - # so the upload cannot be decoupled from the repository association. Packages - # are labeled with their module so that promote-to-stable can identify them; - # pulp-cli does not allow to set labels on upload so the api is used directly. - # The upload tasks are awaited as a batch after the loop: pulp serializes the - # tasks of a repository server-side, so waiting for each task before sending - # the next upload would pay the task-queue latency once per package instead - # of once per delivery. + # TEMP(test-pulp-unstable) experimental batched delivery: packages are + # uploaded as unassociated content (repository-less, so the create tasks + # parallelize across the pulp workers instead of serializing on the + # repository lock), then the whole batch is added to the repository with a + # single modify task. Requires the reconciled rpm/packages access policy + # (delivery-tooling#209). Packages are labeled with their module so that + # promote-to-stable can identify them. TASK_HREFS=() + SHA256S=() for FILE in "${ARCH_FILES[@]}"; do assert_not_in_stable "$FILE" "$ARCH" - echo "[INFO] Uploading $(basename "$FILE") to $REPOSITORY_NAME (module $MODULE_NAME)" + echo "[INFO] Uploading $(basename "$FILE") (module $MODULE_NAME)" + sha256=$(sha256sum "$FILE" | cut -d' ' -f1) + SHA256S+=("$sha256") TASK_HREFS+=("$( pulp_upload \ -F "file=@\"$FILE\"" \ - -F "repository=$REPOSITORY_HREF" \ -F "pulp_labels=$PULP_LABELS" \ "$PULP_URL/api/v3/content/rpm/packages/" )") @@ -125,7 +166,6 @@ for ARCH in noarch x86_64; do base=${base%-*} version=${base##*-} name=${base%-*} - sha256=$(sha256sum "$FILE" | cut -d' ' -f1) manifest_add "$(jq -cn \ --arg filename "$FILENAME" --arg name "$name" --arg version "$version" \ --arg release "$release" --arg arch "$ARCH" --arg sha256 "$sha256" \ @@ -133,8 +173,21 @@ for ARCH in noarch x86_64; do '{filename:$filename,name:$name,version:$version,release:$release,arch:$arch,sha256:$sha256,repository:$repository,base_path:$base_path}')" done - echo "[INFO] Waiting for ${#TASK_HREFS[@]} upload task(s) to complete" - wait_tasks "${TASK_HREFS[@]}" + echo "[INFO] Waiting for ${#TASK_HREFS[@]} upload task(s) and resolving the content" + CONTENT_HREFS=() + for i in "${!TASK_HREFS[@]}"; do + CONTENT_HREFS+=("$(resolve_uploaded_content "${TASK_HREFS[$i]}" "${SHA256S[$i]}")") + done + + echo "[INFO] Adding ${#CONTENT_HREFS[@]} package(s) to $REPOSITORY_NAME in a single task" + ADD_BODY=$(printf '%s\n' "${CONTENT_HREFS[@]}" | jq -R . | jq -cs '{add_content_units: .}') + MODIFY_TASK=$( + curl -fsSL -H "Authorization: Github $PULP_TOKEN" \ + -X POST -H "Content-Type: application/json" \ + -d "$ADD_BODY" \ + "$PULP_URL${REPOSITORY_HREF}modify/" | jq -r '.task' + ) + wait_task "$MODIFY_TASK" echo "[INFO] Publishing repository $REPOSITORY_NAME" pulp rpm publication create --repository "$REPOSITORY_NAME" >/dev/null From d5d0feedf1deb1e7d28d8160aaf35bb9ff37268a Mon Sep 17 00:00:00 2001 From: Kevin Duret Date: Wed, 15 Jul 2026 10:49:26 +0200 Subject: [PATCH 07/49] test(ci): restrict the modify-method load test to el10 and trixie TEMP(test-pulp-unstable): same scope as the sibling batch-wait test for a direct comparison (675 packages per distrib). --- .github/workflows/plugins.yml | 34 ++++------------------------------ 1 file changed, 4 insertions(+), 30 deletions(-) diff --git a/.github/workflows/plugins.yml b/.github/workflows/plugins.yml index 3ed68362d0..ce873be7d7 100644 --- a/.github/workflows/plugins.yml +++ b/.github/workflows/plugins.yml @@ -290,30 +290,14 @@ jobs: max-parallel: 5 matrix: include: - - package_extension: rpm - image: packaging:alma8 - distrib: el8 - - package_extension: rpm - image: packaging:alma9 - distrib: el9 + # TEMP(test-pulp-unstable): restrict the load test to one rpm and one + # deb distrib (el10 + trixie) - package_extension: rpm image: packaging:alma10 distrib: el10 - - package_extension: deb - image: packaging:bullseye - distrib: bullseye - - package_extension: deb - image: packaging:bookworm - distrib: bookworm - package_extension: deb image: packaging:trixie distrib: trixie - - package_extension: deb - image: packaging:jammy - distrib: jammy - - package_extension: deb - image: packaging:noble - distrib: noble container: image: ghcr.io/${{ github.repository }}/${{ matrix.image }} @@ -430,22 +414,12 @@ jobs: fail-fast: false matrix: include: - - distrib: el8 - package_extension: rpm - - distrib: el9 - package_extension: rpm + # TEMP(test-pulp-unstable): restrict the load test to one rpm and + # one deb distrib (el10 + trixie) - distrib: el10 package_extension: rpm - - distrib: bullseye - package_extension: deb - - distrib: bookworm - package_extension: deb - distrib: trixie package_extension: deb - - distrib: jammy - package_extension: deb - - distrib: noble - package_extension: deb name: deliver ${{ matrix.distrib }} steps: From 4dcfdc717997a3e85063b6e803b45ded1adc61de Mon Sep 17 00:00:00 2001 From: Kevin Duret Date: Wed, 15 Jul 2026 11:07:04 +0200 Subject: [PATCH 08/49] chore(ci): retrigger the el10/trixie modify-method test From 413d5d5b70519c8fbc86ee965e4398e7942be412 Mon Sep 17 00:00:00 2001 From: Kevin Duret Date: Wed, 15 Jul 2026 13:38:17 +0200 Subject: [PATCH 09/49] test(ci): restrict the right matrix (the pulp twin) to el10 and trixie [skip ci] The previous restriction hit the first matching matrix, which is the disabled artifactory job's; anchor on the twin's job name this time. [skip ci]: no point running until delivery-tooling#209 is merged (the orphan upload is rejected by the default access policy). --- .github/workflows/plugins.yml | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/.github/workflows/plugins.yml b/.github/workflows/plugins.yml index ce873be7d7..b11409c90b 100644 --- a/.github/workflows/plugins.yml +++ b/.github/workflows/plugins.yml @@ -456,22 +456,12 @@ jobs: fail-fast: false matrix: include: - - distrib: el8 - package_extension: rpm - - distrib: el9 - package_extension: rpm + # TEMP(test-pulp-unstable): restrict the load test to one rpm and + # one deb distrib (el10 + trixie) - distrib: el10 package_extension: rpm - - distrib: bullseye - package_extension: deb - - distrib: bookworm - package_extension: deb - distrib: trixie package_extension: deb - - distrib: jammy - package_extension: deb - - distrib: noble - package_extension: deb name: deliver ${{ matrix.distrib }} on pulp steps: From 62ff80e0fc917bd4f92ed30378ebfbcc22699211 Mon Sep 17 00:00:00 2001 From: Kevin Duret Date: Thu, 16 Jul 2026 08:36:39 +0200 Subject: [PATCH 10/49] chore(ci): retrigger the modify-method test after the policy reconcile From e62326cec9d5512e1110493088939cc95604a10e Mon Sep 17 00:00:00 2001 From: Kevin Duret Date: Thu, 16 Jul 2026 08:37:20 +0200 Subject: [PATCH 11/49] chore(ci): measure the modify-method delivery (policy reconciled) From cf52ee22b128e98283cbe9659ba65c32e3e3fe3b Mon Sep 17 00:00:00 2001 From: Kevin Duret Date: Thu, 16 Jul 2026 08:44:35 +0200 Subject: [PATCH 12/49] TEMP(test-pulp-unstable): cap the delivery to 50 plugins --- .github/scripts/list-plugins-to-build-and-test.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/scripts/list-plugins-to-build-and-test.py b/.github/scripts/list-plugins-to-build-and-test.py index f45508557e..49072a41c4 100644 --- a/.github/scripts/list-plugins-to-build-and-test.py +++ b/.github/scripts/list-plugins-to-build-and-test.py @@ -83,7 +83,9 @@ def get_pack_from_path(path): if common: cmd = ["find", "packaging", "-type", "f", "-name", "pkg.json"] res = subprocess.run(cmd, capture_output=True, text=True, check=False) - for filepath in res.stdout.strip().split('\n'): + # TEMP(test-pulp-unstable): cap the delivery to 50 plugins, matching + # the batch-method test run for a fair comparison + for filepath in sorted(res.stdout.strip().split('\n'))[:50]: add_package_info(filepath) else: with open('packages_directories.json') as packages_directories_file: From e981c7f45335426b1fa5ff4bff4d4479ee76dccf Mon Sep 17 00:00:00 2001 From: Kevin Duret Date: Thu, 16 Jul 2026 09:17:31 +0200 Subject: [PATCH 13/49] chore(ci): retrigger the modify-method test after the policy fix From 0455a8b424a275db15d468b2027cb2e3ed9b8022 Mon Sep 17 00:00:00 2001 From: Kevin Duret Date: Thu, 16 Jul 2026 09:38:01 +0200 Subject: [PATCH 14/49] Revert "TEMP(test-pulp-unstable): cap the delivery to 50 plugins" This reverts commit cf52ee22b128e98283cbe9659ba65c32e3e3fe3b. --- .github/scripts/list-plugins-to-build-and-test.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/scripts/list-plugins-to-build-and-test.py b/.github/scripts/list-plugins-to-build-and-test.py index 49072a41c4..f45508557e 100644 --- a/.github/scripts/list-plugins-to-build-and-test.py +++ b/.github/scripts/list-plugins-to-build-and-test.py @@ -83,9 +83,7 @@ def get_pack_from_path(path): if common: cmd = ["find", "packaging", "-type", "f", "-name", "pkg.json"] res = subprocess.run(cmd, capture_output=True, text=True, check=False) - # TEMP(test-pulp-unstable): cap the delivery to 50 plugins, matching - # the batch-method test run for a fair comparison - for filepath in sorted(res.stdout.strip().split('\n'))[:50]: + for filepath in res.stdout.strip().split('\n'): add_package_info(filepath) else: with open('packages_directories.json') as packages_directories_file: From f3b40790323de0216dfe6bcc656df531796a3ce7 Mon Sep 17 00:00:00 2001 From: Kevin Duret Date: Thu, 16 Jul 2026 10:29:28 +0200 Subject: [PATCH 15/49] fix(ci): avoid SIGPIPE noise in the pulp delivery verification --- .github/scripts/pulp/check-deb.sh | 18 ++++++++++++------ .github/scripts/pulp/check-rpm.sh | 18 +++++++++++------- 2 files changed, 23 insertions(+), 13 deletions(-) diff --git a/.github/scripts/pulp/check-deb.sh b/.github/scripts/pulp/check-deb.sh index e5bdca72f0..0bb99adca0 100755 --- a/.github/scripts/pulp/check-deb.sh +++ b/.github/scripts/pulp/check-deb.sh @@ -58,12 +58,15 @@ done # --- metadata resolvability + fetchability, with a bounded retry window ----- # resolve a package's published Filename from a suite Packages index by sha256 resolve_filename() { - local packages=$1 sha=$2 - printf '%s' "$packages" | awk -v sha="$sha" ' + # read the Packages index from a file: awk exits on the first match, and a + # pipe writer would take a SIGPIPE ("printf: write error: Broken pipe" noise) + local packages_file=$1 sha=$2 + [[ -s "$packages_file" ]] || return 0 + awk -v sha="$sha" ' BEGIN { RS = ""; FS = "\n" } index($0, "SHA256: " sha) { for (i = 1; i <= NF; i++) if ($i ~ /^Filename: /) { sub(/^Filename: /, "", $i); print $i; exit } - }' + }' "$packages_file" } declare -A META_IDX # idx -> true|false @@ -73,9 +76,9 @@ for i in "${!E_FILENAME[@]}"; do META_IDX[$i]=false; done # one resolution round: fetch each suite's Packages indexes once, then resolve # each pending package's published Filename by sha256 resolve_pending() { - local -A pkg_cache=() # key: base_path|suite|arch -> Packages body + local -A pkg_cache=() # key: base_path|suite|arch -> Packages index file local -A arches_cache=() # key: base_path|suite -> space separated arches - local all_resolved=true i base_path suite arch search_arches sk ck a filename + local all_resolved=true i base_path suite arch search_arches sk ck a filename cache_file for i in "${!E_FILENAME[@]}"; do [[ "${META_IDX[$i]}" == "true" ]] && continue base_path=${E_BASEPATH[$i]}; suite=${E_SUITE[$i]}; arch=${E_ARCH[$i]} @@ -96,7 +99,9 @@ resolve_pending() { for a in $search_arches; do ck="$base_path|$suite|$a" if [[ -z "${pkg_cache[$ck]+set}" ]]; then - pkg_cache[$ck]=$(curl -fsSL "$PULP_CONTENT_URL/$base_path/dists/$suite/main/binary-$a/Packages" 2>/dev/null || true) + cache_file=$(mktemp) + curl -fsSL "$PULP_CONTENT_URL/$base_path/dists/$suite/main/binary-$a/Packages" 2>/dev/null > "$cache_file" || true + pkg_cache[$ck]=$cache_file fi filename=$(resolve_filename "${pkg_cache[$ck]}" "${E_SHA256[$i]}") [[ -n "$filename" ]] && break @@ -109,6 +114,7 @@ resolve_pending() { all_resolved=false fi done + rm -f "${pkg_cache[@]}" [[ "$all_resolved" == "true" ]] } diff --git a/.github/scripts/pulp/check-rpm.sh b/.github/scripts/pulp/check-rpm.sh index 14eb3bea55..dc17654cc4 100755 --- a/.github/scripts/pulp/check-rpm.sh +++ b/.github/scripts/pulp/check-rpm.sh @@ -78,8 +78,11 @@ for i in "${!E_FILENAME[@]}"; do META_IDX[$i]=false; done # matches, from a primary.xml body: matching by checksum (not filename) binds # the verification to the exact delivered content, like the DEB by-sha256 match resolve_href() { - local primary=$1 sha=$2 - printf '%s' "$primary" | awk -v sha="$sha" ' + # read the primary.xml from a file: awk exits on the first match, and a pipe + # writer would take a SIGPIPE ("printf: write error: Broken pipe" noise) + local primary_file=$1 sha=$2 + [[ -s "$primary_file" ]] || return 0 + awk -v sha="$sha" ' BEGIN { RS = "" } index($0, ">" sha "") { if (match($0, /]*href="[^"]+"/)) { @@ -87,26 +90,26 @@ resolve_href() { sub(/.*href="/, "", s); sub(/"$/, "", s) print s; exit } - }' + }' "$primary_file" } # one resolution round: fetch each base_path's primary.xml once, then resolve # each pending package's published href by sha256 resolve_pending() { local -A primary_cache=() - local all_resolved=true i base_path repomd primary_href href + local all_resolved=true i base_path repomd primary_href href cache_file for i in "${!E_FILENAME[@]}"; do [[ "${META_IDX[$i]}" == "true" ]] && continue base_path=${E_BASEPATH[$i]} if [[ -z "${primary_cache[$base_path]+set}" ]]; then + cache_file=$(mktemp) repomd=$(curl -fsSL "$PULP_CONTENT_URL/$base_path/repodata/repomd.xml" 2>/dev/null || true) primary_href=$(printf '%s' "$repomd" | grep -oP '/dev/null | gunzip -c 2>/dev/null || true) - else - primary_cache[$base_path]="" + curl -fsSL "$PULP_CONTENT_URL/$base_path/$primary_href" 2>/dev/null | gunzip -c 2>/dev/null > "$cache_file" || true fi + primary_cache[$base_path]=$cache_file fi href=$(resolve_href "${primary_cache[$base_path]}" "${E_SHA256[$i]}") @@ -117,6 +120,7 @@ resolve_pending() { all_resolved=false fi done + rm -f "${primary_cache[@]}" [[ "$all_resolved" == "true" ]] } From 113cb6ad67c44c1f4d4379e4607917d1b525c341 Mon Sep 17 00:00:00 2001 From: Kevin Duret Date: Thu, 16 Jul 2026 11:10:45 +0200 Subject: [PATCH 16/49] TEMP(test-pulp-unstable): purge the 20260714-1 leftovers before delivering --- .../package-delivery-pulp/deliver-rpm.sh | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/.github/actions/package-delivery-pulp/deliver-rpm.sh b/.github/actions/package-delivery-pulp/deliver-rpm.sh index 048a6d4639..01e911068e 100755 --- a/.github/actions/package-delivery-pulp/deliver-rpm.sh +++ b/.github/actions/package-delivery-pulp/deliver-rpm.sh @@ -136,6 +136,32 @@ for ARCH in noarch x86_64; do REPOSITORY_HREF=$(pulp rpm repository show --name "$REPOSITORY_NAME" | jq -r '.pulp_href') + # TEMP(test-pulp-unstable): one-off purge of the 20260714-1 leftovers of an + # earlier test round - they outrank the 20260700- test builds in + # RPM version comparison, so the retention policy evicts every freshly + # delivered package sharing their name. No-op once purged. + refresh_pulp_token + LATEST_VERSION=$(pulp rpm repository show --name "$REPOSITORY_NAME" | jq -r '.latest_version_href') + PURGE_HREFS=$( + curl -fsSL -H "Authorization: Github $PULP_TOKEN" -G \ + --data-urlencode "repository_version=$LATEST_VERSION" \ + --data-urlencode "version=20260714" \ + --data-urlencode "release=1.el10" \ + --data-urlencode "limit=200" \ + "$PULP_URL/api/v3/content/rpm/packages/" | jq -r '.results[].pulp_href' + ) + if [[ -n "$PURGE_HREFS" ]]; then + echo "[INFO] Purging $(wc -l <<< "$PURGE_HREFS") leftover 20260714-1 package(s) from $REPOSITORY_NAME" + PURGE_BODY=$(printf '%s\n' "$PURGE_HREFS" | jq -R . | jq -cs '{remove_content_units: .}') + PURGE_TASK=$( + curl -fsSL -H "Authorization: Github $PULP_TOKEN" \ + -X POST -H "Content-Type: application/json" \ + -d "$PURGE_BODY" \ + "$PULP_URL${REPOSITORY_HREF}modify/" | jq -r '.task' + ) + wait_task "$PURGE_TASK" + fi + # TEMP(test-pulp-unstable) experimental batched delivery: packages are # uploaded as unassociated content (repository-less, so the create tasks # parallelize across the pulp workers instead of serializing on the From d4c12872e47ca92db827db6bcee14b03eda02a38 Mon Sep 17 00:00:00 2001 From: Kevin Duret Date: Thu, 16 Jul 2026 11:20:09 +0200 Subject: [PATCH 17/49] TEMP(test-pulp-unstable): parallelize the client-side uploads (pool of 8) --- .../package-delivery-pulp/deliver-rpm.sh | 47 ++++++++++++++----- 1 file changed, 34 insertions(+), 13 deletions(-) diff --git a/.github/actions/package-delivery-pulp/deliver-rpm.sh b/.github/actions/package-delivery-pulp/deliver-rpm.sh index 01e911068e..55eb319438 100755 --- a/.github/actions/package-delivery-pulp/deliver-rpm.sh +++ b/.github/actions/package-delivery-pulp/deliver-rpm.sh @@ -171,22 +171,42 @@ for ARCH in noarch x86_64; do # promote-to-stable can identify them. TASK_HREFS=() SHA256S=() - for FILE in "${ARCH_FILES[@]}"; do - # refresh from the parent shell: pulp_upload runs in a command substitution - # (subshell), so its internal refresh cannot update this shell's token — - # the guardrail curl below and the token inherited by the subshells must - # be kept fresh from here. - refresh_pulp_token - assert_not_in_stable "$FILE" "$ARCH" - echo "[INFO] Uploading $(basename "$FILE") (module $MODULE_NAME)" - sha256=$(sha256sum "$FILE" | cut -d' ' -f1) - SHA256S+=("$sha256") - TASK_HREFS+=("$( + # TEMP(test-pulp-unstable): the uploads are parallelized client-side (the + # repository-less create tasks already parallelize server-side). A bounded + # pool of background subshells posts the files, each writing its task href + # to a marker file; the parent refreshes the OIDC token between spawn + # chunks so the workers always inherit a fresh token. Worker failures are + # detected through missing/empty marker files after the wait. + UPLOAD_DIR=$(mktemp -d) + MAX_PARALLEL_UPLOADS=8 + for i in "${!ARCH_FILES[@]}"; do + FILE=${ARCH_FILES[$i]} + if ((i % 40 == 0)); then + refresh_pulp_token + fi + ( + assert_not_in_stable "$FILE" "$ARCH" + echo "[INFO] Uploading $(basename "$FILE") (module $MODULE_NAME)" pulp_upload \ -F "file=@\"$FILE\"" \ -F "pulp_labels=$PULP_LABELS" \ - "$PULP_URL/api/v3/content/rpm/packages/" - )") + "$PULP_URL/api/v3/content/rpm/packages/" > "$UPLOAD_DIR/$i.task" + ) & + while (($(jobs -rp | wc -l) >= MAX_PARALLEL_UPLOADS)); do + wait -n || true + done + done + wait || true + + for i in "${!ARCH_FILES[@]}"; do + FILE=${ARCH_FILES[$i]} + if [[ ! -s "$UPLOAD_DIR/$i.task" ]]; then + echo "::error::Upload failed for $(basename "$FILE") (no task href, see the worker error above)" + exit 1 + fi + TASK_HREFS+=("$(cat "$UPLOAD_DIR/$i.task")") + sha256=$(sha256sum "$FILE" | cut -d' ' -f1) + SHA256S+=("$sha256") # record the uploaded package in the manifest (name-version-release parsed # the same way as assert_not_in_stable) for the verification step @@ -203,6 +223,7 @@ for ARCH in noarch x86_64; do --arg repository "$REPOSITORY_NAME" --arg base_path "$BASE_PATH" \ '{filename:$filename,name:$name,version:$version,release:$release,arch:$arch,sha256:$sha256,repository:$repository,base_path:$base_path}')" done + rm -rf "$UPLOAD_DIR" echo "[INFO] Waiting for ${#TASK_HREFS[@]} upload task(s) and resolving the content" CONTENT_HREFS=() From a0934ab8ff3dc917422b55847420bed28659e82f Mon Sep 17 00:00:00 2001 From: Kevin Duret Date: Thu, 16 Jul 2026 11:25:06 +0200 Subject: [PATCH 18/49] chore(ci): retrigger after the skip label re-add race From 03bc03c040f248b21246aa2905f63b83c9bd6554 Mon Sep 17 00:00:00 2001 From: Kevin Duret Date: Thu, 16 Jul 2026 11:42:21 +0200 Subject: [PATCH 19/49] TEMP(test-pulp-unstable): batched deb delivery (orphan uploads + PRC + single modify) --- .../package-delivery-pulp/deliver-deb.sh | 208 ++++++++++++++++-- 1 file changed, 190 insertions(+), 18 deletions(-) diff --git a/.github/actions/package-delivery-pulp/deliver-deb.sh b/.github/actions/package-delivery-pulp/deliver-deb.sh index c4ef23a5a4..a62b0a44ab 100755 --- a/.github/actions/package-delivery-pulp/deliver-deb.sh +++ b/.github/actions/package-delivery-pulp/deliver-deb.sh @@ -91,22 +91,84 @@ PULP_LABELS=$(jq -cn \ --arg workflow "${GITHUB_WORKFLOW:-}" \ '{"module": $mod, "git_commit": $git_commit, "git_ref": $git_ref, "github_run_id": $run_id, "github_actor": $actor, "github_workflow": $workflow}') -# the upload tasks are awaited as a batch after the loop: pulp serializes the -# tasks of the shared repository server-side, so waiting for each task before -# sending the next upload would pay the task-queue latency once per package -# instead of once per delivery. -TASK_HREFS=() +# TEMP(test-pulp-unstable) experimental batched delivery, deb flavor. Unlike +# rpm, a repository-less deb upload ignores the distribution/component +# parameters (pulp_deb only creates the suite association inside the +# repository code path), so the suite association is created explicitly as +# PackageReleaseComponent content units. The release_components api is not +# listable by the OIDC ci-user, so the FIRST package of each architecture is +# delivered through the legacy repository code path - that also +# get_or_creates the ReleaseComponent and the ReleaseArchitecture of the +# suite - and its PackageReleaseComponent (listable) yields the release +# component href for the whole batch. Everything else is uploaded as +# unassociated content in a client-side pool, associated in parallel tasks +# (no repository lock), then added to the repository with a single modify. +lookup_deb_content() { + # emit the href of a deb content unit matching the query, empty if absent + local endpoint=$1 query=$2 + curl -fsSL -H "Authorization: Github $PULP_TOKEN" -G \ + --data-urlencode "limit=1" \ + $query \ + "$PULP_URL/api/v3/content/deb/$endpoint/" | jq -r '.results[0].pulp_href // empty' +} + +resolve_task_content() { + # wait for a content-create task and emit its created content href; fall + # back to a lookup (content already existing on a job re-run) + local task_href=$1 endpoint=$2 fallback_query=$3 + local body state content attempt + for ((attempt = 0; attempt < 200; attempt++)); do + refresh_pulp_token + body=$(curl -fsSL -H "Authorization: Github $PULP_TOKEN" "$PULP_URL$task_href" 2>/dev/null) || body="" + state=$(echo "$body" | jq -r '.state' 2>/dev/null) || state="" + case "$state" in + completed) + content=$(echo "$body" | jq -r '.created_resources[0] // empty') + if [[ -n "$content" ]]; then + echo "$content" + return 0 + fi + break + ;; + failed | canceled) + break + ;; + *) + sleep 3 + ;; + esac + done + content=$(lookup_deb_content "$endpoint" "$fallback_query") + if [[ -z "$content" ]]; then + echo "::error::Cannot resolve the created deb content for task $task_href" >&2 + return 1 + fi + echo "$content" +} + +# group the files by architecture: the first file of each arch goes through +# the legacy path so the suite association targets of that arch exist +declare -A ARCH_SEEN=() +LEGACY_FILES=() +ORPHAN_FILES=() +FILE_ARCHS=() for FILE in "${FILES[@]}"; do - # refresh from the parent shell: pulp_upload runs in a command substitution - # (subshell), so its internal refresh cannot update this shell's token — the - # token inherited by the subshells must be kept fresh from here. - refresh_pulp_token + arch=$(dpkg-deb -f "$FILE" Architecture) + FILE_ARCHS+=("$arch") + if [[ -z "${ARCH_SEEN[$arch]+set}" ]]; then + ARCH_SEEN[$arch]=1 + LEGACY_FILES+=("$FILE") + else + ORPHAN_FILES+=("$FILE") + fi +done + +refresh_pulp_token +LEGACY_TASKS=() +for FILE in "${LEGACY_FILES[@]}"; do assert_not_in_stable "$FILE" - echo "[INFO] Uploading $FILE to $POOL_PATH/ ($SUITE/main, module $MODULE_NAME)" - # packages are labeled with their module so that promote-to-stable can identify - # which packages belong to this module, pulp-cli does not allow to set labels nor - # the relative path of deb packages so the api is used directly - TASK_HREFS+=("$( + echo "[INFO] Uploading $FILE to $POOL_PATH/ ($SUITE/main, module $MODULE_NAME) [legacy path]" + LEGACY_TASKS+=("$( pulp_upload \ -F "file=@\"$FILE\"" \ -F "relative_path=$POOL_PATH/$FILE" \ @@ -116,8 +178,121 @@ for FILE in "${FILES[@]}"; do -F "pulp_labels=$PULP_LABELS" \ "$PULP_URL/api/v3/content/deb/packages/" )") +done +echo "[INFO] Waiting for ${#LEGACY_TASKS[@]} legacy upload task(s)" +wait_tasks "${LEGACY_TASKS[@]}" + +# the release component href of the suite, through the first legacy package +refresh_pulp_token +first_sha256=$(sha256sum "${LEGACY_FILES[0]}" | cut -d' ' -f1) +FIRST_PACKAGE_HREF=$(lookup_deb_content "packages" "--data-urlencode sha256=$first_sha256") +if [[ -z "$FIRST_PACKAGE_HREF" ]]; then + echo "::error::Cannot find the legacy-delivered package ${LEGACY_FILES[0]} (sha256 $first_sha256)" + exit 1 +fi +# the legacy package is a fresh build (unique per-build version), so its only +# suite association is the one the legacy upload just created for $SUITE/main +RELEASE_COMPONENT_HREF=$( + curl -fsSL -H "Authorization: Github $PULP_TOKEN" -G \ + --data-urlencode "package=$FIRST_PACKAGE_HREF" \ + "$PULP_URL/api/v3/content/deb/package_release_components/" \ + | jq -r '.results[0].release_component // empty' +) +if [[ -z "$RELEASE_COMPONENT_HREF" ]]; then + echo "::error::Cannot resolve the $SUITE/main release component from the legacy-delivered package" + exit 1 +fi +echo "[INFO] Release component of $SUITE/main: $RELEASE_COMPONENT_HREF" - # record the uploaded package in the manifest for the verification step +# parallel repository-less uploads (pool of 8), marker-file based like rpm +UPLOAD_DIR=$(mktemp -d) +MAX_PARALLEL_UPLOADS=8 +for i in "${!ORPHAN_FILES[@]}"; do + FILE=${ORPHAN_FILES[$i]} + if ((i % 40 == 0)); then + refresh_pulp_token + fi + ( + assert_not_in_stable "$FILE" + echo "[INFO] Uploading $FILE to $POOL_PATH/ ($SUITE/main, module $MODULE_NAME)" + pulp_upload \ + -F "file=@\"$FILE\"" \ + -F "relative_path=$POOL_PATH/$FILE" \ + -F "pulp_labels=$PULP_LABELS" \ + "$PULP_URL/api/v3/content/deb/packages/" > "$UPLOAD_DIR/$i.task" + ) & + while (($(jobs -rp | wc -l) >= MAX_PARALLEL_UPLOADS)); do + wait -n || true + done +done +wait || true + +echo "[INFO] Resolving ${#ORPHAN_FILES[@]} uploaded package(s)" +PACKAGE_HREFS=() +ORPHAN_SHA256S=() +for i in "${!ORPHAN_FILES[@]}"; do + FILE=${ORPHAN_FILES[$i]} + if [[ ! -s "$UPLOAD_DIR/$i.task" ]]; then + echo "::error::Upload failed for $FILE (no task href, see the worker error above)" + exit 1 + fi + if ((i % 40 == 0)); then + refresh_pulp_token + fi + sha256=$(sha256sum "$FILE" | cut -d' ' -f1) + ORPHAN_SHA256S+=("$sha256") + PACKAGE_HREFS+=("$(resolve_task_content "$(cat "$UPLOAD_DIR/$i.task")" "packages" "--data-urlencode sha256=$sha256")") +done + +# associate every uploaded package with the suite component: one small task +# per association, parallelized (no repository lock involved) +echo "[INFO] Associating ${#PACKAGE_HREFS[@]} package(s) with $SUITE/main" +for i in "${!PACKAGE_HREFS[@]}"; do + if ((i % 40 == 0)); then + refresh_pulp_token + fi + ( + curl -fsSL -H "Authorization: Github $PULP_TOKEN" \ + -X POST -H "Content-Type: application/json" \ + -d "{\"package\": \"${PACKAGE_HREFS[$i]}\", \"release_component\": \"$RELEASE_COMPONENT_HREF\"}" \ + "$PULP_URL/api/v3/content/deb/package_release_components/" \ + | jq -r '.task // empty' > "$UPLOAD_DIR/$i.prctask" + ) & + while (($(jobs -rp | wc -l) >= MAX_PARALLEL_UPLOADS)); do + wait -n || true + done +done +wait || true + +PRC_HREFS=() +for i in "${!PACKAGE_HREFS[@]}"; do + if [[ ! -s "$UPLOAD_DIR/$i.prctask" ]]; then + echo "::error::Suite association failed for ${ORPHAN_FILES[$i]} (no task href)" + exit 1 + fi + if ((i % 40 == 0)); then + refresh_pulp_token + fi + PRC_HREFS+=("$(resolve_task_content "$(cat "$UPLOAD_DIR/$i.prctask")" "package_release_components" \ + "--data-urlencode package=${PACKAGE_HREFS[$i]} --data-urlencode release_component=$RELEASE_COMPONENT_HREF")") +done +rm -rf "$UPLOAD_DIR" + +if ((${#PACKAGE_HREFS[@]} > 0)); then + echo "[INFO] Adding ${#PACKAGE_HREFS[@]} package(s) and their suite associations to $REPOSITORY_NAME in a single task" + refresh_pulp_token + ADD_BODY=$(printf '%s\n' "${PACKAGE_HREFS[@]}" "${PRC_HREFS[@]}" | jq -R . | jq -cs '{add_content_units: .}') + MODIFY_TASK=$( + curl -fsSL -H "Authorization: Github $PULP_TOKEN" \ + -X POST -H "Content-Type: application/json" \ + -d "$ADD_BODY" \ + "$PULP_URL${REPOSITORY_HREF}modify/" | jq -r '.task' + ) + wait_task "$MODIFY_TASK" +fi + +# record every delivered package in the manifest for the verification step +for FILE in "${FILES[@]}"; do name=$(dpkg-deb -f "$FILE" Package) version=$(dpkg-deb -f "$FILE" Version) arch=$(dpkg-deb -f "$FILE" Architecture) @@ -129,9 +304,6 @@ for FILE in "${FILES[@]}"; do '{filename:$filename,name:$name,version:$version,arch:$arch,sha256:$sha256,repository:$repository,base_path:$base_path,suite:$suite,relative_path:$relative_path}')" done -echo "[INFO] Waiting for ${#TASK_HREFS[@]} upload task(s) to complete" -wait_tasks "${TASK_HREFS[@]}" - echo "[INFO] Publishing repository $REPOSITORY_NAME" pulp deb publication create --repository "$REPOSITORY_NAME" --structured >/dev/null From 97fb4e969d24d711099cb2579c5186f5740a0345 Mon Sep 17 00:00:00 2001 From: Kevin Duret Date: Thu, 16 Jul 2026 12:46:17 +0200 Subject: [PATCH 20/49] TEMP(test-pulp-unstable): fix the stale-token modify, parallelize the resolutions --- .../package-delivery-pulp/deliver-deb.sh | 45 ++++++++++++++++--- .../package-delivery-pulp/deliver-rpm.sh | 32 +++++++++++-- 2 files changed, 67 insertions(+), 10 deletions(-) diff --git a/.github/actions/package-delivery-pulp/deliver-deb.sh b/.github/actions/package-delivery-pulp/deliver-deb.sh index a62b0a44ab..e729b00494 100755 --- a/.github/actions/package-delivery-pulp/deliver-deb.sh +++ b/.github/actions/package-delivery-pulp/deliver-deb.sh @@ -228,7 +228,6 @@ done wait || true echo "[INFO] Resolving ${#ORPHAN_FILES[@]} uploaded package(s)" -PACKAGE_HREFS=() ORPHAN_SHA256S=() for i in "${!ORPHAN_FILES[@]}"; do FILE=${ORPHAN_FILES[$i]} @@ -236,12 +235,28 @@ for i in "${!ORPHAN_FILES[@]}"; do echo "::error::Upload failed for $FILE (no task href, see the worker error above)" exit 1 fi + ORPHAN_SHA256S+=("$(sha256sum "$FILE" | cut -d' ' -f1)") +done +for i in "${!ORPHAN_FILES[@]}"; do if ((i % 40 == 0)); then refresh_pulp_token fi - sha256=$(sha256sum "$FILE" | cut -d' ' -f1) - ORPHAN_SHA256S+=("$sha256") - PACKAGE_HREFS+=("$(resolve_task_content "$(cat "$UPLOAD_DIR/$i.task")" "packages" "--data-urlencode sha256=$sha256")") + ( + resolve_task_content "$(cat "$UPLOAD_DIR/$i.task")" "packages" \ + "--data-urlencode sha256=${ORPHAN_SHA256S[$i]}" > "$UPLOAD_DIR/$i.content" + ) & + while (($(jobs -rp | wc -l) >= MAX_PARALLEL_UPLOADS)); do + wait -n || true + done +done +wait || true +PACKAGE_HREFS=() +for i in "${!ORPHAN_FILES[@]}"; do + if [[ ! -s "$UPLOAD_DIR/$i.content" ]]; then + echo "::error::Cannot resolve the uploaded content for ${ORPHAN_FILES[$i]} (see the worker error above)" + exit 1 + fi + PACKAGE_HREFS+=("$(cat "$UPLOAD_DIR/$i.content")") done # associate every uploaded package with the suite component: one small task @@ -264,17 +279,33 @@ for i in "${!PACKAGE_HREFS[@]}"; do done wait || true -PRC_HREFS=() for i in "${!PACKAGE_HREFS[@]}"; do if [[ ! -s "$UPLOAD_DIR/$i.prctask" ]]; then echo "::error::Suite association failed for ${ORPHAN_FILES[$i]} (no task href)" exit 1 fi +done +for i in "${!PACKAGE_HREFS[@]}"; do if ((i % 40 == 0)); then refresh_pulp_token fi - PRC_HREFS+=("$(resolve_task_content "$(cat "$UPLOAD_DIR/$i.prctask")" "package_release_components" \ - "--data-urlencode package=${PACKAGE_HREFS[$i]} --data-urlencode release_component=$RELEASE_COMPONENT_HREF")") + ( + resolve_task_content "$(cat "$UPLOAD_DIR/$i.prctask")" "package_release_components" \ + "--data-urlencode package=${PACKAGE_HREFS[$i]} --data-urlencode release_component=$RELEASE_COMPONENT_HREF" \ + > "$UPLOAD_DIR/$i.prc" + ) & + while (($(jobs -rp | wc -l) >= MAX_PARALLEL_UPLOADS)); do + wait -n || true + done +done +wait || true +PRC_HREFS=() +for i in "${!PACKAGE_HREFS[@]}"; do + if [[ ! -s "$UPLOAD_DIR/$i.prc" ]]; then + echo "::error::Cannot resolve the suite association of ${ORPHAN_FILES[$i]} (see the worker error above)" + exit 1 + fi + PRC_HREFS+=("$(cat "$UPLOAD_DIR/$i.prc")") done rm -rf "$UPLOAD_DIR" diff --git a/.github/actions/package-delivery-pulp/deliver-rpm.sh b/.github/actions/package-delivery-pulp/deliver-rpm.sh index 55eb319438..63f8f9bedd 100755 --- a/.github/actions/package-delivery-pulp/deliver-rpm.sh +++ b/.github/actions/package-delivery-pulp/deliver-rpm.sh @@ -142,14 +142,16 @@ for ARCH in noarch x86_64; do # delivered package sharing their name. No-op once purged. refresh_pulp_token LATEST_VERSION=$(pulp rpm repository show --name "$REPOSITORY_NAME" | jq -r '.latest_version_href') - PURGE_HREFS=$( + PURGE_RESPONSE=$( curl -fsSL -H "Authorization: Github $PULP_TOKEN" -G \ --data-urlencode "repository_version=$LATEST_VERSION" \ --data-urlencode "version=20260714" \ --data-urlencode "release=1.el10" \ --data-urlencode "limit=200" \ - "$PULP_URL/api/v3/content/rpm/packages/" | jq -r '.results[].pulp_href' + "$PULP_URL/api/v3/content/rpm/packages/" ) + echo "[INFO] Purge query on $LATEST_VERSION matched $(echo "$PURGE_RESPONSE" | jq -r '.count // "?"') package(s) at 20260714-1.el10" + PURGE_HREFS=$(echo "$PURGE_RESPONSE" | jq -r '.results[].pulp_href') if [[ -n "$PURGE_HREFS" ]]; then echo "[INFO] Purging $(wc -l <<< "$PURGE_HREFS") leftover 20260714-1 package(s) from $REPOSITORY_NAME" PURGE_BODY=$(printf '%s\n' "$PURGE_HREFS" | jq -R . | jq -cs '{remove_content_units: .}') @@ -226,12 +228,36 @@ for ARCH in noarch x86_64; do rm -rf "$UPLOAD_DIR" echo "[INFO] Waiting for ${#TASK_HREFS[@]} upload task(s) and resolving the content" + # TEMP(test-pulp-unstable): the resolutions are parallelized like the + # uploads - sequential, they paid one task GET per package (~6 min at 675) + RESOLVE_DIR=$(mktemp -d) + for i in "${!TASK_HREFS[@]}"; do + if ((i % 40 == 0)); then + refresh_pulp_token + fi + ( + resolve_uploaded_content "${TASK_HREFS[$i]}" "${SHA256S[$i]}" > "$RESOLVE_DIR/$i.content" + ) & + while (($(jobs -rp | wc -l) >= MAX_PARALLEL_UPLOADS)); do + wait -n || true + done + done + wait || true + CONTENT_HREFS=() for i in "${!TASK_HREFS[@]}"; do - CONTENT_HREFS+=("$(resolve_uploaded_content "${TASK_HREFS[$i]}" "${SHA256S[$i]}")") + if [[ ! -s "$RESOLVE_DIR/$i.content" ]]; then + echo "::error::Cannot resolve the uploaded content of task ${TASK_HREFS[$i]} (see the worker error above)" + exit 1 + fi + CONTENT_HREFS+=("$(cat "$RESOLVE_DIR/$i.content")") done + rm -rf "$RESOLVE_DIR" echo "[INFO] Adding ${#CONTENT_HREFS[@]} package(s) to $REPOSITORY_NAME in a single task" + # refresh from the parent shell: the resolutions above ran in subshells, so + # their refreshes never updated this shell's token (the 401 trap, again) + refresh_pulp_token ADD_BODY=$(printf '%s\n' "${CONTENT_HREFS[@]}" | jq -R . | jq -cs '{add_content_units: .}') MODIFY_TASK=$( curl -fsSL -H "Authorization: Github $PULP_TOKEN" \ From 78c03964b4a5a778bb557d41713e1b365ffb53df Mon Sep 17 00:00:00 2001 From: Kevin Duret Date: Thu, 16 Jul 2026 13:53:33 +0200 Subject: [PATCH 21/49] TEMP(test-pulp-unstable): drop the purge (develop delivers daily), sync PRC create The 2026071x-1 packages are not leftovers: #6301 is merged, so every push to develop delivers the changed plugins to Pulp with nightly versioning - the purge would fight the real pipeline. The verification misses on this test branch are an artifact of the PR versioning (20260700-) losing the RPM version comparison against develop's YYYYMMDD-1 builds. The package_release_components create is a plain synchronous DRF create (201 with the created unit, no task): take the href straight from the response, fall back to a lookup for a re-run. --- .../package-delivery-pulp/deliver-deb.sh | 56 +++++++++---------- .../package-delivery-pulp/deliver-rpm.sh | 28 ---------- 2 files changed, 26 insertions(+), 58 deletions(-) diff --git a/.github/actions/package-delivery-pulp/deliver-deb.sh b/.github/actions/package-delivery-pulp/deliver-deb.sh index e729b00494..2b145bae16 100755 --- a/.github/actions/package-delivery-pulp/deliver-deb.sh +++ b/.github/actions/package-delivery-pulp/deliver-deb.sh @@ -259,19 +259,29 @@ for i in "${!ORPHAN_FILES[@]}"; do PACKAGE_HREFS+=("$(cat "$UPLOAD_DIR/$i.content")") done -# associate every uploaded package with the suite component: one small task -# per association, parallelized (no repository lock involved) +# associate every uploaded package with the suite component. The +# package_release_components create is a plain synchronous DRF create (201 +# with the created unit, no task), so the href comes straight out of the +# response; a failed create (unit already existing on a job re-run) falls +# back to a lookup. echo "[INFO] Associating ${#PACKAGE_HREFS[@]} package(s) with $SUITE/main" for i in "${!PACKAGE_HREFS[@]}"; do if ((i % 40 == 0)); then refresh_pulp_token fi ( - curl -fsSL -H "Authorization: Github $PULP_TOKEN" \ - -X POST -H "Content-Type: application/json" \ - -d "{\"package\": \"${PACKAGE_HREFS[$i]}\", \"release_component\": \"$RELEASE_COMPONENT_HREF\"}" \ - "$PULP_URL/api/v3/content/deb/package_release_components/" \ - | jq -r '.task // empty' > "$UPLOAD_DIR/$i.prctask" + response=$( + curl -fsSL -H "Authorization: Github $PULP_TOKEN" \ + -X POST -H "Content-Type: application/json" \ + -d "{\"package\": \"${PACKAGE_HREFS[$i]}\", \"release_component\": \"$RELEASE_COMPONENT_HREF\"}" \ + "$PULP_URL/api/v3/content/deb/package_release_components/" + ) || response="" + href=$(echo "$response" | jq -r '.pulp_href // .task // empty') + if [[ -z "$href" ]]; then + href=$(lookup_deb_content "package_release_components" \ + "--data-urlencode package=${PACKAGE_HREFS[$i]} --data-urlencode release_component=$RELEASE_COMPONENT_HREF") + fi + printf '%s' "$href" > "$UPLOAD_DIR/$i.prc" ) & while (($(jobs -rp | wc -l) >= MAX_PARALLEL_UPLOADS)); do wait -n || true @@ -279,33 +289,19 @@ for i in "${!PACKAGE_HREFS[@]}"; do done wait || true -for i in "${!PACKAGE_HREFS[@]}"; do - if [[ ! -s "$UPLOAD_DIR/$i.prctask" ]]; then - echo "::error::Suite association failed for ${ORPHAN_FILES[$i]} (no task href)" - exit 1 - fi -done -for i in "${!PACKAGE_HREFS[@]}"; do - if ((i % 40 == 0)); then - refresh_pulp_token - fi - ( - resolve_task_content "$(cat "$UPLOAD_DIR/$i.prctask")" "package_release_components" \ - "--data-urlencode package=${PACKAGE_HREFS[$i]} --data-urlencode release_component=$RELEASE_COMPONENT_HREF" \ - > "$UPLOAD_DIR/$i.prc" - ) & - while (($(jobs -rp | wc -l) >= MAX_PARALLEL_UPLOADS)); do - wait -n || true - done -done -wait || true PRC_HREFS=() for i in "${!PACKAGE_HREFS[@]}"; do - if [[ ! -s "$UPLOAD_DIR/$i.prc" ]]; then - echo "::error::Cannot resolve the suite association of ${ORPHAN_FILES[$i]} (see the worker error above)" + href="" + [[ -s "$UPLOAD_DIR/$i.prc" ]] && href=$(cat "$UPLOAD_DIR/$i.prc") + if [[ "$href" == */tasks/* ]]; then + href=$(resolve_task_content "$href" "package_release_components" \ + "--data-urlencode package=${PACKAGE_HREFS[$i]} --data-urlencode release_component=$RELEASE_COMPONENT_HREF") + fi + if [[ -z "$href" ]]; then + echo "::error::Suite association failed for ${ORPHAN_FILES[$i]} (see the worker error above)" exit 1 fi - PRC_HREFS+=("$(cat "$UPLOAD_DIR/$i.prc")") + PRC_HREFS+=("$href") done rm -rf "$UPLOAD_DIR" diff --git a/.github/actions/package-delivery-pulp/deliver-rpm.sh b/.github/actions/package-delivery-pulp/deliver-rpm.sh index 63f8f9bedd..d201c17471 100755 --- a/.github/actions/package-delivery-pulp/deliver-rpm.sh +++ b/.github/actions/package-delivery-pulp/deliver-rpm.sh @@ -136,34 +136,6 @@ for ARCH in noarch x86_64; do REPOSITORY_HREF=$(pulp rpm repository show --name "$REPOSITORY_NAME" | jq -r '.pulp_href') - # TEMP(test-pulp-unstable): one-off purge of the 20260714-1 leftovers of an - # earlier test round - they outrank the 20260700- test builds in - # RPM version comparison, so the retention policy evicts every freshly - # delivered package sharing their name. No-op once purged. - refresh_pulp_token - LATEST_VERSION=$(pulp rpm repository show --name "$REPOSITORY_NAME" | jq -r '.latest_version_href') - PURGE_RESPONSE=$( - curl -fsSL -H "Authorization: Github $PULP_TOKEN" -G \ - --data-urlencode "repository_version=$LATEST_VERSION" \ - --data-urlencode "version=20260714" \ - --data-urlencode "release=1.el10" \ - --data-urlencode "limit=200" \ - "$PULP_URL/api/v3/content/rpm/packages/" - ) - echo "[INFO] Purge query on $LATEST_VERSION matched $(echo "$PURGE_RESPONSE" | jq -r '.count // "?"') package(s) at 20260714-1.el10" - PURGE_HREFS=$(echo "$PURGE_RESPONSE" | jq -r '.results[].pulp_href') - if [[ -n "$PURGE_HREFS" ]]; then - echo "[INFO] Purging $(wc -l <<< "$PURGE_HREFS") leftover 20260714-1 package(s) from $REPOSITORY_NAME" - PURGE_BODY=$(printf '%s\n' "$PURGE_HREFS" | jq -R . | jq -cs '{remove_content_units: .}') - PURGE_TASK=$( - curl -fsSL -H "Authorization: Github $PULP_TOKEN" \ - -X POST -H "Content-Type: application/json" \ - -d "$PURGE_BODY" \ - "$PULP_URL${REPOSITORY_HREF}modify/" | jq -r '.task' - ) - wait_task "$PURGE_TASK" - fi - # TEMP(test-pulp-unstable) experimental batched delivery: packages are # uploaded as unassociated content (repository-less, so the create tasks # parallelize across the pulp workers instead of serializing on the From f1f358c9dbf22c500eab654ccf0224372fb23909 Mon Sep 17 00:00:00 2001 From: Kevin Duret Date: Thu, 16 Jul 2026 14:23:54 +0200 Subject: [PATCH 22/49] fix(ci): paginate the delivery-verification presence listing A single limit=1000 page silently truncates once the repository holds more module packages than the page size (the shared apt-plugins repository is past 5000), flagging delivered packages as missing. --- .github/scripts/pulp/check-deb.sh | 18 ++++++++++++------ .github/scripts/pulp/check-rpm.sh | 18 ++++++++++++------ 2 files changed, 24 insertions(+), 12 deletions(-) diff --git a/.github/scripts/pulp/check-deb.sh b/.github/scripts/pulp/check-deb.sh index 0bb99adca0..d8a804c0b9 100755 --- a/.github/scripts/pulp/check-deb.sh +++ b/.github/scripts/pulp/check-deb.sh @@ -36,13 +36,19 @@ for repo in $(printf '%s\n' "${E_REPOSITORY[@]}" | sort -u); do PRESENT_BY_REPO[$repo]="" continue fi + # paginate: a single page silently truncates once the repository holds more + # module packages than the page size PRESENT_BY_REPO[$repo]=$( - curl -fsSL -H "Authorization: Github $PULP_TOKEN" -G \ - --data-urlencode "repository_version=$version_href" \ - --data-urlencode "pulp_label_select=module=$MODULE_NAME" \ - --data-urlencode "limit=1000" \ - "$PULP_URL/api/v3/content/deb/packages/" 2>/dev/null \ - | jq -r '.results[].relative_path' + url="$PULP_URL/api/v3/content/deb/packages/?$( + printf 'repository_version=%s&pulp_label_select=%s&limit=1000' \ + "$(jq -rn --arg v "$version_href" '$v | @uri')" \ + "$(jq -rn --arg v "module=$MODULE_NAME" '$v | @uri')" + )" + while [[ -n "$url" ]]; do + page=$(curl -fsSL -H "Authorization: Github $PULP_TOKEN" "$url" 2>/dev/null) || break + echo "$page" | jq -r '.results[].relative_path' + url=$(echo "$page" | jq -r '.next // empty') + done ) done diff --git a/.github/scripts/pulp/check-rpm.sh b/.github/scripts/pulp/check-rpm.sh index dc17654cc4..661e985d92 100755 --- a/.github/scripts/pulp/check-rpm.sh +++ b/.github/scripts/pulp/check-rpm.sh @@ -30,13 +30,19 @@ for repo in $(printf '%s\n' "${E_REPOSITORY[@]}" | sort -u); do PRESENT_BY_REPO[$repo]="" continue fi + # paginate: a single page silently truncates once the repository holds more + # module packages than the page size PRESENT_BY_REPO[$repo]=$( - curl -fsSL -H "Authorization: Github $PULP_TOKEN" -G \ - --data-urlencode "repository_version=$version_href" \ - --data-urlencode "pulp_label_select=module=$MODULE_NAME" \ - --data-urlencode "limit=1000" \ - "$PULP_URL/api/v3/content/rpm/packages/" 2>/dev/null \ - | jq -r '.results[].location_href' | awk -F/ '{print $NF}' + url="$PULP_URL/api/v3/content/rpm/packages/?$( + printf 'repository_version=%s&pulp_label_select=%s&limit=1000' \ + "$(jq -rn --arg v "$version_href" '$v | @uri')" \ + "$(jq -rn --arg v "module=$MODULE_NAME" '$v | @uri')" + )" + while [[ -n "$url" ]]; do + page=$(curl -fsSL -H "Authorization: Github $PULP_TOKEN" "$url" 2>/dev/null) || break + echo "$page" | jq -r '.results[].location_href' | awk -F/ '{print $NF}' + url=$(echo "$page" | jq -r '.next // empty') + done ) done From 9ae6b203817baee701ad8d8d3e7e4facdf56b001 Mon Sep 17 00:00:00 2001 From: Kevin Duret Date: Thu, 16 Jul 2026 14:59:42 +0200 Subject: [PATCH 23/49] TEMP(test-pulp-unstable): instrument the presence listing (page failures, counts) --- .github/scripts/pulp/check-deb.sh | 6 +++++- .github/scripts/pulp/check-rpm.sh | 6 +++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/.github/scripts/pulp/check-deb.sh b/.github/scripts/pulp/check-deb.sh index d8a804c0b9..fbf3142a57 100755 --- a/.github/scripts/pulp/check-deb.sh +++ b/.github/scripts/pulp/check-deb.sh @@ -45,11 +45,15 @@ for repo in $(printf '%s\n' "${E_REPOSITORY[@]}" | sort -u); do "$(jq -rn --arg v "module=$MODULE_NAME" '$v | @uri')" )" while [[ -n "$url" ]]; do - page=$(curl -fsSL -H "Authorization: Github $PULP_TOKEN" "$url" 2>/dev/null) || break + page=$(curl -fsSL -H "Authorization: Github $PULP_TOKEN" "$url") || { + echo "[WARN] presence page fetch failed for $repo ($url)" >&2 + break + } echo "$page" | jq -r '.results[].relative_path' url=$(echo "$page" | jq -r '.next // empty') done ) + echo "[INFO] Repository $repo holds $(printf '%s\n' "${PRESENT_BY_REPO[$repo]}" | grep -c .) module package(s) in its latest version" done declare -A PRESENT_IDX diff --git a/.github/scripts/pulp/check-rpm.sh b/.github/scripts/pulp/check-rpm.sh index 661e985d92..2dd75ec6e9 100755 --- a/.github/scripts/pulp/check-rpm.sh +++ b/.github/scripts/pulp/check-rpm.sh @@ -39,11 +39,15 @@ for repo in $(printf '%s\n' "${E_REPOSITORY[@]}" | sort -u); do "$(jq -rn --arg v "module=$MODULE_NAME" '$v | @uri')" )" while [[ -n "$url" ]]; do - page=$(curl -fsSL -H "Authorization: Github $PULP_TOKEN" "$url" 2>/dev/null) || break + page=$(curl -fsSL -H "Authorization: Github $PULP_TOKEN" "$url") || { + echo "[WARN] presence page fetch failed for $repo ($url)" >&2 + break + } echo "$page" | jq -r '.results[].location_href' | awk -F/ '{print $NF}' url=$(echo "$page" | jq -r '.next // empty') done ) + echo "[INFO] Repository $repo holds $(printf '%s\n' "${PRESENT_BY_REPO[$repo]}" | grep -c .) module package(s) in its latest version" done declare -A PRESENT_IDX # idx -> true|false From ec2579a32b9af919d51d36dcffac4b2d31c81468 Mon Sep 17 00:00:00 2001 From: Kevin Duret Date: Thu, 16 Jul 2026 15:06:23 +0200 Subject: [PATCH 24/49] chore(ci): retrigger From 78e8d1e94d1177e905aeaca0ceb12213ca8154a4 Mon Sep 17 00:00:00 2001 From: Kevin Duret Date: Thu, 16 Jul 2026 15:46:29 +0200 Subject: [PATCH 25/49] fix(ci): bound the delivery-verification presence listing List the module packages newest first and stop as soon as every expected package has been seen: the shared plugins repository holds 10k+ module packages, so an unbounded offset walk is slow, eventually fails server-side, and the expected packages sit in the tail with the default ordering. The unexpected-extras report becomes count-based for the same reason. --- .github/scripts/pulp/check-deb.sh | 44 +++++++++++++------- .github/scripts/pulp/check-rpm.sh | 69 +++++++++++++++++-------------- 2 files changed, 67 insertions(+), 46 deletions(-) diff --git a/.github/scripts/pulp/check-deb.sh b/.github/scripts/pulp/check-deb.sh index fbf3142a57..6a13fce5fd 100755 --- a/.github/scripts/pulp/check-deb.sh +++ b/.github/scripts/pulp/check-deb.sh @@ -28,6 +28,10 @@ mapfile -t E_SUITE < <(echo "$PACKAGES_JSON" | jq -r '.[].suite') mapfile -t E_RELPATH < <(echo "$PACKAGES_JSON" | jq -r '.[].relative_path') # --- physical presence: content units in the repository's latest version ---- +# newest first, stopping as soon as every expected package of the repository +# has been seen: the shared plugins repository holds 10k+ module packages and +# deep offset pagination both costs the server dearly and eventually fails, +# while the freshly delivered packages are the newest content by construction. declare -A PRESENT_BY_REPO for repo in $(printf '%s\n' "${E_REPOSITORY[@]}" | sort -u); do version_href=$(pulp deb repository show --name "$repo" 2>/dev/null | jq -r '.latest_version_href // empty') @@ -36,24 +40,32 @@ for repo in $(printf '%s\n' "${E_REPOSITORY[@]}" | sort -u); do PRESENT_BY_REPO[$repo]="" continue fi - # paginate: a single page silently truncates once the repository holds more - # module packages than the page size - PRESENT_BY_REPO[$repo]=$( - url="$PULP_URL/api/v3/content/deb/packages/?$( - printf 'repository_version=%s&pulp_label_select=%s&limit=1000' \ - "$(jq -rn --arg v "$version_href" '$v | @uri')" \ - "$(jq -rn --arg v "module=$MODULE_NAME" '$v | @uri')" - )" - while [[ -n "$url" ]]; do - page=$(curl -fsSL -H "Authorization: Github $PULP_TOKEN" "$url") || { - echo "[WARN] presence page fetch failed for $repo ($url)" >&2 + PRESENT_BY_REPO[$repo]="" + url="$PULP_URL/api/v3/content/deb/packages/?$( + printf 'repository_version=%s&pulp_label_select=%s&ordering=-pulp_created&limit=1000' \ + "$(jq -rn --arg v "$version_href" '$v | @uri')" \ + "$(jq -rn --arg v "module=$MODULE_NAME" '$v | @uri')" + )" + pages=0 + while [[ -n "$url" ]] && ((pages < 20)); do + page=$(curl -fsSL -H "Authorization: Github $PULP_TOKEN" "$url") || { + echo "[WARN] presence page fetch failed for $repo ($url)" >&2 + break + } + ((pages == 0)) && echo "[INFO] Repository $repo holds $(echo "$page" | jq -r '.count') module package(s) in its latest version" + PRESENT_BY_REPO[$repo]+=$(echo "$page" | jq -r '.results[].relative_path')$'\n' + pages=$((pages + 1)) + all_found=true + for i in "${!E_FILENAME[@]}"; do + [[ "${E_REPOSITORY[$i]}" == "$repo" ]] || continue + if ! printf '%s\n' "${PRESENT_BY_REPO[$repo]}" | grep -Fxq "${E_RELPATH[$i]}"; then + all_found=false break - } - echo "$page" | jq -r '.results[].relative_path' - url=$(echo "$page" | jq -r '.next // empty') + fi done - ) - echo "[INFO] Repository $repo holds $(printf '%s\n' "${PRESENT_BY_REPO[$repo]}" | grep -c .) module package(s) in its latest version" + [[ "$all_found" == "true" ]] && break + url=$(echo "$page" | jq -r '.next // empty') + done done declare -A PRESENT_IDX diff --git a/.github/scripts/pulp/check-rpm.sh b/.github/scripts/pulp/check-rpm.sh index 2dd75ec6e9..31805a3e2b 100755 --- a/.github/scripts/pulp/check-rpm.sh +++ b/.github/scripts/pulp/check-rpm.sh @@ -20,8 +20,13 @@ mapfile -t E_REPOSITORY < <(echo "$PACKAGES_JSON" | jq -r '.[].repository') mapfile -t E_BASEPATH < <(echo "$PACKAGES_JSON" | jq -r '.[].base_path') # --- physical presence: content units in each repository's latest version -- +# newest first, stopping as soon as every expected package of the repository +# has been seen: a large repository makes deep offset pagination costly and +# flaky, while the freshly delivered packages are the newest content by +# construction. declare -A PRESENT_BY_REPO declare -A EXPECTED_COUNT_BY_REPO +declare -A TOTAL_COUNT_BY_REPO for repo in $(printf '%s\n' "${E_REPOSITORY[@]}" | sort -u); do version_href=$(pulp rpm repository show --name "$repo" 2>/dev/null | jq -r '.latest_version_href // empty') @@ -30,24 +35,35 @@ for repo in $(printf '%s\n' "${E_REPOSITORY[@]}" | sort -u); do PRESENT_BY_REPO[$repo]="" continue fi - # paginate: a single page silently truncates once the repository holds more - # module packages than the page size - PRESENT_BY_REPO[$repo]=$( - url="$PULP_URL/api/v3/content/rpm/packages/?$( - printf 'repository_version=%s&pulp_label_select=%s&limit=1000' \ - "$(jq -rn --arg v "$version_href" '$v | @uri')" \ - "$(jq -rn --arg v "module=$MODULE_NAME" '$v | @uri')" - )" - while [[ -n "$url" ]]; do - page=$(curl -fsSL -H "Authorization: Github $PULP_TOKEN" "$url") || { - echo "[WARN] presence page fetch failed for $repo ($url)" >&2 + PRESENT_BY_REPO[$repo]="" + url="$PULP_URL/api/v3/content/rpm/packages/?$( + printf 'repository_version=%s&pulp_label_select=%s&ordering=-pulp_created&limit=1000' \ + "$(jq -rn --arg v "$version_href" '$v | @uri')" \ + "$(jq -rn --arg v "module=$MODULE_NAME" '$v | @uri')" + )" + pages=0 + while [[ -n "$url" ]] && ((pages < 20)); do + page=$(curl -fsSL -H "Authorization: Github $PULP_TOKEN" "$url") || { + echo "[WARN] presence page fetch failed for $repo ($url)" >&2 + break + } + if ((pages == 0)); then + TOTAL_COUNT_BY_REPO[$repo]=$(echo "$page" | jq -r '.count') + echo "[INFO] Repository $repo holds ${TOTAL_COUNT_BY_REPO[$repo]} module package(s) in its latest version" + fi + PRESENT_BY_REPO[$repo]+=$(echo "$page" | jq -r '.results[].location_href' | awk -F/ '{print $NF}')$'\n' + pages=$((pages + 1)) + all_found=true + for i in "${!E_FILENAME[@]}"; do + [[ "${E_REPOSITORY[$i]}" == "$repo" ]] || continue + if ! printf '%s\n' "${PRESENT_BY_REPO[$repo]}" | grep -Fxq "${E_FILENAME[$i]}"; then + all_found=false break - } - echo "$page" | jq -r '.results[].location_href' | awk -F/ '{print $NF}' - url=$(echo "$page" | jq -r '.next // empty') + fi done - ) - echo "[INFO] Repository $repo holds $(printf '%s\n' "${PRESENT_BY_REPO[$repo]}" | grep -c .) module package(s) in its latest version" + [[ "$all_found" == "true" ]] && break + url=$(echo "$page" | jq -r '.next // empty') + done done declare -A PRESENT_IDX # idx -> true|false @@ -61,21 +77,14 @@ for i in "${!E_FILENAME[@]}"; do fi done -# right number (delivery only): flag content-unit filenames present under the -# module that were not expected (e.g. a stale package left behind). the testing -# and unstable repositories retain a single version per package, so any extra is -# suspect; the stable repository legitimately keeps the full version history of -# previous releases, so this sweep would flag every past release on promotion. +# right number (delivery only): warn when the repository holds more module +# packages than expected. Count-based only: with shared repositories and the +# daily develop deliveries, listing every extra name is noise (and the full +# listing is what the bounded pagination above avoids). [[ "${CHECK_MODE:-delivery}" == "delivery" ]] && for repo in "${!EXPECTED_COUNT_BY_REPO[@]}"; do - present_count=$(printf '%s\n' "${PRESENT_BY_REPO[$repo]:-}" | grep -c . || true) - if [[ "$present_count" -gt "${EXPECTED_COUNT_BY_REPO[$repo]}" ]]; then - echo "::warning::Repository $repo holds $present_count module packages but only ${EXPECTED_COUNT_BY_REPO[$repo]} were expected (unexpected extras)" - while read -r fn; do - [[ -z "$fn" ]] && continue - if ! printf '%s\n' "${E_FILENAME[@]}" | grep -Fxq "$fn"; then - record_row "$fn" "?" "false" "false" "false" - fi - done < <(printf '%s\n' "${PRESENT_BY_REPO[$repo]:-}") + total=${TOTAL_COUNT_BY_REPO[$repo]:-0} + if [[ "$total" -gt "${EXPECTED_COUNT_BY_REPO[$repo]}" ]]; then + echo "::warning::Repository $repo holds $total module packages, ${EXPECTED_COUNT_BY_REPO[$repo]} delivered by this run (older builds and other modules accumulate until cleanup)" fi done From 3b7ce3933b47b4d50988b0bb3aba54b846bca8e8 Mon Sep 17 00:00:00 2001 From: Kevin Duret Date: Thu, 16 Jul 2026 16:17:19 +0200 Subject: [PATCH 26/49] TEMP(test-pulp-unstable): debug the deb presence shape --- .github/scripts/pulp/check-deb.sh | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/scripts/pulp/check-deb.sh b/.github/scripts/pulp/check-deb.sh index 6a13fce5fd..c98dafbffb 100755 --- a/.github/scripts/pulp/check-deb.sh +++ b/.github/scripts/pulp/check-deb.sh @@ -52,7 +52,12 @@ for repo in $(printf '%s\n' "${E_REPOSITORY[@]}" | sort -u); do echo "[WARN] presence page fetch failed for $repo ($url)" >&2 break } - ((pages == 0)) && echo "[INFO] Repository $repo holds $(echo "$page" | jq -r '.count') module package(s) in its latest version" + if ((pages == 0)); then + echo "[INFO] Repository $repo holds $(echo "$page" | jq -r '.count') module package(s) in its latest version" + # TEMP(test-pulp-unstable): compare the api shape with the manifest + echo "[DEBUG] first result: $(echo "$page" | jq -c '.results[0] | {relative_path, pulp_created}')" + echo "[DEBUG] first expected relative_path: ${E_RELPATH[0]}" + fi PRESENT_BY_REPO[$repo]+=$(echo "$page" | jq -r '.results[].relative_path')$'\n' pages=$((pages + 1)) all_found=true From 5003637b9265237be6c0b64e075f7866714acc31 Mon Sep 17 00:00:00 2001 From: Kevin Duret Date: Thu, 16 Jul 2026 16:55:18 +0200 Subject: [PATCH 27/49] fix(ci): stop piping the presence listing into grep -q grep -q exits on the first match, the printf writer takes a SIGPIPE, and pipefail turns the pipeline status into 141: every package that IS present gets flagged missing once the listing outgrows the pipe buffer (the shared plugins repositories are past 12k module packages). Write the listing to a file and let grep read the file - no writer left to kill. --- .github/scripts/pulp/check-deb.sh | 17 ++++++++--------- .github/scripts/pulp/check-rpm.sh | 14 ++++++++------ 2 files changed, 16 insertions(+), 15 deletions(-) diff --git a/.github/scripts/pulp/check-deb.sh b/.github/scripts/pulp/check-deb.sh index c98dafbffb..819cd2db9a 100755 --- a/.github/scripts/pulp/check-deb.sh +++ b/.github/scripts/pulp/check-deb.sh @@ -32,15 +32,17 @@ mapfile -t E_RELPATH < <(echo "$PACKAGES_JSON" | jq -r '.[].relative_path') # has been seen: the shared plugins repository holds 10k+ module packages and # deep offset pagination both costs the server dearly and eventually fails, # while the freshly delivered packages are the newest content by construction. -declare -A PRESENT_BY_REPO +# The listing goes to a file and grep reads the file: piping it into grep -q +# would SIGPIPE the writer on the (early-exiting) first match, and pipefail +# then turns every successful match into a false negative. +declare -A PRESENT_BY_REPO # repo -> file holding one relative_path per line for repo in $(printf '%s\n' "${E_REPOSITORY[@]}" | sort -u); do + PRESENT_BY_REPO[$repo]=$(mktemp) version_href=$(pulp deb repository show --name "$repo" 2>/dev/null | jq -r '.latest_version_href // empty') if [[ -z "$version_href" ]]; then echo "[WARN] Repository $repo does not exist or has no version" - PRESENT_BY_REPO[$repo]="" continue fi - PRESENT_BY_REPO[$repo]="" url="$PULP_URL/api/v3/content/deb/packages/?$( printf 'repository_version=%s&pulp_label_select=%s&ordering=-pulp_created&limit=1000' \ "$(jq -rn --arg v "$version_href" '$v | @uri')" \ @@ -54,16 +56,13 @@ for repo in $(printf '%s\n' "${E_REPOSITORY[@]}" | sort -u); do } if ((pages == 0)); then echo "[INFO] Repository $repo holds $(echo "$page" | jq -r '.count') module package(s) in its latest version" - # TEMP(test-pulp-unstable): compare the api shape with the manifest - echo "[DEBUG] first result: $(echo "$page" | jq -c '.results[0] | {relative_path, pulp_created}')" - echo "[DEBUG] first expected relative_path: ${E_RELPATH[0]}" fi - PRESENT_BY_REPO[$repo]+=$(echo "$page" | jq -r '.results[].relative_path')$'\n' + echo "$page" | jq -r '.results[].relative_path' >> "${PRESENT_BY_REPO[$repo]}" pages=$((pages + 1)) all_found=true for i in "${!E_FILENAME[@]}"; do [[ "${E_REPOSITORY[$i]}" == "$repo" ]] || continue - if ! printf '%s\n' "${PRESENT_BY_REPO[$repo]}" | grep -Fxq "${E_RELPATH[$i]}"; then + if ! grep -Fxq "${E_RELPATH[$i]}" "${PRESENT_BY_REPO[$repo]}"; then all_found=false break fi @@ -75,7 +74,7 @@ done declare -A PRESENT_IDX for i in "${!E_FILENAME[@]}"; do - if printf '%s\n' "${PRESENT_BY_REPO[${E_REPOSITORY[$i]}]:-}" | grep -Fxq "${E_RELPATH[$i]}"; then + if grep -Fxq "${E_RELPATH[$i]}" "${PRESENT_BY_REPO[${E_REPOSITORY[$i]}]}"; then PRESENT_IDX[$i]=true else PRESENT_IDX[$i]=false diff --git a/.github/scripts/pulp/check-rpm.sh b/.github/scripts/pulp/check-rpm.sh index 31805a3e2b..31a945fb1b 100755 --- a/.github/scripts/pulp/check-rpm.sh +++ b/.github/scripts/pulp/check-rpm.sh @@ -24,18 +24,20 @@ mapfile -t E_BASEPATH < <(echo "$PACKAGES_JSON" | jq -r '.[].base_path') # has been seen: a large repository makes deep offset pagination costly and # flaky, while the freshly delivered packages are the newest content by # construction. -declare -A PRESENT_BY_REPO +# The listing goes to a file and grep reads the file: piping it into grep -q +# would SIGPIPE the writer on the (early-exiting) first match, and pipefail +# then turns every successful match into a false negative. +declare -A PRESENT_BY_REPO # repo -> file holding one filename per line declare -A EXPECTED_COUNT_BY_REPO declare -A TOTAL_COUNT_BY_REPO for repo in $(printf '%s\n' "${E_REPOSITORY[@]}" | sort -u); do + PRESENT_BY_REPO[$repo]=$(mktemp) version_href=$(pulp rpm repository show --name "$repo" 2>/dev/null | jq -r '.latest_version_href // empty') if [[ -z "$version_href" ]]; then echo "[WARN] Repository $repo does not exist or has no version" - PRESENT_BY_REPO[$repo]="" continue fi - PRESENT_BY_REPO[$repo]="" url="$PULP_URL/api/v3/content/rpm/packages/?$( printf 'repository_version=%s&pulp_label_select=%s&ordering=-pulp_created&limit=1000' \ "$(jq -rn --arg v "$version_href" '$v | @uri')" \ @@ -51,12 +53,12 @@ for repo in $(printf '%s\n' "${E_REPOSITORY[@]}" | sort -u); do TOTAL_COUNT_BY_REPO[$repo]=$(echo "$page" | jq -r '.count') echo "[INFO] Repository $repo holds ${TOTAL_COUNT_BY_REPO[$repo]} module package(s) in its latest version" fi - PRESENT_BY_REPO[$repo]+=$(echo "$page" | jq -r '.results[].location_href' | awk -F/ '{print $NF}')$'\n' + echo "$page" | jq -r '.results[].location_href' | awk -F/ '{print $NF}' >> "${PRESENT_BY_REPO[$repo]}" pages=$((pages + 1)) all_found=true for i in "${!E_FILENAME[@]}"; do [[ "${E_REPOSITORY[$i]}" == "$repo" ]] || continue - if ! printf '%s\n' "${PRESENT_BY_REPO[$repo]}" | grep -Fxq "${E_FILENAME[$i]}"; then + if ! grep -Fxq "${E_FILENAME[$i]}" "${PRESENT_BY_REPO[$repo]}"; then all_found=false break fi @@ -70,7 +72,7 @@ declare -A PRESENT_IDX # idx -> true|false for i in "${!E_FILENAME[@]}"; do repo=${E_REPOSITORY[$i]} EXPECTED_COUNT_BY_REPO[$repo]=$(( ${EXPECTED_COUNT_BY_REPO[$repo]:-0} + 1 )) - if printf '%s\n' "${PRESENT_BY_REPO[$repo]:-}" | grep -Fxq "${E_FILENAME[$i]}"; then + if grep -Fxq "${E_FILENAME[$i]}" "${PRESENT_BY_REPO[$repo]}"; then PRESENT_IDX[$i]=true else PRESENT_IDX[$i]=false From 9406a81270a37080bc284f4699665712ea306fcf Mon Sep 17 00:00:00 2001 From: Kevin Duret Date: Thu, 16 Jul 2026 17:40:29 +0200 Subject: [PATCH 28/49] chore(ci): retrigger From bac65ba64dc76abf8b4ac41facad6deff2120747 Mon Sep 17 00:00:00 2001 From: Kevin Duret Date: Thu, 16 Jul 2026 18:14:57 +0200 Subject: [PATCH 29/49] fix(ci): survive long publications (background task + re-authenticating wait) pulp-cli reads its token once at startup, so its built-in wait dies with "Authentication failed for tasks_read" as soon as the publication of a large repository outlives the OIDC token validity (~5 minutes, and the shared apt-plugins publication now takes longer than that). Start the publication in the background and poll it with wait_task, which re-authenticates on every attempt. --- .../package-delivery-pulp/deliver-deb.sh | 2 +- .../package-delivery-pulp/deliver-rpm.sh | 2 +- .../promote-to-stable-pulp/promote-deb.sh | 2 +- .../promote-to-stable-pulp/promote-rpm.sh | 2 +- .github/scripts/pulp/api.sh | 23 +++++++++++++++++++ 5 files changed, 27 insertions(+), 4 deletions(-) diff --git a/.github/actions/package-delivery-pulp/deliver-deb.sh b/.github/actions/package-delivery-pulp/deliver-deb.sh index 2b145bae16..1dcc5a421b 100755 --- a/.github/actions/package-delivery-pulp/deliver-deb.sh +++ b/.github/actions/package-delivery-pulp/deliver-deb.sh @@ -332,7 +332,7 @@ for FILE in "${FILES[@]}"; do done echo "[INFO] Publishing repository $REPOSITORY_NAME" -pulp deb publication create --repository "$REPOSITORY_NAME" --structured >/dev/null +create_publication deb "$REPOSITORY_NAME" --structured echo "::notice::Packages are available with: deb $PULP_CONTENT_URL/$BASE_PATH/ $SUITE main" diff --git a/.github/actions/package-delivery-pulp/deliver-rpm.sh b/.github/actions/package-delivery-pulp/deliver-rpm.sh index d201c17471..83dbe4b6ba 100755 --- a/.github/actions/package-delivery-pulp/deliver-rpm.sh +++ b/.github/actions/package-delivery-pulp/deliver-rpm.sh @@ -240,7 +240,7 @@ for ARCH in noarch x86_64; do wait_task "$MODIFY_TASK" echo "[INFO] Publishing repository $REPOSITORY_NAME" - pulp rpm publication create --repository "$REPOSITORY_NAME" >/dev/null + create_publication rpm "$REPOSITORY_NAME" echo "::notice::Packages are available at $PULP_CONTENT_URL/$BASE_PATH/" done diff --git a/.github/actions/promote-to-stable-pulp/promote-deb.sh b/.github/actions/promote-to-stable-pulp/promote-deb.sh index c1db6bcd9f..3ad3eb1860 100755 --- a/.github/actions/promote-to-stable-pulp/promote-deb.sh +++ b/.github/actions/promote-to-stable-pulp/promote-deb.sh @@ -115,7 +115,7 @@ while read -r PACKAGE; do done < <(echo "$PACKAGES" | jq -c '.[]') echo "[INFO] Publishing repository $REPOSITORY_NAME" -pulp deb publication create --repository "$REPOSITORY_NAME" --structured >/dev/null +create_publication deb "$REPOSITORY_NAME" --structured echo "::notice::Packages are available with: deb $PULP_CONTENT_URL/$BASE_PATH/ $STABLE_SUITE main" diff --git a/.github/actions/promote-to-stable-pulp/promote-rpm.sh b/.github/actions/promote-to-stable-pulp/promote-rpm.sh index 8f5c404b18..9426e87e65 100755 --- a/.github/actions/promote-to-stable-pulp/promote-rpm.sh +++ b/.github/actions/promote-to-stable-pulp/promote-rpm.sh @@ -98,7 +98,7 @@ for ARCH in noarch x86_64; do wait_task "$TASK_HREF" echo "[INFO] Publishing repository $STABLE_REPOSITORY_NAME" - pulp rpm publication create --repository "$STABLE_REPOSITORY_NAME" >/dev/null + create_publication rpm "$STABLE_REPOSITORY_NAME" # record the promoted packages (with their stable target coordinates) so the # verification step verifies exactly this set against the stable repo diff --git a/.github/scripts/pulp/api.sh b/.github/scripts/pulp/api.sh index 4295604853..48ba2b0be6 100644 --- a/.github/scripts/pulp/api.sh +++ b/.github/scripts/pulp/api.sh @@ -136,3 +136,26 @@ pulp_upload() { echo "::error::Upload failed after 3 attempts (HTTP ${http_code:-network-error})" >&2 return 1 } + +# create a publication in the background and wait for it with re-authenticating +# polling: pulp-cli reads its token once at startup, so its built-in wait fails +# with "Authentication failed for tasks_read" as soon as the publication of a +# large repository outlives the OIDC token validity (~5 minutes) +create_publication() { + local plugin=$1 repository=$2 + shift 2 + local out task + refresh_pulp_token + out=$(pulp --background "$plugin" publication create --repository "$repository" "$@" 2>&1) || { + echo "$out" + echo "::error::Cannot start the publication of repository $repository" + return 1 + } + task=$(grep -oPm1 '/api/v3/tasks/[0-9a-f-]+/' <<< "$out" || true) + if [[ -z "$task" ]]; then + echo "$out" + echo "::error::Cannot find the publication task of repository $repository in the pulp-cli output" + return 1 + fi + wait_task "$task" +} From bb6477fa746df1a4c93ad221ada27492d0e78a44 Mon Sep 17 00:00:00 2001 From: Kevin Duret Date: Thu, 16 Jul 2026 18:17:49 +0200 Subject: [PATCH 30/49] chore(ci): retrigger From 11ec7b173665910c1e4beb2b48ab78e50292a2fb Mon Sep 17 00:00:00 2001 From: Kevin Duret Date: Thu, 16 Jul 2026 18:52:21 +0200 Subject: [PATCH 31/49] fix(ci): stop the metadata retry window once resolution stalls The retry window only covers publication propagation: once a round reads the published metadata and resolves nothing new while some packages already resolved, the remaining ones are not part of the publication at all (e.g. evicted by the retention policy) - report right away instead of burning the whole 5-minute window, and log the per-round progress. --- .github/scripts/pulp/check-common.sh | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/.github/scripts/pulp/check-common.sh b/.github/scripts/pulp/check-common.sh index 8c13d494b3..ce44d8070c 100644 --- a/.github/scripts/pulp/check-common.sh +++ b/.github/scripts/pulp/check-common.sh @@ -39,15 +39,30 @@ load_expected() { # wait_for_metadata — repeatedly call the sourcing script's resolve_pending (one # resolution round over the still-unresolved packages, returning 0 once none -# remain) until everything resolves or METADATA_TIMEOUT is reached. +# remain) until everything resolves, METADATA_TIMEOUT is reached, or the +# resolution stalls. The retry window only covers publication propagation: once +# a round reads the published metadata and resolves nothing new while some +# packages already resolved, the remaining ones are not in the publication at +# all (e.g. evicted by the retention policy) and no amount of waiting will +# surface them - report them right away instead of burning the whole window. wait_for_metadata() { local deadline=$(( SECONDS + METADATA_TIMEOUT )) + local resolved previous_resolved=-1 until resolve_pending; do + resolved=0 + for i in "${!E_FILENAME[@]}"; do + [[ "${META_IDX[$i]}" == "true" ]] && resolved=$((resolved + 1)) + done + if ((resolved > 0 && resolved == previous_resolved)); then + echo "[WARN] ${resolved}/${#E_FILENAME[@]} package(s) resolvable in the published metadata and no progress in the last round; the remaining ones are not part of the publication, giving up early" + break + fi + previous_resolved=$resolved if [[ "$SECONDS" -ge "$deadline" ]]; then - echo "[WARN] Metadata resolution timed out after ${METADATA_TIMEOUT}s" + echo "[WARN] Metadata resolution timed out after ${METADATA_TIMEOUT}s (${resolved}/${#E_FILENAME[@]} resolvable)" break fi - echo "[INFO] Waiting ${METADATA_INTERVAL}s for metadata to publish..." + echo "[INFO] ${resolved}/${#E_FILENAME[@]} package(s) resolvable, waiting ${METADATA_INTERVAL}s for the metadata to propagate..." sleep "$METADATA_INTERVAL" done } From 252e87b30e48ba87249f9d211fdc44ea21257dcb Mon Sep 17 00:00:00 2001 From: Kevin Duret Date: Thu, 16 Jul 2026 18:57:40 +0200 Subject: [PATCH 32/49] TEMP(test-pulp-unstable): purge the pull-request test packages from the shared deb repository --- .../package-delivery-pulp/deliver-deb.sh | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/.github/actions/package-delivery-pulp/deliver-deb.sh b/.github/actions/package-delivery-pulp/deliver-deb.sh index 1dcc5a421b..d21047d65f 100755 --- a/.github/actions/package-delivery-pulp/deliver-deb.sh +++ b/.github/actions/package-delivery-pulp/deliver-deb.sh @@ -82,6 +82,38 @@ fi REPOSITORY_HREF=$(pulp deb repository show --name "$REPOSITORY_NAME" | jq -r '.pulp_href') +# TEMP(test-pulp-unstable): purge the packages delivered by the pull-request +# test rounds (label git_ref=refs/pull/...) - they pile up in the shared +# repository (14k+ module packages, no deb retention) and slow every +# finalization and publication down. The develop deliveries +# (git_ref=refs/heads/develop) are the real pipeline and are preserved. Paged +# removals: re-querying page 1 after each remove avoids deep offsets. +for ((round = 0; round < 30; round++)); do + refresh_pulp_token + LATEST_VERSION=$(pulp deb repository show --name "$REPOSITORY_NAME" | jq -r '.latest_version_href') + PURGE_PAGE=$( + curl -fsSL -H "Authorization: Github $PULP_TOKEN" -G \ + --data-urlencode "repository_version=$LATEST_VERSION" \ + --data-urlencode "pulp_label_select=git_ref~pull" \ + --data-urlencode "limit=1000" \ + "$PULP_URL/api/v3/content/deb/packages/" + ) + PURGE_COUNT=$(echo "$PURGE_PAGE" | jq -r '.count') + if ((round == 0)); then + echo "[INFO] Purging $PURGE_COUNT pull-request test package(s) from $REPOSITORY_NAME" + fi + ((PURGE_COUNT == 0)) && break + PURGE_BODY=$(echo "$PURGE_PAGE" | jq -c '{remove_content_units: [.results[].pulp_href]}') + PURGE_TASK=$( + curl -fsSL -H "Authorization: Github $PULP_TOKEN" \ + -X POST -H "Content-Type: application/json" \ + -d "$PURGE_BODY" \ + "$PULP_URL${REPOSITORY_HREF}modify/" | jq -r '.task' + ) + wait_task "$PURGE_TASK" + echo "[INFO] Purge round $((round + 1)): removed $(echo "$PURGE_PAGE" | jq -r '.results | length') package(s), $PURGE_COUNT were matching" +done + PULP_LABELS=$(jq -cn \ --arg mod "$MODULE_NAME" \ --arg git_commit "${GITHUB_SHA:-}" \ From 6536f40504da1a02d70150dc61334f9f3051347a Mon Sep 17 00:00:00 2001 From: Kevin Duret Date: Fri, 17 Jul 2026 08:52:23 +0200 Subject: [PATCH 33/49] chore(ci): retrigger (resume the deb purge, the dev stack was shut down mid-run) From 898f48fa26c0d80fec1867a4ab554ac07d760810 Mon Sep 17 00:00:00 2001 From: Kevin Duret Date: Fri, 17 Jul 2026 10:10:13 +0200 Subject: [PATCH 34/49] chore(ci): retrigger (validate the delivery with the package retention disabled) From afc814b6a46c7848bcc6012359a5663b9922e86f Mon Sep 17 00:00:00 2001 From: Kevin Duret Date: Fri, 17 Jul 2026 10:37:14 +0200 Subject: [PATCH 35/49] Revert "TEMP(test-pulp-unstable): purge the pull-request test packages from the shared deb repository" This reverts commit 252e87b30e48ba87249f9d211fdc44ea21257dcb. --- .../package-delivery-pulp/deliver-deb.sh | 32 ------------------- 1 file changed, 32 deletions(-) diff --git a/.github/actions/package-delivery-pulp/deliver-deb.sh b/.github/actions/package-delivery-pulp/deliver-deb.sh index d21047d65f..1dcc5a421b 100755 --- a/.github/actions/package-delivery-pulp/deliver-deb.sh +++ b/.github/actions/package-delivery-pulp/deliver-deb.sh @@ -82,38 +82,6 @@ fi REPOSITORY_HREF=$(pulp deb repository show --name "$REPOSITORY_NAME" | jq -r '.pulp_href') -# TEMP(test-pulp-unstable): purge the packages delivered by the pull-request -# test rounds (label git_ref=refs/pull/...) - they pile up in the shared -# repository (14k+ module packages, no deb retention) and slow every -# finalization and publication down. The develop deliveries -# (git_ref=refs/heads/develop) are the real pipeline and are preserved. Paged -# removals: re-querying page 1 after each remove avoids deep offsets. -for ((round = 0; round < 30; round++)); do - refresh_pulp_token - LATEST_VERSION=$(pulp deb repository show --name "$REPOSITORY_NAME" | jq -r '.latest_version_href') - PURGE_PAGE=$( - curl -fsSL -H "Authorization: Github $PULP_TOKEN" -G \ - --data-urlencode "repository_version=$LATEST_VERSION" \ - --data-urlencode "pulp_label_select=git_ref~pull" \ - --data-urlencode "limit=1000" \ - "$PULP_URL/api/v3/content/deb/packages/" - ) - PURGE_COUNT=$(echo "$PURGE_PAGE" | jq -r '.count') - if ((round == 0)); then - echo "[INFO] Purging $PURGE_COUNT pull-request test package(s) from $REPOSITORY_NAME" - fi - ((PURGE_COUNT == 0)) && break - PURGE_BODY=$(echo "$PURGE_PAGE" | jq -c '{remove_content_units: [.results[].pulp_href]}') - PURGE_TASK=$( - curl -fsSL -H "Authorization: Github $PULP_TOKEN" \ - -X POST -H "Content-Type: application/json" \ - -d "$PURGE_BODY" \ - "$PULP_URL${REPOSITORY_HREF}modify/" | jq -r '.task' - ) - wait_task "$PURGE_TASK" - echo "[INFO] Purge round $((round + 1)): removed $(echo "$PURGE_PAGE" | jq -r '.results | length') package(s), $PURGE_COUNT were matching" -done - PULP_LABELS=$(jq -cn \ --arg mod "$MODULE_NAME" \ --arg git_commit "${GITHUB_SHA:-}" \ From 3054366af48268cfb31a31849cfae68bf36b5163 Mon Sep 17 00:00:00 2001 From: Kevin Duret Date: Fri, 17 Jul 2026 11:11:51 +0200 Subject: [PATCH 36/49] chore(ci): retrigger (the one-time version trim of apt-plugins completed) From 140c05a7dcc0e8289ec333c5ae2bf8e2dc11abaa Mon Sep 17 00:00:00 2001 From: Kevin Duret Date: Fri, 17 Jul 2026 11:18:14 +0200 Subject: [PATCH 37/49] feat(packaging): compress the rpm payload with zstd Faster to decompress at install time than xz, supported by rpm >= 4.14 (el8 onwards). --- .github/packaging/centreon-plugin.yaml.template | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/packaging/centreon-plugin.yaml.template b/.github/packaging/centreon-plugin.yaml.template index bdbd8a03ff..5485d7e49d 100644 --- a/.github/packaging/centreon-plugin.yaml.template +++ b/.github/packaging/centreon-plugin.yaml.template @@ -102,7 +102,7 @@ overrides: [@DEB_PROVIDES@] rpm: - compression: xz + compression: zstd signature: key_file: ${RPM_SIGNING_KEY_FILE} key_id: ${RPM_SIGNING_KEY_ID} From 304a08a9f796118cdeb748316adf91f6a8efe5a2 Mon Sep 17 00:00:00 2001 From: Kevin Duret Date: Fri, 17 Jul 2026 11:47:36 +0200 Subject: [PATCH 38/49] fix(ci): retry the fetchability probe of the delivery verification One flaky HEAD out of hundreds (content-app/S3 hiccup) must not fail the whole verification. --- .github/scripts/pulp/check-common.sh | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/.github/scripts/pulp/check-common.sh b/.github/scripts/pulp/check-common.sh index ce44d8070c..7ce28d476d 100644 --- a/.github/scripts/pulp/check-common.sh +++ b/.github/scripts/pulp/check-common.sh @@ -83,13 +83,18 @@ record_row() { # record every package's result row. Uses the E_FILENAME/E_ARCH/E_BASEPATH, # PRESENT_IDX, META_IDX and RESOLVED_IDX arrays filled by the sourcing script. check_fetchable_and_record() { - local i url code fetchable + local i url code fetchable attempt for i in "${!E_FILENAME[@]}"; do fetchable=false if [[ "${META_IDX[$i]}" == "true" ]]; then url="${PULP_CONTENT_URL}/${E_BASEPATH[$i]}/${RESOLVED_IDX[$i]}" - code=$(curl -fsSL -o /dev/null -w '%{http_code}' -I "$url" 2>/dev/null || echo 000) - [[ "$code" == "200" ]] && fetchable=true + # retry: one flaky HEAD out of hundreds (content-app/S3 hiccup) must not + # fail the whole verification + for attempt in 1 2 3; do + code=$(curl -fsSL -o /dev/null -w '%{http_code}' -I "$url" 2>/dev/null || echo 000) + [[ "$code" == "200" ]] && { fetchable=true; break; } + sleep 2 + done fi record_row "${E_FILENAME[$i]}" "${E_ARCH[$i]}" "${PRESENT_IDX[$i]}" "${META_IDX[$i]}" "$fetchable" done From 39b1123f3d6e675c8bc5f6aa2d58215044557630 Mon Sep 17 00:00:00 2001 From: Kevin Duret Date: Fri, 17 Jul 2026 11:58:57 +0200 Subject: [PATCH 39/49] chore(ci): retrigger (apt-plugins version trim completed) From f67ff14127998989b78cd7397475d75558c22fb8 Mon Sep 17 00:00:00 2001 From: Kevin Duret Date: Fri, 17 Jul 2026 12:34:21 +0200 Subject: [PATCH 40/49] TEMP(test-pulp-unstable): full-matrix testing delivery + forced promote --- .github/workflows/plugins.yml | 56 +++++++++++++++++++++++++++++------ 1 file changed, 47 insertions(+), 9 deletions(-) diff --git a/.github/workflows/plugins.yml b/.github/workflows/plugins.yml index b11409c90b..8a11da9b94 100644 --- a/.github/workflows/plugins.yml +++ b/.github/workflows/plugins.yml @@ -290,14 +290,30 @@ jobs: max-parallel: 5 matrix: include: - # TEMP(test-pulp-unstable): restrict the load test to one rpm and one - # deb distrib (el10 + trixie) + - package_extension: rpm + image: packaging:alma8 + distrib: el8 + - package_extension: rpm + image: packaging:alma9 + distrib: el9 - package_extension: rpm image: packaging:alma10 distrib: el10 + - package_extension: deb + image: packaging:bullseye + distrib: bullseye + - package_extension: deb + image: packaging:bookworm + distrib: bookworm - package_extension: deb image: packaging:trixie distrib: trixie + - package_extension: deb + image: packaging:jammy + distrib: jammy + - package_extension: deb + image: packaging:noble + distrib: noble container: image: ghcr.io/${{ github.repository }}/${{ matrix.image }} @@ -414,12 +430,22 @@ jobs: fail-fast: false matrix: include: - # TEMP(test-pulp-unstable): restrict the load test to one rpm and - # one deb distrib (el10 + trixie) + - distrib: el8 + package_extension: rpm + - distrib: el9 + package_extension: rpm - distrib: el10 package_extension: rpm + - distrib: bullseye + package_extension: deb + - distrib: bookworm + package_extension: deb - distrib: trixie package_extension: deb + - distrib: jammy + package_extension: deb + - distrib: noble + package_extension: deb name: deliver ${{ matrix.distrib }} steps: @@ -456,12 +482,22 @@ jobs: fail-fast: false matrix: include: - # TEMP(test-pulp-unstable): restrict the load test to one rpm and - # one deb distrib (el10 + trixie) + - distrib: el8 + package_extension: rpm + - distrib: el9 + package_extension: rpm - distrib: el10 package_extension: rpm + - distrib: bullseye + package_extension: deb + - distrib: bookworm + package_extension: deb - distrib: trixie package_extension: deb + - distrib: jammy + package_extension: deb + - distrib: noble + package_extension: deb name: deliver ${{ matrix.distrib }} on pulp steps: @@ -525,7 +561,7 @@ jobs: distrib: ${{ matrix.distrib }} version: "" cache_key: ${{ github.sha }}-${{ github.run_id }}-${{ matrix.package_extension }}-${{ matrix.distrib }} - stability: unstable # TEMP(test-pulp-unstable): force unstable Pulp delivery + stability: testing # TEMP(test-pulp-unstable): force testing so the promote step has a source release_type: ${{ needs.get-environment.outputs.release_type }} is_cloud: "false" pulp_url: ${{ vars.PULP_API_URL }} @@ -533,7 +569,9 @@ jobs: audience: ${{ vars.PULP_OIDC_AUDIENCE }} - name: Promote packages on Pulp - if: ${{ needs.get-environment.outputs.stability == 'stable' && steps.get_release_type.outputs.release_type != '' }} + # TEMP(test-pulp-unstable): promote right after the testing delivery + # with a forced release type, to validate the promote path at scale + if: ${{ steps.deliver_pulp.outcome == 'success' }} id: promote_pulp continue-on-error: true uses: ./.github/actions/promote-to-stable-pulp @@ -543,7 +581,7 @@ jobs: distrib: ${{ matrix.distrib }} major_version: "" stability: stable - release_type: ${{ steps.get_release_type.outputs.release_type }} + release_type: release # TEMP(test-pulp-unstable): forced is_cloud: "false" pulp_url: ${{ vars.PULP_API_URL }} pulp_content_url: ${{ vars.PULP_CONTENT_URL }} From e96ce87c67ea89de6db035000135ee326befdc1c Mon Sep 17 00:00:00 2001 From: Kevin Duret Date: Fri, 17 Jul 2026 13:23:17 +0200 Subject: [PATCH 41/49] fix(ci): paginate the promote package listing The testing repositories keep every delivered version (and the shared deb repository holds every suite), so the module listing exceeds a single page; the fail-loud truncation guard becomes an actual pagination. --- .../promote-to-stable-pulp/promote-deb.sh | 38 +++++++++---------- .../promote-to-stable-pulp/promote-rpm.sh | 33 ++++++++-------- 2 files changed, 36 insertions(+), 35 deletions(-) diff --git a/.github/actions/promote-to-stable-pulp/promote-deb.sh b/.github/actions/promote-to-stable-pulp/promote-deb.sh index 3ad3eb1860..5df1885a82 100755 --- a/.github/actions/promote-to-stable-pulp/promote-deb.sh +++ b/.github/actions/promote-to-stable-pulp/promote-deb.sh @@ -20,28 +20,28 @@ VERSION_HREF=$(pulp deb repository show --name "$REPOSITORY_NAME" | jq -r '.late # packages of the module are identified by the label set at delivery time, # the testing pool path scopes the stability and the package distrib name -# scopes the distribution as the apt repository holds all the suites -RESPONSE=$( - curl -fsSL -H "Authorization: Github $PULP_TOKEN" -G \ - --data-urlencode "repository_version=$VERSION_HREF" \ - --data-urlencode "pulp_label_select=module=$MODULE_NAME" \ - --data-urlencode "limit=1000" \ - "$PULP_URL/api/v3/content/deb/packages/" -) - -# fail on a truncated page (checked before the pool-path filtering): silently -# promoting a subset of the module's packages would publish an incomplete -# stable suite -if [[ $(echo "$RESPONSE" | jq '.count > (.results | length)') == "true" ]]; then - echo "::error::Package query on $REPOSITORY_NAME is truncated ($(echo "$RESPONSE" | jq -r '.results | length')/$(echo "$RESPONSE" | jq -r '.count') results); pagination is required" - exit 1 -fi +# scopes the distribution as the apt repository holds all the suites. +# Paginated: the shared repository keeps every delivered version across all +# suites, so the module listing exceeds a single page. +RESULTS_FILE=$(mktemp) +url="$PULP_URL/api/v3/content/deb/packages/?$( + printf 'repository_version=%s&pulp_label_select=%s&limit=1000' \ + "$(jq -rn --arg v "$VERSION_HREF" '$v | @uri')" \ + "$(jq -rn --arg v "module=$MODULE_NAME" '$v | @uri')" +)" +while [[ -n "$url" ]]; do + refresh_pulp_token + page=$(curl -fsSL -H "Authorization: Github $PULP_TOKEN" "$url") + echo "$page" | jq -c '.results[]' >> "$RESULTS_FILE" + url=$(echo "$page" | jq -r '.next // empty') +done PACKAGES=$( - echo "$RESPONSE" | \ - jq --arg testing_path "$TESTING_POOL_PATH/" --arg distrib_name "$PACKAGE_DISTRIB_NAME" \ - '[.results[] | select((.relative_path | startswith($testing_path)) and (.relative_path | contains($distrib_name)))]' + jq -s --arg testing_path "$TESTING_POOL_PATH/" --arg distrib_name "$PACKAGE_DISTRIB_NAME" \ + '[.[] | select((.relative_path | startswith($testing_path)) and (.relative_path | contains($distrib_name)))]' \ + "$RESULTS_FILE" ) +rm -f "$RESULTS_FILE" PACKAGES_COUNT=$(echo "$PACKAGES" | jq 'length') if [[ "$PACKAGES_COUNT" -eq 0 ]]; then diff --git a/.github/actions/promote-to-stable-pulp/promote-rpm.sh b/.github/actions/promote-to-stable-pulp/promote-rpm.sh index 9426e87e65..06e64b5c63 100755 --- a/.github/actions/promote-to-stable-pulp/promote-rpm.sh +++ b/.github/actions/promote-to-stable-pulp/promote-rpm.sh @@ -28,22 +28,23 @@ for ARCH in noarch x86_64; do # packages of the module are identified by the label set at delivery time; # keep both the href list (for the content modify call) and the package # identity (name/version/release/arch/filename) to feed the promotion manifest - RESPONSE=$( - curl -fsSL -H "Authorization: Github $PULP_TOKEN" -G \ - --data-urlencode "repository_version=$VERSION_HREF" \ - --data-urlencode "pulp_label_select=module=$MODULE_NAME" \ - --data-urlencode "limit=1000" \ - "$PULP_URL/api/v3/content/rpm/packages/" - ) - - # fail on a truncated page: silently promoting a subset of the module's - # packages would publish an incomplete stable repository - if [[ $(echo "$RESPONSE" | jq '.count > (.results | length)') == "true" ]]; then - echo "::error::Package query on $TESTING_REPOSITORY_NAME is truncated ($(echo "$RESPONSE" | jq -r '.results | length')/$(echo "$RESPONSE" | jq -r '.count') results); pagination is required" - exit 1 - fi - - RESULTS=$(echo "$RESPONSE" | jq '[.results[] | {pulp_href, name, version, release, arch, location_href, sha256}]') + # paginate: the testing repository keeps every delivered version, so the + # module listing can exceed a single page + RESULTS_FILE=$(mktemp) + url="$PULP_URL/api/v3/content/rpm/packages/?$( + printf 'repository_version=%s&pulp_label_select=%s&limit=1000' \ + "$(jq -rn --arg v "$VERSION_HREF" '$v | @uri')" \ + "$(jq -rn --arg v "module=$MODULE_NAME" '$v | @uri')" + )" + while [[ -n "$url" ]]; do + refresh_pulp_token + page=$(curl -fsSL -H "Authorization: Github $PULP_TOKEN" "$url") + echo "$page" | jq -c '.results[]' >> "$RESULTS_FILE" + url=$(echo "$page" | jq -r '.next // empty') + done + + RESULTS=$(jq -s '[.[] | {pulp_href, name, version, release, arch, location_href, sha256}]' "$RESULTS_FILE") + rm -f "$RESULTS_FILE" CONTENT=$(echo "$RESULTS" | jq '[.[].pulp_href]') ARCH_PACKAGES_COUNT=$(echo "$CONTENT" | jq 'length') From ad6625ab9249efcb63c49eddf14165e0c47d9bbf Mon Sep 17 00:00:00 2001 From: Kevin Duret Date: Fri, 17 Jul 2026 14:11:28 +0200 Subject: [PATCH 42/49] TEMP(test-pulp-unstable): batched deb promote (stable associations + single modify) The packages already exist as content units in the shared repository, so promoting is only a matter of stable suite associations: one legacy re-upload per architecture (creates the stable ReleaseComponent and ReleaseArchitecture), synchronous PackageReleaseComponent creates for the rest, one modify, one publication - instead of one repository-serialized upload task and one file download per package. --- .../promote-to-stable-pulp/promote-deb.sh | 133 ++++++++++++++++-- 1 file changed, 119 insertions(+), 14 deletions(-) diff --git a/.github/actions/promote-to-stable-pulp/promote-deb.sh b/.github/actions/promote-to-stable-pulp/promote-deb.sh index 5df1885a82..3b265dfba4 100755 --- a/.github/actions/promote-to-stable-pulp/promote-deb.sh +++ b/.github/actions/promote-to-stable-pulp/promote-deb.sh @@ -63,7 +63,39 @@ mkdir -p promoted-packages # module label, built with jq (safe escaping) — consistent with the delivery scripts PULP_LABELS=$(jq -cn --arg mod "$MODULE_NAME" '{"module": $mod}') +lookup_prcs() { + # emit the release_component hrefs of a package's suite associations + local package_href=$1 + curl -fsSL -H "Authorization: Github $PULP_TOKEN" -G \ + --data-urlencode "package=$package_href" \ + --data-urlencode "limit=100" \ + "$PULP_URL/api/v3/content/deb/package_release_components/" \ + | jq -r '.results[].release_component' +} + +# batched promote, mirroring the batched delivery: the packages already exist +# as content units in the repository (testing suite), so promoting is only a +# matter of stable suite associations. The FIRST package of each architecture +# is promoted through the legacy re-upload path - it get_or_creates the stable +# ReleaseComponent and ReleaseArchitecture, which the ci user cannot create +# nor list directly - then the stable release-component href is deduced from +# its associations, every other association is created synchronously, and one +# modify adds the whole batch to the repository. +declare -A ARCH_SEEN=() +LEGACY_PACKAGES=() +BATCH_PACKAGES=() while read -r PACKAGE; do + ARCH=$(echo "$PACKAGE" | jq -r '.architecture') + if [[ -z "${ARCH_SEEN[$ARCH]+set}" ]]; then + ARCH_SEEN[$ARCH]=1 + LEGACY_PACKAGES+=("$PACKAGE") + else + BATCH_PACKAGES+=("$PACKAGE") + fi +done < <(echo "$PACKAGES" | jq -c '.[]') + +refresh_pulp_token +for PACKAGE in "${LEGACY_PACKAGES[@]}"; do RELATIVE_PATH=$(echo "$PACKAGE" | jq -r '.relative_path') SHA256=$(echo "$PACKAGE" | jq -r '.sha256') ARCH=$(echo "$PACKAGE" | jq -r '.architecture') @@ -86,11 +118,9 @@ while read -r PACKAGE; do curl -fsSL -o "$FILE" "$PULP_CONTENT_URL/$BASE_PATH/$FILENAME" # re-upload the same bytes with the SAME relative_path: pulp reuses the - # existing content unit (a different relative_path would create a second unit - # that evicts the first, as pulp deduplicates by name+version+architecture - # repository wide) and only adds the stable suite association. an upload is - # required because pulp_deb rejects direct PackageReleaseComponent creation. - echo "[INFO] Promoting $FILE_NAME to $STABLE_SUITE/main" + # existing content unit and only adds the stable suite association, + # creating the stable ReleaseComponent/ReleaseArchitecture on the way + echo "[INFO] Promoting $FILE_NAME to $STABLE_SUITE/main [legacy path]" TASK_HREF=$( pulp_upload \ -F "file=@\"$FILE\"" \ @@ -102,16 +132,91 @@ while read -r PACKAGE; do "$PULP_URL/api/v3/content/deb/packages/" ) wait_task "$TASK_HREF" +done - # record the promoted package (with its stable suite coordinates) so the - # verification step verifies exactly this set against the stable suite - NAME=$(echo "$PACKAGE" | jq -r '.package') - VERSION=$(echo "$PACKAGE" | jq -r '.version') - manifest_add "$(jq -cn \ - --arg filename "$FILE_NAME" --arg name "$NAME" --arg version "$VERSION" \ - --arg arch "$ARCH" --arg sha256 "$SHA256" --arg repository "$REPOSITORY_NAME" \ - --arg base_path "$BASE_PATH" --arg suite "$STABLE_SUITE" --arg relative_path "$RELATIVE_PATH" \ - '{filename:$filename,name:$name,version:$version,arch:$arch,sha256:$sha256,repository:$repository,base_path:$base_path,suite:$suite,relative_path:$relative_path}')" +if ((${#BATCH_PACKAGES[@]} > 0)); then + # deduce the stable release-component href: the batch packages only carry + # their testing association, the just-promoted legacy package carries the + # stable one on top of it + refresh_pulp_token + FIRST_BATCH_HREF=$(echo "${BATCH_PACKAGES[0]}" | jq -r '.pulp_href') + LEGACY_HREF=$(echo "${LEGACY_PACKAGES[0]}" | jq -r '.pulp_href') + TESTING_RC=$(lookup_prcs "$FIRST_BATCH_HREF" | head -1) + STABLE_RC=$(lookup_prcs "$LEGACY_HREF" | grep -Fxv "$TESTING_RC" | head -1) + if [[ -z "$STABLE_RC" ]]; then + echo "::error::Cannot deduce the $STABLE_SUITE/main release component from the legacy-promoted package" + exit 1 + fi + echo "[INFO] Release component of $STABLE_SUITE/main: $STABLE_RC" + + # create the stable suite associations: synchronous creates (201 with the + # unit, no task), parallelized; a failed create (existing association on a + # re-run) falls back to a lookup + echo "[INFO] Associating ${#BATCH_PACKAGES[@]} package(s) with $STABLE_SUITE/main" + PRC_DIR=$(mktemp -d) + MAX_PARALLEL=8 + for i in "${!BATCH_PACKAGES[@]}"; do + if ((i % 40 == 0)); then + refresh_pulp_token + fi + ( + package_href=$(echo "${BATCH_PACKAGES[$i]}" | jq -r '.pulp_href') + response=$( + curl -fsSL -H "Authorization: Github $PULP_TOKEN" \ + -X POST -H "Content-Type: application/json" \ + -d "{\"package\": \"$package_href\", \"release_component\": \"$STABLE_RC\"}" \ + "$PULP_URL/api/v3/content/deb/package_release_components/" + ) || response="" + href=$(echo "$response" | jq -r '.pulp_href // empty') + if [[ -z "$href" ]]; then + href=$( + curl -fsSL -H "Authorization: Github $PULP_TOKEN" -G \ + --data-urlencode "package=$package_href" \ + --data-urlencode "release_component=$STABLE_RC" \ + --data-urlencode "limit=1" \ + "$PULP_URL/api/v3/content/deb/package_release_components/" \ + | jq -r '.results[0].pulp_href // empty' + ) + fi + printf '%s\n%s' "$package_href" "$href" > "$PRC_DIR/$i.pair" + ) & + while (($(jobs -rp | wc -l) >= MAX_PARALLEL)); do + wait -n || true + done + done + wait || true + + ADD_UNITS_FILE=$(mktemp) + for i in "${!BATCH_PACKAGES[@]}"; do + if [[ ! -s "$PRC_DIR/$i.pair" ]] || [[ -z "$(sed -n '2p' "$PRC_DIR/$i.pair")" ]]; then + echo "::error::Stable suite association failed for package $(echo "${BATCH_PACKAGES[$i]}" | jq -r '.package') (see the worker error above)" + exit 1 + fi + cat "$PRC_DIR/$i.pair" >> "$ADD_UNITS_FILE" + echo >> "$ADD_UNITS_FILE" + done + rm -rf "$PRC_DIR" + + echo "[INFO] Adding ${#BATCH_PACKAGES[@]} package(s) and their suite associations to $REPOSITORY_NAME in a single task" + refresh_pulp_token + ADD_BODY=$(jq -R . "$ADD_UNITS_FILE" | jq -cs '{add_content_units: [.[] | select(. != "")]}') + rm -f "$ADD_UNITS_FILE" + MODIFY_TASK=$( + curl -fsSL -H "Authorization: Github $PULP_TOKEN" \ + -X POST -H "Content-Type: application/json" \ + -d "$ADD_BODY" \ + "$PULP_URL${REPOSITORY_HREF}modify/" | jq -r '.task' + ) + wait_task "$MODIFY_TASK" +fi + +# record the promoted packages (with their stable suite coordinates) so the +# verification step verifies exactly this set against the stable suite +while read -r PACKAGE; do + manifest_add "$(echo "$PACKAGE" | jq -c \ + --arg repository "$REPOSITORY_NAME" --arg base_path "$BASE_PATH" \ + --arg suite "$STABLE_SUITE" \ + '{filename: (.relative_path | sub(".*/"; "")), name: .package, version, arch: .architecture, sha256, repository: $repository, base_path: $base_path, suite: $suite, relative_path}')" done < <(echo "$PACKAGES" | jq -c '.[]') echo "[INFO] Publishing repository $REPOSITORY_NAME" From 342d621c6d0fa138edb229b96b8dfca4f2dc54bb Mon Sep 17 00:00:00 2001 From: Kevin Duret Date: Fri, 17 Jul 2026 15:05:50 +0200 Subject: [PATCH 43/49] fix(ci): modify bodies through files, bounded retries on transient 5xx - the modify/add_content_units bodies exceed the argv limit past a few thousand hrefs: pass them with curl -d @file - the stable guards and index fetches now retry transient 5xx (404 stays a valid not-in-stable answer), and the upload retry budget grows to 5 attempts: the stack under full-matrix load returns occasional 502s --- .github/actions/package-delivery-pulp/deliver-deb.sh | 8 +++++--- .github/actions/package-delivery-pulp/deliver-rpm.sh | 8 +++++--- .github/actions/promote-to-stable-pulp/promote-deb.sh | 10 ++++++---- .github/actions/promote-to-stable-pulp/promote-rpm.sh | 2 +- .github/scripts/pulp/api.sh | 8 ++++---- 5 files changed, 21 insertions(+), 15 deletions(-) diff --git a/.github/actions/package-delivery-pulp/deliver-deb.sh b/.github/actions/package-delivery-pulp/deliver-deb.sh index 1dcc5a421b..c00a928f33 100755 --- a/.github/actions/package-delivery-pulp/deliver-deb.sh +++ b/.github/actions/package-delivery-pulp/deliver-deb.sh @@ -32,7 +32,7 @@ assert_not_in_stable() { # endpoint error cannot let an already-stable version slip through and evict it # (the rpm guardrail is likewise fail-closed). pkg_file=$(mktemp) - http_code=$(curl -sSL -o "$pkg_file" -w '%{http_code}' \ + http_code=$(curl -sSL --retry 3 --retry-delay 5 -o "$pkg_file" -w '%{http_code}' \ "$PULP_CONTENT_URL/$BASE_PATH/dists/$STABLE_SUITE/main/binary-$arch/Packages" 2>/dev/null || echo 000) case "$http_code" in 404) rm -f "$pkg_file"; return 0 ;; @@ -308,11 +308,13 @@ rm -rf "$UPLOAD_DIR" if ((${#PACKAGE_HREFS[@]} > 0)); then echo "[INFO] Adding ${#PACKAGE_HREFS[@]} package(s) and their suite associations to $REPOSITORY_NAME in a single task" refresh_pulp_token - ADD_BODY=$(printf '%s\n' "${PACKAGE_HREFS[@]}" "${PRC_HREFS[@]}" | jq -R . | jq -cs '{add_content_units: .}') + # body through a file: thousands of hrefs exceed the argv limit + ADD_BODY_FILE=$(mktemp) + printf '%s\n' "${PACKAGE_HREFS[@]}" "${PRC_HREFS[@]}" | jq -R . | jq -cs '{add_content_units: .}' > "$ADD_BODY_FILE" MODIFY_TASK=$( curl -fsSL -H "Authorization: Github $PULP_TOKEN" \ -X POST -H "Content-Type: application/json" \ - -d "$ADD_BODY" \ + -d @"$ADD_BODY_FILE" \ "$PULP_URL${REPOSITORY_HREF}modify/" | jq -r '.task' ) wait_task "$MODIFY_TASK" diff --git a/.github/actions/package-delivery-pulp/deliver-rpm.sh b/.github/actions/package-delivery-pulp/deliver-rpm.sh index 83dbe4b6ba..c756dbc1ed 100755 --- a/.github/actions/package-delivery-pulp/deliver-rpm.sh +++ b/.github/actions/package-delivery-pulp/deliver-rpm.sh @@ -34,7 +34,7 @@ assert_not_in_stable() { repository_version=$(pulp rpm repository show --name "$stable_repository" | jq -r '.latest_version_href') count=$( - curl -fsSL -H "Authorization: Github $PULP_TOKEN" -G \ + curl -fsSL --retry 3 --retry-delay 5 -H "Authorization: Github $PULP_TOKEN" -G \ --data-urlencode "repository_version=$repository_version" \ --data-urlencode "name=$name" \ --data-urlencode "version=$version" \ @@ -230,11 +230,13 @@ for ARCH in noarch x86_64; do # refresh from the parent shell: the resolutions above ran in subshells, so # their refreshes never updated this shell's token (the 401 trap, again) refresh_pulp_token - ADD_BODY=$(printf '%s\n' "${CONTENT_HREFS[@]}" | jq -R . | jq -cs '{add_content_units: .}') + # body through a file: thousands of hrefs exceed the argv limit + ADD_BODY_FILE=$(mktemp) + printf '%s\n' "${CONTENT_HREFS[@]}" | jq -R . | jq -cs '{add_content_units: .}' > "$ADD_BODY_FILE" MODIFY_TASK=$( curl -fsSL -H "Authorization: Github $PULP_TOKEN" \ -X POST -H "Content-Type: application/json" \ - -d "$ADD_BODY" \ + -d @"$ADD_BODY_FILE" \ "$PULP_URL${REPOSITORY_HREF}modify/" | jq -r '.task' ) wait_task "$MODIFY_TASK" diff --git a/.github/actions/promote-to-stable-pulp/promote-deb.sh b/.github/actions/promote-to-stable-pulp/promote-deb.sh index 3b265dfba4..665ef6383f 100755 --- a/.github/actions/promote-to-stable-pulp/promote-deb.sh +++ b/.github/actions/promote-to-stable-pulp/promote-deb.sh @@ -106,7 +106,7 @@ for PACKAGE in "${LEGACY_PACKAGES[@]}"; do # layout, not their upload relative_path, so resolve the published location # from the testing suite Packages indexed by checksum to download the file FILENAME=$( - curl -fsSL "$PULP_CONTENT_URL/$BASE_PATH/dists/$TESTING_SUITE/main/binary-$ARCH/Packages" | + curl -fsSL --retry 3 --retry-delay 5 "$PULP_CONTENT_URL/$BASE_PATH/dists/$TESTING_SUITE/main/binary-$ARCH/Packages" | awk -v sha="$SHA256" 'BEGIN { RS = ""; FS = "\n" } index($0, "SHA256: " sha) { for (i = 1; i <= NF; i++) if ($i ~ /^Filename: /) { sub(/^Filename: /, "", $i); print $i } }' ) if [[ -z "$FILENAME" ]]; then @@ -115,7 +115,7 @@ for PACKAGE in "${LEGACY_PACKAGES[@]}"; do fi echo "[INFO] Downloading $PULP_CONTENT_URL/$BASE_PATH/$FILENAME" - curl -fsSL -o "$FILE" "$PULP_CONTENT_URL/$BASE_PATH/$FILENAME" + curl -fsSL --retry 3 --retry-delay 5 -o "$FILE" "$PULP_CONTENT_URL/$BASE_PATH/$FILENAME" # re-upload the same bytes with the SAME relative_path: pulp reuses the # existing content unit and only adds the stable suite association, @@ -199,12 +199,14 @@ if ((${#BATCH_PACKAGES[@]} > 0)); then echo "[INFO] Adding ${#BATCH_PACKAGES[@]} package(s) and their suite associations to $REPOSITORY_NAME in a single task" refresh_pulp_token - ADD_BODY=$(jq -R . "$ADD_UNITS_FILE" | jq -cs '{add_content_units: [.[] | select(. != "")]}') + # body through a file: thousands of hrefs exceed the argv limit + ADD_BODY_FILE=$(mktemp) + jq -R . "$ADD_UNITS_FILE" | jq -cs '{add_content_units: [.[] | select(. != "")]}' > "$ADD_BODY_FILE" rm -f "$ADD_UNITS_FILE" MODIFY_TASK=$( curl -fsSL -H "Authorization: Github $PULP_TOKEN" \ -X POST -H "Content-Type: application/json" \ - -d "$ADD_BODY" \ + -d @"$ADD_BODY_FILE" \ "$PULP_URL${REPOSITORY_HREF}modify/" | jq -r '.task' ) wait_task "$MODIFY_TASK" diff --git a/.github/actions/promote-to-stable-pulp/promote-rpm.sh b/.github/actions/promote-to-stable-pulp/promote-rpm.sh index 06e64b5c63..7ff64dac3d 100755 --- a/.github/actions/promote-to-stable-pulp/promote-rpm.sh +++ b/.github/actions/promote-to-stable-pulp/promote-rpm.sh @@ -93,7 +93,7 @@ for ARCH in noarch x86_64; do TASK_HREF=$( curl -fsSL -H "Authorization: Github $PULP_TOKEN" \ -X POST -H "Content-Type: application/json" \ - -d "{\"add_content_units\": $CONTENT}" \ + -d @<(echo "$CONTENT" | jq -c '{add_content_units: .}') \ "$PULP_URL${STABLE_REPOSITORY_HREF}modify/" | jq -r '.task' ) wait_task "$TASK_HREF" diff --git a/.github/scripts/pulp/api.sh b/.github/scripts/pulp/api.sh index 48ba2b0be6..3decf472a4 100644 --- a/.github/scripts/pulp/api.sh +++ b/.github/scripts/pulp/api.sh @@ -121,7 +121,7 @@ wait_tasks() { # (concurrent deliveries can race on artifact creation), echoes the task href pulp_upload() { local attempt response http_code body - for attempt in 1 2 3; do + for attempt in 1 2 3 4 5; do refresh_pulp_token response=$(curl -sS -H "Authorization: Github $PULP_TOKEN" -w $'\n%{http_code}' "$@" 2>/dev/null) || response="" http_code=${response##*$'\n'} @@ -130,10 +130,10 @@ pulp_upload() { echo "$body" | jq -r '.task' return 0 fi - echo "[WARN] upload attempt $attempt/3 failed (HTTP ${http_code:-network-error}), retrying..." >&2 - sleep $((attempt * 3)) + echo "[WARN] upload attempt $attempt/5 failed (HTTP ${http_code:-network-error}), retrying..." >&2 + sleep $((attempt * 5)) done - echo "::error::Upload failed after 3 attempts (HTTP ${http_code:-network-error})" >&2 + echo "::error::Upload failed after 5 attempts (HTTP ${http_code:-network-error})" >&2 return 1 } From e80bf1459102e94be0196eecc6a7569275e689d6 Mon Sep 17 00:00:00 2001 From: Kevin Duret Date: Fri, 17 Jul 2026 15:10:08 +0200 Subject: [PATCH 44/49] fix(ci): worker-entry token refresh, promote only the latest build - the parallel workers now refresh the inherited OIDC token on entry: under server slowdowns the parent's every-40-spawns cadence can span the token validity, and the first guard call of a worker then dies on 401 - the promote sweeps only the LATEST version of each package: the testing repositories keep every delivered build now, and promoting the whole accumulated history to stable is never intended --- .github/actions/package-delivery-pulp/deliver-deb.sh | 4 ++++ .github/actions/package-delivery-pulp/deliver-rpm.sh | 3 +++ .github/actions/promote-to-stable-pulp/promote-deb.sh | 8 +++++++- .github/actions/promote-to-stable-pulp/promote-rpm.sh | 5 ++++- 4 files changed, 18 insertions(+), 2 deletions(-) diff --git a/.github/actions/package-delivery-pulp/deliver-deb.sh b/.github/actions/package-delivery-pulp/deliver-deb.sh index c00a928f33..b45e71c5df 100755 --- a/.github/actions/package-delivery-pulp/deliver-deb.sh +++ b/.github/actions/package-delivery-pulp/deliver-deb.sh @@ -213,6 +213,9 @@ for i in "${!ORPHAN_FILES[@]}"; do refresh_pulp_token fi ( + # subshell-local refresh: under server slowdowns the inherited token can + # outlive its validity between two parent refreshes + refresh_pulp_token assert_not_in_stable "$FILE" echo "[INFO] Uploading $FILE to $POOL_PATH/ ($SUITE/main, module $MODULE_NAME)" pulp_upload \ @@ -270,6 +273,7 @@ for i in "${!PACKAGE_HREFS[@]}"; do refresh_pulp_token fi ( + refresh_pulp_token response=$( curl -fsSL -H "Authorization: Github $PULP_TOKEN" \ -X POST -H "Content-Type: application/json" \ diff --git a/.github/actions/package-delivery-pulp/deliver-rpm.sh b/.github/actions/package-delivery-pulp/deliver-rpm.sh index c756dbc1ed..0528ff68b1 100755 --- a/.github/actions/package-delivery-pulp/deliver-rpm.sh +++ b/.github/actions/package-delivery-pulp/deliver-rpm.sh @@ -159,6 +159,9 @@ for ARCH in noarch x86_64; do refresh_pulp_token fi ( + # subshell-local refresh: under server slowdowns the inherited token can + # outlive its validity between two parent refreshes + refresh_pulp_token assert_not_in_stable "$FILE" "$ARCH" echo "[INFO] Uploading $(basename "$FILE") (module $MODULE_NAME)" pulp_upload \ diff --git a/.github/actions/promote-to-stable-pulp/promote-deb.sh b/.github/actions/promote-to-stable-pulp/promote-deb.sh index 665ef6383f..56b8152875 100755 --- a/.github/actions/promote-to-stable-pulp/promote-deb.sh +++ b/.github/actions/promote-to-stable-pulp/promote-deb.sh @@ -36,9 +36,14 @@ while [[ -n "$url" ]]; do url=$(echo "$page" | jq -r '.next // empty') done +# only the LATEST version of each package is promoted: the testing suite +# accumulates every delivered build (deb has no retention mechanism), and +# our version schemes embed a monotonic date/timestamp so the plain string +# max is the newest build PACKAGES=$( jq -s --arg testing_path "$TESTING_POOL_PATH/" --arg distrib_name "$PACKAGE_DISTRIB_NAME" \ - '[.[] | select((.relative_path | startswith($testing_path)) and (.relative_path | contains($distrib_name)))]' \ + '[.[] | select((.relative_path | startswith($testing_path)) and (.relative_path | contains($distrib_name)))] + | group_by(.package, .architecture) | map(max_by(.version))' \ "$RESULTS_FILE" ) rm -f "$RESULTS_FILE" @@ -160,6 +165,7 @@ if ((${#BATCH_PACKAGES[@]} > 0)); then refresh_pulp_token fi ( + refresh_pulp_token package_href=$(echo "${BATCH_PACKAGES[$i]}" | jq -r '.pulp_href') response=$( curl -fsSL -H "Authorization: Github $PULP_TOKEN" \ diff --git a/.github/actions/promote-to-stable-pulp/promote-rpm.sh b/.github/actions/promote-to-stable-pulp/promote-rpm.sh index 7ff64dac3d..222a04a660 100755 --- a/.github/actions/promote-to-stable-pulp/promote-rpm.sh +++ b/.github/actions/promote-to-stable-pulp/promote-rpm.sh @@ -43,7 +43,10 @@ for ARCH in noarch x86_64; do url=$(echo "$page" | jq -r '.next // empty') done - RESULTS=$(jq -s '[.[] | {pulp_href, name, version, release, arch, location_href, sha256}]' "$RESULTS_FILE") + # only the LATEST build of each package is promoted: the version schemes + # embed a monotonic date/timestamp so the plain string max is the newest + RESULTS=$(jq -s '[.[] | {pulp_href, name, version, release, arch, location_href, sha256}] + | group_by(.name, .arch) | map(max_by([.version, .release]))' "$RESULTS_FILE") rm -f "$RESULTS_FILE" CONTENT=$(echo "$RESULTS" | jq '[.[].pulp_href]') ARCH_PACKAGES_COUNT=$(echo "$CONTENT" | jq 'length') From d56a69c980de8f8c222b4b659c1d560e4cb43ae1 Mon Sep 17 00:00:00 2001 From: Kevin Duret Date: Fri, 17 Jul 2026 16:27:02 +0200 Subject: [PATCH 45/49] fix(ci): retry the suite-association creates and the publication start Transient ingress 504s under full-matrix load; a replayed association that already applied answers 400 and falls back to the existing lookup. --- .../actions/package-delivery-pulp/deliver-deb.sh | 2 +- .../actions/promote-to-stable-pulp/promote-deb.sh | 2 +- .github/scripts/pulp/api.sh | 15 ++++++++++----- 3 files changed, 12 insertions(+), 7 deletions(-) diff --git a/.github/actions/package-delivery-pulp/deliver-deb.sh b/.github/actions/package-delivery-pulp/deliver-deb.sh index b45e71c5df..a3090a453a 100755 --- a/.github/actions/package-delivery-pulp/deliver-deb.sh +++ b/.github/actions/package-delivery-pulp/deliver-deb.sh @@ -275,7 +275,7 @@ for i in "${!PACKAGE_HREFS[@]}"; do ( refresh_pulp_token response=$( - curl -fsSL -H "Authorization: Github $PULP_TOKEN" \ + curl -fsSL --retry 3 --retry-delay 5 -H "Authorization: Github $PULP_TOKEN" \ -X POST -H "Content-Type: application/json" \ -d "{\"package\": \"${PACKAGE_HREFS[$i]}\", \"release_component\": \"$RELEASE_COMPONENT_HREF\"}" \ "$PULP_URL/api/v3/content/deb/package_release_components/" diff --git a/.github/actions/promote-to-stable-pulp/promote-deb.sh b/.github/actions/promote-to-stable-pulp/promote-deb.sh index 56b8152875..da88f02c28 100755 --- a/.github/actions/promote-to-stable-pulp/promote-deb.sh +++ b/.github/actions/promote-to-stable-pulp/promote-deb.sh @@ -168,7 +168,7 @@ if ((${#BATCH_PACKAGES[@]} > 0)); then refresh_pulp_token package_href=$(echo "${BATCH_PACKAGES[$i]}" | jq -r '.pulp_href') response=$( - curl -fsSL -H "Authorization: Github $PULP_TOKEN" \ + curl -fsSL --retry 3 --retry-delay 5 -H "Authorization: Github $PULP_TOKEN" \ -X POST -H "Content-Type: application/json" \ -d "{\"package\": \"$package_href\", \"release_component\": \"$STABLE_RC\"}" \ "$PULP_URL/api/v3/content/deb/package_release_components/" diff --git a/.github/scripts/pulp/api.sh b/.github/scripts/pulp/api.sh index 3decf472a4..349d4c8bb1 100644 --- a/.github/scripts/pulp/api.sh +++ b/.github/scripts/pulp/api.sh @@ -144,13 +144,18 @@ pulp_upload() { create_publication() { local plugin=$1 repository=$2 shift 2 - local out task - refresh_pulp_token - out=$(pulp --background "$plugin" publication create --repository "$repository" "$@" 2>&1) || { - echo "$out" + local out task attempt + for attempt in 1 2 3; do + refresh_pulp_token + out=$(pulp --background "$plugin" publication create --repository "$repository" "$@" 2>&1) && break + echo "[WARN] publication start attempt $attempt/3 failed for $repository, retrying..." >&2 + sleep $((attempt * 10)) + out="" + done + if [[ -z "$out" ]]; then echo "::error::Cannot start the publication of repository $repository" return 1 - } + fi task=$(grep -oPm1 '/api/v3/tasks/[0-9a-f-]+/' <<< "$out" || true) if [[ -z "$task" ]]; then echo "$out" From 99bd5aa7476f8cb9cb709a3c039a3ce23daf64e2 Mon Sep 17 00:00:00 2001 From: Kevin Duret Date: Fri, 17 Jul 2026 17:12:25 +0200 Subject: [PATCH 46/49] fix(ci): bound the pulp delivery concurrency to 4 legs A full-matrix delivery (8 distributions uploading, associating and polling at once) saturates the pulp database: postgres dropped connections and the api pods crash-looped. Four concurrent legs keep the throughput without tipping the stack over. --- .github/workflows/plugins.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/plugins.yml b/.github/workflows/plugins.yml index 8a11da9b94..771c5510ad 100644 --- a/.github/workflows/plugins.yml +++ b/.github/workflows/plugins.yml @@ -480,6 +480,10 @@ jobs: pull-requests: read strategy: fail-fast: false + # bounded concurrency: a full-matrix delivery saturates the pulp + # database (postgres dropped connections, api pods crash-looped) when + # the 8 legs upload/associate/poll at once + max-parallel: 4 matrix: include: - distrib: el8 From b6705f4722f8523b93f751712fd65183a428da91 Mon Sep 17 00:00:00 2001 From: Kevin Duret Date: Fri, 17 Jul 2026 19:14:53 +0200 Subject: [PATCH 47/49] fix(ci): fail-closed, retried stable guard through the api The per-package pulp-cli lookups in the rpm stable guard died silently on a transient 502 (killing the upload worker without a message), and a 502 on the existence check would even have bypassed the guard. One retried api call, fail-closed on unreachability - and two python cli startups per package saved. --- .../package-delivery-pulp/deliver-rpm.sh | 24 +++++++++++++++---- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/.github/actions/package-delivery-pulp/deliver-rpm.sh b/.github/actions/package-delivery-pulp/deliver-rpm.sh index 0528ff68b1..1dbad7a650 100755 --- a/.github/actions/package-delivery-pulp/deliver-rpm.sh +++ b/.github/actions/package-delivery-pulp/deliver-rpm.sh @@ -27,11 +27,22 @@ assert_not_in_stable() { name=${base%-*} stable_repository="$STABLE_REPOSITORY_PREFIX-$arch" + # fail closed on an unreachable api (a transient 5xx must not bypass the + # guard nor silently kill the calling worker), fail open only on the repo + # genuinely not existing yet + local repo_page + repo_page=$( + curl -fsSL --retry 3 --retry-delay 5 -H "Authorization: Github $PULP_TOKEN" -G \ + --data-urlencode "name=$stable_repository" \ + --data-urlencode "limit=1" \ + "$PULP_URL/api/v3/repositories/rpm/rpm/" + ) || { + echo "::error::Cannot check the stable repository $stable_repository to guard $name $version-$release ($arch); refusing to deliver. Retry once the api is reachable." + return 1 + } + repository_version=$(echo "$repo_page" | jq -r '.results[0].latest_version_href // empty') # no stable repository yet => nothing can be in stable - if ! pulp rpm repository show --name "$stable_repository" >/dev/null 2>&1; then - return 0 - fi - repository_version=$(pulp rpm repository show --name "$stable_repository" | jq -r '.latest_version_href') + [[ -z "$repository_version" ]] && return 0 count=$( curl -fsSL --retry 3 --retry-delay 5 -H "Authorization: Github $PULP_TOKEN" -G \ @@ -42,7 +53,10 @@ assert_not_in_stable() { --data-urlencode "arch=$arch" \ --data-urlencode "limit=1" \ "$PULP_URL/api/v3/content/rpm/packages/" | jq -r '.count' - ) + ) || { + echo "::error::Cannot check the stable repository $stable_repository content to guard $name $version-$release ($arch); refusing to deliver. Retry once the api is reachable." + return 1 + } if [[ "$count" -gt 0 ]]; then echo "::error::$name $version-$release ($arch) is already published in the stable repository $stable_repository; refusing to deliver it to $REPOSITORY_NAME. Bump the package version for a new build." return 1 From d3543e9e4416abf6b38278c8dc6fcff35bbb5825 Mon Sep 17 00:00:00 2001 From: Kevin Duret Date: Mon, 20 Jul 2026 09:28:10 +0200 Subject: [PATCH 48/49] chore(ci): retrigger (the dev stack is up) From c791a2a0e1203018ec3d59be3c492b5506859211 Mon Sep 17 00:00:00 2001 From: Kevin Duret Date: Mon, 20 Jul 2026 14:46:52 +0200 Subject: [PATCH 49/49] test(ci): full-blast matrix (no max-parallel) on the resized database --- .github/workflows/plugins.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.github/workflows/plugins.yml b/.github/workflows/plugins.yml index 771c5510ad..8a11da9b94 100644 --- a/.github/workflows/plugins.yml +++ b/.github/workflows/plugins.yml @@ -480,10 +480,6 @@ jobs: pull-requests: read strategy: fail-fast: false - # bounded concurrency: a full-matrix delivery saturates the pulp - # database (postgres dropped connections, api pods crash-looped) when - # the 8 legs upload/associate/poll at once - max-parallel: 4 matrix: include: - distrib: el8