diff --git a/.github/actions/package-delivery-pulp/action.yml b/.github/actions/package-delivery-pulp/action.yml index 0950a5e1e29..608899a6cde 100644 --- a/.github/actions/package-delivery-pulp/action.yml +++ b/.github/actions/package-delivery-pulp/action.yml @@ -8,8 +8,9 @@ inputs: description: "The distribution used for packaging" required: true version: - description: "Centreon packaged major version" - required: true + description: "Centreon packaged major version (not used by the plugins repository family)" + required: false + default: "" cache_key: description: "The cached package key" required: true @@ -27,6 +28,10 @@ inputs: description: "Delivery to feature or standard repository" required: false default: "standard" + repository_name: + description: "Target repository family: standard, standard-internal, business, business-internal or plugins" + required: false + default: "standard" verify: description: "Verify the delivered packages afterwards (presence, metadata resolution, fetchability)" required: false @@ -71,6 +76,7 @@ runs: release_type: ${{ inputs.release_type }} is_cloud: ${{ inputs.is_cloud }} delivery_type: ${{ inputs.delivery_type }} + repository_name: ${{ inputs.repository_name }} - name: Restore packages from cache if: steps.pulp-properties.outputs.skip_delivery == 'false' @@ -115,6 +121,7 @@ runs: BASE_PATH: ${{ steps.pulp-properties.outputs.base_path }} SUITE: ${{ steps.pulp-properties.outputs.suite }} STABLE_SUITE: ${{ steps.pulp-properties.outputs.stable_suite }} + STABLE_BASE_PATH: ${{ steps.pulp-properties.outputs.stable_base_path }} POOL_PATH: ${{ steps.pulp-properties.outputs.pool_path }} PULP_URL: ${{ inputs.pulp_url }} PULP_TOKEN: ${{ env.PULP_TOKEN }} diff --git a/.github/actions/package-delivery-pulp/deliver-deb.sh b/.github/actions/package-delivery-pulp/deliver-deb.sh index 1dfd3409b63..331e69ad259 100755 --- a/.github/actions/package-delivery-pulp/deliver-deb.sh +++ b/.github/actions/package-delivery-pulp/deliver-deb.sh @@ -32,8 +32,8 @@ assert_not_in_stable() { # endpoint error cannot let an already-stable version slip through and evict it # (the rpm guardrail is likewise fail-closed). pkg_file=$(mktemp) - http_code=$(curl -sSL -o "$pkg_file" -w '%{http_code}' \ - "$PULP_CONTENT_URL/$BASE_PATH/dists/$STABLE_SUITE/main/binary-$arch/Packages" 2>/dev/null || echo 000) + http_code=$(content_curl -sSL --retry 3 --retry-delay 5 -o "$pkg_file" -w '%{http_code}' \ + "$PULP_CONTENT_URL/${STABLE_BASE_PATH:-$BASE_PATH}/dists/$STABLE_SUITE/main/binary-$arch/Packages" 2>/dev/null || echo 000) case "$http_code" in 404) rm -f "$pkg_file"; return 0 ;; 200) packages=$(cat "$pkg_file"); rm -f "$pkg_file" ;; @@ -91,25 +91,346 @@ PULP_LABELS=$(jq -cn \ --arg workflow "${GITHUB_WORKFLOW:-}" \ '{"module": $mod, "git_commit": $git_commit, "git_ref": $git_ref, "github_run_id": $run_id, "github_actor": $actor, "github_workflow": $workflow}') +# Batched delivery, deb flavor. Unlike +# rpm, a repository-less deb upload ignores the distribution/component +# parameters (pulp_deb only creates the suite association inside the +# repository code path), so the suite association is created explicitly as +# PackageReleaseComponent content units. The release_components api is not +# listable by the OIDC ci-user, so the FIRST package of each architecture is +# delivered through the legacy repository code path - that also +# get_or_creates the ReleaseComponent and the ReleaseArchitecture of the +# suite - and its PackageReleaseComponent (listable) yields the release +# component href for the whole batch. Everything else is uploaded as +# unassociated content in a client-side pool, associated in parallel tasks +# (no repository lock), then added to the repository with a single modify. +lookup_deb_content() { + # emit the href of a deb content unit matching the query, empty if absent. + # Every caller sits on a fallback path where the content is expected to + # exist, so an empty result is retried too: the api has been observed + # answering an empty page for content committed seconds earlier (stale + # read), and a transient error must not read as "content absent" either. + local endpoint=$1 query=$2 out attempt + for attempt in 1 2 3 4 5; do + out=$(curl -fsSL --retry 3 --retry-delay 5 -H "Authorization: Github $PULP_TOKEN" -G \ + --data-urlencode "limit=1" \ + $query \ + "$PULP_URL/api/v3/content/deb/$endpoint/" | jq -r '.results[0].pulp_href // empty') || out="" + if [[ -n "$out" ]]; then + echo "$out" + return 0 + fi + sleep $((attempt * 3)) + refresh_pulp_token + done +} + +resolve_task_content() { + # wait for a content-create task and emit its created content href; fall + # back to a lookup (content already existing on a job re-run) + local task_href=$1 endpoint=$2 fallback_query=$3 + local body state content attempt + for ((attempt = 0; attempt < 200; attempt++)); do + refresh_pulp_token + body=$(curl -fsSL -H "Authorization: Github $PULP_TOKEN" "$PULP_URL$task_href" 2>/dev/null) || body="" + state=$(echo "$body" | jq -r '.state' 2>/dev/null) || state="" + case "$state" in + completed) + content=$(echo "$body" | jq -r '.created_resources[0] // empty') + if [[ -n "$content" ]]; then + echo "$content" + return 0 + fi + break + ;; + failed | canceled) + break + ;; + *) + sleep 3 + ;; + esac + done + content=$(lookup_deb_content "$endpoint" "$fallback_query") + if [[ -z "$content" ]]; then + echo "::error::Cannot resolve the created deb content for task $task_href" >&2 + return 1 + fi + echo "$content" +} + +# emit the release-component hrefs a package is associated with +lookup_prcs() { + local package_href=$1 + curl -fsSL --retry 3 --retry-delay 5 -H "Authorization: Github $PULP_TOKEN" -G \ + --data-urlencode "package=$package_href" \ + --data-urlencode "limit=100" \ + "$PULP_URL/api/v3/content/deb/package_release_components/" \ + | jq -r '.results[].release_component' +} + +# group the files by architecture: one file per arch goes through the legacy +# path so the suite association targets of that arch exist. Prefer a file +# whose content does NOT exist in pulp yet: a fresh upload carries exactly +# one suite association afterwards, which identifies the release component +# unambiguously. Content reused across suites (identical bytes delivered to +# another suite before) keeps its prior associations, so for a reused +# representative the pre-upload association set is captured to diff later. +declare -A LEGACY_FOR_ARCH=() +declare -A LEGACY_FRESH=() +declare -A LEGACY_BEFORE=() +refresh_pulp_token +for i in "${!FILES[@]}"; do + FILE=${FILES[$i]} + arch=$(dpkg-deb -f "$FILE" Architecture) + if [[ -n "${LEGACY_FRESH[$arch]+set}" ]]; then + continue + fi + if ((i % 40 == 0)); then + refresh_pulp_token + fi + sha=$(sha256sum "$FILE" | cut -d' ' -f1) + existing=$(lookup_deb_content "packages" "--data-urlencode sha256=$sha") + if [[ -z "$existing" ]]; then + LEGACY_FOR_ARCH[$arch]="$FILE" + LEGACY_FRESH[$arch]=1 + elif [[ -z "${LEGACY_FOR_ARCH[$arch]+set}" ]]; then + LEGACY_FOR_ARCH[$arch]="$FILE" + LEGACY_BEFORE[$arch]=$(lookup_prcs "$existing" | sort) + fi +done + +declare -A IS_LEGACY=() +LEGACY_FILES=() +LEGACY_ARCHS=() +for arch in "${!LEGACY_FOR_ARCH[@]}"; do + LEGACY_FILES+=("${LEGACY_FOR_ARCH[$arch]}") + LEGACY_ARCHS+=("$arch") + IS_LEGACY["${LEGACY_FOR_ARCH[$arch]}"]=1 +done +ORPHAN_FILES=() for FILE in "${FILES[@]}"; do + [[ -n "${IS_LEGACY[$FILE]+set}" ]] || ORPHAN_FILES+=("$FILE") +done + +refresh_pulp_token +for FILE in "${LEGACY_FILES[@]}"; do assert_not_in_stable "$FILE" - echo "[INFO] Uploading $FILE to $POOL_PATH/ ($SUITE/main, module $MODULE_NAME)" - # packages are labeled with their module so that promote-to-stable can identify - # which packages belong to this module, pulp-cli does not allow to set labels nor - # the relative path of deb packages so the api is used directly - TASK_HREF=$( +done +# sequential with retry: the legacy path creates a repository version, and +# concurrent legs of the shared repository can lose the version race +# (wait_task_race rc 2); re-uploading converges, content and suite structure +# are get_or_created server-side +for FILE in "${LEGACY_FILES[@]}"; do + for legacy_attempt in 1 2 3; do + echo "[INFO] Uploading $FILE to $POOL_PATH/ ($SUITE/main, module $MODULE_NAME) [legacy path, attempt $legacy_attempt]" + LEGACY_TASK=$( + pulp_upload \ + -F "file=@\"$FILE\"" \ + -F "relative_path=$POOL_PATH/$FILE" \ + -F "distribution=$SUITE" \ + -F "component=main" \ + -F "repository=$REPOSITORY_HREF" \ + -F "pulp_labels=$PULP_LABELS" \ + "$PULP_URL/api/v3/content/deb/packages/" + ) + wait_task_race "$LEGACY_TASK" && rc=0 || rc=$? + if [[ $rc -eq 0 ]]; then + break + elif [[ $rc -eq 2 && $legacy_attempt -lt 3 ]]; then + echo "[WARN] Legacy upload of $FILE lost the repository-version race against a concurrent delivery, retrying" + sleep $((legacy_attempt * 15)) + else + echo "::error::Legacy upload of $FILE failed" + exit 1 + fi + done +done + +# the release component href of $SUITE/main, deduced from the association the +# legacy uploads just created. A FRESH representative now carries exactly one +# association; a reused one carries its prior associations too, so the href is +# the difference between its post- and pre-upload association sets. +refresh_pulp_token +RELEASE_COMPONENT_HREF="" +for i in "${!LEGACY_FILES[@]}"; do + FILE=${LEGACY_FILES[$i]} + arch=${LEGACY_ARCHS[$i]} + sha=$(sha256sum "$FILE" | cut -d' ' -f1) + href=$(lookup_deb_content "packages" "--data-urlencode sha256=$sha") + if [[ -z "$href" ]]; then + echo "::error::Cannot find the legacy-delivered package $FILE (sha256 $sha)" + exit 1 + fi + after=$(lookup_prcs "$href" | sort) + if [[ -n "${LEGACY_FRESH[$arch]+set}" ]]; then + if [[ $(echo "$after" | grep -c .) -eq 1 ]]; then + RELEASE_COMPONENT_HREF="$after" + break + fi + else + new_rcs=$(comm -13 <(echo "${LEGACY_BEFORE[$arch]}") <(echo "$after") | grep . || true) + if [[ $(echo "$new_rcs" | grep -c .) -eq 1 ]]; then + RELEASE_COMPONENT_HREF="$new_rcs" + break + elif [[ $(echo "$after" | grep -c .) -eq 1 ]]; then + # rerun of a partially delivered leg: the association was created by + # the previous attempt (empty diff) and is the only one the + # representative carries - it IS the suite release component. + RELEASE_COMPONENT_HREF=$(echo "$after" | grep .) + break + fi + fi +done +if [[ -z "$RELEASE_COMPONENT_HREF" ]]; then + echo "::error::Cannot deduce the $SUITE/main release component: every legacy representative is reused content whose suite association pre-exists. Deliver a rebuilt (fresh) package or clean up the previous partial delivery." + exit 1 +fi +echo "[INFO] Release component of $SUITE/main: $RELEASE_COMPONENT_HREF" + +# parallel repository-less uploads (pool of 8), marker-file based like rpm +UPLOAD_DIR=$(mktemp -d) +MAX_PARALLEL_UPLOADS=8 +for i in "${!ORPHAN_FILES[@]}"; do + FILE=${ORPHAN_FILES[$i]} + if ((i % 40 == 0)); then + refresh_pulp_token + fi + ( + # subshell-local refresh: under server slowdowns the inherited token can + # outlive its validity between two parent refreshes + refresh_pulp_token + assert_not_in_stable "$FILE" + echo "[INFO] Uploading $FILE to $POOL_PATH/ ($SUITE/main, module $MODULE_NAME)" pulp_upload \ -F "file=@\"$FILE\"" \ -F "relative_path=$POOL_PATH/$FILE" \ - -F "distribution=$SUITE" \ - -F "component=main" \ - -F "repository=$REPOSITORY_HREF" \ -F "pulp_labels=$PULP_LABELS" \ - "$PULP_URL/api/v3/content/deb/packages/" - ) - wait_task "$TASK_HREF" + "$PULP_URL/api/v3/content/deb/packages/" > "$UPLOAD_DIR/$i.task" + ) & + while (($(jobs -rp | wc -l) >= MAX_PARALLEL_UPLOADS)); do + wait -n || true + done +done +wait || true + +echo "[INFO] Resolving ${#ORPHAN_FILES[@]} uploaded package(s)" +ORPHAN_SHA256S=() +for i in "${!ORPHAN_FILES[@]}"; do + FILE=${ORPHAN_FILES[$i]} + if [[ ! -s "$UPLOAD_DIR/$i.task" ]]; then + echo "::error::Upload failed for $FILE (no task href, see the worker error above)" + exit 1 + fi + ORPHAN_SHA256S+=("$(sha256sum "$FILE" | cut -d' ' -f1)") +done +for i in "${!ORPHAN_FILES[@]}"; do + if ((i % 40 == 0)); then + refresh_pulp_token + fi + ( + resolve_task_content "$(cat "$UPLOAD_DIR/$i.task")" "packages" \ + "--data-urlencode sha256=${ORPHAN_SHA256S[$i]}" > "$UPLOAD_DIR/$i.content" + ) & + while (($(jobs -rp | wc -l) >= MAX_PARALLEL_UPLOADS)); do + wait -n || true + done +done +wait || true +PACKAGE_HREFS=() +for i in "${!ORPHAN_FILES[@]}"; do + if [[ ! -s "$UPLOAD_DIR/$i.content" ]]; then + echo "::error::Cannot resolve the uploaded content for ${ORPHAN_FILES[$i]} (see the worker error above)" + exit 1 + fi + PACKAGE_HREFS+=("$(cat "$UPLOAD_DIR/$i.content")") +done + +# associate every uploaded package with the suite component. The +# package_release_components create is a plain synchronous DRF create (201 +# with the created unit, no task), so the href comes straight out of the +# response; a failed create (unit already existing on a job re-run) falls +# back to a lookup. +echo "[INFO] Associating ${#PACKAGE_HREFS[@]} package(s) with $SUITE/main" +for i in "${!PACKAGE_HREFS[@]}"; do + if ((i % 40 == 0)); then + refresh_pulp_token + fi + ( + refresh_pulp_token + response=$( + curl -sSL --retry 3 --retry-delay 5 -w '\n%{http_code}' -H "Authorization: Github $PULP_TOKEN" \ + -X POST -H "Content-Type: application/json" \ + -d "{\"package\": \"${PACKAGE_HREFS[$i]}\", \"release_component\": \"$RELEASE_COMPONENT_HREF\"}" \ + "$PULP_URL/api/v3/content/deb/package_release_components/" + ) || response=$'\n000' + code="${response##*$'\n'}" + body="${response%$'\n'*}" + href="" + if [[ "$code" == 2* ]]; then + # tolerate a non-json body (gateway error page behind a 2xx): a jq + # failure inside the substitution would silently kill this subshell + href=$(echo "$body" | jq -r '.pulp_href // .task // empty' 2>/dev/null) || href="" + fi + if [[ -z "$href" ]]; then + href=$(lookup_deb_content "package_release_components" \ + "--data-urlencode package=${PACKAGE_HREFS[$i]} --data-urlencode release_component=$RELEASE_COMPONENT_HREF") + if [[ -n "$href" ]]; then + # the api answers 500 on a duplicate synchronous content create; on a + # rerun the association simply pre-exists, nothing is wrong + echo "[INFO] ${ORPHAN_FILES[$i]}: suite association already exists (HTTP $code on create, expected on a rerun), reusing it" + else + echo "[WARN] Suite association create for ${ORPHAN_FILES[$i]} returned HTTP $code: $(echo "$body" | head -c 300)" >&2 + fi + fi + printf '%s' "$href" > "$UPLOAD_DIR/$i.prc" + ) & + while (($(jobs -rp | wc -l) >= MAX_PARALLEL_UPLOADS)); do + wait -n || true + done +done +wait || true - # record the uploaded package in the manifest for the verification step +PRC_HREFS=() +for i in "${!PACKAGE_HREFS[@]}"; do + href="" + [[ -s "$UPLOAD_DIR/$i.prc" ]] && href=$(cat "$UPLOAD_DIR/$i.prc") + if [[ "$href" == */tasks/* ]]; then + href=$(resolve_task_content "$href" "package_release_components" \ + "--data-urlencode package=${PACKAGE_HREFS[$i]} --data-urlencode release_component=$RELEASE_COMPONENT_HREF") + fi + if [[ -z "$href" ]]; then + echo "::error::Suite association failed for ${ORPHAN_FILES[$i]} (see the worker error above)" + exit 1 + fi + PRC_HREFS+=("$href") +done +rm -rf "$UPLOAD_DIR" + +if ((${#PACKAGE_HREFS[@]} > 0)); then + echo "[INFO] Adding ${#PACKAGE_HREFS[@]} package(s) and their suite associations to $REPOSITORY_NAME in a single task" + refresh_pulp_token + # body through a file: thousands of hrefs exceed the argv limit + ADD_BODY_FILE=$(mktemp) + printf '%s\n' "${PACKAGE_HREFS[@]}" "${PRC_HREFS[@]}" | jq -R . | jq -cs '{add_content_units: .}' > "$ADD_BODY_FILE" + # retried like the legacy uploads: the modify also creates a repository + # version and can lose the same race against a concurrent delivery + for modify_attempt in 1 2 3; do + MODIFY_TASK=$(start_modify_task "$PULP_URL${REPOSITORY_HREF}modify/" "$ADD_BODY_FILE") + wait_task_race "$MODIFY_TASK" && rc=0 || rc=$? + if [[ $rc -eq 0 ]]; then + break + elif [[ $rc -eq 2 && $modify_attempt -lt 3 ]]; then + echo "[WARN] Repository modify lost the repository-version race against a concurrent delivery, retrying" + sleep $((modify_attempt * 15)) + else + echo "::error::Repository modify failed" + exit 1 + fi + done +fi + +# record every delivered package in the manifest for the verification step +for FILE in "${FILES[@]}"; do name=$(dpkg-deb -f "$FILE" Package) version=$(dpkg-deb -f "$FILE" Version) arch=$(dpkg-deb -f "$FILE" Architecture) @@ -122,7 +443,7 @@ for FILE in "${FILES[@]}"; do done echo "[INFO] Publishing repository $REPOSITORY_NAME" -pulp deb publication create --repository "$REPOSITORY_NAME" --structured >/dev/null +create_publication deb "$REPOSITORY_NAME" --structured echo "::notice::Packages are available with: deb $PULP_CONTENT_URL/$BASE_PATH/ $SUITE main" diff --git a/.github/actions/package-delivery-pulp/deliver-rpm.sh b/.github/actions/package-delivery-pulp/deliver-rpm.sh index dd58fa2bf9b..d52b7e37f1f 100755 --- a/.github/actions/package-delivery-pulp/deliver-rpm.sh +++ b/.github/actions/package-delivery-pulp/deliver-rpm.sh @@ -27,14 +27,25 @@ assert_not_in_stable() { name=${base%-*} stable_repository="$STABLE_REPOSITORY_PREFIX-$arch" + # fail closed on an unreachable api (a transient 5xx must not bypass the + # guard nor silently kill the calling worker), fail open only on the repo + # genuinely not existing yet + local repo_page + repo_page=$( + curl -fsSL --retry 3 --retry-delay 5 -H "Authorization: Github $PULP_TOKEN" -G \ + --data-urlencode "name=$stable_repository" \ + --data-urlencode "limit=1" \ + "$PULP_URL/api/v3/repositories/rpm/rpm/" + ) || { + echo "::error::Cannot check the stable repository $stable_repository to guard $name $version-$release ($arch); refusing to deliver. Retry once the api is reachable." + return 1 + } + repository_version=$(echo "$repo_page" | jq -r '.results[0].latest_version_href // empty') # no stable repository yet => nothing can be in stable - if ! pulp rpm repository show --name "$stable_repository" >/dev/null 2>&1; then - return 0 - fi - repository_version=$(pulp rpm repository show --name "$stable_repository" | jq -r '.latest_version_href') + [[ -z "$repository_version" ]] && return 0 count=$( - curl -fsSL -H "Authorization: Github $PULP_TOKEN" -G \ + curl -fsSL --retry 3 --retry-delay 5 -H "Authorization: Github $PULP_TOKEN" -G \ --data-urlencode "repository_version=$repository_version" \ --data-urlencode "name=$name" \ --data-urlencode "version=$version" \ @@ -42,13 +53,57 @@ assert_not_in_stable() { --data-urlencode "arch=$arch" \ --data-urlencode "limit=1" \ "$PULP_URL/api/v3/content/rpm/packages/" | jq -r '.count' - ) + ) || { + echo "::error::Cannot check the stable repository $stable_repository content to guard $name $version-$release ($arch); refusing to deliver. Retry once the api is reachable." + return 1 + } if [[ "$count" -gt 0 ]]; then echo "::error::$name $version-$release ($arch) is already published in the stable repository $stable_repository; refusing to deliver it to $REPOSITORY_NAME. Bump the package version for a new build." return 1 fi } +# Wait for a repository-less upload task and emit +# the created content href. A task that did not produce one (content already +# existing on a job re-run) is resolved by the package sha256 instead of +# failing the delivery. +resolve_uploaded_content() { + local task_href=$1 sha256=$2 + local body state content attempt + for ((attempt = 0; attempt < 200; attempt++)); do + refresh_pulp_token + body=$(curl -fsSL -H "Authorization: Github $PULP_TOKEN" "$PULP_URL$task_href" 2>/dev/null) || body="" + state=$(echo "$body" | jq -r '.state' 2>/dev/null) || state="" + case "$state" in + completed) + content=$(echo "$body" | jq -r '.created_resources[0] // empty') + if [[ -n "$content" ]]; then + echo "$content" + return 0 + fi + break + ;; + failed | canceled) + break + ;; + *) + sleep 3 + ;; + esac + done + content=$( + curl -fsSL -H "Authorization: Github $PULP_TOKEN" -G \ + --data-urlencode "sha256=$sha256" \ + --data-urlencode "limit=1" \ + "$PULP_URL/api/v3/content/rpm/packages/" | jq -r '.results[0].pulp_href // empty' + ) + if [[ -z "$content" ]]; then + echo "::error::Cannot resolve the uploaded content for task $task_href (sha256 $sha256)" >&2 + return 1 + fi + echo "$content" +} + FILES=(*.rpm) if [[ ${#FILES[@]} -eq 0 ]]; then echo "::error::No rpm package found to deliver" @@ -95,22 +150,54 @@ for ARCH in noarch x86_64; do REPOSITORY_HREF=$(pulp rpm repository show --name "$REPOSITORY_NAME" | jq -r '.pulp_href') - # Packages are uploaded straight into the repository: pulpcore requires - # non-admin accounts to provide the destination repository on content upload, - # so the upload cannot be decoupled from the repository association. Packages - # are labeled with their module so that promote-to-stable can identify them; - # pulp-cli does not allow to set labels on upload so the api is used directly. - for FILE in "${ARCH_FILES[@]}"; do - assert_not_in_stable "$FILE" "$ARCH" - echo "[INFO] Uploading $(basename "$FILE") to $REPOSITORY_NAME (module $MODULE_NAME)" - TASK_HREF=$( + # Batched delivery: packages are + # uploaded as unassociated content (repository-less, so the create tasks + # parallelize across the pulp workers instead of serializing on the + # repository lock), then the whole batch is added to the repository with a + # single modify task. Requires the reconciled rpm/packages access policy + # (delivery-tooling#209). Packages are labeled with their module so that + # promote-to-stable can identify them. + TASK_HREFS=() + SHA256S=() + # The uploads are parallelized client-side (the + # repository-less create tasks already parallelize server-side). A bounded + # pool of background subshells posts the files, each writing its task href + # to a marker file; the parent refreshes the OIDC token between spawn + # chunks so the workers always inherit a fresh token. Worker failures are + # detected through missing/empty marker files after the wait. + UPLOAD_DIR=$(mktemp -d) + MAX_PARALLEL_UPLOADS=8 + for i in "${!ARCH_FILES[@]}"; do + FILE=${ARCH_FILES[$i]} + if ((i % 40 == 0)); then + refresh_pulp_token + fi + ( + # subshell-local refresh: under server slowdowns the inherited token can + # outlive its validity between two parent refreshes + refresh_pulp_token + assert_not_in_stable "$FILE" "$ARCH" + echo "[INFO] Uploading $(basename "$FILE") (module $MODULE_NAME)" pulp_upload \ -F "file=@\"$FILE\"" \ - -F "repository=$REPOSITORY_HREF" \ -F "pulp_labels=$PULP_LABELS" \ - "$PULP_URL/api/v3/content/rpm/packages/" - ) - wait_task "$TASK_HREF" + "$PULP_URL/api/v3/content/rpm/packages/" > "$UPLOAD_DIR/$i.task" + ) & + while (($(jobs -rp | wc -l) >= MAX_PARALLEL_UPLOADS)); do + wait -n || true + done + done + wait || true + + for i in "${!ARCH_FILES[@]}"; do + FILE=${ARCH_FILES[$i]} + if [[ ! -s "$UPLOAD_DIR/$i.task" ]]; then + echo "::error::Upload failed for $(basename "$FILE") (no task href, see the worker error above)" + exit 1 + fi + TASK_HREFS+=("$(cat "$UPLOAD_DIR/$i.task")") + sha256=$(sha256sum "$FILE" | cut -d' ' -f1) + SHA256S+=("$sha256") # record the uploaded package in the manifest (name-version-release parsed # the same way as assert_not_in_stable) for the verification step @@ -121,16 +208,53 @@ for ARCH in noarch x86_64; do base=${base%-*} version=${base##*-} name=${base%-*} - sha256=$(sha256sum "$FILE" | cut -d' ' -f1) manifest_add "$(jq -cn \ --arg filename "$FILENAME" --arg name "$name" --arg version "$version" \ --arg release "$release" --arg arch "$ARCH" --arg sha256 "$sha256" \ --arg repository "$REPOSITORY_NAME" --arg base_path "$BASE_PATH" \ '{filename:$filename,name:$name,version:$version,release:$release,arch:$arch,sha256:$sha256,repository:$repository,base_path:$base_path}')" done + rm -rf "$UPLOAD_DIR" + + echo "[INFO] Waiting for ${#TASK_HREFS[@]} upload task(s) and resolving the content" + # The resolutions are parallelized like the + # uploads - sequential, they paid one task GET per package (~6 min at 675) + RESOLVE_DIR=$(mktemp -d) + for i in "${!TASK_HREFS[@]}"; do + if ((i % 40 == 0)); then + refresh_pulp_token + fi + ( + resolve_uploaded_content "${TASK_HREFS[$i]}" "${SHA256S[$i]}" > "$RESOLVE_DIR/$i.content" + ) & + while (($(jobs -rp | wc -l) >= MAX_PARALLEL_UPLOADS)); do + wait -n || true + done + done + wait || true + + CONTENT_HREFS=() + for i in "${!TASK_HREFS[@]}"; do + if [[ ! -s "$RESOLVE_DIR/$i.content" ]]; then + echo "::error::Cannot resolve the uploaded content of task ${TASK_HREFS[$i]} (see the worker error above)" + exit 1 + fi + CONTENT_HREFS+=("$(cat "$RESOLVE_DIR/$i.content")") + done + rm -rf "$RESOLVE_DIR" + + echo "[INFO] Adding ${#CONTENT_HREFS[@]} package(s) to $REPOSITORY_NAME in a single task" + # refresh from the parent shell: the resolutions above ran in subshells, so + # their refreshes never updated this shell's token (the 401 trap, again) + refresh_pulp_token + # body through a file: thousands of hrefs exceed the argv limit + ADD_BODY_FILE=$(mktemp) + printf '%s\n' "${CONTENT_HREFS[@]}" | jq -R . | jq -cs '{add_content_units: .}' > "$ADD_BODY_FILE" + MODIFY_TASK=$(start_modify_task "$PULP_URL${REPOSITORY_HREF}modify/" "$ADD_BODY_FILE") + wait_task "$MODIFY_TASK" echo "[INFO] Publishing repository $REPOSITORY_NAME" - pulp rpm publication create --repository "$REPOSITORY_NAME" >/dev/null + create_publication rpm "$REPOSITORY_NAME" echo "::notice::Packages are available at $PULP_CONTENT_URL/$BASE_PATH/" done diff --git a/.github/actions/promote-to-stable-pulp/action.yml b/.github/actions/promote-to-stable-pulp/action.yml index 068d4659aab..3f50ee882df 100644 --- a/.github/actions/promote-to-stable-pulp/action.yml +++ b/.github/actions/promote-to-stable-pulp/action.yml @@ -8,8 +8,9 @@ inputs: description: "The distribution used for packaging" required: true major_version: - description: "Centreon packaged major version" - required: true + description: "Centreon packaged major version (not used by the plugins repository family)" + required: false + default: "" stability: description: "The package stability (stable, testing, unstable)" required: true @@ -19,6 +20,10 @@ inputs: is_cloud: description: "Release context (cloud or not cloud)" required: true + repository_name: + description: "Target repository family: standard, standard-internal, business, business-internal or plugins" + required: false + default: "standard" verify: description: "Verify the promoted packages afterwards (presence, metadata resolution, fetchability)" required: false @@ -61,6 +66,7 @@ runs: stability: ${{ inputs.stability }} release_type: ${{ inputs.release_type }} is_cloud: ${{ inputs.is_cloud }} + repository_name: ${{ inputs.repository_name }} - name: Setup pulp-cli uses: ./.github/actions/setup-pulp-cli @@ -96,6 +102,8 @@ runs: TESTING_POOL_PATH: ${{ steps.pulp-properties.outputs.testing_pool_path }} TESTING_SUITE: ${{ steps.pulp-properties.outputs.testing_suite }} STABLE_SUITE: ${{ steps.pulp-properties.outputs.stable_suite }} + STABLE_REPOSITORY_NAME: ${{ steps.pulp-properties.outputs.stable_repository_name }} + STABLE_BASE_PATH: ${{ steps.pulp-properties.outputs.stable_base_path }} PACKAGE_DISTRIB_NAME: ${{ steps.pulp-properties.outputs.package_distrib_name }} STABILITY: ${{ inputs.stability }} PULP_URL: ${{ inputs.pulp_url }} diff --git a/.github/actions/promote-to-stable-pulp/promote-deb.sh b/.github/actions/promote-to-stable-pulp/promote-deb.sh index c1db6bcd9fd..3704bd50faa 100755 --- a/.github/actions/promote-to-stable-pulp/promote-deb.sh +++ b/.github/actions/promote-to-stable-pulp/promote-deb.sh @@ -11,59 +11,209 @@ source "$(dirname "$0")/../../scripts/pulp/api.sh" PULP_URL="${PULP_URL:-https://pulp-api.apps.centreon.com}" PULP_CONTENT_URL="${PULP_CONTENT_URL:-https://packages.apps.centreon.com}" +# the stable packages may live in a DEDICATED repository (deb plugins: +# apt-plugins-stable/ubuntu-plugins-stable with plain-codename suites, the +# artifactory-compatible client layout); default to the source repository +# for the families whose stable suite it hosts. +STABLE_REPOSITORY_NAME="${STABLE_REPOSITORY_NAME:-$REPOSITORY_NAME}" +STABLE_BASE_PATH="${STABLE_BASE_PATH:-$BASE_PATH}" + if ! pulp deb repository show --name "$REPOSITORY_NAME" >/dev/null 2>&1; then echo "::error::Nothing to promote, repository $REPOSITORY_NAME does not exist" exit 1 fi +if ! pulp deb repository show --name "$STABLE_REPOSITORY_NAME" >/dev/null 2>&1; then + echo "::error::Stable repository $STABLE_REPOSITORY_NAME does not exist. Pulp repositories are provisioned centrally by delivery-tooling create-repos; run create-repos before promoting." + exit 1 +fi + VERSION_HREF=$(pulp deb repository show --name "$REPOSITORY_NAME" | jq -r '.latest_version_href') -# packages of the module are identified by the label set at delivery time, -# the testing pool path scopes the stability and the package distrib name -# scopes the distribution as the apt repository holds all the suites -RESPONSE=$( - curl -fsSL -H "Authorization: Github $PULP_TOKEN" -G \ - --data-urlencode "repository_version=$VERSION_HREF" \ - --data-urlencode "pulp_label_select=module=$MODULE_NAME" \ - --data-urlencode "limit=1000" \ - "$PULP_URL/api/v3/content/deb/packages/" -) +# fetch a published index into a file: 0=ok, 2=absent (404), 1=error +fetch_index() { + local url=$1 out=$2 code + code=$(content_curl -sSL --retry 3 --retry-delay 5 -o "$out" -w '%{http_code}' "$url") || return 1 + [[ "$code" == "200" ]] && return 0 + [[ "$code" == "404" ]] && return 2 + return 1 +} -# fail on a truncated page (checked before the pool-path filtering): silently -# promoting a subset of the module's packages would publish an incomplete -# stable suite -if [[ $(echo "$RESPONSE" | jq '.count > (.results | length)') == "true" ]]; then - echo "::error::Package query on $REPOSITORY_NAME is truncated ($(echo "$RESPONSE" | jq -r '.results | length')/$(echo "$RESPONSE" | jq -r '.count') results); pagination is required" - exit 1 +# collect the sha256 set of every package published in a suite. The served +# dists/ indexes are the source of truth for suite membership: the packages' +# relative_path cannot be trusted for that (content reused across suites +# keeps its original upload path) and the release_components api is not +# listable by the OIDC ci-user. A missing Release (404) is an empty suite; +# any other fetch failure aborts, a partial set must not drive a promotion. +suite_sha_file() { + local base_path=$1 suite=$2 out release_file arches arch pkg_file rc + out=$(mktemp) + release_file=$(mktemp) + fetch_index "$PULP_CONTENT_URL/$base_path/dists/$suite/Release" "$release_file" && rc=0 || rc=$? + if [[ $rc -eq 2 ]]; then + echo "$out" + return 0 + elif [[ $rc -ne 0 ]]; then + echo "::error::Cannot fetch the $suite Release index; refusing to promote from a partial view." >&2 + return 1 + fi + arches=$(awk -F': ' '/^Architectures:/ {print $2}' "$release_file") + for arch in $arches; do + pkg_file=$(mktemp) + if ! fetch_index "$PULP_CONTENT_URL/$base_path/dists/$suite/main/binary-$arch/Packages" "$pkg_file"; then + echo "::error::Cannot fetch the $suite binary-$arch Packages index; refusing to promote from a partial view." >&2 + return 1 + fi + awk -F': ' '/^SHA256:/ {print $2}' "$pkg_file" >> "$out" + rm -f "$pkg_file" + done + rm -f "$release_file" + sort -u "$out" -o "$out" + echo "$out" +} + +TESTING_SHAS_FILE=$(suite_sha_file "$BASE_PATH" "$TESTING_SUITE") + +# "already promoted" is decided against the stable repository VERSION when a +# dedicated stable repository is used (it only ever receives stable content): +# a promotion whose publication failed then reruns as a plain republish. With +# a shared repository the version cannot discriminate suites, so membership +# stays based on the published stable index. +if [[ "$STABLE_REPOSITORY_NAME" != "$REPOSITORY_NAME" ]]; then + STABLE_SHAS_FILE=$(mktemp) + STABLE_VERSION_HREF=$(pulp deb repository show --name "$STABLE_REPOSITORY_NAME" | jq -r '.latest_version_href') + url="$PULP_URL/api/v3/content/deb/packages/?$( + printf 'repository_version=%s&fields=sha256&limit=1000' \ + "$(jq -rn --arg v "$STABLE_VERSION_HREF" '$v | @uri')" + )" + while [[ -n "$url" ]]; do + refresh_pulp_token + page=$(curl -fsSL --retry 3 --retry-delay 5 -H "Authorization: Github $PULP_TOKEN" "$url") + echo "$page" | jq -r '.results[].sha256' >> "$STABLE_SHAS_FILE" + url=$(echo "$page" | jq -r '.next // empty') + done + sort -u "$STABLE_SHAS_FILE" -o "$STABLE_SHAS_FILE" +else + STABLE_SHAS_FILE=$(suite_sha_file "$STABLE_BASE_PATH" "$STABLE_SUITE") fi +# packages of the module are identified by the label set at delivery time. +# Paginated: the shared repository keeps every delivered version across all +# suites, so the module listing exceeds a single page. +RESULTS_FILE=$(mktemp) +url="$PULP_URL/api/v3/content/deb/packages/?$( + printf 'repository_version=%s&pulp_label_select=%s&limit=1000' \ + "$(jq -rn --arg v "$VERSION_HREF" '$v | @uri')" \ + "$(jq -rn --arg v "module=$MODULE_NAME" '$v | @uri')" +)" +while [[ -n "$url" ]]; do + refresh_pulp_token + page=$(curl -fsSL -H "Authorization: Github $PULP_TOKEN" "$url") + echo "$page" | jq -c '.results[]' >> "$RESULTS_FILE" + url=$(echo "$page" | jq -r '.next // empty') +done + +# only the LATEST version of each package is promoted: the testing suite +# accumulates every delivered build (deb has no retention mechanism). +# Versions are compared segment by segment (numeric segments as numbers): a +# plain string max would rank "0.9" above "0.10", which bites the +# semver-like versions of some modules. Pipeline-built versions carry no +# epoch or tilde. +TESTING_SET_FILE=$(mktemp) +jq -R . "$TESTING_SHAS_FILE" | jq -s 'map({key: ., value: true}) | from_entries' > "$TESTING_SET_FILE" PACKAGES=$( - echo "$RESPONSE" | \ - jq --arg testing_path "$TESTING_POOL_PATH/" --arg distrib_name "$PACKAGE_DISTRIB_NAME" \ - '[.results[] | select((.relative_path | startswith($testing_path)) and (.relative_path | contains($distrib_name)))]' + jq -s --slurpfile testing_set "$TESTING_SET_FILE" \ + 'def vkey: [scan("[0-9]+|[^0-9]+") | (tonumber? // .)]; + [.[] | select($testing_set[0][.sha256])] + | group_by(.package, .architecture) | map(max_by(.version | vkey))' \ + "$RESULTS_FILE" ) +rm -f "$RESULTS_FILE" "$TESTING_SHAS_FILE" "$TESTING_SET_FILE" PACKAGES_COUNT=$(echo "$PACKAGES" | jq 'length') if [[ "$PACKAGES_COUNT" -eq 0 ]]; then - echo "::error::Nothing to promote, no package of module $MODULE_NAME found in $REPOSITORY_NAME ($TESTING_POOL_PATH/)" + echo "::error::Nothing to promote, no package of module $MODULE_NAME found in the $TESTING_SUITE suite of $REPOSITORY_NAME" exit 1 fi -echo "[INFO] $PACKAGES_COUNT packages of module $MODULE_NAME found in $REPOSITORY_NAME ($TESTING_POOL_PATH/)" +echo "[INFO] $PACKAGES_COUNT packages of module $MODULE_NAME found in the $TESTING_SUITE suite of $REPOSITORY_NAME" if [[ "$STABILITY" != "stable" ]]; then echo "[INFO] Dry run, $PACKAGES_COUNT packages would be promoted to $STABLE_SUITE/main" exit 0 fi -REPOSITORY_HREF=$(pulp deb repository show --name "$REPOSITORY_NAME" | jq -r '.pulp_href') +STABLE_REPOSITORY_HREF=$(pulp deb repository show --name "$STABLE_REPOSITORY_NAME" | jq -r '.pulp_href') mkdir -p promoted-packages # module label, built with jq (safe escaping) — consistent with the delivery scripts PULP_LABELS=$(jq -cn --arg mod "$MODULE_NAME" '{"module": $mod}') +# emit the release-component hrefs a package is associated with +lookup_prcs() { + local package_href=$1 + curl -fsSL --retry 3 --retry-delay 5 -H "Authorization: Github $PULP_TOKEN" -G \ + --data-urlencode "package=$package_href" \ + --data-urlencode "limit=100" \ + "$PULP_URL/api/v3/content/deb/package_release_components/" \ + | jq -r '.results[].release_component' +} + +# candidates already published in the stable suite need no new association; +# when EVERYTHING already reached stable (a rerun after the modify), only the +# publication may be missing: republish and stop. +UNPROMOTED_HREFS=() +while read -r PACKAGE; do + sha=$(echo "$PACKAGE" | jq -r '.sha256') + if ! grep -qxF "$sha" "$STABLE_SHAS_FILE"; then + UNPROMOTED_HREFS+=("$(echo "$PACKAGE" | jq -r '.pulp_href')") + fi +done < <(echo "$PACKAGES" | jq -c '.[]') +rm -f "$STABLE_SHAS_FILE" +if ((${#UNPROMOTED_HREFS[@]} == 0)); then + echo "[INFO] All $PACKAGES_COUNT package(s) are already promoted to $STABLE_SUITE; republishing only" + while read -r PACKAGE; do + manifest_add "$(echo "$PACKAGE" | jq -c \ + --arg repository "$STABLE_REPOSITORY_NAME" --arg base_path "$STABLE_BASE_PATH" \ + --arg suite "$STABLE_SUITE" \ + '{filename: (.relative_path | sub(".*/"; "")), name: .package, version, arch: .architecture, sha256, repository: $repository, base_path: $base_path, suite: $suite, relative_path}')" + done < <(echo "$PACKAGES" | jq -c '.[]') + create_publication deb "$STABLE_REPOSITORY_NAME" --structured + echo "::notice::Packages are available with: deb $PULP_CONTENT_URL/$STABLE_BASE_PATH/ $STABLE_SUITE main" + manifest_write "$MODULE_NAME" "${DISTRIB:-}" "deb" "$STABILITY" "promote" "$PULP_CONTENT_URL" + exit 0 +fi + +# batched promote, mirroring the batched delivery: the packages already exist +# as content units in the repository (testing suite), so promoting is only a +# matter of stable suite associations. The FIRST package of each architecture +# is promoted through the legacy re-upload path - it get_or_creates the stable +# ReleaseComponent and ReleaseArchitecture, which the ci user cannot create +# nor list directly - then the stable release-component href is deduced from +# its associations, every other association is created synchronously, and one +# modify adds the whole batch to the repository. +declare -A ARCH_SEEN=() +LEGACY_PACKAGES=() +BATCH_PACKAGES=() while read -r PACKAGE; do + ARCH=$(echo "$PACKAGE" | jq -r '.architecture') + if [[ -z "${ARCH_SEEN[$ARCH]+set}" ]]; then + ARCH_SEEN[$ARCH]=1 + LEGACY_PACKAGES+=("$PACKAGE") + else + BATCH_PACKAGES+=("$PACKAGE") + fi +done < <(echo "$PACKAGES" | jq -c '.[]') + +# the stable release component is deduced from the association the legacy +# re-upload of the FIRST reference creates: capture its association set +# before the upload so the new one stands out as the difference. +refresh_pulp_token +LEGACY_REF_HREF=$(echo "${LEGACY_PACKAGES[0]}" | jq -r '.pulp_href') +LEGACY_REF_BEFORE=$(lookup_prcs "$LEGACY_REF_HREF" | sort) + +for PACKAGE in "${LEGACY_PACKAGES[@]}"; do RELATIVE_PATH=$(echo "$PACKAGE" | jq -r '.relative_path') SHA256=$(echo "$PACKAGE" | jq -r '.sha256') ARCH=$(echo "$PACKAGE" | jq -r '.architecture') @@ -74,7 +224,7 @@ while read -r PACKAGE; do # layout, not their upload relative_path, so resolve the published location # from the testing suite Packages indexed by checksum to download the file FILENAME=$( - curl -fsSL "$PULP_CONTENT_URL/$BASE_PATH/dists/$TESTING_SUITE/main/binary-$ARCH/Packages" | + content_curl -fsSL --retry 3 --retry-delay 5 "$PULP_CONTENT_URL/$BASE_PATH/dists/$TESTING_SUITE/main/binary-$ARCH/Packages" | awk -v sha="$SHA256" 'BEGIN { RS = ""; FS = "\n" } index($0, "SHA256: " sha) { for (i = 1; i <= NF; i++) if ($i ~ /^Filename: /) { sub(/^Filename: /, "", $i); print $i } }' ) if [[ -z "$FILENAME" ]]; then @@ -83,40 +233,205 @@ while read -r PACKAGE; do fi echo "[INFO] Downloading $PULP_CONTENT_URL/$BASE_PATH/$FILENAME" - curl -fsSL -o "$FILE" "$PULP_CONTENT_URL/$BASE_PATH/$FILENAME" + content_curl -fsSL --retry 3 --retry-delay 5 -o "$FILE" "$PULP_CONTENT_URL/$BASE_PATH/$FILENAME" # re-upload the same bytes with the SAME relative_path: pulp reuses the - # existing content unit (a different relative_path would create a second unit - # that evicts the first, as pulp deduplicates by name+version+architecture - # repository wide) and only adds the stable suite association. an upload is - # required because pulp_deb rejects direct PackageReleaseComponent creation. - echo "[INFO] Promoting $FILE_NAME to $STABLE_SUITE/main" - TASK_HREF=$( - pulp_upload \ - -F "file=@\"$FILE\"" \ - -F "relative_path=$RELATIVE_PATH" \ - -F "distribution=$STABLE_SUITE" \ - -F "component=main" \ - -F "repository=$REPOSITORY_HREF" \ - -F "pulp_labels=$PULP_LABELS" \ - "$PULP_URL/api/v3/content/deb/packages/" - ) - wait_task "$TASK_HREF" - - # record the promoted package (with its stable suite coordinates) so the - # verification step verifies exactly this set against the stable suite - NAME=$(echo "$PACKAGE" | jq -r '.package') - VERSION=$(echo "$PACKAGE" | jq -r '.version') - manifest_add "$(jq -cn \ - --arg filename "$FILE_NAME" --arg name "$NAME" --arg version "$VERSION" \ - --arg arch "$ARCH" --arg sha256 "$SHA256" --arg repository "$REPOSITORY_NAME" \ - --arg base_path "$BASE_PATH" --arg suite "$STABLE_SUITE" --arg relative_path "$RELATIVE_PATH" \ - '{filename:$filename,name:$name,version:$version,arch:$arch,sha256:$sha256,repository:$repository,base_path:$base_path,suite:$suite,relative_path:$relative_path}')" + # existing content unit and only adds the stable suite association, + # creating the stable ReleaseComponent/ReleaseArchitecture on the way + # retried: the legacy path creates a repository version and can lose the + # version race against a concurrent promotion or delivery (wait_task_race + # rc 2); re-uploading converges, content and structure are get_or_created + for legacy_attempt in 1 2 3; do + echo "[INFO] Promoting $FILE_NAME to $STABLE_SUITE/main [legacy path, attempt $legacy_attempt]" + TASK_HREF=$( + pulp_upload \ + -F "file=@\"$FILE\"" \ + -F "relative_path=$RELATIVE_PATH" \ + -F "distribution=$STABLE_SUITE" \ + -F "component=main" \ + -F "repository=$STABLE_REPOSITORY_HREF" \ + -F "pulp_labels=$PULP_LABELS" \ + "$PULP_URL/api/v3/content/deb/packages/" + ) + wait_task_race "$TASK_HREF" && rc=0 || rc=$? + if [[ $rc -eq 0 ]]; then + break + elif [[ $rc -eq 2 && $legacy_attempt -lt 3 ]]; then + echo "[WARN] Legacy promotion of $FILE_NAME lost the repository-version race, retrying" + sleep $((legacy_attempt * 15)) + else + echo "::error::Legacy promotion of $FILE_NAME failed" + exit 1 + fi + done +done + +if ((${#BATCH_PACKAGES[@]} > 0)); then + # the stable release component is the association the legacy re-upload just + # added to the reference package. On a rerun the association pre-exists and + # the difference is empty: the reference then carries every suite it belongs + # to (stable, testing, possibly unstable for reused content), and the + # stable one is isolated by subtracting the associations of not-yet-promoted + # candidates - those belong to every suite of the reference EXCEPT stable. + refresh_pulp_token + LEGACY_REF_AFTER=$(lookup_prcs "$LEGACY_REF_HREF" | sort) + STABLE_RC_SET=$(comm -13 <(echo "$LEGACY_REF_BEFORE") <(echo "$LEGACY_REF_AFTER") | grep . || true) + if [[ $(echo "$STABLE_RC_SET" | grep -c .) -ne 1 ]]; then + STABLE_RC_SET=$(echo "$LEGACY_REF_AFTER" | grep . || true) + for u_href in "${UNPROMOTED_HREFS[@]}"; do + [[ "$u_href" == "$LEGACY_REF_HREF" ]] && continue + [[ $(echo "$STABLE_RC_SET" | grep -c .) -le 1 ]] && break + u_prcs=$(lookup_prcs "$u_href" | sort) + STABLE_RC_SET=$(comm -23 <(echo "$STABLE_RC_SET") <(echo "$u_prcs") | grep . || true) + done + fi + STABLE_RC=$(echo "$STABLE_RC_SET" | grep . | head -1) + if [[ (-z "$STABLE_RC" || $(echo "$STABLE_RC_SET" | grep -c .) -ne 1) && "$STABLE_REPOSITORY_NAME" != "$REPOSITORY_NAME" ]]; then + # dedicated stable repository: it only ever receives stable suite + # associations, so any association it already holds points to the stable + # release component. Covers reruns of a promotion interrupted between the + # association creation and the repository modify, where every + # before/after diff above is empty. + refresh_pulp_token + STABLE_LATEST=$(pulp deb repository show --name "$STABLE_REPOSITORY_NAME" | jq -r '.latest_version_href') + STABLE_RC=$( + curl -fsSL --retry 3 --retry-delay 5 -H "Authorization: Github $PULP_TOKEN" \ + "$PULP_URL/api/v3/content/deb/package_release_components/?$( + printf 'repository_version=%s&limit=1' "$(jq -rn --arg v "$STABLE_LATEST" '$v | @uri')" + )" | jq -r '.results[0].release_component // empty' + ) + [[ -n "$STABLE_RC" ]] && STABLE_RC_SET="$STABLE_RC" + fi + if [[ -z "$STABLE_RC" || $(echo "$STABLE_RC_SET" | grep -c .) -ne 1 ]]; then + # a promotion interrupted after its repository modify only misses the + # publication: republish and re-check every candidate before giving up + echo "[WARN] Cannot deduce the $STABLE_SUITE/main release component (got: ${STABLE_RC_SET:-none}); republishing to check for an interrupted promotion" + create_publication deb "$STABLE_REPOSITORY_NAME" --structured + RECHECK_SHAS_FILE=$(suite_sha_file "$STABLE_BASE_PATH" "$STABLE_SUITE") + MISSING_COUNT=0 + while read -r sha; do + grep -qxF "$sha" "$RECHECK_SHAS_FILE" || MISSING_COUNT=$((MISSING_COUNT + 1)) + done < <(echo "$PACKAGES" | jq -r '.[].sha256') + rm -f "$RECHECK_SHAS_FILE" + if ((MISSING_COUNT == 0)); then + echo "[INFO] Every candidate package is published in $STABLE_SUITE; the interrupted promotion only missed the publication" + while read -r PACKAGE; do + manifest_add "$(echo "$PACKAGE" | jq -c \ + --arg repository "$STABLE_REPOSITORY_NAME" --arg base_path "$STABLE_BASE_PATH" \ + --arg suite "$STABLE_SUITE" \ + '{filename: (.relative_path | sub(".*/"; "")), name: .package, version, arch: .architecture, sha256, repository: $repository, base_path: $base_path, suite: $suite, relative_path}')" + done < <(echo "$PACKAGES" | jq -c '.[]') + echo "::notice::Packages are available with: deb $PULP_CONTENT_URL/$STABLE_BASE_PATH/ $STABLE_SUITE main" + manifest_write "$MODULE_NAME" "${DISTRIB:-}" "deb" "$STABILITY" "promote" "$PULP_CONTENT_URL" + exit 0 + fi + echo "::error::Cannot deduce the $STABLE_SUITE/main release component (got: ${STABLE_RC_SET:-none}) and $MISSING_COUNT candidate package(s) are missing from the published $STABLE_SUITE suite. A previous promotion likely left the suite associations without the repository content; deliver a rebuilt (fresh) build or add the packages to the stable repository manually." + exit 1 + fi + echo "[INFO] Release component of $STABLE_SUITE/main: $STABLE_RC" + + # create the stable suite associations: synchronous creates (201 with the + # unit, no task), parallelized; a failed create (existing association on a + # re-run) falls back to a lookup + echo "[INFO] Associating ${#BATCH_PACKAGES[@]} package(s) with $STABLE_SUITE/main" + PRC_DIR=$(mktemp -d) + MAX_PARALLEL=8 + for i in "${!BATCH_PACKAGES[@]}"; do + if ((i % 40 == 0)); then + refresh_pulp_token + fi + ( + refresh_pulp_token + package_href=$(echo "${BATCH_PACKAGES[$i]}" | jq -r '.pulp_href') + response=$( + curl -sSL --retry 3 --retry-delay 5 -w '\n%{http_code}' -H "Authorization: Github $PULP_TOKEN" \ + -X POST -H "Content-Type: application/json" \ + -d "{\"package\": \"$package_href\", \"release_component\": \"$STABLE_RC\"}" \ + "$PULP_URL/api/v3/content/deb/package_release_components/" + ) || response=$'\n000' + code="${response##*$'\n'}" + body="${response%$'\n'*}" + href="" + if [[ "$code" == 2* ]]; then + # tolerate a non-json body (gateway error page behind a 2xx): a jq + # failure inside the substitution would silently kill this subshell + href=$(echo "$body" | jq -r '.pulp_href // empty' 2>/dev/null) || href="" + fi + if [[ -z "$href" ]]; then + # the api answers 500 on a duplicate synchronous content create; on a + # rerun the stable association simply pre-exists (see the lookup below) + echo "[INFO] $(echo "${BATCH_PACKAGES[$i]}" | jq -r '.package'): stable association create answered HTTP $code, falling back to the lookup (expected on a rerun)" + # the association exists (the create just hit its unique constraint): + # empty pages are retried, the api has been observed answering stale + # reads for content committed seconds earlier + for lookup_attempt in 1 2 3 4 5; do + href=$( + curl -fsSL --retry 3 --retry-delay 5 -H "Authorization: Github $PULP_TOKEN" -G \ + --data-urlencode "package=$package_href" \ + --data-urlencode "release_component=$STABLE_RC" \ + --data-urlencode "limit=1" \ + "$PULP_URL/api/v3/content/deb/package_release_components/" \ + | jq -r '.results[0].pulp_href // empty' + ) || href="" + [[ -n "$href" ]] && break + sleep $((lookup_attempt * 3)) + refresh_pulp_token + done + fi + printf '%s\n%s' "$package_href" "$href" > "$PRC_DIR/$i.pair" + ) & + while (($(jobs -rp | wc -l) >= MAX_PARALLEL)); do + wait -n || true + done + done + wait || true + + ADD_UNITS_FILE=$(mktemp) + for i in "${!BATCH_PACKAGES[@]}"; do + if [[ ! -s "$PRC_DIR/$i.pair" ]] || [[ -z "$(sed -n '2p' "$PRC_DIR/$i.pair")" ]]; then + echo "::error::Stable suite association failed for package $(echo "${BATCH_PACKAGES[$i]}" | jq -r '.package') (see the worker error above)" + exit 1 + fi + cat "$PRC_DIR/$i.pair" >> "$ADD_UNITS_FILE" + echo >> "$ADD_UNITS_FILE" + done + rm -rf "$PRC_DIR" + + echo "[INFO] Adding ${#BATCH_PACKAGES[@]} package(s) and their suite associations to $STABLE_REPOSITORY_NAME in a single task" + refresh_pulp_token + # body through a file: thousands of hrefs exceed the argv limit + ADD_BODY_FILE=$(mktemp) + jq -R . "$ADD_UNITS_FILE" | jq -cs '{add_content_units: [.[] | select(. != "")]}' > "$ADD_BODY_FILE" + rm -f "$ADD_UNITS_FILE" + # retried like the legacy uploads: the modify also creates a repository + # version and can lose the same race + for modify_attempt in 1 2 3; do + MODIFY_TASK=$(start_modify_task "$PULP_URL${STABLE_REPOSITORY_HREF}modify/" "$ADD_BODY_FILE") + wait_task_race "$MODIFY_TASK" && rc=0 || rc=$? + if [[ $rc -eq 0 ]]; then + break + elif [[ $rc -eq 2 && $modify_attempt -lt 3 ]]; then + echo "[WARN] Stable repository modify lost the repository-version race, retrying" + sleep $((modify_attempt * 15)) + else + echo "::error::Stable repository modify failed" + exit 1 + fi + done +fi + +# record the promoted packages (with their stable suite coordinates) so the +# verification step verifies exactly this set against the stable suite +while read -r PACKAGE; do + manifest_add "$(echo "$PACKAGE" | jq -c \ + --arg repository "$STABLE_REPOSITORY_NAME" --arg base_path "$STABLE_BASE_PATH" \ + --arg suite "$STABLE_SUITE" \ + '{filename: (.relative_path | sub(".*/"; "")), name: .package, version, arch: .architecture, sha256, repository: $repository, base_path: $base_path, suite: $suite, relative_path}')" done < <(echo "$PACKAGES" | jq -c '.[]') -echo "[INFO] Publishing repository $REPOSITORY_NAME" -pulp deb publication create --repository "$REPOSITORY_NAME" --structured >/dev/null +echo "[INFO] Publishing repository $STABLE_REPOSITORY_NAME" +create_publication deb "$STABLE_REPOSITORY_NAME" --structured -echo "::notice::Packages are available with: deb $PULP_CONTENT_URL/$BASE_PATH/ $STABLE_SUITE main" +echo "::notice::Packages are available with: deb $PULP_CONTENT_URL/$STABLE_BASE_PATH/ $STABLE_SUITE main" manifest_write "$MODULE_NAME" "${DISTRIB:-}" "deb" "$STABILITY" "promote" "$PULP_CONTENT_URL" diff --git a/.github/actions/promote-to-stable-pulp/promote-rpm.sh b/.github/actions/promote-to-stable-pulp/promote-rpm.sh index 8f5c404b183..01bdd84c936 100755 --- a/.github/actions/promote-to-stable-pulp/promote-rpm.sh +++ b/.github/actions/promote-to-stable-pulp/promote-rpm.sh @@ -28,22 +28,29 @@ for ARCH in noarch x86_64; do # packages of the module are identified by the label set at delivery time; # keep both the href list (for the content modify call) and the package # identity (name/version/release/arch/filename) to feed the promotion manifest - RESPONSE=$( - curl -fsSL -H "Authorization: Github $PULP_TOKEN" -G \ - --data-urlencode "repository_version=$VERSION_HREF" \ - --data-urlencode "pulp_label_select=module=$MODULE_NAME" \ - --data-urlencode "limit=1000" \ - "$PULP_URL/api/v3/content/rpm/packages/" - ) - - # fail on a truncated page: silently promoting a subset of the module's - # packages would publish an incomplete stable repository - if [[ $(echo "$RESPONSE" | jq '.count > (.results | length)') == "true" ]]; then - echo "::error::Package query on $TESTING_REPOSITORY_NAME is truncated ($(echo "$RESPONSE" | jq -r '.results | length')/$(echo "$RESPONSE" | jq -r '.count') results); pagination is required" - exit 1 - fi - - RESULTS=$(echo "$RESPONSE" | jq '[.results[] | {pulp_href, name, version, release, arch, location_href, sha256}]') + # paginate: the testing repository keeps every delivered version, so the + # module listing can exceed a single page + RESULTS_FILE=$(mktemp) + url="$PULP_URL/api/v3/content/rpm/packages/?$( + printf 'repository_version=%s&pulp_label_select=%s&limit=1000' \ + "$(jq -rn --arg v "$VERSION_HREF" '$v | @uri')" \ + "$(jq -rn --arg v "module=$MODULE_NAME" '$v | @uri')" + )" + while [[ -n "$url" ]]; do + refresh_pulp_token + page=$(curl -fsSL -H "Authorization: Github $PULP_TOKEN" "$url") + echo "$page" | jq -c '.results[]' >> "$RESULTS_FILE" + url=$(echo "$page" | jq -r '.next // empty') + done + + # only the LATEST build of each package is promoted. Versions are compared + # segment by segment (numeric segments as numbers): a plain string max + # would rank "0.9" above "0.10", which bites the semver-like versions of + # some modules. Pipeline-built versions carry no epoch or tilde. + RESULTS=$(jq -s 'def vkey: [scan("[0-9]+|[^0-9]+") | (tonumber? // .)]; + [.[] | {pulp_href, name, version, release, arch, location_href, sha256}] + | group_by(.name, .arch) | map(max_by([(.version | vkey), (.release | vkey)]))' "$RESULTS_FILE") + rm -f "$RESULTS_FILE" CONTENT=$(echo "$RESULTS" | jq '[.[].pulp_href]') ARCH_PACKAGES_COUNT=$(echo "$CONTENT" | jq 'length') @@ -89,16 +96,13 @@ for ARCH in noarch x86_64; do echo "[INFO] Promoting $ARCH_PACKAGES_COUNT packages to $STABLE_REPOSITORY_NAME" # pulp-cli repository content modify does not resolve content by pulp_href, use the api directly - TASK_HREF=$( - curl -fsSL -H "Authorization: Github $PULP_TOKEN" \ - -X POST -H "Content-Type: application/json" \ - -d "{\"add_content_units\": $CONTENT}" \ - "$PULP_URL${STABLE_REPOSITORY_HREF}modify/" | jq -r '.task' - ) + ADD_BODY_FILE=$(mktemp) + echo "$CONTENT" | jq -c '{add_content_units: .}' > "$ADD_BODY_FILE" + TASK_HREF=$(start_modify_task "$PULP_URL${STABLE_REPOSITORY_HREF}modify/" "$ADD_BODY_FILE") wait_task "$TASK_HREF" echo "[INFO] Publishing repository $STABLE_REPOSITORY_NAME" - pulp rpm publication create --repository "$STABLE_REPOSITORY_NAME" >/dev/null + create_publication rpm "$STABLE_REPOSITORY_NAME" # record the promoted packages (with their stable target coordinates) so the # verification step verifies exactly this set against the stable repo diff --git a/.github/actions/pulp-repository-properties/action.yml b/.github/actions/pulp-repository-properties/action.yml index 7f286cc849d..db74cb83bd6 100644 --- a/.github/actions/pulp-repository-properties/action.yml +++ b/.github/actions/pulp-repository-properties/action.yml @@ -8,8 +8,9 @@ inputs: description: "The distribution used for packaging" required: true version: - description: "Centreon packaged major version" - required: true + description: "Centreon packaged major version (not used by the plugins repository family)" + required: false + default: "" stability: description: "The package stability (stable, testing, unstable)" required: true @@ -24,6 +25,10 @@ inputs: description: "Delivery to feature or standard repository" required: false default: "standard" + repository_name: + description: "Target repository family: standard, standard-internal, business, business-internal or plugins" + required: false + default: "standard" outputs: package_extension: description: "Package extension according to the distribution" @@ -67,6 +72,12 @@ outputs: stable_suite: description: "Apt suite of stable deb packages" value: ${{ steps.properties.outputs.stable_suite }} + stable_repository_name: + description: "Repository holding the stable packages (dedicated for deb plugins, the delivery repository otherwise)" + value: ${{ steps.properties.outputs.stable_repository_name }} + stable_base_path: + description: "Base path serving the stable packages" + value: ${{ steps.properties.outputs.stable_base_path }} pool_path: description: "Module scoped pool path of the deb delivery" value: ${{ steps.properties.outputs.pool_path }} @@ -98,5 +109,6 @@ runs: RELEASE_TYPE: ${{ inputs.release_type }} IS_CLOUD: ${{ inputs.is_cloud }} DELIVERY_TYPE: ${{ inputs.delivery_type }} + REPOSITORY_TYPE: ${{ inputs.repository_name }} GH_HEAD_REF: ${{ github.head_ref || github.ref_name }} run: ${{ github.action_path }}/properties.sh diff --git a/.github/actions/pulp-repository-properties/properties.sh b/.github/actions/pulp-repository-properties/properties.sh index 859e5f05f14..719872b2cd5 100755 --- a/.github/actions/pulp-repository-properties/properties.sh +++ b/.github/actions/pulp-repository-properties/properties.sh @@ -1,27 +1,58 @@ #!/usr/bin/env bash set -euo pipefail -if [[ -z "$MODULE_NAME" || -z "$DISTRIB" || -z "$VERSION" || -z "$STABILITY" || -z "$IS_CLOUD" ]]; then +if [[ -z "$MODULE_NAME" || -z "$DISTRIB" || -z "$STABILITY" || -z "$IS_CLOUD" ]]; then echo "::error::some mandatory inputs are empty, please check the logs." exit 1 fi +# repository_name selects the standard (open-source), business (paid) or plugins +# repository family. standard/business have "-internal" variants: either a cloud +# context (is_cloud) or an explicit "-internal" suffix means the internal repo. +# plugins repositories are not versioned (packages carry their own date-based +# versioning), so the version input is only required for the other families. +REPOSITORY_TYPE="${REPOSITORY_TYPE:-standard}" +case "$REPOSITORY_TYPE" in + standard | standard-internal) REPO_BASE="standard" ;; + business | business-internal) REPO_BASE="business" ;; + plugins) REPO_BASE="plugins" ;; + *) + echo "::error::Unsupported repository_name: $REPOSITORY_TYPE" + exit 1 + ;; +esac + +if [[ "$REPO_BASE" != "plugins" && -z "$VERSION" ]]; then + echo "::error::version input is mandatory for the $REPOSITORY_TYPE repository family." + exit 1 +fi + # parse-distrib emits the el family either generically ("el") or per version # ("el7"/"el8"/"el9"/"el10" on centreon-collect); normalize it to "el" so the # family checks below are portable across both conventions. case "$DISTRIB_FAMILY" in el | el[0-9]*) - ROOT_REPO="rpm-standard" + FAMILY_PREFIX="rpm-" DISTRIB_FAMILY="el" ;; - debian) ROOT_REPO="apt-standard" ;; - ubuntu) ROOT_REPO="ubuntu-standard" ;; + debian) FAMILY_PREFIX="apt-" ;; + ubuntu) FAMILY_PREFIX="ubuntu-" ;; *) echo "::error::Unsupported distribution family: $DISTRIB_FAMILY" exit 1 ;; esac +ROOT_REPO="${FAMILY_PREFIX}${REPO_BASE}" + +# business (paid) rpm content is served under an opaque path segment, mirroring +# the Artifactory layout; only rpm business repos use it (deb business does not). +BUSINESS_HASH="1a97ff9985262bf3daf7a0919f9c59a6" +HASH_SEGMENT="" +if [[ "$REPO_BASE" == "business" && "$DISTRIB_FAMILY" == "el" ]]; then + HASH_SEGMENT="/$BUSINESS_HASH" +fi + TESTING_SEGMENT="testing" TESTING_POOL_SEGMENT="testing" if [[ "$RELEASE_TYPE" == "release" || "$RELEASE_TYPE" == "hotfix" ]]; then @@ -50,7 +81,54 @@ POOL_PATH="" TESTING_POOL_PATH="" STABLE_POOL_PATH="" -if [[ "$DELIVERY_TYPE" == "feature" ]]; then +if [[ "$REPO_BASE" == "plugins" ]]; then + # plugins repositories are not versioned and use their own testing layout: a + # bare "testing" segment for releases and "testing-hotfix" for hotfixes, + # mirroring the artifactory rpm-plugins// layout. deb + # plugins share one apt-plugins/ubuntu-plugins repo with - + # suites. + if [[ "$DELIVERY_TYPE" == "feature" ]]; then + echo "::notice::Feature delivery is not supported for plugins packages, skipping delivery." + echo "skip_delivery=true" >> "$GITHUB_OUTPUT" + exit 0 + fi + + TESTING_SEGMENT="testing" + TESTING_POOL_SEGMENT="testing" + if [[ "$RELEASE_TYPE" == "hotfix" ]]; then + TESTING_SEGMENT="testing-hotfix" + TESTING_POOL_SEGMENT="testing/hotfix" + fi + + STABILITY_SEGMENT="$STABILITY" + POOL_SEGMENT="$STABILITY" + if [[ "$STABILITY" == "testing" ]]; then + STABILITY_SEGMENT="$TESTING_SEGMENT" + POOL_SEGMENT="$TESTING_POOL_SEGMENT" + fi + + if [[ "$DISTRIB_FAMILY" == "el" ]]; then + REPOSITORY_PREFIX="$ROOT_REPO-$DISTRIB-$STABILITY_SEGMENT" + BASE_PATH_PREFIX="$ROOT_REPO/$DISTRIB/$STABILITY_SEGMENT" + TESTING_REPOSITORY_PREFIX="$ROOT_REPO-$DISTRIB-$TESTING_SEGMENT" + STABLE_REPOSITORY_PREFIX="$ROOT_REPO-$DISTRIB-stable" + STABLE_BASE_PATH_PREFIX="$ROOT_REPO/$DISTRIB/stable" + else + REPOSITORY_NAME="$ROOT_REPO" + BASE_PATH="$ROOT_REPO" + SUITE="$DISTRIB-$STABILITY_SEGMENT" + TESTING_SUITE="$DISTRIB-$TESTING_SEGMENT" + # stable lives in a DEDICATED repository with plain-codename suites, so + # the client apt configuration stays identical to the artifactory one + # (deb .../apt-plugins-stable main) + STABLE_REPOSITORY_NAME="$ROOT_REPO-stable" + STABLE_BASE_PATH="$ROOT_REPO-stable" + STABLE_SUITE="$DISTRIB" + POOL_PATH="pool/$POOL_SEGMENT/$MODULE_NAME" + TESTING_POOL_PATH="pool/$TESTING_POOL_SEGMENT/$MODULE_NAME" + STABLE_POOL_PATH="pool/stable/$MODULE_NAME" + fi +elif [[ "$DELIVERY_TYPE" == "feature" ]]; then if [[ "$DISTRIB_FAMILY" != "el" ]]; then echo "::notice::Feature delivery is not supported for $DISTRIB_FAMILY packages, skipping delivery." echo "skip_delivery=true" >> "$GITHUB_OUTPUT" @@ -64,18 +142,18 @@ if [[ "$DELIVERY_TYPE" == "feature" ]]; then fi REPOSITORY_PREFIX="$ROOT_REPO-feature-$FEATURE_TICKET-$VERSION-$DISTRIB-$STABILITY" - BASE_PATH_PREFIX="$ROOT_REPO-feature/$FEATURE_TICKET/$VERSION/$DISTRIB/$STABILITY" + BASE_PATH_PREFIX="$ROOT_REPO-feature$HASH_SEGMENT/$FEATURE_TICKET/$VERSION/$DISTRIB/$STABILITY" else - if [[ "$IS_CLOUD" == "true" ]]; then + if [[ "$IS_CLOUD" == "true" || "$REPOSITORY_TYPE" == *-internal ]]; then ROOT_REPO="$ROOT_REPO-internal" fi if [[ "$DISTRIB_FAMILY" == "el" ]]; then REPOSITORY_PREFIX="$ROOT_REPO-$VERSION-$DISTRIB-$STABILITY_SEGMENT" - BASE_PATH_PREFIX="$ROOT_REPO/$VERSION/$DISTRIB/$STABILITY_SEGMENT" + BASE_PATH_PREFIX="$ROOT_REPO$HASH_SEGMENT/$VERSION/$DISTRIB/$STABILITY_SEGMENT" TESTING_REPOSITORY_PREFIX="$ROOT_REPO-$VERSION-$DISTRIB-$TESTING_SEGMENT" STABLE_REPOSITORY_PREFIX="$ROOT_REPO-$VERSION-$DISTRIB-stable" - STABLE_BASE_PATH_PREFIX="$ROOT_REPO/$VERSION/$DISTRIB/stable" + STABLE_BASE_PATH_PREFIX="$ROOT_REPO$HASH_SEGMENT/$VERSION/$DISTRIB/stable" else REPOSITORY_NAME="$ROOT_REPO" BASE_PATH="$ROOT_REPO" @@ -88,6 +166,13 @@ else fi fi +# unless a dedicated stable repository was selected above, stable shares the +# delivery repository +STABLE_REPOSITORY_NAME="${STABLE_REPOSITORY_NAME:-$REPOSITORY_NAME}" +STABLE_BASE_PATH="${STABLE_BASE_PATH:-$BASE_PATH}" + +echo "[DEBUG] - repository_type: $REPOSITORY_TYPE" +echo "[DEBUG] - root_repo: $ROOT_REPO" echo "[DEBUG] - repository_prefix: $REPOSITORY_PREFIX" echo "[DEBUG] - base_path_prefix: $BASE_PATH_PREFIX" echo "[DEBUG] - testing_repository_prefix: $TESTING_REPOSITORY_PREFIX" @@ -114,6 +199,8 @@ echo "[DEBUG] - stable_pool_path: $STABLE_POOL_PATH" echo "suite=$SUITE" echo "testing_suite=$TESTING_SUITE" echo "stable_suite=$STABLE_SUITE" + echo "stable_repository_name=$STABLE_REPOSITORY_NAME" + echo "stable_base_path=$STABLE_BASE_PATH" echo "pool_path=$POOL_PATH" echo "testing_pool_path=$TESTING_POOL_PATH" echo "stable_pool_path=$STABLE_POOL_PATH" diff --git a/.github/actions/setup-pulp-cli/action.yml b/.github/actions/setup-pulp-cli/action.yml index 44525b501a7..19365c4ec3d 100644 --- a/.github/actions/setup-pulp-cli/action.yml +++ b/.github/actions/setup-pulp-cli/action.yml @@ -63,9 +63,16 @@ runs: exit 1 fi - # keep the token out of any log and expose it to the next steps + # keep the token out of any log and expose it to the next steps. + # the issuance time and audience let the api helpers refresh the token + # mid-delivery: github oidc tokens expire ~5 minutes after issuance, + # which is shorter than a large delivery. echo "::add-mask::$PULP_TOKEN" - echo "PULP_TOKEN=$PULP_TOKEN" >> "$GITHUB_ENV" + { + echo "PULP_TOKEN=$PULP_TOKEN" + echo "PULP_TOKEN_ISSUED_AT=$(date +%s)" + echo "PULP_OIDC_AUDIENCE=$AUDIENCE" + } >> "$GITHUB_ENV" - name: Configure pulp-cli shell: bash diff --git a/.github/scripts/pulp/api.sh b/.github/scripts/pulp/api.sh index 7faf3ace2ce..19dfc6936bb 100644 --- a/.github/scripts/pulp/api.sh +++ b/.github/scripts/pulp/api.sh @@ -2,13 +2,56 @@ # Shared pulp api helpers for the delivery/promotion scripts. Source this file # like manifest.sh; PULP_URL and PULP_TOKEN must be set by the caller. +# the github actions oidc token expires ~5 minutes after issuance, which is +# shorter than a large delivery: refresh it before it goes stale whenever the +# job can mint tokens (id-token: write context + audience exported by +# setup-pulp-cli). The fresh token is propagated to the following steps +# (GITHUB_ENV) and to the pulp-cli config so the pulp commands keep working. +refresh_pulp_token() { + local now token + now=$(date +%s) + if ((now - ${PULP_TOKEN_ISSUED_AT:-0} < 240)); then + return 0 + fi + if [[ -z "${ACTIONS_ID_TOKEN_REQUEST_URL:-}" || -z "${ACTIONS_ID_TOKEN_REQUEST_TOKEN:-}" || -z "${PULP_OIDC_AUDIENCE:-}" ]]; then + return 0 + fi + token=$( + curl -fsSL --retry 5 --retry-delay 3 --retry-all-errors \ + -H "Authorization: bearer $ACTIONS_ID_TOKEN_REQUEST_TOKEN" \ + -G --data-urlencode "audience=$PULP_OIDC_AUDIENCE" \ + "$ACTIONS_ID_TOKEN_REQUEST_URL" 2>/dev/null | jq -r '.value' 2>/dev/null + ) || token="" + if [[ -z "$token" || "$token" == "null" ]]; then + return 0 + fi + PULP_TOKEN="$token" + PULP_TOKEN_ISSUED_AT="$now" + # written to stderr: this function also runs inside command substitutions + # (pulp_upload), where a stdout echo would corrupt the captured output + echo "::add-mask::$token" >&2 + if [[ -n "${GITHUB_ENV:-}" ]]; then + { + echo "PULP_TOKEN=$token" + echo "PULP_TOKEN_ISSUED_AT=$now" + } >> "$GITHUB_ENV" + fi + if command -v pulp >/dev/null 2>&1; then + pulp config create --overwrite --base-url "$PULP_URL" --api-root "/" \ + --header "Authorization:Github $token" --timeout 0 >/dev/null 2>&1 || true + fi +} + # wait for a pulp api task to complete. a failed poll request (network blip, # api 5xx) is retried like an unexpected state instead of aborting the caller -# under set -e; the 200-attempt cap bounds the total wait (~10 min). +# under set -e; the 600-attempt cap bounds the total wait (~30 min): under a +# full-matrix delivery the publication of a large repository can sit several +# minutes in the worker queue before its ~5 min of actual execution. wait_task() { local task_href=$1 local state attempt - for ((attempt = 0; attempt < 200; attempt++)); do + for ((attempt = 0; attempt < 600; attempt++)); do + refresh_pulp_token state=$(curl -fsSL -H "Authorization: Github $PULP_TOKEN" "$PULP_URL$task_href" 2>/dev/null | jq -r '.state' 2>/dev/null) || state="" case "$state" in completed) @@ -23,15 +66,65 @@ wait_task() { ;; esac done - echo "::error::Task $task_href did not complete in time (~10 min)" + echo "::error::Task $task_href did not complete in time (~30 min)" return 1 } +# wait for a batch of pulp api tasks, polling the still-pending ones in sweeps. +# unlike wait_task, the guard aborts only when NO task completes for ~10 min: +# tasks of a repository are serialized server-side, so a long-but-draining +# queue is expected under concurrent deliveries and must not be mistaken for a +# hang. +wait_tasks() { + local pending=("$@") + local total=$# + local next=() stall=0 state href now last_report + last_report=$(date +%s) + while ((${#pending[@]} > 0)); do + refresh_pulp_token + next=() + for href in "${pending[@]}"; do + state=$(curl -fsSL -H "Authorization: Github $PULP_TOKEN" "$PULP_URL$href" 2>/dev/null | jq -r '.state' 2>/dev/null) || state="" + case "$state" in + completed) ;; + failed|canceled) + echo "::error::Task $href $state: $(curl -fsSL -H "Authorization: Github $PULP_TOKEN" "$PULP_URL$href" | jq -c '.error')" + return 1 + ;; + *) + next+=("$href") + ;; + esac + done + if ((${#next[@]} == ${#pending[@]})); then + stall=$((stall + 1)) + if ((stall >= 200)); then + echo "::error::${#pending[@]} task(s) still pending with no progress for ~10 min, e.g. ${pending[0]}" + return 1 + fi + else + stall=0 + fi + pending=("${next[@]+"${next[@]}"}") + # progress heartbeat, at most every 30s: shows how fast the server queue + # drains (sweeps over large batches can take minutes on their own) + now=$(date +%s) + if ((${#pending[@]} > 0 && now - last_report >= 30)); then + echo "[INFO] $((total - ${#pending[@]}))/$total task(s) completed" + last_report=$now + fi + if ((${#pending[@]} > 0)); then + sleep 3 + fi + done +} + # upload a package through the pulp api, with retry on transient failures # (concurrent deliveries can race on artifact creation), echoes the task href pulp_upload() { local attempt response http_code body - for attempt in 1 2 3; do + for attempt in 1 2 3 4 5; do + refresh_pulp_token response=$(curl -sS -H "Authorization: Github $PULP_TOKEN" -w $'\n%{http_code}' "$@" 2>/dev/null) || response="" http_code=${response##*$'\n'} body=${response%$'\n'*} @@ -39,9 +132,124 @@ pulp_upload() { echo "$body" | jq -r '.task' return 0 fi - echo "[WARN] upload attempt $attempt/3 failed (HTTP ${http_code:-network-error}), retrying..." >&2 - sleep $((attempt * 3)) + echo "[WARN] upload attempt $attempt/5 failed (HTTP ${http_code:-network-error}), retrying..." >&2 + sleep $((attempt * 5)) + done + echo "::error::Upload failed after 5 attempts (HTTP ${http_code:-network-error})" >&2 + return 1 +} + +# wait for a task, distinguishing a RETRYABLE failure from a genuine one: +# - lost repository-version race: concurrent legs of a SHARED deb repository +# collide on the version bookkeeping (duplicate (repository_id, number) +# insert, or the retention cleanup deleting a version another transaction +# still references) +# - "Worker has gone missing": the task worker pod was terminated mid-task +# (scale-down, node consolidation) +# Re-running the operation converges, so callers retry on rc 2. +# 0=completed, 2=retryable failure, 1=failed or timed out. +wait_task_race() { + local task_href=$1 body state error attempt + for ((attempt = 0; attempt < 600; attempt++)); do + refresh_pulp_token + body=$(curl -fsSL -H "Authorization: Github $PULP_TOKEN" "$PULP_URL$task_href" 2>/dev/null) || body="" + state=$(echo "$body" | jq -r '.state' 2>/dev/null) || state="" + case "$state" in + completed) + return 0 + ;; + failed | canceled) + error=$(echo "$body" | jq -c '.error // empty' 2>/dev/null) || error="" + if echo "$error" | grep -qE 'core_repositoryversion|Worker has gone missing'; then + return 2 + fi + # the "gone missing" reason lands in a separate field on some tasks + if echo "$body" | jq -r '.reason // empty' 2>/dev/null | grep -q 'Worker has gone missing'; then + return 2 + fi + echo "::error::Task $task_href $state: $error" + return 1 + ;; + *) + sleep 3 + ;; + esac + done + echo "::error::Task $task_href did not complete in time (~30 min)" + return 1 +} + +# start a repository modify task, with retry on transient gateway failures: +# it is the single most critical call of the flow (all uploaded content lands +# in the repository through it) and add_content_units is idempotent, so +# retrying is always safe. Echoes the task href. +start_modify_task() { + local url=$1 body_file=$2 attempt response http_code body + for attempt in 1 2 3 4 5; do + refresh_pulp_token + response=$(curl -sS -H "Authorization: Github $PULP_TOKEN" -w $'\n%{http_code}' \ + -X POST -H "Content-Type: application/json" \ + -d @"$body_file" "$url" 2>/dev/null) || response="" + http_code=${response##*$'\n'} + body=${response%$'\n'*} + if [[ "$http_code" == "202" ]]; then + echo "$body" | jq -r '.task' + return 0 + fi + echo "[WARN] modify attempt $attempt/5 on $url failed (HTTP ${http_code:-network-error}), retrying..." >&2 + sleep $((attempt * 5)) done - echo "::error::Upload failed after 3 attempts (HTTP ${http_code:-network-error})" >&2 + echo "::error::Repository modify failed after 5 attempts on $url (HTTP ${http_code:-network-error}): $(echo "$body" | head -c 300)" >&2 return 1 } + +# create a publication in the background and wait for it with re-authenticating +# polling: pulp-cli reads its token once at startup, so its built-in wait fails +# with "Authentication failed for tasks_read" as soon as the publication of a +# large repository outlives the OIDC token validity (~5 minutes) +# curl against the content endpoint with the CI credentials: the guarded +# *business-internal distributions authorize the download through the same +# GitHub OIDC authentication as the api (the CI user holds the guard +# downloader role); unguarded distributions ignore the header. +content_curl() { + refresh_pulp_token + curl -H "Authorization: Github $PULP_TOKEN" "$@" +} + +create_publication() { + local plugin=$1 repository=$2 + shift 2 + local out task attempt outer rc + # outer retry: the publication task itself can die server-side for + # retryable reasons (task worker terminated mid-task, version race) + for outer in 1 2 3; do + out="" + for attempt in 1 2 3; do + refresh_pulp_token + out=$(pulp --background "$plugin" publication create --repository "$repository" "$@" 2>&1) && break + echo "[WARN] publication start attempt $attempt/3 failed for $repository, retrying..." >&2 + sleep $((attempt * 10)) + out="" + done + if [[ -z "$out" ]]; then + echo "::error::Cannot start the publication of repository $repository" + return 1 + fi + task=$(grep -oPm1 '/api/v3/tasks/[0-9a-f-]+/' <<< "$out" || true) + if [[ -z "$task" ]]; then + echo "$out" + echo "::error::Cannot find the publication task of repository $repository in the pulp-cli output" + return 1 + fi + wait_task_race "$task" && rc=0 || rc=$? + if [[ $rc -eq 0 ]]; then + return 0 + elif [[ $rc -eq 2 && $outer -lt 3 ]]; then + echo "[WARN] Publication of $repository was interrupted server-side, retrying" + sleep $((outer * 15)) + else + echo "::error::Publication of repository $repository failed" + return 1 + fi + done +} diff --git a/.github/scripts/pulp/check-common.sh b/.github/scripts/pulp/check-common.sh index 8c13d494b34..9a6e2662981 100644 --- a/.github/scripts/pulp/check-common.sh +++ b/.github/scripts/pulp/check-common.sh @@ -7,9 +7,15 @@ # and only then is a non-zero status returned if anything failed. Source this # file, call load_expected, verify the packages, then call render_summary. +# authenticated content fetches (content_curl) and token refresh +# shellcheck source=.github/scripts/pulp/api.sh +source "$(dirname "${BASH_SOURCE[0]}")/api.sh" + PULP_URL="${PULP_URL:-https://pulp-api.apps.centreon.com}" PULP_CONTENT_URL="${PULP_CONTENT_URL:-https://packages.apps.centreon.com}" -METADATA_TIMEOUT="${METADATA_TIMEOUT:-300}" +# 900s: the published indexes can lag a finished publication by up to the +# content-app cache TTL (600s), the window must survive that worst case +METADATA_TIMEOUT="${METADATA_TIMEOUT:-900}" METADATA_INTERVAL="${METADATA_INTERVAL:-15}" # accumulated per-package result rows and the aggregate failure flag @@ -39,15 +45,39 @@ load_expected() { # wait_for_metadata — repeatedly call the sourcing script's resolve_pending (one # resolution round over the still-unresolved packages, returning 0 once none -# remain) until everything resolves or METADATA_TIMEOUT is reached. +# remain) until everything resolves, METADATA_TIMEOUT is reached, or the +# resolution stalls. The retry window only covers publication propagation: once +# a round reads the published metadata and resolves nothing new while some +# packages already resolved, the remaining ones are not in the publication at +# all (e.g. evicted by the retention policy) and no amount of waiting will +# surface them - report them right away instead of burning the whole window. wait_for_metadata() { local deadline=$(( SECONDS + METADATA_TIMEOUT )) + local resolved previous_resolved=-1 stall_rounds=0 until resolve_pending; do + resolved=0 + for i in "${!E_FILENAME[@]}"; do + [[ "${META_IDX[$i]}" == "true" ]] && resolved=$((resolved + 1)) + done + if ((resolved > 0 && resolved == previous_resolved)); then + stall_rounds=$((stall_rounds + 1)) + else + stall_rounds=0 + fi + # several consecutive stalled rounds before giving up: a single one is + # not enough, the published index can lag the publication by minutes + # (content-app cache TTL), which read as false negatives on freshly + # delivered builds + if ((stall_rounds >= 6)); then + echo "[WARN] ${resolved}/${#E_FILENAME[@]} package(s) resolvable in the published metadata and no progress in the last 6 rounds; the remaining ones are not part of the publication, giving up early" + break + fi + previous_resolved=$resolved if [[ "$SECONDS" -ge "$deadline" ]]; then - echo "[WARN] Metadata resolution timed out after ${METADATA_TIMEOUT}s" + echo "[WARN] Metadata resolution timed out after ${METADATA_TIMEOUT}s (${resolved}/${#E_FILENAME[@]} resolvable)" break fi - echo "[INFO] Waiting ${METADATA_INTERVAL}s for metadata to publish..." + echo "[INFO] ${resolved}/${#E_FILENAME[@]} package(s) resolvable, waiting ${METADATA_INTERVAL}s for the metadata to propagate..." sleep "$METADATA_INTERVAL" done } @@ -68,13 +98,18 @@ record_row() { # record every package's result row. Uses the E_FILENAME/E_ARCH/E_BASEPATH, # PRESENT_IDX, META_IDX and RESOLVED_IDX arrays filled by the sourcing script. check_fetchable_and_record() { - local i url code fetchable + local i url code fetchable attempt for i in "${!E_FILENAME[@]}"; do fetchable=false if [[ "${META_IDX[$i]}" == "true" ]]; then url="${PULP_CONTENT_URL}/${E_BASEPATH[$i]}/${RESOLVED_IDX[$i]}" - code=$(curl -fsSL -o /dev/null -w '%{http_code}' -I "$url" 2>/dev/null || echo 000) - [[ "$code" == "200" ]] && fetchable=true + # retry: one flaky HEAD out of hundreds (content-app/S3 hiccup) must not + # fail the whole verification + for attempt in 1 2 3; do + code=$(content_curl -fsSL -o /dev/null -w '%{http_code}' -I "$url" 2>/dev/null || echo 000) + [[ "$code" == "200" ]] && { fetchable=true; break; } + sleep 2 + done fi record_row "${E_FILENAME[$i]}" "${E_ARCH[$i]}" "${PRESENT_IDX[$i]}" "${META_IDX[$i]}" "$fetchable" done diff --git a/.github/scripts/pulp/check-deb.sh b/.github/scripts/pulp/check-deb.sh index e5bdca72f01..756ef30f5b4 100755 --- a/.github/scripts/pulp/check-deb.sh +++ b/.github/scripts/pulp/check-deb.sh @@ -28,27 +28,53 @@ mapfile -t E_SUITE < <(echo "$PACKAGES_JSON" | jq -r '.[].suite') mapfile -t E_RELPATH < <(echo "$PACKAGES_JSON" | jq -r '.[].relative_path') # --- physical presence: content units in the repository's latest version ---- -declare -A PRESENT_BY_REPO +# newest first, stopping as soon as every expected package of the repository +# has been seen: the shared plugins repository holds 10k+ module packages and +# deep offset pagination both costs the server dearly and eventually fails, +# while the freshly delivered packages are the newest content by construction. +# The listing goes to a file and grep reads the file: piping it into grep -q +# would SIGPIPE the writer on the (early-exiting) first match, and pipefail +# then turns every successful match into a false negative. +declare -A PRESENT_BY_REPO # repo -> file holding one relative_path per line for repo in $(printf '%s\n' "${E_REPOSITORY[@]}" | sort -u); do + PRESENT_BY_REPO[$repo]=$(mktemp) version_href=$(pulp deb repository show --name "$repo" 2>/dev/null | jq -r '.latest_version_href // empty') if [[ -z "$version_href" ]]; then echo "[WARN] Repository $repo does not exist or has no version" - PRESENT_BY_REPO[$repo]="" continue fi - PRESENT_BY_REPO[$repo]=$( - curl -fsSL -H "Authorization: Github $PULP_TOKEN" -G \ - --data-urlencode "repository_version=$version_href" \ - --data-urlencode "pulp_label_select=module=$MODULE_NAME" \ - --data-urlencode "limit=1000" \ - "$PULP_URL/api/v3/content/deb/packages/" 2>/dev/null \ - | jq -r '.results[].relative_path' - ) + url="$PULP_URL/api/v3/content/deb/packages/?$( + printf 'repository_version=%s&pulp_label_select=%s&ordering=-pulp_created&limit=1000' \ + "$(jq -rn --arg v "$version_href" '$v | @uri')" \ + "$(jq -rn --arg v "module=$MODULE_NAME" '$v | @uri')" + )" + pages=0 + while [[ -n "$url" ]] && ((pages < 20)); do + page=$(curl -fsSL -H "Authorization: Github $PULP_TOKEN" "$url") || { + echo "[WARN] presence page fetch failed for $repo ($url)" >&2 + break + } + if ((pages == 0)); then + echo "[INFO] Repository $repo holds $(echo "$page" | jq -r '.count') module package(s) in its latest version" + fi + echo "$page" | jq -r '.results[].relative_path' >> "${PRESENT_BY_REPO[$repo]}" + pages=$((pages + 1)) + all_found=true + for i in "${!E_FILENAME[@]}"; do + [[ "${E_REPOSITORY[$i]}" == "$repo" ]] || continue + if ! grep -Fxq "${E_RELPATH[$i]}" "${PRESENT_BY_REPO[$repo]}"; then + all_found=false + break + fi + done + [[ "$all_found" == "true" ]] && break + url=$(echo "$page" | jq -r '.next // empty') + done done declare -A PRESENT_IDX for i in "${!E_FILENAME[@]}"; do - if printf '%s\n' "${PRESENT_BY_REPO[${E_REPOSITORY[$i]}]:-}" | grep -Fxq "${E_RELPATH[$i]}"; then + if grep -Fxq "${E_RELPATH[$i]}" "${PRESENT_BY_REPO[${E_REPOSITORY[$i]}]}"; then PRESENT_IDX[$i]=true else PRESENT_IDX[$i]=false @@ -58,12 +84,15 @@ done # --- metadata resolvability + fetchability, with a bounded retry window ----- # resolve a package's published Filename from a suite Packages index by sha256 resolve_filename() { - local packages=$1 sha=$2 - printf '%s' "$packages" | awk -v sha="$sha" ' + # read the Packages index from a file: awk exits on the first match, and a + # pipe writer would take a SIGPIPE ("printf: write error: Broken pipe" noise) + local packages_file=$1 sha=$2 + [[ -s "$packages_file" ]] || return 0 + awk -v sha="$sha" ' BEGIN { RS = ""; FS = "\n" } index($0, "SHA256: " sha) { for (i = 1; i <= NF; i++) if ($i ~ /^Filename: /) { sub(/^Filename: /, "", $i); print $i; exit } - }' + }' "$packages_file" } declare -A META_IDX # idx -> true|false @@ -73,9 +102,9 @@ for i in "${!E_FILENAME[@]}"; do META_IDX[$i]=false; done # one resolution round: fetch each suite's Packages indexes once, then resolve # each pending package's published Filename by sha256 resolve_pending() { - local -A pkg_cache=() # key: base_path|suite|arch -> Packages body + local -A pkg_cache=() # key: base_path|suite|arch -> Packages index file local -A arches_cache=() # key: base_path|suite -> space separated arches - local all_resolved=true i base_path suite arch search_arches sk ck a filename + local all_resolved=true i base_path suite arch search_arches sk ck a filename cache_file for i in "${!E_FILENAME[@]}"; do [[ "${META_IDX[$i]}" == "true" ]] && continue base_path=${E_BASEPATH[$i]}; suite=${E_SUITE[$i]}; arch=${E_ARCH[$i]} @@ -86,7 +115,7 @@ resolve_pending() { if [[ "$arch" == "all" ]]; then sk="$base_path|$suite" if [[ -z "${arches_cache[$sk]+set}" ]]; then - arches_cache[$sk]=$(curl -fsSL "$PULP_CONTENT_URL/$base_path/dists/$suite/Release" 2>/dev/null \ + arches_cache[$sk]=$(content_curl -fsSL "$PULP_CONTENT_URL/$base_path/dists/$suite/Release" 2>/dev/null \ | awk -F': ' '/^Architectures:/ { print $2; exit }') fi search_arches="${arches_cache[$sk]:-amd64 arm64 all}" @@ -96,7 +125,9 @@ resolve_pending() { for a in $search_arches; do ck="$base_path|$suite|$a" if [[ -z "${pkg_cache[$ck]+set}" ]]; then - pkg_cache[$ck]=$(curl -fsSL "$PULP_CONTENT_URL/$base_path/dists/$suite/main/binary-$a/Packages" 2>/dev/null || true) + cache_file=$(mktemp) + content_curl -fsSL "$PULP_CONTENT_URL/$base_path/dists/$suite/main/binary-$a/Packages" 2>/dev/null > "$cache_file" || true + pkg_cache[$ck]=$cache_file fi filename=$(resolve_filename "${pkg_cache[$ck]}" "${E_SHA256[$i]}") [[ -n "$filename" ]] && break @@ -109,6 +140,7 @@ resolve_pending() { all_resolved=false fi done + rm -f "${pkg_cache[@]}" [[ "$all_resolved" == "true" ]] } diff --git a/.github/scripts/pulp/check-rpm.sh b/.github/scripts/pulp/check-rpm.sh index 14eb3bea55a..3d785e0aedf 100755 --- a/.github/scripts/pulp/check-rpm.sh +++ b/.github/scripts/pulp/check-rpm.sh @@ -20,52 +20,73 @@ mapfile -t E_REPOSITORY < <(echo "$PACKAGES_JSON" | jq -r '.[].repository') mapfile -t E_BASEPATH < <(echo "$PACKAGES_JSON" | jq -r '.[].base_path') # --- physical presence: content units in each repository's latest version -- -declare -A PRESENT_BY_REPO +# newest first, stopping as soon as every expected package of the repository +# has been seen: a large repository makes deep offset pagination costly and +# flaky, while the freshly delivered packages are the newest content by +# construction. +# The listing goes to a file and grep reads the file: piping it into grep -q +# would SIGPIPE the writer on the (early-exiting) first match, and pipefail +# then turns every successful match into a false negative. +declare -A PRESENT_BY_REPO # repo -> file holding one filename per line declare -A EXPECTED_COUNT_BY_REPO +declare -A TOTAL_COUNT_BY_REPO for repo in $(printf '%s\n' "${E_REPOSITORY[@]}" | sort -u); do + PRESENT_BY_REPO[$repo]=$(mktemp) version_href=$(pulp rpm repository show --name "$repo" 2>/dev/null | jq -r '.latest_version_href // empty') if [[ -z "$version_href" ]]; then echo "[WARN] Repository $repo does not exist or has no version" - PRESENT_BY_REPO[$repo]="" continue fi - PRESENT_BY_REPO[$repo]=$( - curl -fsSL -H "Authorization: Github $PULP_TOKEN" -G \ - --data-urlencode "repository_version=$version_href" \ - --data-urlencode "pulp_label_select=module=$MODULE_NAME" \ - --data-urlencode "limit=1000" \ - "$PULP_URL/api/v3/content/rpm/packages/" 2>/dev/null \ - | jq -r '.results[].location_href' | awk -F/ '{print $NF}' - ) + url="$PULP_URL/api/v3/content/rpm/packages/?$( + printf 'repository_version=%s&pulp_label_select=%s&ordering=-pulp_created&limit=1000' \ + "$(jq -rn --arg v "$version_href" '$v | @uri')" \ + "$(jq -rn --arg v "module=$MODULE_NAME" '$v | @uri')" + )" + pages=0 + while [[ -n "$url" ]] && ((pages < 20)); do + page=$(curl -fsSL -H "Authorization: Github $PULP_TOKEN" "$url") || { + echo "[WARN] presence page fetch failed for $repo ($url)" >&2 + break + } + if ((pages == 0)); then + TOTAL_COUNT_BY_REPO[$repo]=$(echo "$page" | jq -r '.count') + echo "[INFO] Repository $repo holds ${TOTAL_COUNT_BY_REPO[$repo]} module package(s) in its latest version" + fi + echo "$page" | jq -r '.results[].location_href' | awk -F/ '{print $NF}' >> "${PRESENT_BY_REPO[$repo]}" + pages=$((pages + 1)) + all_found=true + for i in "${!E_FILENAME[@]}"; do + [[ "${E_REPOSITORY[$i]}" == "$repo" ]] || continue + if ! grep -Fxq "${E_FILENAME[$i]}" "${PRESENT_BY_REPO[$repo]}"; then + all_found=false + break + fi + done + [[ "$all_found" == "true" ]] && break + url=$(echo "$page" | jq -r '.next // empty') + done done declare -A PRESENT_IDX # idx -> true|false for i in "${!E_FILENAME[@]}"; do repo=${E_REPOSITORY[$i]} EXPECTED_COUNT_BY_REPO[$repo]=$(( ${EXPECTED_COUNT_BY_REPO[$repo]:-0} + 1 )) - if printf '%s\n' "${PRESENT_BY_REPO[$repo]:-}" | grep -Fxq "${E_FILENAME[$i]}"; then + if grep -Fxq "${E_FILENAME[$i]}" "${PRESENT_BY_REPO[$repo]}"; then PRESENT_IDX[$i]=true else PRESENT_IDX[$i]=false fi done -# right number (delivery only): flag content-unit filenames present under the -# module that were not expected (e.g. a stale package left behind). the testing -# and unstable repositories retain a single version per package, so any extra is -# suspect; the stable repository legitimately keeps the full version history of -# previous releases, so this sweep would flag every past release on promotion. +# right number (delivery only): warn when the repository holds more module +# packages than expected. Count-based only: with shared repositories and the +# daily develop deliveries, listing every extra name is noise (and the full +# listing is what the bounded pagination above avoids). [[ "${CHECK_MODE:-delivery}" == "delivery" ]] && for repo in "${!EXPECTED_COUNT_BY_REPO[@]}"; do - present_count=$(printf '%s\n' "${PRESENT_BY_REPO[$repo]:-}" | grep -c . || true) - if [[ "$present_count" -gt "${EXPECTED_COUNT_BY_REPO[$repo]}" ]]; then - echo "::warning::Repository $repo holds $present_count module packages but only ${EXPECTED_COUNT_BY_REPO[$repo]} were expected (unexpected extras)" - while read -r fn; do - [[ -z "$fn" ]] && continue - if ! printf '%s\n' "${E_FILENAME[@]}" | grep -Fxq "$fn"; then - record_row "$fn" "?" "false" "false" "false" - fi - done < <(printf '%s\n' "${PRESENT_BY_REPO[$repo]:-}") + total=${TOTAL_COUNT_BY_REPO[$repo]:-0} + if [[ "$total" -gt "${EXPECTED_COUNT_BY_REPO[$repo]}" ]]; then + echo "::warning::Repository $repo holds $total module packages, ${EXPECTED_COUNT_BY_REPO[$repo]} delivered by this run (older builds and other modules accumulate until cleanup)" fi done @@ -78,8 +99,11 @@ for i in "${!E_FILENAME[@]}"; do META_IDX[$i]=false; done # matches, from a primary.xml body: matching by checksum (not filename) binds # the verification to the exact delivered content, like the DEB by-sha256 match resolve_href() { - local primary=$1 sha=$2 - printf '%s' "$primary" | awk -v sha="$sha" ' + # read the primary.xml from a file: awk exits on the first match, and a pipe + # writer would take a SIGPIPE ("printf: write error: Broken pipe" noise) + local primary_file=$1 sha=$2 + [[ -s "$primary_file" ]] || return 0 + awk -v sha="$sha" ' BEGIN { RS = "" } index($0, ">" sha "") { if (match($0, /]*href="[^"]+"/)) { @@ -87,26 +111,26 @@ resolve_href() { sub(/.*href="/, "", s); sub(/"$/, "", s) print s; exit } - }' + }' "$primary_file" } # one resolution round: fetch each base_path's primary.xml once, then resolve # each pending package's published href by sha256 resolve_pending() { local -A primary_cache=() - local all_resolved=true i base_path repomd primary_href href + local all_resolved=true i base_path repomd primary_href href cache_file for i in "${!E_FILENAME[@]}"; do [[ "${META_IDX[$i]}" == "true" ]] && continue base_path=${E_BASEPATH[$i]} if [[ -z "${primary_cache[$base_path]+set}" ]]; then - repomd=$(curl -fsSL "$PULP_CONTENT_URL/$base_path/repodata/repomd.xml" 2>/dev/null || true) + cache_file=$(mktemp) + repomd=$(content_curl -fsSL "$PULP_CONTENT_URL/$base_path/repodata/repomd.xml" 2>/dev/null || true) primary_href=$(printf '%s' "$repomd" | grep -oP '/dev/null | gunzip -c 2>/dev/null || true) - else - primary_cache[$base_path]="" + content_curl -fsSL "$PULP_CONTENT_URL/$base_path/$primary_href" 2>/dev/null | gunzip -c 2>/dev/null > "$cache_file" || true fi + primary_cache[$base_path]=$cache_file fi href=$(resolve_href "${primary_cache[$base_path]}" "${E_SHA256[$i]}") @@ -117,6 +141,7 @@ resolve_pending() { all_resolved=false fi done + rm -f "${primary_cache[@]}" [[ "$all_resolved" == "true" ]] } diff --git a/.github/scripts/pulp/manifest.sh b/.github/scripts/pulp/manifest.sh index 76c28b239d5..242f5d76241 100644 --- a/.github/scripts/pulp/manifest.sh +++ b/.github/scripts/pulp/manifest.sh @@ -24,21 +24,22 @@ manifest_add() { manifest_write() { local module=$1 distrib=$2 package_type=$3 stability=$4 mode=$5 content_url=$6 local file="${GITHUB_WORKSPACE:-$PWD}/pulp-delivery-manifest.json" - local packages="[]" - if ((${#MANIFEST_ENTRIES[@]})); then - packages=$(printf '%s\n' "${MANIFEST_ENTRIES[@]}" | jq -s '.') - fi + # the packages array is slurped from stdin: with hundreds of packages an + # --argjson argument would exceed the kernel argv size limit (E2BIG). # NB: the jq variable is named module_name, not module — `module` is a reserved # jq keyword and `$module` fails to parse on stricter jq builds (the CI runner) - jq -n \ + { + if ((${#MANIFEST_ENTRIES[@]})); then + printf '%s\n' "${MANIFEST_ENTRIES[@]}" + fi + } | jq -s \ --arg module_name "$module" \ --arg distrib "$distrib" \ --arg package_type "$package_type" \ --arg stability "$stability" \ --arg mode "$mode" \ --arg content_url "$content_url" \ - --argjson packages "$packages" \ '{ "module": $module_name, distrib: $distrib, @@ -46,10 +47,10 @@ manifest_write() { stability: $stability, mode: $mode, pulp_content_url: $content_url, - packages: $packages + packages: . }' > "$file" - echo "[INFO] Wrote delivery manifest ($(echo "$packages" | jq 'length') package(s)) to $file" + echo "[INFO] Wrote delivery manifest ($(jq '.packages | length' "$file") package(s)) to $file" if [[ -n "${GITHUB_OUTPUT:-}" ]]; then echo "manifest=$file" >> "$GITHUB_OUTPUT" fi