diff --git a/.github/actions/package-delivery-pulp/deliver-deb.sh b/.github/actions/package-delivery-pulp/deliver-deb.sh index a2bcb5b80c..a3090a453a 100755 --- a/.github/actions/package-delivery-pulp/deliver-deb.sh +++ b/.github/actions/package-delivery-pulp/deliver-deb.sh @@ -32,7 +32,7 @@ assert_not_in_stable() { # endpoint error cannot let an already-stable version slip through and evict it # (the rpm guardrail is likewise fail-closed). pkg_file=$(mktemp) - http_code=$(curl -sSL -o "$pkg_file" -w '%{http_code}' \ + http_code=$(curl -sSL --retry 3 --retry-delay 5 -o "$pkg_file" -w '%{http_code}' \ "$PULP_CONTENT_URL/$BASE_PATH/dists/$STABLE_SUITE/main/binary-$arch/Packages" 2>/dev/null || echo 000) case "$http_code" in 404) rm -f "$pkg_file"; return 0 ;; @@ -91,22 +91,84 @@ PULP_LABELS=$(jq -cn \ --arg workflow "${GITHUB_WORKFLOW:-}" \ '{"module": $mod, "git_commit": $git_commit, "git_ref": $git_ref, "github_run_id": $run_id, "github_actor": $actor, "github_workflow": $workflow}') -# the upload tasks are awaited as a batch after the loop: pulp serializes the -# tasks of the shared repository server-side, so waiting for each task before -# sending the next upload would pay the task-queue latency once per package -# instead of once per delivery. -TASK_HREFS=() +# TEMP(test-pulp-unstable) experimental batched delivery, deb flavor. Unlike +# rpm, a repository-less deb upload ignores the distribution/component +# parameters (pulp_deb only creates the suite association inside the +# repository code path), so the suite association is created explicitly as +# PackageReleaseComponent content units. The release_components api is not +# listable by the OIDC ci-user, so the FIRST package of each architecture is +# delivered through the legacy repository code path - that also +# get_or_creates the ReleaseComponent and the ReleaseArchitecture of the +# suite - and its PackageReleaseComponent (listable) yields the release +# component href for the whole batch. Everything else is uploaded as +# unassociated content in a client-side pool, associated in parallel tasks +# (no repository lock), then added to the repository with a single modify. +lookup_deb_content() { + # emit the href of a deb content unit matching the query, empty if absent + local endpoint=$1 query=$2 + curl -fsSL -H "Authorization: Github $PULP_TOKEN" -G \ + --data-urlencode "limit=1" \ + $query \ + "$PULP_URL/api/v3/content/deb/$endpoint/" | jq -r '.results[0].pulp_href // empty' +} + +resolve_task_content() { + # wait for a content-create task and emit its created content href; fall + # back to a lookup (content already existing on a job re-run) + local task_href=$1 endpoint=$2 fallback_query=$3 + local body state content attempt + for ((attempt = 0; attempt < 200; attempt++)); do + refresh_pulp_token + body=$(curl -fsSL -H "Authorization: Github $PULP_TOKEN" "$PULP_URL$task_href" 2>/dev/null) || body="" + state=$(echo "$body" | jq -r '.state' 2>/dev/null) || state="" + case "$state" in + completed) + content=$(echo "$body" | jq -r '.created_resources[0] // empty') + if [[ -n "$content" ]]; then + echo "$content" + return 0 + fi + break + ;; + failed | canceled) + break + ;; + *) + sleep 3 + ;; + esac + done + content=$(lookup_deb_content "$endpoint" "$fallback_query") + if [[ -z "$content" ]]; then + echo "::error::Cannot resolve the created deb content for task $task_href" >&2 + return 1 + fi + echo "$content" +} + +# group the files by architecture: the first file of each arch goes through +# the legacy path so the suite association targets of that arch exist +declare -A ARCH_SEEN=() +LEGACY_FILES=() +ORPHAN_FILES=() +FILE_ARCHS=() for FILE in "${FILES[@]}"; do - # refresh from the parent shell: pulp_upload runs in a command substitution - # (subshell), so its internal refresh cannot update this shell's token — the - # token inherited by the subshells must be kept fresh from here. - refresh_pulp_token + arch=$(dpkg-deb -f "$FILE" Architecture) + FILE_ARCHS+=("$arch") + if [[ -z "${ARCH_SEEN[$arch]+set}" ]]; then + ARCH_SEEN[$arch]=1 + LEGACY_FILES+=("$FILE") + else + ORPHAN_FILES+=("$FILE") + fi +done + +refresh_pulp_token +LEGACY_TASKS=() +for FILE in "${LEGACY_FILES[@]}"; do assert_not_in_stable "$FILE" - echo "[INFO] Uploading $FILE to $POOL_PATH/ ($SUITE/main, module $MODULE_NAME)" - # packages are labeled with their module so that promote-to-stable can identify - # which packages belong to this module, pulp-cli does not allow to set labels nor - # the relative path of deb packages so the api is used directly - TASK_HREFS+=("$( + echo "[INFO] Uploading $FILE to $POOL_PATH/ ($SUITE/main, module $MODULE_NAME) [legacy path]" + LEGACY_TASKS+=("$( pulp_upload \ -F "file=@\"$FILE\"" \ -F "relative_path=$POOL_PATH/$FILE" \ @@ -116,8 +178,154 @@ for FILE in "${FILES[@]}"; do -F "pulp_labels=$PULP_LABELS" \ "$PULP_URL/api/v3/content/deb/packages/" )") +done +echo "[INFO] Waiting for ${#LEGACY_TASKS[@]} legacy upload task(s)" +wait_tasks "${LEGACY_TASKS[@]}" + +# the release component href of the suite, through the first legacy package +refresh_pulp_token +first_sha256=$(sha256sum "${LEGACY_FILES[0]}" | cut -d' ' -f1) +FIRST_PACKAGE_HREF=$(lookup_deb_content "packages" "--data-urlencode sha256=$first_sha256") +if [[ -z "$FIRST_PACKAGE_HREF" ]]; then + echo "::error::Cannot find the legacy-delivered package ${LEGACY_FILES[0]} (sha256 $first_sha256)" + exit 1 +fi +# the legacy package is a fresh build (unique per-build version), so its only +# suite association is the one the legacy upload just created for $SUITE/main +RELEASE_COMPONENT_HREF=$( + curl -fsSL -H "Authorization: Github $PULP_TOKEN" -G \ + --data-urlencode "package=$FIRST_PACKAGE_HREF" \ + "$PULP_URL/api/v3/content/deb/package_release_components/" \ + | jq -r '.results[0].release_component // empty' +) +if [[ -z "$RELEASE_COMPONENT_HREF" ]]; then + echo "::error::Cannot resolve the $SUITE/main release component from the legacy-delivered package" + exit 1 +fi +echo "[INFO] Release component of $SUITE/main: $RELEASE_COMPONENT_HREF" - # record the uploaded package in the manifest for the verification step +# parallel repository-less uploads (pool of 8), marker-file based like rpm +UPLOAD_DIR=$(mktemp -d) +MAX_PARALLEL_UPLOADS=8 +for i in "${!ORPHAN_FILES[@]}"; do + FILE=${ORPHAN_FILES[$i]} + if ((i % 40 == 0)); then + refresh_pulp_token + fi + ( + # subshell-local refresh: under server slowdowns the inherited token can + # outlive its validity between two parent refreshes + refresh_pulp_token + assert_not_in_stable "$FILE" + echo "[INFO] Uploading $FILE to $POOL_PATH/ ($SUITE/main, module $MODULE_NAME)" + pulp_upload \ + -F "file=@\"$FILE\"" \ + -F "relative_path=$POOL_PATH/$FILE" \ + -F "pulp_labels=$PULP_LABELS" \ + "$PULP_URL/api/v3/content/deb/packages/" > "$UPLOAD_DIR/$i.task" + ) & + while (($(jobs -rp | wc -l) >= MAX_PARALLEL_UPLOADS)); do + wait -n || true + done +done +wait || true + +echo "[INFO] Resolving ${#ORPHAN_FILES[@]} uploaded package(s)" +ORPHAN_SHA256S=() +for i in "${!ORPHAN_FILES[@]}"; do + FILE=${ORPHAN_FILES[$i]} + if [[ ! -s "$UPLOAD_DIR/$i.task" ]]; then + echo "::error::Upload failed for $FILE (no task href, see the worker error above)" + exit 1 + fi + ORPHAN_SHA256S+=("$(sha256sum "$FILE" | cut -d' ' -f1)") +done +for i in "${!ORPHAN_FILES[@]}"; do + if ((i % 40 == 0)); then + refresh_pulp_token + fi + ( + resolve_task_content "$(cat "$UPLOAD_DIR/$i.task")" "packages" \ + "--data-urlencode sha256=${ORPHAN_SHA256S[$i]}" > "$UPLOAD_DIR/$i.content" + ) & + while (($(jobs -rp | wc -l) >= MAX_PARALLEL_UPLOADS)); do + wait -n || true + done +done +wait || true +PACKAGE_HREFS=() +for i in "${!ORPHAN_FILES[@]}"; do + if [[ ! -s "$UPLOAD_DIR/$i.content" ]]; then + echo "::error::Cannot resolve the uploaded content for ${ORPHAN_FILES[$i]} (see the worker error above)" + exit 1 + fi + PACKAGE_HREFS+=("$(cat "$UPLOAD_DIR/$i.content")") +done + +# associate every uploaded package with the suite component. The +# package_release_components create is a plain synchronous DRF create (201 +# with the created unit, no task), so the href comes straight out of the +# response; a failed create (unit already existing on a job re-run) falls +# back to a lookup. +echo "[INFO] Associating ${#PACKAGE_HREFS[@]} package(s) with $SUITE/main" +for i in "${!PACKAGE_HREFS[@]}"; do + if ((i % 40 == 0)); then + refresh_pulp_token + fi + ( + refresh_pulp_token + response=$( + curl -fsSL --retry 3 --retry-delay 5 -H "Authorization: Github $PULP_TOKEN" \ + -X POST -H "Content-Type: application/json" \ + -d "{\"package\": \"${PACKAGE_HREFS[$i]}\", \"release_component\": \"$RELEASE_COMPONENT_HREF\"}" \ + "$PULP_URL/api/v3/content/deb/package_release_components/" + ) || response="" + href=$(echo "$response" | jq -r '.pulp_href // .task // empty') + if [[ -z "$href" ]]; then + href=$(lookup_deb_content "package_release_components" \ + "--data-urlencode package=${PACKAGE_HREFS[$i]} --data-urlencode release_component=$RELEASE_COMPONENT_HREF") + fi + printf '%s' "$href" > "$UPLOAD_DIR/$i.prc" + ) & + while (($(jobs -rp | wc -l) >= MAX_PARALLEL_UPLOADS)); do + wait -n || true + done +done +wait || true + +PRC_HREFS=() +for i in "${!PACKAGE_HREFS[@]}"; do + href="" + [[ -s "$UPLOAD_DIR/$i.prc" ]] && href=$(cat "$UPLOAD_DIR/$i.prc") + if [[ "$href" == */tasks/* ]]; then + href=$(resolve_task_content "$href" "package_release_components" \ + "--data-urlencode package=${PACKAGE_HREFS[$i]} --data-urlencode release_component=$RELEASE_COMPONENT_HREF") + fi + if [[ -z "$href" ]]; then + echo "::error::Suite association failed for ${ORPHAN_FILES[$i]} (see the worker error above)" + exit 1 + fi + PRC_HREFS+=("$href") +done +rm -rf "$UPLOAD_DIR" + +if ((${#PACKAGE_HREFS[@]} > 0)); then + echo "[INFO] Adding ${#PACKAGE_HREFS[@]} package(s) and their suite associations to $REPOSITORY_NAME in a single task" + refresh_pulp_token + # body through a file: thousands of hrefs exceed the argv limit + ADD_BODY_FILE=$(mktemp) + printf '%s\n' "${PACKAGE_HREFS[@]}" "${PRC_HREFS[@]}" | jq -R . | jq -cs '{add_content_units: .}' > "$ADD_BODY_FILE" + MODIFY_TASK=$( + curl -fsSL -H "Authorization: Github $PULP_TOKEN" \ + -X POST -H "Content-Type: application/json" \ + -d @"$ADD_BODY_FILE" \ + "$PULP_URL${REPOSITORY_HREF}modify/" | jq -r '.task' + ) + wait_task "$MODIFY_TASK" +fi + +# record every delivered package in the manifest for the verification step +for FILE in "${FILES[@]}"; do name=$(dpkg-deb -f "$FILE" Package) version=$(dpkg-deb -f "$FILE" Version) arch=$(dpkg-deb -f "$FILE" Architecture) @@ -129,9 +337,6 @@ for FILE in "${FILES[@]}"; do '{filename:$filename,name:$name,version:$version,arch:$arch,sha256:$sha256,repository:$repository,base_path:$base_path,suite:$suite,relative_path:$relative_path}')" done -echo "[INFO] Waiting for ${#TASK_HREFS[@]} upload task(s) to complete" -wait_tasks "${TASK_HREFS[@]}" - echo "[INFO] Publishing repository $REPOSITORY_NAME" create_publication deb "$REPOSITORY_NAME" --structured diff --git a/.github/actions/package-delivery-pulp/deliver-rpm.sh b/.github/actions/package-delivery-pulp/deliver-rpm.sh index 4b91d3cc20..1dbad7a650 100755 --- a/.github/actions/package-delivery-pulp/deliver-rpm.sh +++ b/.github/actions/package-delivery-pulp/deliver-rpm.sh @@ -27,14 +27,25 @@ assert_not_in_stable() { name=${base%-*} stable_repository="$STABLE_REPOSITORY_PREFIX-$arch" + # fail closed on an unreachable api (a transient 5xx must not bypass the + # guard nor silently kill the calling worker), fail open only on the repo + # genuinely not existing yet + local repo_page + repo_page=$( + curl -fsSL --retry 3 --retry-delay 5 -H "Authorization: Github $PULP_TOKEN" -G \ + --data-urlencode "name=$stable_repository" \ + --data-urlencode "limit=1" \ + "$PULP_URL/api/v3/repositories/rpm/rpm/" + ) || { + echo "::error::Cannot check the stable repository $stable_repository to guard $name $version-$release ($arch); refusing to deliver. Retry once the api is reachable." + return 1 + } + repository_version=$(echo "$repo_page" | jq -r '.results[0].latest_version_href // empty') # no stable repository yet => nothing can be in stable - if ! pulp rpm repository show --name "$stable_repository" >/dev/null 2>&1; then - return 0 - fi - repository_version=$(pulp rpm repository show --name "$stable_repository" | jq -r '.latest_version_href') + [[ -z "$repository_version" ]] && return 0 count=$( - curl -fsSL -H "Authorization: Github $PULP_TOKEN" -G \ + curl -fsSL --retry 3 --retry-delay 5 -H "Authorization: Github $PULP_TOKEN" -G \ --data-urlencode "repository_version=$repository_version" \ --data-urlencode "name=$name" \ --data-urlencode "version=$version" \ @@ -42,13 +53,57 @@ assert_not_in_stable() { --data-urlencode "arch=$arch" \ --data-urlencode "limit=1" \ "$PULP_URL/api/v3/content/rpm/packages/" | jq -r '.count' - ) + ) || { + echo "::error::Cannot check the stable repository $stable_repository content to guard $name $version-$release ($arch); refusing to deliver. Retry once the api is reachable." + return 1 + } if [[ "$count" -gt 0 ]]; then echo "::error::$name $version-$release ($arch) is already published in the stable repository $stable_repository; refusing to deliver it to $REPOSITORY_NAME. Bump the package version for a new build." return 1 fi } +# TEMP(test-pulp-unstable): wait for a repository-less upload task and emit +# the created content href. A task that did not produce one (content already +# existing on a job re-run) is resolved by the package sha256 instead of +# failing the delivery. +resolve_uploaded_content() { + local task_href=$1 sha256=$2 + local body state content attempt + for ((attempt = 0; attempt < 200; attempt++)); do + refresh_pulp_token + body=$(curl -fsSL -H "Authorization: Github $PULP_TOKEN" "$PULP_URL$task_href" 2>/dev/null) || body="" + state=$(echo "$body" | jq -r '.state' 2>/dev/null) || state="" + case "$state" in + completed) + content=$(echo "$body" | jq -r '.created_resources[0] // empty') + if [[ -n "$content" ]]; then + echo "$content" + return 0 + fi + break + ;; + failed | canceled) + break + ;; + *) + sleep 3 + ;; + esac + done + content=$( + curl -fsSL -H "Authorization: Github $PULP_TOKEN" -G \ + --data-urlencode "sha256=$sha256" \ + --data-urlencode "limit=1" \ + "$PULP_URL/api/v3/content/rpm/packages/" | jq -r '.results[0].pulp_href // empty' + ) + if [[ -z "$content" ]]; then + echo "::error::Cannot resolve the uploaded content for task $task_href (sha256 $sha256)" >&2 + return 1 + fi + echo "$content" +} + FILES=(*.rpm) if [[ ${#FILES[@]} -eq 0 ]]; then echo "::error::No rpm package found to deliver" @@ -95,31 +150,54 @@ for ARCH in noarch x86_64; do REPOSITORY_HREF=$(pulp rpm repository show --name "$REPOSITORY_NAME" | jq -r '.pulp_href') - # Packages are uploaded straight into the repository: pulpcore requires - # non-admin accounts to provide the destination repository on content upload, - # so the upload cannot be decoupled from the repository association. Packages - # are labeled with their module so that promote-to-stable can identify them; - # pulp-cli does not allow to set labels on upload so the api is used directly. - # The upload tasks are awaited as a batch after the loop: pulp serializes the - # tasks of a repository server-side, so waiting for each task before sending - # the next upload would pay the task-queue latency once per package instead - # of once per delivery. + # TEMP(test-pulp-unstable) experimental batched delivery: packages are + # uploaded as unassociated content (repository-less, so the create tasks + # parallelize across the pulp workers instead of serializing on the + # repository lock), then the whole batch is added to the repository with a + # single modify task. Requires the reconciled rpm/packages access policy + # (delivery-tooling#209). Packages are labeled with their module so that + # promote-to-stable can identify them. TASK_HREFS=() - for FILE in "${ARCH_FILES[@]}"; do - # refresh from the parent shell: pulp_upload runs in a command substitution - # (subshell), so its internal refresh cannot update this shell's token — - # the guardrail curl below and the token inherited by the subshells must - # be kept fresh from here. - refresh_pulp_token - assert_not_in_stable "$FILE" "$ARCH" - echo "[INFO] Uploading $(basename "$FILE") to $REPOSITORY_NAME (module $MODULE_NAME)" - TASK_HREFS+=("$( + SHA256S=() + # TEMP(test-pulp-unstable): the uploads are parallelized client-side (the + # repository-less create tasks already parallelize server-side). A bounded + # pool of background subshells posts the files, each writing its task href + # to a marker file; the parent refreshes the OIDC token between spawn + # chunks so the workers always inherit a fresh token. Worker failures are + # detected through missing/empty marker files after the wait. + UPLOAD_DIR=$(mktemp -d) + MAX_PARALLEL_UPLOADS=8 + for i in "${!ARCH_FILES[@]}"; do + FILE=${ARCH_FILES[$i]} + if ((i % 40 == 0)); then + refresh_pulp_token + fi + ( + # subshell-local refresh: under server slowdowns the inherited token can + # outlive its validity between two parent refreshes + refresh_pulp_token + assert_not_in_stable "$FILE" "$ARCH" + echo "[INFO] Uploading $(basename "$FILE") (module $MODULE_NAME)" pulp_upload \ -F "file=@\"$FILE\"" \ - -F "repository=$REPOSITORY_HREF" \ -F "pulp_labels=$PULP_LABELS" \ - "$PULP_URL/api/v3/content/rpm/packages/" - )") + "$PULP_URL/api/v3/content/rpm/packages/" > "$UPLOAD_DIR/$i.task" + ) & + while (($(jobs -rp | wc -l) >= MAX_PARALLEL_UPLOADS)); do + wait -n || true + done + done + wait || true + + for i in "${!ARCH_FILES[@]}"; do + FILE=${ARCH_FILES[$i]} + if [[ ! -s "$UPLOAD_DIR/$i.task" ]]; then + echo "::error::Upload failed for $(basename "$FILE") (no task href, see the worker error above)" + exit 1 + fi + TASK_HREFS+=("$(cat "$UPLOAD_DIR/$i.task")") + sha256=$(sha256sum "$FILE" | cut -d' ' -f1) + SHA256S+=("$sha256") # record the uploaded package in the manifest (name-version-release parsed # the same way as assert_not_in_stable) for the verification step @@ -130,16 +208,55 @@ for ARCH in noarch x86_64; do base=${base%-*} version=${base##*-} name=${base%-*} - sha256=$(sha256sum "$FILE" | cut -d' ' -f1) manifest_add "$(jq -cn \ --arg filename "$FILENAME" --arg name "$name" --arg version "$version" \ --arg release "$release" --arg arch "$ARCH" --arg sha256 "$sha256" \ --arg repository "$REPOSITORY_NAME" --arg base_path "$BASE_PATH" \ '{filename:$filename,name:$name,version:$version,release:$release,arch:$arch,sha256:$sha256,repository:$repository,base_path:$base_path}')" done + rm -rf "$UPLOAD_DIR" + + echo "[INFO] Waiting for ${#TASK_HREFS[@]} upload task(s) and resolving the content" + # TEMP(test-pulp-unstable): the resolutions are parallelized like the + # uploads - sequential, they paid one task GET per package (~6 min at 675) + RESOLVE_DIR=$(mktemp -d) + for i in "${!TASK_HREFS[@]}"; do + if ((i % 40 == 0)); then + refresh_pulp_token + fi + ( + resolve_uploaded_content "${TASK_HREFS[$i]}" "${SHA256S[$i]}" > "$RESOLVE_DIR/$i.content" + ) & + while (($(jobs -rp | wc -l) >= MAX_PARALLEL_UPLOADS)); do + wait -n || true + done + done + wait || true - echo "[INFO] Waiting for ${#TASK_HREFS[@]} upload task(s) to complete" - wait_tasks "${TASK_HREFS[@]}" + CONTENT_HREFS=() + for i in "${!TASK_HREFS[@]}"; do + if [[ ! -s "$RESOLVE_DIR/$i.content" ]]; then + echo "::error::Cannot resolve the uploaded content of task ${TASK_HREFS[$i]} (see the worker error above)" + exit 1 + fi + CONTENT_HREFS+=("$(cat "$RESOLVE_DIR/$i.content")") + done + rm -rf "$RESOLVE_DIR" + + echo "[INFO] Adding ${#CONTENT_HREFS[@]} package(s) to $REPOSITORY_NAME in a single task" + # refresh from the parent shell: the resolutions above ran in subshells, so + # their refreshes never updated this shell's token (the 401 trap, again) + refresh_pulp_token + # body through a file: thousands of hrefs exceed the argv limit + ADD_BODY_FILE=$(mktemp) + printf '%s\n' "${CONTENT_HREFS[@]}" | jq -R . | jq -cs '{add_content_units: .}' > "$ADD_BODY_FILE" + MODIFY_TASK=$( + curl -fsSL -H "Authorization: Github $PULP_TOKEN" \ + -X POST -H "Content-Type: application/json" \ + -d @"$ADD_BODY_FILE" \ + "$PULP_URL${REPOSITORY_HREF}modify/" | jq -r '.task' + ) + wait_task "$MODIFY_TASK" echo "[INFO] Publishing repository $REPOSITORY_NAME" create_publication rpm "$REPOSITORY_NAME" diff --git a/.github/actions/promote-to-stable-pulp/promote-deb.sh b/.github/actions/promote-to-stable-pulp/promote-deb.sh index 3ad3eb1860..da88f02c28 100755 --- a/.github/actions/promote-to-stable-pulp/promote-deb.sh +++ b/.github/actions/promote-to-stable-pulp/promote-deb.sh @@ -20,28 +20,33 @@ VERSION_HREF=$(pulp deb repository show --name "$REPOSITORY_NAME" | jq -r '.late # packages of the module are identified by the label set at delivery time, # the testing pool path scopes the stability and the package distrib name -# scopes the distribution as the apt repository holds all the suites -RESPONSE=$( - curl -fsSL -H "Authorization: Github $PULP_TOKEN" -G \ - --data-urlencode "repository_version=$VERSION_HREF" \ - --data-urlencode "pulp_label_select=module=$MODULE_NAME" \ - --data-urlencode "limit=1000" \ - "$PULP_URL/api/v3/content/deb/packages/" -) - -# fail on a truncated page (checked before the pool-path filtering): silently -# promoting a subset of the module's packages would publish an incomplete -# stable suite -if [[ $(echo "$RESPONSE" | jq '.count > (.results | length)') == "true" ]]; then - echo "::error::Package query on $REPOSITORY_NAME is truncated ($(echo "$RESPONSE" | jq -r '.results | length')/$(echo "$RESPONSE" | jq -r '.count') results); pagination is required" - exit 1 -fi - +# scopes the distribution as the apt repository holds all the suites. +# Paginated: the shared repository keeps every delivered version across all +# suites, so the module listing exceeds a single page. +RESULTS_FILE=$(mktemp) +url="$PULP_URL/api/v3/content/deb/packages/?$( + printf 'repository_version=%s&pulp_label_select=%s&limit=1000' \ + "$(jq -rn --arg v "$VERSION_HREF" '$v | @uri')" \ + "$(jq -rn --arg v "module=$MODULE_NAME" '$v | @uri')" +)" +while [[ -n "$url" ]]; do + refresh_pulp_token + page=$(curl -fsSL -H "Authorization: Github $PULP_TOKEN" "$url") + echo "$page" | jq -c '.results[]' >> "$RESULTS_FILE" + url=$(echo "$page" | jq -r '.next // empty') +done + +# only the LATEST version of each package is promoted: the testing suite +# accumulates every delivered build (deb has no retention mechanism), and +# our version schemes embed a monotonic date/timestamp so the plain string +# max is the newest build PACKAGES=$( - echo "$RESPONSE" | \ - jq --arg testing_path "$TESTING_POOL_PATH/" --arg distrib_name "$PACKAGE_DISTRIB_NAME" \ - '[.results[] | select((.relative_path | startswith($testing_path)) and (.relative_path | contains($distrib_name)))]' + jq -s --arg testing_path "$TESTING_POOL_PATH/" --arg distrib_name "$PACKAGE_DISTRIB_NAME" \ + '[.[] | select((.relative_path | startswith($testing_path)) and (.relative_path | contains($distrib_name)))] + | group_by(.package, .architecture) | map(max_by(.version))' \ + "$RESULTS_FILE" ) +rm -f "$RESULTS_FILE" PACKAGES_COUNT=$(echo "$PACKAGES" | jq 'length') if [[ "$PACKAGES_COUNT" -eq 0 ]]; then @@ -63,7 +68,39 @@ mkdir -p promoted-packages # module label, built with jq (safe escaping) — consistent with the delivery scripts PULP_LABELS=$(jq -cn --arg mod "$MODULE_NAME" '{"module": $mod}') +lookup_prcs() { + # emit the release_component hrefs of a package's suite associations + local package_href=$1 + curl -fsSL -H "Authorization: Github $PULP_TOKEN" -G \ + --data-urlencode "package=$package_href" \ + --data-urlencode "limit=100" \ + "$PULP_URL/api/v3/content/deb/package_release_components/" \ + | jq -r '.results[].release_component' +} + +# batched promote, mirroring the batched delivery: the packages already exist +# as content units in the repository (testing suite), so promoting is only a +# matter of stable suite associations. The FIRST package of each architecture +# is promoted through the legacy re-upload path - it get_or_creates the stable +# ReleaseComponent and ReleaseArchitecture, which the ci user cannot create +# nor list directly - then the stable release-component href is deduced from +# its associations, every other association is created synchronously, and one +# modify adds the whole batch to the repository. +declare -A ARCH_SEEN=() +LEGACY_PACKAGES=() +BATCH_PACKAGES=() while read -r PACKAGE; do + ARCH=$(echo "$PACKAGE" | jq -r '.architecture') + if [[ -z "${ARCH_SEEN[$ARCH]+set}" ]]; then + ARCH_SEEN[$ARCH]=1 + LEGACY_PACKAGES+=("$PACKAGE") + else + BATCH_PACKAGES+=("$PACKAGE") + fi +done < <(echo "$PACKAGES" | jq -c '.[]') + +refresh_pulp_token +for PACKAGE in "${LEGACY_PACKAGES[@]}"; do RELATIVE_PATH=$(echo "$PACKAGE" | jq -r '.relative_path') SHA256=$(echo "$PACKAGE" | jq -r '.sha256') ARCH=$(echo "$PACKAGE" | jq -r '.architecture') @@ -74,7 +111,7 @@ while read -r PACKAGE; do # layout, not their upload relative_path, so resolve the published location # from the testing suite Packages indexed by checksum to download the file FILENAME=$( - curl -fsSL "$PULP_CONTENT_URL/$BASE_PATH/dists/$TESTING_SUITE/main/binary-$ARCH/Packages" | + curl -fsSL --retry 3 --retry-delay 5 "$PULP_CONTENT_URL/$BASE_PATH/dists/$TESTING_SUITE/main/binary-$ARCH/Packages" | awk -v sha="$SHA256" 'BEGIN { RS = ""; FS = "\n" } index($0, "SHA256: " sha) { for (i = 1; i <= NF; i++) if ($i ~ /^Filename: /) { sub(/^Filename: /, "", $i); print $i } }' ) if [[ -z "$FILENAME" ]]; then @@ -83,14 +120,12 @@ while read -r PACKAGE; do fi echo "[INFO] Downloading $PULP_CONTENT_URL/$BASE_PATH/$FILENAME" - curl -fsSL -o "$FILE" "$PULP_CONTENT_URL/$BASE_PATH/$FILENAME" + curl -fsSL --retry 3 --retry-delay 5 -o "$FILE" "$PULP_CONTENT_URL/$BASE_PATH/$FILENAME" # re-upload the same bytes with the SAME relative_path: pulp reuses the - # existing content unit (a different relative_path would create a second unit - # that evicts the first, as pulp deduplicates by name+version+architecture - # repository wide) and only adds the stable suite association. an upload is - # required because pulp_deb rejects direct PackageReleaseComponent creation. - echo "[INFO] Promoting $FILE_NAME to $STABLE_SUITE/main" + # existing content unit and only adds the stable suite association, + # creating the stable ReleaseComponent/ReleaseArchitecture on the way + echo "[INFO] Promoting $FILE_NAME to $STABLE_SUITE/main [legacy path]" TASK_HREF=$( pulp_upload \ -F "file=@\"$FILE\"" \ @@ -102,16 +137,94 @@ while read -r PACKAGE; do "$PULP_URL/api/v3/content/deb/packages/" ) wait_task "$TASK_HREF" +done + +if ((${#BATCH_PACKAGES[@]} > 0)); then + # deduce the stable release-component href: the batch packages only carry + # their testing association, the just-promoted legacy package carries the + # stable one on top of it + refresh_pulp_token + FIRST_BATCH_HREF=$(echo "${BATCH_PACKAGES[0]}" | jq -r '.pulp_href') + LEGACY_HREF=$(echo "${LEGACY_PACKAGES[0]}" | jq -r '.pulp_href') + TESTING_RC=$(lookup_prcs "$FIRST_BATCH_HREF" | head -1) + STABLE_RC=$(lookup_prcs "$LEGACY_HREF" | grep -Fxv "$TESTING_RC" | head -1) + if [[ -z "$STABLE_RC" ]]; then + echo "::error::Cannot deduce the $STABLE_SUITE/main release component from the legacy-promoted package" + exit 1 + fi + echo "[INFO] Release component of $STABLE_SUITE/main: $STABLE_RC" + + # create the stable suite associations: synchronous creates (201 with the + # unit, no task), parallelized; a failed create (existing association on a + # re-run) falls back to a lookup + echo "[INFO] Associating ${#BATCH_PACKAGES[@]} package(s) with $STABLE_SUITE/main" + PRC_DIR=$(mktemp -d) + MAX_PARALLEL=8 + for i in "${!BATCH_PACKAGES[@]}"; do + if ((i % 40 == 0)); then + refresh_pulp_token + fi + ( + refresh_pulp_token + package_href=$(echo "${BATCH_PACKAGES[$i]}" | jq -r '.pulp_href') + response=$( + curl -fsSL --retry 3 --retry-delay 5 -H "Authorization: Github $PULP_TOKEN" \ + -X POST -H "Content-Type: application/json" \ + -d "{\"package\": \"$package_href\", \"release_component\": \"$STABLE_RC\"}" \ + "$PULP_URL/api/v3/content/deb/package_release_components/" + ) || response="" + href=$(echo "$response" | jq -r '.pulp_href // empty') + if [[ -z "$href" ]]; then + href=$( + curl -fsSL -H "Authorization: Github $PULP_TOKEN" -G \ + --data-urlencode "package=$package_href" \ + --data-urlencode "release_component=$STABLE_RC" \ + --data-urlencode "limit=1" \ + "$PULP_URL/api/v3/content/deb/package_release_components/" \ + | jq -r '.results[0].pulp_href // empty' + ) + fi + printf '%s\n%s' "$package_href" "$href" > "$PRC_DIR/$i.pair" + ) & + while (($(jobs -rp | wc -l) >= MAX_PARALLEL)); do + wait -n || true + done + done + wait || true + + ADD_UNITS_FILE=$(mktemp) + for i in "${!BATCH_PACKAGES[@]}"; do + if [[ ! -s "$PRC_DIR/$i.pair" ]] || [[ -z "$(sed -n '2p' "$PRC_DIR/$i.pair")" ]]; then + echo "::error::Stable suite association failed for package $(echo "${BATCH_PACKAGES[$i]}" | jq -r '.package') (see the worker error above)" + exit 1 + fi + cat "$PRC_DIR/$i.pair" >> "$ADD_UNITS_FILE" + echo >> "$ADD_UNITS_FILE" + done + rm -rf "$PRC_DIR" + + echo "[INFO] Adding ${#BATCH_PACKAGES[@]} package(s) and their suite associations to $REPOSITORY_NAME in a single task" + refresh_pulp_token + # body through a file: thousands of hrefs exceed the argv limit + ADD_BODY_FILE=$(mktemp) + jq -R . "$ADD_UNITS_FILE" | jq -cs '{add_content_units: [.[] | select(. != "")]}' > "$ADD_BODY_FILE" + rm -f "$ADD_UNITS_FILE" + MODIFY_TASK=$( + curl -fsSL -H "Authorization: Github $PULP_TOKEN" \ + -X POST -H "Content-Type: application/json" \ + -d @"$ADD_BODY_FILE" \ + "$PULP_URL${REPOSITORY_HREF}modify/" | jq -r '.task' + ) + wait_task "$MODIFY_TASK" +fi - # record the promoted package (with its stable suite coordinates) so the - # verification step verifies exactly this set against the stable suite - NAME=$(echo "$PACKAGE" | jq -r '.package') - VERSION=$(echo "$PACKAGE" | jq -r '.version') - manifest_add "$(jq -cn \ - --arg filename "$FILE_NAME" --arg name "$NAME" --arg version "$VERSION" \ - --arg arch "$ARCH" --arg sha256 "$SHA256" --arg repository "$REPOSITORY_NAME" \ - --arg base_path "$BASE_PATH" --arg suite "$STABLE_SUITE" --arg relative_path "$RELATIVE_PATH" \ - '{filename:$filename,name:$name,version:$version,arch:$arch,sha256:$sha256,repository:$repository,base_path:$base_path,suite:$suite,relative_path:$relative_path}')" +# record the promoted packages (with their stable suite coordinates) so the +# verification step verifies exactly this set against the stable suite +while read -r PACKAGE; do + manifest_add "$(echo "$PACKAGE" | jq -c \ + --arg repository "$REPOSITORY_NAME" --arg base_path "$BASE_PATH" \ + --arg suite "$STABLE_SUITE" \ + '{filename: (.relative_path | sub(".*/"; "")), name: .package, version, arch: .architecture, sha256, repository: $repository, base_path: $base_path, suite: $suite, relative_path}')" done < <(echo "$PACKAGES" | jq -c '.[]') echo "[INFO] Publishing repository $REPOSITORY_NAME" diff --git a/.github/actions/promote-to-stable-pulp/promote-rpm.sh b/.github/actions/promote-to-stable-pulp/promote-rpm.sh index 9426e87e65..222a04a660 100755 --- a/.github/actions/promote-to-stable-pulp/promote-rpm.sh +++ b/.github/actions/promote-to-stable-pulp/promote-rpm.sh @@ -28,22 +28,26 @@ for ARCH in noarch x86_64; do # packages of the module are identified by the label set at delivery time; # keep both the href list (for the content modify call) and the package # identity (name/version/release/arch/filename) to feed the promotion manifest - RESPONSE=$( - curl -fsSL -H "Authorization: Github $PULP_TOKEN" -G \ - --data-urlencode "repository_version=$VERSION_HREF" \ - --data-urlencode "pulp_label_select=module=$MODULE_NAME" \ - --data-urlencode "limit=1000" \ - "$PULP_URL/api/v3/content/rpm/packages/" - ) - - # fail on a truncated page: silently promoting a subset of the module's - # packages would publish an incomplete stable repository - if [[ $(echo "$RESPONSE" | jq '.count > (.results | length)') == "true" ]]; then - echo "::error::Package query on $TESTING_REPOSITORY_NAME is truncated ($(echo "$RESPONSE" | jq -r '.results | length')/$(echo "$RESPONSE" | jq -r '.count') results); pagination is required" - exit 1 - fi - - RESULTS=$(echo "$RESPONSE" | jq '[.results[] | {pulp_href, name, version, release, arch, location_href, sha256}]') + # paginate: the testing repository keeps every delivered version, so the + # module listing can exceed a single page + RESULTS_FILE=$(mktemp) + url="$PULP_URL/api/v3/content/rpm/packages/?$( + printf 'repository_version=%s&pulp_label_select=%s&limit=1000' \ + "$(jq -rn --arg v "$VERSION_HREF" '$v | @uri')" \ + "$(jq -rn --arg v "module=$MODULE_NAME" '$v | @uri')" + )" + while [[ -n "$url" ]]; do + refresh_pulp_token + page=$(curl -fsSL -H "Authorization: Github $PULP_TOKEN" "$url") + echo "$page" | jq -c '.results[]' >> "$RESULTS_FILE" + url=$(echo "$page" | jq -r '.next // empty') + done + + # only the LATEST build of each package is promoted: the version schemes + # embed a monotonic date/timestamp so the plain string max is the newest + RESULTS=$(jq -s '[.[] | {pulp_href, name, version, release, arch, location_href, sha256}] + | group_by(.name, .arch) | map(max_by([.version, .release]))' "$RESULTS_FILE") + rm -f "$RESULTS_FILE" CONTENT=$(echo "$RESULTS" | jq '[.[].pulp_href]') ARCH_PACKAGES_COUNT=$(echo "$CONTENT" | jq 'length') @@ -92,7 +96,7 @@ for ARCH in noarch x86_64; do TASK_HREF=$( curl -fsSL -H "Authorization: Github $PULP_TOKEN" \ -X POST -H "Content-Type: application/json" \ - -d "{\"add_content_units\": $CONTENT}" \ + -d @<(echo "$CONTENT" | jq -c '{add_content_units: .}') \ "$PULP_URL${STABLE_REPOSITORY_HREF}modify/" | jq -r '.task' ) wait_task "$TASK_HREF" diff --git a/.github/packaging/centreon-plugin.yaml.template b/.github/packaging/centreon-plugin.yaml.template index 3acc9262b5..5485d7e49d 100644 --- a/.github/packaging/centreon-plugin.yaml.template +++ b/.github/packaging/centreon-plugin.yaml.template @@ -102,7 +102,8 @@ overrides: [@DEB_PROVIDES@] rpm: - compression: xz + compression: zstd signature: key_file: ${RPM_SIGNING_KEY_FILE} key_id: ${RPM_SIGNING_KEY_ID} +# TEMP(test-pulp-unstable): touch the common template so every plugin is packaged for the Pulp delivery load test; revert before merge diff --git a/.github/scripts/pulp/api.sh b/.github/scripts/pulp/api.sh index 48ba2b0be6..349d4c8bb1 100644 --- a/.github/scripts/pulp/api.sh +++ b/.github/scripts/pulp/api.sh @@ -121,7 +121,7 @@ wait_tasks() { # (concurrent deliveries can race on artifact creation), echoes the task href pulp_upload() { local attempt response http_code body - for attempt in 1 2 3; do + for attempt in 1 2 3 4 5; do refresh_pulp_token response=$(curl -sS -H "Authorization: Github $PULP_TOKEN" -w $'\n%{http_code}' "$@" 2>/dev/null) || response="" http_code=${response##*$'\n'} @@ -130,10 +130,10 @@ pulp_upload() { echo "$body" | jq -r '.task' return 0 fi - echo "[WARN] upload attempt $attempt/3 failed (HTTP ${http_code:-network-error}), retrying..." >&2 - sleep $((attempt * 3)) + echo "[WARN] upload attempt $attempt/5 failed (HTTP ${http_code:-network-error}), retrying..." >&2 + sleep $((attempt * 5)) done - echo "::error::Upload failed after 3 attempts (HTTP ${http_code:-network-error})" >&2 + echo "::error::Upload failed after 5 attempts (HTTP ${http_code:-network-error})" >&2 return 1 } @@ -144,13 +144,18 @@ pulp_upload() { create_publication() { local plugin=$1 repository=$2 shift 2 - local out task - refresh_pulp_token - out=$(pulp --background "$plugin" publication create --repository "$repository" "$@" 2>&1) || { - echo "$out" + local out task attempt + for attempt in 1 2 3; do + refresh_pulp_token + out=$(pulp --background "$plugin" publication create --repository "$repository" "$@" 2>&1) && break + echo "[WARN] publication start attempt $attempt/3 failed for $repository, retrying..." >&2 + sleep $((attempt * 10)) + out="" + done + if [[ -z "$out" ]]; then echo "::error::Cannot start the publication of repository $repository" return 1 - } + fi task=$(grep -oPm1 '/api/v3/tasks/[0-9a-f-]+/' <<< "$out" || true) if [[ -z "$task" ]]; then echo "$out" diff --git a/.github/workflows/plugins.yml b/.github/workflows/plugins.yml index 9c622b7850..771c5510ad 100644 --- a/.github/workflows/plugins.yml +++ b/.github/workflows/plugins.yml @@ -173,12 +173,8 @@ jobs: unit-tests: needs: [get-environment, get-plugins] - if: | - needs.get-environment.outputs.skip_workflow == 'false' && - needs.get-environment.outputs.stability != 'stable' && - ! cancelled() && - ! contains(needs.*.result, 'failure') && - ! contains(needs.*.result, 'cancelled') + # TEMP(test-pulp-unstable): skip the test jobs, this PR only validates the Pulp delivery + if: false strategy: fail-fast: false max-parallel: 3 @@ -363,15 +359,8 @@ jobs: test-plugins: needs: [get-environment, get-plugins, package] - if: | - always() && - needs.get-environment.outputs.skip_workflow == 'false' && - needs.get-plugins.outputs.test_plugins == 'True' && - needs.get-environment.outputs.stability != 'stable' && - (needs.package.result == 'success' || needs.package.result == 'skipped') && - ! cancelled() && - ! contains(needs.*.result, 'failure') && - ! contains(needs.*.result, 'cancelled') + # TEMP(test-pulp-unstable): skip the test jobs, this PR only validates the Pulp delivery + if: false strategy: fail-fast: false matrix: @@ -434,13 +423,8 @@ jobs: deliver-packages: needs: [get-environment, get-plugins, test-plugins] - if: | - needs.get-environment.outputs.skip_workflow == 'false' && - needs.get-plugins.outputs.package_plugins == 'True' && - (contains(fromJson('["testing", "unstable"]'), needs.get-environment.outputs.stability) || (needs.get-environment.outputs.stability == 'stable' && github.event_name != 'workflow_dispatch')) && - ! cancelled() && - ! contains(needs.*.result, 'failure') && - ! contains(needs.*.result, 'cancelled') + # TEMP(test-pulp-unstable): Pulp-only test, Artifactory delivery disabled + if: false runs-on: ubuntu-24.04 strategy: fail-fast: false @@ -479,11 +463,12 @@ jobs: artifactory_token: ${{ secrets.ARTIFACTORY_ACCESS_TOKEN }} deliver-packages-pulp: - needs: [get-environment, get-plugins, test-plugins] + # TEMP(test-pulp-unstable): decouple from test-plugins so the Pulp delivery + # runs regardless of the PR stability (canary) + needs: [get-environment, get-plugins, package] if: | needs.get-environment.outputs.skip_workflow == 'false' && needs.get-plugins.outputs.package_plugins == 'True' && - (contains(fromJson('["testing", "unstable"]'), needs.get-environment.outputs.stability) || (needs.get-environment.outputs.stability == 'stable' && github.event_name != 'workflow_dispatch')) && ! cancelled() && ! contains(needs.*.result, 'failure') && ! contains(needs.*.result, 'cancelled') @@ -495,6 +480,10 @@ jobs: pull-requests: read strategy: fail-fast: false + # bounded concurrency: a full-matrix delivery saturates the pulp + # database (postgres dropped connections, api pods crash-looped) when + # the 8 legs upload/associate/poll at once + max-parallel: 4 matrix: include: - distrib: el8 @@ -566,7 +555,7 @@ jobs: core.setOutput('release_type', releaseType); - name: Deliver packages on Pulp - if: ${{ contains(fromJson('["testing", "unstable"]'), needs.get-environment.outputs.stability) }} + # TEMP(test-pulp-unstable): run on the PR (canary) stability too id: deliver_pulp continue-on-error: true uses: ./.github/actions/package-delivery-pulp @@ -576,7 +565,7 @@ jobs: distrib: ${{ matrix.distrib }} version: "" cache_key: ${{ github.sha }}-${{ github.run_id }}-${{ matrix.package_extension }}-${{ matrix.distrib }} - stability: ${{ needs.get-environment.outputs.stability }} + stability: testing # TEMP(test-pulp-unstable): force testing so the promote step has a source release_type: ${{ needs.get-environment.outputs.release_type }} is_cloud: "false" pulp_url: ${{ vars.PULP_API_URL }} @@ -584,7 +573,9 @@ jobs: audience: ${{ vars.PULP_OIDC_AUDIENCE }} - name: Promote packages on Pulp - if: ${{ needs.get-environment.outputs.stability == 'stable' && steps.get_release_type.outputs.release_type != '' }} + # TEMP(test-pulp-unstable): promote right after the testing delivery + # with a forced release type, to validate the promote path at scale + if: ${{ steps.deliver_pulp.outcome == 'success' }} id: promote_pulp continue-on-error: true uses: ./.github/actions/promote-to-stable-pulp @@ -594,7 +585,7 @@ jobs: distrib: ${{ matrix.distrib }} major_version: "" stability: stable - release_type: ${{ steps.get_release_type.outputs.release_type }} + release_type: release # TEMP(test-pulp-unstable): forced is_cloud: "false" pulp_url: ${{ vars.PULP_API_URL }} pulp_content_url: ${{ vars.PULP_CONTENT_URL }}