diff --git a/.github/actions/package-delivery-pulp/deliver-deb.sh b/.github/actions/package-delivery-pulp/deliver-deb.sh index 1dfd3409b63..394c7645d19 100755 --- a/.github/actions/package-delivery-pulp/deliver-deb.sh +++ b/.github/actions/package-delivery-pulp/deliver-deb.sh @@ -32,7 +32,7 @@ assert_not_in_stable() { # endpoint error cannot let an already-stable version slip through and evict it # (the rpm guardrail is likewise fail-closed). pkg_file=$(mktemp) - http_code=$(curl -sSL -o "$pkg_file" -w '%{http_code}' \ + http_code=$(curl -sSL --retry 3 --retry-delay 5 -o "$pkg_file" -w '%{http_code}' \ "$PULP_CONTENT_URL/$BASE_PATH/dists/$STABLE_SUITE/main/binary-$arch/Packages" 2>/dev/null || echo 000) case "$http_code" in 404) rm -f "$pkg_file"; return 0 ;; @@ -91,13 +91,121 @@ PULP_LABELS=$(jq -cn \ --arg workflow "${GITHUB_WORKFLOW:-}" \ '{"module": $mod, "git_commit": $git_commit, "git_ref": $git_ref, "github_run_id": $run_id, "github_actor": $actor, "github_workflow": $workflow}') +# Batched delivery, deb flavor. Unlike +# rpm, a repository-less deb upload ignores the distribution/component +# parameters (pulp_deb only creates the suite association inside the +# repository code path), so the suite association is created explicitly as +# PackageReleaseComponent content units. The release_components api is not +# listable by the OIDC ci-user, so the FIRST package of each architecture is +# delivered through the legacy repository code path - that also +# get_or_creates the ReleaseComponent and the ReleaseArchitecture of the +# suite - and its PackageReleaseComponent (listable) yields the release +# component href for the whole batch. Everything else is uploaded as +# unassociated content in a client-side pool, associated in parallel tasks +# (no repository lock), then added to the repository with a single modify. +lookup_deb_content() { + # emit the href of a deb content unit matching the query, empty if absent + local endpoint=$1 query=$2 + curl -fsSL -H "Authorization: Github $PULP_TOKEN" -G \ + --data-urlencode "limit=1" \ + $query \ + "$PULP_URL/api/v3/content/deb/$endpoint/" | jq -r '.results[0].pulp_href // empty' +} + +resolve_task_content() { + # wait for a content-create task and emit its created content href; fall + # back to a lookup (content already existing on a job re-run) + local task_href=$1 endpoint=$2 fallback_query=$3 + local body state content attempt + for ((attempt = 0; attempt < 200; attempt++)); do + refresh_pulp_token + body=$(curl -fsSL -H "Authorization: Github $PULP_TOKEN" "$PULP_URL$task_href" 2>/dev/null) || body="" + state=$(echo "$body" | jq -r '.state' 2>/dev/null) || state="" + case "$state" in + completed) + content=$(echo "$body" | jq -r '.created_resources[0] // empty') + if [[ -n "$content" ]]; then + echo "$content" + return 0 + fi + break + ;; + failed | canceled) + break + ;; + *) + sleep 3 + ;; + esac + done + content=$(lookup_deb_content "$endpoint" "$fallback_query") + if [[ -z "$content" ]]; then + echo "::error::Cannot resolve the created deb content for task $task_href" >&2 + return 1 + fi + echo "$content" +} + +# emit the release-component hrefs a package is associated with +lookup_prcs() { + local package_href=$1 + curl -fsSL --retry 3 --retry-delay 5 -H "Authorization: Github $PULP_TOKEN" -G \ + --data-urlencode "package=$package_href" \ + --data-urlencode "limit=100" \ + "$PULP_URL/api/v3/content/deb/package_release_components/" \ + | jq -r '.results[].release_component' +} + +# group the files by architecture: one file per arch goes through the legacy +# path so the suite association targets of that arch exist. Prefer a file +# whose content does NOT exist in pulp yet: a fresh upload carries exactly +# one suite association afterwards, which identifies the release component +# unambiguously. Content reused across suites (identical bytes delivered to +# another suite before) keeps its prior associations, so for a reused +# representative the pre-upload association set is captured to diff later. +declare -A LEGACY_FOR_ARCH=() +declare -A LEGACY_FRESH=() +declare -A LEGACY_BEFORE=() +refresh_pulp_token +for i in "${!FILES[@]}"; do + FILE=${FILES[$i]} + arch=$(dpkg-deb -f "$FILE" Architecture) + if [[ -n "${LEGACY_FRESH[$arch]+set}" ]]; then + continue + fi + if ((i % 40 == 0)); then + refresh_pulp_token + fi + sha=$(sha256sum "$FILE" | cut -d' ' -f1) + existing=$(lookup_deb_content "packages" "--data-urlencode sha256=$sha") + if [[ -z "$existing" ]]; then + LEGACY_FOR_ARCH[$arch]="$FILE" + LEGACY_FRESH[$arch]=1 + elif [[ -z "${LEGACY_FOR_ARCH[$arch]+set}" ]]; then + LEGACY_FOR_ARCH[$arch]="$FILE" + LEGACY_BEFORE[$arch]=$(lookup_prcs "$existing" | sort) + fi +done + +declare -A IS_LEGACY=() +LEGACY_FILES=() +LEGACY_ARCHS=() +for arch in "${!LEGACY_FOR_ARCH[@]}"; do + LEGACY_FILES+=("${LEGACY_FOR_ARCH[$arch]}") + LEGACY_ARCHS+=("$arch") + IS_LEGACY["${LEGACY_FOR_ARCH[$arch]}"]=1 +done +ORPHAN_FILES=() for FILE in "${FILES[@]}"; do + [[ -n "${IS_LEGACY[$FILE]+set}" ]] || ORPHAN_FILES+=("$FILE") +done + +refresh_pulp_token +LEGACY_TASKS=() +for FILE in "${LEGACY_FILES[@]}"; do assert_not_in_stable "$FILE" - echo "[INFO] Uploading $FILE to $POOL_PATH/ ($SUITE/main, module $MODULE_NAME)" - # packages are labeled with their module so that promote-to-stable can identify - # which packages belong to this module, pulp-cli does not allow to set labels nor - # the relative path of deb packages so the api is used directly - TASK_HREF=$( + echo "[INFO] Uploading $FILE to $POOL_PATH/ ($SUITE/main, module $MODULE_NAME) [legacy path]" + LEGACY_TASKS+=("$( pulp_upload \ -F "file=@\"$FILE\"" \ -F "relative_path=$POOL_PATH/$FILE" \ @@ -106,10 +214,168 @@ for FILE in "${FILES[@]}"; do -F "repository=$REPOSITORY_HREF" \ -F "pulp_labels=$PULP_LABELS" \ "$PULP_URL/api/v3/content/deb/packages/" + )") +done +echo "[INFO] Waiting for ${#LEGACY_TASKS[@]} legacy upload task(s)" +wait_tasks "${LEGACY_TASKS[@]}" + +# the release component href of $SUITE/main, deduced from the association the +# legacy uploads just created. A FRESH representative now carries exactly one +# association; a reused one carries its prior associations too, so the href is +# the difference between its post- and pre-upload association sets. +refresh_pulp_token +RELEASE_COMPONENT_HREF="" +for i in "${!LEGACY_FILES[@]}"; do + FILE=${LEGACY_FILES[$i]} + arch=${LEGACY_ARCHS[$i]} + sha=$(sha256sum "$FILE" | cut -d' ' -f1) + href=$(lookup_deb_content "packages" "--data-urlencode sha256=$sha") + if [[ -z "$href" ]]; then + echo "::error::Cannot find the legacy-delivered package $FILE (sha256 $sha)" + exit 1 + fi + after=$(lookup_prcs "$href" | sort) + if [[ -n "${LEGACY_FRESH[$arch]+set}" ]]; then + if [[ $(echo "$after" | grep -c .) -eq 1 ]]; then + RELEASE_COMPONENT_HREF="$after" + break + fi + else + new_rcs=$(comm -13 <(echo "${LEGACY_BEFORE[$arch]}") <(echo "$after") | grep . || true) + if [[ $(echo "$new_rcs" | grep -c .) -eq 1 ]]; then + RELEASE_COMPONENT_HREF="$new_rcs" + break + fi + fi +done +if [[ -z "$RELEASE_COMPONENT_HREF" ]]; then + echo "::error::Cannot deduce the $SUITE/main release component: every legacy representative is reused content whose suite association pre-exists. Deliver a rebuilt (fresh) package or clean up the previous partial delivery." + exit 1 +fi +echo "[INFO] Release component of $SUITE/main: $RELEASE_COMPONENT_HREF" + +# parallel repository-less uploads (pool of 8), marker-file based like rpm +UPLOAD_DIR=$(mktemp -d) +MAX_PARALLEL_UPLOADS=8 +for i in "${!ORPHAN_FILES[@]}"; do + FILE=${ORPHAN_FILES[$i]} + if ((i % 40 == 0)); then + refresh_pulp_token + fi + ( + # subshell-local refresh: under server slowdowns the inherited token can + # outlive its validity between two parent refreshes + refresh_pulp_token + assert_not_in_stable "$FILE" + echo "[INFO] Uploading $FILE to $POOL_PATH/ ($SUITE/main, module $MODULE_NAME)" + pulp_upload \ + -F "file=@\"$FILE\"" \ + -F "relative_path=$POOL_PATH/$FILE" \ + -F "pulp_labels=$PULP_LABELS" \ + "$PULP_URL/api/v3/content/deb/packages/" > "$UPLOAD_DIR/$i.task" + ) & + while (($(jobs -rp | wc -l) >= MAX_PARALLEL_UPLOADS)); do + wait -n || true + done +done +wait || true + +echo "[INFO] Resolving ${#ORPHAN_FILES[@]} uploaded package(s)" +ORPHAN_SHA256S=() +for i in "${!ORPHAN_FILES[@]}"; do + FILE=${ORPHAN_FILES[$i]} + if [[ ! -s "$UPLOAD_DIR/$i.task" ]]; then + echo "::error::Upload failed for $FILE (no task href, see the worker error above)" + exit 1 + fi + ORPHAN_SHA256S+=("$(sha256sum "$FILE" | cut -d' ' -f1)") +done +for i in "${!ORPHAN_FILES[@]}"; do + if ((i % 40 == 0)); then + refresh_pulp_token + fi + ( + resolve_task_content "$(cat "$UPLOAD_DIR/$i.task")" "packages" \ + "--data-urlencode sha256=${ORPHAN_SHA256S[$i]}" > "$UPLOAD_DIR/$i.content" + ) & + while (($(jobs -rp | wc -l) >= MAX_PARALLEL_UPLOADS)); do + wait -n || true + done +done +wait || true +PACKAGE_HREFS=() +for i in "${!ORPHAN_FILES[@]}"; do + if [[ ! -s "$UPLOAD_DIR/$i.content" ]]; then + echo "::error::Cannot resolve the uploaded content for ${ORPHAN_FILES[$i]} (see the worker error above)" + exit 1 + fi + PACKAGE_HREFS+=("$(cat "$UPLOAD_DIR/$i.content")") +done + +# associate every uploaded package with the suite component. The +# package_release_components create is a plain synchronous DRF create (201 +# with the created unit, no task), so the href comes straight out of the +# response; a failed create (unit already existing on a job re-run) falls +# back to a lookup. +echo "[INFO] Associating ${#PACKAGE_HREFS[@]} package(s) with $SUITE/main" +for i in "${!PACKAGE_HREFS[@]}"; do + if ((i % 40 == 0)); then + refresh_pulp_token + fi + ( + refresh_pulp_token + response=$( + curl -fsSL --retry 3 --retry-delay 5 -H "Authorization: Github $PULP_TOKEN" \ + -X POST -H "Content-Type: application/json" \ + -d "{\"package\": \"${PACKAGE_HREFS[$i]}\", \"release_component\": \"$RELEASE_COMPONENT_HREF\"}" \ + "$PULP_URL/api/v3/content/deb/package_release_components/" + ) || response="" + href=$(echo "$response" | jq -r '.pulp_href // .task // empty') + if [[ -z "$href" ]]; then + href=$(lookup_deb_content "package_release_components" \ + "--data-urlencode package=${PACKAGE_HREFS[$i]} --data-urlencode release_component=$RELEASE_COMPONENT_HREF") + fi + printf '%s' "$href" > "$UPLOAD_DIR/$i.prc" + ) & + while (($(jobs -rp | wc -l) >= MAX_PARALLEL_UPLOADS)); do + wait -n || true + done +done +wait || true + +PRC_HREFS=() +for i in "${!PACKAGE_HREFS[@]}"; do + href="" + [[ -s "$UPLOAD_DIR/$i.prc" ]] && href=$(cat "$UPLOAD_DIR/$i.prc") + if [[ "$href" == */tasks/* ]]; then + href=$(resolve_task_content "$href" "package_release_components" \ + "--data-urlencode package=${PACKAGE_HREFS[$i]} --data-urlencode release_component=$RELEASE_COMPONENT_HREF") + fi + if [[ -z "$href" ]]; then + echo "::error::Suite association failed for ${ORPHAN_FILES[$i]} (see the worker error above)" + exit 1 + fi + PRC_HREFS+=("$href") +done +rm -rf "$UPLOAD_DIR" + +if ((${#PACKAGE_HREFS[@]} > 0)); then + echo "[INFO] Adding ${#PACKAGE_HREFS[@]} package(s) and their suite associations to $REPOSITORY_NAME in a single task" + refresh_pulp_token + # body through a file: thousands of hrefs exceed the argv limit + ADD_BODY_FILE=$(mktemp) + printf '%s\n' "${PACKAGE_HREFS[@]}" "${PRC_HREFS[@]}" | jq -R . | jq -cs '{add_content_units: .}' > "$ADD_BODY_FILE" + MODIFY_TASK=$( + curl -fsSL -H "Authorization: Github $PULP_TOKEN" \ + -X POST -H "Content-Type: application/json" \ + -d @"$ADD_BODY_FILE" \ + "$PULP_URL${REPOSITORY_HREF}modify/" | jq -r '.task' ) - wait_task "$TASK_HREF" + wait_task "$MODIFY_TASK" +fi - # record the uploaded package in the manifest for the verification step +# record every delivered package in the manifest for the verification step +for FILE in "${FILES[@]}"; do name=$(dpkg-deb -f "$FILE" Package) version=$(dpkg-deb -f "$FILE" Version) arch=$(dpkg-deb -f "$FILE" Architecture) @@ -122,7 +388,7 @@ for FILE in "${FILES[@]}"; do done echo "[INFO] Publishing repository $REPOSITORY_NAME" -pulp deb publication create --repository "$REPOSITORY_NAME" --structured >/dev/null +create_publication deb "$REPOSITORY_NAME" --structured echo "::notice::Packages are available with: deb $PULP_CONTENT_URL/$BASE_PATH/ $SUITE main" diff --git a/.github/actions/package-delivery-pulp/deliver-rpm.sh b/.github/actions/package-delivery-pulp/deliver-rpm.sh index dd58fa2bf9b..db39bf92ab4 100755 --- a/.github/actions/package-delivery-pulp/deliver-rpm.sh +++ b/.github/actions/package-delivery-pulp/deliver-rpm.sh @@ -27,14 +27,25 @@ assert_not_in_stable() { name=${base%-*} stable_repository="$STABLE_REPOSITORY_PREFIX-$arch" + # fail closed on an unreachable api (a transient 5xx must not bypass the + # guard nor silently kill the calling worker), fail open only on the repo + # genuinely not existing yet + local repo_page + repo_page=$( + curl -fsSL --retry 3 --retry-delay 5 -H "Authorization: Github $PULP_TOKEN" -G \ + --data-urlencode "name=$stable_repository" \ + --data-urlencode "limit=1" \ + "$PULP_URL/api/v3/repositories/rpm/rpm/" + ) || { + echo "::error::Cannot check the stable repository $stable_repository to guard $name $version-$release ($arch); refusing to deliver. Retry once the api is reachable." + return 1 + } + repository_version=$(echo "$repo_page" | jq -r '.results[0].latest_version_href // empty') # no stable repository yet => nothing can be in stable - if ! pulp rpm repository show --name "$stable_repository" >/dev/null 2>&1; then - return 0 - fi - repository_version=$(pulp rpm repository show --name "$stable_repository" | jq -r '.latest_version_href') + [[ -z "$repository_version" ]] && return 0 count=$( - curl -fsSL -H "Authorization: Github $PULP_TOKEN" -G \ + curl -fsSL --retry 3 --retry-delay 5 -H "Authorization: Github $PULP_TOKEN" -G \ --data-urlencode "repository_version=$repository_version" \ --data-urlencode "name=$name" \ --data-urlencode "version=$version" \ @@ -42,13 +53,57 @@ assert_not_in_stable() { --data-urlencode "arch=$arch" \ --data-urlencode "limit=1" \ "$PULP_URL/api/v3/content/rpm/packages/" | jq -r '.count' - ) + ) || { + echo "::error::Cannot check the stable repository $stable_repository content to guard $name $version-$release ($arch); refusing to deliver. Retry once the api is reachable." + return 1 + } if [[ "$count" -gt 0 ]]; then echo "::error::$name $version-$release ($arch) is already published in the stable repository $stable_repository; refusing to deliver it to $REPOSITORY_NAME. Bump the package version for a new build." return 1 fi } +# Wait for a repository-less upload task and emit +# the created content href. A task that did not produce one (content already +# existing on a job re-run) is resolved by the package sha256 instead of +# failing the delivery. +resolve_uploaded_content() { + local task_href=$1 sha256=$2 + local body state content attempt + for ((attempt = 0; attempt < 200; attempt++)); do + refresh_pulp_token + body=$(curl -fsSL -H "Authorization: Github $PULP_TOKEN" "$PULP_URL$task_href" 2>/dev/null) || body="" + state=$(echo "$body" | jq -r '.state' 2>/dev/null) || state="" + case "$state" in + completed) + content=$(echo "$body" | jq -r '.created_resources[0] // empty') + if [[ -n "$content" ]]; then + echo "$content" + return 0 + fi + break + ;; + failed | canceled) + break + ;; + *) + sleep 3 + ;; + esac + done + content=$( + curl -fsSL -H "Authorization: Github $PULP_TOKEN" -G \ + --data-urlencode "sha256=$sha256" \ + --data-urlencode "limit=1" \ + "$PULP_URL/api/v3/content/rpm/packages/" | jq -r '.results[0].pulp_href // empty' + ) + if [[ -z "$content" ]]; then + echo "::error::Cannot resolve the uploaded content for task $task_href (sha256 $sha256)" >&2 + return 1 + fi + echo "$content" +} + FILES=(*.rpm) if [[ ${#FILES[@]} -eq 0 ]]; then echo "::error::No rpm package found to deliver" @@ -95,22 +150,54 @@ for ARCH in noarch x86_64; do REPOSITORY_HREF=$(pulp rpm repository show --name "$REPOSITORY_NAME" | jq -r '.pulp_href') - # Packages are uploaded straight into the repository: pulpcore requires - # non-admin accounts to provide the destination repository on content upload, - # so the upload cannot be decoupled from the repository association. Packages - # are labeled with their module so that promote-to-stable can identify them; - # pulp-cli does not allow to set labels on upload so the api is used directly. - for FILE in "${ARCH_FILES[@]}"; do - assert_not_in_stable "$FILE" "$ARCH" - echo "[INFO] Uploading $(basename "$FILE") to $REPOSITORY_NAME (module $MODULE_NAME)" - TASK_HREF=$( + # Batched delivery: packages are + # uploaded as unassociated content (repository-less, so the create tasks + # parallelize across the pulp workers instead of serializing on the + # repository lock), then the whole batch is added to the repository with a + # single modify task. Requires the reconciled rpm/packages access policy + # (delivery-tooling#209). Packages are labeled with their module so that + # promote-to-stable can identify them. + TASK_HREFS=() + SHA256S=() + # The uploads are parallelized client-side (the + # repository-less create tasks already parallelize server-side). A bounded + # pool of background subshells posts the files, each writing its task href + # to a marker file; the parent refreshes the OIDC token between spawn + # chunks so the workers always inherit a fresh token. Worker failures are + # detected through missing/empty marker files after the wait. + UPLOAD_DIR=$(mktemp -d) + MAX_PARALLEL_UPLOADS=8 + for i in "${!ARCH_FILES[@]}"; do + FILE=${ARCH_FILES[$i]} + if ((i % 40 == 0)); then + refresh_pulp_token + fi + ( + # subshell-local refresh: under server slowdowns the inherited token can + # outlive its validity between two parent refreshes + refresh_pulp_token + assert_not_in_stable "$FILE" "$ARCH" + echo "[INFO] Uploading $(basename "$FILE") (module $MODULE_NAME)" pulp_upload \ -F "file=@\"$FILE\"" \ - -F "repository=$REPOSITORY_HREF" \ -F "pulp_labels=$PULP_LABELS" \ - "$PULP_URL/api/v3/content/rpm/packages/" - ) - wait_task "$TASK_HREF" + "$PULP_URL/api/v3/content/rpm/packages/" > "$UPLOAD_DIR/$i.task" + ) & + while (($(jobs -rp | wc -l) >= MAX_PARALLEL_UPLOADS)); do + wait -n || true + done + done + wait || true + + for i in "${!ARCH_FILES[@]}"; do + FILE=${ARCH_FILES[$i]} + if [[ ! -s "$UPLOAD_DIR/$i.task" ]]; then + echo "::error::Upload failed for $(basename "$FILE") (no task href, see the worker error above)" + exit 1 + fi + TASK_HREFS+=("$(cat "$UPLOAD_DIR/$i.task")") + sha256=$(sha256sum "$FILE" | cut -d' ' -f1) + SHA256S+=("$sha256") # record the uploaded package in the manifest (name-version-release parsed # the same way as assert_not_in_stable) for the verification step @@ -121,16 +208,58 @@ for ARCH in noarch x86_64; do base=${base%-*} version=${base##*-} name=${base%-*} - sha256=$(sha256sum "$FILE" | cut -d' ' -f1) manifest_add "$(jq -cn \ --arg filename "$FILENAME" --arg name "$name" --arg version "$version" \ --arg release "$release" --arg arch "$ARCH" --arg sha256 "$sha256" \ --arg repository "$REPOSITORY_NAME" --arg base_path "$BASE_PATH" \ '{filename:$filename,name:$name,version:$version,release:$release,arch:$arch,sha256:$sha256,repository:$repository,base_path:$base_path}')" done + rm -rf "$UPLOAD_DIR" + + echo "[INFO] Waiting for ${#TASK_HREFS[@]} upload task(s) and resolving the content" + # The resolutions are parallelized like the + # uploads - sequential, they paid one task GET per package (~6 min at 675) + RESOLVE_DIR=$(mktemp -d) + for i in "${!TASK_HREFS[@]}"; do + if ((i % 40 == 0)); then + refresh_pulp_token + fi + ( + resolve_uploaded_content "${TASK_HREFS[$i]}" "${SHA256S[$i]}" > "$RESOLVE_DIR/$i.content" + ) & + while (($(jobs -rp | wc -l) >= MAX_PARALLEL_UPLOADS)); do + wait -n || true + done + done + wait || true + + CONTENT_HREFS=() + for i in "${!TASK_HREFS[@]}"; do + if [[ ! -s "$RESOLVE_DIR/$i.content" ]]; then + echo "::error::Cannot resolve the uploaded content of task ${TASK_HREFS[$i]} (see the worker error above)" + exit 1 + fi + CONTENT_HREFS+=("$(cat "$RESOLVE_DIR/$i.content")") + done + rm -rf "$RESOLVE_DIR" + + echo "[INFO] Adding ${#CONTENT_HREFS[@]} package(s) to $REPOSITORY_NAME in a single task" + # refresh from the parent shell: the resolutions above ran in subshells, so + # their refreshes never updated this shell's token (the 401 trap, again) + refresh_pulp_token + # body through a file: thousands of hrefs exceed the argv limit + ADD_BODY_FILE=$(mktemp) + printf '%s\n' "${CONTENT_HREFS[@]}" | jq -R . | jq -cs '{add_content_units: .}' > "$ADD_BODY_FILE" + MODIFY_TASK=$( + curl -fsSL -H "Authorization: Github $PULP_TOKEN" \ + -X POST -H "Content-Type: application/json" \ + -d @"$ADD_BODY_FILE" \ + "$PULP_URL${REPOSITORY_HREF}modify/" | jq -r '.task' + ) + wait_task "$MODIFY_TASK" echo "[INFO] Publishing repository $REPOSITORY_NAME" - pulp rpm publication create --repository "$REPOSITORY_NAME" >/dev/null + create_publication rpm "$REPOSITORY_NAME" echo "::notice::Packages are available at $PULP_CONTENT_URL/$BASE_PATH/" done diff --git a/.github/actions/promote-to-stable-pulp/promote-deb.sh b/.github/actions/promote-to-stable-pulp/promote-deb.sh index c1db6bcd9fd..5ff5e8a1876 100755 --- a/.github/actions/promote-to-stable-pulp/promote-deb.sh +++ b/.github/actions/promote-to-stable-pulp/promote-deb.sh @@ -18,38 +18,88 @@ fi VERSION_HREF=$(pulp deb repository show --name "$REPOSITORY_NAME" | jq -r '.latest_version_href') -# packages of the module are identified by the label set at delivery time, -# the testing pool path scopes the stability and the package distrib name -# scopes the distribution as the apt repository holds all the suites -RESPONSE=$( - curl -fsSL -H "Authorization: Github $PULP_TOKEN" -G \ - --data-urlencode "repository_version=$VERSION_HREF" \ - --data-urlencode "pulp_label_select=module=$MODULE_NAME" \ - --data-urlencode "limit=1000" \ - "$PULP_URL/api/v3/content/deb/packages/" -) +# fetch a published index into a file: 0=ok, 2=absent (404), 1=error +fetch_index() { + local url=$1 out=$2 code + code=$(curl -sSL --retry 3 --retry-delay 5 -o "$out" -w '%{http_code}' "$url") || return 1 + [[ "$code" == "200" ]] && return 0 + [[ "$code" == "404" ]] && return 2 + return 1 +} -# fail on a truncated page (checked before the pool-path filtering): silently -# promoting a subset of the module's packages would publish an incomplete -# stable suite -if [[ $(echo "$RESPONSE" | jq '.count > (.results | length)') == "true" ]]; then - echo "::error::Package query on $REPOSITORY_NAME is truncated ($(echo "$RESPONSE" | jq -r '.results | length')/$(echo "$RESPONSE" | jq -r '.count') results); pagination is required" - exit 1 -fi +# collect the sha256 set of every package published in a suite. The served +# dists/ indexes are the source of truth for suite membership: the packages' +# relative_path cannot be trusted for that (content reused across suites +# keeps its original upload path) and the release_components api is not +# listable by the OIDC ci-user. A missing Release (404) is an empty suite; +# any other fetch failure aborts, a partial set must not drive a promotion. +suite_sha_file() { + local suite=$1 out release_file arches arch pkg_file rc + out=$(mktemp) + release_file=$(mktemp) + fetch_index "$PULP_CONTENT_URL/$BASE_PATH/dists/$suite/Release" "$release_file" && rc=0 || rc=$? + if [[ $rc -eq 2 ]]; then + echo "$out" + return 0 + elif [[ $rc -ne 0 ]]; then + echo "::error::Cannot fetch the $suite Release index; refusing to promote from a partial view." >&2 + return 1 + fi + arches=$(awk -F': ' '/^Architectures:/ {print $2}' "$release_file") + for arch in $arches; do + pkg_file=$(mktemp) + if ! fetch_index "$PULP_CONTENT_URL/$BASE_PATH/dists/$suite/main/binary-$arch/Packages" "$pkg_file"; then + echo "::error::Cannot fetch the $suite binary-$arch Packages index; refusing to promote from a partial view." >&2 + return 1 + fi + awk -F': ' '/^SHA256:/ {print $2}' "$pkg_file" >> "$out" + rm -f "$pkg_file" + done + rm -f "$release_file" + sort -u "$out" -o "$out" + echo "$out" +} +TESTING_SHAS_FILE=$(suite_sha_file "$TESTING_SUITE") +STABLE_SHAS_FILE=$(suite_sha_file "$STABLE_SUITE") + +# packages of the module are identified by the label set at delivery time. +# Paginated: the shared repository keeps every delivered version across all +# suites, so the module listing exceeds a single page. +RESULTS_FILE=$(mktemp) +url="$PULP_URL/api/v3/content/deb/packages/?$( + printf 'repository_version=%s&pulp_label_select=%s&limit=1000' \ + "$(jq -rn --arg v "$VERSION_HREF" '$v | @uri')" \ + "$(jq -rn --arg v "module=$MODULE_NAME" '$v | @uri')" +)" +while [[ -n "$url" ]]; do + refresh_pulp_token + page=$(curl -fsSL -H "Authorization: Github $PULP_TOKEN" "$url") + echo "$page" | jq -c '.results[]' >> "$RESULTS_FILE" + url=$(echo "$page" | jq -r '.next // empty') +done + +# only the LATEST version of each package is promoted: the testing suite +# accumulates every delivered build (deb has no retention mechanism), and +# our version schemes embed a monotonic date/timestamp so the plain string +# max is the newest build +TESTING_SET_FILE=$(mktemp) +jq -R . "$TESTING_SHAS_FILE" | jq -s 'map({key: ., value: true}) | from_entries' > "$TESTING_SET_FILE" PACKAGES=$( - echo "$RESPONSE" | \ - jq --arg testing_path "$TESTING_POOL_PATH/" --arg distrib_name "$PACKAGE_DISTRIB_NAME" \ - '[.results[] | select((.relative_path | startswith($testing_path)) and (.relative_path | contains($distrib_name)))]' + jq -s --slurpfile testing_set "$TESTING_SET_FILE" \ + '[.[] | select($testing_set[0][.sha256])] + | group_by(.package, .architecture) | map(max_by(.version))' \ + "$RESULTS_FILE" ) +rm -f "$RESULTS_FILE" "$TESTING_SHAS_FILE" "$TESTING_SET_FILE" PACKAGES_COUNT=$(echo "$PACKAGES" | jq 'length') if [[ "$PACKAGES_COUNT" -eq 0 ]]; then - echo "::error::Nothing to promote, no package of module $MODULE_NAME found in $REPOSITORY_NAME ($TESTING_POOL_PATH/)" + echo "::error::Nothing to promote, no package of module $MODULE_NAME found in the $TESTING_SUITE suite of $REPOSITORY_NAME" exit 1 fi -echo "[INFO] $PACKAGES_COUNT packages of module $MODULE_NAME found in $REPOSITORY_NAME ($TESTING_POOL_PATH/)" +echo "[INFO] $PACKAGES_COUNT packages of module $MODULE_NAME found in the $TESTING_SUITE suite of $REPOSITORY_NAME" if [[ "$STABILITY" != "stable" ]]; then echo "[INFO] Dry run, $PACKAGES_COUNT packages would be promoted to $STABLE_SUITE/main" @@ -63,7 +113,70 @@ mkdir -p promoted-packages # module label, built with jq (safe escaping) — consistent with the delivery scripts PULP_LABELS=$(jq -cn --arg mod "$MODULE_NAME" '{"module": $mod}') +# emit the release-component hrefs a package is associated with +lookup_prcs() { + local package_href=$1 + curl -fsSL --retry 3 --retry-delay 5 -H "Authorization: Github $PULP_TOKEN" -G \ + --data-urlencode "package=$package_href" \ + --data-urlencode "limit=100" \ + "$PULP_URL/api/v3/content/deb/package_release_components/" \ + | jq -r '.results[].release_component' +} + +# candidates already published in the stable suite need no new association; +# when EVERYTHING already reached stable (a rerun after the modify), only the +# publication may be missing: republish and stop. +UNPROMOTED_HREFS=() while read -r PACKAGE; do + sha=$(echo "$PACKAGE" | jq -r '.sha256') + if ! grep -qxF "$sha" "$STABLE_SHAS_FILE"; then + UNPROMOTED_HREFS+=("$(echo "$PACKAGE" | jq -r '.pulp_href')") + fi +done < <(echo "$PACKAGES" | jq -c '.[]') +rm -f "$STABLE_SHAS_FILE" +if ((${#UNPROMOTED_HREFS[@]} == 0)); then + echo "[INFO] All $PACKAGES_COUNT package(s) are already published in $STABLE_SUITE; republishing only" + while read -r PACKAGE; do + manifest_add "$(echo "$PACKAGE" | jq -c \ + --arg repository "$REPOSITORY_NAME" --arg base_path "$BASE_PATH" \ + --arg suite "$STABLE_SUITE" \ + '{filename: (.relative_path | sub(".*/"; "")), name: .package, version, arch: .architecture, sha256, repository: $repository, base_path: $base_path, suite: $suite, relative_path}')" + done < <(echo "$PACKAGES" | jq -c '.[]') + create_publication deb "$REPOSITORY_NAME" --structured + echo "::notice::Packages are available with: deb $PULP_CONTENT_URL/$BASE_PATH/ $STABLE_SUITE main" + manifest_write "$MODULE_NAME" "${DISTRIB:-}" "deb" "$STABILITY" "promote" "$PULP_CONTENT_URL" + exit 0 +fi + +# batched promote, mirroring the batched delivery: the packages already exist +# as content units in the repository (testing suite), so promoting is only a +# matter of stable suite associations. The FIRST package of each architecture +# is promoted through the legacy re-upload path - it get_or_creates the stable +# ReleaseComponent and ReleaseArchitecture, which the ci user cannot create +# nor list directly - then the stable release-component href is deduced from +# its associations, every other association is created synchronously, and one +# modify adds the whole batch to the repository. +declare -A ARCH_SEEN=() +LEGACY_PACKAGES=() +BATCH_PACKAGES=() +while read -r PACKAGE; do + ARCH=$(echo "$PACKAGE" | jq -r '.architecture') + if [[ -z "${ARCH_SEEN[$ARCH]+set}" ]]; then + ARCH_SEEN[$ARCH]=1 + LEGACY_PACKAGES+=("$PACKAGE") + else + BATCH_PACKAGES+=("$PACKAGE") + fi +done < <(echo "$PACKAGES" | jq -c '.[]') + +# the stable release component is deduced from the association the legacy +# re-upload of the FIRST reference creates: capture its association set +# before the upload so the new one stands out as the difference. +refresh_pulp_token +LEGACY_REF_HREF=$(echo "${LEGACY_PACKAGES[0]}" | jq -r '.pulp_href') +LEGACY_REF_BEFORE=$(lookup_prcs "$LEGACY_REF_HREF" | sort) + +for PACKAGE in "${LEGACY_PACKAGES[@]}"; do RELATIVE_PATH=$(echo "$PACKAGE" | jq -r '.relative_path') SHA256=$(echo "$PACKAGE" | jq -r '.sha256') ARCH=$(echo "$PACKAGE" | jq -r '.architecture') @@ -74,7 +187,7 @@ while read -r PACKAGE; do # layout, not their upload relative_path, so resolve the published location # from the testing suite Packages indexed by checksum to download the file FILENAME=$( - curl -fsSL "$PULP_CONTENT_URL/$BASE_PATH/dists/$TESTING_SUITE/main/binary-$ARCH/Packages" | + curl -fsSL --retry 3 --retry-delay 5 "$PULP_CONTENT_URL/$BASE_PATH/dists/$TESTING_SUITE/main/binary-$ARCH/Packages" | awk -v sha="$SHA256" 'BEGIN { RS = ""; FS = "\n" } index($0, "SHA256: " sha) { for (i = 1; i <= NF; i++) if ($i ~ /^Filename: /) { sub(/^Filename: /, "", $i); print $i } }' ) if [[ -z "$FILENAME" ]]; then @@ -83,14 +196,12 @@ while read -r PACKAGE; do fi echo "[INFO] Downloading $PULP_CONTENT_URL/$BASE_PATH/$FILENAME" - curl -fsSL -o "$FILE" "$PULP_CONTENT_URL/$BASE_PATH/$FILENAME" + curl -fsSL --retry 3 --retry-delay 5 -o "$FILE" "$PULP_CONTENT_URL/$BASE_PATH/$FILENAME" # re-upload the same bytes with the SAME relative_path: pulp reuses the - # existing content unit (a different relative_path would create a second unit - # that evicts the first, as pulp deduplicates by name+version+architecture - # repository wide) and only adds the stable suite association. an upload is - # required because pulp_deb rejects direct PackageReleaseComponent creation. - echo "[INFO] Promoting $FILE_NAME to $STABLE_SUITE/main" + # existing content unit and only adds the stable suite association, + # creating the stable ReleaseComponent/ReleaseArchitecture on the way + echo "[INFO] Promoting $FILE_NAME to $STABLE_SUITE/main [legacy path]" TASK_HREF=$( pulp_upload \ -F "file=@\"$FILE\"" \ @@ -102,20 +213,109 @@ while read -r PACKAGE; do "$PULP_URL/api/v3/content/deb/packages/" ) wait_task "$TASK_HREF" +done + +if ((${#BATCH_PACKAGES[@]} > 0)); then + # the stable release component is the association the legacy re-upload just + # added to the reference package. On a rerun the association pre-exists and + # the difference is empty: the reference then carries every suite it belongs + # to (stable, testing, possibly unstable for reused content), and the + # stable one is isolated by subtracting the associations of not-yet-promoted + # candidates - those belong to every suite of the reference EXCEPT stable. + refresh_pulp_token + LEGACY_REF_AFTER=$(lookup_prcs "$LEGACY_REF_HREF" | sort) + STABLE_RC_SET=$(comm -13 <(echo "$LEGACY_REF_BEFORE") <(echo "$LEGACY_REF_AFTER") | grep . || true) + if [[ $(echo "$STABLE_RC_SET" | grep -c .) -ne 1 ]]; then + STABLE_RC_SET=$(echo "$LEGACY_REF_AFTER" | grep . || true) + for u_href in "${UNPROMOTED_HREFS[@]}"; do + [[ "$u_href" == "$LEGACY_REF_HREF" ]] && continue + [[ $(echo "$STABLE_RC_SET" | grep -c .) -le 1 ]] && break + u_prcs=$(lookup_prcs "$u_href" | sort) + STABLE_RC_SET=$(comm -23 <(echo "$STABLE_RC_SET") <(echo "$u_prcs") | grep . || true) + done + fi + STABLE_RC=$(echo "$STABLE_RC_SET" | grep . | head -1) + if [[ -z "$STABLE_RC" || $(echo "$STABLE_RC_SET" | grep -c .) -ne 1 ]]; then + echo "::error::Cannot deduce the $STABLE_SUITE/main release component from the reference package associations (got: ${STABLE_RC_SET:-none})" + exit 1 + fi + echo "[INFO] Release component of $STABLE_SUITE/main: $STABLE_RC" + + # create the stable suite associations: synchronous creates (201 with the + # unit, no task), parallelized; a failed create (existing association on a + # re-run) falls back to a lookup + echo "[INFO] Associating ${#BATCH_PACKAGES[@]} package(s) with $STABLE_SUITE/main" + PRC_DIR=$(mktemp -d) + MAX_PARALLEL=8 + for i in "${!BATCH_PACKAGES[@]}"; do + if ((i % 40 == 0)); then + refresh_pulp_token + fi + ( + refresh_pulp_token + package_href=$(echo "${BATCH_PACKAGES[$i]}" | jq -r '.pulp_href') + response=$( + curl -fsSL --retry 3 --retry-delay 5 -H "Authorization: Github $PULP_TOKEN" \ + -X POST -H "Content-Type: application/json" \ + -d "{\"package\": \"$package_href\", \"release_component\": \"$STABLE_RC\"}" \ + "$PULP_URL/api/v3/content/deb/package_release_components/" + ) || response="" + href=$(echo "$response" | jq -r '.pulp_href // empty') + if [[ -z "$href" ]]; then + href=$( + curl -fsSL -H "Authorization: Github $PULP_TOKEN" -G \ + --data-urlencode "package=$package_href" \ + --data-urlencode "release_component=$STABLE_RC" \ + --data-urlencode "limit=1" \ + "$PULP_URL/api/v3/content/deb/package_release_components/" \ + | jq -r '.results[0].pulp_href // empty' + ) + fi + printf '%s\n%s' "$package_href" "$href" > "$PRC_DIR/$i.pair" + ) & + while (($(jobs -rp | wc -l) >= MAX_PARALLEL)); do + wait -n || true + done + done + wait || true - # record the promoted package (with its stable suite coordinates) so the - # verification step verifies exactly this set against the stable suite - NAME=$(echo "$PACKAGE" | jq -r '.package') - VERSION=$(echo "$PACKAGE" | jq -r '.version') - manifest_add "$(jq -cn \ - --arg filename "$FILE_NAME" --arg name "$NAME" --arg version "$VERSION" \ - --arg arch "$ARCH" --arg sha256 "$SHA256" --arg repository "$REPOSITORY_NAME" \ - --arg base_path "$BASE_PATH" --arg suite "$STABLE_SUITE" --arg relative_path "$RELATIVE_PATH" \ - '{filename:$filename,name:$name,version:$version,arch:$arch,sha256:$sha256,repository:$repository,base_path:$base_path,suite:$suite,relative_path:$relative_path}')" + ADD_UNITS_FILE=$(mktemp) + for i in "${!BATCH_PACKAGES[@]}"; do + if [[ ! -s "$PRC_DIR/$i.pair" ]] || [[ -z "$(sed -n '2p' "$PRC_DIR/$i.pair")" ]]; then + echo "::error::Stable suite association failed for package $(echo "${BATCH_PACKAGES[$i]}" | jq -r '.package') (see the worker error above)" + exit 1 + fi + cat "$PRC_DIR/$i.pair" >> "$ADD_UNITS_FILE" + echo >> "$ADD_UNITS_FILE" + done + rm -rf "$PRC_DIR" + + echo "[INFO] Adding ${#BATCH_PACKAGES[@]} package(s) and their suite associations to $REPOSITORY_NAME in a single task" + refresh_pulp_token + # body through a file: thousands of hrefs exceed the argv limit + ADD_BODY_FILE=$(mktemp) + jq -R . "$ADD_UNITS_FILE" | jq -cs '{add_content_units: [.[] | select(. != "")]}' > "$ADD_BODY_FILE" + rm -f "$ADD_UNITS_FILE" + MODIFY_TASK=$( + curl -fsSL -H "Authorization: Github $PULP_TOKEN" \ + -X POST -H "Content-Type: application/json" \ + -d @"$ADD_BODY_FILE" \ + "$PULP_URL${REPOSITORY_HREF}modify/" | jq -r '.task' + ) + wait_task "$MODIFY_TASK" +fi + +# record the promoted packages (with their stable suite coordinates) so the +# verification step verifies exactly this set against the stable suite +while read -r PACKAGE; do + manifest_add "$(echo "$PACKAGE" | jq -c \ + --arg repository "$REPOSITORY_NAME" --arg base_path "$BASE_PATH" \ + --arg suite "$STABLE_SUITE" \ + '{filename: (.relative_path | sub(".*/"; "")), name: .package, version, arch: .architecture, sha256, repository: $repository, base_path: $base_path, suite: $suite, relative_path}')" done < <(echo "$PACKAGES" | jq -c '.[]') echo "[INFO] Publishing repository $REPOSITORY_NAME" -pulp deb publication create --repository "$REPOSITORY_NAME" --structured >/dev/null +create_publication deb "$REPOSITORY_NAME" --structured echo "::notice::Packages are available with: deb $PULP_CONTENT_URL/$BASE_PATH/ $STABLE_SUITE main" diff --git a/.github/actions/promote-to-stable-pulp/promote-rpm.sh b/.github/actions/promote-to-stable-pulp/promote-rpm.sh index 8f5c404b183..222a04a660f 100755 --- a/.github/actions/promote-to-stable-pulp/promote-rpm.sh +++ b/.github/actions/promote-to-stable-pulp/promote-rpm.sh @@ -28,22 +28,26 @@ for ARCH in noarch x86_64; do # packages of the module are identified by the label set at delivery time; # keep both the href list (for the content modify call) and the package # identity (name/version/release/arch/filename) to feed the promotion manifest - RESPONSE=$( - curl -fsSL -H "Authorization: Github $PULP_TOKEN" -G \ - --data-urlencode "repository_version=$VERSION_HREF" \ - --data-urlencode "pulp_label_select=module=$MODULE_NAME" \ - --data-urlencode "limit=1000" \ - "$PULP_URL/api/v3/content/rpm/packages/" - ) - - # fail on a truncated page: silently promoting a subset of the module's - # packages would publish an incomplete stable repository - if [[ $(echo "$RESPONSE" | jq '.count > (.results | length)') == "true" ]]; then - echo "::error::Package query on $TESTING_REPOSITORY_NAME is truncated ($(echo "$RESPONSE" | jq -r '.results | length')/$(echo "$RESPONSE" | jq -r '.count') results); pagination is required" - exit 1 - fi - - RESULTS=$(echo "$RESPONSE" | jq '[.results[] | {pulp_href, name, version, release, arch, location_href, sha256}]') + # paginate: the testing repository keeps every delivered version, so the + # module listing can exceed a single page + RESULTS_FILE=$(mktemp) + url="$PULP_URL/api/v3/content/rpm/packages/?$( + printf 'repository_version=%s&pulp_label_select=%s&limit=1000' \ + "$(jq -rn --arg v "$VERSION_HREF" '$v | @uri')" \ + "$(jq -rn --arg v "module=$MODULE_NAME" '$v | @uri')" + )" + while [[ -n "$url" ]]; do + refresh_pulp_token + page=$(curl -fsSL -H "Authorization: Github $PULP_TOKEN" "$url") + echo "$page" | jq -c '.results[]' >> "$RESULTS_FILE" + url=$(echo "$page" | jq -r '.next // empty') + done + + # only the LATEST build of each package is promoted: the version schemes + # embed a monotonic date/timestamp so the plain string max is the newest + RESULTS=$(jq -s '[.[] | {pulp_href, name, version, release, arch, location_href, sha256}] + | group_by(.name, .arch) | map(max_by([.version, .release]))' "$RESULTS_FILE") + rm -f "$RESULTS_FILE" CONTENT=$(echo "$RESULTS" | jq '[.[].pulp_href]') ARCH_PACKAGES_COUNT=$(echo "$CONTENT" | jq 'length') @@ -92,13 +96,13 @@ for ARCH in noarch x86_64; do TASK_HREF=$( curl -fsSL -H "Authorization: Github $PULP_TOKEN" \ -X POST -H "Content-Type: application/json" \ - -d "{\"add_content_units\": $CONTENT}" \ + -d @<(echo "$CONTENT" | jq -c '{add_content_units: .}') \ "$PULP_URL${STABLE_REPOSITORY_HREF}modify/" | jq -r '.task' ) wait_task "$TASK_HREF" echo "[INFO] Publishing repository $STABLE_REPOSITORY_NAME" - pulp rpm publication create --repository "$STABLE_REPOSITORY_NAME" >/dev/null + create_publication rpm "$STABLE_REPOSITORY_NAME" # record the promoted packages (with their stable target coordinates) so the # verification step verifies exactly this set against the stable repo diff --git a/.github/actions/setup-pulp-cli/action.yml b/.github/actions/setup-pulp-cli/action.yml index 44525b501a7..19365c4ec3d 100644 --- a/.github/actions/setup-pulp-cli/action.yml +++ b/.github/actions/setup-pulp-cli/action.yml @@ -63,9 +63,16 @@ runs: exit 1 fi - # keep the token out of any log and expose it to the next steps + # keep the token out of any log and expose it to the next steps. + # the issuance time and audience let the api helpers refresh the token + # mid-delivery: github oidc tokens expire ~5 minutes after issuance, + # which is shorter than a large delivery. echo "::add-mask::$PULP_TOKEN" - echo "PULP_TOKEN=$PULP_TOKEN" >> "$GITHUB_ENV" + { + echo "PULP_TOKEN=$PULP_TOKEN" + echo "PULP_TOKEN_ISSUED_AT=$(date +%s)" + echo "PULP_OIDC_AUDIENCE=$AUDIENCE" + } >> "$GITHUB_ENV" - name: Configure pulp-cli shell: bash diff --git a/.github/scripts/pulp/api.sh b/.github/scripts/pulp/api.sh index 7faf3ace2ce..349d4c8bb16 100644 --- a/.github/scripts/pulp/api.sh +++ b/.github/scripts/pulp/api.sh @@ -2,6 +2,46 @@ # Shared pulp api helpers for the delivery/promotion scripts. Source this file # like manifest.sh; PULP_URL and PULP_TOKEN must be set by the caller. +# the github actions oidc token expires ~5 minutes after issuance, which is +# shorter than a large delivery: refresh it before it goes stale whenever the +# job can mint tokens (id-token: write context + audience exported by +# setup-pulp-cli). The fresh token is propagated to the following steps +# (GITHUB_ENV) and to the pulp-cli config so the pulp commands keep working. +refresh_pulp_token() { + local now token + now=$(date +%s) + if ((now - ${PULP_TOKEN_ISSUED_AT:-0} < 240)); then + return 0 + fi + if [[ -z "${ACTIONS_ID_TOKEN_REQUEST_URL:-}" || -z "${ACTIONS_ID_TOKEN_REQUEST_TOKEN:-}" || -z "${PULP_OIDC_AUDIENCE:-}" ]]; then + return 0 + fi + token=$( + curl -fsSL --retry 5 --retry-delay 3 --retry-all-errors \ + -H "Authorization: bearer $ACTIONS_ID_TOKEN_REQUEST_TOKEN" \ + -G --data-urlencode "audience=$PULP_OIDC_AUDIENCE" \ + "$ACTIONS_ID_TOKEN_REQUEST_URL" 2>/dev/null | jq -r '.value' 2>/dev/null + ) || token="" + if [[ -z "$token" || "$token" == "null" ]]; then + return 0 + fi + PULP_TOKEN="$token" + PULP_TOKEN_ISSUED_AT="$now" + # written to stderr: this function also runs inside command substitutions + # (pulp_upload), where a stdout echo would corrupt the captured output + echo "::add-mask::$token" >&2 + if [[ -n "${GITHUB_ENV:-}" ]]; then + { + echo "PULP_TOKEN=$token" + echo "PULP_TOKEN_ISSUED_AT=$now" + } >> "$GITHUB_ENV" + fi + if command -v pulp >/dev/null 2>&1; then + pulp config create --overwrite --base-url "$PULP_URL" --api-root "/" \ + --header "Authorization:Github $token" --timeout 0 >/dev/null 2>&1 || true + fi +} + # wait for a pulp api task to complete. a failed poll request (network blip, # api 5xx) is retried like an unexpected state instead of aborting the caller # under set -e; the 200-attempt cap bounds the total wait (~10 min). @@ -9,6 +49,7 @@ wait_task() { local task_href=$1 local state attempt for ((attempt = 0; attempt < 200; attempt++)); do + refresh_pulp_token state=$(curl -fsSL -H "Authorization: Github $PULP_TOKEN" "$PULP_URL$task_href" 2>/dev/null | jq -r '.state' 2>/dev/null) || state="" case "$state" in completed) @@ -27,11 +68,61 @@ wait_task() { return 1 } +# wait for a batch of pulp api tasks, polling the still-pending ones in sweeps. +# unlike wait_task, the guard aborts only when NO task completes for ~10 min: +# tasks of a repository are serialized server-side, so a long-but-draining +# queue is expected under concurrent deliveries and must not be mistaken for a +# hang. +wait_tasks() { + local pending=("$@") + local total=$# + local next=() stall=0 state href now last_report + last_report=$(date +%s) + while ((${#pending[@]} > 0)); do + refresh_pulp_token + next=() + for href in "${pending[@]}"; do + state=$(curl -fsSL -H "Authorization: Github $PULP_TOKEN" "$PULP_URL$href" 2>/dev/null | jq -r '.state' 2>/dev/null) || state="" + case "$state" in + completed) ;; + failed|canceled) + echo "::error::Task $href $state: $(curl -fsSL -H "Authorization: Github $PULP_TOKEN" "$PULP_URL$href" | jq -c '.error')" + return 1 + ;; + *) + next+=("$href") + ;; + esac + done + if ((${#next[@]} == ${#pending[@]})); then + stall=$((stall + 1)) + if ((stall >= 200)); then + echo "::error::${#pending[@]} task(s) still pending with no progress for ~10 min, e.g. ${pending[0]}" + return 1 + fi + else + stall=0 + fi + pending=("${next[@]+"${next[@]}"}") + # progress heartbeat, at most every 30s: shows how fast the server queue + # drains (sweeps over large batches can take minutes on their own) + now=$(date +%s) + if ((${#pending[@]} > 0 && now - last_report >= 30)); then + echo "[INFO] $((total - ${#pending[@]}))/$total task(s) completed" + last_report=$now + fi + if ((${#pending[@]} > 0)); then + sleep 3 + fi + done +} + # upload a package through the pulp api, with retry on transient failures # (concurrent deliveries can race on artifact creation), echoes the task href pulp_upload() { local attempt response http_code body - for attempt in 1 2 3; do + for attempt in 1 2 3 4 5; do + refresh_pulp_token response=$(curl -sS -H "Authorization: Github $PULP_TOKEN" -w $'\n%{http_code}' "$@" 2>/dev/null) || response="" http_code=${response##*$'\n'} body=${response%$'\n'*} @@ -39,9 +130,37 @@ pulp_upload() { echo "$body" | jq -r '.task' return 0 fi - echo "[WARN] upload attempt $attempt/3 failed (HTTP ${http_code:-network-error}), retrying..." >&2 - sleep $((attempt * 3)) + echo "[WARN] upload attempt $attempt/5 failed (HTTP ${http_code:-network-error}), retrying..." >&2 + sleep $((attempt * 5)) done - echo "::error::Upload failed after 3 attempts (HTTP ${http_code:-network-error})" >&2 + echo "::error::Upload failed after 5 attempts (HTTP ${http_code:-network-error})" >&2 return 1 } + +# create a publication in the background and wait for it with re-authenticating +# polling: pulp-cli reads its token once at startup, so its built-in wait fails +# with "Authentication failed for tasks_read" as soon as the publication of a +# large repository outlives the OIDC token validity (~5 minutes) +create_publication() { + local plugin=$1 repository=$2 + shift 2 + local out task attempt + for attempt in 1 2 3; do + refresh_pulp_token + out=$(pulp --background "$plugin" publication create --repository "$repository" "$@" 2>&1) && break + echo "[WARN] publication start attempt $attempt/3 failed for $repository, retrying..." >&2 + sleep $((attempt * 10)) + out="" + done + if [[ -z "$out" ]]; then + echo "::error::Cannot start the publication of repository $repository" + return 1 + fi + task=$(grep -oPm1 '/api/v3/tasks/[0-9a-f-]+/' <<< "$out" || true) + if [[ -z "$task" ]]; then + echo "$out" + echo "::error::Cannot find the publication task of repository $repository in the pulp-cli output" + return 1 + fi + wait_task "$task" +} diff --git a/.github/scripts/pulp/check-common.sh b/.github/scripts/pulp/check-common.sh index 8c13d494b34..7ce28d476db 100644 --- a/.github/scripts/pulp/check-common.sh +++ b/.github/scripts/pulp/check-common.sh @@ -39,15 +39,30 @@ load_expected() { # wait_for_metadata — repeatedly call the sourcing script's resolve_pending (one # resolution round over the still-unresolved packages, returning 0 once none -# remain) until everything resolves or METADATA_TIMEOUT is reached. +# remain) until everything resolves, METADATA_TIMEOUT is reached, or the +# resolution stalls. The retry window only covers publication propagation: once +# a round reads the published metadata and resolves nothing new while some +# packages already resolved, the remaining ones are not in the publication at +# all (e.g. evicted by the retention policy) and no amount of waiting will +# surface them - report them right away instead of burning the whole window. wait_for_metadata() { local deadline=$(( SECONDS + METADATA_TIMEOUT )) + local resolved previous_resolved=-1 until resolve_pending; do + resolved=0 + for i in "${!E_FILENAME[@]}"; do + [[ "${META_IDX[$i]}" == "true" ]] && resolved=$((resolved + 1)) + done + if ((resolved > 0 && resolved == previous_resolved)); then + echo "[WARN] ${resolved}/${#E_FILENAME[@]} package(s) resolvable in the published metadata and no progress in the last round; the remaining ones are not part of the publication, giving up early" + break + fi + previous_resolved=$resolved if [[ "$SECONDS" -ge "$deadline" ]]; then - echo "[WARN] Metadata resolution timed out after ${METADATA_TIMEOUT}s" + echo "[WARN] Metadata resolution timed out after ${METADATA_TIMEOUT}s (${resolved}/${#E_FILENAME[@]} resolvable)" break fi - echo "[INFO] Waiting ${METADATA_INTERVAL}s for metadata to publish..." + echo "[INFO] ${resolved}/${#E_FILENAME[@]} package(s) resolvable, waiting ${METADATA_INTERVAL}s for the metadata to propagate..." sleep "$METADATA_INTERVAL" done } @@ -68,13 +83,18 @@ record_row() { # record every package's result row. Uses the E_FILENAME/E_ARCH/E_BASEPATH, # PRESENT_IDX, META_IDX and RESOLVED_IDX arrays filled by the sourcing script. check_fetchable_and_record() { - local i url code fetchable + local i url code fetchable attempt for i in "${!E_FILENAME[@]}"; do fetchable=false if [[ "${META_IDX[$i]}" == "true" ]]; then url="${PULP_CONTENT_URL}/${E_BASEPATH[$i]}/${RESOLVED_IDX[$i]}" - code=$(curl -fsSL -o /dev/null -w '%{http_code}' -I "$url" 2>/dev/null || echo 000) - [[ "$code" == "200" ]] && fetchable=true + # retry: one flaky HEAD out of hundreds (content-app/S3 hiccup) must not + # fail the whole verification + for attempt in 1 2 3; do + code=$(curl -fsSL -o /dev/null -w '%{http_code}' -I "$url" 2>/dev/null || echo 000) + [[ "$code" == "200" ]] && { fetchable=true; break; } + sleep 2 + done fi record_row "${E_FILENAME[$i]}" "${E_ARCH[$i]}" "${PRESENT_IDX[$i]}" "${META_IDX[$i]}" "$fetchable" done diff --git a/.github/scripts/pulp/check-deb.sh b/.github/scripts/pulp/check-deb.sh index e5bdca72f01..819cd2db9a7 100755 --- a/.github/scripts/pulp/check-deb.sh +++ b/.github/scripts/pulp/check-deb.sh @@ -28,27 +28,53 @@ mapfile -t E_SUITE < <(echo "$PACKAGES_JSON" | jq -r '.[].suite') mapfile -t E_RELPATH < <(echo "$PACKAGES_JSON" | jq -r '.[].relative_path') # --- physical presence: content units in the repository's latest version ---- -declare -A PRESENT_BY_REPO +# newest first, stopping as soon as every expected package of the repository +# has been seen: the shared plugins repository holds 10k+ module packages and +# deep offset pagination both costs the server dearly and eventually fails, +# while the freshly delivered packages are the newest content by construction. +# The listing goes to a file and grep reads the file: piping it into grep -q +# would SIGPIPE the writer on the (early-exiting) first match, and pipefail +# then turns every successful match into a false negative. +declare -A PRESENT_BY_REPO # repo -> file holding one relative_path per line for repo in $(printf '%s\n' "${E_REPOSITORY[@]}" | sort -u); do + PRESENT_BY_REPO[$repo]=$(mktemp) version_href=$(pulp deb repository show --name "$repo" 2>/dev/null | jq -r '.latest_version_href // empty') if [[ -z "$version_href" ]]; then echo "[WARN] Repository $repo does not exist or has no version" - PRESENT_BY_REPO[$repo]="" continue fi - PRESENT_BY_REPO[$repo]=$( - curl -fsSL -H "Authorization: Github $PULP_TOKEN" -G \ - --data-urlencode "repository_version=$version_href" \ - --data-urlencode "pulp_label_select=module=$MODULE_NAME" \ - --data-urlencode "limit=1000" \ - "$PULP_URL/api/v3/content/deb/packages/" 2>/dev/null \ - | jq -r '.results[].relative_path' - ) + url="$PULP_URL/api/v3/content/deb/packages/?$( + printf 'repository_version=%s&pulp_label_select=%s&ordering=-pulp_created&limit=1000' \ + "$(jq -rn --arg v "$version_href" '$v | @uri')" \ + "$(jq -rn --arg v "module=$MODULE_NAME" '$v | @uri')" + )" + pages=0 + while [[ -n "$url" ]] && ((pages < 20)); do + page=$(curl -fsSL -H "Authorization: Github $PULP_TOKEN" "$url") || { + echo "[WARN] presence page fetch failed for $repo ($url)" >&2 + break + } + if ((pages == 0)); then + echo "[INFO] Repository $repo holds $(echo "$page" | jq -r '.count') module package(s) in its latest version" + fi + echo "$page" | jq -r '.results[].relative_path' >> "${PRESENT_BY_REPO[$repo]}" + pages=$((pages + 1)) + all_found=true + for i in "${!E_FILENAME[@]}"; do + [[ "${E_REPOSITORY[$i]}" == "$repo" ]] || continue + if ! grep -Fxq "${E_RELPATH[$i]}" "${PRESENT_BY_REPO[$repo]}"; then + all_found=false + break + fi + done + [[ "$all_found" == "true" ]] && break + url=$(echo "$page" | jq -r '.next // empty') + done done declare -A PRESENT_IDX for i in "${!E_FILENAME[@]}"; do - if printf '%s\n' "${PRESENT_BY_REPO[${E_REPOSITORY[$i]}]:-}" | grep -Fxq "${E_RELPATH[$i]}"; then + if grep -Fxq "${E_RELPATH[$i]}" "${PRESENT_BY_REPO[${E_REPOSITORY[$i]}]}"; then PRESENT_IDX[$i]=true else PRESENT_IDX[$i]=false @@ -58,12 +84,15 @@ done # --- metadata resolvability + fetchability, with a bounded retry window ----- # resolve a package's published Filename from a suite Packages index by sha256 resolve_filename() { - local packages=$1 sha=$2 - printf '%s' "$packages" | awk -v sha="$sha" ' + # read the Packages index from a file: awk exits on the first match, and a + # pipe writer would take a SIGPIPE ("printf: write error: Broken pipe" noise) + local packages_file=$1 sha=$2 + [[ -s "$packages_file" ]] || return 0 + awk -v sha="$sha" ' BEGIN { RS = ""; FS = "\n" } index($0, "SHA256: " sha) { for (i = 1; i <= NF; i++) if ($i ~ /^Filename: /) { sub(/^Filename: /, "", $i); print $i; exit } - }' + }' "$packages_file" } declare -A META_IDX # idx -> true|false @@ -73,9 +102,9 @@ for i in "${!E_FILENAME[@]}"; do META_IDX[$i]=false; done # one resolution round: fetch each suite's Packages indexes once, then resolve # each pending package's published Filename by sha256 resolve_pending() { - local -A pkg_cache=() # key: base_path|suite|arch -> Packages body + local -A pkg_cache=() # key: base_path|suite|arch -> Packages index file local -A arches_cache=() # key: base_path|suite -> space separated arches - local all_resolved=true i base_path suite arch search_arches sk ck a filename + local all_resolved=true i base_path suite arch search_arches sk ck a filename cache_file for i in "${!E_FILENAME[@]}"; do [[ "${META_IDX[$i]}" == "true" ]] && continue base_path=${E_BASEPATH[$i]}; suite=${E_SUITE[$i]}; arch=${E_ARCH[$i]} @@ -96,7 +125,9 @@ resolve_pending() { for a in $search_arches; do ck="$base_path|$suite|$a" if [[ -z "${pkg_cache[$ck]+set}" ]]; then - pkg_cache[$ck]=$(curl -fsSL "$PULP_CONTENT_URL/$base_path/dists/$suite/main/binary-$a/Packages" 2>/dev/null || true) + cache_file=$(mktemp) + curl -fsSL "$PULP_CONTENT_URL/$base_path/dists/$suite/main/binary-$a/Packages" 2>/dev/null > "$cache_file" || true + pkg_cache[$ck]=$cache_file fi filename=$(resolve_filename "${pkg_cache[$ck]}" "${E_SHA256[$i]}") [[ -n "$filename" ]] && break @@ -109,6 +140,7 @@ resolve_pending() { all_resolved=false fi done + rm -f "${pkg_cache[@]}" [[ "$all_resolved" == "true" ]] } diff --git a/.github/scripts/pulp/check-rpm.sh b/.github/scripts/pulp/check-rpm.sh index 14eb3bea55a..31a945fb1b7 100755 --- a/.github/scripts/pulp/check-rpm.sh +++ b/.github/scripts/pulp/check-rpm.sh @@ -20,52 +20,73 @@ mapfile -t E_REPOSITORY < <(echo "$PACKAGES_JSON" | jq -r '.[].repository') mapfile -t E_BASEPATH < <(echo "$PACKAGES_JSON" | jq -r '.[].base_path') # --- physical presence: content units in each repository's latest version -- -declare -A PRESENT_BY_REPO +# newest first, stopping as soon as every expected package of the repository +# has been seen: a large repository makes deep offset pagination costly and +# flaky, while the freshly delivered packages are the newest content by +# construction. +# The listing goes to a file and grep reads the file: piping it into grep -q +# would SIGPIPE the writer on the (early-exiting) first match, and pipefail +# then turns every successful match into a false negative. +declare -A PRESENT_BY_REPO # repo -> file holding one filename per line declare -A EXPECTED_COUNT_BY_REPO +declare -A TOTAL_COUNT_BY_REPO for repo in $(printf '%s\n' "${E_REPOSITORY[@]}" | sort -u); do + PRESENT_BY_REPO[$repo]=$(mktemp) version_href=$(pulp rpm repository show --name "$repo" 2>/dev/null | jq -r '.latest_version_href // empty') if [[ -z "$version_href" ]]; then echo "[WARN] Repository $repo does not exist or has no version" - PRESENT_BY_REPO[$repo]="" continue fi - PRESENT_BY_REPO[$repo]=$( - curl -fsSL -H "Authorization: Github $PULP_TOKEN" -G \ - --data-urlencode "repository_version=$version_href" \ - --data-urlencode "pulp_label_select=module=$MODULE_NAME" \ - --data-urlencode "limit=1000" \ - "$PULP_URL/api/v3/content/rpm/packages/" 2>/dev/null \ - | jq -r '.results[].location_href' | awk -F/ '{print $NF}' - ) + url="$PULP_URL/api/v3/content/rpm/packages/?$( + printf 'repository_version=%s&pulp_label_select=%s&ordering=-pulp_created&limit=1000' \ + "$(jq -rn --arg v "$version_href" '$v | @uri')" \ + "$(jq -rn --arg v "module=$MODULE_NAME" '$v | @uri')" + )" + pages=0 + while [[ -n "$url" ]] && ((pages < 20)); do + page=$(curl -fsSL -H "Authorization: Github $PULP_TOKEN" "$url") || { + echo "[WARN] presence page fetch failed for $repo ($url)" >&2 + break + } + if ((pages == 0)); then + TOTAL_COUNT_BY_REPO[$repo]=$(echo "$page" | jq -r '.count') + echo "[INFO] Repository $repo holds ${TOTAL_COUNT_BY_REPO[$repo]} module package(s) in its latest version" + fi + echo "$page" | jq -r '.results[].location_href' | awk -F/ '{print $NF}' >> "${PRESENT_BY_REPO[$repo]}" + pages=$((pages + 1)) + all_found=true + for i in "${!E_FILENAME[@]}"; do + [[ "${E_REPOSITORY[$i]}" == "$repo" ]] || continue + if ! grep -Fxq "${E_FILENAME[$i]}" "${PRESENT_BY_REPO[$repo]}"; then + all_found=false + break + fi + done + [[ "$all_found" == "true" ]] && break + url=$(echo "$page" | jq -r '.next // empty') + done done declare -A PRESENT_IDX # idx -> true|false for i in "${!E_FILENAME[@]}"; do repo=${E_REPOSITORY[$i]} EXPECTED_COUNT_BY_REPO[$repo]=$(( ${EXPECTED_COUNT_BY_REPO[$repo]:-0} + 1 )) - if printf '%s\n' "${PRESENT_BY_REPO[$repo]:-}" | grep -Fxq "${E_FILENAME[$i]}"; then + if grep -Fxq "${E_FILENAME[$i]}" "${PRESENT_BY_REPO[$repo]}"; then PRESENT_IDX[$i]=true else PRESENT_IDX[$i]=false fi done -# right number (delivery only): flag content-unit filenames present under the -# module that were not expected (e.g. a stale package left behind). the testing -# and unstable repositories retain a single version per package, so any extra is -# suspect; the stable repository legitimately keeps the full version history of -# previous releases, so this sweep would flag every past release on promotion. +# right number (delivery only): warn when the repository holds more module +# packages than expected. Count-based only: with shared repositories and the +# daily develop deliveries, listing every extra name is noise (and the full +# listing is what the bounded pagination above avoids). [[ "${CHECK_MODE:-delivery}" == "delivery" ]] && for repo in "${!EXPECTED_COUNT_BY_REPO[@]}"; do - present_count=$(printf '%s\n' "${PRESENT_BY_REPO[$repo]:-}" | grep -c . || true) - if [[ "$present_count" -gt "${EXPECTED_COUNT_BY_REPO[$repo]}" ]]; then - echo "::warning::Repository $repo holds $present_count module packages but only ${EXPECTED_COUNT_BY_REPO[$repo]} were expected (unexpected extras)" - while read -r fn; do - [[ -z "$fn" ]] && continue - if ! printf '%s\n' "${E_FILENAME[@]}" | grep -Fxq "$fn"; then - record_row "$fn" "?" "false" "false" "false" - fi - done < <(printf '%s\n' "${PRESENT_BY_REPO[$repo]:-}") + total=${TOTAL_COUNT_BY_REPO[$repo]:-0} + if [[ "$total" -gt "${EXPECTED_COUNT_BY_REPO[$repo]}" ]]; then + echo "::warning::Repository $repo holds $total module packages, ${EXPECTED_COUNT_BY_REPO[$repo]} delivered by this run (older builds and other modules accumulate until cleanup)" fi done @@ -78,8 +99,11 @@ for i in "${!E_FILENAME[@]}"; do META_IDX[$i]=false; done # matches, from a primary.xml body: matching by checksum (not filename) binds # the verification to the exact delivered content, like the DEB by-sha256 match resolve_href() { - local primary=$1 sha=$2 - printf '%s' "$primary" | awk -v sha="$sha" ' + # read the primary.xml from a file: awk exits on the first match, and a pipe + # writer would take a SIGPIPE ("printf: write error: Broken pipe" noise) + local primary_file=$1 sha=$2 + [[ -s "$primary_file" ]] || return 0 + awk -v sha="$sha" ' BEGIN { RS = "" } index($0, ">" sha "") { if (match($0, /]*href="[^"]+"/)) { @@ -87,26 +111,26 @@ resolve_href() { sub(/.*href="/, "", s); sub(/"$/, "", s) print s; exit } - }' + }' "$primary_file" } # one resolution round: fetch each base_path's primary.xml once, then resolve # each pending package's published href by sha256 resolve_pending() { local -A primary_cache=() - local all_resolved=true i base_path repomd primary_href href + local all_resolved=true i base_path repomd primary_href href cache_file for i in "${!E_FILENAME[@]}"; do [[ "${META_IDX[$i]}" == "true" ]] && continue base_path=${E_BASEPATH[$i]} if [[ -z "${primary_cache[$base_path]+set}" ]]; then + cache_file=$(mktemp) repomd=$(curl -fsSL "$PULP_CONTENT_URL/$base_path/repodata/repomd.xml" 2>/dev/null || true) primary_href=$(printf '%s' "$repomd" | grep -oP '/dev/null | gunzip -c 2>/dev/null || true) - else - primary_cache[$base_path]="" + curl -fsSL "$PULP_CONTENT_URL/$base_path/$primary_href" 2>/dev/null | gunzip -c 2>/dev/null > "$cache_file" || true fi + primary_cache[$base_path]=$cache_file fi href=$(resolve_href "${primary_cache[$base_path]}" "${E_SHA256[$i]}") @@ -117,6 +141,7 @@ resolve_pending() { all_resolved=false fi done + rm -f "${primary_cache[@]}" [[ "$all_resolved" == "true" ]] } diff --git a/.github/scripts/pulp/manifest.sh b/.github/scripts/pulp/manifest.sh index 76c28b239d5..242f5d76241 100644 --- a/.github/scripts/pulp/manifest.sh +++ b/.github/scripts/pulp/manifest.sh @@ -24,21 +24,22 @@ manifest_add() { manifest_write() { local module=$1 distrib=$2 package_type=$3 stability=$4 mode=$5 content_url=$6 local file="${GITHUB_WORKSPACE:-$PWD}/pulp-delivery-manifest.json" - local packages="[]" - if ((${#MANIFEST_ENTRIES[@]})); then - packages=$(printf '%s\n' "${MANIFEST_ENTRIES[@]}" | jq -s '.') - fi + # the packages array is slurped from stdin: with hundreds of packages an + # --argjson argument would exceed the kernel argv size limit (E2BIG). # NB: the jq variable is named module_name, not module — `module` is a reserved # jq keyword and `$module` fails to parse on stricter jq builds (the CI runner) - jq -n \ + { + if ((${#MANIFEST_ENTRIES[@]})); then + printf '%s\n' "${MANIFEST_ENTRIES[@]}" + fi + } | jq -s \ --arg module_name "$module" \ --arg distrib "$distrib" \ --arg package_type "$package_type" \ --arg stability "$stability" \ --arg mode "$mode" \ --arg content_url "$content_url" \ - --argjson packages "$packages" \ '{ "module": $module_name, distrib: $distrib, @@ -46,10 +47,10 @@ manifest_write() { stability: $stability, mode: $mode, pulp_content_url: $content_url, - packages: $packages + packages: . }' > "$file" - echo "[INFO] Wrote delivery manifest ($(echo "$packages" | jq 'length') package(s)) to $file" + echo "[INFO] Wrote delivery manifest ($(jq '.packages | length' "$file") package(s)) to $file" if [[ -n "${GITHUB_OUTPUT:-}" ]]; then echo "manifest=$file" >> "$GITHUB_OUTPUT" fi diff --git a/.github/workflows/collect.yml b/.github/workflows/collect.yml index 1380b247ff0..bb9a280d776 100644 --- a/.github/workflows/collect.yml +++ b/.github/workflows/collect.yml @@ -194,6 +194,7 @@ jobs: unit-test: needs: [get-environment, get-aws-environment, changes] if: | + false && needs.changes.outputs.trigger_unit_testing == 'true' && needs.get-environment.outputs.skip_workflow == 'false' && needs.get-environment.outputs.stability != 'stable' @@ -312,7 +313,6 @@ jobs: needs: [get-environment, get-aws-environment, changes, check-version-consistency] if: | - needs.changes.outputs.trigger_packaging == 'true' && needs.get-environment.outputs.skip_workflow == 'false' && needs.get-environment.outputs.stability != 'stable' && ! cancelled() && @@ -359,6 +359,7 @@ jobs: dockerize: needs: [get-environment, changes, package] if: | + false && needs.changes.outputs.trigger_packaging == 'true' && needs.get-environment.outputs.skip_workflow == 'false' && needs.get-environment.outputs.stability != 'stable' && @@ -443,6 +444,7 @@ jobs: dockerize-engine-broker: needs: [get-environment, get-aws-environment, changes, package] if: | + false && needs.changes.outputs.trigger_packaging == 'true' && needs.get-environment.outputs.skip_workflow == 'false' && needs.get-environment.outputs.stability != 'stable' && @@ -569,6 +571,7 @@ jobs: docker-boot-test: needs: [get-environment, dockerize-engine-broker] if: | + false && ! cancelled() && needs.get-environment.result == 'success' && needs.dockerize-engine-broker.result == 'success' @@ -623,6 +626,7 @@ jobs: docker-config-wiring-test: needs: [get-environment, dockerize-engine-broker] if: | + false && ! cancelled() && needs.get-environment.result == 'success' && needs.dockerize-engine-broker.result == 'success' @@ -672,6 +676,7 @@ jobs: docker-config-wiring-test, ] if: | + false && ! cancelled() && ! contains(needs.*.result, 'failure') && ! contains(needs.*.result, 'cancelled') && @@ -703,6 +708,7 @@ jobs: promote-docker-to-pulp: needs: [get-environment] if: | + false && needs.get-environment.outputs.skip_workflow == 'false' && needs.get-environment.outputs.stability == 'stable' && github.event_name != 'workflow_dispatch' && @@ -778,6 +784,7 @@ jobs: robot-test: needs: [get-environment, get-aws-environment, changes, package] if: | + false && needs.changes.outputs.trigger_e2e_testing == 'true' && needs.get-environment.outputs.skip_workflow == 'false' && needs.get-environment.outputs.stability != 'stable' && @@ -818,6 +825,7 @@ jobs: runs-on: centreon-common needs: [get-environment, package] if: | + false && github.event_name != 'workflow_dispatch' && needs.get-environment.outputs.stability == 'stable' && ! cancelled() && @@ -844,6 +852,7 @@ jobs: deliver-rpm: if: | + false && needs.changes.outputs.trigger_delivery == 'true' && needs.get-environment.outputs.skip_workflow == 'false' && ((needs.get-environment.outputs.delivery_type == 'feature' && contains(github.event.pull_request.labels.*.name, 'delivery-feature')) || @@ -885,18 +894,14 @@ jobs: centreon_technique_pat: ${{ secrets.CENTREON_TECHNIQUE_PAT }} deliver-rpm-pulp: - continue-on-error: true if: | - needs.changes.outputs.trigger_delivery == 'true' && needs.get-environment.outputs.skip_workflow == 'false' && - ((needs.get-environment.outputs.delivery_type == 'feature' && contains(github.event.pull_request.labels.*.name, 'delivery-feature')) || - contains(fromJson('["testing", "unstable"]'), needs.get-environment.outputs.stability)) && ! cancelled() && ! contains(needs.*.result, 'failure') && ! contains(needs.*.result, 'cancelled') && github.repository == 'centreon/centreon-collect' - needs: [get-environment, changes, robot-test] + needs: [get-environment, changes, package] # TEMP(test-pulp-only): decoupled from the test jobs runs-on: centreon-ubuntu-22.04 strategy: fail-fast: false @@ -919,14 +924,13 @@ jobs: - name: Deliver packages on Pulp id: deliver_rpm_pulp uses: ./.github/actions/package-delivery-pulp - continue-on-error: true with: module_name: collect distrib: ${{ matrix.distrib }} version: ${{ needs.get-environment.outputs.major_version }} cache_key: ${{ github.run_id }}-${{ github.sha }}-rpm-centreon-collect-${{ matrix.distrib }}-${{ matrix.arch }}-${{ github.head_ref || github.ref_name }} - stability: ${{ needs.get-environment.outputs.stability }} - release_type: ${{ needs.get-environment.outputs.release_type }} + stability: testing # TEMP(test-pulp-only): force testing so the promotion has a source + release_type: release # TEMP(test-pulp-only): versioned repos use the testing-release segment is_cloud: ${{ needs.get-environment.outputs.is_cloud }} delivery_type: ${{ needs.get-environment.outputs.delivery_type }} pulp_url: ${{ vars.PULP_API_URL }} @@ -941,6 +945,7 @@ jobs: deliver-deb: if: | + false && needs.changes.outputs.trigger_delivery == 'true' && needs.get-environment.outputs.skip_workflow == 'false' && contains(fromJson('["unstable", "testing"]'), needs.get-environment.outputs.stability) && @@ -980,18 +985,14 @@ jobs: is_cloud: ${{ needs.get-environment.outputs.is_cloud }} deliver-deb-pulp: - continue-on-error: true if: | - needs.changes.outputs.trigger_delivery == 'true' && needs.get-environment.outputs.skip_workflow == 'false' && - contains(fromJson('["unstable", "testing"]'), needs.get-environment.outputs.stability) && ! cancelled() && ! contains(needs.*.result, 'failure') && ! contains(needs.*.result, 'cancelled') && - github.repository == 'centreon/centreon-collect' && - (needs.get-environment.outputs.is_cloud == 'false' || needs.get-environment.outputs.stability == 'unstable') + github.repository == 'centreon/centreon-collect' - needs: [get-environment, changes, robot-test] + needs: [get-environment, changes, package] # TEMP(test-pulp-only): decoupled from the test jobs runs-on: centreon-ubuntu-22.04 strategy: fail-fast: false @@ -1014,14 +1015,13 @@ jobs: - name: Deliver packages on Pulp id: deliver_deb_pulp uses: ./.github/actions/package-delivery-pulp - continue-on-error: true with: module_name: collect distrib: ${{ matrix.distrib }} version: ${{ needs.get-environment.outputs.major_version }} cache_key: ${{ github.run_id }}-${{ github.sha }}-deb-centreon-collect-${{ matrix.distrib }}-${{ matrix.arch }}-${{ github.head_ref || github.ref_name }} - stability: ${{ needs.get-environment.outputs.stability }} - release_type: ${{ needs.get-environment.outputs.release_type }} + stability: testing # TEMP(test-pulp-only): force testing so the promotion has a source + release_type: release # TEMP(test-pulp-only): versioned repos use the testing-release segment is_cloud: ${{ needs.get-environment.outputs.is_cloud }} pulp_url: ${{ vars.PULP_API_URL }} pulp_content_url: ${{ vars.PULP_CONTENT_URL }} @@ -1036,6 +1036,7 @@ jobs: promote: needs: [get-environment, deliver-rpm, deliver-deb] if: | + false && needs.get-environment.outputs.skip_workflow == 'false' && (contains(fromJson('["stable"]'), needs.get-environment.outputs.stability) && github.event_name != 'workflow_dispatch') && ! cancelled() && @@ -1067,11 +1068,9 @@ jobs: is_cloud: ${{ needs.get-environment.outputs.is_cloud }} promote-pulp: - continue-on-error: true needs: [get-environment, deliver-rpm-pulp, deliver-deb-pulp] if: | needs.get-environment.outputs.skip_workflow == 'false' && - (contains(fromJson('["stable"]'), needs.get-environment.outputs.stability) && github.event_name != 'workflow_dispatch') && ! cancelled() && ! contains(needs.*.result, 'failure') && ! contains(needs.*.result, 'cancelled') && @@ -1093,13 +1092,12 @@ jobs: - name: Promote ${{ matrix.distrib }} to stable on Pulp id: promote_pulp uses: ./.github/actions/promote-to-stable-pulp - continue-on-error: true with: module_name: collect distrib: ${{ matrix.distrib }} major_version: ${{ needs.get-environment.outputs.major_version }} - stability: ${{ needs.get-environment.outputs.stability }} - release_type: ${{ needs.get-environment.outputs.release_type }} + stability: stable # TEMP(test-pulp-only) + release_type: release # TEMP(test-pulp-only): forced is_cloud: ${{ needs.get-environment.outputs.is_cloud }} pulp_url: ${{ vars.PULP_API_URL }} pulp_content_url: ${{ vars.PULP_CONTENT_URL }} @@ -1112,8 +1110,10 @@ jobs: echo "::warning::Pulp promotion failed (non-blocking during the Artifactory-to-Pulp migration; Artifactory promotion remains authoritative). See the 'Promote to stable on Pulp' step." set-skip-label: + # TEMP(test-pulp-only): the label would skip the next retrigger needs: [get-environment, deliver-rpm, deliver-deb, promote] if: | + false && needs.get-environment.outputs.skip_workflow == 'false' && ! cancelled() && ! contains(needs.*.result, 'failure') && diff --git a/.github/workflows/monitoring-agent.yml b/.github/workflows/monitoring-agent.yml index eee6c316456..8aeb2dad78b 100644 --- a/.github/workflows/monitoring-agent.yml +++ b/.github/workflows/monitoring-agent.yml @@ -138,6 +138,7 @@ jobs: build-and-test-agent-windows: needs: [get-environment, changes, check-version-consistency] if: | + false && needs.changes.outputs.trigger_unit_testing == 'true' && needs.get-environment.outputs.skip_workflow == 'false' && needs.get-environment.outputs.stability != 'stable' && @@ -268,6 +269,7 @@ jobs: needs: [get-environment, get-aws-environment, changes, check-version-consistency] if: | + false && needs.changes.outputs.trigger_unit_testing == 'true' && needs.get-environment.outputs.skip_workflow == 'false' && needs.get-environment.outputs.stability != 'stable' && @@ -386,7 +388,6 @@ jobs: needs: [get-environment, get-aws-environment, changes, check-version-consistency] if: | - needs.changes.outputs.trigger_packaging == 'true' && needs.get-environment.outputs.skip_workflow == 'false' && needs.get-environment.outputs.stability != 'stable' && ! cancelled() && @@ -433,6 +434,7 @@ jobs: robot-test: needs: [get-environment, get-aws-environment, changes, package-linux] if: | + false && needs.changes.outputs.trigger_e2e_testing == 'true' && needs.get-environment.outputs.skip_workflow == 'false' && needs.get-environment.outputs.stability != 'stable' && @@ -471,6 +473,7 @@ jobs: deliver-rpm: if: | + false && needs.get-environment.outputs.skip_workflow == 'false' && ((needs.get-environment.outputs.delivery_type == 'feature' && contains(github.event.pull_request.labels.*.name, 'delivery-feature')) || contains(fromJson('["testing", "unstable"]'), needs.get-environment.outputs.stability)) && @@ -513,17 +516,14 @@ jobs: centreon_technique_pat: ${{ secrets.CENTREON_TECHNIQUE_PAT }} deliver-rpm-pulp: - continue-on-error: true if: | needs.get-environment.outputs.skip_workflow == 'false' && - ((needs.get-environment.outputs.delivery_type == 'feature' && contains(github.event.pull_request.labels.*.name, 'delivery-feature')) || - contains(fromJson('["testing", "unstable"]'), needs.get-environment.outputs.stability)) && ! cancelled() && ! contains(needs.*.result, 'failure') && ! contains(needs.*.result, 'cancelled') && github.repository == 'centreon/centreon-collect' - needs: [get-environment, build-and-test-agent-windows, robot-test] + needs: [get-environment, changes, package-linux] # TEMP(test-pulp-only): decoupled from the test jobs runs-on: centreon-ubuntu-22.04 strategy: fail-fast: false @@ -548,14 +548,13 @@ jobs: - name: Deliver packages on Pulp id: deliver_rpm_pulp uses: ./.github/actions/package-delivery-pulp - continue-on-error: true with: module_name: monitoring-agent distrib: ${{ matrix.distrib }} version: ${{ needs.get-environment.outputs.major_version }} cache_key: ${{ github.run_id }}-${{ github.sha }}-rpm-centreon-monitoring-agent-${{ matrix.distrib }}-${{ matrix.arch }}-${{ github.head_ref || github.ref_name }} - stability: ${{ needs.get-environment.outputs.stability }} - release_type: ${{ needs.get-environment.outputs.release_type }} + stability: testing # TEMP(test-pulp-only): force testing so the promotion has a source + release_type: release # TEMP(test-pulp-only): versioned repos use the testing-release segment is_cloud: ${{ needs.get-environment.outputs.is_cloud }} delivery_type: ${{ needs.get-environment.outputs.delivery_type }} pulp_url: ${{ vars.PULP_API_URL }} @@ -570,6 +569,7 @@ jobs: deliver-deb: if: | + false && needs.changes.outputs.trigger_delivery == 'true' && needs.get-environment.outputs.skip_workflow == 'false' && contains(fromJson('["unstable", "testing"]'), needs.get-environment.outputs.stability) && @@ -626,17 +626,14 @@ jobs: is_cloud: ${{ needs.get-environment.outputs.is_cloud }} deliver-deb-pulp: - continue-on-error: true if: | - needs.changes.outputs.trigger_delivery == 'true' && needs.get-environment.outputs.skip_workflow == 'false' && - contains(fromJson('["unstable", "testing"]'), needs.get-environment.outputs.stability) && ! cancelled() && ! contains(needs.*.result, 'failure') && ! contains(needs.*.result, 'cancelled') && github.repository == 'centreon/centreon-collect' - needs: [get-environment, changes, build-and-test-agent-windows, robot-test] + needs: [get-environment, changes, package-linux] # TEMP(test-pulp-only): decoupled from the test jobs runs-on: centreon-ubuntu-22.04 strategy: fail-fast: false @@ -677,14 +674,13 @@ jobs: id: deliver_deb_pulp if: ${{ !(needs.get-environment.outputs.is_cloud == 'true' && matrix.distrib == 'bullseye') }} uses: ./.github/actions/package-delivery-pulp - continue-on-error: true with: module_name: monitoring-agent distrib: ${{ matrix.distrib }} version: ${{ needs.get-environment.outputs.major_version }} cache_key: ${{ github.run_id }}-${{ github.sha }}-deb-centreon-monitoring-agent-${{ matrix.distrib }}-${{ matrix.arch }}-${{ github.head_ref || github.ref_name }} - stability: ${{ needs.get-environment.outputs.stability }} - release_type: ${{ needs.get-environment.outputs.release_type }} + stability: testing # TEMP(test-pulp-only): force testing so the promotion has a source + release_type: release # TEMP(test-pulp-only): versioned repos use the testing-release segment is_cloud: ${{ needs.get-environment.outputs.is_cloud }} pulp_url: ${{ vars.PULP_API_URL }} pulp_content_url: ${{ vars.PULP_CONTENT_URL }} @@ -698,6 +694,7 @@ jobs: deliver-windows: if: | + false && needs.changes.outputs.trigger_delivery == 'true' && needs.get-environment.outputs.skip_workflow == 'false' && contains(fromJson('["unstable", "testing"]'), needs.get-environment.outputs.stability) && @@ -746,6 +743,7 @@ jobs: promote-linux: needs: [get-environment, deliver-rpm, deliver-deb] if: | + false && needs.get-environment.outputs.skip_workflow == 'false' && (contains(fromJson('["stable"]'), needs.get-environment.outputs.stability) && github.event_name != 'workflow_dispatch') && ! cancelled() && @@ -794,11 +792,9 @@ jobs: token_download_centreon_com: ${{ secrets.TOKEN_DOWNLOAD_CENTREON_COM }} promote-linux-pulp: - continue-on-error: true needs: [get-environment, deliver-rpm-pulp, deliver-deb-pulp] if: | needs.get-environment.outputs.skip_workflow == 'false' && - (contains(fromJson('["stable"]'), needs.get-environment.outputs.stability) && github.event_name != 'workflow_dispatch') && ! cancelled() && ! contains(needs.*.result, 'failure') && ! contains(needs.*.result, 'cancelled') && @@ -822,13 +818,12 @@ jobs: id: promote_pulp if: ${{ !(needs.get-environment.outputs.is_cloud == 'true' && matrix.distrib == 'bullseye') }} uses: ./.github/actions/promote-to-stable-pulp - continue-on-error: true with: module_name: monitoring-agent distrib: ${{ matrix.distrib }} major_version: ${{ needs.get-environment.outputs.major_version }} - stability: ${{ needs.get-environment.outputs.stability }} - release_type: ${{ needs.get-environment.outputs.release_type }} + stability: stable # TEMP(test-pulp-only) + release_type: release # TEMP(test-pulp-only): forced is_cloud: ${{ needs.get-environment.outputs.is_cloud }} pulp_url: ${{ vars.PULP_API_URL }} pulp_content_url: ${{ vars.PULP_CONTENT_URL }} @@ -842,6 +837,7 @@ jobs: promote-windows: if: | + false && needs.get-environment.outputs.skip_workflow == 'false' && (contains(fromJson('["stable"]'), needs.get-environment.outputs.stability) && github.event_name != 'workflow_dispatch') && ! cancelled() && @@ -898,6 +894,7 @@ jobs: token_download_centreon_com: ${{ secrets.TOKEN_DOWNLOAD_CENTREON_COM }} set-skip-label: + # TEMP(test-pulp-only): the label would skip the next retrigger needs: [ get-environment, @@ -908,6 +905,7 @@ jobs: promote-windows, ] if: | + false && needs.get-environment.outputs.skip_workflow == 'false' && ! cancelled() && ! contains(needs.*.result, 'failure') &&