diff --git a/.github/workflows/deploy-kuiper-artifacts.yml b/.github/workflows/deploy-kuiper-artifacts.yml new file mode 100644 index 00000000000000..fb2436307a4968 --- /dev/null +++ b/.github/workflows/deploy-kuiper-artifacts.yml @@ -0,0 +1,203 @@ +name: Deploy Kuiper Artifacts to Cloudsmith + +on: + workflow_call: + inputs: + artifacts: + description: 'Space-separated patterns of artifact names to download' + required: true + type: string + pr-target-branch: + description: 'PR target branch (for PR deployments)' + required: false + type: string + default: '' + pr-number: + description: 'PR number (for PR deployments)' + required: false + type: string + default: '' + scripts-ref: + description: 'Branch/tag to fetch CI scripts from (default: ci)' + required: false + type: string + default: 'ci' + +permissions: + id-token: write + contents: read + actions: read + +jobs: + deploy-artifacts: + runs-on: ubuntu-latest + timeout-minutes: 30 + permissions: + id-token: write + contents: read + actions: read + + steps: + - name: Detect artifact type from branch + id: detect + run: | + BRANCH_NAME="${{ github.head_ref || github.ref_name }}" + echo "::notice::Branch: ${BRANCH_NAME}" + + if [[ "${BRANCH_NAME}" == *"rpi"* ]]; then + echo "ARTIFACT_TYPE=rpi" >> $GITHUB_ENV + echo "CLOUDSMITH_REPO=sdg-linux-rpi" >> $GITHUB_ENV + else + echo "ARTIFACT_TYPE=kuiper" >> $GITHUB_ENV + echo "CLOUDSMITH_REPO=sdg-linux" >> $GITHUB_ENV + fi + + - name: Prepare directories + run: | + sudo apt-get update && sudo apt-get install -y tree + rm -rf artifacts raw dist + mkdir -p artifacts raw dist + + - name: Get sources + run: | + # Fetch CI scripts from the specified branch (scripts-ref input) + org_repo="analogdevicesinc/linux" + ref="${{ inputs.scripts-ref }}" + + echo "::notice::Fetching scripts from: $org_repo @ $ref" + + get_file() { + curl -sL -H "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}" \ + -o "$1" \ + "https://raw.githubusercontent.com/${org_repo}/${ref}/$1" + } + + mkdir -p ci + get_file ci/lib.sh + get_file ci/lib_github.sh + + if [[ "${{ env.ARTIFACT_TYPE }}" == "kuiper" ]]; then + get_file ci/expand_kernel_images.sh + get_file ci/prepare_kuiper_artifacts_structure.sh + else + get_file ci/prepare_rpi_artifacts.sh + fi + + chmod +x ci/*.sh + + curl -sL -o upload_to_cloudsmith.py \ + https://raw.githubusercontent.com/analogdevicesinc/wiki-scripts/main/utils/cloudsmith_utils/upload_to_cloudsmith.py + + echo "::debug::Scripts downloaded:" + ls -la ci/ + + - name: Download, extract, and process artifacts + run: | + WORK_DIR="$(pwd)" + source ci/lib.sh + source ci/lib_github.sh + + # Download matching artifacts from workflow run + download_matching_artifacts \ + "${{ secrets.GITHUB_TOKEN }}" \ + "${{ github.repository }}" \ + "${{ github.run_id }}" \ + "${{ inputs.artifacts }}" \ + "artifacts" + + # Extract to raw/ + extract_artifacts "artifacts" "raw" + echo "::debug::Contents of raw/:" + ls -la raw/ + + # Process based on artifact type + if [[ "${{ env.ARTIFACT_TYPE }}" == "kuiper" ]]; then + # Expand linux artifacts into structured dist/ for Kuiper format + source ci/expand_kernel_images.sh + prepare_kernel_dist + + else + export SOURCE_DIRECTORY="${WORK_DIR}/raw" + export OUTPUT_DIRECTORY="${WORK_DIR}/dist" + export BUILD_SOURCEBRANCHNAME="${{ github.head_ref || github.ref_name }}" + source ci/prepare_rpi_artifacts.sh + create_rpi_boot_archives + fi + + - name: Setup Cloudsmith OIDC + uses: cloudsmith-io/cloudsmith-cli-action@v1.0.5 + with: + oidc-namespace: ${{ vars.CLOUDSMITH_NAMESPACE }} + oidc-service-slug: ${{ secrets.CLOUDSMITH_SERVICE_SLUG }} + oidc-auth-only: 'true' + + - name: Prepare structure and upload to Cloudsmith + run: | + # Setup Python environment + python3 -m venv venv + source ./venv/bin/activate + python3 -m ensurepip + pip3 install cloudsmith-cli + + source ci/lib_github.sh + + # Set timestamp and git info + TIMESTAMP=$(date +%Y_%m_%d-%H_%M) + GIT_SHA="${{ github.event.pull_request.head.sha || github.sha }}" + GIT_SHA_DATE="${{ github.event.head_commit.timestamp }}" + + # Get branch name + # ToDo: Test PR: github.head_ref = feature-branch, github.ref_name = refs/pull/123/merge + BRANCH_NAME="${{ github.head_ref || github.ref_name }}" + + # Build version path + VERSION_PATH=$(get_version_path \ + "${{ env.ARTIFACT_TYPE }}" \ + "${BRANCH_NAME}" \ + "${TIMESTAMP}" \ + "${{ inputs.pr-target-branch }}" \ + "${{ inputs.pr-number }}") + + echo "::group::Upload configuration" + echo " Artifact type: ${{ env.ARTIFACT_TYPE }}" + echo " Repository: ${{ env.CLOUDSMITH_REPO }}" + echo " Version path: ${VERSION_PATH}" + echo " Git SHA: ${GIT_SHA}" + echo " Git SHA date: ${GIT_SHA_DATE}" + echo "::endgroup::" + + export TIMESTAMP GIT_SHA GIT_SHA_DATE + export BUILD_SOURCEBRANCHNAME="${BRANCH_NAME}" + export SOURCE_DIRECTORY="$(pwd)" + + # Determine upload directory based on artifact type + if [[ "${{ env.ARTIFACT_TYPE }}" == "kuiper" ]]; then + # Kuiper: prepare_kuiper_artifacts_structure.sh creates timestamped directory + export TIMESTAMP GIT_SHA GIT_SHA_DATE + export BUILD_SOURCEBRANCHNAME="${BRANCH_NAME}" + export SOURCE_DIRECTORY="$(pwd)" + source ci/lib.sh + source ci/prepare_kuiper_artifacts_structure.sh + UPLOAD_PATH="${TIMESTAMP}" + else + # RPI: artifacts are already in dist/ + UPLOAD_PATH="dist" + fi + + if [[ ! -d "${UPLOAD_PATH}" ]] || [[ -z "$(ls -A "${UPLOAD_PATH}" 2>/dev/null)" ]]; then + echo "::warning::No artifacts to upload in ${UPLOAD_PATH}" + exit 0 + fi + + echo "::notice::Uploading from: ${UPLOAD_PATH}" + ls -la "${UPLOAD_PATH}" + + python3 upload_to_cloudsmith.py \ + --repo="${{ env.CLOUDSMITH_REPO }}" \ + --version="${VERSION_PATH}" \ + --local_path="${UPLOAD_PATH}" \ + --token="${CLOUDSMITH_API_KEY}" \ + --max_workers=10 + + echo "::notice::Upload complete: ${VERSION_PATH}." + echo "::notice::Check Cloudsmith repository for details on https://cloudsmith.io/~${{ vars.CLOUDSMITH_NAMESPACE }}/repos/${{ env.CLOUDSMITH_REPO }}/packages/" diff --git a/.github/workflows/upload-to-cloudsmith.yml b/.github/workflows/upload-to-cloudsmith.yml index b394b062602f69..4e4ee7900ad349 100644 --- a/.github/workflows/upload-to-cloudsmith.yml +++ b/.github/workflows/upload-to-cloudsmith.yml @@ -6,6 +6,14 @@ on: artifacts: required: true type: string + cloudsmith-repository: + required: false + default: ${{ vars.CLOUDSMITH_REPOSITORY }} + type: string + version: + required: false + default: ${{ github.sha }} + type: string secrets: CLOUDSMITH_SERVICE_SLUG: required: false @@ -143,15 +151,15 @@ jobs: package_file=$( cs-package-upload ${{ env.CLOUDSMITH_API_KEY }} \ ${{ vars.CLOUDSMITH_NAMESPACE }} \ - ${{ vars.CLOUDSMITH_REPOSITORY }} \ + ${{ inputs.cloudsmith-repository }} \ "$file" ) cs-package-store-raw ${{ env.CLOUDSMITH_API_KEY }} \ ${{ vars.CLOUDSMITH_NAMESPACE }} \ - ${{ vars.CLOUDSMITH_REPOSITORY }} \ + ${{ inputs.cloudsmith-repository }} \ $package_file \ $file \ "$tags_" \ - ${{ github.sha }} + ${{ inputs.version }} done diff --git a/ci/expand_kernel_images.sh b/ci/expand_kernel_images.sh new file mode 100644 index 00000000000000..8c532ccc7d6397 --- /dev/null +++ b/ci/expand_kernel_images.sh @@ -0,0 +1,276 @@ +#!/bin/bash +# +# Process downloaded artifacts: unpack and expand images +# +# This script processes artifacts downloaded to raw/ and outputs to dist/ +# - Unpacks DTBs and kernel images +# - Expands microblaze/nios2 images by embedding DTBs +# + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "${SCRIPT_DIR}/lib.sh" + +# Read 4 bytes as big-endian integer +_read_be32() { + local bytes=$(dd if="$1" bs=1 skip=$2 count=4 2>/dev/null | xxd -p) + echo $((16#$bytes)) +} + +# Read 4 bytes as little-endian integer +_read_le32() { + local bytes=$(dd if="$1" bs=1 skip=$2 count=4 2>/dev/null | xxd -p) + echo $((16#${bytes:6:2}${bytes:4:2}${bytes:2:2}${bytes:0:2})) +} + +# Write 4 bytes as little-endian +_write_le32() { + local value=$1 + printf '\x%02x\x%02x\x%02x\x%02x' 2>/dev/null \ + $((value & 0xff)) \ + $(((value >> 8) & 0xff)) \ + $(((value >> 16) & 0xff)) \ + $(((value >> 24) & 0xff)) +} + +_find_empty_dtb() { + # Finds dtb slot in a Image + # Assumes you built the Image with an 'empty' dtb (/dts-v1/; / { };) + # Returns: dtb_start dtb_size dtb_boundary dtb_available + local image=$1 + + python3 -c " +import struct +import sys + +# 'Empty' DTB struct block: FDT_BEGIN_NODE, '', FDT_END_NODE, FDT_END +EMPTY_STRUCT = bytes.fromhex('00000001000000000000000200000009') +DTB_MAGIC = 0xd00dfeed +OFF_DT_STRUCT = 56 + +with open('$image', 'rb') as f: + data = f.read() + +pos = 0 +while True: + pos = data.find(EMPTY_STRUCT, pos) + if pos == -1: + break + + dtb_start = pos - OFF_DT_STRUCT + if dtb_start < 0: + pos += 1 + continue + + magic = struct.unpack('>I', data[dtb_start:dtb_start+4])[0] + if magic != DTB_MAGIC: + pos += 1 + continue + + off_dt_struct = struct.unpack('>I', data[dtb_start+8:dtb_start+12])[0] + if off_dt_struct != OFF_DT_STRUCT: + pos += 1 + continue + + dtb_size = struct.unpack('>I', data[dtb_start+4:dtb_start+8])[0] + dtb_end = dtb_start + dtb_size + + # Find boundary (first non-zero byte after DTB) + boundary = dtb_end + while boundary < len(data) and data[boundary] == 0: + boundary += 1 + + available = boundary - dtb_start + print(f'{dtb_start} {dtb_size} {boundary} {available}') + sys.exit(0) + +sys.exit(1) +" +} + +_unpack_dtb () { + local name="$1" + local arch= + + echo " Unpack dtb" + + for path in "raw/$name/dtb/arch"/*; do + arch=$(echo "$path" | cut -d'/' -f5) + # microblaze/nios2 DTBs are embedded into kernel images, not distributed separately + [[ "$arch" == "microblaze" ]] && continue + [[ "$arch" == "nios2" ]] && continue + + echo " $arch" + + mkdir -p "dist/$arch/boot" + cp -a "$path/boot/dts" "dist/$arch/boot" + done +} + +_unpack_kernel () { + local name="$1" + + ctx="raw/$name/context.txt" + [[ ! -f "$ctx" ]] && return + source "$ctx" + out="dist/$compiler_arch" + + echo " Unpack $kernel_defconfig" + + mkdir -p "$out/boot/kernel/$kernel_defconfig" + cp -a "raw/$name/boot/$kernel" "$out/boot/kernel/$kernel_defconfig" + + mkdir -p "$out/lib/modules_set/$kernel_defconfig" + cp -a "raw/$name/lib/modules/$kernel_release" "$out/lib/modules_set/$kernel_defconfig" +} + +# Create per-board simpleImages by embedding each DTB into the generic kernel +_expand_microblaze () { + local image="dist/microblaze/boot/kernel/adi_mb_defconfig/simpleImage.generic.strip" + local path="raw/dtb-gcc/dtb/arch/microblaze/boot/dts" + local count=0 + + log_info "Expand microblaze" + + [[ ! -d "$path" ]] && return + [[ ! -f "$image" ]] && return + + local dtb_info + dtb_info=$(_find_empty_dtb "$image") + [ -z "$dtb_info" ] && { log_error "Empty DTB not found (expected /dts-v1/; / { };)" ; return 1 ; } + + local dtb_start dtb_size dtb_boundary dtb_available + read dtb_start dtb_size dtb_boundary dtb_available <<< "$dtb_info" + + echo " Found empty DTB at 0x$(printf '%x' $dtb_start) (size: $dtb_size, available: $dtb_available bytes)" + + local dtb_length=$((dtb_boundary - dtb_start)) + + for file in "$path"/*; do + filename=$(basename $file) + filename="${filename%.*}" + image_out="$(dirname $image)/simpleImage.$filename.strip" + + echo -n " $filename" + [ -f $image_out ] && { echo " (cached)" ; continue ;} + echo "" + + local new_dtb_size=$(stat -c%s "$file") + [ $new_dtb_size -gt $dtb_available ] && { log_error "DTB too large: $new_dtb_size > $dtb_available bytes" ; return 1 ; } + + local src_magic=$(_read_be32 "$file" 0) + [ $src_magic -ne $((0xd00dfeed)) ] && { log_error "Source DTB has invalid magic: 0x$(printf '%x' $src_magic)" ; return 1 ;} + + cp "$image" "$image_out" + dd if=/dev/zero status=none \ + of="$image_out" \ + bs=1 seek=$dtb_start conv=notrunc count=$dtb_length + dd if="$file" status=none \ + of="$image_out" \ + bs=1 seek=$dtb_start conv=notrunc + ((count+=1)) + done + + command rm $image + + echo " Expanded $count simpleImages" +} + +# Create per-board zImages by embedding each DTB into the compressed kernel +_expand_nios2 () { + local image="dist/nios2/boot/kernel/adi_nios2_defconfig/zImage" + local path="raw/dtb-gcc/dtb/arch/nios2/boot/dts" + local vmlinux_gz_offset=$((16#4064)) + local input_len_offset=$((16#4060)) + local count=0 + + log_info "Expand nios2" + + [[ ! -d "$path" ]] && return + [[ ! -f "$image" ]] && return + + local tmpdir=$(mktemp -d) + local size=$(_read_le32 "$image" $input_len_offset) + + dd if="$image" status=none \ + of="$tmpdir/vmlinux.gz" \ + bs=1 skip=$vmlinux_gz_offset count=$size + gunzip -c "$tmpdir/vmlinux.gz" > "$tmpdir/vmlinux.bin" + + local dtb_info + dtb_info=$(_find_empty_dtb "$tmpdir/vmlinux.bin") + [ -z "$dtb_info" ] && { log_error "Empty DTB not found (expected /dts-v1/; / { };)" ; command rm -r "$tmpdir" ; return 1 ; } + + local dtb_start dtb_size dtb_boundary dtb_available + read dtb_start dtb_size dtb_boundary dtb_available <<< "$dtb_info" + + echo " Found empty DTB at 0x$(printf '%x' $dtb_start) (size: $dtb_size, available: $dtb_available bytes)" + + for file in "$path"/*; do + filename=$(basename "${file%.*}") + image_out="$(dirname "$image")/zImage.$filename" + + echo -n " $filename" + [ -f "$image_out" ] && { echo " (cached)" ; continue ;} + + echo "" + local new_dtb_size=$(stat -c%s "$file") + [ $new_dtb_size -gt $dtb_available ] && { log_error "DTB too large: $new_dtb_size > $dtb_available bytes" ; return 1 ; } + local src_magic=$(_read_be32 "$file" 0) + [ $src_magic -ne $((0xd00dfeed)) ] && { log_error "Source DTB has invalid magic: 0x$(printf '%x' $src_magic)" ; return 1 ;} + + cp "$tmpdir/vmlinux.bin" "$tmpdir/vmlinux.0.bin" + dd if="$file" status=none \ + of="$tmpdir/vmlinux.0.bin" \ + bs=1 seek=$dtb_start conv=notrunc status=none + gzip -n -9 < "$tmpdir/vmlinux.0.bin" > "$tmpdir/vmlinux.0.gz" + + local new_size=$(stat -c %s "$tmpdir/vmlinux.0.gz") + cp "$image" "$image_out" + _write_le32 $new_size | dd status=none \ + of="$image_out" \ + bs=1 seek=$input_len_offset conv=notrunc + dd if="$tmpdir/vmlinux.0.gz" status=none \ + of="$image_out" \ + bs=1 seek=$vmlinux_gz_offset conv=notrunc + ((count+=1)) + done + + command rm -r "$tmpdir" + command rm "$image" + + echo " Expanded $count zImages" +} + + +prepare_kernel_dist() { +# 1. Unpacks DTBs from raw/ to dist/ +# 2. Unpacks kernel images and modules +# 3. Expands microblaze images (embeds DTBs into simpleImage) +# 4. Expands nios2 images (embeds DTBs into zImage) + log_step "Process artifacts" + + mkdir -p dist + + # Unpack all raw artifacts + for dir in raw/*/; do + [[ ! -d "$dir" ]] && continue + name=$(basename "$dir") + if [[ "$name" == "dtb-gcc" ]]; then + _unpack_dtb "$name" + elif [[ "$name" != *-headers ]]; then + _unpack_kernel "$name" + fi + done + + # Expand images + [ "$SKIP_EXPAND_MICROBLAZE" == "true" ] && log_info "Expand microblaze skipped" || _expand_microblaze + [ "$SKIP_EXPAND_NIOS2" == "true" ] && log_info "Expand nios2 skipped" || _expand_nios2 + + log_info "Wrote to dist/" + ls -la dist/ +} + +# Run if executed directly +if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then + prepare_kernel_dist "$@" +fi diff --git a/ci/lib_github.sh b/ci/lib_github.sh new file mode 100755 index 00000000000000..6808ee7fa29e35 --- /dev/null +++ b/ci/lib_github.sh @@ -0,0 +1,181 @@ +#!/bin/bash +# SPDX-License-Identifier: GPL-2.0-only +# +# GitHub Actions artifact utilities +# Shared functions for downloading and extracting workflow artifacts +# + +# Source logging utilities if available +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +[[ -f "${SCRIPT_DIR}/lib.sh" ]] && source "${SCRIPT_DIR}/lib.sh" + +####################################### +# Get workflow artifacts metadata from GitHub API +# Note: This only retrieves metadata (names, download URLs), not the artifact content +# This is not used in the main workflow but can be useful in other flows from ci branch +# Arguments: +# $1 - GitHub token +# $2 - Repository (owner/repo) +# $3 - Workflow run ID +# Outputs: +# JSON response with artifacts list +####################################### +gh_get_workflow_artifacts() { + local token="$1" + local repository="$2" + local run_id="$3" + + curl -sfL \ + -H "Accept: application/vnd.github+json" \ + -H "Authorization: Bearer ${token}" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + "https://api.github.com/repos/${repository}/actions/runs/${run_id}/artifacts" +} + +####################################### +# Download a single artifact from GitHub +# Arguments: +# $1 - GitHub token +# $2 - Output file path +# $3 - Download URL +####################################### +gh_download_artifact() { + local token="$1" + local output="$2" + local url="$3" + + curl -sfL \ + -H "Authorization: Bearer ${token}" \ + -H "Accept: application/vnd.github+json" \ + -o "${output}" \ + "${url}" +} + +####################################### +# Download matching artifacts from current workflow run +# Arguments: +# $1 - GitHub token +# $2 - Repository (owner/repo) +# $3 - Workflow run ID +# $4 - Space-separated patterns to match +# $5 - Output directory (default: artifacts) +# Returns: +# 0 on success, 1 on failure +####################################### +download_matching_artifacts() { + local token="$1" + local repository="$2" + local run_id="$3" + local patterns="$4" + local output_dir="${5:-artifacts}" + + mkdir -p "${output_dir}" + + local artifacts + artifacts=$(gh_get_workflow_artifacts "${token}" "${repository}" "${run_id}") + + local total_count + total_count=$(echo "${artifacts}" | jq '.total_count' -r) + + if [[ "${total_count}" == "null" ]] || [[ "${total_count}" == "0" ]]; then + echo "::warning::No artifacts found for run ${run_id}" + return 0 + fi + + local artifacts_list + artifacts_list=$(echo "${artifacts}" | jq '[.artifacts[] | [.name, .archive_download_url]]' -r) + + local downloaded=0 + while IFS=$'\t' read -r name url; do + local matched=0 + for p in ${patterns}; do + if [[ "${name}" == ${p} ]]; then + matched=1 + break + fi + done + + if [[ "${matched}" == "1" ]]; then + echo " Downloading: ${name}" + gh_download_artifact "${token}" "${output_dir}/${name}.zip" "${url}" + downloaded=$((downloaded + 1)) + else + echo " Skipped: ${name} (no pattern match)" + fi + done < <(echo "${artifacts_list}" | jq -r '.[] | @tsv') + + echo "Downloaded ${downloaded} artifact(s) to ${output_dir}/" +} + +####################################### +# Extract downloaded artifacts to raw directory +# Arguments: +# $1 - Source directory (with .zip files) +# $2 - Target directory (default: raw) +####################################### +extract_artifacts() { + local source_dir="$1" + local target_dir="${2:-raw}" + + mkdir -p "${target_dir}" + + local count=0 + for zip in "${source_dir}"/*.zip; do + [[ ! -f "${zip}" ]] && continue + + local name + name=$(basename "${zip%.zip}") + mkdir -p "${target_dir}/${name}" + unzip -q "${zip}" -d "${target_dir}/${name}" + rm "${zip}" + echo " Extracted: ${name}" + count=$((count + 1)) + done + + echo "Extracted ${count} artifact(s) to ${target_dir}/" +} + +####################################### +# Get version path for Cloudsmith upload +# Arguments: +# $1 - Artifact type (kuiper or rpi) +# $2 - Branch name +# $3 - Timestamp +# $4 - PR target branch (optional) +# $5 - PR number (optional) +# Outputs: +# Version path string +####################################### +get_version_path() { + local artifact_type="$1" + local branch="$2" + local timestamp="$3" + local pr_target="${4:-}" + local pr_number="${5:-}" + + if [[ "${artifact_type}" == "rpi" ]]; then + if [[ -n "${pr_target}" && -n "${pr_number}" ]]; then + echo "linux_rpi/PRs/${pr_target}/pr_${pr_number}/${timestamp}" + elif [[ "${branch}" == "main" ]]; then + echo "linux_rpi/main/${timestamp}" + else + echo "linux_rpi/releases/${branch}/${timestamp}" + fi + else + if [[ -n "${pr_target}" && -n "${pr_number}" ]]; then + echo "linux/PRs/${pr_target}/pr_${pr_number}/${timestamp}" + elif [[ "${branch}" == "main" ]]; then + echo "linux/main/${timestamp}" + else + echo "linux/releases/${branch}/${timestamp}" + fi + fi +} + + +# Export functions for use in subshells +export -f gh_get_workflow_artifacts +export -f gh_download_artifact +export -f download_matching_artifacts +export -f extract_artifacts +export -f get_version_path diff --git a/ci/prepare_kuiper_artifacts_structure.sh b/ci/prepare_kuiper_artifacts_structure.sh new file mode 100755 index 00000000000000..35fb60fe34f8f3 --- /dev/null +++ b/ci/prepare_kuiper_artifacts_structure.sh @@ -0,0 +1,209 @@ +#!/bin/bash +# SPDX-License-Identifier: GPL-2.0-only +# Artifacts Structure Preparer +# Reorganizes CI Linux build artifacts in a structured format. The full path +# will be used for versioning the artefact. +# +# Environment variables (should be exported before sourcing this script): +# SOURCE_DIRECTORY - Directory containing build artifacts (default: pwd) +# TIMESTAMP - Timestamp for output directory (default: auto-generated) +# BUILD_SOURCEBRANCHNAME - Git branch name (default: main) +# GIT_SHA - Git commit SHA (default: unknown) +# GIT_SHA_DATE - Git commit date in YYYY-MM-DD-HH-MM format (default: unknown) + +set -e +shopt -s nullglob # Prevent glob patterns from returning themselves when no match + +# Configuration +SOURCE_DIRECTORY="${SOURCE_DIRECTORY:-$(pwd)}" +TIMESTAMP="${TIMESTAMP:-$(date +%Y_%m_%d-%H_%M)}" +BUILD_SOURCEBRANCHNAME="${BUILD_SOURCEBRANCHNAME:?ERROR: BUILD_SOURCEBRANCHNAME must be set}" +GIT_SHA="${GIT_SHA:?ERROR: GIT_SHA must be set}" +GIT_SHA_DATE="${GIT_SHA_DATE:-}" + +# Paths +DIST_DIR="${SOURCE_DIRECTORY}/dist" +OUTPUT_DIR="${SOURCE_DIRECTORY}/${TIMESTAMP}" + +# Architecture to platform mapping (used for directory structure and DTB filtering) +declare -A typeARCH +typeARCH=( + ["arm"]="arria10 cyclone5 zynq" + ["arm64"]="versal zynqmp" + ["microblaze"]="kc705 kcu105 vc707 vcu118 vcu128" +) + +# Kernel image locations per platform (paths within dist/) +declare -A image_to_copy +image_to_copy=( + ["arria10"]="arm/boot/kernel/socfpga_adi_defconfig/zImage" + ["cyclone5"]="arm/boot/kernel/socfpga_adi_defconfig/zImage" + ["zynq"]="arm/boot/kernel/zynq_xcomm_adv7511_defconfig/uImage" + ["versal"]="arm64/boot/kernel/adi_versal_defconfig/Image" + ["zynqmp"]="arm64/boot/kernel/adi_zynqmp_defconfig/Image" +) + + + +####################################### +# Log message with timestamp +####################################### +log() { + echo "[$(date '+%Y-%m-%d %H:%M:%S')] $*" +} + +####################################### +# Create directory structure +####################################### +create_structure() { + log "Creating directory structure: ${OUTPUT_DIR}" + + mkdir -p "${OUTPUT_DIR}" + + for arch in "${!typeARCH[@]}"; do + mkdir -p "${OUTPUT_DIR}/${arch}" + + # Create platform subdirectories for non-microblaze architectures + if [[ "${arch}" != "microblaze" ]]; then + for platform in ${typeARCH[$arch]}; do + mkdir -p "${OUTPUT_DIR}/${arch}/${platform}" + done + fi + done +} + +####################################### +# Generate git properties file +####################################### +generate_git_properties() { + log "Generating git_properties.txt" + + cat > "${OUTPUT_DIR}/git_properties.txt" << EOF + git_branch=${BUILD_SOURCEBRANCHNAME} + git_sha=${GIT_SHA} + git_sha_date=${GIT_SHA_DATE} +EOF +} + +####################################### +# Create extlinux.conf for a platform +####################################### +create_extlinux() { + local platform=$1 + local output_dir=$2 + + if [[ "${platform}" == "arria10" ]]; then + dtb_name="socfpga_arria10_socdk_sdmmc.dtb" + else + dtb_name="socfpga.dtb" + fi + + cat > "${output_dir}/extlinux.conf" </dev/null || true + # Microblaze .strip files (kernel images with debug symbols stripped) + find "${DIST_DIR}/microblaze/boot/kernel/adi_mb_defconfig" -name "*.strip" -exec cp {} "${dtb_flat}/" \; 2>/dev/null || true + + for arch in "${!typeARCH[@]}"; do + local arch_dtb_count=0 + + for platform in ${typeARCH[$arch]}; do + # Copy kernel images (skip microblaze - no kernel images) + if [[ "${arch}" != "microblaze" ]]; then + local image_src="${DIST_DIR}/${image_to_copy[$platform]}" + local image_dst="${OUTPUT_DIR}/${arch}/${platform}/" + + if [[ -f "${image_src}" ]]; then + cp "${image_src}" "${image_dst}" + log " Copied: ${image_to_copy[$platform]} -> ${arch}/${platform}/" + + # Create extlinux.conf for Intel platforms + if [[ "${platform}" == "arria10" || "${platform}" == "cyclone5" ]]; then + create_extlinux "${platform}" "${image_dst}" + fi + else + log " WARNING: Image not found: ${image_src}" + fi + fi + + # Match DTBs by platform prefix (e.g., zynq_*, socfpga_arria10_*, vcu118_*) + local dtbs_to_copy + if [[ "${arch}" == "microblaze" ]]; then + dtbs_to_copy=$(ls "${dtb_flat}"/*.dtb "${dtb_flat}"/*.strip 2>/dev/null | xargs -n1 basename | grep -E "(kc705|kcu105|vc707|vcu118|vcu128)" || true) + else + dtbs_to_copy=$(ls "${dtb_flat}"/*.dtb 2>/dev/null | xargs -n1 basename | grep -E "^${platform}[-_]|^socfpga_${platform}" || true) + fi + + for dtb in ${dtbs_to_copy}; do + cp "${dtb_flat}/${dtb}" "${OUTPUT_DIR}/${arch}/" + ((++arch_dtb_count)) + done + done + + log " Copied ${arch_dtb_count} ${arch} DTBs" + done +} + +####################################### +# Print summary +####################################### +print_summary() { + echo "" + echo "==========================================" + echo "Artifacts Structure Complete" + echo "==========================================" + echo "" + echo "Output directory: ${OUTPUT_DIR}" + echo "" + tree -I '*.dtb' --dirsfirst --noreport "${OUTPUT_DIR}" + echo "" + printf "DTBs: arm=%d, arm64=%d, microblaze=%d\n" \ + "$(find "${OUTPUT_DIR}/arm" -maxdepth 1 -name '*.dtb' | wc -l)" \ + "$(find "${OUTPUT_DIR}/arm64" -maxdepth 1 -name '*.dtb' | wc -l)" \ + "$(find "${OUTPUT_DIR}/microblaze" -maxdepth 1 -name '*.dtb' | wc -l)" + echo "" +} + +####################################### +# Main function +####################################### +main() { + + log "Starting artifacts structure" + log "Source: ${SOURCE_DIRECTORY}" + log "Output: ${OUTPUT_DIR}" + + # Validate source directory + if [[ ! -d "${DIST_DIR}" ]]; then + echo "ERROR: Dist directory not found: ${DIST_DIR}" + exit 1 + fi + + # Execute steps + create_structure + generate_git_properties + copy_artifacts + print_summary + + log "Done!" +} + +# Run main function +main "$@" diff --git a/ci/prepare_rpi_artifacts.sh b/ci/prepare_rpi_artifacts.sh new file mode 100755 index 00000000000000..daaa1aadb6c895 --- /dev/null +++ b/ci/prepare_rpi_artifacts.sh @@ -0,0 +1,194 @@ +#!/bin/bash -e +# SPDX-License-Identifier: GPL-2.0-only +# +# RPI Artifact Processor +# Prepares Raspberry Pi boot files and kernel modules as tar.gz archives +# +# Environment variables: +# SOURCE_DIRECTORY: where the downloaded artifacts are located +# OUTPUT_DIRECTORY: where to create the tar.gz archives (defaults to current directory) +# BUILD_SOURCEBRANCHNAME: Git branch name for versioning +# + +SOURCE_DIRECTORY="${SOURCE_DIRECTORY:-$(pwd)}" +OUTPUT_DIRECTORY="${OUTPUT_DIRECTORY:-$(pwd)}" +BUILD_SOURCEBRANCHNAME="${BUILD_SOURCEBRANCHNAME:?ERROR: BUILD_SOURCEBRANCHNAME must be set}" + +timestamp=$(date +%Y_%m_%d-%H_%M) + +# Extract git info from context.txt (replaces git commands for CI artifacts) +for ctx in "${SOURCE_DIRECTORY}"/adi_bcm*-gcc-arm*/context.txt; do + if [[ -f "$ctx" ]]; then + GIT_SHA=$(grep "^git_sha=" "$ctx" | cut -d'=' -f2) + git_sha_at=$(grep "^git_sha_at=" "$ctx" | cut -d'=' -f2) + GIT_SHA_DATE=$(date -d "@$git_sha_at" +"%Y-%m-%d-%H-%M" 2>/dev/null || echo "unknown") + break + fi +done + +# Temporary working directory +WORK_DIR=$(mktemp -d) +trap "rm -rf ${WORK_DIR}" EXIT + +#create version file found in the boot partition +create_version_file() { + mkdir -p ${WORK_DIR}/${1} + echo -e "RPI Boot Files: ${BUILD_SOURCEBRANCHNAME} ${GIT_SHA_DATE}\n" > ${WORK_DIR}/${1}/version_rpi.txt + echo -e " Linux repository: https://github.com/analogdevicesinc/linux" >> ${WORK_DIR}/${1}/version_rpi.txt + echo -e " Linux branch: ${BUILD_SOURCEBRANCHNAME}" >> ${WORK_DIR}/${1}/version_rpi.txt + echo -e " Linux git sha: ${GIT_SHA}\n" >> ${WORK_DIR}/${1}/version_rpi.txt + echo -e "Supported RaspberryPi platforms:\n" >> ${WORK_DIR}/${1}/version_rpi.txt + list=($2) + for platform in "${list[@]}"; do + echo " ${platform}" >> ${WORK_DIR}/${1}/version_rpi.txt + done +} + +# extract and rename kernel image from boot directory +# Usage: extract_kernel_image +extract_kernel_image() { + local boot_dir="$1" + local output_file="$2" + local kernel_name=$(basename "$output_file") + + if [[ -f "${boot_dir}/Image.gz" ]]; then + echo " Extracting Image.gz -> ${kernel_name}" + gunzip -c "${boot_dir}/Image.gz" > "${output_file}" + elif [[ -f "${boot_dir}/Image" ]]; then + echo " Copying Image -> ${kernel_name}" + cp "${boot_dir}/Image" "${output_file}" + elif [[ -f "${boot_dir}/zImage" ]]; then + echo " Copying zImage -> ${kernel_name}" + cp "${boot_dir}/zImage" "${output_file}" + elif [[ -f "${boot_dir}/uImage" ]]; then + echo " Copying uImage -> ${kernel_name}" + cp "${boot_dir}/uImage" "${output_file}" + else + echo "ERROR: No kernel image found in ${boot_dir}" + return 1 + fi +} + +#prepare the structure of the folder containing artifacts +artifacts_structure() { + # BCM defconfig names map to RPi kernel image names + typeBCM_32bit=( "bcm2709" "bcm2711" "bcmrpi" ) + typeKERNEL_32bit=( "kernel7" "kernel7l" "kernel" ) + typeBCM_64bit=( "bcm2711" "bcm2712" ) + typeKERNEL_64bit=( "kernel8" "kernel_2712" ) + + echo "=== Processing 32-bit artifacts ===" + mkdir -p ${WORK_DIR}/32bit + mkdir -p ${WORK_DIR}/32bit/modules + create_version_file 32bit "${typeBCM_32bit[*]}" + for index in "${!typeBCM_32bit[@]}"; do + local artifact_dir="adi_${typeBCM_32bit[$index]}_defconfig-gcc-arm" + echo "Processing ${artifact_dir}..." + + # Extract and rename kernel image + extract_kernel_image \ + "${SOURCE_DIRECTORY}/${artifact_dir}/boot" \ + "${WORK_DIR}/32bit/${typeKERNEL_32bit[$index]}.img" + + # Copy modules + cp -r "${SOURCE_DIRECTORY}/${artifact_dir}/lib/modules"/* "${WORK_DIR}/32bit/modules/" + done + + # Copy DTBs and overlays from dtb-gcc artifact + cp ${SOURCE_DIRECTORY}/dtb-gcc/dtb/arch/arm/boot/dts/broadcom/*.dtb ${WORK_DIR}/32bit/ + mkdir -p ${WORK_DIR}/32bit/overlays + # RPi firmware expects overlays without the -overlay suffix + for overlay in ${SOURCE_DIRECTORY}/dtb-gcc/dtb/arch/arm/boot/dts/overlays/*-overlay.dtbo; do + base=$(basename "$overlay" -overlay.dtbo) + cp "$overlay" ${WORK_DIR}/32bit/overlays/${base}.dtbo + done + + if [ -z "$(ls ${WORK_DIR}/32bit/*.dtb 2>/dev/null)" ] || [ -z "$(ls ${WORK_DIR}/32bit/overlays/*.dtbo 2>/dev/null)" ]; then + echo "Missing one or more required files from the 32bit artifacts." + exit 1 + fi + + echo "Creating rpi_modules_32bit.tar.gz..." + tar -C ${WORK_DIR}/32bit/modules -czvf ${OUTPUT_DIRECTORY}/rpi_modules_32bit.tar.gz . >/dev/null + rm -r ${WORK_DIR}/32bit/modules + + echo "" + echo "=== Processing 64-bit artifacts ===" + mkdir -p ${WORK_DIR}/64bit + mkdir -p ${WORK_DIR}/64bit/modules + create_version_file 64bit "${typeBCM_64bit[*]}" + for index in "${!typeBCM_64bit[@]}"; do + local artifact_dir="adi_${typeBCM_64bit[$index]}_defconfig-gcc-arm64" + echo "Processing ${artifact_dir}..." + + # Extract and rename kernel image + extract_kernel_image \ + "${SOURCE_DIRECTORY}/${artifact_dir}/boot" \ + "${WORK_DIR}/64bit/${typeKERNEL_64bit[$index]}.img" + + # Copy modules + cp -r "${SOURCE_DIRECTORY}/${artifact_dir}/lib/modules"/* "${WORK_DIR}/64bit/modules/" + done + + # Copy DTBs and overlays from dtb-gcc artifact + cp ${SOURCE_DIRECTORY}/dtb-gcc/dtb/arch/arm64/boot/dts/broadcom/*.dtb ${WORK_DIR}/64bit/ + mkdir -p ${WORK_DIR}/64bit/overlays + for overlay in ${SOURCE_DIRECTORY}/dtb-gcc/dtb/arch/arm/boot/dts/overlays/*-overlay.dtbo; do + base=$(basename "$overlay" -overlay.dtbo) + cp "$overlay" ${WORK_DIR}/64bit/overlays/${base}.dtbo + done + + if [ -z "$(ls ${WORK_DIR}/64bit/*.dtb 2>/dev/null)" ] || [ -z "$(ls ${WORK_DIR}/64bit/overlays/*.dtbo 2>/dev/null)" ]; then + echo "Missing one or more required files from the 64bit artifacts." + exit 1 + fi + + echo "Creating rpi_modules_64bit.tar.gz..." + tar -C ${WORK_DIR}/64bit/modules -czvf ${OUTPUT_DIRECTORY}/rpi_modules_64bit.tar.gz . >/dev/null + rm -r ${WORK_DIR}/64bit/modules + + echo "" + echo "=== Artifacts structure created ===" +} + +#create final boot archives and properties file +create_rpi_boot_archives() { + artifacts_structure + + echo "" + echo "=== Creating boot archives ===" + + cd ${WORK_DIR}/32bit || exit 1 + tar -czvf ${OUTPUT_DIRECTORY}/rpi_latest_boot_32bit.tar.gz . >/dev/null + md5_modules_32bit=$(md5sum ${OUTPUT_DIRECTORY}/rpi_modules_32bit.tar.gz | cut -d ' ' -f 1) + md5_boot_32bit=$(md5sum ${OUTPUT_DIRECTORY}/rpi_latest_boot_32bit.tar.gz | cut -d ' ' -f 1) + + cd ${WORK_DIR}/64bit || exit 1 + tar -czvf ${OUTPUT_DIRECTORY}/rpi_latest_boot_64bit.tar.gz . >/dev/null + md5_modules_64bit=$(md5sum ${OUTPUT_DIRECTORY}/rpi_modules_64bit.tar.gz | cut -d ' ' -f 1) + md5_boot_64bit=$(md5sum ${OUTPUT_DIRECTORY}/rpi_latest_boot_64bit.tar.gz | cut -d ' ' -f 1) + + cat > ${OUTPUT_DIRECTORY}/rpi_archives_properties.txt << EOF +git_branch=${BUILD_SOURCEBRANCHNAME} +https://swdownloads.analog.com/cse/linux_rpi/${BUILD_SOURCEBRANCHNAME}/rpi_modules_32bit.tar.gz +https://swdownloads.analog.com/cse/linux_rpi/${BUILD_SOURCEBRANCHNAME}/rpi_latest_boot_32bit.tar.gz +checksum_modules_32bit=${md5_modules_32bit} +checksum_boot_files_32bit=${md5_boot_32bit} +https://swdownloads.analog.com/cse/linux_rpi/${BUILD_SOURCEBRANCHNAME}/rpi_modules_64bit.tar.gz +https://swdownloads.analog.com/cse/linux_rpi/${BUILD_SOURCEBRANCHNAME}/rpi_latest_boot_64bit.tar.gz +checksum_modules_64bit=${md5_modules_64bit} +checksum_boot_files_64bit=${md5_boot_64bit} +git_sha=${GIT_SHA} +git_sha_date=${GIT_SHA_DATE} +EOF + + echo "" + echo "=== Done ===" + echo "Output: ${OUTPUT_DIRECTORY}/" + ls -lh ${OUTPUT_DIRECTORY}/rpi_*.tar.gz ${OUTPUT_DIRECTORY}/rpi_archives_properties.txt +} + +# Run if executed directly (not sourced) +if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then + artifacts_${1:-local} +fi