diff --git a/.buildkite/Manifest.toml b/.buildkite/Manifest.toml index f00140bf443..acd64b6ea8f 100644 --- a/.buildkite/Manifest.toml +++ b/.buildkite/Manifest.toml @@ -1,6 +1,6 @@ # This file is machine-generated - editing it directly is not advised -julia_version = "1.12.4" +julia_version = "1.12.5" manifest_format = "2.0" project_hash = "5023fc07bb1b56800f97a2d66fbd8bd8c9c8d7a1" @@ -116,9 +116,9 @@ version = "1.12.1" [[deps.Preferences]] deps = ["TOML"] -git-tree-sha1 = "522f093a29b31a93e34eaea17ba055d850edea28" +git-tree-sha1 = "8b770b60760d4451834fe79dd483e318eee709c4" uuid = "21216c6a-2e73-6563-6e65-726566657250" -version = "1.5.1" +version = "1.5.2" [[deps.Printf]] deps = ["Unicode"] diff --git a/.buildkite/build.sh b/.buildkite/build.sh index f813d0f62a1..6e5719ae0ee 100755 --- a/.buildkite/build.sh +++ b/.buildkite/build.sh @@ -2,12 +2,22 @@ # Fail on error set -e +SCRIPT_DIR="$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" +YGGDRASIL_BASE="$(dirname "${SCRIPT_DIR}")" +JULIA_PROJECT="${YGGDRASIL_BASE}/${JULIA_PROJECT:-/foo}" + +# Early-exit if someone is blindly running this manually +if [[ ! -d "${JULIA_PROJECT:-}" ]]; then + echo "ERROR: Must set JULIA_PROJECT to one of:" >&2 + echo " - ${YGGDRASIL_BASE}/.ci/bb1_project" >&2 + echo " - ${YGGDRASIL_BASE}/.ci/bb2_project" >&2 + exit 1 +fi + # Clear secrets from environment export BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET="" export AWS_SECRET_ACCESS_KEY="" -export JULIA_PROJECT="${BUILDKITE_BUILD_CHECKOUT_PATH}/.ci" - # Add our shared depot cache to the end of JULIA_DEPOT_PATH which is already # filled out by `julia-buildkite-plugin` to our agent/pipeline-specific depot path. # Make sure to append a colon at the end to allow use of shipped stdlib caches. @@ -17,17 +27,15 @@ echo "--- Set JULIA_DEPOT_PATH to ${JULIA_DEPOT_PATH}" echo "--- Setup Julia packages" julia --color=yes -e 'import Pkg; Pkg.instantiate(); Pkg.precompile()' -# Cleanup temporary things that might have been left-over -echo "--- Cleanup" -./clean_builds.sh -./clean_products.sh - -echo "+++ Build" -cd "${PROJECT}" - # Parallel auditor can end up opening loads of files and causing # "Too many open files" errors. Increase the limit. ulimit -n 65536 -# Start Julia with multiple thread to make auditor parallel. -julia --threads "${BINARYBUILDER_NPROC:-16}" ./build_tarballs.jl --verbose "${PLATFORM}" +# Cleanup temporary things that might have been left-over +echo "--- Cleanup" +"${YGGDRASIL_BASE}/clean_builds.sh" +"${YGGDRASIL_BASE}/clean_products.sh" + +echo "+++ Build" +# Start Julia with multiple threads to make auditor parallel. +julia --threads "${BINARYBUILDER_NPROC:-16}" ./build_tarballs.jl --verbose "$@" diff --git a/.buildkite/generator.jl b/.buildkite/generator.jl index 230ce77aec3..6dd5e271117 100755 --- a/.buildkite/generator.jl +++ b/.buildkite/generator.jl @@ -1,8 +1,25 @@ #!/bin/env julia -if VERSION < v"1.8.0" - Base.ACTIVE_PROJECT[] = @__DIR__ +import Pkg + +Base.set_active_project(@__DIR__) +using YAML + +const PROJECTS = copy(ARGS) +const DEBUG = !haskey(ENV, "BUILDKITE") +const IS_PR = get(ENV, "BUILDKITE_PULL_REQUEST", "false") != "false" +const SKIP_BUILD_COOKIE="[skip build]" + +if IS_PR + # If we're on a PR though, we look at the entire branch at once + BASE_BRANCH = get(ENV, "BUILDKITE_PULL_REQUEST_BASE_BRANCH", "master") + @assert !isempty(BASE_BRANCH) + + PR_NUMBER = ENV["BUILDKITE_PULL_REQUEST"] + run(ignorestatus(`git fetch origin "refs/pull/$(PR_NUMBER)/head:refs/remotes/origin/pr/$(PR_NUMBER)"`)) + + COMMIT_MSG = readchomp(`git show -s --format=%B origin/pr/$(PR_NUMBER)`) else - Base.set_active_project(@__DIR__) + COMMIT_MSG = readchomp(`git show -s --format=%B`) end # Force ourselves to use the shared depot as well, if it exists @@ -10,125 +27,155 @@ if isdir("/sharedcache/depot") push!(Base.DEPOT_PATH, "/sharedcache/depot") end -import Pkg +# Instantiate, to install all necessary packages like YAML (not BinaryBuilder) Pkg.instantiate() -using Downloads, Dates, SHA +include("utils.jl") -const PROJECT = only(ARGS) -const DEBUG = !haskey(ENV, "BUILDKITE") +# Give a warning if the pkgserver is too far behind +check_pkgserver_latency() -include("utils.jl") +# If there are scary projects we need to exclude, we list them here. (Used to contain `LLVM`) +EXCLUDED_NAMES = Set{String}([]) -if !isnothing(Pkg.pkg_server()) - resp = try - headers = Pkg.PlatformEngines.get_metadata_headers(Pkg.pkg_server()) - Downloads.request("$(Pkg.pkg_server())/registries"; headers) - catch e - # Let us know the download of the registry went wrong, but do not hard fail - @error "Could not download the registry" exception=(e, catch_backtrace()) - end - last_mod_idx = findfirst(h -> first(h) == "last-modified", resp.headers) - msg = "PkgServer: " * resp.url - delay = if !isnothing(last_mod_idx) - last_mod = last(resp.headers[last_mod_idx]) - msg *= " -- last updated: " * last_mod - # Manually strip the "GMT" timezone and hope it never changes. - # Do not error out if parsing fails. - dt = tryparse(DateTime, replace(last_mod, " GMT"=>""), dateformat"e, d u y H:M:S") - # If parsing did fail, set the delay to 0. - isnothing(dt) ? Second(0) : now(UTC) - dt - else - Second(0) - end - delay > Second(0) && (msg *= " (" * string(Dates.canonicalize(round(delay, Second))) * " ago)") - @info(msg) - annotate(msg, style="info", context="pkg") - tolerance = Hour(1) - if delay > tolerance - @warn "The PkgServer registry is older than $(tolerance)" - annotate("The PkgServer registry is older than $(tolerance)", style = "warning", context="pkg") +filter!(PROJECTS) do project + if project ∈ EXCLUDED_NAMES + @info("Skipping project since it is excluded.", project) + return false end + return true end -# If there are scary projects we need to exclude, we list them here. (Used to contain `LLVM`) -EXCLUDED_NAMES = Set{String}([]) +# Clear out any old `.meta.json` files in the Yggdrasil root +cleanup_metadata!() -if PROJECT ∈ EXCLUDED_NAMES - @info "Skipping project since it is excluded." PROJECT - exit() +# Immediately read in all projects, and ensure that they are either all BB2 projects, or none of them are: +bb2_projects = String[] +bb1_projects = String[] +for project in PROJECTS + if uses_bb2(project) + push!(bb2_projects, project) + else + push!(bb1_projects, project) + end end -# Remove secret from environment -sanitize(cmd) = addenv(cmd, Dict("BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET" => nothing)) -exec(cmd) = @assert success(pipeline(sanitize(cmd), stderr=stderr, stdout=stdout)) +function bb1_build_steps!(group_steps, project) + # determine the name, removing any trailing version number + # (`L/LLVM/LLVM@14` results in `NAME = "LLVM"`) + project_basename = basename(project) -YGGDRASIL_BASE = dirname(@__DIR__) -julia(args) = `$(Base.julia_cmd()) --project=$(YGGDRASIL_BASE)/.ci $args` + # We always invoke a `build_tarballs.jl` file from its own directory to generate the platform list + cd(project) do + println("[$(project)] Generating meta.json...") + json_path = "$(YGGDRASIL_BASE)/$(project_basename).meta.json" + julia(`--compile=min ./build_tarballs.jl --meta-json="$(json_path)"`; julia_project = bb1_julia_project) -# Next, we're going to ensure that our BB is up to date and precompiled -julia(`-e "import Pkg; Pkg.instantiate(); Pkg.precompile()"`) |> exec + # Generate platforms + julia(`$(bb1_julia_project)/generate_platforms.jl "$(json_path)" $(YGGDRASIL_BASE)/$(project_basename).platforms.list`) + end -TEMP = mktempdir() + platforms = split(readchomp(joinpath(YGGDRASIL_BASE, "$(project_basename).platforms.list"))) + if isempty(platforms) + @error("Unable to determine the proper platforms", project_basename) + exit(1) + end -# determine the name, removing any trailing version number -# (`L/LLVM/LLVM@14` results in `NAME = "LLVM@14"`) -const NAME = first(split(basename(PROJECT), "@")) + should_skip_builds = contains(COMMIT_MSG, SKIP_BUILD_COOKIE) + steps = [] + for platform in platforms + if !should_skip_builds + println("[$(project)] $(platform): building") + bs = BB1BuildStep( + project_basename, + project, + platform, + ) + push!(steps, render(bs)) + end + end -# We always invoke a `build_tarballs.jl` file from its own directory -# generate platform list -cd(PROJECT) do - println("Generating meta.json...") - JSON_PATH = "$(TEMP)/$(NAME).meta.json" - julia(`--compile=min ./build_tarballs.jl --meta-json="$(JSON_PATH)"`) |> exec + # If this is not a pull request, we're going to register this project + if !IS_PR + push!(steps, wait_step()) + push!(steps, render(BB1RegisterStep( + project_basename, + project, + should_skip_builds, + length(platforms), + ))) + #push!(steps, wait_step()) + #push!(steps, register_step(project_name, project, should_skip_builds, length(platforms))) + end - # Generate platforms - julia(`$(YGGDRASIL_BASE)/.ci/generate_platforms.jl "$(JSON_PATH)" $(TEMP)/$(NAME).platforms.list`) |> exec + # Group this project's full build under `project_name` + if !isempty(steps) + push!(group_steps, group_step(project_basename, steps)) + end end -println("Determining builds to queue...") - -# Load in the platforms -PLATFORMS = split(readchomp(joinpath(TEMP, "$(NAME).platforms.list"))) -if isempty(PLATFORMS) - @error "Unable to determine the proper platforms" NAME +function bb2_build_steps!(group_steps, projects) + # Invoke `scheduler.jl` to build up our list of jobs with their dependencies + io = IOBuffer() + julia(`$(bb2_julia_project)/scheduler.jl $(projects)`; julia_project=bb2_julia_project, stdout=io) + groups = YAML.load(String(take!(io))) + + # We will create one group per packaging project, and each build will belong to one of those + for (group_name, group) in groups + builds = [] + steps = [] + for build in group["builds"] + bs = BB2BuildStep( + build["name"], + group_name, + build["platform"], + build["build_hash"], + build["dependencies"], + ) + push!(builds, bs) + push!(steps, render(bs)) + end + + for packaging in group["packagings"] + deps = vcat( + # Other packages that must be registered + packaging["dependencies"], + # Our own builds + builds, + ) + rs = BB2RegisterStep(packaging["name"], group_name, deps) + push!(steps, render(rs)) + end + push!(group_steps, group_step(group_name, steps)) + end end -const IS_PR = get(ENV, "BUILDKITE_PULL_REQUEST", "false") != "false" -const SKIP_BUILD_COOKIE="[skip build]" -if IS_PR - # If we're on a PR though, we look at the entire branch at once - BASE_BRANCH = get(ENV, "BUILDKITE_PULL_REQUEST_BASE_BRANCH", "master") - @assert !isempty(BASE_BRANCH) +function build_steps!(group_steps, bb1_projects, bb2_projects) + if !isempty(bb1_projects) + # Ensure BB1 is instantiated + julia(`-e "import Pkg; Pkg.instantiate(); Pkg.precompile()"`; julia_project=bb1_julia_project) - PR_NUMBER = ENV["BUILDKITE_PULL_REQUEST"] - exec(`git fetch origin "refs/pull/$(PR_NUMBER)/head:refs/remotes/origin/pr/$(PR_NUMBER)"`) + # Each project is processed individually, with a group of builds for each platform. + for project in bb1_projects + bb1_build_steps!(group_steps, project) + end + end - COMMIT_MSG = readchomp(`git show -s --format=%B origin/pr/$(PR_NUMBER)`) -else - COMMIT_MSG = readchomp(`git show -s --format=%B`) -end -# This variable will tell us whether we want to skip the build -const SKIP_BUILD = contains(COMMIT_MSG, SKIP_BUILD_COOKIE) + if !isempty(bb2_projects) + # Ensure BB2 is instantiated + julia(`-e "import Pkg; Pkg.instantiate(); Pkg.precompile()"`; julia_project=bb2_julia_project) -STEPS = Any[] -# Create the BUILD_STEPS -if SKIP_BUILD - println("The commit messages contains $(SKIP_BUILD_COOKIE), skipping build") -else - for PLATFORM in PLATFORMS - println(" $(PLATFORM): building") - push!(STEPS, build_step(NAME, PLATFORM, PROJECT)) + # For BB2, we jointly process all `build_tarballs.jl` files, building a dependency graph: + bb2_build_steps!(group_steps, bb2_projects) end end -if !IS_PR - push!(STEPS, wait_step()) - push!(STEPS, register_step(NAME, PROJECT, SKIP_BUILD, length(PLATFORMS))) -end -if !isempty(STEPS) - definition = Dict( - :steps => Any[group_step(NAME, STEPS)] - ) - upload_pipeline(definition) + + +group_steps = [] +build_steps!(group_steps, bb1_projects, bb2_projects) + +# Only upload the pipieline if we actually have something to upload +if !isempty(group_steps) + upload_pipeline(Dict(:steps => group_steps)) end diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index d70a78b813f..a7d647a4f96 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -3,7 +3,7 @@ steps: plugins: - JuliaCI/julia#v1: persist_depot_dirs: packages,artifacts,compiled - version: "1.12.4" + version: "1.12.5" artifacts_size_limit: "214748364800" # 200GB - JuliaCI/merge-commit: ~ - staticfloat/forerunner#v1: @@ -13,6 +13,9 @@ steps: path_processor: .buildkite/path_processors/per-project target: .buildkite/generator.jl target_type: command + artifacts: + - "*.meta.json" + - "*.platforms.list" agents: queue: "yggdrasil" arch: "x86_64" diff --git a/.buildkite/pipeline.yml.signature b/.buildkite/pipeline.yml.signature index f0dce700971..f904dbdd952 100644 Binary files a/.buildkite/pipeline.yml.signature and b/.buildkite/pipeline.yml.signature differ diff --git a/.buildkite/register.sh b/.buildkite/register.sh index c65f4192f7e..bed82110d6c 100755 --- a/.buildkite/register.sh +++ b/.buildkite/register.sh @@ -2,19 +2,20 @@ # Fail on error set -e -export JULIA_PROJECT="${BUILDKITE_BUILD_CHECKOUT_PATH}/.ci" +SCRIPT_DIR="$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" +YGGDRASIL_BASE="$(dirname "${SCRIPT_DIR}")" +JULIA_PROJECT="${YGGDRASIL_BASE}/${JULIA_PROJECT:-/foo}" + +# Early-exit if someone is blindly running this manually +if [[ ! -d "${JULIA_PROJECT:-}" ]]; then + echo "ERROR: Must set JULIA_PROJECT to one of:" >&2 + echo " - ${YGGDRASIL_BASE}/.ci/bb1_project" >&2 + echo " - ${YGGDRASIL_BASE}/.ci/bb2_project" >&2 + exit 1 +fi echo "--- Setup Julia packages" GITHUB_TOKEN="" julia --color=yes -e 'import Pkg; Pkg.instantiate(); Pkg.precompile()' -cd "${PROJECT}" -echo "--- Generating meta.json..." -GITHUB_TOKEN="" julia --compile=min ./build_tarballs.jl --meta-json=${NAME}.meta.json - -echo "--- Setting up git" -git config --global user.name "jlbuild" -git config --global user.email "juliabuildbot@gmail.com" - -echo "--- Registering ${NAME}..." -julia ${BUILDKITE_BUILD_CHECKOUT_PATH}/.ci/register_package.jl "${NAME}.meta.json" --verbose - +echo "--- Registering..." +julia ${JULIA_PROJECT}/register_package.jl "${META_JSON:-}" --verbose diff --git a/.buildkite/utils.jl b/.buildkite/utils.jl index e6f29be85d6..6fe48320cb7 100644 --- a/.buildkite/utils.jl +++ b/.buildkite/utils.jl @@ -1,15 +1,61 @@ -import YAML +import Downloads, Pkg, YAML +using Dates + +function check_pkgserver_latency(; tolerance = Hour(1)) + # Nothing to do if this is empty + if isnothing(Pkg.pkg_server()) + return + end + + resp = try + headers = Pkg.PlatformEngines.get_metadata_headers(Pkg.pkg_server()) + Downloads.request("$(Pkg.pkg_server())/registries"; headers) + catch e + # Let us know the download of the registry went wrong, but do not hard fail + @error "Could not download the registry" exception=(e, catch_backtrace()) + end + + last_mod_idx = findfirst(h -> first(h) == "last-modified", resp.headers) + msg = "PkgServer: " * resp.url + delay = if !isnothing(last_mod_idx) + last_mod = last(resp.headers[last_mod_idx]) + msg *= " -- last updated: " * last_mod + # Manually strip the "GMT" timezone and hope it never changes. + # Do not error out if parsing fails. + dt = tryparse(DateTime, replace(last_mod, " GMT"=>""), dateformat"e, d u y H:M:S") + # If parsing did fail, set the delay to 0. + isnothing(dt) ? Second(0) : now(UTC) - dt + else + Second(0) + end + delay > Second(0) && (msg *= " (" * string(Dates.canonicalize(round(delay, Second))) * " ago)") + @info(msg) + annotate(msg, style="info", context="pkg") + if delay > tolerance + @warn "The PkgServer registry is older than $(tolerance)" + annotate("The PkgServer registry is older than $(tolerance)", style = "warning", context="pkg") + end +end + +# Cleanup `.meta.json` or `.platform.list` files from previous BB1 runs +function cleanup_metadata!() + for (root, _, files) in walkdir(YGGDRASIL_BASE) + for file in files + if endswith(file, ".meta.json") || endswith(file, ".platforms.list") + rm(joinpath(root, file)) + end + end + end +end function upload_pipeline(definition) try open(`buildkite-agent pipeline upload --no-interpolation`, stdout, write=true) do io YAML.write(io, definition) end - catch - println(stderr, "pipeline upload failed:") - YAML.write(stderr, definition) - println(stderr) - rethrow() + catch e + @error("pipeline upload failed", e) + YAML.write(stdout, definition) end end @@ -17,8 +63,12 @@ function annotate(annotation; context="default", style="info", append=true) @assert style in ("success", "info", "warning", "error") append = append ? `--append` : `` cmd = `buildkite-agent annotate --style $(style) --context $(context) $(append)` - open(cmd, stdout, write=true) do io - write(io, annotation) + try + open(cmd, stdout, write=true) do io + write(io, annotation) + end + catch err + @error("Unable to annotate", err) end end @@ -32,33 +82,73 @@ agent() = Dict( plugins() = Pair{String, Union{Nothing, Dict}}[ "JuliaCI/julia#v1" => Dict( "persist_depot_dirs" => "packages,artifacts,compiled", - "version" => "1.12.4", + "version" => "1.12.5", "artifacts_size_limit" => string(120 << 30), # 120 GiB ), "JuliaCI/merge-commit" => nothing ] -env(NAME, PROJECT) = Dict( + +safe_name(fn::AbstractString) = replace(fn, r"[^A-Za-z0-9_\-:]"=>"-") + +wait_step() = Dict(:wait => nothing) +group_step(name, steps) = Dict(:group => name, :steps => steps) + +function init_git_config() + git_config_dir = mktempdir() + git_config = joinpath(git_config_dir, "config") + open(git_config, write=true) do io + println(io, "[user]") + println(io, "\name = jlbuild") + println(io, "\temail = juliabuildbot@gmail.com") + end + return git_config +end + +env() = Dict( "JULIA_PKG_SERVER" => "us-east.pkg.julialang.org", "JULIA_PKG_SERVER_REGISTRY_PREFERENCE" => "eager", "JULIA_REGISTRYCI_AUTOMERGE" => "true", - "NAME" => NAME, - "PROJECT" => PROJECT, "YGGDRASIL" => "true", # Inherit the secret so that we can decrypt cryptic secrets "BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET" => get(ENV, "BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET", ""), + + # Store the (global) git config here, seed it with jlbuild values, and tell git to use only it + "GIT_CONFIG_GLOBAL" => init_git_config(), + "GIT_CONFIG_NOSYSTEM" => "true", ) -safe_name(fn::AbstractString) = replace(fn, r"[^A-Za-z0-9_\-:]"=>"-") +abstract type AbstractStep; end -wait_step() = Dict(:wait => nothing) -group_step(name, steps) = Dict(:group => name, :steps => steps) +abstract type AbstractBuildStep <: AbstractStep; end +struct BB1BuildStep <: AbstractBuildStep + name::String + project::String + platform::String +end -function build_step(NAME, PLATFORM, PROJECT) - script = raw""" - .buildkite/build.sh - """ +bb1_step_key(project_name::String, platform::String) = "$(safe_name(project_name))--$(safe_name(platform))" +step_key(bs::BB1BuildStep) = bb1_step_key(bs.project, bs.platform) +struct BB2BuildStep <: AbstractBuildStep + name::String + project::String + platform::String + build_hash::String + dependencies::Vector{String} +end +step_key(bs::BB2BuildStep) = "$(safe_name(bs.project))--$(bs.build_hash[5:13])" +universe_name(bs::BB2BuildStep) = "$(safe_name(bs.project))-$(YGGDRASIL_COMMIT)" + +function universe_flag(s) + # If we're on a PR, deploy to a custom universe on a custom org + if get(ENV, "BUILDKITE_PULL_REQUEST", "false") != "false" + "--universe=$(universe_name(s))" + end + return "" +end + +function render(bs::AbstractBuildStep) build_plugins = plugins() push!(build_plugins, "staticfloat/cryptic#v2" => Dict( @@ -66,51 +156,89 @@ function build_step(NAME, PLATFORM, PROJECT) "AWS_SECRET_ACCESS_KEY=\"U2FsdGVkX1846b0BRbZjwIWSFV+Fiv1C/Hds/vB3aTkxubHPnRP6lVxGkAkOcFuvAntkoLF6J64QrOHWvjz8xg==\"", ] ), - "staticfloat/coppermind#v2" => Dict( - "inputs" => [ - PROJECT, - ".ci/", - # ?meta.json - ], - "s3_prefix" => "s3://julia-bb-buildcache/" - ), ) - build_env = env(NAME, PROJECT) - merge!(build_env, Dict( - "PLATFORM" => PLATFORM, + + # Only use coppermind on BB1, we're going to do caching with BB2 differently. + if isa(bs, BB1BuildStep) + push!(build_plugins, + "staticfloat/coppermind#v2" => Dict( + "inputs" => [ + bs.project, + ".ci/", + # ?meta.json + ], + "s3_prefix" => "s3://julia-bb-buildcache/" + ), + ) + end + + # On BB1, we pass the platform to build for, on BB2 we identify by build hash + if isa(bs, BB1BuildStep) + build_sh_args = "$(bs.platform)" + else + build_sh_args = "$(universe_flag(bs)) --build-hashes=$(bs.build_hash) --archive-dir=products" + end + + build_env = merge(env(), Dict( + # These are all BinaryBuilder1 specific, but they don't hurt to send to BB2 "BINARYBUILDER_AUTOMATIC_APPLE" => "true", "BINARYBUILDER_USE_CCACHE" => "true", "BINARYBUILDER_STORAGE_DIR" => "/cache/yggdrasil", "BINARYBUILDER_CCACHE_DIR" => "/sharedcache/ccache", "BINARYBUILDER_NPROC" => "16", # Limit parallelism somewhat to avoid OOM for LLVM + + # This is for coppermind "AWS_ACCESS_KEY_ID" => "AKIA4WZGSTHCB2YWWN46", "AWS_DEFAULT_REGION" => "us-east-1", + + # Set the julia project to either the BB1 or BB2 project + "JULIA_PROJECT" => julia_project(bs), )) - Dict( - :key => "$(safe_name(PROJECT))--$(safe_name(PLATFORM))", - :label => "build -- $PROJECT -- $PLATFORM", + return Dict( + :key => step_key(bs), + :label => "build -- $(bs.name) -- $(bs.platform)", :agents => agent(), :plugins => build_plugins, :timeout_in_minutes => 240, :priority => -1, # Reduce concurrency for Reactant builds, which are extremely intensive and grind # the system to a halt when run with several parallel jobs. - :concurrency => NAME == "Reactant" ? 8 : 12, - :concurrency_group => "yggdrasil/build/$NAME", # Could use ENV["BUILDKITE_JOB_ID"] - :commands => [script], + :concurrency => startswith(bs.name, "Reactant") ? 8 : 12, + :concurrency_group => "yggdrasil/build/$(bs.project)", + :commands => [""" + YGGDRASIL_BASE=\$(pwd) + cd $(bs.project) + "\${YGGDRASIL_BASE}/.buildkite/build.sh" $(build_sh_args) + cd "\${YGGDRASIL_BASE}" + """], :env => build_env, :artifacts => [ - "**/products/$NAME*.tar.*" + "$(bs.project)/products/*" ] ) end -function register_step(NAME, PROJECT, SKIP_BUILD, NUM_PLATFORMS) - script = raw""" - BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET="" .buildkite/register.sh - """ +abstract type AbstractRegisterStep <: AbstractStep; end +struct BB1RegisterStep <: AbstractRegisterStep + name::String + project::String + skip_build::Bool + num_platforms::Int +end + +step_key(rs::BB1RegisterStep) = "register-$(rs.name)" + +struct BB2RegisterStep <: AbstractRegisterStep + name::String + project::String + + dependencies::Vector{AbstractStep} +end +step_key(rs::BB2RegisterStep) = "register-$(rs.name)" +universe_name(rs::BB2RegisterStep) = "$(safe_name(rs.project))-$(YGGDRASIL_COMMIT)" +function render(rs::AbstractRegisterStep) register_plugins = plugins() push!(register_plugins, "staticfloat/cryptic#v2" => Dict( @@ -119,27 +247,78 @@ function register_step(NAME, PROJECT, SKIP_BUILD, NUM_PLATFORMS) ] )) - register_env = env(NAME, PROJECT) - if SKIP_BUILD - register_env["SKIP_BUILD"] = "true" + register_env = env() + register_env["JULIA_PROJECT"] = julia_project(rs) + if isa(rs, BB1RegisterStep) + if rs.skip_build + register_env["SKIP_BUILD"] = "true" + end + register_env["META_JSON"] = joinpath(YGGDRASIL_BASE, "$(rs.name).meta.json") + + # For packages with a large number of platforms, trying to upload several release + # artifacts at once with `ghr` results in exceeding GitHub's API secondary rate limits. + # Ref: . + if rs.num_platforms > 80 + concurrency = 4 + @info "Reducing ghr concurrency" NAME NUM_PLATFORMS concurrency + register_env["BINARYBUILDER_GHR_CONCURRENCY"] = string(concurrency) + end end - # For packages with a large number of platforms, trying to upload several release - # artifacts at once with `ghr` results in exceeding GitHub's API secondary rate limits. - # Ref: . - if NUM_PLATFORMS > 80 - concurrency = 4 - @info "Reducing ghr concurrency" NAME NUM_PLATFORMS concurrency - register_env["BINARYBUILDER_GHR_CONCURRENCY"] = string(concurrency) + + artifact_download = "" + if isa(rs, BB2RegisterStep) + artifact_download = "mkdir -p products; buildkite-agent artifact download '$(rs.project)/products/*' 'products'" end - Dict( - :label => "register -- $NAME", + out = Dict( + :key => step_key(rs), + :label => "register -- $(rs.name)", :agents => agent(), :plugins => register_plugins, :timeout_in_minutes => 90, :concurrency => 1, :concurrency_group => "yggdrasil/register", - :commands => [script], - :env => register_env + :commands => [""" + set -u BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET + YGGDRASIL_BASE=\$(pwd) + cd $(rs.project) + $(artifact_download) + "\${YGGDRASIL_BASE}/.buildkite/register.sh" + cd "\${YGGDRASIL_BASE}" + """], + :env => register_env, ) + + if isa(rs, BB2RegisterStep) + out[:depends_on] = step_key.(rs.dependencies) + end + + return out +end + +# Remove secret from environment +sanitize(cmd) = addenv(cmd, Dict("BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET" => nothing)) +exec(cmd; stdout=Base.stdout, stderr=Base.stderr) = @assert success(pipeline(sanitize(cmd); stderr, stdout)) + +YGGDRASIL_BASE = dirname(@__DIR__) +YGGDRASIL_COMMIT = get(ENV, "BUILDKITE_COMMIT", "1a2b3c")[1:6] +bb1_julia_project = joinpath(YGGDRASIL_BASE, ".ci/bb1_project") +bb2_julia_project = joinpath(YGGDRASIL_BASE, ".ci/bb2_project") +function julia(args; julia_project=bb1_julia_project, stdout = Base.stdout, stderr = Base.stderr) + return exec(`$(Base.julia_cmd()) --project=$(julia_project) $args`; stdout, stderr) +end + +function julia_project(s) + if isa(s, BB1BuildStep) || isa(s, BB1RegisterStep) + return ".ci/bb1_project" + else + return ".ci/bb2_project" + end +end + +# We will use the BinaryBuilder2 environment if there is a `using BinaryBuilder2` in the build_tarballs.jl file +function uses_bb2(project) + build_tarballs_path = joinpath(YGGDRASIL_BASE, project, "build_tarballs.jl") + build_tarballs_contents = String(read(open(build_tarballs_path))) + return any(startswith.(split(build_tarballs_contents, "\n"), ("using BinaryBuilder2",))) end diff --git a/.ci/Manifest.toml b/.ci/bb1_project/Manifest.toml similarity index 100% rename from .ci/Manifest.toml rename to .ci/bb1_project/Manifest.toml diff --git a/.ci/Project.toml b/.ci/bb1_project/Project.toml similarity index 100% rename from .ci/Project.toml rename to .ci/bb1_project/Project.toml diff --git a/.ci/download_sources.jl b/.ci/bb1_project/download_sources.jl similarity index 100% rename from .ci/download_sources.jl rename to .ci/bb1_project/download_sources.jl diff --git a/.ci/generate_platforms.jl b/.ci/bb1_project/generate_platforms.jl similarity index 100% rename from .ci/generate_platforms.jl rename to .ci/bb1_project/generate_platforms.jl diff --git a/.ci/register_package.jl b/.ci/bb1_project/register_package.jl similarity index 100% rename from .ci/register_package.jl rename to .ci/bb1_project/register_package.jl diff --git a/.ci/update_manifest.jl b/.ci/bb1_project/update_manifest.jl similarity index 100% rename from .ci/update_manifest.jl rename to .ci/bb1_project/update_manifest.jl diff --git a/.ci/bb2_project/Manifest.toml b/.ci/bb2_project/Manifest.toml new file mode 100644 index 00000000000..58f11556bae --- /dev/null +++ b/.ci/bb2_project/Manifest.toml @@ -0,0 +1,511 @@ +# This file is machine-generated - editing it directly is not advised + +julia_version = "1.12.5" +manifest_format = "2.0" +project_hash = "f43ce102636991893b567e0ead79cd7d961942b7" + +[[deps.ArgTools]] +uuid = "0dad84c5-d112-42e6-8d28-ef12dabb789f" +version = "1.1.2" + +[[deps.Artifacts]] +uuid = "56f22d72-fd6d-98f1-02f0-08ddc0907c33" +version = "1.11.0" + +[[deps.Attr_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl"] +git-tree-sha1 = "b8747cda97b6cedbd8dc9fc6218e1d53ce8de32f" +uuid = "1fd713ca-387f-5abc-8002-d8b8b1623b73" +version = "2.5.3+0" + +[[deps.Base64]] +uuid = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f" +version = "1.11.0" + +[[deps.BinaryBuilder2]] +deps = ["Artifacts", "BinaryBuilderAuditor", "BinaryBuilderGitUtils", "BinaryBuilderPlatformExtensions", "BinaryBuilderProducts", "BinaryBuilderSources", "BinaryBuilderToolchains", "Ccache_jll", "Dates", "Downloads", "Git", "HistoricalStdlibVersions", "Infiltrator", "JLLGenerator", "JLLPrefixes", "KeywordArgumentExtraction", "LazyJLLWrappers", "Libdl", "LocalRegistry", "Logging", "LoggingExtras", "MultiHashParsing", "ObjectFile", "OrderedCollections", "OutputCollectors", "Pkg", "PrecompileTools", "Preferences", "Random", "Reexport", "RegistryTools", "SHA", "Sandbox", "Scratch", "ScratchSpaceGarbageCollector", "StructEquality", "StyledStrings", "TOML", "Test", "TimerOutputs", "TreeArchival", "UserNSSandbox_jll", "gh_cli_jll"] +git-tree-sha1 = "98005986ee38c5913246f9a6fa42c89b5ef1cd60" +repo-rev = "main" +repo-url = "https://github.com/JuliaPackaging/BinaryBuilder2.jl.git" +uuid = "12aac903-9f7c-5d81-afc2-d9565ea332af" +version = "1.0.1" + +[[deps.BinaryBuilderAuditor]] +deps = ["BinaryBuilderPlatformExtensions", "BinaryBuilderProducts", "BinaryBuilderSources", "BinaryBuilderToolchains", "JLLGenerator", "ObjectFile", "Pkg", "StyledStrings"] +git-tree-sha1 = "4d9cc98c984347619a2ec1640016e1875c68806c" +uuid = "53524979-5234-6e31-4220-6120654b694c" +version = "0.2.0" + +[[deps.BinaryBuilderGitUtils]] +deps = ["Git", "Git_jll", "MultiHashParsing", "gh_cli_jll"] +git-tree-sha1 = "14d6018f14d9df6cf92e90e97a98b3027ab4244f" +uuid = "654d7472-7548-7361-4832-74694762694c" +version = "0.2.0" + +[[deps.BinaryBuilderPlatformExtensions]] +deps = ["OrderedCollections", "PrecompileTools", "Reexport"] +git-tree-sha1 = "067afea6e559ffd71dca3125cdbe665635c6a85c" +uuid = "213f2928-4d72-6f46-7461-4c705f634367" +version = "0.2.0" + +[[deps.BinaryBuilderProducts]] +deps = ["BinaryBuilderSources", "JLLGenerator", "KeywordArgumentExtraction", "Libdl"] +git-tree-sha1 = "d1145b66a8a9ed1a9972f4051637b94230189a62" +uuid = "21737265-7a69-6e4f-4974-6375646f7270" +version = "0.2.0" + +[[deps.BinaryBuilderSources]] +deps = ["BinaryBuilderGitUtils", "Dates", "Downloads", "Git", "JLLPrefixes", "MultiHashParsing", "Pkg", "Random", "SHA", "Scratch", "TOML", "TimerOutputs", "TreeArchival"] +git-tree-sha1 = "602033da2f110123543db1ac45861bded509df0c" +uuid = "316c416d-4527-6863-7465-466137743047" +version = "0.2.0" + +[[deps.BinaryBuilderToolchains]] +deps = ["Artifacts", "BinaryBuilderPlatformExtensions", "BinaryBuilderSources", "HistoricalStdlibVersions", "JLLPrefixes", "LazyJLLWrappers", "NetworkOptions", "Pkg", "PrecompileTools", "Reexport", "SHA", "Scratch", "TreeArchival"] +git-tree-sha1 = "a9e66df160930be64aa67d6dea0a5be6ad05c3f3" +uuid = "33566f4c-336c-3150-6d30-4374274e6143" +version = "0.3.0" + +[[deps.Ccache_jll]] +deps = ["Artifacts", "CompilerSupportLibraries_jll", "JLLWrappers", "Libdl", "Zlib_jll", "Zstd_jll"] +git-tree-sha1 = "ff8d49ca66b9731888e3a7dcdda74b5342a5a6fc" +uuid = "c6bc53e4-e6d8-51d9-af6d-d7969aa08f60" +version = "4.9.1+0" + +[[deps.Compat]] +deps = ["TOML", "UUIDs"] +git-tree-sha1 = "9d8a54ce4b17aa5bdce0ea5c34bc5e7c340d16ad" +uuid = "34da2185-b29b-5c13-b0c7-acf172513d20" +version = "4.18.1" + + [deps.Compat.extensions] + CompatLinearAlgebraExt = "LinearAlgebra" + + [deps.Compat.weakdeps] + Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" + LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" + +[[deps.CompilerSupportLibraries_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "e66e0078-7015-5450-92f7-15fbd957f2ae" +version = "1.3.0+1" + +[[deps.Dates]] +deps = ["Printf"] +uuid = "ade2ca70-3891-5945-98fb-dc099432e06a" +version = "1.11.0" + +[[deps.Downloads]] +deps = ["ArgTools", "FileWatching", "LibCURL", "NetworkOptions"] +uuid = "f43a241f-c20a-4ad4-852c-f6b1247861c6" +version = "1.7.0" + +[[deps.EnumX]] +git-tree-sha1 = "c49898e8438c828577f04b92fc9368c388ac783c" +uuid = "4e289a0a-7415-4d19-859d-a7e5c4648b56" +version = "1.0.7" + +[[deps.Expat_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl"] +git-tree-sha1 = "9cb7fe11da6adb8683cbacf8aa9b5237941e3a75" +uuid = "2e619515-83b5-522b-bb60-26c02a35a201" +version = "2.7.5+0" + +[[deps.ExprTools]] +git-tree-sha1 = "27415f162e6028e81c72b82ef756bf321213b6ec" +uuid = "e2ba6199-217a-4e67-a87a-7c52f15ade04" +version = "0.1.10" + +[[deps.FileWatching]] +uuid = "7b1f6079-737a-58dc-b8bc-7a2ca5c1b5ee" +version = "1.11.0" + +[[deps.Git]] +deps = ["Git_LFS_jll", "Git_jll", "JLLWrappers", "OpenSSH_jll"] +git-tree-sha1 = "824a1890086880696fc908fe12a17bcf61738bd8" +uuid = "d7ba0133-e1db-5d97-8f8c-041e4b3a1eb2" +version = "1.5.0" + +[[deps.Git_LFS_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl"] +git-tree-sha1 = "bb8471f313ed941f299aa53d32a94ab3bee08844" +uuid = "020c3dae-16b3-5ae5-87b3-4cb189e250b2" +version = "3.7.0+0" + +[[deps.Git_jll]] +deps = ["Artifacts", "Expat_jll", "JLLWrappers", "LibCURL_jll", "Libdl", "Libiconv_jll", "OpenSSL_jll", "PCRE2_jll", "Zlib_jll"] +git-tree-sha1 = "0dd4cfb426924210c8f42742751cbde74b27bfa3" +uuid = "f8c6e375-362e-5223-8a59-34ff63f689eb" +version = "2.54.0+0" + +[[deps.HistoricalStdlibVersions]] +deps = ["Pkg", "PrecompileTools"] +git-tree-sha1 = "abdd6437ede003ee3c412b7da655a8dde356a573" +uuid = "6df8b67a-e8a0-4029-b4b7-ac196fe72102" +version = "2.0.6" + +[[deps.Infiltrator]] +deps = ["InteractiveUtils", "Markdown", "REPL", "UUIDs"] +git-tree-sha1 = "b1eccca4ebc42ac306496c5a5677e50251a19ab1" +uuid = "5903a43b-9cc3-4c30-8d17-598619ec4e9b" +version = "1.9.10" + +[[deps.InteractiveUtils]] +deps = ["Markdown"] +uuid = "b77e0a4c-d291-57a0-90e8-8db25a27a240" +version = "1.11.0" + +[[deps.JLLGenerator]] +deps = ["BinaryBuilderPlatformExtensions", "Dates", "Libdl", "LicenseCheck", "MultiHashParsing", "Pkg", "Reexport", "SHA", "StructEquality", "TOML"] +git-tree-sha1 = "3f8d48349031d53fb0f19cd84074dcd37f07a809" +uuid = "73536572-7074-4e69-5270-206c4c6a2061" +version = "0.4.1" +weakdeps = ["BinaryBuilderSources"] + + [deps.JLLGenerator.extensions] + BinaryBuilderSourcesExt = "BinaryBuilderSources" + +[[deps.JLLPrefixes]] +deps = ["Artifacts", "Git", "HistoricalStdlibVersions", "Pkg", "Preferences", "SHA", "Scratch"] +git-tree-sha1 = "a5bb3dd3e86ad7fdd07820ffea7bba4e06c133f8" +uuid = "afc68a34-7891-4c5a-9da1-1c62935e7b0d" +version = "0.4.2" + +[[deps.JLLWrappers]] +deps = ["Artifacts", "Preferences"] +git-tree-sha1 = "0533e564aae234aff59ab625543145446d8b6ec2" +uuid = "692b3bcd-3c85-4b1f-b108-f13ce0eb3210" +version = "1.7.1" + +[[deps.JuliaSyntaxHighlighting]] +deps = ["StyledStrings"] +uuid = "ac6e5ff7-fb65-4e79-a425-ec3bc9c03011" +version = "1.12.0" + +[[deps.KeywordArgumentExtraction]] +deps = ["ExprTools", "InteractiveUtils"] +git-tree-sha1 = "e7bb4202f8213e1d10989fc34cb68b3f1687c121" +uuid = "45533465-3150-2073-4772-41774b207255" +version = "1.2.0" + +[[deps.LazilyInitializedFields]] +git-tree-sha1 = "0f2da712350b020bc3957f269c9caad516383ee0" +uuid = "0e77f7df-68c5-4e49-93ce-4cd80f5598bf" +version = "1.3.0" + +[[deps.LazyArtifacts]] +deps = ["Artifacts", "Pkg"] +uuid = "4af54fe1-eca0-43a8-85a7-787d91b784e3" +version = "1.11.0" + +[[deps.LazyJLLWrappers]] +deps = ["Artifacts", "Libdl", "Pkg", "Preferences", "TOML"] +git-tree-sha1 = "a02a187f0c4c8c9be99b22f8ad5fe41da34ff14f" +uuid = "21706172-204c-4d4f-5420-656854206f44" +version = "1.1.0" + +[[deps.LibCURL]] +deps = ["LibCURL_jll", "MozillaCACerts_jll"] +uuid = "b27032c2-a3e7-50c8-80cd-2d36dbcbfd21" +version = "0.6.4" + +[[deps.LibCURL_jll]] +deps = ["Artifacts", "LibSSH2_jll", "Libdl", "OpenSSL_jll", "Zlib_jll", "nghttp2_jll"] +uuid = "deac9b47-8bc7-5906-a0fe-35ac56dc84c0" +version = "8.15.0+0" + +[[deps.LibGit2]] +deps = ["LibGit2_jll", "NetworkOptions", "Printf", "SHA"] +uuid = "76f85450-5226-5b5a-8eaa-529ad045b433" +version = "1.11.0" + +[[deps.LibGit2_jll]] +deps = ["Artifacts", "LibSSH2_jll", "Libdl", "OpenSSL_jll"] +uuid = "e37daf67-58a4-590a-8e99-b0245dd2ffc5" +version = "1.9.0+0" + +[[deps.LibSSH2_jll]] +deps = ["Artifacts", "Libdl", "OpenSSL_jll"] +uuid = "29816b5a-b9ab-546f-933c-edad1886dfa8" +version = "1.11.3+1" + +[[deps.Libdl]] +uuid = "8f399da3-3557-5675-b5ff-fb832c97cbdb" +version = "1.11.0" + +[[deps.Libiconv_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl"] +git-tree-sha1 = "be484f5c92fad0bd8acfef35fe017900b0b73809" +uuid = "94ce4f54-9a6c-5748-9c1c-f9c7231a4531" +version = "1.18.0+0" + +[[deps.LicenseCheck]] +deps = ["licensecheck_jll"] +git-tree-sha1 = "9c36d53a16450b2302720c822c62b0fb903d5df7" +uuid = "726dbf0d-6eb6-41af-b36c-cd770e0f00cc" +version = "0.2.3" + +[[deps.LocalRegistry]] +deps = ["Random", "RegistryInstances", "RegistryTools", "TOML", "UUIDs"] +git-tree-sha1 = "80d096310838ebd9dfbe7934034da35c26ff4df4" +uuid = "89398ba2-070a-4b16-a995-9893c55d93cf" +version = "0.5.7" + +[[deps.Logging]] +uuid = "56ddb016-857b-54e1-b83d-db4d58db5568" +version = "1.11.0" + +[[deps.LoggingExtras]] +deps = ["Dates", "Logging"] +git-tree-sha1 = "f00544d95982ea270145636c181ceda21c4e2575" +uuid = "e6f89c97-d47a-5376-807f-9c37f3926c36" +version = "1.2.0" + +[[deps.Markdown]] +deps = ["Base64", "JuliaSyntaxHighlighting", "StyledStrings"] +uuid = "d6f4376e-aef5-505a-96c1-9c027394607a" +version = "1.11.0" + +[[deps.MozillaCACerts_jll]] +uuid = "14a3606d-f60d-562e-9121-12d972cd8159" +version = "2025.11.4" + +[[deps.MultiHashParsing]] +deps = ["SHA"] +git-tree-sha1 = "5b50aad47daa2f19b61fae3abb3bfe431eff1347" +uuid = "73786568-6863-756d-6873-6168796e616d" +version = "0.2.1" + +[[deps.NetworkOptions]] +uuid = "ca575930-c2e3-43a9-ace4-1e988b2c1908" +version = "1.3.0" + +[[deps.ObjectFile]] +deps = ["Reexport", "StructIO"] +git-tree-sha1 = "09b1fe6ff16e6587fa240c165347474322e77cf1" +uuid = "d8793406-e978-5875-9003-1fc021f44a92" +version = "0.4.4" + +[[deps.OpenSSH_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "OpenSSL_jll", "Zlib_jll"] +git-tree-sha1 = "57baa4b81a24c2910afbb6d853aa0685e4312bf7" +uuid = "9bd350c2-7e96-507f-8002-3f2e150b4e1b" +version = "10.3.1+0" + +[[deps.OpenSSL_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "458c3c95-2e84-50aa-8efc-19380b2a3a95" +version = "3.5.4+0" + +[[deps.OrderedCollections]] +git-tree-sha1 = "05868e21324cede2207c6f0f466b4bfef6d5e7ee" +uuid = "bac558e1-5e72-5ebc-8fee-abe8a469f55d" +version = "1.8.1" + +[[deps.OutputCollectors]] +deps = ["StyledStrings"] +git-tree-sha1 = "eb19f9f90e662259412f4e17bbab2b7d6292c503" +uuid = "6c11c7d4-943b-4e2b-80de-f2cfc2930a8c" +version = "1.0.0" + +[[deps.PCRE2_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "efcefdf7-47ab-520b-bdef-62a2eaa19f15" +version = "10.44.0+1" + +[[deps.Pkg]] +deps = ["Artifacts", "Dates", "Downloads", "FileWatching", "LibGit2", "Libdl", "Logging", "Markdown", "Printf", "Random", "SHA", "TOML", "Tar", "UUIDs", "p7zip_jll"] +uuid = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" +version = "1.12.1" +weakdeps = ["REPL"] + + [deps.Pkg.extensions] + REPLExt = "REPL" + +[[deps.PrecompileTools]] +deps = ["Preferences"] +git-tree-sha1 = "07a921781cab75691315adc645096ed5e370cb77" +uuid = "aea7be01-6a6a-4083-8856-8a6e6704d82a" +version = "1.3.3" + +[[deps.Preferences]] +deps = ["TOML"] +git-tree-sha1 = "8b770b60760d4451834fe79dd483e318eee709c4" +uuid = "21216c6a-2e73-6563-6e65-726566657250" +version = "1.5.2" + +[[deps.Printf]] +deps = ["Unicode"] +uuid = "de0858da-6303-5e67-8744-51eddeeeb8d7" +version = "1.11.0" + +[[deps.REPL]] +deps = ["InteractiveUtils", "JuliaSyntaxHighlighting", "Markdown", "Sockets", "StyledStrings", "Unicode"] +uuid = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb" +version = "1.11.0" + +[[deps.Random]] +deps = ["SHA"] +uuid = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" +version = "1.11.0" + +[[deps.Reexport]] +git-tree-sha1 = "45e428421666073eab6f2da5c9d310d99bb12f9b" +uuid = "189a3867-3050-52da-a836-e630ba90ab69" +version = "1.2.2" + +[[deps.RegistryInstances]] +deps = ["LazilyInitializedFields", "Pkg", "TOML", "Tar"] +git-tree-sha1 = "ffd19052caf598b8653b99404058fce14828be51" +uuid = "2792f1a3-b283-48e8-9a74-f99dce5104f3" +version = "0.1.0" + +[[deps.RegistryTools]] +deps = ["LibGit2", "Pkg", "SHA", "UUIDs"] +git-tree-sha1 = "c887ec1a5f4fe2872dfa01640a1dd2050fac48f4" +uuid = "d1eb7eb1-105f-429d-abf5-b0f65cb9e2c4" +version = "2.4.3" + +[[deps.SHA]] +uuid = "ea8e919c-243c-51af-8825-aaa63cd721ce" +version = "0.7.0" + +[[deps.Sandbox]] +deps = ["EnumX", "LazyArtifacts", "Libdl", "Preferences", "Random", "SHA", "Scratch", "TOML", "Tar", "Tar_jll", "UserNSSandbox_jll"] +git-tree-sha1 = "0ef566f01a76acf5a98f1371e05e07ebec2e982a" +uuid = "9307e30f-c43e-9ca7-d17c-c2dc59df670d" +version = "2.1.3" + +[[deps.Scratch]] +deps = ["Dates"] +git-tree-sha1 = "9b81b8393e50b7d4e6d0a9f14e192294d3b7c109" +uuid = "6c6a2e73-6563-6170-7368-637461726353" +version = "1.3.0" + +[[deps.ScratchSpaceGarbageCollector]] +deps = ["Dates", "Scratch", "TOML"] +git-tree-sha1 = "23fb4904a86f00eff119865bbbbc0a8f58928ef3" +uuid = "2b218182-314f-4389-a4eb-69d351d2272a" +version = "0.1.1" + +[[deps.Serialization]] +uuid = "9e88b42a-f829-5b0c-bbe9-9e923198166b" +version = "1.11.0" + +[[deps.Sockets]] +uuid = "6462fe0b-24de-5631-8697-dd941f90decc" +version = "1.11.0" + +[[deps.StringEncodings]] +deps = ["Libiconv_jll"] +git-tree-sha1 = "b765e46ba27ecf6b44faf70df40c57aa3a547dcb" +uuid = "69024149-9ee7-55f6-a4c4-859efe599b68" +version = "0.3.7" + +[[deps.StructEquality]] +deps = ["Compat"] +git-tree-sha1 = "192a9f1de3cfef80ab1a4ba7b150bb0e11ceedcf" +uuid = "6ec83bb0-ed9f-11e9-3b4c-2b04cb4e219c" +version = "2.1.0" + +[[deps.StructIO]] +git-tree-sha1 = "c581be48ae1cbf83e899b14c07a807e1787512cc" +uuid = "53d494c1-5632-5724-8f4c-31dff12d585f" +version = "0.3.1" + +[[deps.StyledStrings]] +uuid = "f489334b-da3d-4c2e-b8f0-e476e12c162b" +version = "1.11.0" + +[[deps.TOML]] +deps = ["Dates"] +uuid = "fa267f1f-6049-4f14-aa54-33bafae1ed76" +version = "1.0.3" + +[[deps.Tar]] +deps = ["ArgTools", "SHA"] +uuid = "a4e569a6-e804-4fa4-b0f3-eef7a1d5b13e" +version = "1.10.0" + +[[deps.Tar_jll]] +deps = ["Artifacts", "Attr_jll", "JLLWrappers", "Libdl", "Libiconv_jll"] +git-tree-sha1 = "85e7d0ef5248971fbd824f29c52ab6168b895dfd" +uuid = "9b64493d-8859-5bf3-93d7-7c32dd38186f" +version = "1.35.0+0" + +[[deps.Test]] +deps = ["InteractiveUtils", "Logging", "Random", "Serialization"] +uuid = "8dfed614-e22c-5e08-85e1-65c5234f0b40" +version = "1.11.0" + +[[deps.TimerOutputs]] +deps = ["ExprTools", "Printf"] +git-tree-sha1 = "3748bd928e68c7c346b52125cf41fff0de6937d0" +uuid = "a759f4b9-e2f1-59dc-863e-4aeb61b1ea8f" +version = "0.5.29" + + [deps.TimerOutputs.extensions] + FlameGraphsExt = "FlameGraphs" + + [deps.TimerOutputs.weakdeps] + FlameGraphs = "08572546-2f56-4bcf-ba4e-bab62c3a3f89" + +[[deps.TreeArchival]] +deps = ["SHA", "Tar", "Tar_jll", "Zstd_jll", "p7zip_jll"] +git-tree-sha1 = "c8fdba0b5836047e2060b887da3761effa6f6864" +uuid = "216c6a2e-6c61-7669-6863-726165657274" +version = "0.2.0" + +[[deps.UUIDs]] +deps = ["Random", "SHA"] +uuid = "cf7118a7-6976-5b1a-9a39-7adc72f591a4" +version = "1.11.0" + +[[deps.Unicode]] +uuid = "4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5" +version = "1.11.0" + +[[deps.UserNSSandbox_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl"] +git-tree-sha1 = "79dc57555c873394aa67457f9c6022068da7b04a" +uuid = "b88861f7-1d72-59dd-91e7-a8cc876a4984" +version = "2025.1.28+0" + +[[deps.YAML]] +deps = ["Base64", "Dates", "Printf", "StringEncodings"] +git-tree-sha1 = "a1c0c7585346251353cddede21f180b96388c403" +uuid = "ddb6d928-2868-570f-bddf-ab3f9cf99eb6" +version = "0.4.16" + +[[deps.Zlib_jll]] +deps = ["Libdl"] +uuid = "83775a58-1f1d-513f-b197-d71354ab007a" +version = "1.3.1+2" + +[[deps.Zstd_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl"] +git-tree-sha1 = "446b23e73536f84e8037f5dce465e92275f6a308" +uuid = "3161d3a3-bdf6-5164-811a-617609db77b4" +version = "1.5.7+1" + +[[deps.gh_cli_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl"] +git-tree-sha1 = "894b4ad6ac54832386f8d01e84cab691b588f364" +uuid = "5d31d589-30fb-542f-b82d-10325e863e38" +version = "2.83.2+0" + +[[deps.licensecheck_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl"] +git-tree-sha1 = "ec4140cff0cecddeb1bcf0253194ea655bae1c21" +uuid = "4ecb348a-8b88-51ea-b912-4c460483ee91" +version = "0.4.0+0" + +[[deps.nghttp2_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "8e850ede-7688-5339-a07c-302acd2aaf8d" +version = "1.64.0+1" + +[[deps.p7zip_jll]] +deps = ["Artifacts", "CompilerSupportLibraries_jll", "Libdl"] +uuid = "3f19e933-33d8-53b3-aaab-bd5110c3b7a0" +version = "17.7.0+0" diff --git a/.ci/bb2_project/Project.toml b/.ci/bb2_project/Project.toml new file mode 100644 index 00000000000..105ef0b97e1 --- /dev/null +++ b/.ci/bb2_project/Project.toml @@ -0,0 +1,3 @@ +[deps] +BinaryBuilder2 = "12aac903-9f7c-5d81-afc2-d9565ea332af" +YAML = "ddb6d928-2868-570f-bddf-ab3f9cf99eb6" diff --git a/.ci/bb2_project/build_test.jl b/.ci/bb2_project/build_test.jl new file mode 100644 index 00000000000..22155c8000c --- /dev/null +++ b/.ci/bb2_project/build_test.jl @@ -0,0 +1,76 @@ +# This script is meant to run a simple test of the BB2 Yggdrasil build integration +using YAML + +# Mark ourselves as a pull request by default. The `git branch` thing +# is so that `generator.jl` can get a commit message for this fictitious pull request. +ENV["BUILDKITE_PULL_REQUEST"] = "-1" +run(`git branch -f origin/pr/-1`) + +YGGDRASIL_BASE = dirname(dirname(@__DIR__)) + +# Load YAML definition of buildkite job +yaml = YAML.load(readchomp( + Cmd(`$(Base.julia_cmd()) --project .buildkite/generator.jl H/HelloWorldCxx`; + dir=YGGDRASIL_BASE + ) +)) + +function run_buildkite_cmd(cmd) + env = copy(ENV) + env["JULIA_PROJECT"] = @__DIR__ + return run(Cmd(`/bin/bash -c $(cmd)`; dir=YGGDRASIL_BASE, env)) +end + +# We're going to "upload" our build products to this archive folder +# This also preserves the artifacts from our `clean_products.sh` script +archive_dir = joinpath(YGGDRASIL_BASE, "archive") +mkpath(archive_dir) + +# Run all the build steps +for bk_group in yaml["steps"] + for bk_step in bk_group["steps"] + continue + if !startswith(bk_step["label"], "build") + continue + end + + for cmd in bk_step["commands"] + @info(bk_step["label"]) + run_buildkite_cmd(cmd) + + # Now, "upload those builds" + products_dir = joinpath(YGGDRASIL_BASE, bk_group["group"], "products") + for file in readdir(products_dir) + @info("Upload: ", file) + mv(joinpath(products_dir, file), joinpath(archive_dir, file)) + end + end + end +end + +# Now we're going to run the register steps +for bk_group in yaml["steps"] + for bk_step in bk_group["steps"] + if !startswith(bk_step["label"], "register") + println("skipping $(bk_step["label"])") + continue + end + + # "download" the artifacts + products_dir = joinpath(YGGDRASIL_BASE, bk_group["group"], "products") + for file in readdir(archive_dir) + @info("Download: ", file) + cp(joinpath(archive_dir, file), joinpath(products_dir, file)) + end + + # Run registration + for cmd in bk_step["commands"] + @info(bk_step["label"]) + run_buildkite_cmd(cmd) + end + end +end + + +# Cleanup our fake branch +run(`git branch -d origin/pr/-1`) diff --git a/.ci/bb2_project/extract_artifacts.jl b/.ci/bb2_project/extract_artifacts.jl new file mode 100644 index 00000000000..241284a6806 --- /dev/null +++ b/.ci/bb2_project/extract_artifacts.jl @@ -0,0 +1,3 @@ +using BinaryBuilder2 + +# Load in the set of artifacts, diff --git a/.ci/bb2_project/register_package.jl b/.ci/bb2_project/register_package.jl new file mode 100644 index 00000000000..0cdd27e534b --- /dev/null +++ b/.ci/bb2_project/register_package.jl @@ -0,0 +1,13 @@ +using BinaryBuilder2 +using BinaryBuilder2: import_archives + +meta = BuildMeta(; + deploy_org="JuliaBinaryWrappers", + register=true, +) + +# Import archives from `products` +import_archives(meta.build_cache, "products") + +# Then run build_tarballs.jl; this shouldn't build anything +run_build_tarballs(meta, "build_tarballs.jl") diff --git a/.ci/bb2_project/scheduler.jl b/.ci/bb2_project/scheduler.jl new file mode 100644 index 00000000000..5fd8aa27108 --- /dev/null +++ b/.ci/bb2_project/scheduler.jl @@ -0,0 +1,72 @@ +using BinaryBuilder2, YAML + +# Given a list of projects (which each have a `build_tarballs.jl` file), use BB2's dry-run capability +# to get a list of `BuildConfig` objects. We will then emit a structured output back to the calling +# script that allows it to emit buildkite jobs. + +# Start by ingesting all build_tarballs into `meta` +const YGGDRASIL_BASE = dirname(dirname(@__DIR__)) +meta = BuildMeta(; verbose=false, dry_run=["build"]) +for project in ARGS + run_build_tarballs(meta, joinpath(YGGDRASIL_BASE, project, "build_tarballs.jl")) +end + +package_names = [package_config.name for package_config in keys(meta.packagings)] +function get_dep_names(pr::PackageResult) + dep_names = String[] + for er in first.(values(pr.config.named_extractions)) + for ts in er.config.build.config.target_specs + for dep in ts.dependencies + if !isa(dep, JLLSource) + continue + end + + dep_name = BinaryBuilder2.strip_jll_suffix(dep.package.name) + if dep_name ∈ package_names + push!(dep_names, dep_name) + end + end + end + end + return dep_names +end + +function project_path(build_result::BuildResult) + build_script_path = build_result.config.build_script_path + if isfile(build_script_path) + build_script_path = dirname(build_script_path) + end + return relpath(build_script_path, YGGDRASIL_BASE) +end + + +# Next, iterate over `PackageResult` objects; each `PackageResult` will become a registration job, +# and the `BuildConfig`'s from each `PackageResult` will become a build job. Note that we +# deduplicate `BuildConfig`'s because it's possible for a single build to be packaged into multiple +# separate JLLs. + +groups = Dict() +get_group(name) = get!(groups, name, Dict("builds" => [], "packagings" => [])) + +for (_, package_result) in meta.packagings + package_deps = get_dep_names(package_result) + for build_result in collect_builds(package_result) + group = get_group(project_path(build_result)) + push!(group["builds"], Dict( + "name" => build_result.config.src_name, + "script" => build_result.config.build_script_path, + "build_hash" => spec_hash(build_result.config), + "platform" => triplet(BinaryBuilder2.get_default_target_spec(build_result.config).platform.target), + "dependencies" => package_deps, + )) + end + group = get_group(project_path(first(collect_builds(package_result)))) + push!(group["packagings"], Dict( + "name" => package_result.config.name, + "dependencies" => package_deps, + )) +end + +# Emit structured output that our caller can interpret. +# We use YAML because it already exists in the `.buildkite` project +YAML.write(stdout, groups) diff --git a/.ci/bb2_project/test.sh b/.ci/bb2_project/test.sh new file mode 100755 index 00000000000..236f1de3e50 --- /dev/null +++ b/.ci/bb2_project/test.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +for x in "$@"; do + echo "$x" +done diff --git a/.github/workflows/update_manifest.yml b/.github/workflows/update_manifest.yml index e90208988cd..8804c51c6a2 100644 --- a/.github/workflows/update_manifest.yml +++ b/.github/workflows/update_manifest.yml @@ -30,7 +30,7 @@ jobs: Pkg.instantiate() Pkg.precompile() - name: Generate commit message - run: julia --project=.ci --color=yes .ci/update_manifest.jl + run: julia --project=.ci/bb1_project --color=yes .ci/bb1_project/update_manifest.jl id: commit_message - name: Show git status run: git status diff --git a/.gitignore b/.gitignore index 305d2f4a9f7..d4ba67e9006 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ # Ignore intermediate and final build products build/ +archive/ */**/build */**/products @@ -7,7 +8,8 @@ build/ *.swp # Ignore meta.json files -meta.json +*.meta.json +*.platform.list .DS_Store diff --git a/H/HelloWorldC/build_tarballs.jl b/H/HelloWorldC/build_tarballs.jl index 2fbe0c62d73..89d42141737 100644 --- a/H/HelloWorldC/build_tarballs.jl +++ b/H/HelloWorldC/build_tarballs.jl @@ -1,7 +1,7 @@ using BinaryBuilder name = "HelloWorldC" -version = v"1.4.1" +version = v"1.4.2" # No sources, we're just building the testsuite sources = [ diff --git a/H/HelloWorldCxx/build_tarballs.jl b/H/HelloWorldCxx/build_tarballs.jl index 50bd39d9288..5d922a5b437 100644 --- a/H/HelloWorldCxx/build_tarballs.jl +++ b/H/HelloWorldCxx/build_tarballs.jl @@ -1,16 +1,17 @@ -using BinaryBuilder +using BinaryBuilder2, BinaryBuilder2.Compat name = "HelloWorldCxx" -version = v"1.1.1" +version = v"1.1.2" # No sources, we're just building the testsuite sources = [ + DirectorySource("./bundled"), ] # Bash recipe for building across all platforms script = raw""" mkdir -p ${prefix}/bin -c++ -o ${prefix}/bin/hello_world${exeext} -g -O2 /usr/share/testsuite/cxx/hello_world/hello_world.cc +c++ -o ${prefix}/bin/hello_world${exeext} -g -O2 hello_world.cc install_license /usr/share/licenses/MIT """ diff --git a/H/HelloWorldCxx/bundled/hello_world.cc b/H/HelloWorldCxx/bundled/hello_world.cc new file mode 100644 index 00000000000..5309443c087 --- /dev/null +++ b/H/HelloWorldCxx/bundled/hello_world.cc @@ -0,0 +1,6 @@ +#include + +int main(void) { + std::cout << "Hello, World!\n"; + return 0; +}