Add BB2 CI setup

.buildkite/Manifest.toml

@@ -1,6 +1,6 @@
 # This file is machine-generated - editing it directly is not advised
 
-julia_version = "1.12.4"
+julia_version = "1.12.5"
 manifest_format = "2.0"
 project_hash = "5023fc07bb1b56800f97a2d66fbd8bd8c9c8d7a1"
 
@@ -116,9 +116,9 @@ version = "1.12.1"
 
 [[deps.Preferences]]
 deps = ["TOML"]
-git-tree-sha1 = "522f093a29b31a93e34eaea17ba055d850edea28"
+git-tree-sha1 = "8b770b60760d4451834fe79dd483e318eee709c4"
 uuid = "21216c6a-2e73-6563-6e65-726566657250"
-version = "1.5.1"
+version = "1.5.2"
 
 [[deps.Printf]]
 deps = ["Unicode"]

.buildkite/build.sh

@@ -2,12 +2,22 @@
 # Fail on error
 set -e
 
+SCRIPT_DIR="$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"
+YGGDRASIL_BASE="$(dirname "${SCRIPT_DIR}")"
+JULIA_PROJECT="${YGGDRASIL_BASE}/${JULIA_PROJECT:-/foo}"
+
+# Early-exit if someone is blindly running this manually
+if [[ ! -d "${JULIA_PROJECT:-}" ]]; then
+    echo "ERROR: Must set JULIA_PROJECT to one of:" >&2
+    echo "  - ${YGGDRASIL_BASE}/.ci/bb1_project" >&2
+    echo "  - ${YGGDRASIL_BASE}/.ci/bb2_project" >&2
+    exit 1
+fi
+
 # Clear secrets from environment
 export BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET=""
 export AWS_SECRET_ACCESS_KEY=""
 
-export JULIA_PROJECT="${BUILDKITE_BUILD_CHECKOUT_PATH}/.ci"
-
 # Add our shared depot cache to the end of JULIA_DEPOT_PATH which is already
 # filled out by `julia-buildkite-plugin` to our agent/pipeline-specific depot path.
 # Make sure to append a colon at the end to allow use of shipped stdlib caches.
@@ -17,17 +27,15 @@ echo "--- Set JULIA_DEPOT_PATH to ${JULIA_DEPOT_PATH}"
 echo "--- Setup Julia packages"
 julia --color=yes -e 'import Pkg; Pkg.instantiate(); Pkg.precompile()'
 
-# Cleanup temporary things that might have been left-over
-echo "--- Cleanup"
-./clean_builds.sh
-./clean_products.sh
-
-echo "+++ Build"
-cd "${PROJECT}"
-
 # Parallel auditor can end up opening loads of files and causing
 # "Too many open files" errors.  Increase the limit.
 ulimit -n 65536
 
-# Start Julia with multiple thread to make auditor parallel.
-julia --threads "${BINARYBUILDER_NPROC:-16}" ./build_tarballs.jl --verbose "${PLATFORM}"
+# Cleanup temporary things that might have been left-over
+echo "--- Cleanup"
+"${YGGDRASIL_BASE}/clean_builds.sh"
+"${YGGDRASIL_BASE}/clean_products.sh"
+
+echo "+++ Build"
+# Start Julia with multiple threads to make auditor parallel.
+julia --threads "${BINARYBUILDER_NPROC:-16}" ./build_tarballs.jl --verbose "$@"

.buildkite/generator.jl

@@ -1,134 +1,181 @@
 #!/bin/env julia
-if VERSION < v"1.8.0"
-    Base.ACTIVE_PROJECT[] = @__DIR__
+import Pkg
+
+Base.set_active_project(@__DIR__)
+using YAML
+
+const PROJECTS = copy(ARGS)
+const DEBUG = !haskey(ENV, "BUILDKITE")
+const IS_PR = get(ENV, "BUILDKITE_PULL_REQUEST", "false") != "false"
+const SKIP_BUILD_COOKIE="[skip build]"
+
+if IS_PR
+    # If we're on a PR though, we look at the entire branch at once
+    BASE_BRANCH = get(ENV, "BUILDKITE_PULL_REQUEST_BASE_BRANCH", "master")
+    @assert !isempty(BASE_BRANCH)
+
+    PR_NUMBER = ENV["BUILDKITE_PULL_REQUEST"]
+    run(ignorestatus(`git fetch origin "refs/pull/$(PR_NUMBER)/head:refs/remotes/origin/pr/$(PR_NUMBER)"`))
+
+    COMMIT_MSG = readchomp(`git show -s --format=%B origin/pr/$(PR_NUMBER)`)
 else
-    Base.set_active_project(@__DIR__)
+    COMMIT_MSG = readchomp(`git show -s --format=%B`)
 end
 
 # Force ourselves to use the shared depot as well, if it exists
 if isdir("/sharedcache/depot")
     push!(Base.DEPOT_PATH, "/sharedcache/depot")
 end
 
-import Pkg
+# Instantiate, to install all necessary packages like YAML (not BinaryBuilder)
 Pkg.instantiate()
 
-using Downloads, Dates, SHA
+include("utils.jl")
 
-const PROJECT = only(ARGS)
-const DEBUG = !haskey(ENV, "BUILDKITE")
+# Give a warning if the pkgserver is too far behind
+check_pkgserver_latency()
 
-include("utils.jl")
+# If there are scary projects we need to exclude, we list them here. (Used to contain `LLVM`)
+EXCLUDED_NAMES = Set{String}([])
 
-if !isnothing(Pkg.pkg_server())
-    resp = try
-        headers = Pkg.PlatformEngines.get_metadata_headers(Pkg.pkg_server())
-        Downloads.request("$(Pkg.pkg_server())/registries"; headers)
-    catch e
-        # Let us know the download of the registry went wrong, but do not hard fail
-        @error "Could not download the registry" exception=(e, catch_backtrace())
-    end
-    last_mod_idx = findfirst(h -> first(h) == "last-modified", resp.headers)
-    msg = "PkgServer: " * resp.url
-    delay = if !isnothing(last_mod_idx)
-        last_mod = last(resp.headers[last_mod_idx])
-        msg *= " -- last updated: " * last_mod
-        # Manually strip the "GMT" timezone and hope it never changes.
-        # Do not error out if parsing fails.
-        dt = tryparse(DateTime, replace(last_mod, " GMT"=>""), dateformat"e, d u y H:M:S")
-        # If parsing did fail, set the delay to 0.
-        isnothing(dt) ? Second(0) : now(UTC) - dt
-    else
-        Second(0)
-    end
-    delay > Second(0) && (msg *= " (" * string(Dates.canonicalize(round(delay, Second))) * " ago)")
-    @info(msg)
-    annotate(msg, style="info", context="pkg")
-    tolerance = Hour(1)
-    if delay > tolerance
-        @warn "The PkgServer registry is older than $(tolerance)"
-        annotate("The PkgServer registry is older than $(tolerance)", style = "warning", context="pkg")
+filter!(PROJECTS) do project
+    if project ∈ EXCLUDED_NAMES
+        @info("Skipping project since it is excluded.", project)
+        return false
     end
+    return true
 end
 
-# If there are scary projects we need to exclude, we list them here. (Used to contain `LLVM`)
-EXCLUDED_NAMES = Set{String}([])
+# Clear out any old `.meta.json` files in the Yggdrasil root
+cleanup_metadata!()
 
-if PROJECT ∈ EXCLUDED_NAMES
-    @info "Skipping project since it is excluded." PROJECT
-    exit()
+# Immediately read in all projects, and ensure that they are either all BB2 projects, or none of them are:
+bb2_projects = String[]
+bb1_projects = String[]
+for project in PROJECTS
+    if uses_bb2(project)
+        push!(bb2_projects, project)
+    else
+        push!(bb1_projects, project)
+    end
 end
 
-# Remove secret from environment
-sanitize(cmd) = addenv(cmd, Dict("BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET" => nothing))
-exec(cmd) = @assert success(pipeline(sanitize(cmd), stderr=stderr, stdout=stdout))
+function bb1_build_steps!(group_steps, project)
+    # determine the name, removing any trailing version number
+    # (`L/LLVM/LLVM@14` results in `NAME = "LLVM"`)
+    project_basename = basename(project)
 
-YGGDRASIL_BASE = dirname(@__DIR__)
-julia(args) = `$(Base.julia_cmd()) --project=$(YGGDRASIL_BASE)/.ci $args`
+    # We always invoke a `build_tarballs.jl` file from its own directory to generate the platform list
+    cd(project) do
+        println("[$(project)] Generating meta.json...")
+        json_path = "$(YGGDRASIL_BASE)/$(project_basename).meta.json"
+        julia(`--compile=min ./build_tarballs.jl --meta-json="$(json_path)"`; julia_project = bb1_julia_project)
 
-# Next, we're going to ensure that our BB is up to date and precompiled
-julia(`-e "import Pkg; Pkg.instantiate(); Pkg.precompile()"`) |> exec
+        # Generate platforms
+        julia(`$(bb1_julia_project)/generate_platforms.jl "$(json_path)" $(YGGDRASIL_BASE)/$(project_basename).platforms.list`)
+    end
 
-TEMP = mktempdir()
+    platforms = split(readchomp(joinpath(YGGDRASIL_BASE, "$(project_basename).platforms.list")))
+    if isempty(platforms)
+        @error("Unable to determine the proper platforms", project_basename)
+        exit(1)
+    end
 
-# determine the name, removing any trailing version number
-# (`L/LLVM/LLVM@14` results in `NAME = "LLVM@14"`)
-const NAME = first(split(basename(PROJECT), "@"))
+    should_skip_builds = contains(COMMIT_MSG, SKIP_BUILD_COOKIE)
+    steps = []
+    for platform in platforms
+        if !should_skip_builds
+            println("[$(project)] $(platform): building")
+            bs = BB1BuildStep(
+                project_basename,
+                project,
+                platform,
+            )
+            push!(steps, render(bs))
+        end
+    end
 
-# We always invoke a `build_tarballs.jl` file from its own directory
-# generate platform list
-cd(PROJECT) do
-    println("Generating meta.json...")
-    JSON_PATH = "$(TEMP)/$(NAME).meta.json"
-    julia(`--compile=min ./build_tarballs.jl --meta-json="$(JSON_PATH)"`) |> exec
+    # If this is not a pull request, we're going to register this project
+    if !IS_PR
+        push!(steps, wait_step())
+        push!(steps, render(BB1RegisterStep(
+            project_basename,
+            project,
+            should_skip_builds,
+            length(platforms),
+        )))
+        #push!(steps, wait_step())
+        #push!(steps, register_step(project_name, project, should_skip_builds, length(platforms)))
+    end
 
-    # Generate platforms
-    julia(`$(YGGDRASIL_BASE)/.ci/generate_platforms.jl "$(JSON_PATH)" $(TEMP)/$(NAME).platforms.list`) |> exec
+    # Group this project's full build under `project_name`
+    if !isempty(steps)
+        push!(group_steps, group_step(project_basename, steps))
+    end
 end
 
-println("Determining builds to queue...")
-
-# Load in the platforms
-PLATFORMS = split(readchomp(joinpath(TEMP, "$(NAME).platforms.list")))
-if isempty(PLATFORMS)
-    @error "Unable to determine the proper platforms" NAME
+function bb2_build_steps!(group_steps, projects)
+    # Invoke `scheduler.jl` to build up our list of jobs with their dependencies
+    io = IOBuffer()
+    julia(`$(bb2_julia_project)/scheduler.jl $(projects)`; julia_project=bb2_julia_project, stdout=io)
+    groups = YAML.load(String(take!(io)))
+
+    # We will create one group per packaging project, and each build will belong to one of those 
+    for (group_name, group) in groups
+        builds = []
+        steps = []
+        for build in group["builds"]
+            bs = BB2BuildStep(
+                build["name"],
+                group_name,
+                build["platform"],
+                build["build_hash"],
+                build["dependencies"],
+            )
+            push!(builds, bs)
+            push!(steps, render(bs))
+        end
+
+        for packaging in group["packagings"]
+            deps = vcat(
+                # Other packages that must be registered
+                packaging["dependencies"],
+                # Our own builds
+                builds,
+            )
+            rs = BB2RegisterStep(packaging["name"], group_name, deps)
+            push!(steps, render(rs))
+        end
+        push!(group_steps, group_step(group_name, steps))
+    end
 end
 
-const IS_PR = get(ENV, "BUILDKITE_PULL_REQUEST", "false") != "false"
-const SKIP_BUILD_COOKIE="[skip build]"
 
-if IS_PR
-    # If we're on a PR though, we look at the entire branch at once
-    BASE_BRANCH = get(ENV, "BUILDKITE_PULL_REQUEST_BASE_BRANCH", "master")
-    @assert !isempty(BASE_BRANCH)
+function build_steps!(group_steps, bb1_projects, bb2_projects)
+    if !isempty(bb1_projects)
+        # Ensure BB1 is instantiated
+        julia(`-e "import Pkg; Pkg.instantiate(); Pkg.precompile()"`; julia_project=bb1_julia_project)
 
-    PR_NUMBER = ENV["BUILDKITE_PULL_REQUEST"]
-    exec(`git fetch origin "refs/pull/$(PR_NUMBER)/head:refs/remotes/origin/pr/$(PR_NUMBER)"`)
+        # Each project is processed individually, with a group of builds for each platform.
+        for project in bb1_projects
+            bb1_build_steps!(group_steps, project)
+        end
+    end
 
-    COMMIT_MSG = readchomp(`git show -s --format=%B origin/pr/$(PR_NUMBER)`)
-else
-    COMMIT_MSG = readchomp(`git show -s --format=%B`)
-end
-# This variable will tell us whether we want to skip the build
-const SKIP_BUILD = contains(COMMIT_MSG, SKIP_BUILD_COOKIE)
+    if !isempty(bb2_projects)
+        # Ensure BB2 is instantiated
+        julia(`-e "import Pkg; Pkg.instantiate(); Pkg.precompile()"`; julia_project=bb2_julia_project)
 
-STEPS = Any[]
-# Create the BUILD_STEPS
-if SKIP_BUILD
-    println("The commit messages contains $(SKIP_BUILD_COOKIE), skipping build")
-else
-    for PLATFORM in PLATFORMS
-        println("    $(PLATFORM): building")
-        push!(STEPS, build_step(NAME, PLATFORM, PROJECT))
+        # For BB2, we jointly process all `build_tarballs.jl` files, building a dependency graph:
+        bb2_build_steps!(group_steps, bb2_projects)
     end
 end
-if !IS_PR
-    push!(STEPS, wait_step())
-    push!(STEPS, register_step(NAME, PROJECT, SKIP_BUILD, length(PLATFORMS)))
-end
-if !isempty(STEPS)
-    definition = Dict(
-        :steps => Any[group_step(NAME, STEPS)]
-    )
-    upload_pipeline(definition)
+
+
+group_steps = []
+build_steps!(group_steps, bb1_projects, bb2_projects)
+
+# Only upload the pipieline if we actually have something to upload
+if !isempty(group_steps)
+    upload_pipeline(Dict(:steps => group_steps))
 end

.buildkite/pipeline.yml

@@ -3,7 +3,7 @@ steps:
     plugins:
       - JuliaCI/julia#v1:
           persist_depot_dirs: packages,artifacts,compiled
-          version: "1.12.4"
+          version: "1.12.5"
           artifacts_size_limit: "214748364800" # 200GB
       - JuliaCI/merge-commit: ~
       - staticfloat/forerunner#v1:
@@ -13,6 +13,9 @@ steps:
           path_processor: .buildkite/path_processors/per-project
           target: .buildkite/generator.jl
           target_type: command
+    artifacts:
+      - "*.meta.json"
+      - "*.platforms.list"
     agents:
       queue: "yggdrasil"
       arch:  "x86_64"