ci: publish edge from main and reserve latest for stable releases

.github/scripts/get_release_version.py

@@ -26,7 +26,9 @@
 # '1.0.0-rc1' (the full version): for a tagged prerelease
 # '1.0' (major.minor): for a tagged release
 
-# We set the environment variable UPDATE_RELEASE if it's a full release (tagged and not prerelease)
+# LATEST_STABLE_CHANNEL is the first non-prerelease channel in versions.yaml.
+# We set UPDATE_RELEASE for any full release and UPDATE_LATEST when that release
+# belongs to LATEST_STABLE_CHANNEL.
 
 # We set the environment variable CHART_VERSION based on the kind of build. This is used for
 # versioning our helm chart ONLY.
@@ -51,19 +53,97 @@
 import os
 import re
 import sys
+from pathlib import Path
+
+
+def get_supported_releases(versions_file):
+    in_supported = False
+    channel_pattern = re.compile(
+        r"^\s*-\s+channel:\s*(['\"]?)(?P<channel>\d+\.\d+)\1\s*(?:#.*)?$"
+    )
+    version_pattern = re.compile(
+        r"^\s+version:\s*(['\"]?)(?P<version>v[^'\"\s]+)\1\s*(?:#.*)?$"
+    )
+    candidateChannel = None
+    releases = []
+
+    with open(versions_file, encoding="utf-8") as versions:
+        for line in versions:
+            if line.strip() == "supported:":
+                in_supported = True
+                continue
+
+            if in_supported and line and not line[0].isspace():
+                break
+
+            if in_supported:
+                channelMatch = channel_pattern.match(line)
+                if channelMatch is not None:
+                    candidateChannel = channelMatch.group("channel")
+                    continue
+
+                versionMatch = version_pattern.match(line)
+                if versionMatch is not None and candidateChannel is not None:
+                    releases.append(
+                        (
+                            candidateChannel,
+                            versionMatch.group("version").removeprefix("v"),
+                        )
+                    )
+                    candidateChannel = None
+
+    if not releases:
+        raise ValueError(
+            "Could not find supported releases in {}".format(versions_file)
+        )
+    return releases
+
 
 gitRef = os.getenv("GITHUB_REF")
+githubEnvPath = os.getenv("GITHUB_ENV")
+if githubEnvPath is None:
+    raise ValueError("GITHUB_ENV must be set")
 
 # From https://semver.org/#is-there-a-suggested-regular-expression-regex-to-check-a-semver-string
 # Group 'version' returns the whole version
 # other named groups return the components
 tagRefRegex = r"^refs/tags/v(?P<version>0|(?P<major>0|[1-9]\d*)\.(?P<minor>0|[1-9]\d*)\.(?P<patch>0|[1-9]\d*)(?:-(?P<prerelease>(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+(?P<buildmetadata>[0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?)$"
 pullRefRegex = r"^refs/pull/(.*)/(.*)$"
 
-with open(os.getenv("GITHUB_ENV"), "a") as githubEnv:
+with open(githubEnvPath, "a", encoding="utf-8") as githubEnv:
+    versionsFile = os.getenv(
+        "VERSIONS_FILE", Path(__file__).resolve().parents[2] / "versions.yaml"
+    )
+    supportedReleases = get_supported_releases(versionsFile)
+    latestStableRelease = next(
+        (
+            (releaseChannel, releaseVersion.partition("+")[0])
+            for releaseChannel, releaseVersion in supportedReleases
+            if "-" not in releaseVersion.partition("+")[0]
+        ),
+        None,
+    )
+    if latestStableRelease is None:
+        raise ValueError(
+            "Could not find the latest stable release in {}".format(versionsFile)
+        )
+    latestChannel, latestVersion = latestStableRelease
+    latestStableChannel = "LATEST_STABLE_CHANNEL={}".format(latestChannel)
+    print("Setting: {}".format(latestStableChannel))
+    githubEnv.write(latestStableChannel + "\n")
+    latestStableVersion = "LATEST_STABLE_VERSION={}".format(latestVersion)
+    print("Setting: {}".format(latestStableVersion))
+    githubEnv.write(latestStableVersion + "\n")
+
+    print("Setting: UPDATE_LATEST=false")
+    githubEnv.write("UPDATE_LATEST=false" + "\n")
+    print("Setting: UPDATE_CHANNEL=false")
+    githubEnv.write("UPDATE_CHANNEL=false" + "\n")
+
     if gitRef is None:
         print(
-            "This is not running in github, GITHUB_REF is null. Assuming a local build...")
+            "This is not running in github, GITHUB_REF is null. Assuming a local build..."
+        )
 
         version = "REL_VERSION=edge"
         print("Setting: {}".format(version))
@@ -113,12 +193,23 @@
             githubEnv.write(chart + "\n")
 
             channel = "REL_CHANNEL={}.{}".format(
-                match.group("major"), match.group("minor"))
+                match.group("major"), match.group("minor")
+            )
             print("Setting: {}".format(channel))
             githubEnv.write(channel + "\n")
 
             print("Setting: UPDATE_RELEASE=true")
             githubEnv.write("UPDATE_RELEASE=true" + "\n")
+
+            releaseChannel = channel.removeprefix("REL_CHANNEL=")
+            supportedVersion = dict(supportedReleases).get(releaseChannel)
+            if match.group("version") == supportedVersion:
+                print("Setting: UPDATE_CHANNEL=true")
+                githubEnv.write("UPDATE_CHANNEL=true" + "\n")
+
+            if match.group("version") == latestVersion:
+                print("Setting: UPDATE_LATEST=true")
+                githubEnv.write("UPDATE_LATEST=true" + "\n")
             sys.exit(0)
 
         else:

.github/scripts/test_get_release_version.py

@@ -0,0 +1,119 @@
+import os
+import subprocess
+import sys
+import tempfile
+import unittest
+from pathlib import Path
+
+
+SCRIPT_PATH = Path(__file__).with_name("get_release_version.py")
+
+
+class GetReleaseVersionTests(unittest.TestCase):
+    def run_script(self, git_ref, versions=None):
+        with tempfile.TemporaryDirectory() as temp_dir:
+            temp_path = Path(temp_dir)
+            github_env = temp_path / "github-env"
+            versions_file = temp_path / "versions.yaml"
+            versions_file.write_text(
+                versions
+                or (
+                    "supported:\n"
+                    "  - channel: '0.60'\n"
+                    "    version: 'v0.60.0-rc1'\n"
+                    "  - channel: '0.59'\n"
+                    "    version: 'v0.59.0'\n"
+                    "  - channel: '0.58'\n"
+                    "    version: 'v0.58.0'\n"
+                ),
+                encoding="utf-8",
+            )
+
+            environment = os.environ.copy()
+            environment.update(
+                {
+                    "GITHUB_ENV": str(github_env),
+                    "GITHUB_REF": git_ref,
+                    "VERSIONS_FILE": str(versions_file),
+                }
+            )
+            subprocess.run(
+                [sys.executable, str(SCRIPT_PATH)],
+                check=True,
+                env=environment,
+                capture_output=True,
+                text=True,
+            )
+
+            return dict(
+                line.split("=", 1)
+                for line in github_env.read_text(encoding="utf-8").splitlines()
+            )
+
+    def test_latest_stable_release_updates_latest(self):
+        values = self.run_script("refs/tags/v0.59.0")
+
+        self.assertEqual("0.59", values["REL_CHANNEL"])
+        self.assertEqual("true", values["UPDATE_RELEASE"])
+        self.assertEqual("true", values["UPDATE_CHANNEL"])
+        self.assertEqual("true", values["UPDATE_LATEST"])
+
+    def test_stale_patch_in_latest_channel_does_not_update_latest(self):
+        values = self.run_script(
+            "refs/tags/v0.59.1",
+            "supported:\n"
+            "  - channel: '0.60'\n"
+            "    version: 'v0.60.0-rc1'\n"
+            "  - channel: '0.59'\n"
+            "    version: 'v0.59.2'\n",
+        )
+
+        self.assertEqual("0.59", values["REL_CHANNEL"])
+        self.assertEqual("0.59.2", values["LATEST_STABLE_VERSION"])
+        self.assertEqual("false", values["UPDATE_CHANNEL"])
+        self.assertEqual("false", values["UPDATE_LATEST"])
+
+    def test_older_channel_full_release_does_not_update_latest(self):
+        values = self.run_script("refs/tags/v0.58.0")
+
+        self.assertEqual("0.58", values["REL_CHANNEL"])
+        self.assertEqual("true", values["UPDATE_RELEASE"])
+        self.assertEqual("true", values["UPDATE_CHANNEL"])
+        self.assertEqual("false", values["UPDATE_LATEST"])
+
+    def test_newly_finalized_channel_updates_latest(self):
+        values = self.run_script(
+            "refs/tags/v0.60.0",
+            "supported:\n"
+            "  - channel: '0.60'\n"
+            "    version: 'v0.60.0'\n"
+            "  - channel: '0.59'\n"
+            "    version: 'v0.59.0'\n",
+        )
+
+        self.assertEqual("0.60", values["LATEST_STABLE_CHANNEL"])
+        self.assertEqual("true", values["UPDATE_CHANNEL"])
+        self.assertEqual("true", values["UPDATE_LATEST"])
+
+    def test_prerelease_does_not_update_latest(self):
+        values = self.run_script("refs/tags/v0.60.0-rc1")
+
+        self.assertEqual("0.60.0-rc1", values["REL_CHANNEL"])
+        self.assertNotIn("UPDATE_RELEASE", values)
+        self.assertEqual("false", values["UPDATE_CHANNEL"])
+        self.assertEqual("false", values["UPDATE_LATEST"])
+
+    def test_main_build_uses_edge_channel(self):
+        values = self.run_script("refs/heads/main")
+
+        self.assertEqual("edge", values["REL_VERSION"])
+        self.assertEqual("edge", values["REL_CHANNEL"])
+        self.assertEqual("0.59", values["LATEST_STABLE_CHANNEL"])
+        self.assertEqual("0.59.0", values["LATEST_STABLE_VERSION"])
+        self.assertNotIn("UPDATE_RELEASE", values)
+        self.assertEqual("false", values["UPDATE_CHANNEL"])
+        self.assertEqual("false", values["UPDATE_LATEST"])
+
+
+if __name__ == "__main__":
+    unittest.main()

.github/workflows/build.yaml

@@ -41,9 +41,9 @@ on:
 permissions: {}
 
 concurrency:
-  # Cancel the previously triggered build for only PR build.
-  group: build-${{ github.ref }}-${{ github.event.pull_request.number || github.sha }}
-  cancel-in-progress: true
+  # Cancel stale PR and main builds. Release tags use independent groups so no release is discarded.
+  group: build-${{ github.ref }}-${{ github.event.pull_request.number || (github.ref == 'refs/heads/main' && 'main') || github.sha }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' || github.ref == 'refs/heads/main' }}
 
 env:
   # GitHub Actor for pushing images to GHCR
@@ -217,6 +217,19 @@ jobs:
         with:
           python-version-file: .python-version
 
+      - name: Test release version parsing
+        run: python ./.github/scripts/test_get_release_version.py
+
+      - name: Test container image promotion
+        run: bash ./build/scripts/test-promote-container-image.sh
+
+      - name: Read release metadata from main
+        if: github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/v')
+        run: |
+          git fetch --no-tags --depth=1 origin main
+          git show FETCH_HEAD:versions.yaml > "${RUNNER_TEMP}/versions-main.yaml"
+          echo "VERSIONS_FILE=${RUNNER_TEMP}/versions-main.yaml" >> "${GITHUB_ENV}"
+
       - name: Parse release version and set environment variables
         run: python ./.github/scripts/get_release_version.py
 
@@ -267,32 +280,57 @@ jobs:
           path: ./dist/images/
           retention-days: 1
 
-      - name: Push container images (latest)
+      - name: Push container images (edge)
         run: |
           make docker-multi-arch-push
-        if: (github.ref == 'refs/heads/main') # push image to latest on merge to main
+        if: (github.ref == 'refs/heads/main') # push image to edge on merge to main
         env:
           DOCKER_REGISTRY: ${{ env.CONTAINER_REGISTRY }}
-          DOCKER_TAG_VERSION: latest
+          DOCKER_TAG_VERSION: edge
           DOCKER_CACHE_GHA: 1
 
-      - name: Push container images (release)
+      - name: Push container images (release version)
         run: |
           make docker-multi-arch-push
-        if: startsWith(github.ref, 'refs/tags/v') # push image on tag
+        if: startsWith(github.ref, 'refs/tags/v') # publish the immutable full version before updating mutable aliases
         env:
           DOCKER_REGISTRY: ${{ env.CONTAINER_REGISTRY }}
-          DOCKER_TAG_VERSION: ${{ env.REL_CHANNEL }}
+          DOCKER_TAG_VERSION: ${{ env.REL_VERSION }}
           DOCKER_CACHE_GHA: 1
 
+      - name: Refresh release metadata from main
+        if: startsWith(github.ref, 'refs/tags/v')
+        run: |
+          git fetch --no-tags --depth=1 origin main
+          git show FETCH_HEAD:versions.yaml > "${VERSIONS_FILE}"
+          python ./.github/scripts/get_release_version.py
+
+      - name: Promote container images to the release channel
+        run: |
+          make docker-promote-tag
+        if: startsWith(github.ref, 'refs/tags/v') && env.UPDATE_RELEASE == 'true' && env.UPDATE_CHANNEL == 'true'
+        env:
+          DOCKER_REGISTRY: ${{ env.CONTAINER_REGISTRY }}
+          DOCKER_SOURCE_TAG_VERSION: ${{ env.REL_VERSION }}
+          DOCKER_TAG_VERSION: ${{ env.REL_CHANNEL }}
+
+      - name: Ensure latest points to the stable release
+        run: |
+          make docker-promote-tag
+        if: github.ref == 'refs/heads/main' || (startsWith(github.ref, 'refs/tags/v') && env.UPDATE_LATEST == 'true')
+        env:
+          DOCKER_REGISTRY: ${{ env.CONTAINER_REGISTRY }}
+          DOCKER_SOURCE_TAG_VERSION: ${{ github.ref == 'refs/heads/main' && env.LATEST_STABLE_CHANNEL || env.REL_VERSION }}
+          DOCKER_TAG_VERSION: latest
+
       - name: Collect build metrics
         if: always()
         run: |
           make build-metrics
           cat dist/metrics/metrics.txt >> $GITHUB_STEP_SUMMARY
         env:
           DOCKER_REGISTRY: ${{ github.event_name == 'pull_request' && format('{0}/dev', env.CONTAINER_REGISTRY) || env.CONTAINER_REGISTRY }}
-          DOCKER_TAG_VERSION: ${{ github.event_name == 'pull_request' && env.REL_VERSION || (github.ref == 'refs/heads/main' && 'latest' || env.REL_CHANNEL) }}
+          DOCKER_TAG_VERSION: ${{ github.ref == 'refs/heads/main' && 'edge' || env.REL_VERSION }}
 
       - name: Upload build metrics
         if: always()