Package Sync

.github/workflows/tests.yml

@@ -4,7 +4,7 @@ on:
     branches:
       - main
     tags:
-      - '*'
+      - "*"
   pull_request:
   schedule:
     # Runs "At 00:01" (see https://crontab.guru)
@@ -22,100 +22,16 @@ defaults:
     shell: bash -l {0}
 
 jobs:
-  software:
-    name: Setup
-    runs-on: ubuntu-latest
-    strategy:
-      fail-fast: false
-      matrix:
-        python-version: ["3.8", "3.9", "3.10"]
-
-    steps:
-      - uses: actions/checkout@v2
-        with:
-          fetch-depth: 0
-
-      - name: Set up environment
-        uses: conda-incubator/setup-miniconda@v2
-        with:
-          miniforge-variant: Mambaforge
-          use-mamba: true
-          condarc-file: ci/condarc
-          python-version: ${{ matrix.python-version }}
-          environment-file: ci/environment.yml
-
-      - name: Check upstream
-        run: |
-          if [[ ${{ github.event_name }} == "pull_request" ]]
-          then
-              export REF="HEAD^2"
-          else
-              export REF="HEAD"
-          fi
-
-          COMMIT="$(git log -n 1 --pretty=format:%s $REF)"
-          if [[ "$COMMIT" == *"test-upstream"* || ${{ github.event_name }} == "schedule" ]]
-          then
-            export TEST_UPSTREAM="true"
-          else
-            export TEST_UPSTREAM="false"
-          fi
-
-          # Put TEST_UPSTREAM into $GITHUB_ENV so it can be used in subsequent workflow steps
-          echo $TEST_UPSTREAM
-          echo TEST_UPSTREAM=$TEST_UPSTREAM >> $GITHUB_ENV
-
-          # Put TEST_UPSTREAM into a file so it can be downloaded in subsequent workflow jobs
-          echo $TEST_UPSTREAM > test_upstream.txt
-
-      - name: Build Coiled Software Environment
-        env:
-          DASK_COILED__TOKEN: ${{ secrets.COILED_BENCHMARK_BOT_TOKEN }}
-        run: |
-          export PYTHON_VERSION_FORMATTED=$(echo "${{ matrix.python-version }}" | sed 's/\.//g' )
-          export REF_NAME_FORMATTED=$(echo "$GITHUB_REF_NAME" | sed 's/\./-/g' )
-          export COILED_SOFTWARE_NAME_HEAD=dask-engineering/coiled-runtime-${{ github.event_name }}
-          export COILED_SOFTWARE_NAME_TAIL=$GITHUB_RUN_ID-py$PYTHON_VERSION_FORMATTED
-
-          if [[ ${{ github.event_name }} = 'pull_request' ]]
-          then
-            export COILED_SOFTWARE_NAME=$COILED_SOFTWARE_NAME_HEAD-${{ github.event.number }}-$COILED_SOFTWARE_NAME_TAIL
-          else
-            export COILED_SOFTWARE_NAME=$COILED_SOFTWARE_NAME_HEAD-$GITHUB_REF_TYPE-$REF_NAME_FORMATTED-$COILED_SOFTWARE_NAME_TAIL
-          fi
-
-          # Create conda environment.yaml file for the latest software environment
-          python ci/create_latest_runtime_meta.py
-          export ENV_FILE=latest.yaml
-          cat $ENV_FILE
-
-          mamba install coiled
-          echo "Creating Coiled software environment for $COILED_SOFTWARE_NAME"
-          coiled env create --name $COILED_SOFTWARE_NAME --conda $ENV_FILE
-
-          # Put COILED_SOFTWARE_NAME into a file so it can be downloaded in subsequent workflow jobs
-          echo $COILED_SOFTWARE_NAME > software_name.txt
-
-      - name: Upload environment file
-        uses: actions/upload-artifact@v3
-        with:
-          name: software-environment-py${{ matrix.python-version }}
-          path: |
-            latest.yaml
-            software_name.txt
-            test_upstream.txt
-
   runtime:
     name: Runtime - ${{ matrix.os }}, Python ${{ matrix.python-version }}, Runtime ${{ matrix.runtime-version }}
-    needs: software
     runs-on: ${{ matrix.os }}
     timeout-minutes: 120
     strategy:
       fail-fast: false
       matrix:
         os: ["ubuntu-latest"]
         python-version: ["3.9"]
-        runtime-version: ["latest", "0.0.3", "0.0.4"]
+        runtime-version: ["upstream", "latest", "0.0.3", "0.0.4"]
 
     steps:
       - uses: actions/checkout@v2
@@ -131,16 +47,12 @@ jobs:
           python-version: ${{ matrix.python-version }}
           environment-file: ci/environment.yml
 
-      - name: Download software environment assets
-        if: matrix.runtime-version == 'latest'
-        uses: actions/download-artifact@v3
-        with:
-          name: software-environment-py${{ matrix.python-version }}
-
       - name: Install coiled-runtime
         env:
           COILED_RUNTIME_VERSION: ${{ matrix.runtime-version }}
-        run: source ci/scripts/install_coiled_runtime.sh
+        run: |
+          python ci/create_runtime_meta.py
+          source ci/scripts/install_coiled_runtime.sh
 
       - name: Run Coiled Runtime Tests
         id: test
@@ -162,15 +74,14 @@ jobs:
 
   benchmarks:
     name: Benchmarks - ${{ matrix.os }}, Python ${{ matrix.python-version }}, Runtime ${{ matrix.runtime-version }}
-    needs: software
     runs-on: ${{ matrix.os }}
     timeout-minutes: 120
     strategy:
       fail-fast: false
       matrix:
         os: ["ubuntu-latest"]
         python-version: ["3.9"]
-        runtime-version: ["latest", "0.0.3", "0.0.4"]
+        runtime-version: ["upstream", "latest", "0.0.3", "0.0.4"]
 
     steps:
       - uses: actions/checkout@v2
@@ -186,16 +97,12 @@ jobs:
           python-version: ${{ matrix.python-version }}
           environment-file: ci/environment.yml
 
-      - name: Download software environment assets
-        if: matrix.runtime-version == 'latest'
-        uses: actions/download-artifact@v3
-        with:
-          name: software-environment-py${{ matrix.python-version }}
-
       - name: Install coiled-runtime
         env:
           COILED_RUNTIME_VERSION: ${{ matrix.runtime-version }}
-        run: source ci/scripts/install_coiled_runtime.sh
+        run: |
+          python ci/create_runtime_meta.py
+          source ci/scripts/install_coiled_runtime.sh
 
       - name: Run benchmarking tests
         id: benchmarking_tests
@@ -217,15 +124,14 @@ jobs:
 
   stability:
     name: Stability - ${{ matrix.os }}, Python ${{ matrix.python-version }}, Runtime ${{ matrix.runtime-version }}
-    needs: software
     runs-on: ${{ matrix.os }}
     timeout-minutes: 120
     strategy:
       fail-fast: false
       matrix:
         os: ["ubuntu-latest"]
         python-version: ["3.8", "3.9", "3.10"]
-        runtime-version: ["latest", "0.0.3", "0.0.4"]
+        runtime-version: ["upstream", "latest", "0.0.3", "0.0.4"]
         include:
           - python-version: "3.9"
             runtime-version: "latest"
@@ -256,16 +162,12 @@ jobs:
           python-version: ${{ matrix.python-version }}
           environment-file: ci/environment.yml
 
-      - name: Download software environment assets
-        if: matrix.runtime-version == 'latest'
-        uses: actions/download-artifact@v3
-        with:
-          name: software-environment-py${{ matrix.python-version }}
-
       - name: Install coiled-runtime
         env:
           COILED_RUNTIME_VERSION: ${{ matrix.runtime-version }}
-        run: source ci/scripts/install_coiled_runtime.sh
+        run: |
+          python ci/create_runtime_meta.py
+          source ci/scripts/install_coiled_runtime.sh
 
       - name: Run stability tests
         id: stability_tests
@@ -292,39 +194,6 @@ jobs:
           name: stability-${{ matrix.os }}-${{ matrix.runtime-version }}-py${{ matrix.python-version }}
           path: stability-${{ matrix.os }}-${{ matrix.runtime-version }}-py${{ matrix.python-version }}.db
 
-  cleanup:
-    needs: [software, runtime, benchmarks, stability]
-    if: always()
-    name: Cleanup
-    runs-on: ubuntu-latest
-    strategy:
-      fail-fast: false
-      matrix:
-        python-version: ["3.8", "3.9", "3.10"]
-    steps:
-      - uses: actions/checkout@v2
-
-      - name: Set up Python
-        uses: actions/setup-python@v3
-        with:
-          python-version: ${{ matrix.python-version }}
-
-      - name: Install coiled
-        run: python -m pip install coiled
-
-      - name: Download software environment assets
-        uses: actions/download-artifact@v3
-        with:
-          name: software-environment-py${{ matrix.python-version }}
-
-      - name: Remove Coiled software environment
-        env:
-          DASK_COILED__TOKEN: ${{ secrets.COILED_BENCHMARK_BOT_TOKEN }}
-        run: |
-          export SOFTWARE_NAME=$(cat software_name.txt)
-          echo "Deleting $SOFTWARE_NAME"
-          coiled env delete $SOFTWARE_NAME
-
   process-results:
     needs: [runtime, benchmarks, stability]
     name: Combine separate benchmark results
@@ -351,7 +220,7 @@ jobs:
         env:
           AWS_ACCESS_KEY_ID: ${{ secrets.RUNTIME_CI_BOT_AWS_ACCESS_KEY_ID }}
           AWS_SECRET_ACCESS_KEY: ${{ secrets.RUNTIME_CI_BOT_AWS_SECRET_ACCESS_KEY }}
-          AWS_DEFAULT_REGION: us-east-2  # this is needed for boto for some reason
+          AWS_DEFAULT_REGION: us-east-2 # this is needed for boto for some reason
           DB_NAME: benchmark.db
         run: |
           aws s3 cp s3://coiled-runtime-ci/benchmarks/$DB_NAME . || true
@@ -366,15 +235,15 @@ jobs:
         env:
           AWS_ACCESS_KEY_ID: ${{ secrets.RUNTIME_CI_BOT_AWS_ACCESS_KEY_ID }}
           AWS_SECRET_ACCESS_KEY: ${{ secrets.RUNTIME_CI_BOT_AWS_SECRET_ACCESS_KEY }}
-          AWS_DEFAULT_REGION: us-east-2  # this is needed for boto for some reason
+          AWS_DEFAULT_REGION: us-east-2 # this is needed for boto for some reason
           DB_NAME: benchmark.db
         run: |
           aws s3 cp $DB_NAME s3://coiled-runtime-ci/benchmarks/
 
       - name: Upload benchmark results as artifact
         uses: actions/upload-artifact@v3
         with:
-          name: benchmark.db
+          name: benchmark
           path: benchmark.db
 
   regressions:
@@ -390,7 +259,7 @@ jobs:
 
       - uses: actions/download-artifact@v3
         with:
-          name: benchmark.db
+          name: benchmark
 
       - name: Set up environment
         uses: conda-incubator/setup-miniconda@v2
@@ -446,7 +315,7 @@ jobs:
   static-site:
     needs: process-results
     # Always generate the site, as this can be skipped even if an indirect dependency fails (like a test run)
-    if: always() && github.ref == 'refs/heads/main' && github.repository == 'coiled/coiled-runtime'
+    if: always()
     name: Build static dashboards
     runs-on: ubuntu-latest
     steps:
@@ -456,7 +325,7 @@ jobs:
 
       - uses: actions/download-artifact@v3
         with:
-          name: benchmark.db
+          name: benchmark
 
       - name: Set up environment
         uses: conda-incubator/setup-miniconda@v2
@@ -470,7 +339,14 @@ jobs:
         run: |
           python dashboard.py
 
+      - name: Upload benchmark results as artifact
+        uses: actions/upload-artifact@v3
+        with:
+          name: static-site
+          path: static/
+
       - name: Deploy 🚀
+        if: github.ref == 'refs/heads/main' && github.repository == 'coiled/coiled-runtime'
         uses: JamesIves/github-pages-deploy-action@4.1.7
         with:
           branch: gh-pages

README.md

@@ -50,10 +50,11 @@ The `coiled-runtime` test suite can be run locally with the following steps:
    the Coiled Dask Engineering AWS S3 account.
 2. Create a Python environment and install development dependencies as
    specified in `ci/environment.yml`.
-3. (Optional) If testing against an unreleased version of `coiled-runtime`,
-   create a Coiled software environment with the unreleased `coiled-runtime` installed
-   and set a local `COILED_SOFTWARE_NAME` environment variable to the name
-   of the software environment (e.g. `export COILED_SOFTWARE_NAME="account/software-name"`)
+3. Install a coiled runtime environment. This might be from one of the environments
+   listed in ``environments/``, or it could be a development environment if you are
+   testing feature branches of dask or distributed. This test suite is configured
+   to run Coiled's ``package_sync`` feature, so your local environment should be copied
+   to the cluster.
 4. Run tests with `python -m pytest tests`
 
 Additionally, tests are automatically run on pull requests to this repository.

ci/create_latest_runtime_meta.py → ci/create_runtime_meta.py

@@ -6,7 +6,6 @@
 import shlex
 import subprocess
 import sys
-from distutils.util import strtobool
 
 import yaml
 from jinja2 import Environment, FileSystemLoader, select_autoescape
@@ -40,8 +39,7 @@ def main():
 
     # Optionally use the development version of `dask` and `distributed`
     # from `dask/label/dev` conda channel
-    upstream = strtobool(os.environ.get("TEST_UPSTREAM", "false"))
-    if upstream:
+    if os.environ.get("COILED_RUNTIME_VERSION", "unknown") == "upstream":
         upstream_packages = {"dask", "distributed"}
         for idx, req in enumerate(requirements):
             package_name = Requirement(req).name
@@ -53,7 +51,7 @@ def main():
         "channels": ["conda-forge"],
         "dependencies": requirements,
     }
-    with open("latest.yaml", "w") as f:
+    with open("runtime.yaml", "w") as f:
         yaml.dump(env, f)
 
 

ci/scripts/combine-dbs.sh

@@ -16,6 +16,11 @@ DB_NAME=benchmark.tmp.db alembic upgrade head
 # Merge in the individual job dbs into our working copy
 for FILE in $(find benchmarks -name "*.db")
 do
+  # Skip the output DB if we see it
+  if [ ${FILE##*/} == $DB_NAME ]; then
+    echo "Skipping $FILE"
+    continue
+  fi
   echo "Processing $FILE"
   # Copy the individual table into the primary one. We make an intermediate
   # temp table so that we can null out the primary keys and reset the

ci/scripts/install_coiled_runtime.sh

@@ -5,14 +5,14 @@ set -o errexit
 set -o nounset
 set -o xtrace
 
-if [[ "$COILED_RUNTIME_VERSION" = 'latest' ]]
+if [[ "$COILED_RUNTIME_VERSION" = 'latest' || "$COILED_RUNTIME_VERSION" = 'upstream' ]]
 then
-  cat latest.yaml
-  mamba env update --file latest.yaml
+  cat runtime.yaml
+  mamba env update --file runtime.yaml
 else
   mamba install -c conda-forge coiled-runtime=$COILED_RUNTIME_VERSION
 fi
 
 # For debugging
 echo -e "--\n--Conda Environment (re-create this with \`conda env create --name <name> -f <output_file>\`)\n--"
-mamba env export | grep -E -v '^prefix:.*$'
+mamba env export | grep -E -v '^prefix:.*$'