diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..4f3a6d3 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,12 @@ +root = true + +[*] +indent_style = space +indent_size = 4 +end_of_line = lf +charset = utf-8 +trim_trailing_whitespace = true +insert_final_newline = true +# options for shfmt(1) +binary_next_line = true +switch_case_indent = true diff --git a/.envrc b/.envrc new file mode 100644 index 0000000..5dae8de --- /dev/null +++ b/.envrc @@ -0,0 +1,4 @@ +watch_file devenv.nix +watch_file devenv.yaml +watch_file devenv.lock +eval "$(devenv print-dev-env)" \ No newline at end of file diff --git a/.github/dependabot.yaml b/.github/dependabot.yaml new file mode 100644 index 0000000..2d9bd29 --- /dev/null +++ b/.github/dependabot.yaml @@ -0,0 +1,9 @@ +# Set update schedule for GitHub Actions + +version: 2 +updates: + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "weekly" + diff --git a/.github/workflows/issue-notifier.yml b/.github/workflows/issue-notifier.yml index e485396..e2ffbb9 100644 --- a/.github/workflows/issue-notifier.yml +++ b/.github/workflows/issue-notifier.yml @@ -8,9 +8,9 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v5 - name: issue-notifier - uses: timheuer/issue-notifier@v1.0.2 + uses: timheuer/issue-notifier@v1.0.4 env: SENDGRID_API_KEY: ${{ secrets.SENDGRID_API_KEY }} with: diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index d741721..1228fe8 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -21,7 +21,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout source code - uses: actions/checkout@v2 + uses: actions/checkout@v5 - name: Install dependencies run: | # Get Nix @@ -32,6 +32,7 @@ jobs: echo "/nix/var/nix/profiles/per-user/runner/profile/bin" >> $GITHUB_PATH echo "/nix/var/nix/profiles/default/bin" >> $GITHUB_PATH + echo "${HOME}/.nix-profile/bin" >> $GITHUB_PATH source "${HOME}/.nix-profile/etc/profile.d/nix.sh" # Install dependencies diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..715bef4 --- /dev/null +++ b/.gitignore @@ -0,0 +1,5 @@ + +# Devenv +.devenv* +devenv.local.nix + diff --git a/README.adoc b/README.adoc index f9aa1d8..7925a9b 100644 --- a/README.adoc +++ b/README.adoc @@ -1,16 +1,16 @@ -= niv-updater: Automated dependency updates with niv += niv-updater: Automated dependency updates with niv -image:https://github.com/knl/niv-updater-action/workflows/CI/badge.svg[CI] +image:https://github.com/knl/niv-updater-action/actions/workflows/main.yml/badge.svg[CI] image:https://img.shields.io/github/v/release/knl/niv-updater-action[GitHub release (latest by date)] -This action will open a pull request to master branch (or otherwise specified +This action will open a pull request to `master` branch (or otherwise specified branch) whenever https://github.com/nmattia/niv[niv] detects updates to `nix/sources.json` in your repository, for each dependency separately. Each PR will contain a beautiful Changelog of all the changes in the update, like this: image:./assets/niv-update-action-changelog.png[title="Changelog generated by niv-updater-action] -The best way to use niv-updater-action is to set up a scheduled workflow. This +The best way to use `niv-updater-action` is to set up a scheduled workflow. This way, whenever there are new updates, you will get a PR that you can just approve and avoid a lot of manual work. @@ -34,7 +34,7 @@ jobs: steps: # notice there is no checkout step - name: niv-updater-action - uses: knl/niv-updater-action@v9 + uses: knl/niv-updater-action@v15 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} ---- @@ -53,13 +53,14 @@ jobs: This value will be passed to niv via `--sources-file` option. Defaults to `nix/sources.json`. * `niv_version`: (Optional) The niv version to be used. Defaults to `master`, - meaning niv-updater-action will take the latest niv for each run. You may want - to fix a particular version and avoid future breaks to your workflow. + meaning `niv-updater-action` will take the latest niv for each run. You may want + to fix a particular version and avoid future breaks to your workflow. If you're + using a self-hosted runner, set this to `*from-nixpkgs*`. * `branch_prefix`: (Optional) The prefix used for update branches, created by this action. The action does not sanitize the branch name. For a description of what a valid branch name is, please consult: https://mirrors.edge.kernel.org/pub/software/scm/git/docs/git-check-ref-format.html. - Defaults to "update/". + Defaults to `update/`. * `keep_updating`: (Optional) If PR already exists, keep it updated with new changes. The branch will be force updated, as this process keeps a single commit on a branch. Defaults to `false` to maintain the old behaviour. @@ -101,12 +102,12 @@ jobs: https://github.com/knl/niv-updater-action/issues/26[Issue #26]. Defaults to `true`. * `debug_output`: (Optional, a boolean) If `true`, `set -x` will be turned on - for the updater script, outputing every step the action takes. This will show + for the updater script, outputting every step the action takes. This will show up in the action log, and could be useful for trying to reproduce issues locally. Defaults to `false`. As the above list suggests, `niv-updater-action` is highly configurable. -The following example exposes some of the nobs, many with their default values: +The following example exposes some of the knobs, many with their default values: [source,yaml] ---- @@ -123,7 +124,7 @@ jobs: steps: # notice there is no checkout step - name: niv-updater-action - uses: knl/niv-updater-action@v9 + uses: knl/niv-updater-action@v15 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: @@ -165,3 +166,37 @@ Actions workflow YAML file. * `GITHUB_TOKEN` - (Required) The GitHub API token used to create pull requests and get content from all repositories tracked by `niv`. + +== Self hosted runner + +Self-hosted runners are running with dynamic users so nix profile is not +accessible, as well as nix-env. As this action relies on nix-env to install +niv, the default configuration will not work. Thus, to use niv from available +nixpkgs, set `niv_version` to `pass:[*from-nixpkgs*]`. It will install `niv` +using `nixpkgs` with nix-shell instead of nix-env. + +To avoid using `sudo` (also unavailable on self-hosted runners), the input +`pass:[skip_ssh_repos]` should be set to `true`. + +Example: + +[source,yaml] +---- +name: Automated niv-managed dependency updates +on: + schedule: + # * is a special character in YAML so you have to quote this string + # run this every day at 4:00am + - cron: '0 4 * * *' +jobs: + niv-updater: + name: 'Create PRs for niv-managed dependencies' + runs-on: self-hosted + steps: + # notice there is no checkout step + - name: niv-updater-action + uses: knl/niv-updater-action@v15 + with: + niv_version: '*from-nixpkgs*' + skip_ssh_repos: true +---- diff --git a/action.yml b/action.yml index cd05a7b..452cc55 100644 --- a/action.yml +++ b/action.yml @@ -2,7 +2,7 @@ name: 'niv Updater Action' description: 'A GitHub action that detects updates to dependencies tracked by niv and creates pull requests to keep them up to date.' author: 'knl' runs: - using: 'node12' + using: 'node24' main: 'main.js' inputs: pull_request_base: diff --git a/devenv.lock b/devenv.lock new file mode 100644 index 0000000..ff1babe --- /dev/null +++ b/devenv.lock @@ -0,0 +1,103 @@ +{ + "nodes": { + "devenv": { + "locked": { + "dir": "src/modules", + "lastModified": 1757928168, + "owner": "cachix", + "repo": "devenv", + "rev": "d9019631e0e965b78f94a63246863fa7d8315d17", + "type": "github" + }, + "original": { + "dir": "src/modules", + "owner": "cachix", + "repo": "devenv", + "type": "github" + } + }, + "flake-compat": { + "flake": false, + "locked": { + "lastModified": 1747046372, + "owner": "edolstra", + "repo": "flake-compat", + "rev": "9100a0f413b0c601e0533d1d94ffd501ce2e7885", + "type": "github" + }, + "original": { + "owner": "edolstra", + "repo": "flake-compat", + "type": "github" + } + }, + "git-hooks": { + "inputs": { + "flake-compat": "flake-compat", + "gitignore": "gitignore", + "nixpkgs": [ + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1757974173, + "owner": "cachix", + "repo": "git-hooks.nix", + "rev": "302af509428169db34f268324162712d10559f74", + "type": "github" + }, + "original": { + "owner": "cachix", + "repo": "git-hooks.nix", + "type": "github" + } + }, + "gitignore": { + "inputs": { + "nixpkgs": [ + "git-hooks", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1709087332, + "owner": "hercules-ci", + "repo": "gitignore.nix", + "rev": "637db329424fd7e46cf4185293b9cc8c88c95394", + "type": "github" + }, + "original": { + "owner": "hercules-ci", + "repo": "gitignore.nix", + "type": "github" + } + }, + "nixpkgs": { + "locked": { + "lastModified": 1757967192, + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "0d7c15863b251a7a50265e57c1dca1a7add2e291", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixpkgs-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, + "root": { + "inputs": { + "devenv": "devenv", + "git-hooks": "git-hooks", + "nixpkgs": "nixpkgs", + "pre-commit-hooks": [ + "git-hooks" + ] + } + } + }, + "root": "root", + "version": 7 +} diff --git a/devenv.nix b/devenv.nix new file mode 100644 index 0000000..cbd57fa --- /dev/null +++ b/devenv.nix @@ -0,0 +1,27 @@ +{ pkgs, ... }: + +{ + # https://devenv.sh/basics/ + # env.GREET = "devenv"; + + # https://devenv.sh/packages/ + packages = [ pkgs.asciidoctor ]; + + # enterShell = '' + # ''; + + # https://devenv.sh/languages/ + # languages.nix.enable = true; + + # https://devenv.sh/scripts/ + # scripts.hello.exec = "echo hello from $GREET"; + + # https://devenv.sh/pre-commit-hooks/ + pre-commit.hooks.shellcheck.enable = true; + pre-commit.hooks.actionlint.enable = true; + pre-commit.hooks.shfmt.enable = true; + # pre-commit.hooks.typos.enable = true; + + # https://devenv.sh/processes/ + # processes.ping.exec = "ping example.com"; +} diff --git a/devenv.yaml b/devenv.yaml new file mode 100644 index 0000000..8de0039 --- /dev/null +++ b/devenv.yaml @@ -0,0 +1,4 @@ +inputs: + nixpkgs: + url: github:NixOS/nixpkgs/nixpkgs-unstable +# can't point to the local modules here as it's used as a template \ No newline at end of file diff --git a/niv-updater b/niv-updater index ba716a3..0ebc8ff 100755 --- a/niv-updater +++ b/niv-updater @@ -25,11 +25,68 @@ setupPrerequisites() { # Check if some other step already installed Nix if [[ ! -d /nix/store ]] || ! nix --version >/dev/null 2>&1; then - sudo mkdir -p /etc/nix + # Create a temporary workdir + workdir=$(mktemp -d) + trap 'rm -rf "$workdir"' EXIT + + # Configure Nix + add_config() { + echo "$1" >>"$workdir/nix.conf" + } + # Workaround segfault: https://github.com/NixOS/nix/issues/2733 - sudo sh -c 'echo "http2 = false" >> /etc/nix/nix.conf' + add_config "http2 = false" - sh <(curl -sSL https://nixos.org/nix/install) --no-daemon + add_config "max-jobs = auto" + if [[ $OSTYPE =~ darwin ]]; then + add_config "ssl-cert-file = /etc/ssl/cert.pem" + fi + # Allow binary caches for user + add_config "trusted-users = root ${USER:-}" + add_config "experimental-features = nix-command flakes" + # Add a GitHub access token. + # Token-less access is subject to lower rate limits. + # Use the default GitHub token if available. + # Skip this step if running an Enterprise instance. The default token there does not work for github.com. + if [[ -n ${GITHUB_TOKEN:-} && $GITHUB_SERVER_URL == "https://github.com" ]]; then + echo "::debug::Using the default GITHUB_TOKEN for github.com" + add_config "access-tokens = github.com=$GITHUB_TOKEN" + else + echo "::debug::Continuing without a GitHub access token" + fi + # Nix installer flags + installer_options=( + --no-channel-add + --darwin-use-unencrypted-nix-store-volume + --nix-extra-conf-file "$workdir/nix.conf" + --no-daemon + ) + + # "fix" the following error when running nix* + # error: the group 'nixbld' specified in 'build-users-group' does not exist + add_config "build-users-group =" + sudo mkdir -p /etc/nix + sudo chmod 0755 /etc/nix + sudo cp "$workdir/nix.conf" /etc/nix/nix.conf + + # There is --retry-on-errors, but only newer curl versions support that + curl_retries=5 + # pin to ensure future upgrades do not interfere with the subsequent steps + while ! curl -sS -o "$workdir/install" -v --fail -L "https://releases.nixos.org/nix/nix-2.24.9/install"; do + sleep 1 + ((curl_retries--)) + if [[ $curl_retries -le 0 ]]; then + echo "curl retries failed" >&2 + exit 1 + fi + done + + sh "$workdir/install" "${installer_options[@]}" + + # Set paths + echo "/nix/var/nix/profiles/default/bin" >>"$GITHUB_PATH" + # new path for nix 2.14 + echo "$HOME/.nix-profile/bin" >>"$GITHUB_PATH" fi echo 'Installing Nix - done' @@ -37,16 +94,30 @@ setupPrerequisites() { export PATH="${PATH}:/nix/var/nix/profiles/per-user/runner/profile/bin:/nix/var/nix/profiles/default/bin" - # shellcheck disable=SC1090 - source "${HOME}/.nix-profile/etc/profile.d/nix.sh" + # shellcheck disable=SC1091 + test -f "${HOME}/.nix-profile/etc/profile.d/nix.sh" && source "${HOME}/.nix-profile/etc/profile.d/nix.sh" - # we also need hub, and git, but they both come with ubuntu-latest with GitHub Actions - # https://github.com/actions/virtual-environments/blob/master/images/linux/Ubuntu1804-README.md - nix-env -iA niv -f "https://github.com/nmattia/niv/tarball/$INPUT_NIV_VERSION" \ - --substituters https://niv.cachix.org \ - --trusted-public-keys niv.cachix.org-1:X32PCg2e/zAm3/uD1ScqW2z/K0LtDyNV7RdaxIuLgQM= + PACKAGES=(jq moreutils curl) + if [[ $INPUT_NIV_VERSION == "*from-nixpkgs*" ]]; then + PACKAGES+=(niv) + else + # we also need hub, and git, but they both come with ubuntu-latest with GitHub Actions + # https://github.com/actions/virtual-environments/blob/master/images/linux/Ubuntu1804-README.md + # NOTE: We add 'cache.nixos.org' as a substituter because niv uses it as an upstream cache: + # https://blog.cachix.org/posts/2020-07-28-upstream-caches-avoiding-pushing-paths-in-cache-nixos-org/ + nix-env -iA niv -f "https://github.com/nmattia/niv/tarball/$INPUT_NIV_VERSION" \ + --substituters 'https://niv.cachix.org https://cache.nixos.org' \ + --trusted-public-keys 'niv.cachix.org-1:X32PCg2e/zAm3/uD1ScqW2z/K0LtDyNV7RdaxIuLgQM= cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY=' + fi # We need --rawfile for jq, available in jq 1.6, which is only available on ubuntu 20.04 and macos - nix-env -iA nixpkgs.jq + # We need moreutils because of the sponge utility (to simplify the code) + + # Install if hub dependency is missing on self hosted runner + [[ $(type -P "hub") ]] || PACKAGES+=(hub) + + # shellcheck disable=SC2016 + PATH="${PATH}:$(nix-shell -I "nixpkgs=https://github.com/NixOS/nixpkgs/archive/refs/heads/nixos-24.05.tar.gz" -p "${PACKAGES[@]}" --command 'echo $PATH')" + echo 'Installing dependencies - done' } @@ -126,17 +197,16 @@ formatIssueLinksPlain() { } # Like formatIssuesLinkPlain, this function turns any #123 a direct reference to the original repository. -# Unlike formatIssuesLink, it will use a redirection service (DuckDuckGo), so that referenced PRs/Issues +# Unlike formatIssuesLink, it will use a redirection service (togithub.com), so that referenced PRs/Issues # do not contain back references. # For more details, see https://github.com/knl/niv-updater-action/issues/26 -# NOTE: It has to be http://r.duckduckgo.com, https variant will not redirect formatIssueLinksNoBackreferences() { dep_owner="$1" dep_repo="$2" # The sequence e2 81 a0 is \u2060 - UNICODE WORD JOINER # We need it in order prevent GitHub from making a backreference due to # the commit message text. - sed "s~#\([0-9]\+\)~[$dep_owner/$dep_repo\xe2\x81\xa0#\1](http://r.duckduckgo.com/l/?uddg=https://github.com/$dep_owner/$dep_repo/issues/\1)~g" + sed "s~#\([0-9]\+\)~[$dep_owner/$dep_repo\xe2\x81\xa0#\1](https://togithub.com/$dep_owner/$dep_repo/issues/\1)~g" } # This function formats mentions in the generated changelog in such a way that @@ -160,7 +230,8 @@ formatIssueLinks() { createPullRequestsOnUpdate() { echo 'Checking for updates' if [[ -z $INPUT_PULL_REQUEST_BASE ]]; then - INPUT_PULL_REQUEST_BASE="$GITHUB_REF" + INPUT_PULL_REQUEST_BASE="${GITHUB_REF#refs/heads/}" + INPUT_PULL_REQUEST_BASE="${INPUT_PULL_REQUEST_BASE#refs/tags/}" base="$GITHUB_SHA" else # get the SHA of the current base, so that it remains fixed during the run @@ -338,23 +409,37 @@ createPullRequestsOnUpdate() { printf 'Branch: %s\n' "$niv_branch" printf 'Commits: [%s/%s@%.8s...%.8s](https://github.com/%s/%s/compare/%s...%s)\n\n' "$dep_owner" "$dep_repo" "$revision" "$new_revision" "$dep_owner" "$dep_repo" "$revision" "$new_revision" { - hub api "/repos/$dep_owner/$dep_repo/compare/${revision}...${new_revision}" || true + # In order for this to work, one has to use both paginate and per_page + hub api --paginate "/repos/$dep_owner/$dep_repo/compare/${revision}...${new_revision}?per_page=100" || true } | jq -r '.commits[] '"$merges_filter"' | "* [`\(.sha[0:8])`](\(.html_url)) \(.commit.message | split("\n") | first)"' \ | formatIssueLinks "$dep_owner" "$dep_repo" \ | formatMentions } >>"$message" fi - # print with a new line appended, as yaml swallows those - printf '%s%s' "$INPUT_MESSAGE_SUFFIX" "${INPUT_MESSAGE_SUFFIX:+$'\n'}" >>"$message" + message_size=$(wc -c <"$message") + truncation_string="* ... _the rest of the list is truncated due to the maximum length of the PR message on GitHub. Please take a look at the commit message._" + max_message_size=$((65536 - ${#INPUT_MESSAGE_SUFFIX} - 1 - ${#truncation_string} - 1)) + # This is now split in two different, interleaved actions: + # - generating the commit message of any size + # - generating the PR description with max size of 64k commit_message=$(mktemp) { cat "$title" printf '\n\n' cat "$message" + # print with a new line appended, as yaml swallows those + printf '%s%s' "$INPUT_MESSAGE_SUFFIX" "${INPUT_MESSAGE_SUFFIX:+$'\n'}" } >"$commit_message" + if ((message_size > max_message_size)); then + head <"$message" -c $max_message_size | sed '$d' | sponge "$message" + printf "%s\n" "$truncation_string" >>"$message" + fi + # print with a new line appended, as yaml swallows those + printf '%s%s' "$INPUT_MESSAGE_SUFFIX" "${INPUT_MESSAGE_SUFFIX:+$'\n'}" >>"$message" + new_content=$(mktemp) base64 "$wdir/$SOURCES_JSON" >>"$new_content" @@ -362,11 +447,12 @@ createPullRequestsOnUpdate() { if [[ $branch_exists == 'no' ]]; then # create the branch echo "Creating branch '$branch_name' for the update of '$dep'" + hub_api_out=$(mktemp) if ! hub api \ -F ref="refs/heads/$branch_name" \ -F sha="$base" \ - "/repos/$GITHUB_REPOSITORY/git/refs" >/dev/null; then - error "could not create branch $branch_name" + "/repos/$GITHUB_REPOSITORY/git/refs" >"$hub_api_out"; then + error "could not create branch $branch_name for base $base, due to $(cat "$hub_api_out")" fi # upload the content @@ -537,10 +623,11 @@ createPullRequestsOnUpdate() { if [[ $INPUT_DEBUG_OUTPUT == 'true' ]]; then set -x + export HUB_VERBOSE=true fi -checkCredentials setupPrerequisites +checkCredentials setupNetrc sanitizeInputs createPullRequestsOnUpdate