diff --git a/.github/generate-test-groups.groovy b/.github/generate-test-groups.groovy deleted file mode 100644 index e4a7e7ec..00000000 --- a/.github/generate-test-groups.groovy +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -import groovy.json.JsonOutput; - -final int MAX_GROUPS = 10 -final List> GROUPS = new ArrayList<>() -int groupId = 0 - -// Distribute example projects across a bounded set of test groups and output as JSON -new File(".").eachFileRecurse { file -> - if (file.getName() == "pom.xml" && file.getParentFile().getParentFile()?.getName() == ".") { - if (file.getParentFile().getName() == "saga") { - // saga is put into a dedicated group as it's a multi-module build - return - } - - if (GROUPS[groupId] == null) { - GROUPS[groupId] = [:] - GROUPS[groupId].name = "group-${String.format("%02d", groupId + 1)}" - GROUPS[groupId].tests = "" - } - - String separator = GROUPS[groupId].tests == "" ? "" : "," - - GROUPS[groupId].tests = "${GROUPS[groupId].tests}${separator}${file.parentFile.name}" - - groupId += 1; - if (groupId == MAX_GROUPS) { - groupId = 0 - } - } -} - -// Add saga to a dedicated group -GROUPS[MAX_GROUPS] = [:] -GROUPS[MAX_GROUPS].name = "group-${String.format("%02d", MAX_GROUPS + 1)}" -GROUPS[MAX_GROUPS].tests = "saga" - -print JsonOutput.toJson(["include": GROUPS]) diff --git a/.github/generate-test-matrix.sh b/.github/generate-test-matrix.sh new file mode 100755 index 00000000..07fbe120 --- /dev/null +++ b/.github/generate-test-matrix.sh @@ -0,0 +1,96 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Reads a Scalpel report and emits GitHub Actions matrix + modules outputs. +# Usage: generate-test-matrix.sh +# Output lines are in key=value format suitable for >> $GITHUB_OUTPUT. + +set -euo pipefail + +REPORT="${1:?Usage: $0 }" +MAX_GROUPS=10 + +# Unique top-level modules (first path component of each affected module path) +ALL=() +if [ -f "$REPORT" ]; then + while IFS= read -r line; do + ALL+=("$line") + done < <(jq -r '[.affectedModules[] | select(.path != "") | .path | split("/")[0]] | unique[]' "$REPORT") +fi + +# Fall back to all reactor modules when Scalpel found nothing or wrote no report +# (no base branch configured, e.g. a plain push with no PR context) +if [ ${#ALL[@]} -eq 0 ]; then + while IFS= read -r line; do + ALL+=("$line") + done < <(grep '' pom.xml | sed 's|.*\(.*\).*|\1|' | sort) +fi + +# Separate single-module and multi-module projects. +# Multi-module projects get a dedicated native test group (they run from their own +# root directory) and are expanded with sub-modules for the -pl modules list. +SINGLE=() +MULTI=() +for m in "${ALL[@]}"; do + if grep -q '' "$m/pom.xml" 2>/dev/null; then + MULTI+=("$m") + else + SINGLE+=("$m") + fi +done + +# Round-robin distribution of single-module projects into at most MAX_GROUPS groups +GROUP_TESTS=() +for i in "${!SINGLE[@]}"; do + gid=$((i % MAX_GROUPS)) + if [[ -z "${GROUP_TESTS[$gid]+x}" ]]; then + GROUP_TESTS[$gid]="${SINGLE[$i]}" + else + GROUP_TESTS[$gid]="${GROUP_TESTS[$gid]},${SINGLE[$i]}" + fi +done + +# Build matrix JSON include array +INCLUDE="[" +SEP="" +for gid in $(echo "${!GROUP_TESTS[@]}" | tr ' ' '\n' | sort -n); do + INCLUDE+="${SEP}$(printf '{"name":"group-%02d","tests":"%s"}' $((gid + 1)) "${GROUP_TESTS[$gid]}")" + SEP="," +done +# Multi-module projects each get their own dedicated group at the end +NUM_GROUPS=${#GROUP_TESTS[@]} +for i in "${!MULTI[@]}"; do + INCLUDE+="${SEP}$(printf '{"name":"group-%02d","tests":"%s"}' $((NUM_GROUPS + i + 1)) "${MULTI[$i]}")" + SEP="," +done +INCLUDE+="]" + +# Build expanded module list for -pl: multi-module projects include their sub-modules +# so that Maven builds the full project tree rather than just the parent POM. +EXPANDED=() +for m in "${ALL[@]}"; do + EXPANDED+=("$m") + if grep -q '' "$m/pom.xml" 2>/dev/null; then + while IFS= read -r submod; do + [ -n "$submod" ] && EXPANDED+=("$m/$submod") + done < <(grep '' "$m/pom.xml" | sed 's|.*\(.*\).*|\1|') + fi +done + +echo "matrix={\"include\":${INCLUDE}}" +echo "modules=$(IFS=,; echo "${EXPANDED[*]}")" diff --git a/.github/workflows/ci-build.yaml b/.github/workflows/ci-build.yaml index 3f083a1a..b5ff8dea 100644 --- a/.github/workflows/ci-build.yaml +++ b/.github/workflows/ci-build.yaml @@ -57,6 +57,7 @@ jobs: runs-on: ubuntu-latest outputs: matrix: ${{ steps.set-itest-matrix.outputs.matrix }} + modules: ${{ steps.set-itest-matrix.outputs.modules }} steps: - name: Set up JDK 17 uses: actions/setup-java@v5 @@ -103,13 +104,22 @@ jobs: license:check \ net.revelc.code.formatter:formatter-maven-plugin:validate \ net.revelc.code:impsort-maven-plugin:check + - name: Fetch base branch for Scalpel + if: github.event_name == 'pull_request' && !contains(github.event.pull_request.labels.*.name, 'ci/disable-incremental') + run: git fetch --unshallow origin ${{ github.base_ref }} - name: Build Camel Quarkus Examples run: | - ./mvnw ${MAVEN_ARGS} clean install -DskipTests -Dquarkus.build.skip + ./mvnw ${MAVEN_ARGS} clean install -DskipTests -Dquarkus.build.skip \ + -Dscalpel.enabled=true \ + -Dscalpel.mode=report \ + -Dscalpel.reportFile=${{ runner.temp }}/scalpel-report.json - name: Fail if there are uncommitted changes shell: bash run: | [[ -z $(git status --porcelain) ]] || { echo 'There are uncommitted changes'; git status; exit 1; } + - name: Setup Integration Test Matrix + id: set-itest-matrix + run: bash .github/generate-test-matrix.sh '${{ runner.temp }}/scalpel-report.json' >> $GITHUB_OUTPUT - name: Tar Maven Repo shell: bash run: | @@ -120,12 +130,6 @@ jobs: name: maven-repo path: ${{ runner.temp }}/maven-repo.tar.zst retention-days: 1 - - name: Setup Integration Test Matrix - id: set-itest-matrix - run: | - sudo apt install groovy -y --no-install-recommends - TEST_GROUPS=$(groovy .github/generate-test-groups.groovy) - echo "matrix=${TEST_GROUPS}" >> $GITHUB_OUTPUT integration-tests: name: Integration Tests - ${{matrix.name}} @@ -214,7 +218,8 @@ jobs: - name: Integration Tests shell: bash run: | - ./mvnw ${MAVEN_ARGS} clean test + ./mvnw ${MAVEN_ARGS} clean test \ + -pl ${{ needs.initial-mvn-install.outputs.modules }} - name: Fail if there are uncommitted changes shell: bash run: | @@ -248,7 +253,8 @@ jobs: - name: Integration Tests shell: bash run: | - ./mvnw ${MAVEN_ARGS} -Dskip-testcontainers-tests clean test + ./mvnw ${MAVEN_ARGS} -Dskip-testcontainers-tests clean test \ + -pl ${{ needs.initial-mvn-install.outputs.modules }} - name: Fail if there are uncommitted changes shell: bash run: | diff --git a/.mvn/extensions.xml b/.mvn/extensions.xml new file mode 100644 index 00000000..fb9b1d4b --- /dev/null +++ b/.mvn/extensions.xml @@ -0,0 +1,8 @@ + + + + eu.maveniverse.maven.scalpel + extension + 0.3.4 + + diff --git a/.mvn/maven.config b/.mvn/maven.config new file mode 100644 index 00000000..351e49e2 --- /dev/null +++ b/.mvn/maven.config @@ -0,0 +1 @@ +-Dscalpel.enabled=false