From d6ccb6c78a2137420e425d2cd53a11d0f0434796 Mon Sep 17 00:00:00 2001 From: Komal Yadav Date: Tue, 10 Feb 2026 06:21:12 +0000 Subject: [PATCH] Implemented exit gate updated Implemented exit gate --- .github/workflows/build-and-unit-test.yml | 85 ++++++---------- .github/workflows/cloudbuild.yaml | 118 ++++++++++++++++++++++ pom.xml | 34 +++++++ 3 files changed, 181 insertions(+), 56 deletions(-) create mode 100644 .github/workflows/cloudbuild.yaml diff --git a/.github/workflows/build-and-unit-test.yml b/.github/workflows/build-and-unit-test.yml index c6b8a348..0685b199 100644 --- a/.github/workflows/build-and-unit-test.yml +++ b/.github/workflows/build-and-unit-test.yml @@ -1,4 +1,4 @@ -# Copyright © 2022 Cask Data, Inc. +# Copyright © 2025 Cask Data, Inc. # Licensed under the Apache License, Version 2.0 (the "License"); you may not # use this file except in compliance with the License. You may obtain a copy of # the License at @@ -85,10 +85,10 @@ jobs: if: ${{ matrix.branch == 'develop' || startsWith(matrix.branch, 'release/') }} with: secrets: |- - CDAP_OSSRH_USERNAME:cdapio-github-builds/CDAP_OSSRH_USERNAME - CDAP_OSSRH_PASSWORD:cdapio-github-builds/CDAP_OSSRH_PASSWORD - CDAP_GPG_PASSPHRASE:cdapio-github-builds/CDAP_GPG_PASSPHRASE CDAP_GPG_PRIVATE_KEY:cdapio-github-builds/CDAP_GPG_PRIVATE_KEY + CDAP_GPG_PASSPHRASE:cdapio-github-builds/CDAP_GPG_PASSPHRASE + ARTIFACT_BUCKET:cdapio-github-builds/artifacts_bucket + SECURE_PUBLISH_BUCKET:cdapio-github-builds/publish_bucket - name: Recursively Checkout Repository uses: actions/checkout@v4 @@ -160,8 +160,8 @@ jobs: retry_on: error on_retry_command: echo "Build Standalone failed in this attempt, retrying ..." command: | - cd cdap-build - MAVEN_OPTS="-Xmx12G" mvn -e -T2 clean package -Dgpg.skip -DskipTests -Ddocker.skip=true -nsu -am -amd -P templates,dist,release -Dadditional.artifacts.dir=$(pwd)/app-artifacts -Dsecurity.extensions.dir=$(pwd)/security-extensions -Dmaven.wagon.http.retryHandler.count=5 -Dmaven.wagon.httpconnectionManager.ttlSeconds=30 + cd cdap-build + MAVEN_OPTS="-Xmx12G" mvn -e -T2 clean package -Dgpg.skip -DskipTests -Ddocker.skip=true -nsu -am -amd -P templates,dist,release -Dadditional.artifacts.dir=$(pwd)/app-artifacts -Dsecurity.extensions.dir=$(pwd)/security-extensions -Dmaven.wagon.http.retryHandler.count=5 -Dmaven.wagon.httpconnectionManager.ttlSeconds=30 - name: Find Build Version working-directory: cdap-build/cdap @@ -189,65 +189,38 @@ jobs: - name: Set up GPG conf if: ${{ matrix.branch == 'develop' || startsWith(matrix.branch, 'release/') }} - run: | - echo "pinentry-mode loopback" >> ~/.gnupg/gpg.conf - echo "allow-loopback-pinentry" >> ~/.gnupg/gpg-agent.conf - name: Import GPG key if: ${{ matrix.branch == 'develop' || startsWith(matrix.branch, 'release/') }} run: | - echo "$GPG_PRIVATE_KEY" > private.key - gpg --import --batch private.key + echo "$GPG_PRIVATE_KEY" > private.key + gpg --import --batch private.key env: GPG_PRIVATE_KEY: ${{ steps.secrets.outputs.CDAP_GPG_PRIVATE_KEY }} - - name: Maven Deploy + - name: Submit Build to GCB + id: gcb + working-directory: cdap-build run: | - cd cdap-build - if [[ (${{ matrix.branch }} == "develop") || (${{ matrix.branch }} == release/*) ]]; - then - retry_count=0 - failed_module="" - while [ $retry_count -lt 3 ]; do - if [ -n "$failed_module" ]; - then - build_output=$(mvn deploy -B -V -DskipTests -DskipLocalStaging=true -Ddocker.skip=true -P templates,dist,release,rpm-prepare,rpm,deb-prepare,deb,tgz,unit-tests -Dadditional.artifacts.dir=$(pwd)/app-artifacts -Dsecurity.extensions.dir=$(pwd)/security-extensions -Dmaven.wagon.http.retryHandler.count=5 -Dmaven.wagon.httpconnectionManager.ttlSeconds=30 -Dmaven.wagon.http.retryHandler.requestSentEnabled=true -Dgpg.passphrase=$CDAP_GPG_PASSPHRASE -rf :"$failed_module" 2>&1 || true) - else - build_output=$(mvn deploy -B -V -DskipTests -DskipLocalStaging=true -Ddocker.skip=true -P templates,dist,release,rpm-prepare,rpm,deb-prepare,deb,tgz,unit-tests -Dadditional.artifacts.dir=$(pwd)/app-artifacts -Dsecurity.extensions.dir=$(pwd)/security-extensions -Dmaven.wagon.http.retryHandler.count=5 -Dmaven.wagon.httpconnectionManager.ttlSeconds=30 -Dmaven.wagon.http.retryHandler.requestSentEnabled=true -Dgpg.passphrase=$CDAP_GPG_PASSPHRASE 2>&1 || true) - fi - echo "$build_output" - if [ $(echo "$build_output" | grep -c "BUILD FAILURE") -gt 0 ]; - then - echo "[WARNING] Deployment failed, retrying..." - failed_module=$(echo "$build_output" | grep -e 'mvn -rf' | sed -n 's/.*mvn -rf ://p') - echo "[INFO] FAILED MODULE = $failed_module" - retry_count=$((retry_count + 1)) - else - echo "[INFO] Deployment successful" - break - fi - done - if [ $retry_count -ge 3 ]; - then - echo "[ERROR] Max retries reached..., deployment failed" - exit 1 - fi - else - mvn verify -B -V -T2 -DskipTests -Dgpg.skip -Ddocker.skip=true -P templates,dist,release,rpm-prepare,rpm,deb-prepare,deb,tgz,unit-tests -Dadditional.artifacts.dir=$(pwd)/app-artifacts -Dsecurity.extensions.dir=$(pwd)/security-extensions -Dmaven.wagon.http.retryHandler.count=5 -Dmaven.wagon.httpconnectionManager.ttlSeconds=30 -Dmaven.wagon.http.retryHandler.requestSentEnabled=true + BUILD_OUTPUT=$(gcloud builds submit . \ + --config=cloudbuild-release.yaml \ + --project='cdapio-github-builds' \ + --substitutions="_SECURE_PUBLISH_BUCKET_NAME=${{ SECURE_PUBLISH_BUCKET }}, GPG_PASSPHRASE=${{CDAP_GPG_PASSPHRASE}}) + echo "$BUILD_OUTPUT" + BUILD_ID=$(echo "$BUILD_OUTPUT" | grep -oP 'ID: \K[a-f0-9-]+' | head -n 1) + if [ -z "$BUILD_ID" ]; then + echo "Failed to extract BUILD_ID from gcloud output." >&2 + # Fallback: Get the last build submitted by this service account. + # This assumes no other concurrent builds are started by this SA. + BUILD_ID=$(gcloud builds list --project='cdapio-github-builds' --limit=1 --format='value(ID)' --filter="buildTriggerId='' AND status!=WORKING AND status!=QUEUED" --sort-by=~CREATE_TIME) + if [ -z "$BUILD_ID" ]; then + echo "Fallback failed to get BUILD_ID." >&2 + exit 1 fi - env: - CDAP_OSSRH_USERNAME: ${{ steps.secrets.outputs.CDAP_OSSRH_USERNAME }} - CDAP_OSSRH_PASSWORD: ${{ steps.secrets.outputs.CDAP_OSSRH_PASSWORD }} - CDAP_GPG_PASSPHRASE: ${{ steps.secrets.outputs.CDAP_GPG_PASSPHRASE }} - MAVEN_OPTS: "-Xmx12G" - - - name: Build DEB Bundle - working-directory: cdap-build/cdap - run: | - mkdir -p cdap-distributions/target/deb-bundle-tmp - cd cdap-distributions/target/deb-bundle-tmp - cp ../../../*/target/*.deb . - tar zcf ../cdap-distributed-deb-bundle-${{env.CDAP_VERSION}}.tgz *.deb + echo "Fallback BUILD_ID: $BUILD_ID" + fi + echo "build_id=$BUILD_ID" >> $GITHUB_OUTPUT + echo "GCB Build ID: $BUILD_ID" - name: Set Up Tag working-directory: cdap-build diff --git a/.github/workflows/cloudbuild.yaml b/.github/workflows/cloudbuild.yaml new file mode 100644 index 00000000..cb2a7727 --- /dev/null +++ b/.github/workflows/cloudbuild.yaml @@ -0,0 +1,118 @@ +# Copyright © 2025 Cask Data, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. + +steps: + - name: 'gcr.io/cloud-builders/mvn:3.8-jdk-8' + id: mvn-deploy + entrypoint: 'bash' + args: + - '-c' + - | + if [[ ("$BRANCH" == "develop") || ("$BRANCH" == release/*) ]]; + then + echo "--- Running 'mvn deploy' with retries for branch: $BRANCH ---" + retry_count=0 + failed_module="" + while [ $retry_count -lt 3 ]; do + MVN_CMD="mvn deploy -B -V -DskipTests -DskipLocalStaging=true -Ddocker.skip=true -P templates,dist,release,rpm-prepare,rpm,deb-prepare,deb,tgz,unit-tests -Dadditional.artifacts.dir=$(pwd)/app-artifacts -Dsecurity.extensions.dir=$(pwd)/security-extensions -Dmaven.wagon.http.retryHandler.count=5 -Dmaven.wagon.httpconnectionManager.ttlSeconds=30 -Dmaven.wagon.http.retryHandler.requestSentEnabled=true -Dgpg.passphrase=$GPG_PASSPHRASE" + + if [ -n "$failed_module" ]; then + MVN_CMD="$MVN_CMD -rf :$failed_module" + echo "[INFO] Retrying build from module: $failed_module" + fi + + echo "Executing Maven command attempt $((retry_count + 1)): $MVN_CMD" + # Execute the Maven command, redirecting stderr to stdout, and continuing on failure + build_output=$(eval "$MVN_CMD" 2>&1 || true) + echo "$build_output" + + if echo "$build_output" | grep -q "BUILD FAILURE"; then + echo "[WARNING] Deployment failed, attempt $((retry_count + 1)) of 3." + # Extract the failed module suggested by Maven's output for resuming + failed_module=$(echo "$build_output" | grep -e 'mvn -rf' | sed -n 's/.*mvn -rf ://p' | head -n 1) + + if [ -z "$failed_module" ]; then + echo "[WARNING] Could not extract specific failed module. Retrying full build." + else + echo "[INFO] Found failed module: $failed_module" + fi + retry_count=$((retry_count + 1)) + else + echo "[INFO] Deployment successful" + break + fi + done + + if [ $retry_count -ge 3 ]; then + echo "[ERROR] Max retries reached..., deployment failed" + exit 1 + fi + else + echo "--- Running 'mvn verify' for branch: $BRANCH ---" + mvn verify -B -V -T2 -DskipTests -Dgpg.skip -Ddocker.skip=true -P templates,dist,release,rpm-prepare,rpm,deb-prepare,deb,tgz,unit-tests -Dadditional.artifacts.dir=$(pwd)/app-artifacts -Dsecurity.extensions.dir=$(pwd)/security-extensions -Dmaven.wagon.http.retryHandler.count=5 -Dmaven.wagon.httpconnectionManager.ttlSeconds=30 -Dmaven.wagon.http.retryHandler.requestSentEnabled=true + fi + secretEnv: + - 'GPG_PASSPHRASE' + + - name: 'bash' + id: stage-artifacts + entrypoint: 'bash' + args: + - '-c' + - | + set -e + GCS_MANIFEST_DIR="gs://oss-exit-gate-prod-projects-bucket/cloud-data-fusion/mavencentral/manifests/" + MANIFEST_FILE="/workspace/exit_gate_manifest.textproto" + MANIFEST_FILENAME="release_$$(date -u +%Y%m%d%H%M%S).textproto" + echo "Uploading manifest to $${GCS_MANIFEST_DIR}$${MANIFEST_FILENAME}" + gsutil cp "$${MANIFEST_FILE}" "$${GCS_MANIFEST_DIR}$${MANIFEST_FILENAME}" + echo "Manifest uploaded successfully. OSS Exit Gate process should now be triggered." + waitFor: ["mvn-deploy"] + + # 5. Create Exit Gate Manifest (ONLY for RELEASE) + - name: 'bash' + id: create-exit-gate-manifest + entrypoint: 'bash' + args: + - '-c' + - | + set -e + MANIFEST_FILE="/workspace/exit_gate_manifest.textproto" + echo '' >> "$${MANIFEST_FILE}" + echo "Created manifest file: $${MANIFEST_FILE}" + waitFor: ['stage-artifact'] + + # 6. Upload Exit Gate Manifest (ONLY for RELEASE) + - name: 'gcr.io/cloud-builders/gsutil' + id: upload-exit-gate-manifest + entrypoint: 'bash' + args: + - '-c' + - | + set -e + GCS_MANIFEST_DIR="gs://oss-exit-gate-prod-projects-bucket/cloud-data-fusion/mavencentral/manifests/" + MANIFEST_FILE="/workspace/exit_gate_manifest.textproto" + MANIFEST_FILENAME="release_$$(date -u +%Y%m%d%H%M%S).textproto" + echo "Uploading manifest to $${GCS_MANIFEST_DIR}$${MANIFEST_FILENAME}" + gsutil cp "$${MANIFEST_FILE}" "$${GCS_MANIFEST_DIR}$${MANIFEST_FILENAME}" + echo "Manifest uploaded successfully. OSS Exit Gate process should now be triggered." + else + echo "Skipping Exit Gate manifest upload for SNAPSHOT build." + fi + waitFor: ['create-exit-gate-manifest'] + +options: + requestedVerifyOption: VERIFIED + machineType: 'E2_HIGHCPU_32' + diff --git a/pom.xml b/pom.xml index 4bc350f3..16ded95c 100644 --- a/pom.xml +++ b/pom.xml @@ -53,6 +53,30 @@ HEAD + + + artifact-registry + artifactregistry://us-maven.pkg.dev/oss-exit-gate-prod/cloud-data-fusion--mavencentral + + + artifact-registry + artifactregistry://us-maven.pkg.dev/oss-exit-gate-prod/cloud-data-fusion--mavencentral + + + + + + artifact-registry + artifactregistry://us-maven.pkg.dev/oss-exit-gate-prod/cloud-data-fusion--mavencentral + + true + + + true + + + + app-artifacts/bigquery-delta-plugins app-artifacts/database-delta-plugins @@ -72,6 +96,16 @@ cdap + + + + com.google.cloud.artifactregistry + artifactregistry-maven-wagon + 2.2.5 + + + + release