diff --git a/docker/pypi/wmcore-dev/ContainerScripts/AggregatePylint.py b/docker/pypi/wmcore-dev/ContainerScripts/AggregatePylint.py deleted file mode 100755 index 84559c69b..000000000 --- a/docker/pypi/wmcore-dev/ContainerScripts/AggregatePylint.py +++ /dev/null @@ -1,95 +0,0 @@ -#! /usr/bin/env python3 - -import json - -from optparse import OptionParser - -usage = "usage: %prog [options] message" -parser = OptionParser(usage) -(options, args) = parser.parse_args() -if len(args) != 1: - parser.error("You must supply a label") - -label = args[0] - -try: - with open('pylintReport.json', 'r') as reportFile: - report = json.load(reportFile) -except IOError: - report = {} - -warnings = 0 -errors = 0 -comments = 0 -refactors = 0 -score = 0 - -with open('pylint.out', 'r') as pylintFile: - for line in pylintFile: - if line.startswith('Your code has been rated at '): - scorePart = line.strip('Your code has been rated at ') - score = scorePart.split('/')[0] - try: - if not filename in report: - report[filename] = {} - if not label in report[filename]: - report[filename][label] = {} - if filename and label: - report[filename][label]['score'] = score - except NameError: - print("Score of %s found, but no filename" % score) - - parts = line.split(':') - if len(parts) != 3: - continue - try: - newFilename, lineNumber, rawMessage = parts - newFilename = newFilename.strip() - if not newFilename: # Don't update filename if we didn't find one - continue - lineNumber = int(lineNumber) - filename = newFilename - rmParts = rawMessage.split(']', 1) - rawCode = rmParts[0].strip() - message = rmParts[1].strip() - severity = rawCode[1:2] - code = rawCode[2:6] - shortMsg = rawCode[7:] - msgParts = shortMsg.split(',') - objectName = msgParts[1].strip() - - if severity == 'R': - refactors += 1 - elif severity == 'W': - warnings += 1 - elif severity == 'E': - errors += 1 - elif severity == 'C': - comments += 1 - - if not filename in report: - report[filename] = {} - - if not label in report[filename]: - report[filename][label] = {} - if not 'events' in report[filename][label]: - report[filename][label]['events'] = [] - report[filename][label]['events'].append((lineNumber, severity, code, objectName, message)) - - report[filename][label]['refactors'] = refactors - report[filename][label]['warnings'] = warnings - report[filename][label]['errors'] = errors - report[filename][label]['comments'] = comments - - except ValueError: - continue - -with open('pylintReport.json', 'w') as reportFile: - json.dump(report, reportFile, indent=2) - reportFile.write('\n') - - - - - - diff --git a/docker/pypi/wmcore-dev/ContainerScripts/AnalyzePyFuture.py b/docker/pypi/wmcore-dev/ContainerScripts/AnalyzePyFuture.py deleted file mode 100755 index 94092147d..000000000 --- a/docker/pypi/wmcore-dev/ContainerScripts/AnalyzePyFuture.py +++ /dev/null @@ -1,20 +0,0 @@ -#! /usr/bin/env python - -from __future__ import print_function, division - -import os - -with open('addedFiles.txt', 'r') as addedFiles: - for fileName in addedFiles: - fileName = fileName.strip() - if fileName.endswith('__init__.py'): - continue - with open(fileName, 'r') as pyFile: - pyLines = pyFile.readlines() - if fileName.endswith('.py') or 'python' in pyLines[0]: - foundDivision = False - for line in pyLines: - if '__future__' in line and 'division' in line: - foundDivision = True - if not foundDivision: - print ("* New file %s does not use python 3 division. Please add `from __future__ import division`.\n" % fileName) diff --git a/docker/pypi/wmcore-dev/ContainerScripts/CompareTests.py b/docker/pypi/wmcore-dev/ContainerScripts/CompareTests.py deleted file mode 100755 index 195e667d7..000000000 --- a/docker/pypi/wmcore-dev/ContainerScripts/CompareTests.py +++ /dev/null @@ -1,114 +0,0 @@ -#!/usr/bin/env python - -from __future__ import print_function - -import glob -import os -import sys - -try: - from github import Github -except ImportError: - # DMWM-WMCore-UnitTests and DMWM-WMCorePy3-UnitTests don't really push anything! - Github = None - -import xunitparser - -testResults = {} - -unstableTests = [] - -try: - with open('code/test/etc/UnstableTests.txt') as unstableFile: - for line in unstableFile: - unstableTests.append(line.strip()) -except: - print("Was not able to open list of unstable tests") - -# Parse all the various nose xunit test reports looking for changes -filePattern = '*/nosetestspy3-*.xml' -if len(sys.argv) == 2: - filePattern = "*/%s-*.xml" % sys.argv[1] -for kind, directory in [('base', './MasterUnitTests/'), ('test', './LatestUnitTests/')]: - for xunitFile in glob.iglob(directory + filePattern): - - ts, tr = xunitparser.parse(open(xunitFile)) - for tc in ts: - testName = '%s:%s' % (tc.classname, tc.methodname) - if testName in testResults: - testResults[testName].update({kind: tc.result}) - else: - testResults[testName] = {kind: tc.result} - -# Generate a Github report of any changes found - -issueID, mode = None, None - -if 'ghprbPullId' in os.environ: - issueID = os.environ['ghprbPullId'] - mode = 'PR' -elif 'TargetIssueID' in os.environ: - issueID = os.environ['TargetIssueID'] - mode = 'Daily' - -print("Comparing tests for issueID: {} in mode: {}".format(issueID, mode)) - -message = 'Unit test changes for pull request %s:\n' % issueID -if mode == 'Daily': - message = 'Unit test changes for most recent test of master branch:\n' - -changed = False -stableChanged = False -failed = False -errorConditions = ['error', 'failure'] - -for testName, testResult in sorted(testResults.items()): - if 'base' in testResult and 'test' in testResult and testName in unstableTests: - if testResult['base'] != testResult['test']: - changed = True - message += "* %s (unstable) changed from %s to %s\n" % (testName, testResult['base'], testResult['test']) - elif 'base' in testResult and 'test' in testResult: - if testResult['base'] != testResult['test']: - changed = True - stableChanged = True - message += "* %s changed from %s to %s\n" % (testName, testResult['base'], testResult['test']) - if testResult['test'] in errorConditions: - failed = True - elif 'test' in testResult: - changed = True - stableChanged = True - message += "* %s was added. Status is %s\n" % (testName, testResult['test']) - if testResult['test'] in errorConditions: - failed = True - elif 'base' in testResult: - changed = True - stableChanged = True - message += "* %s was deleted. Prior status was %s\n" % (testName, testResult['base']) -if failed: - message += '\n\nPreviously working unit tests have failed!\n' - -if mode == 'Daily': - # Alan on 25/may/2021: then there is nothing else to be done - print(message) - sys.exit(0) - -gh = Github(os.environ['DMWMBOT_TOKEN']) -codeRepo = os.environ.get('CODE_REPO', 'WMCore') -repoName = '%s/%s' % (os.environ['WMCORE_REPO'], codeRepo) - -issue = gh.get_repo(repoName).get_issue(int(issueID)) - -if not changed and mode == 'Daily': - message = "No changes to unit tests for latest build\n" -elif not changed: - message = "No changes to unit tests for pull request %s\n" % issueID - -if mode == 'Daily' and stableChanged: - issue.create_comment('%s' % message) -elif mode != 'Daily': - issue.create_comment('%s' % message) - -if failed: - print('Testing of python code. DMWM-FAIL-UNIT') -else: - print('Testing of python code. DMWM-SUCCEED-UNIT') diff --git a/docker/pypi/wmcore-dev/ContainerScripts/IdentifyPythonFiles.py b/docker/pypi/wmcore-dev/ContainerScripts/IdentifyPythonFiles.py deleted file mode 100755 index f632f426e..000000000 --- a/docker/pypi/wmcore-dev/ContainerScripts/IdentifyPythonFiles.py +++ /dev/null @@ -1,31 +0,0 @@ -#! /usr/bin/env python3 - -from __future__ import print_function, division - -import os -from optparse import OptionParser - -usage = "usage: %prog [options] list_of_files.txt" -parser = OptionParser(usage) -(options, args) = parser.parse_args() -if len(args) != 1: - parser.error("You must supply a file with a list of files to check") - -list_of_files = args[0] - -with open(list_of_files, 'r') as changedFiles: - for fileName in changedFiles: - fileName = fileName.strip() - if not fileName: - continue - if fileName.endswith('.py'): - print(fileName) - continue - try: - with open(fileName, 'r') as pyFile: - pyLines = pyFile.readlines() - if 'python' in pyLines[0]: - print(fileName) - continue - except IOError: - pass diff --git a/docker/pypi/wmcore-dev/ContainerScripts/deployWMAgent.sh b/docker/pypi/wmcore-dev/ContainerScripts/deployWMAgent.sh deleted file mode 100755 index d304b98fb..000000000 --- a/docker/pypi/wmcore-dev/ContainerScripts/deployWMAgent.sh +++ /dev/null @@ -1,12 +0,0 @@ -#! /bin/bash - -. ./cms-bot/DMWM/setup-secrets.sh -. ./cms-bot/DMWM/update-deployment.sh -. ./cms-bot/DMWM/latest-dmwm-versions.sh - -if [ -z "$WMAGENT_VERSION" ]; then - export WMAGENT_VERSION=$WMAGENT_LATEST -fi - -. ./cms-bot/DMWM/deploy-wmagent.sh - diff --git a/docker/pypi/wmcore-dev/ContainerScripts/deploy_unittest_py3.sh b/docker/pypi/wmcore-dev/ContainerScripts/deploy_unittest_py3.sh deleted file mode 100755 index bc65e42a1..000000000 --- a/docker/pypi/wmcore-dev/ContainerScripts/deploy_unittest_py3.sh +++ /dev/null @@ -1,94 +0,0 @@ -#!/bin/bash - -### -# usage -# deploy current version of wmagent external library and use the dmwm/master WMCore code to set up unittest -# sh ./deploy_unittest.sh -# -# Also optional values can be specified. -v deploy agent version, -r git repository for code to test, -# -b branch name of the repository. -# following -# i.e) sh ./deploy_unittest.sh -v 1.0.6 -r ticonaan -v 1.0.5_wmagent -# -# for running the test check the tutorial, https://github.com/dmwm/WMCore/wiki/Setup-wmcore-unittest -### -DMWM_ARCH=slc7_amd64_gcc630 -VERSION=$(curl -s "http://cmsrep.cern.ch/cgi-bin/repos/comp/$DMWM_ARCH?C=M;O=D" | grep -oP "(?<=>cms\+wmagentpy3-dev\+).*(?=-1-1)" | head -1) - -REPOSITORY=dmwm -BRANCH= -UPDATE=false - -deploy_agent() { - - git clone https://github.com/dmwm/deployment.git - curl -s https://raw.githubusercontent.com/dmwm/WMCore/master/test/deploy/init.sh > init.sh - curl -s https://raw.githubusercontent.com/dmwm/WMCore/master/test/deploy/env_unittest_py3.sh > env_unittest_py3.sh - curl -s https://raw.githubusercontent.com/dmwm/WMCore/master/test/deploy/WMAgent_unittest.secrets > WMAgent_unittest.secrets - source ./init.sh - # set -e - for step in prep sw post; do - echo -e "\n*** Deploying WMAgent py3: running $step step ***" - $PWD/deployment/Deploy -R wmagentpy3-dev@$1 -r comp=comp -t $1 -A $DMWM_ARCH -s $step $INSTALL_DIR wmagentpy3/devtools - if [ $? -ne 0 ]; then - ls $INSTALL_DIR - cat $INSTALL_DIR/.deploy/*-$step.log - exit 1 - fi - done - # set +e -} - -setup_test_src() { - ( - mkdir $TEST_DIR; - cd $TEST_DIR; - git clone https://github.com/$1/WMCore.git; - cd WMCore - # if branch is set check out the branch - if [ -n $2 ] - then - git checkout $2 - fi; - ) -} - -update_src() { - ( - rm -rf $TEST_DIR; - setup_test_src $1 $2 - ) -} - -while [ $# -gt 0 ] -do - case "$1" in - -v) VERSION=$2; shift;; - -r) REPOSITORY=$2; shift;; - -b) BRANCH=$2; shift;; - -u) UPDATE=true;; - *) break;; # terminate while loop - esac - shift -done - -if [ $UPDATE = "true" ] -then - source ./env_unittest_py3.sh - update_src $REPOSITORY $BRANCH -else - echo "--- deploying agent $VERSION with local user $USER" - # deploy agent - deploy_agent $VERSION - echo "--- updating agent source repository $REPOSITORY, branch $BRANCH" - # checkout test source - setup_test_src $REPOSITORY $BRANCH - - # swap the source code from deployed one test source - source ./env_unittest_py3.sh - echo "--- starting services" - $manage start-services - - echo "--- creating MariaDB database" - mysql -u $USER --socket=$INSTALL_DIR/current/install/mysql/logs/mysql.sock --execute "create database wmcore_unittest" -fi diff --git a/docker/pypi/wmcore-dev/ContainerScripts/fixCertificates.sh b/docker/pypi/wmcore-dev/ContainerScripts/fixCertificates.sh deleted file mode 100755 index 7967f144d..000000000 --- a/docker/pypi/wmcore-dev/ContainerScripts/fixCertificates.sh +++ /dev/null @@ -1,10 +0,0 @@ -#! /bin/bash - -set -x - -# Certificates are either not readable by the container or have the wrong permissions - -cp orig-certs/servicecert.pem.orig certs/servicecert.pem -cp orig-certs/servicekey.pem.orig certs/servicekey.pem -chmod 600 certs/servicecert.pem -chmod 400 certs/servicekey.pem diff --git a/docker/pypi/wmcore-dev/ContainerScripts/installCMSBot.sh b/docker/pypi/wmcore-dev/ContainerScripts/installCMSBot.sh deleted file mode 100755 index 701d34698..000000000 --- a/docker/pypi/wmcore-dev/ContainerScripts/installCMSBot.sh +++ /dev/null @@ -1,9 +0,0 @@ -git clone https://github.com/cms-sw/cms-bot -pushd cms-bot/ - -wget https://pypi.python.org/packages/source/r/requests/requests-2.3.0.tar.gz#md5=7449ffdc8ec9ac37bbcd286003c80f00 -tar -xvf requests-2.3.0.tar.gz -rm -rf requests || true -mv requests-2.3.0/requests/ requests - -popd diff --git a/docker/pypi/wmcore-dev/ContainerScripts/installWMCore.sh b/docker/pypi/wmcore-dev/ContainerScripts/installWMCore.sh deleted file mode 100755 index b293340de..000000000 --- a/docker/pypi/wmcore-dev/ContainerScripts/installWMCore.sh +++ /dev/null @@ -1,12 +0,0 @@ -#! /bin/sh - -# Download the script to install everything -curl https://raw.githubusercontent.com/dmwm/WMCore/master/test/deploy/deploy_unittest.sh > /home/dmwm/ContainerScripts/deploy_unittest.sh -chmod +x /home/dmwm/ContainerScripts/deploy_unittest.sh -sh /home/dmwm/ContainerScripts/deploy_unittest.sh - -echo "export PYTHONPATH=/home/dmwm/wmcore_unittest/WMCore/src/python:\$PYTHONPATH" >> ./env_unittest.sh -# Shut down services so the docker container doesn't have stale PID & socket files -source ./env_unittest.sh -$manage stop-services - diff --git a/docker/pypi/wmcore-dev/ContainerScripts/installWMCorePy3.sh b/docker/pypi/wmcore-dev/ContainerScripts/installWMCorePy3.sh deleted file mode 100755 index 5074bdd22..000000000 --- a/docker/pypi/wmcore-dev/ContainerScripts/installWMCorePy3.sh +++ /dev/null @@ -1,12 +0,0 @@ -#! /bin/sh - -# Download the script to install everything -curl https://raw.githubusercontent.com/dmwm/WMCore/master/test/deploy/deploy_unittest_py3.sh > /home/dmwm/ContainerScripts/deploy_unittest_py3.sh -chmod +x /home/dmwm/ContainerScripts/deploy_unittest_py3.sh -sh /home/dmwm/ContainerScripts/deploy_unittest_py3.sh - -echo "export PYTHONPATH=/home/dmwm/wmcore_unittest/WMCore/src/python:\$PYTHONPATH" >> ./env_unittest_py3.sh -# Shut down services so the docker container doesn't have stale PID & socket files -source ./env_unittest_py3.sh -$manage stop-services - diff --git a/docker/pypi/wmcore-dev/ContainerScripts/pyfutureTest.sh b/docker/pypi/wmcore-dev/ContainerScripts/pyfutureTest.sh deleted file mode 100755 index 7cd05fa0a..000000000 --- a/docker/pypi/wmcore-dev/ContainerScripts/pyfutureTest.sh +++ /dev/null @@ -1,40 +0,0 @@ -#!/usr/bin/env bash - -if [ -z "$ghprbPullId" -o -z "$ghprbTargetBranch" ]; then - echo "Not all necessary environment variables set: ghprbPullId, ghprbTargetBranch" - exit 1 -fi - -echo "$(TZ=GMT date): running pyfutureTest.sh script" -source ./env_unittest.sh - -pushd wmcore_unittest/WMCore -export PYTHONPATH=`pwd`/test/python:`pwd`/src/python:${PYTHONPATH} - -git config remote.origin.url https://github.com/dmwm/WMCore.git -git fetch origin pull/${ghprbPullId}/merge:PR_MERGE -export COMMIT=`git rev-parse "PR_MERGE^{commit}"` -git checkout -f ${COMMIT} - -echo "$(TZ=GMT date): figuring out what are the files that changed" -# Find all the changed files and filter python-only -git diff --name-only ${ghprbTargetBranch}..${COMMIT} > allChangedFiles.txt -${HOME}/ContainerScripts/IdentifyPythonFiles.py allChangedFiles.txt > changedFiles.txt -git diff-tree --name-status -r ${ghprbTargetBranch}..${COMMIT} | egrep "^A" | cut -f 2 > allAddedFiles.txt -${HOME}/ContainerScripts/IdentifyPythonFiles.py allAddedFiles.txt > addedFiles.txt -rm -f allChangedFiles.txt allAddedFiles.txt - -echo "$(TZ=GMT date): running futurize 1st stage and some fixers" -while read name; do - futurize -1 $name >> test.patch - futurize -f execfile -f filter -f raw_input $name >> test.patch || true - futurize -f idioms $name >> idioms.patch || true -done added.message - -cp test.patch idioms.patch added.message ${HOME}/artifacts/ -echo "$(TZ=GMT date): done with all tests and copying files over to artifacts!" -popd diff --git a/docker/pypi/wmcore-dev/ContainerScripts/tagLatestWMCore.sh b/docker/pypi/wmcore-dev/ContainerScripts/tagLatestWMCore.sh deleted file mode 100755 index d1580c060..000000000 --- a/docker/pypi/wmcore-dev/ContainerScripts/tagLatestWMCore.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/usr/bin/env bash - -if [ -z "$DMWMBOT_TOKEN" -o -z "$WMCORE_REPO" -o -z "$CODE_REPO" -o -z "$TAG_PREFIX" ]; then - echo "Not all necessary environment variables set: DMWMBOT_TOKEN, WMCORE_REPO, CODE_REPO" - exit 1 -fi - -pushd $WORKSPACE/WMCore - -set +x -git remote set-url origin https://${DMWMBOT_TOKEN}:x-oauth-basic@github.com/${WMCORE_REPO}/${CODE_REPO} -set -x - -git checkout master -git pull - -# Create a new tag and push it -export TAG=${TAG_PREFIX}_`date "+%Y%m%d%H%M%S"` -git tag ${TAG} -echo Will add tag: $TAG -#git push origin ${TAG} - -# Deletes tags from previous days -export VALID_TAGS=`date "+JENKINS_%Y%m%d"` -git tag -d `git tag | grep JENKINS` -git fetch --tags -#git push --delete origin `git tag | grep JENKINS | grep -Ev ${VALID_TAGS}` || true - -popd diff --git a/docker/pypi/wmcore-dev/ContainerScripts/updateGit.sh b/docker/pypi/wmcore-dev/ContainerScripts/updateGit.sh deleted file mode 100755 index 70938df23..000000000 --- a/docker/pypi/wmcore-dev/ContainerScripts/updateGit.sh +++ /dev/null @@ -1,12 +0,0 @@ -#! /bin/sh - -pushd /home/dmwm/wmcore_unittest/WMCore/ -git pull -git fetch --quiet --tags https://github.com/dmwm/WMCore.git "+refs/heads/*:refs/remotes/origin/*" -git config remote.origin.url https://github.com/dmwm/WMCore.git -git config --add remote.origin.fetch "+refs/heads/*:refs/remotes/origin/*" -git fetch --quiet --tags https://github.com/dmwm/WMCore.git "+refs/pull/*:refs/remotes/origin/pr/*" - - - -popd diff --git a/docker/pypi/wmcore-dev/Dockerfile b/docker/pypi/wmcore-dev/Dockerfile index 2f1d721f3..7672e7bc8 100644 --- a/docker/pypi/wmcore-dev/Dockerfile +++ b/docker/pypi/wmcore-dev/Dockerfile @@ -1,11 +1,27 @@ -FROM --platform=linux/amd64 registry.cern.ch/cmsweb/cmsweb-alma9-base:20240530 +ARG PYTHON_VERSION=3.12 -LABEL version="0.2.0" +FROM --platform=linux/amd64 registry.cern.ch/cmsweb/gfal:alma9-py${PYTHON_VERSION}-gfal-1.13.0-stable as gfal2 +FROM --platform=linux/amd64 registry.cern.ch/cmsweb/dmwm-base:alma9-py3.12-20250728-stable + +ARG PYTHON_VERSION +ARG WMCORE_TAG=2.4.1 + +LABEL org.opencontainers.image.authors="Dennis Lee dylee@fnal.gov" + + +ENV WDIR="/home/cmsbld" + +# Install EPEL repository (required for voms, fetch-crl and CA-related packages) +RUN dnf -y install epel-release && dnf -y upgrade && \ + dnf install -y http://linuxsoft.cern.ch/wlcg/el9/x86_64/wlcg-repo-1.0.0-1.el9.noarch.rpm && \ + dnf clean all # Install cern repo RUN dnf install -y 'dnf-command(config-manager)' && \ dnf config-manager --add-repo https://linuxsoft.cern.ch/cern/alma/9/devel/x86_64/os +ADD http://repository.egi.eu/sw/production/cas/1/current/repo-files/egi-trustanchors.repo /etc/yum.repos.d/egi.repo + # Install basic tools RUN dnf upgrade -y && \ dnf install -y \ @@ -38,31 +54,45 @@ RUN dnf install -y http://linuxsoft.cern.ch/wlcg/el9/x86_64/wlcg-repo-1.0.0-1.el # Tools for getting proxies RUN dnf install -y \ wlcg-voms-cms \ + fetch-crl cern-get-certificate CERN-CA-certs ca-certificates \ + dummy-ca-certs ca-policy-lcg ca-policy-egi-core \ + ca_CERN-GridCA ca_CERN-Root-2 \ voms-clients-cpp && \ dnf clean all # Create the working directory -RUN mkdir -p /home/cmsbld -WORKDIR /home/cmsbld +RUN mkdir -p ${WDIR}/.local +WORKDIR $WDIR + +# Install uv and make a venv based on the python version +ENV UV_PYTHON_INSTALL_DIR=$WDIR/.local/bin +RUN pip3 install uv && \ + uv python install ${PYTHON_VERSION} +ENV VIRTUAL_ENV="$WDIR/.local/wmcore-venv" +RUN uv venv -p ${PYTHON_VERSION} ${VIRTUAL_ENV} +ENV PATH="${VIRTUAL_ENV}/bin:$PATH" # get requirements.txt and remove gfal2 to use RPM gfal2 -RUN curl -O https://raw.githubusercontent.com/dmwm/WMCore/master/requirements.txt && \ - sed '/gfal2/d' /home/cmsbld/requirements.txt > /home/cmsbld/requirements-dev.txt +RUN curl -O "https://raw.githubusercontent.com/dmwm/WMCore/$WMCORE_TAG/requirements.txt" && \ + sed -i '/gfal2/d' /home/cmsbld/requirements.txt -# clone dmwm/WMCore repo and remove gfal2 to use RPM gfal2 -# RUN git clone https://github.com/dmwm/WMCore && \ -# sed '/gfal2/d' /home/cmsbld/WMCore/requirements.txt > /home/cmsbld/WMCore/requirements-dev.txt +RUN uv pip install --no-cache-dir --upgrade pip setuptools wheel && \ + uv pip install --no-cache-dir -r /home/cmsbld/requirements.txt -RUN pip3 install --no-cache-dir --upgrade pip setuptools wheel && \ - pip3 install --no-cache-dir --ignore-installed -r /home/cmsbld/requirements-dev.txt +# copy gfal2 and libboost +COPY --from=gfal2 /tmp/.venv/lib/python${PYTHON_VERSION}/site-packages/gfal2.so $WDIR/.local/wmcore-venv/lib/python${PYTHON_VERSION}/site-packages/gfal2.so +COPY --from=gfal2 /usr/local/lib/* /usr/local/lib/. ADD entrypoint.sh /entrypoint.sh +ADD etc ${WDIR}/etc ENV X509_HOST_CERT=/data/certs/servicecert.pem ENV X509_HOST_KEY=/data/certs/servicekey.pem ENV X509_USER_CERT=/data/certs/servicecert.pem ENV X509_USER_KEY=/data/certs/servicekey.pem +ENV USER=cmsbld +ENV WMCORE_TAG=$WMCORE_TAG ENTRYPOINT ["/entrypoint.sh"] -CMD ["sleep", "infinity"] \ No newline at end of file +CMD ["sleep", "infinity"] diff --git a/docker/pypi/wmcore-dev/Dockerfile.dev b/docker/pypi/wmcore-dev/Dockerfile.dev new file mode 100644 index 000000000..8e643a876 --- /dev/null +++ b/docker/pypi/wmcore-dev/Dockerfile.dev @@ -0,0 +1,22 @@ +ARG WMCORE_TAG=2.4.2rc8 +ARG PYTHON_VERSION=3.12 +FROM --platform=linux/amd64 registry.cern.ch/cmsweb/wmcore-dev:py${PYTHON_VERSION}-${WMCORE_TAG}-stable +LABEL org.opencontainers.image.authors="Dennis Lee dylee@fnal.gov" + +RUN dnf install -y https://repo.osg-htc.org/osg/24-main/osg-24-main-el9-release-latest.rpm + +RUN dnf install -y \ + osg-ca-certs && \ + dnf clean all + +# Creat wmdev user +RUN useradd -m -u 1000 wmdev && \ + chown -R wmdev /home/cmsbld && \ + chmod -R 755 /home/cmsbld + +USER wmdev + +WORKDIR /workspaces/WMCore + +ENV SHELL=/bin/bash +ENV USER=wmdev diff --git a/docker/pypi/wmcore-dev/Makefile b/docker/pypi/wmcore-dev/Makefile index 524091205..4ef9c0cdd 100644 --- a/docker/pypi/wmcore-dev/Makefile +++ b/docker/pypi/wmcore-dev/Makefile @@ -1,23 +1,22 @@ -VERSION=0.2.0 +WMCORE_TAG=2.4.2rc8 DATE := $(shell date +%Y%m%d) -#DATE = 20241202 build: - docker buildx build --platform=linux/amd64 --tag registry.cern.ch/cmsweb/wmcore-dev . + docker buildx build --platform=linux/amd64 --build-arg PYTHON_VERSION=3.12 --build-arg WMCORE_TAG=$(WMCORE_TAG) --tag registry.cern.ch/cmsweb/wmcore-dev:py3.12-$(WMCORE_TAG) . build-no-cache: - docker buildx build --no-cache --platform=linux/amd64 --tag registry.cern.ch/cmsweb/wmcore-dev . + docker buildx build --no-cache --platform=linux/amd64 --build-arg PYTHON_VERSION=3.12 --build-arg WMCORE_TAG=$(WMCORE_TAG) --tag registry.cern.ch/cmsweb/wmcore-dev:py3.12-$(WMCORE_TAG) . -push: - docker tag registry.cern.ch/cmsweb/wmcore-dev registry.cern.ch/cmsweb/wmcore-dev:$(DATE) - docker push registry.cern.ch/cmsweb/wmcore-dev:$(DATE) - docker tag registry.cern.ch/cmsweb/wmcore-dev registry.cern.ch/cmsweb/wmcore-dev:latest - docker push registry.cern.ch/cmsweb/wmcore-dev:latest +build-dev: + docker buildx build -f ./Dockerfile.dev --platform=linux/amd64 --build-arg PYTHON_VERSION=3.12 --build-arg WMCORE_TAG=$(WMCORE_TAG) --tag registry.cern.ch/cmsweb/wmcore-development:py3.12-${WMCORE_TAG} . push-preprod: - docker tag registry.cern.ch/cmsweb/wmcore-dev registry.cern.ch/cmsweb/wmcore-dev:$(VERSION) - docker push registry.cern.ch/cmsweb/wmcore-dev:$(VERSION) + docker push registry.cern.ch/cmsweb/wmcore-dev:py3.12-$(WMCORE_TAG) push-stable: - docker tag registry.cern.ch/cmsweb/wmcore-dev registry.cern.ch/cmsweb/wmcore-dev:$(VERSION)-stable - docker push registry.cern.ch/cmsweb/wmcore-dev:$(VERSION)-stable \ No newline at end of file + docker tag registry.cern.ch/cmsweb/wmcore-dev:py3.12-$(WMCORE_TAG) registry.cern.ch/cmsweb/wmcore-dev:py3.12-$(WMCORE_TAG)-stable + docker push registry.cern.ch/cmsweb/wmcore-dev:py3.12-$(WMCORE_TAG)-stable + +push-dev: + docker tag registry.cern.ch/cmsweb/wmcore-development:py3.12-${WMCORE_TAG} registry.cern.ch/cmsweb/wmcore-development:py3.12-${WMCORE_TAG}-stable + docker push registry.cern.ch/cmsweb/wmcore-development:py3.12-${WMCORE_TAG} diff --git a/docker/pypi/wmcore-dev/README.md b/docker/pypi/wmcore-dev/README.md index 507586f9f..ef2505948 100644 --- a/docker/pypi/wmcore-dev/README.md +++ b/docker/pypi/wmcore-dev/README.md @@ -5,12 +5,31 @@ This image is used for running unit tests and linting for the [WMCore](https://g ## Usage 1. To build the image, run `make build` -2. Push to the CERN registry with `make push`, `make push-preprod`, or `make push-stable` + * `make build` builds the `wmcore-dev:` tagged image using `Dockerfile` + * `make build-no-cache` builds the `wmcore-dev:` tagged image using `Dockerfile` without using the Docker cache + * `make-build-dev` builds the `wmcore-develpment:` image using `Dockerfile.dev` +2. Push to the CERN registry with `make push-preprod`, `make push-stable`, or `make push-dev` * Special account privileges are needed to push to the CERN registry. - * `*-stable` tagged releases are not automatically cleared from the registry. * Currently, the date is used as a tag for `make push`, while `make push-preprod` and `make push-stable` uses a version number. - * `make push-preprod`: Pushes the current version - * `make push-stable`: Pushes the current version, appended with `-stable` + * `make push-preprod` pushes the image tagged in `make build` + * `make push-stable` retags the image in `make build` by adding a `-stable` suffix + * `*-stable` tagged releases are not automatically cleared from the registry. + * `make push-dev` pushes the `wmcore-development` image built with `make build-dev` + +## Dockerfiles +* `Dockerfile`: Used to build the image for [WMCore-Jenkins](https://github.com/dmwm/WMCore-Jenkins) +* `Dockerfile-dev`: Used to build the image for [wmcore-devcontainer](https://github.com/d-ylee/WMCore-Jenkins) + +## Environment Variables +These are the environment variables used in `entrypoint.sh`. + +| Variable | Description | +| :------------------- | :---------------------------------------------------------------------------------- | +| `BUILD_ID` | Set when run from Jenkins. Indicates whether we are in Jenkins or in developer mode | +| `MY_ID`, `MY_GROUP` | Used in Jenkins. The id and group of the user running tests | +| `WMCORE_ORG` | default: `dmwm`, the organization/user of the cloned WMCore repository | +| `WMCORE_JENKINS_REF` | default: `main`, the ref of the WMCore-Jenkins repository to pull scripts from | +| `WMCORE_JENKINS_ORG` | default: `dmwm`, the organization/user of the cloned WMCore-Jenkins repository | ## Maintainers [@d-ylee](https://github.com/d-ylee) diff --git a/docker/pypi/wmcore-dev/TestScripts/env_unittest.sh b/docker/pypi/wmcore-dev/TestScripts/env_unittest.sh deleted file mode 100644 index 9ae5ff79a..000000000 --- a/docker/pypi/wmcore-dev/TestScripts/env_unittest.sh +++ /dev/null @@ -1,60 +0,0 @@ -#!/bin/bash - -export USER=`whoami` -export BASE_DIR=$PWD -export TEST_DIR=$BASE_DIR/wmcore_unittest -export TEST_SRC=$TEST_DIR/WMCore/src -export TEST_SRC_PYTHON=$TEST_SRC/python -export INSTALL_DIR=$BASE_DIR/unittestdeploy/wmagent -export ADMIN_DIR=/data/admin/wmagent -export CERT_DIR=/data/certs - -#export ORG_SRC_PYTHON=$INSTALL_DIR/current/apps/wmagentpy3/lib/python3.8/site-packages/ -#export ORG_SRC_OTHER=$INSTALL_DIR/current/apps/wmagentpy3/data -#export DBSOCK=$INSTALL_DIR/current/install/mysql/logs/mysql.sock - -# TODO: Somewhat redundant in TestScripts/test-wmcorepy3.sh -export DATABASE=mysql://${MDB_USER}:${MDB_PASS}@127.0.0.1/${MDB_UNITTEST_DB} -export COUCHURL=http://unittestagent:passwd@localhost:6994 -export DIALECT=MySQL - -#rm -rf $ORG_SRC_PYTHON/* - -# ln -s $TEST_SRC_PYTHON/WMCore/ $ORG_SRC_PYTHON -# ln -s $TEST_SRC_PYTHON/WMComponent/ $ORG_SRC_PYTHON -# ln -s $TEST_SRC_PYTHON/PSetTweaks/ $ORG_SRC_PYTHON -# ln -s $TEST_SRC_PYTHON/WMQuality/ $ORG_SRC_PYTHON -# ln -s $TEST_SRC_PYTHON/Utils/ $ORG_SRC_PYTHON - -# rm -rf $ORG_SRC_OTHER/* - -# ln -s $TEST_SRC/couchapps/ $ORG_SRC_OTHER -# ln -s $TEST_SRC/css/ $ORG_SRC_OTHER -# ln -s $TEST_SRC/html/ $ORG_SRC_OTHER -# ln -s $TEST_SRC/javascript/ $ORG_SRC_OTHER -# ln -s $TEST_SRC/template/ $ORG_SRC_OTHER - -export WMAGENT_SECRETS_LOCATION=$ADMIN_DIR/WMAgent.secrets -export X509_HOST_CERT=$CERT_DIR/servicecert.pem -export X509_HOST_KEY=$CERT_DIR/servicekey.pem -export X509_USER_CERT=$CERT_DIR/servicecert.pem -export X509_USER_KEY=$CERT_DIR/servicekey.pem - -# export install=$INSTALL_DIR/current/install/wmagentpy3 -# export config=$INSTALL_DIR/current/config/wmagentpy3 -# export manage=$config/manage - -# source $INSTALL_DIR/current/apps/wmagentpy3/etc/profile.d/init.sh -# source $INSTALL_DIR/current/apps/wmcorepy3-devtools/etc/profile.d/init.sh - -### some Rucio setup needed for jenkins and docker unit tests -# fetch the values defined in the secrets file and update rucio.cfg file -export RUCIO_HOME=$BASE_DIR # TODO: Change to specific rucio directory -MATCH_RUCIO_HOST=`cat $WMAGENT_SECRETS_LOCATION | grep RUCIO_HOST | sed s/RUCIO_HOST=//` -MATCH_RUCIO_AUTH=`cat $WMAGENT_SECRETS_LOCATION | grep RUCIO_AUTH | sed s/RUCIO_AUTH=//` -sed "s+^rucio_host.*+rucio_host = $MATCH_RUCIO_HOST+" $RUCIO_HOME/etc/rucio.cfg-temp > $RUCIO_HOME/etc/rucio.cfg -sed "s+^auth_host.*+auth_host = $MATCH_RUCIO_AUTH+" $RUCIO_HOME/etc/rucio.cfg-temp > $RUCIO_HOME/etc/rucio.cfg -echo "Updated RUCIO_HOME file under: $RUCIO_HOME" - -export PYTHONPATH=/home/cmsbld/WMCore/test/python:$PYTHONPATH -export PYTHONPATH=/home/cmsbld/WMCore/src/python:$PYTHONPATH \ No newline at end of file diff --git a/docker/pypi/wmcore-dev/TestScripts/pylint3kWMCore.sh b/docker/pypi/wmcore-dev/TestScripts/pylint3kWMCore.sh deleted file mode 100755 index ecc1072f6..000000000 --- a/docker/pypi/wmcore-dev/TestScripts/pylint3kWMCore.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/usr/bin/env bash - -# Run pylint over the entire WMCore code base for compatibility checks - -source ./env_unittest.sh -pushd wmcore_unittest/WMCore -export PYTHONPATH=`pwd`/test/python:`pwd`/src/python:$PYTHONPATH - -git checkout master -git pull origin - -# Run the python3 compatibility checkers in python -echo "Printing Pylint version" -pylint --version - -# Disable: [W1618(no-absolute-import), ] import missing `from __future__ import absolute_import` -pylint --py3k -j 2 -f parseable -d W1618 src/python/* test/python/* diff --git a/docker/pypi/wmcore-dev/TestScripts/pylintPRTest.sh b/docker/pypi/wmcore-dev/TestScripts/pylintPRTest.sh deleted file mode 100755 index 31e0f543d..000000000 --- a/docker/pypi/wmcore-dev/TestScripts/pylintPRTest.sh +++ /dev/null @@ -1,61 +0,0 @@ -#!/usr/bin/env bash - -WORKDIR=/home/cmsbld - -if [ -z "$ghprbPullId" -o -z "$ghprbTargetBranch" ]; then - echo "Not all necessary environment variables set: ghprbPullId, ghprbTargetBranch" - exit 1 -fi - -echo "Executing Pylint for PR ID $ghprbPullId and target branch $ghprbTargetBranch" - -# Setup the environment -echo "Sourcing a python3 unittest environment" -source $WORKDIR/TestScripts/env_unittest.sh -JSON_FILENAME=pylintpy3Report.json -PEP8_FILENAME=pep8py3.txt - -pushd $WORKDIR/WMCore -export PYTHONPATH=`pwd`/test/python:`pwd`/src/python:$PYTHONPATH - -# Figure out the one commit we are interested in and what happens to the repo if we were to merge it -git config remote.origin.url https://github.com/dmwm/WMCore.git -git fetch origin pull/${ghprbPullId}/merge:PR_MERGE -export COMMIT=`git rev-parse "PR_MERGE^{commit}"` -git checkout ${ghprbTargetBranch} -git pull - -# Which python files changed? -git diff --name-only ${ghprbTargetBranch}..${COMMIT} > allChangedFiles.txt -$WORKDIR/ContainerScripts/IdentifyPythonFiles.py allChangedFiles.txt > changedFiles.txt - -echo "Printing Pylint version" -pylint --version - -# Get pylint report for master -git checkout -f $ghprbTargetBranch -echo "{}" > pylintReport.json -echo "*** Running Pylint on the changed files against the $ghprbTargetBranch" -while read name; do - pylint --evaluation='10.0 - ((float(5 * error + warning) / statement) * 10)' --rcfile standards/.pylintrc --msg-template='{path}:{line}: [{msg_id}({symbol}), {obj}] {msg}' $name > pylint.out || true - $WORKDIR/ContainerScripts/AggregatePylint.py base -done pylint.out || true - $WORKDIR/ContainerScripts/AggregatePylint.py test -done ${PEP8_FILENAME} -cp ${PEP8_FILENAME} $WORKDIR/artifacts/ - -popd diff --git a/docker/pypi/wmcore-dev/TestScripts/pylintWMCore.sh b/docker/pypi/wmcore-dev/TestScripts/pylintWMCore.sh deleted file mode 100755 index ccec534bb..000000000 --- a/docker/pypi/wmcore-dev/TestScripts/pylintWMCore.sh +++ /dev/null @@ -1,37 +0,0 @@ -#!/usr/bin/env bash - -# Run pylint and pep8 (pycodestyle) over the entire WMCore code base - -# Setup the environment -if [[ -f env_unittest_py3.sh ]] -then - echo "Sourcing a python3 unittest environment" - source env_unittest_py3.sh - OUT_FILENAME=pylintpy3.txt -else - echo "Sourcing a python2 unittest environment" - source env_unittest.sh - OUT_FILENAME=pylint.txt -fi - -pushd wmcore_unittest/WMCore -export PYTHONPATH=`pwd`/test/python:`pwd`/src/python:$PYTHONPATH - -git checkout master -git pull origin - -echo "Printing Pylint version" -pylint --version - -# Run pylint on the whole code base -pylint --rcfile=standards/.pylintrc -j 2 -f parseable src/python/* test/python/* - -# Fix pep8 which has the wrong python executable -echo "#! /usr/bin/env python" > ../pep8 -cat `which pep8` >> ../pep8 -chmod +x ../pep8 - -# Run PEP-8 checker but not in pylint format -pycodestyle --format=default --exclude=test/data,.svn,CVS,.bzr,.hg,.git,__pycache__,.tox. - -cp ${PEP8_FILENAME} ${HOME}/artifacts/ \ No newline at end of file diff --git a/docker/pypi/wmcore-dev/TestScripts/runPath.sh b/docker/pypi/wmcore-dev/TestScripts/runPath.sh deleted file mode 100755 index 402a585bd..000000000 --- a/docker/pypi/wmcore-dev/TestScripts/runPath.sh +++ /dev/null @@ -1,41 +0,0 @@ -#! /bin/bash - -echo Running tests in path $1 -set -x - -# Make sure we the certs we use are readable by us and only us - -/home/dmwm/ContainerScripts/fixCertificates.sh - -# Start up services (Couch and MySQL) - -source ./env_unittest.sh -$manage start-services - -cd /home/dmwm/wmcore_unittest/WMCore/ - -# Make sure we base our tests on the latest Jenkins-tested master - -/home/dmwm/ContainerScripts/updateGit.sh - -export LATEST_TAG=`git tag |grep JENKINS| sort | tail -1` -if [ -z "$2" ]; then - export COMMIT=$LATEST_TAG -else - export COMMIT=`git rev-parse "origin/pr/$3/merge^{commit}"` -fi - -# First try to merge this PR into the same tag used for the baseline -# Finally give up and just test the tip of the branch -(git checkout $LATEST_TAG && git merge $COMMIT) || git checkout -f $COMMIT - -# Run tests and watchdog to shut it down if needed - -cd /home/dmwm/wmcore_unittest/WMCore/ -rm test/python/WMCore_t/REST_t/*_t.py -/home/dmwm/cms-bot/DMWM/TestWatchdog.py & -python setup.py test --buildBotMode=true --reallyDeleteMyDatabaseAfterEveryTest=true --testCertainPath=$1 --testTotalSlices=1 --testCurrentSlice=0 - -# Save the results - -cp nosetests.xml /home/dmwm/artifacts/nosetests-path.xml diff --git a/docker/pypi/wmcore-dev/TestScripts/runSlice.sh b/docker/pypi/wmcore-dev/TestScripts/runSlice.sh deleted file mode 100755 index 641a3a10b..000000000 --- a/docker/pypi/wmcore-dev/TestScripts/runSlice.sh +++ /dev/null @@ -1,50 +0,0 @@ -#! /bin/bash - -if [ -z "$1" -o -z "$2" ]; then - echo "Not all necessary environment variables set: Two parameters for slice and number of slices" - exit 1 -fi - -echo Running slice $1 of $2 - -# Make sure we the certs we use are readable by us and only us - -/home/dmwm/ContainerScripts/fixCertificates.sh - -# Start up services (Couch and MySQL) - -source ./env_unittest.sh -$manage start-services - -pushd /home/dmwm/wmcore_unittest/WMCore/ - -# Make sure we base our tests on the latest Jenkins-tested master -# sometimes GitHub has issues, so try each command twice - -timeout -s 9 5m git fetch --tags || timeout -s 9 5m git fetch --tags -timeout -s 9 5m git pull || timeout -s 9 5m git fetch --tags -export LATEST_TAG=`git tag |grep JENKINS| sort | tail -1` - -# Find the commit that represents the tip of the PR the latest tag -if [ -z "$ghprbPullId" ]; then - export COMMIT=$LATEST_TAG -else - git config remote.origin.url https://github.com/dmwm/WMCore.git - timeout -s 9 5m git fetch origin pull/${ghprbPullId}/merge:PR_MERGE - export COMMIT=`git rev-parse "PR_MERGE^{commit}"` -fi - -# First try to merge this PR into the same tag used for the baseline -# If it doesn't merge, just test the tip of the branch -(git checkout $LATEST_TAG && git merge $COMMIT) || git checkout -f $COMMIT - -# Run tests and watchdog to shut it down if needed -export USER=`whoami` -/home/dmwm/cms-bot/DMWM/TestWatchdog.py & -timeout 80m python setup.py test --buildBotMode=true --reallyDeleteMyDatabaseAfterEveryTest=true --testCertainPath=test/python --testTotalSlices=$2 --testCurrentSlice=$1 - -# Save the results - -cp nosetests.xml /home/dmwm/artifacts/nosetests-$1.xml - -popd diff --git a/docker/pypi/wmcore-dev/TestScripts/test-wmcorepy3.sh b/docker/pypi/wmcore-dev/TestScripts/test-wmcorepy3.sh deleted file mode 100755 index 986bc4fbb..000000000 --- a/docker/pypi/wmcore-dev/TestScripts/test-wmcorepy3.sh +++ /dev/null @@ -1,104 +0,0 @@ -#! /bin/bash -e - -WORKDIR=/home/cmsbld -CODE=$WORKDIR/WMCore - -pushd $WORKDIR - -start=`date +%s` - -set +x -# load environment -echo "Sourcing a python3 unittest environment" -. $WORKDIR/TestScripts/env_unittest.sh - -# load wmagent secrets -. /data/admin/wmagent/WMAgent.secrets - -# Load CMS environment from CVMFS -source /cvmfs/cms.cern.ch/cmsset_default.sh - -# This is what the production wmagent-mariadb image defines (via WMA_DATABASE) has as of 2024/07/15 -# TODO: We should either make an option in the mariadb image to define the default WMA_DATABASE or make a custom test mariadb image -MDB_UNITTEST_DB=wmagent - -export DATABASE=mysql://${MDB_USER}:${MDB_PASS}@127.0.0.1/${MDB_UNITTEST_DB} -export COUCHURL="http://${COUCH_USER}:${COUCH_PASS}@${COUCH_HOST}:${COUCH_PORT}" - -# ensure db exists -# retry if fails - - -for i in 1 2 3 4 5; do - mysql -u ${MDB_USER} -h 127.0.0.1 -p${MDB_PASS} --execute "CREATE DATABASE IF NOT EXISTS ${MDB_UNITTEST_DB}" && break - - echo "Attempt $i failed. Trying again..." - sleep 5 -done - -# rucio -export RUCIO_HOST=$RUCIO_HOST -export RUCIO_AUTH=$RUCIO_AUTH -set -x - -pushd $CODE - -git pull origin master - -# use ghprbPullId if triggered from a PR -if [[ ! -z "${ghprbPullId}" ]]; then - git fetch --tags https://github.com/dmwm/WMCore.git "+refs/heads/*:refs/remotes/origin/*" - git config remote.origin.url https://github.com/dmwm/WMCore.git - git config --add remote.origin.fetch "+refs/heads/*:refs/remotes/origin/*" - git fetch --tags --quiet https://github.com/dmwm/WMCore.git "+refs/pull/*:refs/remotes/origin/pr/*" - export COMMIT=`git rev-parse "origin/pr/$ghprbPullId/merge^{commit}"` - export LATEST_TAG=`git tag |grep JENKINS| sort | tail -1` - - # First try to merge this PR into the same tag used for the baseline - # Next try to merge this tag onto current master - # Finally give up and just test the tip of the branch - (git checkout $LATEST_TAG && git merge $COMMIT) || (git checkout master && git merge $COMMIT) || git checkout -f $COMMIT -fi - -popd - -### Some tweaks for the nose run (in practice, there is nothing to change in setup_test.py...) -# working dir includes entire python source - ignore -perl -p -i -e 's/--cover-inclusive//' $CODE/setup_test.py -# cover branches but not external python modules -perl -p -i -e 's/--cover-inclusive/--cover-branches/' $CODE/setup_test.py -perl -p -i -e "s/'--cover-html',//" $CODE/setup_test.py - -#export NOSE_EXCLUDE='AlertGenerator' -# timeout tests after 5 mins -#export NOSE_PROCESSES=1 -#export NOSE_PROCESS_TIMEOUT=300 -#export NOSE_PROCESS_RESTARTWORKER=1 - -# include FWCore.ParameterSet.Config - -export PYTHONPATH=/var/lib/jenkins/additional-library:$PYTHONPATH - -# remove old coverage data -coverage erase - -# debugging python interpreters -echo "Python version is: " && python --version || true -echo "Python3 version is: " && python3 --version || true - -# run test - force success though - failure stops coverage report -rm nosetests*.xml || true -# FIXME Alan on 25/may/2021: ImportError: cannot import name _psutil_linux -#python3 cms-bot/DMWM/TestWatchdog.py & - -python3 $CODE/setup.py test --buildBotMode=true --reallyDeleteMyDatabaseAfterEveryTest=true --testCertainPath=$CODE/test/python --testTotalSlices=$SLICES --testCurrentSlice=$SLICE || true #--testCertainPath=test/python/WMCore_t/WMBS_t || true -mv nosetests.xml artifacts/nosetestspy3-$SLICE-$BUILD_ID.xml - -# Add these here as they need the same environment as the main run -#FIXME: change so initial coverage command skips external code -# coverage xml -i --include=$PWD/install* || true - -end=`date +%s` -runtime=$((end-start)) - -echo "Total time to test slice $SLICE: $runtime" diff --git a/docker/pypi/wmcore-dev/entrypoint.sh b/docker/pypi/wmcore-dev/entrypoint.sh index 7bb5e6dc8..0a0be4635 100755 --- a/docker/pypi/wmcore-dev/entrypoint.sh +++ b/docker/pypi/wmcore-dev/entrypoint.sh @@ -5,25 +5,23 @@ if [[ -z "${BUILD_ID}" ]]; then echo "No BUILD_ID set, we in dev mode" else echo "BUILD_ID set, setting up for CI/CD" - # give proper permissions to home directory - #chown -R ${MY_ID}:${MY_GROUP} /home/cmsbld - - #USERN=$(id -un ${MY_ID}) pushd /home/cmsbld # clone main repo - git clone https://github.com/dmwm/WMCore + git clone https://github.com/${WMCORE_ORG:-dmwm}/WMCore # clone jenkins-test scripts - git clone https://github.com/dmwm/WMCore-Jenkins + git clone --depth 1 --branch ${WMCORE_JENKINS_REF:-main} https://github.com/${WMCORE_JENKINS_ORG:-dmwm}/WMCore-Jenkins + + # copy scripts + cp -r WMCore-Jenkins/{TestScripts,ContainerScripts} . - # TODO: Move scripts to a more sensible location in WMCore-Jenkins - cp -r WMCore-Jenkins/docker/wmcore-dev/TestScripts . - cp -r WMCore-Jenkins/docker/wmcore-dev/ContainerScripts . - cp -r WMCore-Jenkins/docker/wmcore-dev/etc . popd + # give proper ownership and permissions to home directory + chown -R ${MY_ID}:${MY_GROUP} /home/cmsbld + fi -$@ \ No newline at end of file +$@