diff --git a/docker/cmsweb-alma9-base/Dockerfile b/docker/cmsweb-alma9-base/Dockerfile new file mode 100644 index 000000000..269e7f0ea --- /dev/null +++ b/docker/cmsweb-alma9-base/Dockerfile @@ -0,0 +1,23 @@ +FROM cern/alma9-base:latest +LABEL author="Dennis Lee dylee@fnal.gov" + +# Install EPEL repository (required for voms, fetch-crl and CA-related packages) +RUN dnf -y install epel-release && dnf -y upgrade && \ + dnf install -y http://linuxsoft.cern.ch/wlcg/el9/x86_64/wlcg-repo-1.0.0-1.el9.noarch.rpm && \ + dnf clean all + +ADD http://repository.egi.eu/sw/production/cas/1/current/repo-files/egi-trustanchors.repo /etc/yum.repos.d/egi.repo + +# Upgrade packages from the base image and install CMSWEB required packages +RUN dnf -y install fetch-crl cronie cern-get-certificate CERN-CA-certs ca-certificates \ + dummy-ca-certs ca-policy-lcg ca-policy-egi-core \ + ca_CERN-GridCA ca_CERN-Root-2 \ + wlcg-voms-cms && \ + dnf clean all && \ + echo "32 */6 * * * root ! /usr/sbin/fetch-crl -q -r 360" > /etc/cron.d/fetch-crl-docker + +# Required OS packages +RUN dnf -y install vim less procps python3-pycurl pip && dnf clean all +RUN ln -s /usr/bin/python3 /usr/bin/python + +RUN update-ca-trust diff --git a/docker/cmsweb-base/Dockerfile b/docker/cmsweb-base/Dockerfile index 7f37ac338..1ffd1eabd 100644 --- a/docker/cmsweb-base/Dockerfile +++ b/docker/cmsweb-base/Dockerfile @@ -7,7 +7,10 @@ ADD slc7-cernonly.repo /etc/yum.repos.d/slc7-cernonly.repo # see https://developers.redhat.com/blog/2016/03/09/more-about-docker-images-size/ RUN yum install -y sudo cern-get-certificate fetch-crl \ CERN-CA-certs ca-certificates dummy-ca-certs ca-policy-lcg ca-policy-egi-core \ - ca_EG-GRID ca_CERN-GridCA ca_CERN-LCG-IOTA-CA ca_CERN-Root-2 \ - && yum install -y --nogpgcheck wlcg-voms-cms && yum clean all && rm -rf /var/cache/yum && ln -s /bin/bash /usr/bin/bashs && echo "32 */6 * * * root ! /usr/sbin/fetch-crl -q -r 360" > /etc/cron.d/fetch-crl-docker + ca_EG-GRID ca_CERN-GridCA ca_CERN-LCG-IOTA-CA ca_CERN-Root-2 && \ + yum install -y --nogpgcheck wlcg-voms-cms && \ + yum clean all && rm -rf /var/cache/yum && \ + ln -s /bin/bash /usr/bin/bashs && \ + echo "32 */6 * * * root ! /usr/sbin/fetch-crl -q -r 360" > /etc/cron.d/fetch-crl-docker RUN update-ca-trust diff --git a/docker/pypi/alma-base/Dockerfile b/docker/pypi/alma-base/Dockerfile deleted file mode 100644 index 1b2148e9b..000000000 --- a/docker/pypi/alma-base/Dockerfile +++ /dev/null @@ -1,4 +0,0 @@ -FROM almalinux:latest -MAINTAINER Valentin Kuznetsov vkuznet@gmail.com -RUN yum install -y curl-minimal libcurl-minimal vim python3-pycurl pip sudo less \ - && yum clean all && rm -rf /var/cache/yum diff --git a/docker/pypi/alma-base/errors.txt b/docker/pypi/alma-base/errors.txt deleted file mode 100644 index 70e1e2973..000000000 --- a/docker/pypi/alma-base/errors.txt +++ /dev/null @@ -1,14 +0,0 @@ -Step 3/3 : RUN yum install -y curl vim python3 pip sudo less && yum clean all && rm -rf /var/cache/yum - ---> Running in 77dbadded671 -AlmaLinux 9 - AppStream 12 MB/s | 9.1 MB 00:00 -AlmaLinux 9 - BaseOS 12 MB/s | 4.7 MB 00:00 -AlmaLinux 9 - Extras 47 kB/s | 17 kB 00:00 -Package python3-3.9.18-1.el9_3.x86_64 is already installed. -Package less-590-2.el9_2.x86_64 is already installed. -Error: - Problem: problem with installed package curl-minimal-7.76.1-26.el9_3.2.x86_64 - - package curl-minimal-7.76.1-26.el9_3.2.x86_64 from @System conflicts with curl provided by curl-7.76.1-26.el9_3.2.x86_64 from baseos - - package curl-minimal-7.76.1-26.el9.x86_64 from baseos conflicts with curl provided by curl-7.76.1-26.el9_3.2.x86_64 from baseos - - package curl-minimal-7.76.1-26.el9_3.2.x86_64 from baseos conflicts with curl provided by curl-7.76.1-26.el9_3.2.x86_64 from baseos - - cannot install the best candidate for the job -(try to add '--allowerasing' to command line to replace conflicting packages or '--skip-broken' to skip uninstallable packages or '--nobest' to use not only best candidate packages) diff --git a/docker/pypi/dmwm-alma9-base/Dockerfile b/docker/pypi/dmwm-alma9-base/Dockerfile new file mode 100644 index 000000000..45057b253 --- /dev/null +++ b/docker/pypi/dmwm-alma9-base/Dockerfile @@ -0,0 +1,26 @@ +FROM registry.cern.ch/cmsweb/cmsweb-alma9-base:20250529 AS cmsweb-base +FROM registry.cern.ch/cmsweb/exporters AS exporters +FROM almalinux:latest +LABEL org.opencontainers.image.authors="Alan Malta alan.malta@cern.ch, Dennis Lee dylee@fnal.gov" + +# base image stuff: certificates, monitoring, exporters, etc +RUN mkdir /etc/grid-security +COPY --from=cmsweb-base /etc/grid-security/certificates /etc/grid-security/certificates +COPY --from=cmsweb-base /etc/grid-security/vomsdir /etc/grid-security/vomsdir +COPY --from=exporters /data/cmsweb-ping /usr/bin/cmsweb-ping +COPY --from=exporters /data/process_exporter /usr/bin/process_exporter +COPY --from=exporters /data/cpy_exporter /usr/bin/cpy_exporter + +# Required OS packages +RUN dnf -y upgrade && \ + dnf -y install --skip-broken curl libcurl && \ + dnf -y install sudo vim less procps && \ + dnf -y install python3-pycurl pip && \ + dnf clean all +RUN ln -s /usr/bin/python3.9 /usr/bin/python + +ENV WDIR=/data +ADD run.sh $WDIR/run.sh +ADD monitor.sh $WDIR/monitor.sh +ADD manage $WDIR/manage +WORKDIR /data diff --git a/docker/pypi/dmwm-alma9-base/Dockerfile.py312 b/docker/pypi/dmwm-alma9-base/Dockerfile.py312 new file mode 100644 index 000000000..a994f0c43 --- /dev/null +++ b/docker/pypi/dmwm-alma9-base/Dockerfile.py312 @@ -0,0 +1,26 @@ +FROM registry.cern.ch/cmsweb/cmsweb-alma9-base:20250529 AS cmsweb-base +FROM registry.cern.ch/cmsweb/exporters AS exporters +FROM almalinux:latest +LABEL org.opencontainers.image.authors="Alan Malta alan.malta@cern.ch" + +# base image stuff: certificates, monitoring, exporters, etc +RUN mkdir /etc/grid-security +COPY --from=cmsweb-base /etc/grid-security/certificates /etc/grid-security/certificates +COPY --from=cmsweb-base /etc/grid-security/vomsdir /etc/grid-security/vomsdir +COPY --from=exporters /data/cmsweb-ping /usr/bin/cmsweb-ping +COPY --from=exporters /data/process_exporter /usr/bin/process_exporter +COPY --from=exporters /data/cpy_exporter /usr/bin/cpy_exporter + +# Required OS packages +RUN dnf -y upgrade && \ + dnf -y install --skip-broken curl libcurl && \ + dnf -y install sudo vim less procps && \ + dnf -y install python3.12 python3.12-pip python3-pycurl pip && \ + dnf clean all +RUN ln -s /usr/bin/python3.12 /usr/bin/python + +ENV WDIR=/data +ADD run.sh $WDIR/run.sh +ADD monitor.sh $WDIR/monitor.sh +ADD manage $WDIR/manage +WORKDIR /data \ No newline at end of file diff --git a/docker/pypi/dmwm-alma9-base/manage b/docker/pypi/dmwm-alma9-base/manage new file mode 100755 index 000000000..044d271d9 --- /dev/null +++ b/docker/pypi/dmwm-alma9-base/manage @@ -0,0 +1,105 @@ +#!/bin/bash +##H Usage: manage ACTION [ATTRIBUTE] [SECURITY-STRING] +##H +##H Available actions: +##H help show this help +##H version get current version of the service +##H status show current service's status +##H restart (re)start the service +##H start (re)start the service +##H stop stop the service + +# common settings to prettify output +echo_e=-e +COLOR_OK="\\033[0;32m" +COLOR_WARN="\\033[0;31m" +COLOR_NORMAL="\\033[0;39m" + +# service settings +srv=`echo $USER | sed -e "s,_,,g" | sed -e "s,t0req,t0_req,g"` +LOGDIR=/data/srv/logs/$srv +AUTHDIR=/data/srv/current/auth/$srv +STATEDIR=/data/srv/state/$srv +CFGDIR=/data/srv/current/config/$srv +CFGFILE=$CFGDIR/config.py +# some MS services uses different config naming convention, therefore we'll +# adjust CFGFILE assingment +for c in monitor output ruleCleaner transferor unmerged; do + if [ -f $CFGDIR/config-${c}.py ]; then + CFGFILE=$CFGDIR/config-${c}.py + fi +done + +# necessary env settings for all WM services +export PYTHONPATH=$PYTHONPATH:/etc/secrets:/data/srv/current/config/$srv +export X509_USER_KEY=$AUTHDIR/dmwm-service-key.pem +export X509_USER_CERT=$AUTHDIR/dmwm-service-cert.pem +export REQMGR_CACHE_DIR=$STATEDIR +export WMCORE_CACHE_DIR=$STATEDIR +# MSUnmerged also needs to access a proxy with additional voms roles +if [ -f $AUTHDIR/proxy.cert ]; then + export X509_USER_PROXY=$AUTHDIR/proxy.cert +fi + +# by default Rucio relies on /opt/rucio/etc/config.cfg +# if necessary we may setup RUCIO_HOME which should provide this location +# but in k8s we mount rucio config.cfg under /opt/rucio/etc area + +usage() +{ + cat $0 | grep "^##H" | sed -e "s,##H,,g" +} + +start_srv() +{ + wmc-httpd -r -d $STATEDIR -l "$LOGDIR/$srv-`hostname -s`.log" $CFGFILE +} + +stop_srv() +{ + local pid=`ps auxwww | egrep "wmc-httpd" | grep -v grep | awk 'BEGIN{ORS=" "} {print $2}'` + echo "Stop $srv service... ${pid}" + if [ -n "${pid}" ]; then + kill -9 ${pid} + fi +} + +status_srv() +{ + local pid=`ps auxwww | egrep "wmc-httpd" | grep -v grep | awk 'BEGIN{ORS=" "} {print $2}'` + if [ -z "${pid}" ]; then + echo "$srv service is not running" + return + fi + if [ ! -z "${pid}" ]; then + echo $echo_e "$srv service is ${COLOR_OK}RUNNING${COLOR_NORMAL}, PID=${pid}" + ps -f -wwww -p ${pid} + else + echo $echo_e "$srv service is ${COLOR_WARN}NOT RUNNING${COLOR_NORMAL}" + fi +} + +# Main routine, perform action requested on command line. +case ${1:-status} in + start | restart ) + stop_srv + start_srv + ;; + + status ) + status_srv + ;; + + stop ) + stop_srv + ;; + + help ) + usage + ;; + + * ) + echo "$0: unknown action '$1', please try '$0 help' or documentation." 1>&2 + exit 1 + ;; +esac diff --git a/docker/pypi/dmwm-alma9-base/monitor.sh b/docker/pypi/dmwm-alma9-base/monitor.sh new file mode 100755 index 000000000..d2b22a343 --- /dev/null +++ b/docker/pypi/dmwm-alma9-base/monitor.sh @@ -0,0 +1,26 @@ +#!/bin/bash + +echo -e "\nTrying to start process_exporter..." +# start process exporter +configs="config config-monitor config-output config-transferor config-ruleCleaner config-unmerged" +for p in $configs; do + if [ -f /etc/secrets/${p}.py ]; then + echo " Using configuration file: /etc/secrets/${p}.py" + pat="wmc-httpd.*$p" + pid=`ps axjfwww | grep "$pat" | grep -v grep | grep -v process_monitor | grep -v " 1 " | awk '{print $1}'` + if [ -n "$pid" ]; then + app=`grep ^main.application /etc/secrets/${p}.py | grep -v application_dir | sed -e 's,#.*,,g' | awk '{split($0,a,"="); print a[2]}' | sed -e "s, ,,g" -e 's,",,g' -e "s,-,_,g"` + echo " Using PID: $pid and app name: '$app'" + if [ -n "$app" ]; then + prefix=${app} + port=`grep main.port /etc/secrets/${p}.py | sed -e 's,#.*,,g' | awk '{split($0,a,"="); print a[2]}' | sed -e "s, ,,g"` + address=":1${port}" + echo " Starting process_exporter with prefix ${prefix} on ${address}" + nohup process_exporter -pid $pid -prefix $prefix -address "$address" 2>&1 1>& ${prefix}.log < /dev/null & + #cpyAddr=`echo ${address} | sed "s,8,9,g"` + #echo "Start cpy_exporter on ${cpyAddr}" + #nohup cpy_exporter -address "$address" 2>&1 1>& cpy_${prefix}.log < /dev/null & + fi + fi + fi +done diff --git a/docker/pypi/dmwm-alma9-base/run.sh b/docker/pypi/dmwm-alma9-base/run.sh new file mode 100755 index 000000000..dce420d92 --- /dev/null +++ b/docker/pypi/dmwm-alma9-base/run.sh @@ -0,0 +1,86 @@ +#!/bin/bash +# script to start ReqMgr2 + +srv=`echo $USER | sed -e "s,_,,g"` +STATEDIR=/data/srv/state/$srv +LOGDIR=/data/srv/logs/$srv +AUTHDIR=/data/srv/current/auth/$srv +CONFIGDIR=/data/srv/current/config/$srv +CONFIGFILE=${CONFIGFILE:-config.py} +CFGFILE=/etc/secrets/$CONFIGFILE + +### permission update to workaround issues with mounting logs volume +sudo chown -R $USER.$USER /data + +mkdir -p $LOGDIR +mkdir -p $STATEDIR +mkdir -p $AUTHDIR +mkdir -p $CONFIGDIR +mkdir -p $AUTHDIR/../wmcore-auth + +# environment variables required to run some of the WMCore services +export REQMGR_CACHE_DIR=$STATEDIR +export WMCORE_CACHE_DIR=$STATEDIR + +# overwrite host PEM files in /data/srv area by the robot certificate +# Note that the proxy file is not required and used +if [ -f /etc/robots/robotkey.pem ]; then + sudo cp /etc/robots/robotkey.pem $AUTHDIR/dmwm-service-key.pem + sudo cp /etc/robots/robotcert.pem $AUTHDIR/dmwm-service-cert.pem + sudo chown $USER.$USER $AUTHDIR/dmwm-service-key.pem + sudo chown $USER.$USER $AUTHDIR/dmwm-service-cert.pem + sudo chmod 0400 $AUTHDIR/dmwm-service-key.pem +fi + +if [ -e $AUTHDIR/dmwm-service-cert.pem ] && [ -e $AUTHDIR/dmwm-service-key.pem ]; then + export X509_USER_CERT=$AUTHDIR/dmwm-service-cert.pem + export X509_USER_KEY=$AUTHDIR/dmwm-service-key.pem +fi + +# overwrite header-auth key file with one from secrets +if [ -f /etc/hmac/hmac ]; then + sudo cp /etc/hmac/hmac $AUTHDIR/../wmcore-auth/header-auth-key + sudo chown $USER.$USER $AUTHDIR/../wmcore-auth/header-auth-key + sudo chmod 0600 $AUTHDIR/../wmcore-auth/header-auth-key +fi + +# use service configuration files from /etc/secrets if they are present +files=`ls /etc/secrets` +for fname in $files; do + if [ -f /etc/secrets/$fname ]; then + if [ -f $CONFIGDIR/$fname ]; then + rm $CONFIGDIR/$fname + fi + sudo cp /etc/secrets/$fname $CONFIGDIR/$fname + sudo chown $USER.$USER $CONFIGDIR/$fname + if [ "$fname" == "$CONFIGFILE" ]; then + CFGFILE=$CONFIGDIR/$CONFIGFILE + fi + fi +done +files=`ls /etc/secrets` +for fname in $files; do + if [ ! -f $CONFIGDIR/$fname ]; then + sudo cp /etc/secrets/$fname $AUTHDIR/$fname + sudo chown $USER.$USER $AUTHDIR/$fname + fi +done + +export PYTHONPATH=$PYTHONPATH:/etc/secrets:$AUTHDIR/$fname + +# backward compatible changes for RPM based deployment location of aux files +if [ -d /usr/local/data ] && [ "$USER" == "_reqmgr2" ]; then + sudo mkdir -p /data/srv/current/apps/reqmgr2 + sudo ln -s /usr/local/data /data/srv/current/apps/reqmgr2 +fi + +# start the service +wmc-httpd -r -d $STATEDIR -l "|rotatelogs $LOGDIR/$srv-%Y%m%d-`hostname -s`.log 86400" $CFGFILE + +# start monitor.sh script +if [ -f /data/monitor.sh ]; then + /data/monitor.sh +fi + +# hack to keep the container running +tail -f /etc/hosts diff --git a/docker/pypi/gfal/Dockerfile.alma9 b/docker/pypi/gfal/Dockerfile.alma9 new file mode 100644 index 000000000..05864f5bd --- /dev/null +++ b/docker/pypi/gfal/Dockerfile.alma9 @@ -0,0 +1,33 @@ +FROM registry.cern.ch/cmsweb/dmwm-base:alma9-py3.12-20250529 +LABEL org.opencontainers.image.authors="Dennis Lee dylee@fnal.gov" + +WORKDIR /tmp +RUN dnf -y upgrade && \ + dnf install epel-release -y && dnf clean all && \ + dnf -y install git bzip2 gfal2-devel python3.12-devel + +RUN git clone --branch v1.13.0 https://github.com/cern-fts/gfal2-python.git + +WORKDIR /tmp/gfal2-python +RUN ./ci/fedora-packages.sh + +WORKDIR /tmp +# get and install boost +# RUN curl -ksL https://archives.boost.io/release/1.82.0/source/boost_1_82_0.tar.bz2 -o boost_1_82_0.tar.bz2 && \ +# tar --bzip2 -xf /tmp/boost_1_82_0.tar.bz2 && \ +# cd boost_1_82_0 && \ +# ./bootstrap.sh --with-python=/usr/bin/python3.12 && \ +# ./b2 install + +RUN git clone https://github.com/boostorg/boost.git -b boost-1.85.0 boost_1_85_0 --depth 1 && \ + cd boost_1_85_0 && \ + git submodule update --depth 1 -q --init tools/boostdep && \ + git submodule update --depth 1 -q --init libs/python && \ + python tools/boostdep/depinst/depinst.py -X test -g "--depth 1" python && \ + ./bootstrap.sh --with-python=/usr/bin/python3.12 && \ + ./b2 install --with-python + +RUN python -m venv .venv +ENV PATH="$WDIR/.venv/bin:$PATH" + +RUN .venv/bin/pip install -v gfal2-python diff --git a/docker/pypi/msunmerged/11922.patch b/docker/pypi/msunmerged/11922.patch new file mode 100644 index 000000000..b13d739b8 --- /dev/null +++ b/docker/pypi/msunmerged/11922.patch @@ -0,0 +1,343 @@ +diff --git a/src/python/WMCore/REST/Main.py b/src/python/WMCore/REST/Main.py +index 82e4e89e7..82921a04d 100644 +--- a/src/python/WMCore/REST/Main.py ++++ b/src/python/WMCore/REST/Main.py +@@ -5,7 +5,11 @@ + + Manages a web server application. Loads configuration and all views, starting + up an appropriately configured CherryPy instance. Views are loaded dynamically +-and can be turned on/off via configuration file.""" ++and can be turned on/off via configuration file. ++ ++If LOG-FILE does not contain a date in %Y%m%d format, ++a date will be added before the file extension. ++""" + + from __future__ import print_function + from builtins import object +@@ -13,8 +17,10 @@ from future.utils import viewitems + from future import standard_library + standard_library.install_aliases() + ++from datetime import date + import errno + import logging ++from multiprocessing import Queue, Process + import os + import os.path + import re +@@ -35,19 +41,21 @@ from cherrypy import Application + from cherrypy._cplogging import LogManager + from cherrypy.lib import profiler + +-### Tools is needed for CRABServer startup: it sets up the tools attributes ++# Tools is needed for CRABServer startup: it sets up the tools attributes + import WMCore.REST.Tools ++ + from WMCore.Configuration import ConfigSection, loadConfigurationFile ++from WMCore.WMLogging import getTimeRotatingLogger, log_listener + from Utils.Utilities import lowerCmsHeaders + from Utils.PythonVersion import PY2 + +-#: Terminal controls to switch to "OK" status message colour. ++# Terminal controls to switch to "OK" status message colour. + COLOR_OK = "\033[0;32m" + +-#: Terminal controls to switch to "warning" status message colour. ++# Terminal controls to switch to "warning" status message colour. + COLOR_WARN = "\033[0;31m" + +-#: Terminal controls to restore normal message colour. ++# Terminal controls to restore normal message colour. + COLOR_NORMAL = "\033[0;39m" + + +@@ -101,7 +109,8 @@ class ProfiledApp(Application): + self.profiler = profiler.ProfileAggregator(path) + + def __call__(self, env, handler): +- def gather(): return Application.__call__(self, env, handler) ++ def gather(): ++ return Application.__call__(self, env, handler) + + return self.profiler.run(gather) + +@@ -213,10 +222,15 @@ class RESTMain(object): + if local_base.find(':') == -1: + local_base = '%s:%d' % (local_base, port) + +- # Set default server configuration. ++ # setup the cherrypy access logs to log to a queue + cherrypy.log = Logger() + +- cpconfig.update({'tools.add_content_length_if_missing.on': True}) ++ # log to stdout ++ cherrypy.log.screen = True ++ cherrypy.log.access_file = "" ++ cherrypy.log.error_file = "" ++ ++ # Set default server configuration. + cpconfig.update({'server.max_request_body_size': 0}) + cpconfig.update({'server.environment': 'production'}) + cpconfig.update({'server.socket_host': '0.0.0.0'}) +@@ -234,7 +248,7 @@ class RESTMain(object): + # as we previously used sys.setcheckinterval + interval = getattr(self.srvconfig, 'sys_check_interval', 10000) + # set check interval in seconds for sys.setswitchinterval +- sys.setswitchinterval(interval/1000) ++ sys.setswitchinterval(interval / 1000) + self.silent = getattr(self.srvconfig, 'silent', False) + + # Apply any override options from app config file. +@@ -324,7 +338,8 @@ class RESTDaemon(RESTMain): + :arg str statedir: server state directory.""" + RESTMain.__init__(self, config, statedir) + self.pidfile = "%s/%s.pid" % (self.statedir, self.appname) +- self.logfile = ["rotatelogs", "%s/%s-%%Y%%m%%d.log" % (self.statedir, self.appname), "86400"] ++ todayStr = date.today().strftime("%Y%m%d") ++ self.logfile = f"{self.statedir}/{self.appname}-{todayStr}.log" + + def daemon_pid(self): + """Check if there is a daemon running, and if so return its pid. +@@ -413,24 +428,6 @@ class RESTDaemon(RESTMain): + os.chdir(self.statedir) + + if daemonize: +- # Redirect all output to the logging daemon. +- devnull = open(os.devnull, "w") +- if isinstance(self.logfile, list): +- subproc = Popen(self.logfile, stdin=PIPE, stdout=devnull, stderr=devnull, +- bufsize=0, close_fds=True, shell=False) +- logger = subproc.stdin +- elif isinstance(self.logfile, str): +- # if a unix pipe is set as the logfile, it must be opened to append to the end of the file +- # if file/pipe does not exist, create it +- logger = open(self.logfile, "a") +- else: +- raise TypeError("'logfile' must be a string or array") +- os.dup2(logger.fileno(), sys.stdout.fileno()) +- os.dup2(logger.fileno(), sys.stderr.fileno()) +- os.dup2(devnull.fileno(), sys.stdin.fileno()) +- logger.close() +- devnull.close() +- + # First fork. Discard the parent. + pid = os.fork() + if pid > 0: +@@ -481,7 +478,8 @@ class RESTDaemon(RESTMain): + cherrypy.log("WATCHDOG: starting server daemon (pid %d)" % os.getpid()) + while True: + serverpid = os.fork() +- if not serverpid: break ++ if not serverpid: ++ break + signal.signal(signal.SIGINT, signal.SIG_IGN) + signal.signal(signal.SIGTERM, signal.SIG_IGN) + signal.signal(signal.SIGQUIT, signal.SIG_IGN) +@@ -581,18 +579,15 @@ def main(): + running, pid = server.daemon_pid() + if running: + if not opts.quiet: +- print("%s is %sRUNNING%s, PID %d" \ +- % (app, COLOR_OK, COLOR_NORMAL, pid)) ++ print(f"{app} is {COLOR_OK}RUNNING{COLOR_NORMAL}, PID {pid}") + sys.exit(0) + elif pid != None: + if not opts.quiet: +- print("%s is %sNOT RUNNING%s, stale PID %d" \ +- % (app, COLOR_WARN, COLOR_NORMAL, pid)) ++ print(f"{app} is {COLOR_WARN}NOT RUNNING{COLOR_NORMAL}, PID {pid}") + sys.exit(2) + else: + if not opts.quiet: +- print("%s is %sNOT RUNNING%s" \ +- % (app, COLOR_WARN, COLOR_NORMAL)) ++ print(f"{app} is {COLOR_WARN}NOT RUNNING{COLOR_NORMAL}") + sys.exit(1) + + elif opts.kill: +@@ -623,14 +618,8 @@ def main(): + print("Refusing to start over an already running daemon, pid %d" % pid, file=sys.stderr) + sys.exit(1) + +- # If we are (re)starting and were given a log file option, convert +- # the logfile option to a list if it looks like a pipe request, i.e. +- # starts with "|", such as "|rotatelogs foo/bar-%Y%m%d.log". + if opts.logfile: +- if opts.logfile.startswith("|"): +- server.logfile = re.split(r"\s+", opts.logfile[1:]) +- else: +- server.logfile = opts.logfile ++ server.logfile = opts.logfile + + # Actually start the daemon now. + server.start_daemon() +diff --git a/src/python/WMCore/WMLogging.py b/src/python/WMCore/WMLogging.py +index 55d713b4e..3f2623623 100644 +--- a/src/python/WMCore/WMLogging.py ++++ b/src/python/WMCore/WMLogging.py +@@ -4,16 +4,23 @@ _WMLogging_ + + Logging facilities used in WMCore. + """ ++from datetime import datetime, date + import logging +-from datetime import date, timedelta +-from logging.handlers import HTTPHandler, RotatingFileHandler, TimedRotatingFileHandler ++from logging.handlers import (HTTPHandler, ++ RotatingFileHandler, ++ TimedRotatingFileHandler, ++ QueueHandler, ++ QueueListener) ++from pathlib import Path ++ + + # a new log level which is lower than debug + # to prevent a tsunami of log messages in debug + # mode but to have the possibility to see all + # database queries if necessary. + logging.SQLDEBUG = 5 +-logging.addLevelName(logging.SQLDEBUG,"SQLDEBUG") ++logging.addLevelName(logging.SQLDEBUG, "SQLDEBUG") ++ + + def sqldebug(msg): + """ +@@ -22,7 +29,34 @@ def sqldebug(msg): + """ + logging.log(logging.SQLDEBUG, msg) + +-def setupRotatingHandler(fileName, maxBytes = 200000000, backupCount = 3): ++ ++def log_listener(queue, name, logFile): ++ """ ++ Listener process for process to fetch logs from a queue and log to the ++ root logger. The root logger is set to a TimeRotatingLogger ++ ++ From ++ https://docs.python.org/3/howto/logging-cookbook.html#logging-to-a-single-file-from-multiple-processes ++ ++ :param queue: The queue to listen to ++ :param name: The logger name ++ :param logFile: The TimeRotatingLogger filename ++ """ ++ getTimeRotatingLogger(name, logFile) ++ while True: ++ try: ++ record = queue.get() ++ if record is None: ++ break ++ logger = logger.getLogger(record.name) ++ logger.handle(record) ++ except Exception: ++ import sys, traceback ++ print('Problem with log listener:', file=sys.stderr) ++ traceback.print_exc(file=sys.stderr) ++ ++ ++def setupRotatingHandler(fileName, maxBytes=200000000, backupCount=3): + """ + _setupRotatingHandler_ + +@@ -30,55 +64,75 @@ def setupRotatingHandler(fileName, maxBytes = 200000000, backupCount = 3): + """ + handler = RotatingFileHandler(fileName, "a", maxBytes, backupCount) + logging.getLogger().addHandler(handler) +- return + + +-def getTimeRotatingLogger(name, logFile, duration = 'midnight'): +- """ Set the logger for time based lotaing. ++def getTimeRotatingLogger(name, logFile, duration='midnight'): ++ """ ++ Set the logger for time based rotating. + """ + logger = logging.getLogger(name) + if duration == 'midnight': +- handler = MyTimedRotatingFileHandler(logFile, duration, backupCount = 10) ++ handler = WMTimedRotatingFileHandler(logFile, when=duration, backupCount=10) + else: +- handler = TimedRotatingFileHandler(logFile, duration, backupCount = 10) ++ handler = TimedRotatingFileHandler(logFile, when=duration, backupCount=10) + formatter = logging.Formatter("%(asctime)s:%(levelname)s:%(module)s:%(message)s") + handler.setFormatter(formatter) + logger.addHandler(handler) +- logger.setLevel(logging.INFO) ++ # logger.setLevel(logging.INFO) + + return logger + + +-class MyTimedRotatingFileHandler(TimedRotatingFileHandler): ++class WMTimedRotatingFileHandler(TimedRotatingFileHandler): + """ +- _MyTimedRotatingFileHandler_ ++ _WMTimedRotatingFileHandler_ + + Overwrite the standard filename functionality from +- logging.handlers.MyTimedRotatingFileHandler ++ logging.handlers.TimedRotatingFileHandler + such that it mimics the same behaviour as rotatelogs tool. + +- Source code from: +- https://stackoverflow.com/questions/338450/timedrotatingfilehandler-changing-file-name ++ More information here: ++ https://docs.python.org/3/library/logging.handlers.html#baserotatinghandler + """ +- def __init__(self, logName, interval, backupCount): +- super(MyTimedRotatingFileHandler, self).__init__(logName, when=interval, +- backupCount=backupCount) ++ def __init__(self, filename, when='midnight', backupCount=10, **kwargs): ++ """ ++ Initializes WMTimedRotatingFileHandler ++ """ ++ super().__init__(filename, when=when, backupCount=backupCount, ++ delay=True, **kwargs) ++ ++ today = datetime.now().strftime(self.suffix).replace('-', '') ++ # store our filename and set the filename to be used ++ self.loggerBaseName = self.baseFilename ++ filepath = Path(self.baseFilename) ++ self.baseFilename = f"{filepath.parent}/{filepath.stem}-{today}{filepath.suffix}" ++ ++ self.stream = self._open() + +- def doRollover(self): ++ def namer(self, default_name): ++ """ ++ namer called by rotation_filename in TimedRotatingFileHandler.doRollover ++ ++ For us to replicate rotatelogs, we need to add the time_str to the ++ filename before the .log suffix ++ """ ++ # get the time from default_name ++ today = datetime.now().strftime(self.suffix).replace('-', '') ++ time_str = today.replace('-', '') ++ log_path = Path(self.loggerBaseName) ++ logPath = f"{log_path.parent}/{log_path.stem}-{time_str}{log_path.suffix}" ++ return logPath ++ ++ def rotator(self, src, dest): + """ +- _doRollover_ ++ rotator is called by the self.rotate + +- Rotate the log file and add the date between the log name +- and its extension, e.g.: +- reqmgr2.log becomes reqmgr2-20170815.log ++ Gets the new date, then sets the proper self.baseFilename ++ We don't use the default behavior of the current file being renamed + """ +- self.stream.close() +- # replace yesterday's date by today +- yesterdayStr = (date.today() - timedelta(1)).strftime("%Y%m%d") +- todayStr = date.today().strftime("%Y%m%d") +- self.baseFilename = self.baseFilename.replace(yesterdayStr, todayStr) +- self.stream = open(self.baseFilename, 'w', encoding='utf-8') +- self.rolloverAt = self.rolloverAt + self.interval ++ today = datetime.now().strftime(self.suffix).replace('-', '') ++ filepath = Path(self.loggerBaseName) ++ self.baseFilename = f"{filepath.parent}/{filepath.stem}-{today}{filepath.suffix}" + + + class CouchHandler(logging.handlers.HTTPHandler): diff --git a/docker/pypi/msunmerged/11955.patch b/docker/pypi/msunmerged/11955.patch new file mode 100644 index 000000000..cd268f99d --- /dev/null +++ b/docker/pypi/msunmerged/11955.patch @@ -0,0 +1,379 @@ +diff --git a/src/python/WMCore/REST/Main.py b/src/python/WMCore/REST/Main.py +index 82e4e89e7..85a0c4aa3 100644 +--- a/src/python/WMCore/REST/Main.py ++++ b/src/python/WMCore/REST/Main.py +@@ -5,7 +5,11 @@ + + Manages a web server application. Loads configuration and all views, starting + up an appropriately configured CherryPy instance. Views are loaded dynamically +-and can be turned on/off via configuration file.""" ++and can be turned on/off via configuration file. ++ ++If LOG-FILE does not contain a date in %Y%m%d format, ++a date will be added before the file extension. ++""" + + from __future__ import print_function + from builtins import object +@@ -13,6 +17,7 @@ from future.utils import viewitems + from future import standard_library + standard_library.install_aliases() + ++from datetime import date + import errno + import logging + import os +@@ -22,6 +27,7 @@ import signal + import socket + import sys + import _thread ++import threading + import time + import traceback + from argparse import ArgumentParser +@@ -29,25 +35,28 @@ from io import BytesIO, StringIO + from glob import glob + from subprocess import Popen, PIPE + from pprint import pformat ++import queue + + import cherrypy + from cherrypy import Application + from cherrypy._cplogging import LogManager + from cherrypy.lib import profiler + +-### Tools is needed for CRABServer startup: it sets up the tools attributes ++# Tools is needed for CRABServer startup: it sets up the tools attributes + import WMCore.REST.Tools ++ + from WMCore.Configuration import ConfigSection, loadConfigurationFile ++from WMCore.WMLogging import getTimeRotatingLogger, log_listener + from Utils.Utilities import lowerCmsHeaders + from Utils.PythonVersion import PY2 + +-#: Terminal controls to switch to "OK" status message colour. ++# Terminal controls to switch to "OK" status message colour. + COLOR_OK = "\033[0;32m" + +-#: Terminal controls to switch to "warning" status message colour. ++# Terminal controls to switch to "warning" status message colour. + COLOR_WARN = "\033[0;31m" + +-#: Terminal controls to restore normal message colour. ++# Terminal controls to restore normal message colour. + COLOR_NORMAL = "\033[0;39m" + + +@@ -101,7 +110,8 @@ class ProfiledApp(Application): + self.profiler = profiler.ProfileAggregator(path) + + def __call__(self, env, handler): +- def gather(): return Application.__call__(self, env, handler) ++ def gather(): ++ return Application.__call__(self, env, handler) + + return self.profiler.run(gather) + +@@ -182,6 +192,9 @@ class RESTMain(object): + self.extensions = {} + self.views = {} + ++ # setup the queue for the QueueHandler ++ self.logqueue = queue.Queue() ++ + def validate_config(self): + """Check the server configuration has the required sections.""" + for key in ('admin', 'description', 'title'): +@@ -213,10 +226,15 @@ class RESTMain(object): + if local_base.find(':') == -1: + local_base = '%s:%d' % (local_base, port) + +- # Set default server configuration. ++ # setup the cherrypy access logs to log to a queue + cherrypy.log = Logger() ++ handler = logging.handlers.QueueHandler(self.logqueue) ++ cherrypy.log.access_file = "" ++ cherrypy.log.error_file = "" ++ cherrypy.log.access_log.addHandler(handler) ++ cherrypy.log.error_log.addHandler(handler) + +- cpconfig.update({'tools.add_content_length_if_missing.on': True}) ++ # Set default server configuration. + cpconfig.update({'server.max_request_body_size': 0}) + cpconfig.update({'server.environment': 'production'}) + cpconfig.update({'server.socket_host': '0.0.0.0'}) +@@ -234,7 +252,7 @@ class RESTMain(object): + # as we previously used sys.setcheckinterval + interval = getattr(self.srvconfig, 'sys_check_interval', 10000) + # set check interval in seconds for sys.setswitchinterval +- sys.setswitchinterval(interval/1000) ++ sys.setswitchinterval(interval / 1000) + self.silent = getattr(self.srvconfig, 'silent', False) + + # Apply any override options from app config file. +@@ -324,7 +342,16 @@ class RESTDaemon(RESTMain): + :arg str statedir: server state directory.""" + RESTMain.__init__(self, config, statedir) + self.pidfile = "%s/%s.pid" % (self.statedir, self.appname) +- self.logfile = ["rotatelogs", "%s/%s-%%Y%%m%%d.log" % (self.statedir, self.appname), "86400"] ++ todayStr = date.today().strftime("%Y%m%d") ++ self.logfile = f"{self.statedir}/{self.appname}-{todayStr}.log" ++ ++ # setup the log listener ++ self.log_listener = threading.Thread(target=log_listener, ++ args=(self.logqueue, self.logfile)) ++ ++ # set up rotating logger ++ getTimeRotatingLogger(None, self.logfile) ++ + + def daemon_pid(self): + """Check if there is a daemon running, and if so return its pid. +@@ -413,24 +440,6 @@ class RESTDaemon(RESTMain): + os.chdir(self.statedir) + + if daemonize: +- # Redirect all output to the logging daemon. +- devnull = open(os.devnull, "w") +- if isinstance(self.logfile, list): +- subproc = Popen(self.logfile, stdin=PIPE, stdout=devnull, stderr=devnull, +- bufsize=0, close_fds=True, shell=False) +- logger = subproc.stdin +- elif isinstance(self.logfile, str): +- # if a unix pipe is set as the logfile, it must be opened to append to the end of the file +- # if file/pipe does not exist, create it +- logger = open(self.logfile, "a") +- else: +- raise TypeError("'logfile' must be a string or array") +- os.dup2(logger.fileno(), sys.stdout.fileno()) +- os.dup2(logger.fileno(), sys.stderr.fileno()) +- os.dup2(devnull.fileno(), sys.stdin.fileno()) +- logger.close() +- devnull.close() +- + # First fork. Discard the parent. + pid = os.fork() + if pid > 0: +@@ -451,6 +460,7 @@ class RESTDaemon(RESTMain): + # following code is executed both in daemon and not daemon mode + error = False + try: ++ self.log_listener.start() + self.run() + except Exception as e: + error = True +@@ -481,7 +491,8 @@ class RESTDaemon(RESTMain): + cherrypy.log("WATCHDOG: starting server daemon (pid %d)" % os.getpid()) + while True: + serverpid = os.fork() +- if not serverpid: break ++ if not serverpid: ++ break + signal.signal(signal.SIGINT, signal.SIG_IGN) + signal.signal(signal.SIGTERM, signal.SIG_IGN) + signal.signal(signal.SIGQUIT, signal.SIG_IGN) +@@ -581,18 +592,15 @@ def main(): + running, pid = server.daemon_pid() + if running: + if not opts.quiet: +- print("%s is %sRUNNING%s, PID %d" \ +- % (app, COLOR_OK, COLOR_NORMAL, pid)) ++ print(f"{app} is {COLOR_OK}RUNNING{COLOR_NORMAL}, PID {pid}") + sys.exit(0) + elif pid != None: + if not opts.quiet: +- print("%s is %sNOT RUNNING%s, stale PID %d" \ +- % (app, COLOR_WARN, COLOR_NORMAL, pid)) ++ print(f"{app} is {COLOR_WARN}NOT RUNNING{COLOR_NORMAL}, PID {pid}") + sys.exit(2) + else: + if not opts.quiet: +- print("%s is %sNOT RUNNING%s" \ +- % (app, COLOR_WARN, COLOR_NORMAL)) ++ print(f"{app} is {COLOR_WARN}NOT RUNNING{COLOR_NORMAL}") + sys.exit(1) + + elif opts.kill: +@@ -623,14 +631,8 @@ def main(): + print("Refusing to start over an already running daemon, pid %d" % pid, file=sys.stderr) + sys.exit(1) + +- # If we are (re)starting and were given a log file option, convert +- # the logfile option to a list if it looks like a pipe request, i.e. +- # starts with "|", such as "|rotatelogs foo/bar-%Y%m%d.log". + if opts.logfile: +- if opts.logfile.startswith("|"): +- server.logfile = re.split(r"\s+", opts.logfile[1:]) +- else: +- server.logfile = opts.logfile ++ server.logfile = opts.logfile + + # Actually start the daemon now. + server.start_daemon() +diff --git a/src/python/WMCore/WMLogging.py b/src/python/WMCore/WMLogging.py +index 55d713b4e..a39575ebd 100644 +--- a/src/python/WMCore/WMLogging.py ++++ b/src/python/WMCore/WMLogging.py +@@ -4,16 +4,21 @@ _WMLogging_ + + Logging facilities used in WMCore. + """ ++from datetime import datetime, date + import logging +-from datetime import date, timedelta +-from logging.handlers import HTTPHandler, RotatingFileHandler, TimedRotatingFileHandler ++from logging.handlers import (HTTPHandler, ++ RotatingFileHandler, ++ TimedRotatingFileHandler) ++from pathlib import Path ++ + + # a new log level which is lower than debug + # to prevent a tsunami of log messages in debug + # mode but to have the possibility to see all + # database queries if necessary. + logging.SQLDEBUG = 5 +-logging.addLevelName(logging.SQLDEBUG,"SQLDEBUG") ++logging.addLevelName(logging.SQLDEBUG, "SQLDEBUG") ++ + + def sqldebug(msg): + """ +@@ -22,7 +27,34 @@ def sqldebug(msg): + """ + logging.log(logging.SQLDEBUG, msg) + +-def setupRotatingHandler(fileName, maxBytes = 200000000, backupCount = 3): ++ ++def log_listener(queue, name, logFile): ++ """ ++ Listener process for process to fetch logs from a queue and log to the ++ root logger. The root logger is set to a TimeRotatingLogger ++ ++ From ++ https://docs.python.org/3/howto/logging-cookbook.html#logging-to-a-single-file-from-multiple-processes ++ ++ :param queue: The queue to listen to ++ :param name: The logger name ++ :param logFile: The TimeRotatingLogger filename ++ """ ++ getTimeRotatingLogger(name, logFile) ++ while True: ++ try: ++ record = queue.get() ++ if record is None: ++ break ++ logger = logging.getLogger(record.name) ++ logger.handle(record) ++ except Exception: ++ import sys, traceback ++ print('Problem with log listener:', file=sys.stderr) ++ traceback.print_exc(file=sys.stderr) ++ ++ ++def setupRotatingHandler(fileName, maxBytes=200000000, backupCount=3): + """ + _setupRotatingHandler_ + +@@ -30,55 +62,75 @@ def setupRotatingHandler(fileName, maxBytes = 200000000, backupCount = 3): + """ + handler = RotatingFileHandler(fileName, "a", maxBytes, backupCount) + logging.getLogger().addHandler(handler) +- return + + +-def getTimeRotatingLogger(name, logFile, duration = 'midnight'): +- """ Set the logger for time based lotaing. ++def getTimeRotatingLogger(name, logFile, duration='midnight'): ++ """ ++ Set the logger for time based rotating. + """ + logger = logging.getLogger(name) + if duration == 'midnight': +- handler = MyTimedRotatingFileHandler(logFile, duration, backupCount = 10) ++ handler = WMTimedRotatingFileHandler(logFile, when=duration, backupCount=10) + else: +- handler = TimedRotatingFileHandler(logFile, duration, backupCount = 10) ++ handler = TimedRotatingFileHandler(logFile, when=duration, backupCount=10) + formatter = logging.Formatter("%(asctime)s:%(levelname)s:%(module)s:%(message)s") + handler.setFormatter(formatter) + logger.addHandler(handler) +- logger.setLevel(logging.INFO) ++ # logger.setLevel(logging.INFO) + + return logger + + +-class MyTimedRotatingFileHandler(TimedRotatingFileHandler): ++class WMTimedRotatingFileHandler(TimedRotatingFileHandler): + """ +- _MyTimedRotatingFileHandler_ ++ _WMTimedRotatingFileHandler_ + + Overwrite the standard filename functionality from +- logging.handlers.MyTimedRotatingFileHandler ++ logging.handlers.TimedRotatingFileHandler + such that it mimics the same behaviour as rotatelogs tool. + +- Source code from: +- https://stackoverflow.com/questions/338450/timedrotatingfilehandler-changing-file-name ++ More information here: ++ https://docs.python.org/3/library/logging.handlers.html#baserotatinghandler + """ +- def __init__(self, logName, interval, backupCount): +- super(MyTimedRotatingFileHandler, self).__init__(logName, when=interval, +- backupCount=backupCount) ++ def __init__(self, filename, when='midnight', backupCount=10, **kwargs): ++ """ ++ Initializes WMTimedRotatingFileHandler ++ """ ++ super().__init__(filename, when=when, backupCount=backupCount, ++ delay=True, **kwargs) ++ ++ today = datetime.now().strftime(self.suffix).replace('-', '') ++ # store our filename and set the filename to be used ++ self.loggerBaseName = self.baseFilename ++ filepath = Path(self.baseFilename) ++ self.baseFilename = f"{filepath.parent}/{filepath.stem}-{today}{filepath.suffix}" ++ ++ self.stream = self._open() + +- def doRollover(self): ++ def namer(self, default_name): ++ """ ++ namer called by rotation_filename in TimedRotatingFileHandler.doRollover ++ ++ For us to replicate rotatelogs, we need to add the time_str to the ++ filename before the .log suffix ++ """ ++ # get the time from default_name ++ today = datetime.now().strftime(self.suffix).replace('-', '') ++ time_str = today.replace('-', '') ++ log_path = Path(self.loggerBaseName) ++ logPath = f"{log_path.parent}/{log_path.stem}-{time_str}{log_path.suffix}" ++ return logPath ++ ++ def rotator(self, src, dest): + """ +- _doRollover_ ++ rotator is called by the self.rotate + +- Rotate the log file and add the date between the log name +- and its extension, e.g.: +- reqmgr2.log becomes reqmgr2-20170815.log ++ Gets the new date, then sets the proper self.baseFilename ++ We don't use the default behavior of the current file being renamed + """ +- self.stream.close() +- # replace yesterday's date by today +- yesterdayStr = (date.today() - timedelta(1)).strftime("%Y%m%d") +- todayStr = date.today().strftime("%Y%m%d") +- self.baseFilename = self.baseFilename.replace(yesterdayStr, todayStr) +- self.stream = open(self.baseFilename, 'w', encoding='utf-8') +- self.rolloverAt = self.rolloverAt + self.interval ++ today = datetime.now().strftime(self.suffix).replace('-', '') ++ filepath = Path(self.loggerBaseName) ++ self.baseFilename = f"{filepath.parent}/{filepath.stem}-{today}{filepath.suffix}" + + + class CouchHandler(logging.handlers.HTTPHandler): diff --git a/docker/pypi/msunmerged/Dockerfile b/docker/pypi/msunmerged/Dockerfile index c75c2156f..42d7bdb50 100644 --- a/docker/pypi/msunmerged/Dockerfile +++ b/docker/pypi/msunmerged/Dockerfile @@ -1,23 +1,49 @@ -FROM registry.cern.ch/cmsweb/gfal:latest as gfal -FROM registry.cern.ch/cmsweb/dmwm-base:pypi-20240923-stable -MAINTAINER Valentin Kuznetsov vkuznet@gmail.com -COPY --from=gfal /data/miniconda /data/miniconda +FROM registry.cern.ch/cmsweb/dmwm-base:alma9-py3.9-20250703 +LABEL org.opencontainers.image.authors="Alan Malta alan.malta@cern.ch, Dennis Lee dylee@fnal.gov" + +# Specific MSUnmerged requirements from epel repository +RUN dnf install epel-release -y && dnf clean all && \ + dnf -y install patch gfal2 python3-gfal2 python3-gfal2-util gfal2-plugin-http gfal2-plugin-dcap gfal2-plugin-file \ + gfal2-plugin-srm gfal2-plugin-xrootd gfal2-plugin-gridftp gfal2-plugin-sftp && \ + dnf clean all + ENV WDIR=/data -ENV PATH $PATH:$WDIR/miniconda/bin -ENV PYTHONPATH $WDIR/miniconda/lib/python3.8/site-packages/ +WORKDIR $WDIR + # TAG to be passed at build time through `--build-arg TAG=`. Default: None ARG TAG=None -WORKDIR $WDIR -ADD run.sh $WDIR/run.sh -# since we install gfal via external image we'll skip it for installation -# of msunmerged, but to satisfy dependencies we'll install them first +# gfal2 is built and copied from gfal2 stage, so we only install non-gfal2 service dependencies +# FIXME: it is probably best to remove it from the requirements.txt file RUN curl -ksLO https://raw.githubusercontent.com/dmwm/WMCore/$TAG/requirements.txt -RUN cat requirements.txt | grep -v gfal2 > req.txt -RUN pip install -r req.txt -RUN pip install --no-deps msunmerged==$TAG +RUN cat requirements.txt | grep dbs3-client > req.txt +RUN cat requirements.txt | grep msunmerged | grep -v gfal2 >> req.txt + +# Create venv based on environment python to use proper python version, +# install modified requirements, +# and install MSUnmerged itself, without any dependencies +#RUN python -m venv .venv && \ +# .venv/bin/pip install -r req.txt && \ +# .venv/bin/pip install --no-deps msunmerged==$TAG +RUN python3 -m pip install -r req.txt && \ + python3 -m pip install --no-deps msunmerged==$TAG + +# add venv to the path to use the proper python3 +#ENV PATH="$WDIR/.venv/bin:$PATH" + +# patch dmwm/WMCore https://github.com/dmwm/WMCore/pull/11955 +#ADD 11955.patch $WDIR/11955.patch +#ADD 11922.patch $WDIR/11922.patch +#RUN cd /usr/local/lib/python3.9/site-packages/WMCore && \ +# patch -p 4 -t -i $WDIR/11922.patch && \ +# cd $WDIR + +# Specific run.sh for MSUnmerged +ADD run.sh $WDIR/run.sh + +# and now setup run.sh and manage scripts accordingly RUN sed -i -e "s,-config.py,-config-unmerged.py,g" /data/run.sh RUN sed -i -e "s,config.py,config-unmerged.py,g" /data/manage -ENV WDIR=/data + ENV USER=_reqmgr2ms RUN useradd ${USER} && install -o ${USER} -d ${WDIR} RUN echo "%$USER ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers diff --git a/docker/pypi/msunmerged/Dockerfile.cc7 b/docker/pypi/msunmerged/Dockerfile.cc7 new file mode 100644 index 000000000..c75c2156f --- /dev/null +++ b/docker/pypi/msunmerged/Dockerfile.cc7 @@ -0,0 +1,26 @@ +FROM registry.cern.ch/cmsweb/gfal:latest as gfal +FROM registry.cern.ch/cmsweb/dmwm-base:pypi-20240923-stable +MAINTAINER Valentin Kuznetsov vkuznet@gmail.com +COPY --from=gfal /data/miniconda /data/miniconda +ENV WDIR=/data +ENV PATH $PATH:$WDIR/miniconda/bin +ENV PYTHONPATH $WDIR/miniconda/lib/python3.8/site-packages/ +# TAG to be passed at build time through `--build-arg TAG=`. Default: None +ARG TAG=None +WORKDIR $WDIR +ADD run.sh $WDIR/run.sh +# since we install gfal via external image we'll skip it for installation +# of msunmerged, but to satisfy dependencies we'll install them first +RUN curl -ksLO https://raw.githubusercontent.com/dmwm/WMCore/$TAG/requirements.txt +RUN cat requirements.txt | grep -v gfal2 > req.txt +RUN pip install -r req.txt +RUN pip install --no-deps msunmerged==$TAG +RUN sed -i -e "s,-config.py,-config-unmerged.py,g" /data/run.sh +RUN sed -i -e "s,config.py,config-unmerged.py,g" /data/manage +ENV WDIR=/data +ENV USER=_reqmgr2ms +RUN useradd ${USER} && install -o ${USER} -d ${WDIR} +RUN echo "%$USER ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers +USER ${USER} +RUN sudo chown -R $USER.$USER $WDIR +CMD ["python3"] diff --git a/docker/pypi/msunmerged/Dockerfile.py312 b/docker/pypi/msunmerged/Dockerfile.py312 new file mode 100644 index 000000000..c0a9a4da3 --- /dev/null +++ b/docker/pypi/msunmerged/Dockerfile.py312 @@ -0,0 +1,54 @@ +FROM registry.cern.ch/cmsweb/gfal:alma9-py3.12-1.13.0 AS gfal2 +FROM registry.cern.ch/cmsweb/dmwm-base:alma9-py3.12-20250529 +LABEL org.opencontainers.image.authors="Alan Malta alan.malta@cern.ch, Dennis Lee dylee@fnal.gov" + +# Specific MSUnmerged requirements from epel repository +RUN dnf install epel-release -y && dnf clean all && \ + dnf -y install patch gfal2 python3-gfal2-util gfal2-plugin-http gfal2-plugin-dcap gfal2-plugin-file \ + gfal2-plugin-srm gfal2-plugin-xrootd gfal2-plugin-gridftp gfal2-plugin-sftp && \ + dnf clean all + +ENV WDIR=/data +WORKDIR $WDIR + +# TAG to be passed at build time through `--build-arg TAG=`. Default: None +ARG TAG=None +# gfal2 is built and copied from gfal2 stage, so we only install non-gfal2 service dependencies +# FIXME: it is probably best to remove it from the requirements.txt file +RUN curl -ksLO https://raw.githubusercontent.com/dmwm/WMCore/$TAG/requirements.txt +RUN cat requirements.txt | grep dbs3-client > req.txt +RUN cat requirements.txt | grep msunmerged | grep -v gfal2 >> req.txt + +# Create venv based on environment python to use proper python version, +# install modified requirements, +# and install MSUnmerged itself, without any dependencies +RUN python -m venv .venv && \ + .venv/bin/pip install -r req.txt && \ + .venv/bin/pip install --no-deps msunmerged==$TAG + +# copy gfal2.so and boost libs from gfal2 stage +COPY --from=gfal2 /tmp/.venv/lib/python3.12/site-packages/gfal2.so $WDIR/.venv/lib/python3.12/site-packages/gfal2.so +COPY --from=gfal2 /usr/local/lib/* /usr/local/lib/. + +# add venv to the path to use the proper python3 +ENV PATH="$WDIR/.venv/bin:$PATH" + +# patch dmwm/WMCore https://github.com/dmwm/WMCore/pull/11955 +ADD 11955.patch $WDIR/11955.patch +RUN cd $WDIR/.venv/lib/python3.12/site-packages/WMCore && \ + patch -p 4 -t -i $WDIR/11955.patch && \ + cd $WDIR + +# Specific run.sh for MSUnmerged +ADD run.sh $WDIR/run.sh + +# and now setup run.sh and manage scripts accordingly +RUN sed -i -e "s,-config.py,-config-unmerged.py,g" /data/run.sh +RUN sed -i -e "s,config.py,config-unmerged.py,g" /data/manage + +ENV USER=_reqmgr2ms +RUN useradd ${USER} && install -o ${USER} -d ${WDIR} +RUN echo "%$USER ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers +USER ${USER} +RUN sudo chown -R $USER.$USER $WDIR +CMD ["python3"] \ No newline at end of file diff --git a/docker/pypi/msunmerged/run.sh b/docker/pypi/msunmerged/run.sh index eb110e74c..abec0557f 100755 --- a/docker/pypi/msunmerged/run.sh +++ b/docker/pypi/msunmerged/run.sh @@ -142,7 +142,7 @@ fi [[ -n $rseExpr ]] && sed -i -e "s/^[[:blank:]]*RSEEXPR.*/RSEEXPR = \"${rseExpr}\"/g" $CFGFILE # start the service -wmc-httpd -r -d $STATEDIR -l "|rotatelogs $LOGDIR/$srv-%Y%m%d-`hostname -s`.log 86400" $CFGFILE +wmc-httpd -r -d $STATEDIR -l "$LOGDIR/$srv-`hostname -s`.log" $CFGFILE # start monitor.sh script if [ -f /data/monitor.sh ]; then