From 48015b93d72acaf2d76de434bc6d5aa7cb5b4f9a Mon Sep 17 00:00:00 2001 From: LinaresToine Date: Fri, 20 Mar 2026 19:53:04 +0100 Subject: [PATCH 1/7] add mergeL1SCOUTNANOAOD flag for L1 nano output --- src/python/WMCore/WMSpec/StdSpecs/StdBase.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/python/WMCore/WMSpec/StdSpecs/StdBase.py b/src/python/WMCore/WMSpec/StdSpecs/StdBase.py index 8a2abe70cd..5d0d408575 100644 --- a/src/python/WMCore/WMSpec/StdSpecs/StdBase.py +++ b/src/python/WMCore/WMSpec/StdSpecs/StdBase.py @@ -687,6 +687,7 @@ def addMergeTask(self, parentTask, parentTaskSplitting, parentOutputModuleName, parentTaskCmssw = parentTask.getStep(parentStepName) parentOutputModule = parentTaskCmssw.getOutputModule(parentOutputModuleName) + primaryDataset = getattr(parentOutputModule, "primaryDataset") dataTier = getattr(parentOutputModule, "dataTier") mergeTask.setInputReference(parentTaskCmssw, outputModule=parentOutputModuleName, dataTier=dataTier) @@ -715,8 +716,12 @@ def addMergeTask(self, parentTask, parentTaskSplitting, parentOutputModuleName, mergeTaskCmsswHelper.setDataProcessingConfig("do_not_use", "merge", newDQMIO=True) elif dataTier in ("NANOAOD", "NANOAODSIM"): - mergeTaskCmsswHelper.setDataProcessingConfig("do_not_use", "merge", - mergeNANO=True) + if primaryDataset in ("L1Scouting", "L1ScoutingSelection"): + mergeTaskCmsswHelper.setDataProcessingConfig("do_not_use", "merge", + mergeL1SCOUTNANO=True) + else: + mergeTaskCmsswHelper.setDataProcessingConfig("do_not_use", "merge", + mergeNANO=True) else: mergeTaskCmsswHelper.setDataProcessingConfig("do_not_use", "merge") From 0143e0539bb9c424da61d4a1abec6208ad344a3d Mon Sep 17 00:00:00 2001 From: LinaresToine Date: Tue, 24 Mar 2026 11:04:53 +0100 Subject: [PATCH 2/7] Use startswith so we have more flexibility on the PDs that use mergeL1SCOUTNANO --- src/python/WMCore/WMSpec/StdSpecs/StdBase.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/python/WMCore/WMSpec/StdSpecs/StdBase.py b/src/python/WMCore/WMSpec/StdSpecs/StdBase.py index 5d0d408575..a12bbd3cce 100644 --- a/src/python/WMCore/WMSpec/StdSpecs/StdBase.py +++ b/src/python/WMCore/WMSpec/StdSpecs/StdBase.py @@ -687,7 +687,6 @@ def addMergeTask(self, parentTask, parentTaskSplitting, parentOutputModuleName, parentTaskCmssw = parentTask.getStep(parentStepName) parentOutputModule = parentTaskCmssw.getOutputModule(parentOutputModuleName) - primaryDataset = getattr(parentOutputModule, "primaryDataset") dataTier = getattr(parentOutputModule, "dataTier") mergeTask.setInputReference(parentTaskCmssw, outputModule=parentOutputModuleName, dataTier=dataTier) @@ -716,7 +715,8 @@ def addMergeTask(self, parentTask, parentTaskSplitting, parentOutputModuleName, mergeTaskCmsswHelper.setDataProcessingConfig("do_not_use", "merge", newDQMIO=True) elif dataTier in ("NANOAOD", "NANOAODSIM"): - if primaryDataset in ("L1Scouting", "L1ScoutingSelection"): + primaryDataset = getattr(parentOutputModule, "primaryDataset", "") + if primaryDataset.startswith("L1Scouting"): mergeTaskCmsswHelper.setDataProcessingConfig("do_not_use", "merge", mergeL1SCOUTNANO=True) else: From cac9da092bbe0e5002377a32579ca2daa8eda25d Mon Sep 17 00:00:00 2001 From: Muhammad Hassan Ahmed <33365802+hassan11196@users.noreply.github.com> Date: Thu, 2 Apr 2026 12:18:49 +0000 Subject: [PATCH 3/7] Update pip upgrade step to include setuptools constraint for build isolation --- .github/workflows/pypi_build_and_images.yaml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pypi_build_and_images.yaml b/.github/workflows/pypi_build_and_images.yaml index cd26d13066..494c97f2f7 100644 --- a/.github/workflows/pypi_build_and_images.yaml +++ b/.github/workflows/pypi_build_and_images.yaml @@ -52,11 +52,13 @@ jobs: sudo apt update sudo apt install -y curl libcurl4-openssl-dev - # upgrade pip + # upgrade pip and set setuptools constraint for build isolation - name: Upgrade pip3 and install build tools run: | python3 -m pip install --upgrade pip - python3 -m pip install "setuptools>=45,<82" wheel + echo 'setuptools<80' > /tmp/constraints.txt + echo "PIP_CONSTRAINT=/tmp/constraints.txt" >> "$GITHUB_ENV" + python3 -m pip install setuptools wheel # create appropriate setup.py for python builds - name: Update the setup script template with package name From 29a9030cecc0fdc343bb7c0f4a354c20ff00e54c Mon Sep 17 00:00:00 2001 From: LinaresToine Date: Tue, 24 Mar 2026 11:17:14 +0100 Subject: [PATCH 4/7] Do not introduce new flag --- src/python/WMCore/WMSpec/StdSpecs/StdBase.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/src/python/WMCore/WMSpec/StdSpecs/StdBase.py b/src/python/WMCore/WMSpec/StdSpecs/StdBase.py index a12bbd3cce..ad5ad2ffbd 100644 --- a/src/python/WMCore/WMSpec/StdSpecs/StdBase.py +++ b/src/python/WMCore/WMSpec/StdSpecs/StdBase.py @@ -711,24 +711,21 @@ def addMergeTask(self, parentTask, parentTaskSplitting, parentOutputModuleName, max_wait_time=self.maxWaitTime, initial_lfn_counter=lfn_counter) + primaryDataset = getattr(parentOutputModule, "primaryDataset") if dataTier == "DQMIO": mergeTaskCmsswHelper.setDataProcessingConfig("do_not_use", "merge", newDQMIO=True) elif dataTier in ("NANOAOD", "NANOAODSIM"): - primaryDataset = getattr(parentOutputModule, "primaryDataset", "") - if primaryDataset.startswith("L1Scouting"): - mergeTaskCmsswHelper.setDataProcessingConfig("do_not_use", "merge", - mergeL1SCOUTNANO=True) - else: - mergeTaskCmsswHelper.setDataProcessingConfig("do_not_use", "merge", - mergeNANO=True) + mergeTaskCmsswHelper.setDataProcessingConfig("do_not_use", "merge", + mergeNANO=True, + isL1Scouting=(primaryDataset.startswith("L1Scouting"))) else: mergeTaskCmsswHelper.setDataProcessingConfig("do_not_use", "merge") mergeTaskStageHelper.setMinMergeSize(0, 0) self.addOutputModule(mergeTask, "Merged", - primaryDataset=getattr(parentOutputModule, "primaryDataset"), + primaryDataset=primaryDataset, dataTier=getattr(parentOutputModule, "dataTier"), filterName=getattr(parentOutputModule, "filterName"), forceMerged=True, taskConf=taskConf) From 59d6011ad1999a25c4aa87394e445772f8c70b1b Mon Sep 17 00:00:00 2001 From: LinaresToine Date: Mon, 9 Feb 2026 11:15:08 +0100 Subject: [PATCH 5/7] Fix AlcaHarvest PFN --- src/python/WMCore/WMSpec/Steps/Executors/AlcaHarvest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/python/WMCore/WMSpec/Steps/Executors/AlcaHarvest.py b/src/python/WMCore/WMSpec/Steps/Executors/AlcaHarvest.py index 1c1112f80d..4ff90d4b19 100644 --- a/src/python/WMCore/WMSpec/Steps/Executors/AlcaHarvest.py +++ b/src/python/WMCore/WMSpec/Steps/Executors/AlcaHarvest.py @@ -126,7 +126,7 @@ def execute(self, emulator=None): logging.info("==> copy %s", file2copy) targetLFN = os.path.join(self.step.condition.lfnbase, file2copy) - targetPFN = "root://eoscms//eos/cms%s" % targetLFN + targetPFN = "root://eoscms.cern.ch//eos/cms%s" % targetLFN command = "env XRD_WRITERECOVERY=0 xrdcp -s -f %s %s" % (file2copy, targetPFN) From 9791eddeb65b19bf8dab53e2a13d2308ac4ebc1d Mon Sep 17 00:00:00 2001 From: LinaresToine Date: Mon, 9 Feb 2026 11:23:38 +0100 Subject: [PATCH 6/7] Use eoscms.cern.ch in testing --- .../WMCore_t/Storage_t/StageOutImpl_t.py | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/test/python/WMCore_t/Storage_t/StageOutImpl_t.py b/test/python/WMCore_t/Storage_t/StageOutImpl_t.py index 5453248565..2f49de2d6b 100644 --- a/test/python/WMCore_t/Storage_t/StageOutImpl_t.py +++ b/test/python/WMCore_t/Storage_t/StageOutImpl_t.py @@ -16,28 +16,28 @@ def setUp(self): def testSplitPFN_noOpaque(self): self.assertEqual(("splitable/test/test1", "test1", "splitable/test/test1", ""), StageOutImpl.splitPFN("splitable/test/test1")) - self.assertEqual(('//eoscms//eos/cms/store/', 'eoscms', '//eoscms//eos/cms/store/', ""), - StageOutImpl.splitPFN("//eoscms//eos/cms/store/")) + self.assertEqual(('//eoscms.cern.ch//eos/cms/store/', 'eoscms.cern.ch', '//eoscms.cern.ch//eos/cms/store/', ""), + StageOutImpl.splitPFN("//eoscms.cern.ch//eos/cms/store/")) def testSplitPFN_doubleSlashRoot(self): - self.assertEqual(('root', 'eoscms', '/eos/cms/store/', ""), - StageOutImpl.splitPFN("root://eoscms//eos/cms/store/")) + self.assertEqual(('root', 'eoscms.cern.ch', '/eos/cms/store/', ""), + StageOutImpl.splitPFN("root://eoscms.cern.ch//eos/cms/store/")) def testSplitPFN_path(self): - self.assertEqual(('root', 'eoscms', 'default', "?"), - StageOutImpl.splitPFN("root://eoscms//eos/cms/store?path=default")) + self.assertEqual(('root', 'eoscms.cern.ch', 'default', "?"), + StageOutImpl.splitPFN("root://eoscms.cern.ch//eos/cms/store?path=default")) def testSplitPFN_pathConnector(self): - self.assertEqual(('root', 'eoscms', 'default', "?default2"), - StageOutImpl.splitPFN("root://eoscms//eos/cms/store?path=default&default2")) + self.assertEqual(('root', 'eoscms.cern.ch', 'default', "?default2"), + StageOutImpl.splitPFN("root://eoscms.cern.ch//eos/cms/store?path=default&default2")) def testSplitPFN_path2(self): - self.assertEqual(('root', 'eoscms', 'cms', "?cms2ss&path=default"), - StageOutImpl.splitPFN("root://eoscms//eos/cms/store?path=cms&cms2ss&path=default")) + self.assertEqual(('root', 'eoscms.cern.ch', 'cms', "?cms2ss&path=default"), + StageOutImpl.splitPFN("root://eoscms.cern.ch//eos/cms/store?path=cms&cms2ss&path=default")) def testSplitPFN_pathConnector2(self): - self.assertEqual(('root', 'eoscms', 'cms', "?path=default&default2"), - StageOutImpl.splitPFN("root://eoscms//eos/cms/store?path=cms&path=default&default2")) + self.assertEqual(('root', 'eoscms.cern.ch', 'cms', "?path=default&default2"), + StageOutImpl.splitPFN("root://eoscms.cern.ch//eos/cms/store?path=cms&path=default&default2")) @mock.patch('WMCore.Storage.StageOutImpl.runCommandWithOutput') def testExecuteCommand_stageOutError(self, mock_runCommand): From 96bf1c640beb3a53e24b5e23aad1e5c08979649a Mon Sep 17 00:00:00 2001 From: hassan11196 Date: Thu, 15 Jan 2026 12:12:14 +0100 Subject: [PATCH 7/7] Bypass NoPendingSlot limit for Merge job type to prevent blocking by pending slot limits --- src/python/WMComponent/JobSubmitter/JobSubmitterPoller.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/python/WMComponent/JobSubmitter/JobSubmitterPoller.py b/src/python/WMComponent/JobSubmitter/JobSubmitterPoller.py index e64b06544f..c0fedac4ff 100644 --- a/src/python/WMComponent/JobSubmitter/JobSubmitterPoller.py +++ b/src/python/WMComponent/JobSubmitter/JobSubmitterPoller.py @@ -639,6 +639,11 @@ def _getJobSubmitCondition(self, jobPrio, siteName, jobType): totalJobThreshold = totalPendingThreshold + totalRunningSlots totalTaskTheshold = taskPendingThreshold + taskRunningSlots + # Bypass NoPendingSlot limit for job type (Merge) + # These jobs should not be blocked by pending slot limits + if jobType == 'Merge': + totalPendingThreshold = float('inf') + jobStats = [{"Condition": "NoPendingSlot", "Current": totalPendingJobs, "Threshold": totalPendingThreshold},