From b679106d91ddf2ec23d8368f34fbe75ff1a051e3 Mon Sep 17 00:00:00 2001 From: Evan Rees Date: Fri, 10 Apr 2026 11:23:43 -0400 Subject: [PATCH 1/2] Add uv directive for Python package management Add a new `uv` process directive that allows Nextflow processes to declare Python dependencies managed by the uv package manager, following the same pattern as the existing `conda` and `spack` directives. The directive accepts package names, requirements.txt files, pyproject.toml files, or paths to existing virtual environments. Nextflow automatically creates, caches, and activates uv virtual environments for each unique set of dependencies. Includes CLI options (-with-uv / -without-uv), configuration scope (uv.enabled, uv.cacheDir, uv.pythonVersion, etc.), environment variables (NXF_UV_ENABLED, NXF_UV_CACHEDIR), lineage tracking, task hash integration, tests, and documentation. Signed-off-by: Evan Rees Co-Authored-By: Claude Opus 4.6 (1M context) --- docs/reference/config.md | 23 ++ docs/reference/env-vars.md | 6 + docs/reference/process.md | 29 ++ docs/uv.md | 130 +++++++ .../src/main/groovy/nextflow/Session.groovy | 7 + .../main/groovy/nextflow/cli/CmdRun.groovy | 9 + .../main/groovy/nextflow/cli/Launcher.groovy | 4 + .../nextflow/config/ConfigBuilder.groovy | 13 + .../executor/BashWrapperBuilder.groovy | 10 + .../processor/TaskArrayCollector.groovy | 1 + .../groovy/nextflow/processor/TaskBean.groovy | 3 + .../nextflow/processor/TaskHasher.groovy | 6 + .../groovy/nextflow/processor/TaskRun.groovy | 25 ++ .../nextflow/script/dsl/ProcessBuilder.groovy | 3 +- .../main/groovy/nextflow/uv/UvCache.groovy | 343 ++++++++++++++++++ .../main/groovy/nextflow/uv/UvConfig.groovy | 98 +++++ .../nextflow/executor/command-run.txt | 1 + .../groovy/nextflow/cli/LauncherTest.groovy | 4 + .../nextflow/config/ConfigBuilderTest.groovy | 43 +++ .../executor/BashWrapperBuilderTest.groovy | 18 + .../groovy/nextflow/uv/UvCacheTest.groovy | 212 +++++++++++ .../groovy/nextflow/uv/UvConfigTest.groovy | 67 ++++ .../main/nextflow/lineage/LinObserver.groovy | 1 + .../lineage/model/v1beta1/TaskRun.groovy | 4 + .../lineage/cli/LinCommandImplTest.groovy | 4 +- .../lineage/serde/LinEncoderTest.groovy | 3 +- 26 files changed, 1063 insertions(+), 4 deletions(-) create mode 100644 docs/uv.md create mode 100644 modules/nextflow/src/main/groovy/nextflow/uv/UvCache.groovy create mode 100644 modules/nextflow/src/main/groovy/nextflow/uv/UvConfig.groovy create mode 100644 modules/nextflow/src/test/groovy/nextflow/uv/UvCacheTest.groovy create mode 100644 modules/nextflow/src/test/groovy/nextflow/uv/UvConfigTest.groovy diff --git a/docs/reference/config.md b/docs/reference/config.md index 7519a4fe9f..c5617efddf 100644 --- a/docs/reference/config.md +++ b/docs/reference/config.md @@ -1603,6 +1603,29 @@ The following settings are available: `spack.parallelBuilds` : The maximum number of parallel package builds (default: the number of available CPUs). +(config-uv)= + +## `uv` + +The `uv` scope controls the creation of Python virtual environments by the [uv](https://docs.astral.sh/uv/) package manager. + +The following settings are available: + +`uv.cacheDir` +: The path where uv virtual environments are stored. It should be accessible from all compute nodes when using a shared file system. + +`uv.createTimeout` +: The amount of time to wait for the uv environment to be created before failing (default: `20 min`). + +`uv.enabled` +: Execute tasks with uv virtual environments (default: `false`). + +`uv.installOptions` +: Extra command line options for the `uv pip install` command. See the [uv documentation](https://docs.astral.sh/uv/) for more information. + +`uv.pythonVersion` +: The Python version to use when creating virtual environments (e.g. `3.12`). If not specified, uv will use its default Python resolution. + (config-timeline)= ## `timeline` diff --git a/docs/reference/env-vars.md b/docs/reference/env-vars.md index 91cbb0373e..fbebb1c986 100644 --- a/docs/reference/env-vars.md +++ b/docs/reference/env-vars.md @@ -232,6 +232,12 @@ The following environment variables control the configuration of the Nextflow ru ::: : Enable the use of Spack recipes defined by using the {ref}`process-spack` directive. (default: `false`). +`NXF_UV_CACHEDIR` +: Directory where uv virtual environments are stored. When using a computing cluster it must be a shared folder accessible from all compute nodes. + +`NXF_UV_ENABLED` +: Enable the use of uv environments defined by using the {ref}`process-uv` directive. (default: `false`). + `NXF_SYNTAX_PARSER` : :::{versionadded} 25.02.0-edge ::: diff --git a/docs/reference/process.md b/docs/reference/process.md index bb6ecb0ebb..563ae40c85 100644 --- a/docs/reference/process.md +++ b/docs/reference/process.md @@ -1574,6 +1574,35 @@ Multiple packages can be specified separating them with a blank space, e.g. `bwa The `spack` directive also accepts a Spack environment file path or the path of an existing Spack environment. See {ref}`spack-page` for more information. +(process-uv)= + +### uv + +The `uv` directive defines the set of Python packages to be installed using the [uv](https://docs.astral.sh/uv/) package manager for each task. For example: + +```nextflow +process hello { + uv 'numpy pandas matplotlib' + + script: + """ + python my_script.py + """ +} +``` + +Nextflow automatically creates a uv virtual environment for each unique set of packages. + +Multiple packages can be specified separating them with a blank space, e.g. `numpy pandas>=2.0 scikit-learn`. + +The `uv` directive also accepts: + +- A `requirements.txt` file path: `uv '/path/to/requirements.txt'` +- A `pyproject.toml` file path: `uv '/path/to/pyproject.toml'` +- The path of an existing virtual environment directory + +See {ref}`uv-page` for more information. + (process-stageinmode)= ### stageInMode diff --git a/docs/uv.md b/docs/uv.md new file mode 100644 index 0000000000..7cfa6119ef --- /dev/null +++ b/docs/uv.md @@ -0,0 +1,130 @@ +(uv-page)= + +# uv environments + +[uv](https://docs.astral.sh/uv/) is an extremely fast Python package and project manager, written in Rust. It can install Python packages, manage virtual environments, and handle Python versions. + +Nextflow has built-in support for uv that allows the configuration of workflow dependencies using Python packages, requirements files, or pyproject.toml files. + +This allows Nextflow applications to use Python packages managed by uv, taking advantage of its speed and reliability for creating reproducible Python environments. + +## Prerequisites + +This feature requires the [uv](https://docs.astral.sh/uv/getting-started/installation/) package manager to be installed on your system. + +## How it works + +Nextflow automatically creates and activates uv virtual environments given the dependencies specified by each process. + +Dependencies are specified by using the {ref}`process-uv` directive, providing either the names of the required Python packages, the path of a requirements file, the path of a pyproject.toml file, or the path of an existing virtual environment directory. + +You can specify the directory where the uv environments are stored using the `uv.cacheDir` configuration property (see the {ref}`configuration page ` for details). When using a computing cluster, make sure to use a shared file system path accessible from all compute nodes. + +:::{warning} +The uv environment feature is not supported by executors that use remote object storage as the work directory, e.g. AWS Batch. +::: + +### Enabling uv environments + +The use of uv packages specified using the {ref}`process-uv` directive needs to be enabled explicitly by setting the option shown below in the pipeline configuration file (i.e. `nextflow.config`): + +```groovy +uv.enabled = true +``` + +Alternatively, it can be specified by setting the variable `NXF_UV_ENABLED=true` in your environment or by using the `-with-uv` command line option. + +### Use Python package names + +Python package names can be specified using the `uv` directive. Multiple package names can be specified by separating them with a blank space. For example: + +```nextflow +process hello { + uv 'numpy pandas matplotlib' + + script: + ''' + python my_script.py + ''' +} +``` + +Using the above definition, a uv virtual environment that includes NumPy, Pandas, and Matplotlib is created and activated when the process is executed. + +The usual pip package syntax and naming conventions can be used. The version of a package can be specified using pip version specifiers like so: `numpy>=1.24 pandas==2.0.0`. + +### Use requirements files + +uv environments can also be defined using a requirements file. For example, given a `requirements.txt` file: + +``` +numpy>=1.24.0 +pandas>=2.0 +scikit-learn +matplotlib +``` + +The environment for a process can be specified like so: + +```nextflow +process hello { + uv '/path/to/requirements.txt' + + script: + ''' + python my_script.py + ''' +} +``` + +### Use pyproject.toml files + +uv can also install dependencies from a `pyproject.toml` file: + +```nextflow +process hello { + uv '/path/to/pyproject.toml' + + script: + ''' + python my_script.py + ''' +} +``` + +### Use existing environments + +If you already have a uv virtual environment, you can use it directly by specifying the path: + +```nextflow +process hello { + uv '/path/to/existing/venv' + + script: + ''' + python my_script.py + ''' +} +``` + +### Environment caching + +Nextflow caches uv environments so that they are created only once for each unique set of packages. The cache directory can be configured using the `uv.cacheDir` setting or the `NXF_UV_CACHEDIR` environment variable. + +### Python version + +You can specify the Python version to use when creating virtual environments: + +```groovy +uv.pythonVersion = '3.12' +``` + +### Advanced settings + +The following settings are available in the `uv` scope of the Nextflow configuration: + +- `uv.enabled`: Enable the use of uv environments (default: `false`) +- `uv.cacheDir`: The path where uv environments are stored +- `uv.createTimeout`: Timeout for environment creation (default: `20 min`) +- `uv.installOptions`: Extra command line options for `uv pip install` +- `uv.pythonVersion`: Python version for virtual environment creation diff --git a/modules/nextflow/src/main/groovy/nextflow/Session.groovy b/modules/nextflow/src/main/groovy/nextflow/Session.groovy index 720a79a6e0..a30564a1a3 100644 --- a/modules/nextflow/src/main/groovy/nextflow/Session.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/Session.groovy @@ -69,6 +69,7 @@ import nextflow.script.ScriptRunner import nextflow.script.WorkflowMetadata import nextflow.script.dsl.ProcessConfigBuilder import nextflow.spack.SpackConfig +import nextflow.uv.UvConfig import nextflow.trace.LogObserver import nextflow.trace.TraceObserver import nextflow.trace.TraceObserverFactory @@ -1160,6 +1161,12 @@ class Session implements ISession { return new SpackConfig(opts, getSystemEnv()) } + @Memoized + UvConfig getUvConfig() { + final opts = config.uv as Map ?: Collections.emptyMap() + return new UvConfig(opts, getSystemEnv()) + } + /** * Get the container engine configuration for the specified engine. If no engine is specified * if returns the one enabled in the configuration file. If no configuration is found diff --git a/modules/nextflow/src/main/groovy/nextflow/cli/CmdRun.groovy b/modules/nextflow/src/main/groovy/nextflow/cli/CmdRun.groovy index 60e5c174a4..a8310bc558 100644 --- a/modules/nextflow/src/main/groovy/nextflow/cli/CmdRun.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/cli/CmdRun.groovy @@ -257,6 +257,12 @@ class CmdRun extends CmdBase implements HubOptions { @Parameter(names=['-without-spack'], description = 'Disable the use of Spack environments') Boolean withoutSpack + @Parameter(names=['-with-uv'], description = 'Use the specified uv environment packages or requirements file') + String withUv + + @Parameter(names=['-without-uv'], description = 'Disable the use of uv environments') + Boolean withoutUv + @Parameter(names=['-offline'], description = 'Do not check for remote project updates') boolean offline = System.getenv('NXF_OFFLINE')=='true' @@ -321,6 +327,9 @@ class CmdRun extends CmdBase implements HubOptions { if( withSpack && withoutSpack ) throw new AbortOperationException("Command line options `-with-spack` and `-without-spack` cannot be specified at the same time") + if( withUv && withoutUv ) + throw new AbortOperationException("Command line options `-with-uv` and `-without-uv` cannot be specified at the same time") + if( offline && latest ) throw new AbortOperationException("Command line options `-latest` and `-offline` cannot be specified at the same time") diff --git a/modules/nextflow/src/main/groovy/nextflow/cli/Launcher.groovy b/modules/nextflow/src/main/groovy/nextflow/cli/Launcher.groovy index 387b58b67b..582477d739 100644 --- a/modules/nextflow/src/main/groovy/nextflow/cli/Launcher.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/cli/Launcher.groovy @@ -302,6 +302,10 @@ class Launcher { normalized << '-' } + else if( current == '-with-uv' && (i==args.size() || args[i].startsWith('-'))) { + normalized << '-' + } + else if( current == '-with-weblog' && (i==args.size() || args[i].startsWith('-'))) { normalized << '-' } diff --git a/modules/nextflow/src/main/groovy/nextflow/config/ConfigBuilder.groovy b/modules/nextflow/src/main/groovy/nextflow/config/ConfigBuilder.groovy index 087c0efd39..3b2bc81840 100644 --- a/modules/nextflow/src/main/groovy/nextflow/config/ConfigBuilder.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/config/ConfigBuilder.groovy @@ -625,6 +625,19 @@ class ConfigBuilder { config.spack.enabled = true } + if( cmdRun.withoutUv && config.uv instanceof Map ) { + // disable uv execution + log.debug "Disabling execution with uv as requested by command-line option `-without-uv`" + config.uv.enabled = false + } + + // -- apply the uv environment + if( cmdRun.withUv ) { + if( cmdRun.withUv != '-' ) + config.process.uv = cmdRun.withUv + config.uv.enabled = true + } + // -- sets the resume option if( cmdRun.resume ) config.resume = cmdRun.resume diff --git a/modules/nextflow/src/main/groovy/nextflow/executor/BashWrapperBuilder.groovy b/modules/nextflow/src/main/groovy/nextflow/executor/BashWrapperBuilder.groovy index 3005364cdf..49f1362109 100644 --- a/modules/nextflow/src/main/groovy/nextflow/executor/BashWrapperBuilder.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/executor/BashWrapperBuilder.groovy @@ -354,6 +354,7 @@ class BashWrapperBuilder { binding.before_script = getBeforeScriptSnippet() binding.conda_activate = getCondaActivateSnippet() binding.spack_activate = getSpackActivateSnippet() + binding.uv_activate = getUvActivateSnippet() /* * add the task environment @@ -573,6 +574,15 @@ class BashWrapperBuilder { return result } + private String getUvActivateSnippet() { + if( !uvEnv ) + return null + return """\ + # uv environment + source ${Escape.path(uvEnv)}/bin/activate + """.stripIndent() + } + protected String getTraceCommand(String interpreter) { String result = "${interpreter} ${fileStr(scriptFile)}" if( input != null ) diff --git a/modules/nextflow/src/main/groovy/nextflow/processor/TaskArrayCollector.groovy b/modules/nextflow/src/main/groovy/nextflow/processor/TaskArrayCollector.groovy index 5555d440cc..8a79e2924e 100644 --- a/modules/nextflow/src/main/groovy/nextflow/processor/TaskArrayCollector.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/processor/TaskArrayCollector.groovy @@ -57,6 +57,7 @@ class TaskArrayCollector { 'containerOptions', // only needed when using Wave 'conda', + 'uv', ] private TaskProcessor processor diff --git a/modules/nextflow/src/main/groovy/nextflow/processor/TaskBean.groovy b/modules/nextflow/src/main/groovy/nextflow/processor/TaskBean.groovy index 1458dee615..127a58609b 100644 --- a/modules/nextflow/src/main/groovy/nextflow/processor/TaskBean.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/processor/TaskBean.groovy @@ -51,6 +51,8 @@ class TaskBean implements Serializable, Cloneable { Path spackEnv + Path uvEnv + List moduleNames Path workDir @@ -140,6 +142,7 @@ class TaskBean implements Serializable, Cloneable { this.condaEnv = task.getCondaEnv() this.useMicromamba = task.getCondaConfig()?.useMicromamba() this.spackEnv = task.getSpackEnv() + this.uvEnv = task.getUvEnv() this.moduleNames = task.config.getModule() this.shell = task.config.getShell() ?: BashWrapperBuilder.BASH this.script = task.getScript() diff --git a/modules/nextflow/src/main/groovy/nextflow/processor/TaskHasher.groovy b/modules/nextflow/src/main/groovy/nextflow/processor/TaskHasher.groovy index 723488a6de..c885a49902 100644 --- a/modules/nextflow/src/main/groovy/nextflow/processor/TaskHasher.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/processor/TaskHasher.groovy @@ -120,6 +120,12 @@ class TaskHasher { } } + // add uv packages (`uv` directive) + final uv = task.getUvEnv() + if( uv ) { + keys.add(uv) + } + // add stub run marker if enabled if( session.stubRun && task.config.getStubBlock() ) { keys.add('stub-run') diff --git a/modules/nextflow/src/main/groovy/nextflow/processor/TaskRun.groovy b/modules/nextflow/src/main/groovy/nextflow/processor/TaskRun.groovy index 4537cf623c..1b2cf58cf9 100644 --- a/modules/nextflow/src/main/groovy/nextflow/processor/TaskRun.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/processor/TaskRun.groovy @@ -51,6 +51,8 @@ import nextflow.script.params.InParam import nextflow.script.params.OutParam import nextflow.script.params.ValueOutParam import nextflow.spack.SpackCache +import nextflow.uv.UvCache +import nextflow.uv.UvConfig import nextflow.util.ArrayBag /** * Models a task instance @@ -691,6 +693,29 @@ class TaskRun implements Cloneable { cache.getCachePathFor(config.spack as String, arch) } + Path getUvEnv() { + // note: use an explicit function instead of a closure or lambda syntax, otherwise + // when calling this method from a subclass it will result into a MissingMethodExeception + // see https://issues.apache.org/jira/browse/GROOVY-2433 + cache0.computeIfAbsent('uvEnv', new Function() { + @Override + Path apply(String it) { + return getUvEnv0() + }}) + } + + private Path getUvEnv0() { + if( !config.uv || !getUvConfig().isEnabled() ) + return null + + final cache = new UvCache(getUvConfig()) + cache.getCachePathFor(config.uv as String) + } + + UvConfig getUvConfig() { + return processor.session.getUvConfig() + } + protected ContainerInfo containerInfo() { // note: use an explicit function instead of a closure or lambda syntax, otherwise // when calling this method from a subclass it will result into a MissingMethodException diff --git a/modules/nextflow/src/main/groovy/nextflow/script/dsl/ProcessBuilder.groovy b/modules/nextflow/src/main/groovy/nextflow/script/dsl/ProcessBuilder.groovy index 2f63c30d62..355df7612d 100644 --- a/modules/nextflow/src/main/groovy/nextflow/script/dsl/ProcessBuilder.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/script/dsl/ProcessBuilder.groovy @@ -81,7 +81,8 @@ class ProcessBuilder { 'stageOutMode', 'storeDir', 'tag', - 'time' + 'time', + 'uv' ] protected BaseScript ownerScript diff --git a/modules/nextflow/src/main/groovy/nextflow/uv/UvCache.groovy b/modules/nextflow/src/main/groovy/nextflow/uv/UvCache.groovy new file mode 100644 index 0000000000..6c58863a78 --- /dev/null +++ b/modules/nextflow/src/main/groovy/nextflow/uv/UvCache.groovy @@ -0,0 +1,343 @@ +/* + * Copyright 2013-2026, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package nextflow.uv + +import java.nio.file.FileSystems +import java.nio.file.Path +import java.nio.file.Paths +import java.util.concurrent.ConcurrentHashMap + +import groovy.transform.CompileStatic +import groovy.transform.PackageScope +import groovy.util.logging.Slf4j +import groovyx.gpars.dataflow.DataflowVariable +import groovyx.gpars.dataflow.LazyDataflowVariable +import nextflow.Global +import nextflow.SysEnv +import nextflow.file.FileMutex +import nextflow.util.CacheHelper +import nextflow.util.Duration +import nextflow.util.Escape +import nextflow.util.TestOnly + +/** + * Handle uv virtual environment creation and caching + * + * @author Evan Floden + */ +@Slf4j +@CompileStatic +class UvCache { + + /** + * Cache the prefix path for each uv environment + */ + static final private Map> uvPrefixPaths = new ConcurrentHashMap<>() + + /** + * The uv settings defined in the nextflow config file + */ + private UvConfig config + + /** + * Timeout after which the environment creation is aborted + */ + private Duration createTimeout + + private String installOptions + + private String pythonVersion + + private Path configCacheDir0 + + @PackageScope String getInstallOptions() { installOptions } + + @PackageScope Duration getCreateTimeout() { createTimeout } + + @PackageScope Map getEnv() { SysEnv.get() } + + @PackageScope Path getConfigCacheDir0() { configCacheDir0 } + + @PackageScope String getPythonVersion() { pythonVersion } + + @TestOnly + protected UvCache() {} + + /** + * Create a uv env cache object + * + * @param config A {@link UvConfig} object + */ + UvCache(UvConfig config) { + this.config = config + + if( config.createTimeout() ) + createTimeout = config.createTimeout() + + if( config.installOptions() ) + installOptions = config.installOptions() + + if( config.cacheDir() ) + configCacheDir0 = config.cacheDir().toAbsolutePath() + + if( config.pythonVersion() ) + pythonVersion = config.pythonVersion() + } + + /** + * Retrieve the directory where store the uv environment. + * + * It tries these settings in the following order: + * 1) {@code uv.cacheDir} setting in the nextflow config file; + * 2) the {@code $workDir/uv} path + * + * @return + * the {@code Path} where store the uv envs + */ + @PackageScope + Path getCacheDir() { + + def cacheDir = configCacheDir0 + + if( !cacheDir && getEnv().NXF_UV_CACHEDIR ) + cacheDir = getEnv().NXF_UV_CACHEDIR as Path + + if( !cacheDir ) + cacheDir = getSessionWorkDir().resolve('uv') + + if( cacheDir.fileSystem != FileSystems.default ) { + throw new IOException("Cannot store uv environments to a remote work directory -- Use a POSIX compatible work directory or specify an alternative path with the `uv.cacheDir` config setting") + } + + if( !cacheDir.exists() && !cacheDir.mkdirs() ) { + throw new IOException("Failed to create uv cache directory: $cacheDir -- Make sure a file with the same name does not exist and you have write permission") + } + + return cacheDir + } + + @PackageScope Path getSessionWorkDir() { + Global.session.workDir + } + + @PackageScope + boolean isRequirementsFile(String str) { + (str.endsWith('.txt') || str.endsWith('.in')) && !str.contains('\n') + } + + @PackageScope + boolean isPyProjectFile(String str) { + str.endsWith('pyproject.toml') && !str.contains('\n') + } + + /** + * Get the path on the file system where store a uv environment + * + * @param uvEnv The uv environment specification + * @return the uv unique prefix {@link Path} where the env is created + */ + @PackageScope + Path uvPrefixPath(String uvEnv) { + assert uvEnv + + String content + String name = 'env' + // check if it's a requirements file + if( isRequirementsFile(uvEnv) || isPyProjectFile(uvEnv) ) { + try { + final path = uvEnv as Path + content = path.text + } + catch( Exception e ) { + throw new IllegalArgumentException("Error reading uv environment file: $uvEnv -- Check the log file for details", e) + } + } + // it's interpreted as user provided prefix directory + else if( uvEnv.contains('/') ) { + final prefix = uvEnv as Path + if( !prefix.isDirectory() ) + throw new IllegalArgumentException("uv environment path does not exist or is not a directory: $prefix") + if( prefix.fileSystem != FileSystems.default ) + throw new IllegalArgumentException("uv environment path must be a POSIX file path: $prefix") + + return prefix + } + else if( uvEnv.contains('\n') ) { + throw new IllegalArgumentException("Invalid uv environment definition: $uvEnv") + } + else { + content = uvEnv + } + + // include python version in hash if specified + if( pythonVersion ) content += "\npython:$pythonVersion" + + final hash = CacheHelper.hasher(content).hash().toString() + getCacheDir().resolve("$name-$hash") + } + + /** + * Run the uv tool to create a virtual environment and install packages. + * + * @param uvEnv The uv environment definition + * @param prefixPath The target path for the virtual environment + * @return the uv environment prefix {@link Path} + */ + @PackageScope + Path createLocalUvEnv(String uvEnv, Path prefixPath) { + + if( prefixPath.isDirectory() ) { + log.debug "uv found local env for environment=$uvEnv; path=$prefixPath" + return prefixPath + } + + final file = new File("${prefixPath.parent}/.${prefixPath.name}.lock") + final wait = "Another Nextflow instance is creating the uv environment $uvEnv -- please wait till it completes" + final err = "Unable to acquire exclusive lock after $createTimeout on file: $file" + + final mutex = new FileMutex(target: file, timeout: createTimeout, waitMessage: wait, errorMessage: err) + try { + mutex .lock { createLocalUvEnv0(uvEnv, prefixPath) } + } + finally { + file.delete() + } + + return prefixPath + } + + @PackageScope + Path makeAbsolute( String envFile ) { + Paths.get(envFile).toAbsolutePath() + } + + @PackageScope + Path createLocalUvEnv0(String uvEnv, Path prefixPath) { + if( prefixPath.isDirectory() ) { + log.debug "uv found local env for environment=$uvEnv; path=$prefixPath" + return prefixPath + } + + log.info "Creating env using uv: $uvEnv [cache $prefixPath]" + + String opts = installOptions ? "$installOptions " : '' + String pythonOpt = pythonVersion ? "--python $pythonVersion " : '' + + def cmd + // First create the virtual environment + cmd = "uv venv ${pythonOpt}${Escape.path(prefixPath)} && " + + if( isRequirementsFile(uvEnv) ) { + cmd += "uv pip install ${opts}--python ${Escape.path(prefixPath.resolve('bin/python'))} -r ${Escape.path(makeAbsolute(uvEnv))}" + } + else if( isPyProjectFile(uvEnv) ) { + cmd += "uv pip install ${opts}--python ${Escape.path(prefixPath.resolve('bin/python'))} -r ${Escape.path(makeAbsolute(uvEnv))}" + } + else { + // space-separated package list e.g. 'numpy pandas matplotlib' + cmd += "uv pip install ${opts}--python ${Escape.path(prefixPath.resolve('bin/python'))} $uvEnv" + } + + try { + runCommand( cmd ) + log.debug "'uv' create complete env=$uvEnv path=$prefixPath" + } + catch( Exception e ){ + // clean-up to avoid to keep eventually corrupted environment + prefixPath.deleteDir() + throw e + } + return prefixPath + } + + @PackageScope + int runCommand( String cmd ) { + log.trace """uv create + command: $cmd + timeout: $createTimeout""".stripIndent(true) + + final max = createTimeout.toMillis() + final builder = new ProcessBuilder(['bash','-c',cmd]) + final proc = builder.redirectErrorStream(true).start() + final err = new StringBuilder() + final consumer = proc.consumeProcessOutputStream(err) + proc.waitForOrKill(max) + def status = proc.exitValue() + if( status != 0 ) { + consumer.join() + def msg = "Failed to create uv environment\n command: $cmd\n status : $status\n message:\n" + msg += err.toString().trim().indent(' ') + throw new IllegalStateException(msg) + } + return status + } + + /** + * Given a uv environment string returns a {@link DataflowVariable} which holds + * the local environment path. + * + * This method synchronise multiple concurrent requests so that only one + * environment creation is actually executed. + * + * @param uvEnv + * uv environment string + * @return + * The {@link DataflowVariable} which hold (and create) the local environment + */ + @PackageScope + DataflowVariable getLazyImagePath(String uvEnv) { + final prefixPath = uvPrefixPath(uvEnv) + final uvEnvPath = prefixPath.toString() + if( uvEnvPath in uvPrefixPaths ) { + log.trace "uv found local environment `$uvEnv`" + return uvPrefixPaths[uvEnvPath] + } + + synchronized (uvPrefixPaths) { + def result = uvPrefixPaths[uvEnvPath] + if( result == null ) { + result = new LazyDataflowVariable({ createLocalUvEnv(uvEnv, prefixPath) }) + uvPrefixPaths[uvEnvPath] = result + } + else { + log.trace "uv found local cache for environment `$uvEnv` (2)" + } + return result + } + } + + /** + * Create a uv environment caching it in the file system. + * + * This method synchronise multiple concurrent requests so that only one + * environment is actually created. + * + * @param uvEnv The uv environment string + * @return the local environment path prefix {@link Path} + */ + Path getCachePathFor(String uvEnv) { + def promise = getLazyImagePath(uvEnv) + def result = promise.getVal() + if( promise.isError() ) + throw new IllegalStateException(promise.getError()) + if( !result ) + throw new IllegalStateException("Cannot create uv environment `$uvEnv`") + log.trace "uv cache for env `$uvEnv` path=$result" + return result + } + +} diff --git a/modules/nextflow/src/main/groovy/nextflow/uv/UvConfig.groovy b/modules/nextflow/src/main/groovy/nextflow/uv/UvConfig.groovy new file mode 100644 index 0000000000..bc738ec784 --- /dev/null +++ b/modules/nextflow/src/main/groovy/nextflow/uv/UvConfig.groovy @@ -0,0 +1,98 @@ +/* + * Copyright 2013-2026, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package nextflow.uv + +import java.nio.file.Path + +import groovy.transform.CompileStatic +import nextflow.config.spec.ConfigOption +import nextflow.config.spec.ConfigScope +import nextflow.config.spec.ScopeName +import nextflow.script.dsl.Description +import nextflow.util.Duration + +/** + * Model uv configuration + * + * @author Evan Floden + */ +@ScopeName("uv") +@Description(""" + The `uv` scope controls the creation of Python virtual environments by the uv package manager. +""") +@CompileStatic +class UvConfig implements ConfigScope { + + @ConfigOption + @Description(""" + Execute tasks with uv virtual environments (default: `false`). + """) + final boolean enabled + + @ConfigOption + @Description(""" + The path where uv virtual environments are stored. It should be accessible from all compute nodes when using a shared file system. + """) + final String cacheDir + + @ConfigOption + @Description(""" + Extra command line options for the `uv pip install` command. See the [uv documentation](https://docs.astral.sh/uv/) for more information. + """) + final String installOptions + + @ConfigOption + @Description(""" + The amount of time to wait for the uv environment to be created before failing (default: `20 min`). + """) + final Duration createTimeout + + @ConfigOption + @Description(""" + The Python version to use when creating virtual environments (e.g. `3.12`). If not specified, uv will use its default Python resolution. + """) + final String pythonVersion + + /* required by extension point -- do not remove */ + UvConfig() {} + + UvConfig(Map opts, Map env) { + enabled = opts.enabled != null + ? opts.enabled as boolean + : (env.NXF_UV_ENABLED?.toString() == 'true') + cacheDir = opts.cacheDir + installOptions = opts.installOptions + createTimeout = opts.createTimeout as Duration ?: Duration.of('20min') + pythonVersion = opts.pythonVersion + } + + Duration createTimeout() { + createTimeout + } + + String installOptions() { + installOptions + } + + Path cacheDir() { + cacheDir as Path + } + + String pythonVersion() { + pythonVersion + } +} diff --git a/modules/nextflow/src/main/resources/nextflow/executor/command-run.txt b/modules/nextflow/src/main/resources/nextflow/executor/command-run.txt index 7c267ae96b..c0ba06b0aa 100644 --- a/modules/nextflow/src/main/resources/nextflow/executor/command-run.txt +++ b/modules/nextflow/src/main/resources/nextflow/executor/command-run.txt @@ -165,6 +165,7 @@ nxf_main() { {{module_load}} {{conda_activate}} {{spack_activate}} + {{uv_activate}} set -u {{task_env}} {{secrets_env}} diff --git a/modules/nextflow/src/test/groovy/nextflow/cli/LauncherTest.groovy b/modules/nextflow/src/test/groovy/nextflow/cli/LauncherTest.groovy index eec797be06..e62682e624 100644 --- a/modules/nextflow/src/test/groovy/nextflow/cli/LauncherTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/cli/LauncherTest.groovy @@ -239,6 +239,10 @@ class LauncherTest extends Specification { launcher.normalizeArgs('run','-with-spack', '-x') == ['run', '-with-spack','-', '-x'] launcher.normalizeArgs('run','-with-spack', 'busybox') == ['run', '-with-spack','busybox'] + launcher.normalizeArgs('run','-with-uv') == ['run', '-with-uv','-'] + launcher.normalizeArgs('run','-with-uv', '-x') == ['run', '-with-uv','-', '-x'] + launcher.normalizeArgs('run','-with-uv', 'numpy') == ['run', '-with-uv','numpy'] + launcher.normalizeArgs('run','-dump-channels') == ['run', '-dump-channels','*'] launcher.normalizeArgs('run','-dump-channels', '-x') == ['run', '-dump-channels','*', '-x'] launcher.normalizeArgs('run','-dump-channels', 'foo,bar') == ['run', '-dump-channels','foo,bar'] diff --git a/modules/nextflow/src/test/groovy/nextflow/config/ConfigBuilderTest.groovy b/modules/nextflow/src/test/groovy/nextflow/config/ConfigBuilderTest.groovy index 991dd9368f..d975661aba 100644 --- a/modules/nextflow/src/test/groovy/nextflow/config/ConfigBuilderTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/config/ConfigBuilderTest.groovy @@ -1407,6 +1407,49 @@ class ConfigBuilderTest extends Specification { !config.process.spack } + def 'should set uv env' () { + given: + def env = [:] + def builder = [:] as ConfigBuilder + + when: + def config = new ConfigObject() + builder.configRunOptions(config, env, new CmdRun(withUv: 'numpy pandas')) + then: + config.uv instanceof Map + config.uv.enabled + config.process.uv == 'numpy pandas' + + when: + config = new ConfigObject() + config.process.uv = 'numpy' + builder.configRunOptions(config, env, new CmdRun(withUv: '-')) + then: + config.uv instanceof Map + config.uv.enabled + config.process.uv == 'numpy' + } + + def 'should disable uv env' () { + given: + def file = Files.createTempFile('test','config') + file.deleteOnExit() + file.text = + ''' + uv { + enabled = true + } + ''' + + when: + def opt = new CliOptions(config: [file.toFile().canonicalPath] ) + def run = new CmdRun(withoutUv: true) + def config = new ConfigBuilder().setOptions(opt).setCmdRun(run).build() + then: + !config.uv.enabled + !config.process.uv + } + def 'SHOULD SET `RESUME` OPTION'() { given: diff --git a/modules/nextflow/src/test/groovy/nextflow/executor/BashWrapperBuilderTest.groovy b/modules/nextflow/src/test/groovy/nextflow/executor/BashWrapperBuilderTest.groovy index 0a34b825df..2da65c36d5 100644 --- a/modules/nextflow/src/test/groovy/nextflow/executor/BashWrapperBuilderTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/executor/BashWrapperBuilderTest.groovy @@ -834,6 +834,24 @@ class BashWrapperBuilderTest extends Specification { } + def 'should create uv activate snippet' () { + when: + def binding = newBashWrapperBuilder().makeBinding() + then: + binding.uv_activate == null + binding.containsKey('uv_activate') + + when: + def UV = Paths.get('/some/uv/env/foo') + binding = newBashWrapperBuilder(uvEnv: UV).makeBinding() + then: + binding.uv_activate == '''\ + # uv environment + source /some/uv/env/foo/bin/activate + '''.stripIndent() + + } + def 'should cleanup scratch dir' () { when: def binding = newBashWrapperBuilder().makeBinding() diff --git a/modules/nextflow/src/test/groovy/nextflow/uv/UvCacheTest.groovy b/modules/nextflow/src/test/groovy/nextflow/uv/UvCacheTest.groovy new file mode 100644 index 0000000000..9faaa7c1d1 --- /dev/null +++ b/modules/nextflow/src/test/groovy/nextflow/uv/UvCacheTest.groovy @@ -0,0 +1,212 @@ +/* + * Copyright 2013-2026, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package nextflow.uv + +import java.nio.file.Files +import java.nio.file.Paths + +import nextflow.SysEnv +import spock.lang.Specification + +/** + * + * @author Evan Floden + */ +class UvCacheTest extends Specification { + + def setupSpec() { + SysEnv.push([:]) + } + + def cleanupSpec() { + SysEnv.pop() + } + + def 'should detect requirements file' () { + given: + def cache = new UvCache() + + expect: + !cache.isRequirementsFile('numpy') + cache.isRequirementsFile('requirements.txt') + cache.isRequirementsFile('requirements.in') + !cache.isRequirementsFile("requirements.txt\nfoo") + } + + def 'should detect pyproject file' () { + given: + def cache = new UvCache() + + expect: + !cache.isPyProjectFile('numpy') + cache.isPyProjectFile('pyproject.toml') + cache.isPyProjectFile('/path/to/pyproject.toml') + !cache.isPyProjectFile("pyproject.toml\nfoo") + } + + def 'should create uv env prefix path for a string env' () { + given: + def ENV = 'numpy pandas' + def cache = Spy(UvCache) + def BASE = Paths.get('/uv/envs') + + when: + def prefix = cache.uvPrefixPath(ENV) + then: + 1 * cache.isRequirementsFile(ENV) + 1 * cache.getCacheDir() >> BASE + prefix.toString().startsWith('/uv/envs/env-') + } + + def 'should create uv env prefix path for a requirements file' () { + given: + def folder = Files.createTempDirectory('test') + def cache = Spy(UvCache) + def BASE = Paths.get('/uv/envs') + def ENV = folder.resolve('requirements.txt') + ENV.text = '''\ + numpy==1.24.0 + pandas>=2.0 + '''.stripIndent() + + when: + def prefix = cache.uvPrefixPath(ENV.toString()) + then: + 1 * cache.isRequirementsFile(ENV.toString()) + 1 * cache.getCacheDir() >> BASE + prefix.toString().startsWith('/uv/envs/env-') + + cleanup: + folder?.deleteDir() + } + + def 'should return existing directory for path with slash' () { + given: + def folder = Files.createTempDirectory('test') + def cache = Spy(UvCache) + + when: + def prefix = cache.uvPrefixPath(folder.toString()) + then: + prefix == folder + + cleanup: + folder?.deleteDir() + } + + def 'should throw for non-existing directory path' () { + given: + def cache = Spy(UvCache) + def ENV = '/non/existing/path' + + when: + cache.uvPrefixPath(ENV) + then: + thrown(IllegalArgumentException) + } + + def 'should throw for multi-line env' () { + given: + def cache = Spy(UvCache) + def ENV = "numpy\npandas" + + when: + cache.uvPrefixPath(ENV) + then: + thrown(IllegalArgumentException) + } + + def 'should create the correct uv venv command for package list' () { + given: + def folder = Files.createTempDirectory('test') + def prefixPath = folder.resolve('env-abc123') + def cache = Spy(UvCache) + cache.@installOptions = null + cache.@pythonVersion = null + cache.@createTimeout = nextflow.util.Duration.of('20min') + + when: + cache.createLocalUvEnv0('numpy pandas', prefixPath) + then: + 1 * cache.runCommand({ String cmd -> + cmd.contains('uv venv') && cmd.contains('uv pip install') && cmd.contains('numpy pandas') + }) >> 0 + + cleanup: + folder?.deleteDir() + } + + def 'should create the correct uv venv command with python version' () { + given: + def folder = Files.createTempDirectory('test') + def prefixPath = folder.resolve('env-abc123') + def cache = Spy(UvCache) + cache.@installOptions = null + cache.@pythonVersion = '3.11' + cache.@createTimeout = nextflow.util.Duration.of('20min') + + when: + cache.createLocalUvEnv0('numpy', prefixPath) + then: + 1 * cache.runCommand({ String cmd -> + cmd.contains('uv venv --python 3.11') && cmd.contains('uv pip install') + }) >> 0 + + cleanup: + folder?.deleteDir() + } + + def 'should create the correct uv venv command for requirements file' () { + given: + def folder = Files.createTempDirectory('test') + def reqFile = folder.resolve('requirements.txt') + reqFile.text = 'numpy==1.24.0\npandas>=2.0' + def prefixPath = folder.resolve('env-abc123') + def cache = Spy(UvCache) + cache.@installOptions = null + cache.@pythonVersion = null + cache.@createTimeout = nextflow.util.Duration.of('20min') + + when: + cache.createLocalUvEnv0(reqFile.toString(), prefixPath) + then: + 1 * cache.runCommand({ String cmd -> + cmd.contains('uv venv') && cmd.contains('-r') && cmd.contains('requirements.txt') + }) >> 0 + + cleanup: + folder?.deleteDir() + } + + def 'should include python version in hash' () { + given: + def cache1 = Spy(UvCache) + cache1.@pythonVersion = null + def cache2 = Spy(UvCache) + cache2.@pythonVersion = '3.12' + def BASE = Paths.get('/uv/envs') + + when: + def prefix1 = cache1.uvPrefixPath('numpy') + def prefix2 = cache2.uvPrefixPath('numpy') + then: + _ * cache1.getCacheDir() >> BASE + _ * cache2.getCacheDir() >> BASE + prefix1 != prefix2 + } + +} diff --git a/modules/nextflow/src/test/groovy/nextflow/uv/UvConfigTest.groovy b/modules/nextflow/src/test/groovy/nextflow/uv/UvConfigTest.groovy new file mode 100644 index 0000000000..77c07333d0 --- /dev/null +++ b/modules/nextflow/src/test/groovy/nextflow/uv/UvConfigTest.groovy @@ -0,0 +1,67 @@ +/* + * Copyright 2013-2026, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package nextflow.uv + +import spock.lang.Specification +import spock.lang.Unroll + +/** + * + * @author Evan Floden + */ +class UvConfigTest extends Specification { + + @Unroll + def 'should check enabled flag'() { + given: + def uv = new UvConfig(CONFIG, ENV) + expect: + uv.isEnabled() == EXPECTED + + where: + EXPECTED | CONFIG | ENV + false | [:] | [:] + false | [enabled: false] | [:] + true | [enabled: true] | [:] + and: + false | [:] | [NXF_UV_ENABLED: 'false'] + true | [:] | [NXF_UV_ENABLED: 'true'] + false | [enabled: false] | [NXF_UV_ENABLED: 'true'] // <-- config has priority + true | [enabled: true] | [NXF_UV_ENABLED: 'true'] + } + + def 'should check python version'() { + given: + def uv = new UvConfig([pythonVersion: '3.12'], [:]) + expect: + uv.pythonVersion() == '3.12' + } + + def 'should check install options'() { + given: + def uv = new UvConfig([installOptions: '--no-cache'], [:]) + expect: + uv.installOptions() == '--no-cache' + } + + def 'should have default create timeout'() { + given: + def uv = new UvConfig([:], [:]) + expect: + uv.createTimeout().toMillis() == 20 * 60 * 1000 + } +} diff --git a/modules/nf-lineage/src/main/nextflow/lineage/LinObserver.groovy b/modules/nf-lineage/src/main/nextflow/lineage/LinObserver.groovy index d6fb7175c7..115f67ce5a 100644 --- a/modules/nf-lineage/src/main/nextflow/lineage/LinObserver.groovy +++ b/modules/nf-lineage/src/main/nextflow/lineage/LinObserver.groovy @@ -264,6 +264,7 @@ class LinObserver implements TraceObserverV2 { task.isContainerEnabled() ? task.getContainerFingerprint() : null, normalizer.normalizePath(task.getCondaEnv()), normalizer.normalizePath(task.getSpackEnv()), + normalizer.normalizePath(task.getUvEnv()), task.config?.getArchitecture()?.toString(), getTaskGlobalVars(task), getTaskBinEntries(task).collect { Path p -> new DataPath( diff --git a/modules/nf-lineage/src/main/nextflow/lineage/model/v1beta1/TaskRun.groovy b/modules/nf-lineage/src/main/nextflow/lineage/model/v1beta1/TaskRun.groovy index 7a1a39a05c..3645667a10 100644 --- a/modules/nf-lineage/src/main/nextflow/lineage/model/v1beta1/TaskRun.groovy +++ b/modules/nf-lineage/src/main/nextflow/lineage/model/v1beta1/TaskRun.groovy @@ -60,6 +60,10 @@ class TaskRun implements LinSerializable { * Spack environment used for the task run */ String spack + /** + * uv environment used for the task run + */ + String uv /** * Architecture defined in the Spack environment used for the task run */ diff --git a/modules/nf-lineage/src/test/nextflow/lineage/cli/LinCommandImplTest.groovy b/modules/nf-lineage/src/test/nextflow/lineage/cli/LinCommandImplTest.groovy index 41640c9a97..93a01162a5 100644 --- a/modules/nf-lineage/src/test/nextflow/lineage/cli/LinCommandImplTest.groovy +++ b/modules/nf-lineage/src/test/nextflow/lineage/cli/LinCommandImplTest.groovy @@ -225,7 +225,7 @@ class LinCommandImplTest extends Specification{ new Parameter("path","reads", ["lid://45678/output.txt"] ), new Parameter("path","input", [new DataPath("path/to/file",new Checksum("45372qe","nextflow","standard"))]) ], - null, null, null, null, [:],[]) + null, null, null, null, null, [:],[]) lidFile3.text = encoder.encode(entry) entry = new FileOutput("path/to/file",new Checksum("45372qe","nextflow","standard"), "lid://45678", "lid://45678", null, 1234, time, time, null) @@ -233,7 +233,7 @@ class LinCommandImplTest extends Specification{ entry = new TaskRun("u345-2346-1stw2", "bar", new Checksum("abfs2556","nextflow","standard"), 'this is a script', - null,null, null, null, null, [:],[]) + null,null, null, null, null, null, [:],[]) lidFile5.text = encoder.encode(entry) final network = """\ flowchart TB diff --git a/modules/nf-lineage/src/test/nextflow/lineage/serde/LinEncoderTest.groovy b/modules/nf-lineage/src/test/nextflow/lineage/serde/LinEncoderTest.groovy index fbe1d439f9..2c62066aff 100644 --- a/modules/nf-lineage/src/test/nextflow/lineage/serde/LinEncoderTest.groovy +++ b/modules/nf-lineage/src/test/nextflow/lineage/serde/LinEncoderTest.groovy @@ -203,7 +203,7 @@ class LinEncoderTest extends Specification{ def uniqueId = UUID.randomUUID() def taskRun = new TaskRun( uniqueId.toString(),"name", new Checksum("78910", "nextflow", "standard"), 'this is a script', - [new Parameter("String", "param1", "value1")], "container:version", "conda", "spack", "amd64", + [new Parameter("String", "param1", "value1")], "container:version", "conda", "spack", "uv-env", "amd64", [a: "A", b: "B"], [new DataPath("path/to/file", new Checksum("78910", "nextflow", "standard"))] ) when: @@ -221,6 +221,7 @@ class LinEncoderTest extends Specification{ result.container == "container:version" result.conda == "conda" result.spack == "spack" + result.uv == "uv-env" result.architecture == "amd64" result.globalVars == [a: "A", b: "B"] result.binEntries.size() == 1 From bfd069f63dbce96d35a1318b7b2ad3203c30d52b Mon Sep 17 00:00:00 2001 From: Evan Rees Date: Fri, 10 Apr 2026 12:50:11 -0400 Subject: [PATCH 2/2] Fix lineage TaskRun constructor calls for uv field Add the missing `uv` positional argument to all lineage model TaskRun constructor calls in tests that were missed in the initial commit (LinObserverTest, CmdLineageTest). Signed-off-by: Evan Rees Co-Authored-By: Claude Opus 4.6 (1M context) --- .../src/test/groovy/nextflow/cli/CmdLineageTest.groovy | 4 ++-- .../src/test/nextflow/lineage/LinObserverTest.groovy | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/modules/nextflow/src/test/groovy/nextflow/cli/CmdLineageTest.groovy b/modules/nextflow/src/test/groovy/nextflow/cli/CmdLineageTest.groovy index 26c9a9e27f..152b4e3822 100644 --- a/modules/nextflow/src/test/groovy/nextflow/cli/CmdLineageTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/cli/CmdLineageTest.groovy @@ -197,7 +197,7 @@ class CmdLineageTest extends Specification { 'this is a script', [new Parameter( "val", "sample_id","ggal_gut"), new Parameter("path","reads",["lid://45678/output.txt"])], - null, null, null, null, [:],[], null) + null, null, null, null, null, [:],[], null) lidFile3.text = encoder.encode(entry) entry = new FileOutput("path/to/file",new Checksum("45372qe","nextflow","standard"), "lid://45678", "lid://45678", null, 1234, time, time, null) @@ -205,7 +205,7 @@ class CmdLineageTest extends Specification { entry = new TaskRun("u345-2346-1stw2", "bar", new Checksum("abfs2556","nextflow","standard"), 'this is a script', - null,null, null, null, null, [:],[], null) + null,null, null, null, null, null, [:],[], null) lidFile5.text = encoder.encode(entry) final network = """\ flowchart TB diff --git a/modules/nf-lineage/src/test/nextflow/lineage/LinObserverTest.groovy b/modules/nf-lineage/src/test/nextflow/lineage/LinObserverTest.groovy index 976fec99b5..ecbb8cf149 100644 --- a/modules/nf-lineage/src/test/nextflow/lineage/LinObserverTest.groovy +++ b/modules/nf-lineage/src/test/nextflow/lineage/LinObserverTest.groovy @@ -682,7 +682,7 @@ class LinObserverTest extends Specification { new Parameter("path", "file1", ['lid://78567890/file1.txt']), new Parameter("path", "file2", [[path: normalizer.normalizePath(file), checksum: [value:fileHash, algorithm: "nextflow", mode: "standard"]]]), new Parameter("val", "id", "value") - ], null, null, null, null, [:], [], "lid://hash") + ], null, null, null, null, null, [:], [], "lid://hash") def dataOutput1 = new FileOutput(outFile1.toString(), new Checksum(fileHash1, "nextflow", "standard"), "lid://1234567890", "lid://hash", "lid://1234567890", attrs1.size(), LinUtils.toDate(attrs1.creationTime()), LinUtils.toDate(attrs1.lastModifiedTime()) ) def dataOutput2 = new FileOutput(outFile2.toString(), new Checksum(fileHash2, "nextflow", "standard"),