diff --git a/Cargo.lock b/Cargo.lock index 2d93f83ae7..1d2a7a831d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6493,6 +6493,7 @@ dependencies = [ "fs-err", "futures", "indexmap 2.14.0", + "indicatif", "insta", "itertools 0.14.0", "miette 7.6.0", @@ -6515,7 +6516,11 @@ dependencies = [ "pixi_consts", "pixi_git", "pixi_glob", + "pixi_install_pypi", + "pixi_manifest", "pixi_path", + "pixi_pypi_spec", + "pixi_python_status", "pixi_record", "pixi_spec", "pixi_spec_containers", @@ -6523,10 +6528,13 @@ dependencies = [ "pixi_test_utils", "pixi_url", "pixi_utils", + "pixi_uv_context", + "pixi_uv_conversions", "pixi_variant", "rattler", "rattler_conda_types", "rattler_digest", + "rattler_lock", "rattler_networking", "rattler_package_streaming", "rattler_repodata_gateway", @@ -6547,6 +6555,7 @@ dependencies = [ "tracing", "typed-path", "url", + "uv-normalize", "which", "xxhash-rust", ] @@ -6698,7 +6707,6 @@ dependencies = [ "miette 7.6.0", "once_cell", "ordermap", - "pathdiff", "pep440_rs", "pep508_rs", "pixi_build_backend_passthrough", @@ -6729,7 +6737,6 @@ dependencies = [ "pypi_modifiers", "rattler", "rattler_conda_types", - "rattler_digest", "rattler_lock", "rattler_networking", "rattler_repodata_gateway", @@ -6751,30 +6758,18 @@ dependencies = [ "tracing-test", "typed-path", "url", - "uv-build-frontend", - "uv-cache", - "uv-cache-key", "uv-client", "uv-configuration", - "uv-dispatch", "uv-distribution", "uv-distribution-filename", "uv-distribution-types", - "uv-git", "uv-git-types", - "uv-install-wheel", "uv-normalize", "uv-pep440", "uv-pep508", - "uv-preview", "uv-pypi-types", - "uv-python", "uv-redacted", - "uv-requirements", - "uv-requirements-txt", "uv-resolver", - "uv-types", - "uv-workspace", "xxhash-rust", ] @@ -6875,6 +6870,7 @@ dependencies = [ "miette 7.6.0", "once_cell", "ordermap", + "pep508_rs", "pixi_build_frontend", "pixi_command_dispatcher", "pixi_config", @@ -6883,11 +6879,16 @@ dependencies = [ "pixi_manifest", "pixi_path", "pixi_progress", + "pixi_pypi_spec", + "pixi_python_status", + "pixi_record", "pixi_reporters", "pixi_spec", "pixi_spec_containers", "pixi_toml", "pixi_utils", + "pixi_uv_context", + "pixi_uv_conversions", "rattler", "rattler_conda_types", "rattler_lock", @@ -6917,12 +6918,19 @@ version = "0.1.0" dependencies = [ "ahash", "assert_matches", + "async-once-cell", "console", "csv", "fancy_display", "fs-err", + "futures", + "indexmap 2.14.0", + "indicatif", "itertools 0.14.0", "miette 7.6.0", + "once_cell", + "ordermap", + "pathdiff", "pep440_rs", "pep508_rs", "percent-encoding", @@ -6931,15 +6939,19 @@ dependencies = [ "pixi_manifest", "pixi_path", "pixi_progress", + "pixi_pypi_spec", "pixi_python_status", "pixi_record", - "pixi_reporters", + "pixi_spec", "pixi_utils", "pixi_uv_context", "pixi_uv_conversions", + "pixi_uv_reporter", + "pypi_mapping", "pypi_modifiers", "rattler", "rattler_conda_types", + "rattler_digest", "rattler_lock", "rayon", "serde", @@ -6950,8 +6962,10 @@ dependencies = [ "tracing", "typed-path", "url", + "uv-build-frontend", "uv-cache", "uv-cache-info", + "uv-cache-key", "uv-client", "uv-configuration", "uv-dispatch", @@ -6960,6 +6974,7 @@ dependencies = [ "uv-distribution-types", "uv-flags", "uv-git", + "uv-git-types", "uv-install-wheel", "uv-installer", "uv-normalize", @@ -6969,6 +6984,8 @@ dependencies = [ "uv-pypi-types", "uv-python", "uv-redacted", + "uv-requirements", + "uv-requirements-txt", "uv-resolver", "uv-types", "uv-workspace", @@ -7114,13 +7131,13 @@ dependencies = [ "human_bytes", "indexmap 2.14.0", "indicatif", - "itertools 0.14.0", "parking_lot 0.12.5", "pixi_build_discovery", "pixi_command_dispatcher", "pixi_compute_reporters", "pixi_git", "pixi_progress", + "pixi_uv_reporter", "rattler", "rattler_conda_types", "rattler_repodata_gateway", @@ -7129,12 +7146,6 @@ dependencies = [ "tracing", "url", "uv-configuration", - "uv-distribution", - "uv-distribution-types", - "uv-installer", - "uv-normalize", - "uv-redacted", - "uv-resolver", ] [[package]] @@ -7389,6 +7400,22 @@ dependencies = [ "uv-types", ] +[[package]] +name = "pixi_uv_reporter" +version = "0.1.0" +dependencies = [ + "indicatif", + "itertools 0.14.0", + "pixi_git", + "pixi_progress", + "uv-distribution", + "uv-distribution-types", + "uv-installer", + "uv-normalize", + "uv-redacted", + "uv-resolver", +] + [[package]] name = "pixi_variant" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index d545bf8b20..4fb28d715b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -132,6 +132,7 @@ pixi_url = { path = "crates/pixi_url" } pixi_utils = { path = "crates/pixi_utils", default-features = false } pixi_uv_context = { path = "crates/pixi_uv_context" } pixi_uv_conversions = { path = "crates/pixi_uv_conversions" } +pixi_uv_reporter = { path = "crates/pixi_uv_reporter" } pixi_variant = { path = "crates/pixi_variant" } pypi_mapping = { path = "crates/pypi_mapping" } pypi_modifiers = { path = "crates/pypi_modifiers" } diff --git a/crates/pixi_cli/src/global/add.rs b/crates/pixi_cli/src/global/add.rs index 6ede6fd47c..3f4216f6a0 100644 --- a/crates/pixi_cli/src/global/add.rs +++ b/crates/pixi_cli/src/global/add.rs @@ -2,9 +2,11 @@ use crate::global::global_specs::GlobalSpecs; use crate::global::revert_environment_after_error; use clap::Parser; +use miette::IntoDiagnostic; use pixi_config::{Config, ConfigCli}; use pixi_global::project::GlobalSpec; use pixi_global::{EnvironmentName, Mapping, Project, StateChange, StateChanges}; +use pixi_pypi_spec::{PixiPypiSpec, PypiPackageName}; /// Adds dependencies to an environment /// @@ -12,6 +14,7 @@ use pixi_global::{EnvironmentName, Mapping, Project, StateChange, StateChanges}; /// /// - `pixi global add --environment python numpy` /// - `pixi global add --environment my_env pytest pytest-cov --expose pytest=pytest` +/// - `pixi global add --environment python --pypi httpx` #[derive(Parser, Debug, Clone)] #[clap(arg_required_else_help = true, verbatim_doc_comment)] pub struct Args { @@ -29,6 +32,11 @@ pub struct Args { #[arg(long)] expose: Vec, + /// Add a PyPI package to the environment, in PEP 508 format. + /// The environment must contain a python interpreter in its dependencies. + #[arg(long)] + pypi: Vec, + #[clap(flatten)] config: ConfigCli, } @@ -49,6 +57,7 @@ pub async fn execute(args: Args) -> miette::Result<()> { async fn apply_changes( env_name: &EnvironmentName, specs: &[GlobalSpec], + pypi_requirements: &[pep508_rs::Requirement], expose: &[Mapping], project: &mut Project, ) -> miette::Result { @@ -59,6 +68,15 @@ pub async fn execute(args: Args) -> miette::Result<()> { project.manifest.add_dependency(env_name, spec)?; } + // Add PyPI requirements to the manifest + for requirement in pypi_requirements { + let name = PypiPackageName::from_normalized(requirement.name.clone()); + let spec = PixiPypiSpec::try_from(requirement.clone()).into_diagnostic()?; + project + .manifest + .add_pypi_dependency(env_name, &name, &spec)?; + } + // Add expose mappings to the manifest for mapping in expose { project.manifest.add_exposed_mapping(env_name, mapping)?; @@ -116,6 +134,7 @@ pub async fn execute(args: Args) -> miette::Result<()> { match apply_changes( &args.environment, &specs, + args.pypi.as_slice(), args.expose.as_slice(), &mut project_modified, ) diff --git a/crates/pixi_cli/src/global/global_specs.rs b/crates/pixi_cli/src/global/global_specs.rs index 40856302eb..36ddbe958f 100644 --- a/crates/pixi_cli/src/global/global_specs.rs +++ b/crates/pixi_cli/src/global/global_specs.rs @@ -19,7 +19,7 @@ use crate::has_specs::HasSpecs; #[derive(Parser, Debug, Default, Clone)] pub struct GlobalSpecs { /// The dependency as names, conda MatchSpecs - #[arg(num_args = 1.., required_unless_present_any = ["git", "path"], value_name = "PACKAGE")] + #[arg(num_args = 1.., required_unless_present_any = ["git", "path", "pypi"], value_name = "PACKAGE")] pub specs: Vec, /// The git url, e.g. `https://github.com/user/repo.git` diff --git a/crates/pixi_cli/src/global/install.rs b/crates/pixi_cli/src/global/install.rs index a77ca9a5a2..e45c98ca5c 100644 --- a/crates/pixi_cli/src/global/install.rs +++ b/crates/pixi_cli/src/global/install.rs @@ -5,7 +5,8 @@ use indexmap::IndexMap; use clap::Parser; use fancy_display::FancyDisplay; use itertools::Itertools; -use miette::Report; +use miette::{IntoDiagnostic, Report}; +use pixi_pypi_spec::{PixiPypiSpec, PypiPackageName}; use rattler_conda_types::{MatchSpec, NamedChannelOrUrl, Platform}; use crate::global::{global_specs::GlobalSpecs, revert_environment_after_error}; @@ -25,6 +26,7 @@ use pixi_global::{ /// - `pixi global install jupyter --with polars` /// - `pixi global install --expose python3.8=python python=3.8` /// - `pixi global install --environment science --expose jupyter --expose ipython jupyter ipython polars` +/// - `pixi global install python --pypi httpx --pypi "flask>=2"` #[derive(Parser, Debug, Clone, Default)] #[clap(arg_required_else_help = true, verbatim_doc_comment)] pub struct Args { @@ -64,6 +66,11 @@ pub struct Args { #[arg(long)] with: Vec, + /// Add a PyPI package to the environment, in PEP 508 format. + /// The environment must contain a python interpreter in its dependencies. + #[arg(long)] + pypi: Vec, + #[clap(flatten)] config: ConfigCli, @@ -121,6 +128,12 @@ pub async fn execute(args: Args) -> miette::Result<()> { miette::bail!("Can't add packages with `--with` for more than one environment"); } + if !args.pypi.is_empty() && env_to_specs.len() != 1 { + miette::bail!( + "`--pypi` requires exactly one environment; use `--environment` to specify it" + ); + } + let mut env_changes = EnvChanges::default(); let mut last_updated_project = project_original; let mut errors: Vec<(EnvironmentName, Report)> = Vec::new(); @@ -224,6 +237,15 @@ async fn setup_environment( project.manifest.add_dependency(env_name, spec)?; } + // Add the PyPI dependencies to the environment + for requirement in &args.pypi { + let name = PypiPackageName::from_normalized(requirement.name.clone()); + let spec = PixiPypiSpec::try_from(requirement.clone()).into_diagnostic()?; + project + .manifest + .add_pypi_dependency(env_name, &name, &spec)?; + } + if !args.expose.is_empty() { project.manifest.remove_all_exposed_mappings(env_name)?; // Only add the exposed mappings that were requested diff --git a/crates/pixi_cli/src/global/remove.rs b/crates/pixi_cli/src/global/remove.rs index 566fe3dba1..aac521d437 100644 --- a/crates/pixi_cli/src/global/remove.rs +++ b/crates/pixi_cli/src/global/remove.rs @@ -2,7 +2,8 @@ use clap::Parser; use itertools::Itertools; use miette::Context; use pixi_config::{Config, ConfigCli}; -use pixi_global::{EnvironmentName, ExposedName, Project, StateChanges}; +use pixi_global::{EnvironmentName, ExposedName, Project, StateChange, StateChanges}; +use pixi_pypi_spec::PypiPackageName; use rattler_conda_types::MatchSpec; use std::str::FromStr; @@ -59,14 +60,40 @@ pub async fn execute(args: Args) -> miette::Result<()> { specs: &[MatchSpec], project: &mut Project, ) -> miette::Result { - // Remove specs from the manifest + // Snapshot the executables of the declared PyPI dependencies while + // they are still installed, so the exposed mappings of removed + // packages can be cleaned up below. + let pypi_executables = project.executables_of_pypi_dependencies(env_name).await?; + + // Remove specs from the manifest. A name that is not a conda + // dependency may refer to a PyPI dependency instead. let mut removed_dependencies = vec![]; + let mut removed_pypi_dependencies = vec![]; for spec in specs { let package_name = spec.name.as_exact().expect("package name must be exact"); - project - .manifest - .remove_dependency(env_name, package_name) - .map(|removed_name| removed_dependencies.push(removed_name))?; + let pypi_name = (!project + .environment(env_name) + .is_some_and(|env| env.dependencies.specs.contains_key(package_name))) + .then(|| PypiPackageName::from_str(package_name.as_source()).ok()) + .flatten() + .filter(|name| { + project.environment(env_name).is_some_and(|env| { + env.pypi_dependencies + .keys() + .any(|key| key.as_normalized() == name.as_normalized()) + }) + }); + + match pypi_name { + Some(name) => { + project.manifest.remove_pypi_dependency(env_name, &name)?; + removed_pypi_dependencies.push(name); + } + None => project + .manifest + .remove_dependency(env_name, package_name) + .map(|removed_name| removed_dependencies.push(removed_name))?, + } } // Figure out which package the exposed binaries belong to @@ -93,10 +120,40 @@ pub async fn execute(args: Args) -> miette::Result<()> { } } - // Sync environment - let state_changes = project - .sync_environment(env_name, Some(removed_dependencies)) - .await?; + // Remove the exposed mappings of removed PyPI dependencies. + for name in &removed_pypi_dependencies { + let Some(executables) = pypi_executables.get(name) else { + continue; + }; + for executable in executables { + if let Ok(exposed_name) = ExposedName::from_str(executable.name.as_str()) { + project + .manifest + .remove_exposed_name(env_name, &exposed_name) + .ok(); + } + } + } + + // Sync environment. A removed PyPI dependency cannot be detected by + // the sync check (its distribution lingers in site-packages), so the + // environment is reinstalled explicitly; the PyPI installer then + // removes the now-extraneous distributions. + let state_changes = if removed_pypi_dependencies.is_empty() { + project + .sync_environment(env_name, Some(removed_dependencies)) + .await? + } else { + let mut state_changes = StateChanges::new_with_env(env_name.clone()); + let mut environment_update = project.install_environment(env_name).await?; + environment_update.add_removed_packages(removed_dependencies); + state_changes.insert_change( + env_name, + StateChange::UpdatedEnvironment(environment_update), + ); + state_changes |= project.sync_environment_expose(env_name).await?; + state_changes + }; project.clear_progress(); project.manifest.save().await?; diff --git a/crates/pixi_command_dispatcher/Cargo.toml b/crates/pixi_command_dispatcher/Cargo.toml index 1c84bca721..2d305821d0 100644 --- a/crates/pixi_command_dispatcher/Cargo.toml +++ b/crates/pixi_command_dispatcher/Cargo.toml @@ -17,6 +17,7 @@ dunce = { workspace = true } fs-err = { workspace = true } futures = { workspace = true } indexmap = { workspace = true } +indicatif = { workspace = true } itertools = { workspace = true } miette = { workspace = true } nanoid = { workspace = true } @@ -39,6 +40,7 @@ xxhash-rust = { workspace = true, features = ["xxh3"] } rattler = { workspace = true } rattler_conda_types = { workspace = true } rattler_digest = { workspace = true } +rattler_lock = { workspace = true } rattler_networking = { workspace = true } rattler_package_streaming = { workspace = true } rattler_repodata_gateway = { workspace = true } @@ -59,15 +61,23 @@ pixi_compute_sources = { workspace = true } pixi_consts = { workspace = true } pixi_git = { workspace = true } pixi_glob = { workspace = true } +pixi_install_pypi = { workspace = true } +pixi_manifest = { workspace = true, features = ["rattler_lock"] } pixi_path = { workspace = true } +pixi_pypi_spec = { workspace = true } +pixi_python_status = { workspace = true } pixi_record = { workspace = true } pixi_spec = { workspace = true, features = ["pixi_build_types"] } pixi_spec_containers = { workspace = true } pixi_stable_hash = { workspace = true, features = ["serde_json"] } pixi_url = { workspace = true } pixi_utils = { workspace = true } +pixi_uv_context = { workspace = true } +pixi_uv_conversions = { workspace = true } pixi_variant = { workspace = true } +uv-normalize = { workspace = true } + [dev-dependencies] indexmap = { workspace = true } insta = { workspace = true, features = ["json"] } diff --git a/crates/pixi_command_dispatcher/src/install_pypi.rs b/crates/pixi_command_dispatcher/src/install_pypi.rs new file mode 100644 index 0000000000..5f21aa2e53 --- /dev/null +++ b/crates/pixi_command_dispatcher/src/install_pypi.rs @@ -0,0 +1,144 @@ +//! Installs PyPI packages into a previously installed conda prefix through +//! the [`CommandDispatcher`]. +//! +//! This mirrors [`crate::install_pixi`] for the PyPI side of an environment: +//! callers (the workspace install pipeline, `pixi global`, ...) construct an +//! [`InstallPypiEnvironmentSpec`] from their own manifest/lock-file types and +//! hand it to [`CommandDispatcher::install_pypi_environment`], which drives +//! the uv-based installer in `pixi_install_pypi`. + +use std::{collections::HashSet, path::PathBuf}; + +use pixi_install_pypi::{ + InstallablePypiRecord, LazyEnvironmentVariables, PyPIBuildConfig, PyPIContextConfig, + PyPIEnvironmentUpdater, PyPIUpdateConfig, derive_link_mode, +}; +use pixi_manifest::{ + EnvironmentName, PixiPlatform, + pypi::{ + ResolvedPypiExcludeNewer, + pypi_options::{IndexStrategy, NoBinary, NoBuild, NoBuildIsolation}, + }, +}; +use pixi_python_status::PythonStatus; +use pixi_record::PixiRecord; +use pixi_utils::prefix::Prefix; +use pixi_uv_context::UvResolutionContext; +use rattler_lock::PypiIndexes; + +use crate::CommandDispatcher; + +/// A specification for installing PyPI packages into a conda prefix. +/// +/// The conda packages of the environment must already be installed (see +/// [`crate::InstallPixiEnvironmentSpec`]); this spec only synchronizes the +/// PyPI packages in the prefix's `site-packages` with `pypi_records`. +pub struct InstallPypiEnvironmentSpec { + /// The name of the environment, only used for progress reporting. + pub name: EnvironmentName, + + /// The prefix in which the python interpreter lives and into which the + /// packages are installed. + pub prefix: Prefix, + + /// The platform for which the packages are installed. + pub platform: PixiPlatform, + + /// The directory against which relative paths in the records (e.g. local + /// wheels or editable installs) are resolved. For workspaces this is the + /// directory that holds the lock file. + pub lock_file_dir: PathBuf, + + /// The state of the python interpreter in the prefix, as reported by the + /// conda install transaction. Installation is skipped when no + /// interpreter is present, and outdated site-packages are removed when + /// the interpreter changed. + pub python_status: PythonStatus, + + /// The conda packages installed in the environment. Used to locate the + /// python interpreter record and to derive wheel tags. + pub pixi_records: Vec, + + /// The PyPI packages to install. + pub pypi_records: Vec, + + /// The PyPI indexes the records were locked against. + pub pypi_indexes: Option, + + /// Packages that should be built without build isolation. + pub no_build_isolation: NoBuildIsolation, + + /// Packages that must not be built from source. + pub no_build: NoBuild, + + /// Packages that must be built from source. + pub no_binary: NoBinary, + + /// The index strategy to use when fetching distributions. + pub index_strategy: Option, + + /// Exclude distributions uploaded after the given cutoffs. + pub exclude_newer: ResolvedPypiExcludeNewer, + + /// Whether to skip the wheel filename check when installing wheels. + pub skip_wheel_filename_check: Option, + + /// Package names that are never considered extraneous, i.e. they are not + /// removed from the prefix even though they are missing from + /// `pypi_records`. + pub ignored_extraneous: HashSet, + + /// The shared uv context (cache, concurrency, http settings) to use. + pub uv_context: UvResolutionContext, +} + +impl CommandDispatcher { + /// Install PyPI packages into a previously installed conda prefix. + /// + /// This method takes the PyPI side of a previously solved environment and + /// synchronizes the prefix's `site-packages` with it: missing packages + /// are installed (downloading or building them as needed), outdated ones + /// are reinstalled, and extraneous ones are removed. + /// + /// `env_variables` is resolved lazily, and only when a source + /// distribution actually has to be built; workspace callers use it to + /// expose the activated environment to PEP 517 backends. Pass `None` + /// when no extra build environment is required. + pub async fn install_pypi_environment( + &self, + spec: InstallPypiEnvironmentSpec, + env_variables: Option<&dyn LazyEnvironmentVariables>, + ) -> miette::Result<()> { + let update_config = PyPIUpdateConfig { + environment_name: &spec.name, + prefix: &spec.prefix, + platform: &spec.platform, + lock_file_dir: &spec.lock_file_dir, + }; + + let build_config = PyPIBuildConfig { + no_build_isolation: &spec.no_build_isolation, + no_build: &spec.no_build, + no_binary: &spec.no_binary, + index_strategy: spec.index_strategy.as_ref(), + exclude_newer: &spec.exclude_newer, + skip_wheel_filename_check: spec.skip_wheel_filename_check, + link_mode: Some(derive_link_mode( + self.allow_symbolic_links(), + self.allow_hard_links(), + self.allow_ref_links(), + )), + }; + + let context_config = PyPIContextConfig { + uv_context: &spec.uv_context, + pypi_indexes: spec.pypi_indexes.as_ref(), + environment_variables_lazy: env_variables, + }; + + PyPIEnvironmentUpdater::new(update_config, build_config, context_config) + .with_ignored_extraneous(spec.ignored_extraneous) + .update(&spec.python_status, &spec.pixi_records, &spec.pypi_records) + .await + } +} diff --git a/crates/pixi_command_dispatcher/src/lib.rs b/crates/pixi_command_dispatcher/src/lib.rs index 6f8a54bfb1..2927613245 100644 --- a/crates/pixi_command_dispatcher/src/lib.rs +++ b/crates/pixi_command_dispatcher/src/lib.rs @@ -55,6 +55,7 @@ mod input_globs; mod input_hash; mod install_binary; mod install_pixi; +mod install_pypi; mod installed_source_hints; mod instantiate_backend_key; mod instantiate_tool_env; @@ -63,6 +64,7 @@ pub mod reporter; mod resolved_backend_command; mod solve_binary; mod solve_conda; +mod solve_pypi; mod util; pub use backend_source_build::{ @@ -111,6 +113,7 @@ pub use install_pixi::{ InstallPixiEnvironmentError, InstallPixiEnvironmentExt, InstallPixiEnvironmentResult, InstallPixiEnvironmentSpec, }; +pub use install_pypi::InstallPypiEnvironmentSpec; pub use installed_source_hints::{InstalledSourceHint, InstalledSourceHints}; pub use instantiate_backend_key::{ BackendHandle, InstantiateBackendError, InstantiateBackendKey, ProjectModelOverrides, @@ -129,6 +132,13 @@ pub use pixi_compute_sources::{ GitCheckoutReporter, GitDir, InvalidPathError, SourceCheckout, SourceCheckoutError, SourceCheckoutExt, UrlCheckoutReporter, UrlDir, }; +// Re-export the record/config types callers need to build an +// `InstallPypiEnvironmentSpec` or `SolvePypiEnvironmentSpec`. +pub use pixi_install_pypi::{ + InstallablePypiRecord, LazyEnvironmentVariables, LockedPypiRecord, ManifestData, + UnresolvedPypiRecord, + resolve::{CondaPrefixProvider, LazyBuildDispatchDependencies, ProvidedCondaPrefix}, +}; pub use reporter::{ BackendSourceBuildReporter, BuildBackendMetadataReporter, CondaSolveReporter, GatewayReporter, InstantiateBackendReporter, PixiInstallReporter, PixiSolveEnvironmentSpec, PixiSolveReporter, @@ -138,6 +148,7 @@ pub use reporter::{ pub use resolved_backend_command::{ResolvedBackendCommand, ResolvedBackendCommandKey}; use serde::Serialize; pub use solve_conda::SolveCondaEnvironmentSpec; +pub use solve_pypi::SolvePypiEnvironmentSpec; pub use util::executor; pub use util::{Executor, Limit, Limits, PtrArc}; diff --git a/crates/pixi_command_dispatcher/src/solve_pypi.rs b/crates/pixi_command_dispatcher/src/solve_pypi.rs new file mode 100644 index 0000000000..bcaba5025c --- /dev/null +++ b/crates/pixi_command_dispatcher/src/solve_pypi.rs @@ -0,0 +1,119 @@ +//! Solves (resolves) the PyPI side of an environment through the +//! [`CommandDispatcher`]. +//! +//! This mirrors [`crate::install_pypi`] for resolution: callers construct a +//! [`SolvePypiEnvironmentSpec`] from their own manifest/lock-file types and +//! hand it to [`CommandDispatcher::solve_pypi_environment`], which drives the +//! uv-based resolver in `pixi_install_pypi::resolve`. The conda packages of +//! the environment must already be solved; conda-installed python packages +//! override their PyPI counterparts during resolution. + +use std::{path::PathBuf, sync::Arc}; + +use indexmap::IndexMap; +use indicatif::ProgressBar; +use ordermap::OrderSet; +use pixi_install_pypi::{ + LockedPypiRecord, UnresolvedPypiRecord, derive_link_mode, + resolve::{CondaPrefixProvider, LazyBuildDispatchDependencies, resolve_pypi}, +}; +use pixi_manifest::{ + PixiPlatform, SolveStrategy, + pypi::{ResolvedPypiExcludeNewer, pypi_options::PypiOptions}, +}; +use pixi_pypi_spec::PixiPypiSpec; +use pixi_record::PixiRecord; +use pixi_uv_context::UvResolutionContext; +use pixi_uv_conversions::to_exclude_newer; + +use crate::CommandDispatcher; + +/// A specification for resolving the PyPI packages of an environment. +pub struct SolvePypiEnvironmentSpec { + /// The requested PyPI dependencies. + pub dependencies: IndexMap>, + + /// The PyPI options (indexes, no-build, prerelease mode, ...) that apply + /// to the environment. + pub pypi_options: PypiOptions, + + /// The solved conda records of the environment for the target platform. + /// Used to locate the python interpreter, derive wheel tags, and detect + /// PyPI packages that are already installed by conda. + pub pixi_records: Vec, + + /// Previously locked PyPI packages. Used as resolution preferences to + /// minimize lock-file churn. + pub locked_pypi_records: Vec, + + /// The platform to resolve for. + pub platform: PixiPlatform, + + /// The directory against which relative paths (e.g. local wheels or + /// editable installs) are resolved. For workspaces this is the directory + /// that holds the lock file. + pub project_root: PathBuf, + + /// When set, fail instead of installing a conda prefix when a source + /// distribution must be built (e.g. `--no-install`). + pub disallow_install_conda_prefix: bool, + + /// Exclude distributions uploaded after the given cutoffs. + pub exclude_newer: ResolvedPypiExcludeNewer, + + /// The resolution strategy (highest, lowest, lowest-direct). + pub solve_strategy: SolveStrategy, + + /// Cache of lazily initialized build-dispatch resources (interpreter, + /// python environment, ...). Reusing the same cache across repeated + /// solves of the same environment avoids re-querying the interpreter. + pub build_dispatch_cache: Arc, + + /// The shared uv context (cache, concurrency, http settings) to use. + pub uv_context: UvResolutionContext, + + /// Progress bar to report resolution progress on. A hidden bar is used + /// when not provided. + pub progress_bar: Option, +} + +impl CommandDispatcher { + /// Resolve the PyPI packages of an environment. + /// + /// Returns the locked PyPI records for the requested dependencies, + /// resolved against the conda records in the spec. + /// + /// `prefix_provider` supplies a conda prefix (python interpreter plus + /// activation environment) on demand; it is only invoked when a source + /// distribution actually has to be built to obtain metadata. + pub async fn solve_pypi_environment( + &self, + spec: SolvePypiEnvironmentSpec, + prefix_provider: &dyn CondaPrefixProvider, + ) -> miette::Result> { + let link_mode = derive_link_mode( + self.allow_symbolic_links(), + self.allow_hard_links(), + self.allow_ref_links(), + ); + let progress_bar = spec.progress_bar.unwrap_or_else(ProgressBar::hidden); + + resolve_pypi( + spec.uv_context, + &spec.pypi_options, + spec.dependencies, + &spec.pixi_records, + &spec.locked_pypi_records, + &spec.platform, + &progress_bar, + &spec.project_root, + prefix_provider, + spec.disallow_install_conda_prefix, + to_exclude_newer(&spec.exclude_newer), + spec.solve_strategy, + &spec.build_dispatch_cache, + link_mode, + ) + .await + } +} diff --git a/crates/pixi_core/Cargo.toml b/crates/pixi_core/Cargo.toml index c963539d34..1074674439 100644 --- a/crates/pixi_core/Cargo.toml +++ b/crates/pixi_core/Cargo.toml @@ -32,7 +32,6 @@ itertools = { workspace = true } miette = { workspace = true } once_cell = { workspace = true } ordermap = { workspace = true } -pathdiff = { workspace = true } pep440_rs = { workspace = true } pep508_rs = { workspace = true } pixi_build_discovery = { workspace = true } @@ -61,7 +60,6 @@ pypi_mapping = { workspace = true } pypi_modifiers = { workspace = true } rattler = { workspace = true } rattler_conda_types = { workspace = true } -rattler_digest = { workspace = true } rattler_lock = { workspace = true } rattler_networking = { workspace = true, default-features = false } rattler_repodata_gateway = { workspace = true } @@ -82,30 +80,18 @@ toml_edit = { workspace = true } tracing = { workspace = true } typed-path = { workspace = true } url = { workspace = true } -uv-build-frontend = { workspace = true } -uv-cache = { workspace = true } -uv-cache-key = { workspace = true } uv-client = { workspace = true } uv-configuration = { workspace = true } -uv-dispatch = { workspace = true } uv-distribution = { workspace = true } uv-distribution-filename = { workspace = true } uv-distribution-types = { workspace = true } -uv-git = { workspace = true } uv-git-types = { workspace = true } -uv-install-wheel = { workspace = true } uv-normalize = { workspace = true } uv-pep440 = { workspace = true } uv-pep508 = { workspace = true } -uv-preview = { workspace = true } uv-pypi-types = { workspace = true } -uv-python = { workspace = true } uv-redacted = { workspace = true } -uv-requirements = { workspace = true } -uv-requirements-txt = { workspace = true } uv-resolver = { workspace = true } -uv-types = { workspace = true } -uv-workspace = { workspace = true } xxhash-rust = { workspace = true } [target.'cfg(unix)'.dependencies] diff --git a/crates/pixi_core/src/lock_file/mod.rs b/crates/pixi_core/src/lock_file/mod.rs index 0a505687bd..13477ab58e 100644 --- a/crates/pixi_core/src/lock_file/mod.rs +++ b/crates/pixi_core/src/lock_file/mod.rs @@ -21,7 +21,7 @@ pub use rattler_lock::Verbatim; pub use records_by_name::{ HasNameVersion, PixiRecordsByName, PypiRecordsByName, UnresolvedPixiRecordsByName, }; -pub use resolve::pypi::resolve_pypi; +pub use resolve::WorkspaceCondaPrefixProvider; pub(crate) use satisfiability::resolve_lock_platform; pub use satisfiability::{ Dependency, EnvironmentUnsat, PlatformUnsat, resolve_dev_dependencies, diff --git a/crates/pixi_core/src/lock_file/outdated.rs b/crates/pixi_core/src/lock_file/outdated.rs index 7a11b4e1a8..1e48352b2e 100644 --- a/crates/pixi_core/src/lock_file/outdated.rs +++ b/crates/pixi_core/src/lock_file/outdated.rs @@ -6,7 +6,6 @@ use std::{ use super::{ CondaPrefixUpdater, - resolve::build_dispatch::LazyBuildDispatchDependencies, satisfiability::{ VerifySatisfiabilityContext, pypi_metadata, verify_environment_satisfiability, }, @@ -28,6 +27,7 @@ use once_cell::sync::OnceCell; use pixi_command_dispatcher::executor::CancellationAwareFutures; use pixi_command_dispatcher::{CommandDispatcher, CommandDispatcherError}; use pixi_consts::consts; +use pixi_install_pypi::resolve::LazyBuildDispatchDependencies; use pixi_manifest::{EnvironmentName, FeaturesExt, PixiPlatformName}; use pixi_record::LockFileResolver; use pixi_uv_context::UvResolutionContext; @@ -38,7 +38,7 @@ use rattler_lock::{LockFile, LockedPackage}; #[derive(Default)] pub struct PypiEnvironmentBuildCache { /// Lazily initialized build dispatch dependencies (interpreter, env, etc.) - pub lazy_build_dispatch_deps: LazyBuildDispatchDependencies, + pub lazy_build_dispatch_deps: Arc, /// Optional conda prefix updater (created during satisfiability checking) pub conda_prefix_updater: OnceCell, } diff --git a/crates/pixi_core/src/lock_file/package_identifier.rs b/crates/pixi_core/src/lock_file/package_identifier.rs index 069375a373..00b8d24885 100644 --- a/crates/pixi_core/src/lock_file/package_identifier.rs +++ b/crates/pixi_core/src/lock_file/package_identifier.rs @@ -1,145 +1,31 @@ -use pixi_uv_conversions::{ - ConversionError as PixiConversionError, to_normalize, to_uv_normalize, to_uv_version, -}; -use rattler_conda_types::{PackageRecord, PackageUrl, RepoDataRecord}; -use std::{collections::HashSet, str::FromStr}; -use thiserror::Error; +//! Satisfiability support for [`PypiPackageIdentifier`]. +//! +//! The identifier itself lives in `pixi_install_pypi` (it is shared with the +//! PyPI resolve pipeline); this module adds the workspace-side check that a +//! locked PyPI requirement is satisfied by a conda-installed package. + +pub use pixi_install_pypi::package_identifier::{ConversionError, PypiPackageIdentifier}; + +use pixi_consts::consts; +use pixi_uv_conversions::{to_uv_normalize, to_uv_version}; use crate::lock_file::PlatformUnsat; use crate::lock_file::PlatformUnsat::{ DirectUrlDependencyOnCondaInstalledPackage, DirectoryDependencyOnCondaInstalledPackage, GitDependencyOnCondaInstalledPackage, PathDependencyOnCondaInstalledPackage, }; -use pixi_consts::consts; -use pixi_pypi_spec::PypiPackageName; -use uv_normalize::{ExtraName, InvalidNameError}; -/// Defines information about a Pypi package extracted from either a python -/// package or from a conda package. That can be used for comparison in both -#[derive(Debug)] -pub struct PypiPackageIdentifier { - pub name: PypiPackageName, - pub version: pep440_rs::Version, - pub extras: HashSet, +/// Extension trait that checks whether a found pypi requirement satisfies +/// the information in a [`PypiPackageIdentifier`]. +pub(crate) trait PypiPackageIdentifierSatisfies { + fn satisfies( + &self, + requirement: &uv_distribution_types::Requirement, + ) -> Result>; } -impl PypiPackageIdentifier { - /// Extracts the python packages that will be installed when the specified - /// conda package is installed. - pub(crate) fn from_repodata_record( - record: &RepoDataRecord, - ) -> Result, ConversionError> { - let mut result = Vec::new(); - Self::from_record_into(record, &mut result)?; - - Ok(result) - } - - pub fn from_package_record(record: &PackageRecord) -> Result, ConversionError> { - let mut result = Vec::new(); - if let Some(purls) = &record.purls { - for purl in purls.iter() { - if let Some(entry) = Self::convert_from_purl(purl, &record.version.as_str())? { - result.push(entry); - } - } - } - Ok(result) - } - - /// Helper function to write the result of extract the python packages that - /// will be installed into a pre-allocated vector. - fn from_record_into( - record: &RepoDataRecord, - result: &mut Vec, - ) -> Result<(), ConversionError> { - let mut has_pypi_purl = false; - let identifiers = Self::from_package_record(&record.package_record)?; - if !identifiers.is_empty() { - has_pypi_purl = true; - result.extend(identifiers); - } - - // Backwards compatibility: - // If lock file don't have a purl - // but the package is a conda-forge package, we just assume that - // the name of the package is equivalent to the name of the python package. - // In newer versions of the lock file, we should always have a purl - // where empty purls means that the package is not a pypi-one. - if record.package_record.purls.is_none() - && !has_pypi_purl - && pypi_mapping::is_conda_forge_record(record) - { - tracing::trace!( - "Using backwards compatibility purl logic for conda package: {}", - record.package_record.name.as_source() - ); - // Convert the conda package names to pypi package names. If the conversion fails we - // just assume that its not a valid python package. - let name = - uv_normalize::PackageName::from_str(record.package_record.name.as_source()).ok(); - let version = - pep440_rs::Version::from_str(&record.package_record.version.as_str()).ok(); - if let (Some(name), Some(version)) = (name, version) { - let pep_name = to_normalize(&name)?; - - result.push(PypiPackageIdentifier { - name: PypiPackageName::from_normalized(pep_name), - version, - // TODO: We can't really tell which python extras are enabled in a conda - // package. - extras: Default::default(), - }) - } - } - - Ok(()) - } - - /// Tries to construct an instance from a generic PURL. - /// - /// The `fallback_version` is used if the PURL does not contain a version. - pub(crate) fn convert_from_purl( - package_url: &PackageUrl, - fallback_version: &str, - ) -> Result, ConversionError> { - if package_url.package_type() == "pypi" { - Self::from_pypi_purl(package_url, fallback_version).map(Some) - } else { - Ok(None) - } - } - - /// Constructs a new instance from a PyPI package URL. - /// - /// The `fallback_version` is used if the PURL does not contain a version. - pub(crate) fn from_pypi_purl( - package_url: &PackageUrl, - fallback_version: &str, - ) -> Result { - assert_eq!(package_url.package_type(), "pypi"); - let name = package_url.name(); - let name = uv_normalize::PackageName::from_str(name) - .map_err(|e| ConversionError::PackageName(name.to_string(), e))?; - - let version_str = package_url.version().unwrap_or(fallback_version); - let version = pep440_rs::Version::from_str(version_str) - .map_err(|_| ConversionError::Version(version_str.to_string()))?; - - // TODO: We can't really tell which python extras are enabled from a PURL. - let extras = HashSet::new(); - let pep_name = to_normalize(&name)?; - - Ok(Self { - name: PypiPackageName::from_normalized(pep_name), - version, - extras, - }) - } - - /// Checks of a found pypi requirement satisfies with the information - /// in this package identifier. - pub(crate) fn satisfies( +impl PypiPackageIdentifierSatisfies for PypiPackageIdentifier { + fn satisfies( &self, requirement: &uv_distribution_types::Requirement, ) -> Result> { @@ -199,16 +85,3 @@ impl PypiPackageIdentifier { } } } - -#[derive(Error, Debug)] -pub enum ConversionError { - #[error("'{0}' is not a valid python package name")] - PackageName(String, #[source] InvalidNameError), - - #[error("'{0}' is not a valid python version")] - Version(String), - // #[error("'{0}' is not a valid python extra")] - // Extra(String), - #[error(transparent)] - NameConversion(#[from] PixiConversionError), -} diff --git a/crates/pixi_core/src/lock_file/resolve/mod.rs b/crates/pixi_core/src/lock_file/resolve/mod.rs index 72ac82419b..15828a2f76 100644 --- a/crates/pixi_core/src/lock_file/resolve/mod.rs +++ b/crates/pixi_core/src/lock_file/resolve/mod.rs @@ -1,7 +1,97 @@ -//! This module contains code to resolve python package from PyPi or Conda packages. +//! Workspace-side support for the PyPI resolve pipeline. //! -//! See [`pypi::resolve_pypi`] for more information. +//! The resolution itself lives in `pixi_install_pypi::resolve` and is driven +//! through the command dispatcher +//! ([`CommandDispatcher::solve_pypi_environment`](pixi_command_dispatcher::CommandDispatcher::solve_pypi_environment)). +//! This module provides the workspace implementation of +//! [`CondaPrefixProvider`]: it instantiates a conda prefix (and computes the +//! activated environment variables) on demand when a source distribution has +//! to be built during resolution. -pub(crate) mod build_dispatch; -pub(crate) mod pypi; -mod resolver_provider; +use std::{cell::Cell, collections::HashMap, pin::Pin}; + +use pixi_install_pypi::resolve::{CondaPrefixProvider, ProvidedCondaPrefix}; +use pixi_manifest::EnvironmentName; +use pixi_record::PixiRecord; + +use crate::{ + activation::CurrentEnvVarBehavior, + environment::CondaPrefixUpdater, + workspace::{Environment, EnvironmentVars, get_activated_environment_variables}, +}; + +/// Provides a conda prefix for PyPI source builds by installing the +/// environment's conda packages through a (memoized) [`CondaPrefixUpdater`] +/// and activating the environment. +pub struct WorkspaceCondaPrefixProvider<'p> { + /// Instantiates (and memoizes) the conda prefix. + prefix_updater: CondaPrefixUpdater, + + /// The conda records to install into the prefix. Consumed on first use; + /// carries the error of the upstream solve, if any, so it surfaces only + /// when a prefix is actually required. + repodata_records: Cell>>>, + + /// Project environment variables used to compute the activated + /// environment of `environment`. + project_env_vars: HashMap, + environment: Environment<'p>, +} + +impl<'p> WorkspaceCondaPrefixProvider<'p> { + pub fn new( + prefix_updater: CondaPrefixUpdater, + repodata_records: miette::Result>, + project_env_vars: HashMap, + environment: Environment<'p>, + ) -> Self { + Self { + prefix_updater, + repodata_records: Cell::new(Some(repodata_records)), + project_env_vars, + environment, + } + } +} + +impl CondaPrefixProvider for WorkspaceCondaPrefixProvider<'_> { + fn provide(&self) -> Pin> + '_>> { + Box::pin(async move { + tracing::debug!( + "PyPI solve requires instantiation of conda prefix for '{}'", + self.prefix_updater.name().as_str() + ); + + let repodata_records = self + .repodata_records + .replace(None) + .expect("the conda prefix can only be provided once")?; + + let prefix = self + .prefix_updater + .update( + repodata_records.into_iter().map(Into::into).collect(), + None, + None, + ) + .await?; + + // Get the activation vars to expose to PEP 517 build backends. + let env_vars = get_activated_environment_variables( + &self.project_env_vars, + &self.environment, + CurrentEnvVarBehavior::Exclude, + None, + false, + false, + ) + .await?; + + Ok(ProvidedCondaPrefix { + prefix: prefix.prefix.clone(), + python_status: (*prefix.python_status).clone(), + env_vars: env_vars.clone(), + }) + }) + } +} diff --git a/crates/pixi_core/src/lock_file/satisfiability/platform.rs b/crates/pixi_core/src/lock_file/satisfiability/platform.rs index 03af129d22..4e83681f66 100644 --- a/crates/pixi_core/src/lock_file/satisfiability/platform.rs +++ b/crates/pixi_core/src/lock_file/satisfiability/platform.rs @@ -45,7 +45,7 @@ use crate::{ lock_file::{ PixiRecordsByName, PypiRecordsByName, outdated::{BuildCacheKey, PypiEnvironmentBuildCache}, - package_identifier::ConversionError, + package_identifier::{ConversionError, PypiPackageIdentifierSatisfies}, records_by_name::{HasNameVersion, LockedPypiRecordsByName}, }, workspace::{Environment, EnvironmentVars, HasWorkspaceRef, PlatformOverrides, PlatformSource}, diff --git a/crates/pixi_core/src/lock_file/satisfiability/pypi.rs b/crates/pixi_core/src/lock_file/satisfiability/pypi.rs index c390b99616..f97b5c26cf 100644 --- a/crates/pixi_core/src/lock_file/satisfiability/pypi.rs +++ b/crates/pixi_core/src/lock_file/satisfiability/pypi.rs @@ -41,16 +41,16 @@ use super::platform::{RequirementOrigin, VerifySatisfiabilityContext}; use super::pypi_metadata; use crate::{ lock_file::{ - CondaPrefixUpdater, PixiRecordsByName, PypiRecordsByName, + CondaPrefixUpdater, PixiRecordsByName, PypiRecordsByName, WorkspaceCondaPrefixProvider, outdated::{BuildCacheKey, PypiEnvironmentBuildCache}, records_by_name::LockedPypiRecordsByName, - resolve::build_dispatch::{LazyBuildDispatch, UvBuildDispatchParams}, }, workspace::{ Environment, EnvironmentVars, HasWorkspaceRef, PlatformOverrides, PlatformSource, grouped_environment::GroupedEnvironment, }, }; +use pixi_install_pypi::resolve::build_dispatch::{LazyBuildDispatch, UvBuildDispatchParams}; /// Compare two PyPI index URLs ignoring trailing slashes. fn pypi_index_urls_match(a: &Url, b: &Url) -> bool { @@ -725,15 +725,17 @@ async fn read_local_package_metadata( .as_ref() .map(|r| r.records.clone()) .map_err(|e| miette::miette!("{}", e)); - let lazy_build_dispatch = LazyBuildDispatch::new( - build_params, + let prefix_provider = WorkspaceCondaPrefixProvider::new( conda_prefix_updater, + building_records, ctx.project_env_vars.clone(), ctx.environment.clone(), - building_records, + ); + let lazy_build_dispatch = LazyBuildDispatch::new( + build_params, + &prefix_provider, pypi_options.no_build_isolation.clone(), &cache.lazy_build_dispatch_deps, - None, false, Arc::clone(&last_error), ); diff --git a/crates/pixi_core/src/lock_file/update.rs b/crates/pixi_core/src/lock_file/update.rs index d6f6be071a..739cbb636f 100644 --- a/crates/pixi_core/src/lock_file/update.rs +++ b/crates/pixi_core/src/lock_file/update.rs @@ -25,16 +25,14 @@ use miette::{Diagnostic, IntoDiagnostic, MietteDiagnostic, Report, WrapErr}; use ordermap::{OrderMap, OrderSet}; use pixi_command_dispatcher::{ BuildEnvironment, CommandDispatcher, CommandDispatcherError, CommandDispatcherErrorResultExt, - ComputeResultExt, EnvironmentRef, EnvironmentSpec, SolvePixiEnvironmentError, + ComputeResultExt, EnvironmentRef, EnvironmentSpec, InstallPypiEnvironmentSpec, + SolvePixiEnvironmentError, SolvePypiEnvironmentSpec, executor::CancellationAwareFutures, keys::{SolvePixiEnvironmentKey, SolvePixiEnvironmentSpec}, }; use pixi_consts::consts; use pixi_glob::GlobHashCache; -use pixi_install_pypi::{ - LazyEnvironmentVariables, PyPIBuildConfig, PyPIContextConfig, PyPIEnvironmentUpdater, - PyPIUpdateConfig, derive_link_mode, -}; +use pixi_install_pypi::LazyEnvironmentVariables; use pixi_manifest::{ ChannelPriority, EnvironmentName, FeaturesExt, HasWorkspaceManifest, PixiPlatform, PixiPlatformName, @@ -44,8 +42,8 @@ use pixi_record::{LockFileResolver, ParseLockFileError, PixiRecord, UnresolvedPi use pixi_utils::{prefix::Prefix, variants::VariantConfig}; use pixi_uv_context::UvResolutionContext; use pixi_uv_conversions::{ - ConversionError, to_exclude_newer, to_extra_name, to_marker_environment, to_normalize, - to_uv_extra_name, to_uv_normalize, + ConversionError, to_extra_name, to_marker_environment, to_normalize, to_uv_extra_name, + to_uv_normalize, }; use pypi_mapping::{self, PurlDerivationClient}; use pypi_modifiers::pypi_marker_env::determine_marker_environment; @@ -56,13 +54,12 @@ use serde::{Deserialize, Serialize}; use thiserror::Error; use tokio::sync::Semaphore; use tracing::Instrument; -use uv_install_wheel::LinkMode; use uv_normalize::ExtraName; use super::{ CondaPrefixUpdater, InstallSubset, PixiRecordsByName, PypiRecordsByName, - UnresolvedPixiRecordsByName, outdated::OutdatedEnvironments, resolve_lock_platform, - utils::IoConcurrencyLimit, + UnresolvedPixiRecordsByName, WorkspaceCondaPrefixProvider, outdated::OutdatedEnvironments, + resolve_lock_platform, utils::IoConcurrencyLimit, }; use crate::{ Workspace, @@ -80,7 +77,7 @@ use crate::{ }, }, workspace::{ - Environment, EnvironmentVars, HasWorkspaceRef, + Environment, EnvironmentVars, HasWorkspaceRef, PlatformOverrides, PlatformSource, errors::VariantsError, get_activated_environment_variables, grouped_environment::{GroupedEnvironment, GroupedEnvironmentName}, @@ -1061,52 +1058,41 @@ impl<'p> LockFileDerivedData<'p> { // Update the prefix with Pypi records { - let pypi_indexes = self.locked_env(environment)?.pypi_indexes().cloned(); - let index_strategy = environment.pypi_options().index_strategy.clone(); - let pypi_exclude_newer = environment.pypi_exclude_newer_config_resolved(); - let skip_wheel_filename_check = - environment.pypi_options().skip_wheel_filename_check; - - let pypi_update_config = PyPIUpdateConfig { - environment_name: environment.name(), - prefix: &prefix, - platform: best_declared_platform, - lock_file_dir: self.workspace.root(), - }; - - let workspace_config = self.workspace.config(); - let build_config = PyPIBuildConfig { - no_build_isolation: &non_isolated_packages, - no_build: &no_build, - no_binary: &no_binary, - index_strategy: index_strategy.as_ref(), - exclude_newer: &pypi_exclude_newer, - skip_wheel_filename_check, - link_mode: Some(derive_link_mode( - workspace_config.allow_symbolic_links, - workspace_config.allow_hard_links, - workspace_config.allow_ref_links, - )), - }; + // Ignored pypi records are never considered extraneous, so + // they are not removed from the prefix. + let ignored_extraneous = ignored_pypi + .iter() + .map(to_uv_normalize) + .collect::, _>>() + .into_diagnostic()?; let lazy_env_vars = LazyPixiEnvironmentVars { environment: environment.clone(), }; - let context_config = PyPIContextConfig { - uv_context: &uv_context, - pypi_indexes: pypi_indexes.as_ref(), - environment_variables_lazy: Some(&lazy_env_vars), + + let spec = InstallPypiEnvironmentSpec { + name: environment.name().clone(), + prefix: prefix.clone(), + platform: best_declared_platform.clone(), + lock_file_dir: self.workspace.root().to_path_buf(), + python_status, + pixi_records: resolved_pixi_records, + pypi_records, + pypi_indexes: self.locked_env(environment)?.pypi_indexes().cloned(), + no_build_isolation: non_isolated_packages, + no_build, + no_binary, + index_strategy: environment.pypi_options().index_strategy.clone(), + exclude_newer: environment.pypi_exclude_newer_config_resolved(), + skip_wheel_filename_check: environment + .pypi_options() + .skip_wheel_filename_check, + ignored_extraneous, + uv_context, }; - // Ignored pypi records - let names = ignored_pypi - .iter() - .map(to_uv_normalize) - .collect::, _>>() - .into_diagnostic()?; - PyPIEnvironmentUpdater::new(pypi_update_config, build_config, context_config) - .with_ignored_extraneous(names) - .update(&python_status, &resolved_pixi_records, &pypi_records) + self.command_dispatcher + .install_pypi_environment(spec, Some(&lazy_env_vars)) .await } .with_context(|| { @@ -2127,14 +2113,6 @@ impl<'p> UpdateContext<'p> { } // Spawn tasks to update the pypi packages. - let project_link_mode = { - let config = project.config(); - derive_link_mode( - config.allow_symbolic_links, - config.allow_hard_links, - config.allow_ref_links, - ) - }; for (environment, platform) in self.outdated_envs .pypi @@ -2232,7 +2210,6 @@ impl<'p> UpdateContext<'p> { locked_group_records, self.no_install, build_cache, - project_link_mode, ); pending_futures.push( @@ -3224,7 +3201,6 @@ async fn spawn_solve_pypi_task<'p>( locked_pypi_packages: Arc, disallow_install_conda_prefix: bool, build_cache: Arc, - link_mode: LinkMode, ) -> miette::Result { let pixi_platform = environment .workspace_manifest() @@ -3244,7 +3220,7 @@ async fn spawn_solve_pypi_task<'p>( )); } - let exclude_newer = to_exclude_newer(&grouped_environment.pypi_exclude_newer_config_resolved()); + let exclude_newer = grouped_environment.pypi_exclude_newer_config_resolved(); // Wait until the conda records and prefix are available. let (repodata_records, repodata_building_records) = match repodata_building_records { @@ -3266,12 +3242,9 @@ async fn spawn_solve_pypi_task<'p>( let environment_name = grouped_environment.name().clone(); let solve_strategy = grouped_environment.solve_strategy(); - let pixi_solve_records = &repodata_records.records; - let locked_pypi_records = &locked_pypi_packages.records; - let pypi_options = environment.pypi_options(); let platform_for_async = platform.clone(); - let (pypi_packages, duration, prefix_task_result) = async move { + let (pypi_packages, duration) = async move { let platform = platform_for_async; let pb = SolveProgressBar::new( global_multi_progress().add(ProgressBar::hidden()), @@ -3290,42 +3263,77 @@ async fn spawn_solve_pypi_task<'p>( let requirements = IndexMap::from_iter(dependencies); - let (records, prefix_task_result) = lock_file::resolve_pypi( - resolution_context, - &pypi_options, - requirements, - pixi_solve_records, - locked_pypi_records, - platform.clone(), - &pb.pb, - &project_root, - command_dispatcher, - repodata_building_records, + // The conda prefix for source builds has to run on the current + // system; cross-platform pypi resolves still need a local Python to + // compute wheel tags. Fall back to a bare current-subdir platform + // when no declared workspace platform matches this machine. The + // updater is cached so repeated solves share one prefix. + let conda_prefix_updater = build_cache + .conda_prefix_updater + .get_or_try_init(|| { + let host_platform; + let prefix_platform: &PixiPlatform = match environment.best_declared_platform() { + Some(p) => p, + None => { + host_platform = environment.workspace().host_platform( + PlatformSource::Defaults, + PlatformOverrides::EnvironmentVariableOverrides, + ); + &host_platform + } + }; + let group = GroupedEnvironment::Environment(environment.clone()); + let virtual_packages = environment.virtual_packages(prefix_platform); + + CondaPrefixUpdater::builder( + group, + prefix_platform.clone(), + virtual_packages + .into_iter() + .map(GenericVirtualPackage::from) + .collect(), + command_dispatcher.clone(), + ) + .finish() + })? + .clone(); + let prefix_provider = WorkspaceCondaPrefixProvider::new( + conda_prefix_updater, + repodata_building_records.map(|r| r.records.clone()), project_variables, environment, + ); + + let spec = SolvePypiEnvironmentSpec { + dependencies: requirements, + pypi_options, + pixi_records: repodata_records.records.clone(), + locked_pypi_records: locked_pypi_packages.records.clone(), + platform: pixi_platform.clone(), + project_root, disallow_install_conda_prefix, exclude_newer, solve_strategy, - build_cache, - link_mode, - ) - .await - .with_context(|| { - format!( - "failed to solve the pypi requirements of environment '{}' for platform '{}'", - environment_name.fancy_display(), - consts::PLATFORM_STYLE.apply_to(&platform) - ) - })?; + build_dispatch_cache: build_cache.lazy_build_dispatch_deps.clone(), + uv_context: resolution_context, + progress_bar: Some(pb.pb.clone()), + }; + + let records = command_dispatcher + .solve_pypi_environment(spec, &prefix_provider) + .await + .with_context(|| { + format!( + "failed to solve the pypi requirements of environment '{}' for platform '{}'", + environment_name.fancy_display(), + consts::PLATFORM_STYLE.apply_to(&platform) + ) + })?; let end = Instant::now(); pb.finish(); - Ok::<(_, _, _), miette::Report>(( - LockedPypiRecordsByName::from_iter(records), - end - start, - prefix_task_result, - )) + Ok::<(_, _), miette::Report>((LockedPypiRecordsByName::from_iter(records), end - start)) } .instrument(tracing::info_span!( "resolve_pypi", @@ -3339,7 +3347,9 @@ async fn spawn_solve_pypi_task<'p>( platform, pypi_packages, duration, - prefix_task_result, + // The resolve no longer instantiates conda prefixes outside the + // (cached) prefix updater, so there is never a prefix to forward. + None, )) } diff --git a/crates/pixi_global/Cargo.toml b/crates/pixi_global/Cargo.toml index 74df9ed142..d0a3f5155d 100644 --- a/crates/pixi_global/Cargo.toml +++ b/crates/pixi_global/Cargo.toml @@ -22,6 +22,7 @@ itertools = { workspace = true } miette = { workspace = true } once_cell = { workspace = true } ordermap = { workspace = true } +pep508_rs = { workspace = true } pixi_build_frontend = { workspace = true } pixi_command_dispatcher = { workspace = true } pixi_config = { workspace = true } @@ -30,11 +31,16 @@ pixi_core = { workspace = true } pixi_manifest = { workspace = true } pixi_path = { workspace = true } pixi_progress = { workspace = true } +pixi_pypi_spec = { workspace = true } +pixi_python_status = { workspace = true } +pixi_record = { workspace = true } pixi_reporters = { workspace = true } pixi_spec = { workspace = true } pixi_spec_containers = { workspace = true } pixi_toml = { workspace = true } pixi_utils = { workspace = true, default-features = false } +pixi_uv_context = { workspace = true } +pixi_uv_conversions = { workspace = true } rattler = { workspace = true } rattler_conda_types = { workspace = true } rattler_lock = { workspace = true } diff --git a/crates/pixi_global/src/project/environment.rs b/crates/pixi_global/src/project/environment.rs index d85c533bc0..69fa63ba86 100644 --- a/crates/pixi_global/src/project/environment.rs +++ b/crates/pixi_global/src/project/environment.rs @@ -1,9 +1,15 @@ use crate::install::local_environment_matches_spec; use console::StyledObject; use fancy_display::FancyDisplay; -use indexmap::IndexSet; -use miette::Diagnostic; +use indexmap::{IndexMap, IndexSet}; +use is_executable::IsExecutable; +use miette::{Diagnostic, IntoDiagnostic}; use pixi_consts::consts; +use pixi_pypi_spec::{PixiPypiSpec, PypiPackageName}; +use pixi_utils::is_binary_folder; +use pixi_utils::prefix::{Executable, Prefix}; +use pixi_utils::strip_executable_extension; +use rattler::install::PythonInfo; use rattler_conda_types::{MatchSpec, PackageName, Platform, PrefixRecord}; use regex::Regex; use serde::{self, Deserialize, Deserializer, Serialize}; @@ -99,6 +105,208 @@ pub(crate) async fn environment_specs_in_sync( Ok(true) } +/// A PyPI distribution found in a prefix's site-packages. +pub(crate) struct InstalledPypiDistribution { + /// The distribution name as found in the `.dist-info` directory name, + /// lowercased. Following the wheel spec this uses `_` where + /// pep508-normalized names use `-`. + pub dist_info_name: String, + /// True when the distribution was installed by pixi (rather than e.g. + /// pip run by the user inside the environment). + pub pixi_installed: bool, + /// The path of the `.dist-info` directory. + pub dist_info_path: std::path::PathBuf, +} + +/// Scans `site_packages` for installed distributions by looking at the +/// `{name}-{version}.dist-info` directories. +pub(crate) fn installed_pypi_distributions( + site_packages: &std::path::Path, +) -> Vec { + let mut result = Vec::new(); + if let Ok(entries) = fs_err::read_dir(site_packages) { + for entry in entries.flatten() { + let file_name = entry.file_name(); + let Some(dir_name) = file_name.to_str() else { + continue; + }; + if let Some(dist) = dir_name.strip_suffix(".dist-info") + && let Some((name, _version)) = dist.split_once('-') + { + let pixi_installed = fs_err::read_to_string(entry.path().join("INSTALLER")) + .map(|installer| installer.trim() == consts::PIXI_UV_INSTALLER) + .unwrap_or(false); + result.push(InstalledPypiDistribution { + dist_info_name: name.to_lowercase(), + pixi_installed, + dist_info_path: entry.path(), + }); + } + } + } + result +} + +/// Converts a pep508-normalized package name to the spelling used in +/// `.dist-info` directory names, where runs of `-_.` are replaced by `_`. +pub(crate) fn dist_info_name(name: &PypiPackageName) -> String { + name.as_normalized().to_string().replace('-', "_") +} + +/// Extracts the path field from a line in a `.dist-info/RECORD` file. The +/// format is CSV with three fields, `path,hash,size`, where the path is +/// quoted when it contains special characters. +fn record_entry_path(line: &str) -> Option { + if let Some(rest) = line.strip_prefix('"') { + let mut path = String::new(); + let mut chars = rest.chars().peekable(); + while let Some(c) = chars.next() { + if c == '"' { + if chars.peek() == Some(&'"') { + path.push('"'); + chars.next(); + } else { + break; + } + } else { + path.push(c); + } + } + Some(path) + } else { + line.split(',') + .next() + .map(str::to_string) + .filter(|path| !path.is_empty()) + } +} + +/// Logically normalizes a path by resolving `.` and `..` components. +fn normalize_path(path: &std::path::Path) -> std::path::PathBuf { + let mut result = std::path::PathBuf::new(); + for component in path.components() { + match component { + std::path::Component::CurDir => {} + std::path::Component::ParentDir => { + result.pop(); + } + other => result.push(other), + } + } + result +} + +/// Returns the executables a PyPI distribution installed into the prefix's +/// binary folders, based on the `RECORD` of its `.dist-info` directory. +pub(crate) fn pypi_distribution_executables( + prefix: &Prefix, + site_packages: &std::path::Path, + dist_info_path: &std::path::Path, +) -> Vec { + let Ok(record) = fs_err::read_to_string(dist_info_path.join("RECORD")) else { + return Vec::new(); + }; + + let mut executables = Vec::new(); + for line in record.lines() { + let Some(path) = record_entry_path(line) else { + continue; + }; + // Paths in RECORD are relative to site-packages; scripts use + // `../../../bin/...` style paths to reach the prefix's binary folder. + let absolute = normalize_path(&site_packages.join(path)); + let Ok(relative) = absolute.strip_prefix(prefix.root()) else { + continue; + }; + let Some(parent) = relative.parent() else { + continue; + }; + if !is_binary_folder(parent) || !absolute.is_executable() { + continue; + } + if let Some(name) = relative.file_name().and_then(|name| name.to_str()) { + executables.push(Executable::new( + strip_executable_extension(name.to_string()), + relative.to_path_buf(), + )); + } + } + executables +} + +/// Returns the executables of the pixi-installed PyPI distributions in +/// `site_packages`. When `only_dists` is given, only distributions whose +/// (dist-info spelled) name is in the set are considered. +pub(crate) fn pypi_executables( + prefix: &Prefix, + site_packages: &std::path::Path, + only_dists: Option<&HashSet>, +) -> Vec { + installed_pypi_distributions(site_packages) + .into_iter() + .filter(|dist| dist.pixi_installed) + .filter(|dist| only_dists.is_none_or(|names| names.contains(&dist.dist_info_name))) + .flat_map(|dist| pypi_distribution_executables(prefix, site_packages, &dist.dist_info_path)) + .collect() +} + +/// Returns the site-packages directory of the prefix, if it contains a +/// python interpreter. +pub(crate) fn find_site_packages( + python_record: Option<&rattler_conda_types::PackageRecord>, + prefix: &Prefix, + platform: Platform, +) -> miette::Result> { + let Some(python_record) = python_record else { + return Ok(None); + }; + let python_info = PythonInfo::from_python_record(python_record, platform).into_diagnostic()?; + Ok(Some(prefix.root().join(&python_info.site_packages_path))) +} + +/// Checks whether the PyPI packages declared in the manifest match what is +/// present in the prefix's site-packages. +/// +/// This is a name-presence check only: it detects added or removed +/// pypi-dependencies (including pixi-installed leftovers that should be +/// removed), but a changed version requirement for an already-installed +/// package does not mark the environment as out of sync. The PyPI installer +/// reconciles versions the next time the environment is (re)installed. +pub(crate) fn pypi_dependencies_in_sync( + pypi_dependencies: &IndexMap, + prefix_records: &[PrefixRecord], + prefix: &Prefix, + platform: Platform, +) -> miette::Result { + // Locate the python interpreter among the installed conda packages; its + // record determines where site-packages lives. + let python_record = prefix_records + .iter() + .map(|r| &r.repodata_record.package_record) + .find(|r| r.name.as_normalized() == "python"); + let Some(site_packages) = find_site_packages(python_record, prefix, platform)? else { + // PyPI packages cannot be installed without an interpreter; trigger a + // sync so installation can surface a proper error. + return Ok(pypi_dependencies.is_empty()); + }; + + let installed = installed_pypi_distributions(&site_packages); + + // When nothing is declared anymore, any distribution previously + // installed by pixi has to be removed. + if pypi_dependencies.is_empty() { + return Ok(!installed.iter().any(|dist| dist.pixi_installed)); + } + + let installed_names: HashSet<&str> = installed + .iter() + .map(|dist| dist.dist_info_name.as_str()) + .collect(); + Ok(pypi_dependencies + .keys() + .all(|name| installed_names.contains(dist_info_name(name).as_str()))) +} + #[cfg(test)] mod tests { diff --git a/crates/pixi_global/src/project/manifest.rs b/crates/pixi_global/src/project/manifest.rs index 46a5dd3e90..d918b3bb39 100644 --- a/crates/pixi_global/src/project/manifest.rs +++ b/crates/pixi_global/src/project/manifest.rs @@ -10,6 +10,7 @@ use miette::IntoDiagnostic; use pixi_config::Config; use pixi_consts::consts; use pixi_manifest::{PrioritizedChannel, toml::TomlDocument}; +use pixi_pypi_spec::{PixiPypiSpec, PypiPackageName}; use pixi_toml::TomlIndexMap; use pixi_utils::{executable_from_path, strip_executable_extension}; use rattler_conda_types::{NamedChannelOrUrl, PackageName, Platform}; @@ -170,6 +171,81 @@ impl Manifest { Ok(()) } + /// Adds a PyPI dependency to the manifest + pub fn add_pypi_dependency( + &mut self, + env_name: &EnvironmentName, + name: &PypiPackageName, + spec: &PixiPypiSpec, + ) -> miette::Result<()> { + // Update self.parsed + self.parsed + .envs + .get_mut(env_name) + .ok_or_else(|| { + miette::miette!("Environment {} doesn't exist.", env_name.fancy_display()) + })? + .pypi_dependencies + .insert(name.clone(), spec.clone()); + + // Update self.document + self.document.insert_into_inline_table( + &["envs", env_name.as_str(), "pypi-dependencies"], + name.as_source(), + toml_edit::Value::from(spec.clone()), + )?; + + tracing::debug!( + "Added PyPI dependency {} to toml document for environment {}", + name.as_source(), + env_name.fancy_display() + ); + Ok(()) + } + + /// Removes a PyPI dependency from the manifest + pub fn remove_pypi_dependency( + &mut self, + env_name: &EnvironmentName, + name: &PypiPackageName, + ) -> miette::Result<()> { + let pypi_dependencies = &mut self + .parsed + .envs + .get_mut(env_name) + .ok_or_else(|| { + miette::miette!("Environment {} doesn't exist.", env_name.fancy_display()) + })? + .pypi_dependencies; + + // Match on the normalized name so the manifest's spelling of the key + // does not have to match the spelling on the command line. + let stored_name = pypi_dependencies + .keys() + .find(|key| key.as_normalized() == name.as_normalized()) + .cloned() + .ok_or(miette::miette!( + "PyPI dependency {} not found in {}", + console::style(name.as_source()).green(), + env_name.fancy_display() + ))?; + + // Update self.parsed + pypi_dependencies.swap_remove(&stored_name); + + // Update self.document + self.document + .get_or_insert_nested_table(&["envs", env_name.as_str(), "pypi-dependencies"])? + .remove(stored_name.as_source()); + + tracing::debug!( + "Removed PyPI dependency {} from toml document for environment {}", + console::style(stored_name.as_source()).green(), + env_name.fancy_display() + ); + Ok(()) + } + /// Removes a dependency from the manifest pub fn remove_dependency( &mut self, @@ -1192,4 +1268,74 @@ dependencies = { "python" = "*", pytest = "*"} // Verify parsing works assert!(manifest.parsed.envs.contains_key(&env_name)); } + + #[test] + fn test_add_and_remove_pypi_dependency() { + let env_name = EnvironmentName::from_str("test-env").unwrap(); + let mut manifest = Manifest::from_str( + Path::new("global.toml"), + r#" + [envs.test-env] + channels = ["conda-forge"] + dependencies = { python = "3.12.*" } + "#, + ) + .unwrap(); + + let requirement = pep508_rs::Requirement::from_str("flask>=2.0").unwrap(); + let name = PypiPackageName::from_normalized(requirement.name.clone()); + let spec = PixiPypiSpec::try_from(requirement).unwrap(); + + // Add the pypi dependency + manifest + .add_pypi_dependency(&env_name, &name, &spec) + .unwrap(); + + // Check document + let actual_value = manifest + .document + .get_or_insert_nested_table(&["envs", env_name.as_str(), "pypi-dependencies"]) + .unwrap() + .get(name.as_source()); + assert!(actual_value.is_some()); + + // Check parsed + assert!( + manifest + .parsed + .envs + .get(&env_name) + .unwrap() + .pypi_dependencies + .contains_key(&name) + ); + + // The manifest must round-trip through its TOML representation. + let contents = manifest.document.to_string(); + let reparsed = ParsedManifest::from_toml_str(&contents).unwrap(); + assert!( + reparsed + .envs + .get(&env_name) + .unwrap() + .pypi_dependencies + .contains_key(&name) + ); + + // Remove the pypi dependency again + manifest.remove_pypi_dependency(&env_name, &name).unwrap(); + assert!( + manifest + .parsed + .envs + .get(&env_name) + .unwrap() + .pypi_dependencies + .is_empty() + ); + assert!(!manifest.document.to_string().contains("flask")); + + // Removing a missing dependency errors + assert!(manifest.remove_pypi_dependency(&env_name, &name).is_err()); + } } diff --git a/crates/pixi_global/src/project/mod.rs b/crates/pixi_global/src/project/mod.rs index 879320b4a1..20777ff51b 100644 --- a/crates/pixi_global/src/project/mod.rs +++ b/crates/pixi_global/src/project/mod.rs @@ -1,8 +1,9 @@ use std::{ - collections::{BTreeMap, HashSet}, + collections::{BTreeMap, HashMap, HashSet}, ffi::OsStr, fmt::{Debug, Formatter}, path::{Path, PathBuf}, + pin::Pin, str::FromStr, sync::Arc, }; @@ -22,8 +23,9 @@ pub use parsed_manifest::{ExposedName, ParsedEnvironment, ParsedManifest}; use pixi_build_frontend::BackendOverride; use pixi_command_dispatcher::{ BuildBackendMetadataSpec, BuildEnvironment, CommandDispatcher, ComputeResultExt, - EnvironmentRef, EnvironmentSpec, EphemeralEnv, InstallPixiEnvironmentSpec, Limits, - SourceCheckoutExt, + CondaPrefixProvider, EnvironmentRef, EnvironmentSpec, EphemeralEnv, InstallPixiEnvironmentSpec, + InstallPypiEnvironmentSpec, InstallablePypiRecord, Limits, ManifestData, ProvidedCondaPrefix, + SolvePypiEnvironmentSpec, SourceCheckoutExt, keys::{SolvePixiEnvironmentKey, SolvePixiEnvironmentSpec}, }; use pixi_config::{Config, RunPostLinkScripts, default_channel_config, pixi_home}; @@ -31,6 +33,9 @@ use pixi_consts::consts::{self}; use pixi_core::repodata::Repodata; use pixi_manifest::PrioritizedChannel; use pixi_path::AbsPathBuf; +use pixi_pypi_spec::PypiPackageName; +use pixi_python_status::PythonStatus; +use pixi_record::PixiRecord; use pixi_reporters::TopLevelProgress; use pixi_spec::{BinarySpec, PathBinarySpec}; use pixi_spec_containers::DependencyMap; @@ -40,6 +45,8 @@ use pixi_utils::{ prefix::{Executable, Prefix}, rlimit::try_increase_rlimit_to_sensible, }; +use pixi_uv_context::UvResolutionContext; +use pixi_uv_conversions::to_uv_normalize; use rattler_conda_types::{ ChannelConfig, GenericVirtualPackage, MatchSpec, PackageName, Platform, PrefixRecord, menuinst::MenuMode, package::CondaArchiveIdentifier, @@ -67,7 +74,10 @@ use crate::{ }, find_executables, find_executables_for_many_records, install::{create_executable_trampolines, script_exec_mapping}, - project::environment::environment_specs_in_sync, + project::environment::{ + dist_info_name, environment_specs_in_sync, find_site_packages, + installed_pypi_distributions, pypi_dependencies_in_sync, pypi_executables, + }, }; mod environment; @@ -77,6 +87,29 @@ mod parsed_manifest; pub use global_spec::{FromMatchSpecError, GlobalSpec}; use pixi_utils::reqwest::{LazyReqwestClient, build_lazy_reqwest_clients}; +/// A [`CondaPrefixProvider`] for global environments. The conda prefix is +/// always installed before the PyPI packages are resolved, so this simply +/// hands out the already-installed prefix when a source distribution has to +/// be built during resolution. +struct InstalledCondaPrefixProvider { + prefix: Prefix, + python_status: PythonStatus, +} + +impl CondaPrefixProvider for InstalledCondaPrefixProvider { + fn provide( + &self, + ) -> Pin> + '_>> { + Box::pin(std::future::ready(Ok(ProvidedCondaPrefix { + prefix: self.prefix.clone(), + python_status: self.python_status.clone(), + // Global environments have no activation scripts of their own to + // expose to PEP 517 build backends. + env_vars: HashMap::new(), + }))) + } +} + #[derive(Debug, thiserror::Error, miette::Diagnostic)] pub enum CommandDispatcherError { #[error("could not determine cache directory")] @@ -686,7 +719,7 @@ impl Project { let result = command_dispatcher .install_pixi_environment(InstallPixiEnvironmentSpec { name: env_name.to_string(), - records: pixi_records.into_iter().map(Into::into).collect(), + records: pixi_records.clone().into_iter().map(Into::into).collect(), prefix: rattler_conda_types::prefix::Prefix::create(prefix.root()) .into_diagnostic()?, build_environment, @@ -700,10 +733,164 @@ impl Project { }) .await?; + // Synchronize the PyPI packages declared in the manifest into the + // freshly installed conda prefix. Also runs when nothing is declared + // but pixi-installed PyPI packages linger in site-packages, so that + // removing the last pypi-dependency cleans them up. The pin keeps the + // resolve/install state machine off this future's layout; without it + // callers that embed this future overflow the compiler's layout depth + // limit. + if !environment.pypi_dependencies.is_empty() + || self.has_pypi_leftovers(&prefix, &pixi_records, platform)? + { + let python_status = PythonStatus::from_transaction(&result.transaction); + Box::pin(self.sync_pypi_packages( + env_name, + environment, + &prefix, + platform, + pixi_records, + python_status, + )) + .await + .with_context(|| { + format!( + "failed to install the PyPI dependencies of environment {}", + env_name.fancy_display() + ) + })?; + } + let install_changes = get_install_changes(result.transaction); Ok(EnvironmentUpdate::new(install_changes, dependencies_names)) } + /// Returns true when the prefix's site-packages contains distributions + /// that were installed by pixi. + fn has_pypi_leftovers( + &self, + prefix: &Prefix, + pixi_records: &[PixiRecord], + platform: Platform, + ) -> miette::Result { + let python_record = pixi_records + .iter() + .map(|record| record.package_record()) + .find(|record| record.name.as_normalized() == "python"); + let Some(site_packages) = find_site_packages(python_record, prefix, platform)? else { + return Ok(false); + }; + Ok(installed_pypi_distributions(&site_packages) + .iter() + .any(|dist| dist.pixi_installed)) + } + + /// Resolves and installs the PyPI dependencies of an environment into its + /// (already installed) conda prefix through the command dispatcher. + /// When no PyPI dependencies are declared (anymore), previously + /// pixi-installed packages are removed from site-packages. + async fn sync_pypi_packages( + &self, + env_name: &EnvironmentName, + environment: &ParsedEnvironment, + prefix: &Prefix, + platform: Platform, + pixi_records: Vec, + python_status: PythonStatus, + ) -> miette::Result<()> { + let command_dispatcher = self.command_dispatcher()?; + + let uv_context = UvResolutionContext::from_config( + self.config(), + self.lazy_client_and_authenticated_client()?.0.clone(), + )?; + + // Group the requested specs by normalized package name. + let mut dependencies = IndexMap::new(); + for (name, spec) in &environment.pypi_dependencies { + dependencies + .entry(to_uv_normalize(name.as_normalized()).into_diagnostic()?) + .or_insert_with(ordermap::OrderSet::new) + .insert(spec.clone()); + } + + let pixi_platform = pixi_manifest::PixiPlatform::from(platform); + + // The conda prefix was installed right before this call, so the + // provider can hand it out directly when a source distribution has + // to be built during resolution. + let prefix_provider = InstalledCondaPrefixProvider { + prefix: prefix.clone(), + python_status: python_status.clone(), + }; + + // With no declared dependencies there is nothing to resolve; the + // installer is still invoked below so it removes any pixi-installed + // leftovers from site-packages. + let solved_records = if dependencies.is_empty() { + Vec::new() + } else { + command_dispatcher + .solve_pypi_environment( + SolvePypiEnvironmentSpec { + dependencies, + pypi_options: Default::default(), + pixi_records: pixi_records.clone(), + locked_pypi_records: Vec::new(), + platform: pixi_platform.clone(), + project_root: prefix.root().to_path_buf(), + disallow_install_conda_prefix: false, + exclude_newer: Default::default(), + solve_strategy: Default::default(), + build_dispatch_cache: Arc::default(), + uv_context: uv_context.clone(), + progress_bar: None, + }, + &prefix_provider, + ) + .await + .with_context(|| { + format!( + "failed to resolve the PyPI dependencies of environment {}; make sure \ + the environment contains a python interpreter, e.g. by adding `python` \ + to its dependencies", + env_name.fancy_display() + ) + })? + }; + + let pypi_records = solved_records + .iter() + .map(|record| { + InstallablePypiRecord::from_locked(record, ManifestData { editable: false }) + }) + .collect(); + + command_dispatcher + .install_pypi_environment( + InstallPypiEnvironmentSpec { + name: pixi_manifest::EnvironmentName::Named(env_name.to_string()), + prefix: prefix.clone(), + platform: pixi_platform, + lock_file_dir: prefix.root().to_path_buf(), + python_status, + pixi_records, + pypi_records, + pypi_indexes: None, + no_build_isolation: Default::default(), + no_build: Default::default(), + no_binary: Default::default(), + index_strategy: None, + exclude_newer: Default::default(), + skip_wheel_filename_check: None, + ignored_extraneous: Default::default(), + uv_context, + }, + None, + ) + .await + } + /// Remove an environment from the manifest and the global installation. pub async fn remove_environment( &mut self, @@ -791,11 +978,62 @@ impl Project { let prefix_records = &prefix.find_installed_packages()?; - let all_executables = find_executables_for_many_records(&prefix, prefix_records); + let mut all_executables = find_executables_for_many_records(&prefix, prefix_records); + + // Include the executables installed by PyPI distributions; they are + // not part of any conda record. The prefix records loaded above are + // reused to locate site-packages, since parsing conda-meta is by far + // the most expensive part of this function. + let platform = self + .environment(env_name) + .and_then(|environment| environment.platform) + .unwrap_or_else(Platform::current); + let python_record = prefix_records + .iter() + .map(|record| &record.repodata_record.package_record) + .find(|record| record.name.as_normalized() == "python"); + if let Some(site_packages) = find_site_packages(python_record, &prefix, platform)? { + all_executables.extend(pypi_executables(&prefix, &site_packages, None)); + } Ok(all_executables) } + /// Returns the executables installed by the environment's declared PyPI + /// dependencies, named by package. + pub async fn executables_of_pypi_dependencies( + &self, + env_name: &EnvironmentName, + ) -> miette::Result>> { + let Some(environment) = self.environment(env_name) else { + return Ok(IndexMap::new()); + }; + // Don't touch the prefix at all for environments without PyPI + // dependencies; this function runs on every expose sync. + if environment.pypi_dependencies.is_empty() { + return Ok(IndexMap::new()); + } + let platform = environment.platform.unwrap_or_else(Platform::current); + let declared = environment.pypi_dependencies.keys().cloned().collect_vec(); + let prefix = self.environment_prefix(env_name).await?; + let prefix_records = prefix.find_installed_packages()?; + let python_record = prefix_records + .iter() + .map(|record| &record.repodata_record.package_record) + .find(|record| record.name.as_normalized() == "python"); + let Some(site_packages) = find_site_packages(python_record, &prefix, platform)? else { + return Ok(IndexMap::new()); + }; + + let mut result = IndexMap::new(); + for name in declared { + let dists = HashSet::from([dist_info_name(&name)]); + let executables = pypi_executables(&prefix, &site_packages, Some(&dists)); + result.insert(name, executables); + } + Ok(result) + } + /// Get installed executables of direct dependencies of a specific /// environment. pub async fn executables_of_direct_dependencies( @@ -880,12 +1118,22 @@ impl Project { let execs_direct_deps = self.executables_of_direct_dependencies(env_name).await?; + // Executables of the environment's declared PyPI dependencies; like + // conda packages, only direct dependencies are auto-exposed. + let execs_pypi = self + .executables_of_pypi_dependencies(env_name) + .await? + .into_values() + .flatten() + .collect_vec(); + match expose_type { ExposedType::All => { // Add new binaries that are not yet exposed let executable_names = execs_direct_deps .into_iter() .flat_map(|(_, executables)| executables) + .chain(execs_pypi) .map(|executable| executable.name); for executable_name in executable_names { let mapping = Mapping::new( @@ -909,6 +1157,7 @@ impl Project { } }) .flatten() + .chain(execs_pypi) .map(|executable| executable.name); for executable_name in executable_names { @@ -984,6 +1233,16 @@ impl Project { return Ok(false); } + // For update operations, environments with PyPI dependencies must be + // re-resolved to pick up new releases. + if is_update_operation && !environment.pypi_dependencies.is_empty() { + tracing::debug!( + "Update operation: Environment {} has PyPI dependencies, considering out of sync", + env_name.fancy_display() + ); + return Ok(false); + } + let env_dir = EnvDir::from_path(self.env_root.clone().path().join(env_name.clone().as_str())); @@ -1001,6 +1260,17 @@ impl Project { return Ok(false); } + let pypi_in_sync = pypi_dependencies_in_sync( + &environment.pypi_dependencies, + &prefix_records, + &prefix, + environment.platform.unwrap_or_else(Platform::current), + )?; + if !pypi_in_sync { + tracing::debug!("Environment out of sync because PyPI packages don't match"); + return Ok(false); + } + tracing::debug!("Verify that the binaries are in sync with the environment"); let (exec_to_remove, exec_to_add) = expose_scripts_sync_status(&self.bin_dir, &env_dir, &environment.exposed).await?; diff --git a/crates/pixi_global/src/project/parsed_manifest.rs b/crates/pixi_global/src/project/parsed_manifest.rs index 40fe65768f..d95781e2be 100644 --- a/crates/pixi_global/src/project/parsed_manifest.rs +++ b/crates/pixi_global/src/project/parsed_manifest.rs @@ -7,6 +7,7 @@ use itertools::{Either, Itertools}; use miette::{Context, Diagnostic, IntoDiagnostic, LabeledSpan, NamedSource, Report}; use pixi_consts::consts; use pixi_manifest::{PrioritizedChannel, toml::TomlPlatform, utils::package_map::UniquePackageMap}; +use pixi_pypi_spec::{PixiPypiSpec, PypiPackageName}; use pixi_spec::PixiSpec; use pixi_toml::{TomlFromStr, TomlIndexMap, TomlIndexSet, TomlWith}; use rattler_conda_types::{NamedChannelOrUrl, PackageName, Platform}; @@ -297,6 +298,10 @@ pub struct ParsedEnvironment { /// Platform used by the environment. pub platform: Option, pub dependencies: UniquePackageMap, + /// PyPI packages installed into the environment's site-packages after + /// the conda packages. + #[serde(default, skip_serializing_if = "IndexMap::is_empty")] + pub pypi_dependencies: IndexMap, #[serde(default, serialize_with = "serialize_expose_mappings")] pub exposed: IndexSet, pub shortcuts: Option>, @@ -312,6 +317,10 @@ impl<'de> toml_span::Deserialize<'de> for ParsedEnvironment { .unwrap_or_default(); let platform = th.optional::("platform").map(Platform::from); let dependencies = th.optional("dependencies").unwrap_or_default(); + let pypi_dependencies = th + .optional::>("pypi-dependencies") + .map(TomlIndexMap::into_inner) + .unwrap_or_default(); let exposed = th .optional::("exposed") .map(TomlMapping::into_inner) @@ -326,6 +335,7 @@ impl<'de> toml_span::Deserialize<'de> for ParsedEnvironment { channels, platform, dependencies, + pypi_dependencies, exposed, shortcuts, }) @@ -413,6 +423,8 @@ impl AsRef for ExposedName { #[cfg(test)] mod tests { + use std::str::FromStr; + use insta::assert_snapshot; use super::ParsedManifest; @@ -543,4 +555,36 @@ mod tests { "#; let _manifest = ParsedManifest::from_toml_str(contents).unwrap(); } + + #[test] + fn test_pypi_dependencies_deserialization() { + let contents = r#" + [envs.jupyter] + channels = ["conda-forge"] + + [envs.jupyter.dependencies] + python = "3.12.*" + + [envs.jupyter.pypi-dependencies] + jupyterlab = "*" + flask = { version = ">=2.0", extras = ["async"] } + + [envs.jupyter.exposed] + jupyter = "jupyter" + "#; + let manifest = ParsedManifest::from_toml_str(contents).unwrap(); + let env_name = super::EnvironmentName::from_str("jupyter").unwrap(); + let env = manifest.envs.get(&env_name).unwrap(); + assert_eq!(env.pypi_dependencies.len(), 2); + assert!( + env.pypi_dependencies + .keys() + .any(|name| name.as_normalized().to_string() == "jupyterlab") + ); + assert!( + env.pypi_dependencies + .keys() + .any(|name| name.as_normalized().to_string() == "flask") + ); + } } diff --git a/crates/pixi_global/src/project/snapshots/pixi_global__project__parsed_manifest__tests__invalid_key.snap b/crates/pixi_global/src/project/snapshots/pixi_global__project__parsed_manifest__tests__invalid_key.snap index c7676070e8..64d8b3a1d5 100644 --- a/crates/pixi_global/src/project/snapshots/pixi_global__project__parsed_manifest__tests__invalid_key.snap +++ b/crates/pixi_global/src/project/snapshots/pixi_global__project__parsed_manifest__tests__invalid_key.snap @@ -1,9 +1,9 @@ --- -source: src/global/project/parsed_manifest.rs +source: crates/pixi_global/src/project/parsed_manifest.rs expression: "examples.into_iter().map(|example|\nParsedManifest::from_toml_str(example).unwrap_err().to_string()).collect::>().join(\"\\n\")" --- unexpected keys in table: `[("invalid", Span { start: 1, end: 8 })]` expected: ["version", "envs"] unexpected keys in table: `[("invalid", Span { start: 14, end: 21 })]` -expected: ["channels", "platform", "dependencies", "exposed", "shortcuts"] +expected: ["channels", "platform", "dependencies", "pypi-dependencies", "exposed", "shortcuts"] Failed to parse environment name 'python;3', please use only lowercase letters, numbers, dashes, underscores and dots diff --git a/crates/pixi_install_pypi/Cargo.toml b/crates/pixi_install_pypi/Cargo.toml index 9560f828cd..4007809375 100644 --- a/crates/pixi_install_pypi/Cargo.toml +++ b/crates/pixi_install_pypi/Cargo.toml @@ -10,12 +10,19 @@ version = "0.1.0" [dependencies] ahash = { workspace = true } +async-once-cell = { workspace = true } console = { workspace = true } csv = { workspace = true } fancy_display = { workspace = true } fs-err = { workspace = true } +futures = { workspace = true } +indexmap = { workspace = true } +indicatif = { workspace = true } itertools = { workspace = true } miette = { workspace = true } +once_cell = { workspace = true } +ordermap = { workspace = true } +pathdiff = { workspace = true } pep440_rs = { workspace = true } pep508_rs = { workspace = true } percent-encoding = { workspace = true } @@ -24,15 +31,19 @@ pixi_git = { workspace = true } pixi_manifest = { workspace = true, features = ["rattler_lock"] } pixi_path = { workspace = true } pixi_progress = { workspace = true } +pixi_pypi_spec = { workspace = true } pixi_python_status = { workspace = true } pixi_record = { workspace = true } -pixi_reporters = { workspace = true } +pixi_spec = { workspace = true } pixi_utils = { workspace = true, default-features = false } pixi_uv_context = { workspace = true } pixi_uv_conversions = { workspace = true } +pixi_uv_reporter = { workspace = true } +pypi_mapping = { workspace = true } pypi_modifiers = { workspace = true } rattler = { workspace = true } rattler_conda_types = { workspace = true } +rattler_digest = { workspace = true } rattler_lock = { workspace = true } rayon = { workspace = true } serde = { workspace = true } @@ -42,8 +53,10 @@ tokio = { workspace = true } tracing = { workspace = true } typed-path = { workspace = true } url = { workspace = true } +uv-build-frontend = { workspace = true } uv-cache = { workspace = true } uv-cache-info = { workspace = true } +uv-cache-key = { workspace = true } uv-client = { workspace = true } uv-configuration = { workspace = true } uv-dispatch = { workspace = true } @@ -52,6 +65,7 @@ uv-distribution-filename = { workspace = true } uv-distribution-types = { workspace = true } uv-flags = { workspace = true } uv-git = { workspace = true } +uv-git-types = { workspace = true } uv-install-wheel = { workspace = true } uv-installer = { workspace = true } uv-normalize = { workspace = true } @@ -61,6 +75,8 @@ uv-preview = { workspace = true } uv-pypi-types = { workspace = true } uv-python = { workspace = true } uv-redacted = { workspace = true } +uv-requirements = { workspace = true } +uv-requirements-txt = { workspace = true } uv-resolver = { workspace = true } uv-types = { workspace = true } uv-workspace = { workspace = true } diff --git a/crates/pixi_install_pypi/src/lib.rs b/crates/pixi_install_pypi/src/lib.rs index ffccfe7a0d..0372c0779c 100644 --- a/crates/pixi_install_pypi/src/lib.rs +++ b/crates/pixi_install_pypi/src/lib.rs @@ -19,13 +19,13 @@ use pixi_manifest::{ use pixi_progress::await_in_progress; use pixi_python_status::PythonStatus; use pixi_record::PixiRecord; -use pixi_reporters::{UvReporter, UvReporterOptions}; use pixi_utils::prefix::Prefix; use pixi_uv_context::UvResolutionContext; use pixi_uv_conversions::{ BuildIsolation, configure_insecure_hosts_for_tls_bypass, locked_indexes_to_index_locations, pypi_cache_config_settings, pypi_options_to_build_options, to_exclude_newer, to_index_strategy, }; +use pixi_uv_reporter::{UvReporter, UvReporterOptions}; use plan::{InstallPlanner, InstallReason, NeedReinstall, PyPIInstallationPlan}; use pypi_modifiers::{ Tags, @@ -138,7 +138,9 @@ pub(crate) mod cache_scoped_build_context; pub(crate) mod conda_pypi_clobber; pub(crate) mod conversions; pub(crate) mod install_wheel; +pub mod package_identifier; pub(crate) mod plan; +pub mod resolve; pub(crate) mod utils; use cache_scoped_build_context::CacheScopedBuildContext; diff --git a/crates/pixi_install_pypi/src/package_identifier.rs b/crates/pixi_install_pypi/src/package_identifier.rs new file mode 100644 index 0000000000..9ed8badb6c --- /dev/null +++ b/crates/pixi_install_pypi/src/package_identifier.rs @@ -0,0 +1,142 @@ +use pixi_uv_conversions::{ConversionError as PixiConversionError, to_normalize}; +use rattler_conda_types::{PackageRecord, PackageUrl, RepoDataRecord}; +use std::{collections::HashSet, str::FromStr}; +use thiserror::Error; + +use pixi_pypi_spec::PypiPackageName; +use uv_normalize::{ExtraName, InvalidNameError}; + +/// Defines information about a Pypi package extracted from either a python +/// package or from a conda package. That can be used for comparison in both +#[derive(Debug)] +pub struct PypiPackageIdentifier { + pub name: PypiPackageName, + pub version: pep440_rs::Version, + pub extras: HashSet, +} + +impl PypiPackageIdentifier { + /// Extracts the python packages that will be installed when the specified + /// conda package is installed. + pub fn from_repodata_record(record: &RepoDataRecord) -> Result, ConversionError> { + let mut result = Vec::new(); + Self::from_record_into(record, &mut result)?; + + Ok(result) + } + + pub fn from_package_record(record: &PackageRecord) -> Result, ConversionError> { + let mut result = Vec::new(); + if let Some(purls) = &record.purls { + for purl in purls.iter() { + if let Some(entry) = Self::convert_from_purl(purl, &record.version.as_str())? { + result.push(entry); + } + } + } + Ok(result) + } + + /// Helper function to write the result of extract the python packages that + /// will be installed into a pre-allocated vector. + fn from_record_into( + record: &RepoDataRecord, + result: &mut Vec, + ) -> Result<(), ConversionError> { + let mut has_pypi_purl = false; + let identifiers = Self::from_package_record(&record.package_record)?; + if !identifiers.is_empty() { + has_pypi_purl = true; + result.extend(identifiers); + } + + // Backwards compatibility: + // If lock file don't have a purl + // but the package is a conda-forge package, we just assume that + // the name of the package is equivalent to the name of the python package. + // In newer versions of the lock file, we should always have a purl + // where empty purls means that the package is not a pypi-one. + if record.package_record.purls.is_none() + && !has_pypi_purl + && pypi_mapping::is_conda_forge_record(record) + { + tracing::trace!( + "Using backwards compatibility purl logic for conda package: {}", + record.package_record.name.as_source() + ); + // Convert the conda package names to pypi package names. If the conversion fails we + // just assume that its not a valid python package. + let name = + uv_normalize::PackageName::from_str(record.package_record.name.as_source()).ok(); + let version = + pep440_rs::Version::from_str(&record.package_record.version.as_str()).ok(); + if let (Some(name), Some(version)) = (name, version) { + let pep_name = to_normalize(&name)?; + + result.push(PypiPackageIdentifier { + name: PypiPackageName::from_normalized(pep_name), + version, + // TODO: We can't really tell which python extras are enabled in a conda + // package. + extras: Default::default(), + }) + } + } + + Ok(()) + } + + /// Tries to construct an instance from a generic PURL. + /// + /// The `fallback_version` is used if the PURL does not contain a version. + pub fn convert_from_purl( + package_url: &PackageUrl, + fallback_version: &str, + ) -> Result, ConversionError> { + if package_url.package_type() == "pypi" { + Self::from_pypi_purl(package_url, fallback_version).map(Some) + } else { + Ok(None) + } + } + + /// Constructs a new instance from a PyPI package URL. + /// + /// The `fallback_version` is used if the PURL does not contain a version. + pub fn from_pypi_purl( + package_url: &PackageUrl, + fallback_version: &str, + ) -> Result { + assert_eq!(package_url.package_type(), "pypi"); + let name = package_url.name(); + let name = uv_normalize::PackageName::from_str(name) + .map_err(|e| ConversionError::PackageName(name.to_string(), e))?; + + let version_str = package_url.version().unwrap_or(fallback_version); + let version = pep440_rs::Version::from_str(version_str) + .map_err(|_| ConversionError::Version(version_str.to_string()))?; + + // TODO: We can't really tell which python extras are enabled from a PURL. + let extras = HashSet::new(); + let pep_name = to_normalize(&name)?; + + Ok(Self { + name: PypiPackageName::from_normalized(pep_name), + version, + extras, + }) + } +} + +#[derive(Error, Debug)] +pub enum ConversionError { + #[error("'{0}' is not a valid python package name")] + PackageName(String, #[source] InvalidNameError), + + #[error("'{0}' is not a valid python version")] + Version(String), + // #[error("'{0}' is not a valid python extra")] + // Extra(String), + #[error(transparent)] + NameConversion(#[from] PixiConversionError), +} diff --git a/crates/pixi_core/src/lock_file/resolve/build_dispatch.rs b/crates/pixi_install_pypi/src/resolve/build_dispatch.rs similarity index 87% rename from crates/pixi_core/src/lock_file/resolve/build_dispatch.rs rename to crates/pixi_install_pypi/src/resolve/build_dispatch.rs index 188e1305f9..1ed2545820 100644 --- a/crates/pixi_core/src/lock_file/resolve/build_dispatch.rs +++ b/crates/pixi_install_pypi/src/resolve/build_dispatch.rs @@ -16,22 +16,16 @@ //! the parameters needed to create a `BuildContext` uv implementation. //! and holds struct that is used to instantiate the conda prefix when its //! needed. -use std::cell::Cell; -use std::collections::HashSet; +use std::pin::Pin; use std::sync::Arc; use std::{collections::HashMap, path::Path}; -use crate::environment::{CondaPrefixUpdated, CondaPrefixUpdater}; -use crate::{ - activation::CurrentEnvVarBehavior, - workspace::{Environment, EnvironmentVars, get_activated_environment_variables}, -}; +use crate::initialize_uv_flags; use async_once_cell::OnceCell as AsyncCell; use once_cell::sync::OnceCell; -use pixi_install_pypi::initialize_uv_flags; -use pixi_manifest::EnvironmentName; use pixi_manifest::pypi::pypi_options::NoBuildIsolation; -use pixi_record::PixiRecord; +use pixi_python_status::PythonStatus; +use pixi_utils::prefix::Prefix; use pixi_uv_conversions::BuildIsolation; use uv_build_frontend::SourceBuild; use uv_cache::Cache; @@ -143,7 +137,6 @@ impl<'a> UvBuildDispatchParams<'a> { } /// Set the constraints for the build dispatch - #[expect(unused)] pub fn with_constraints(mut self, constraints: Constraints) -> Self { self.constraints = constraints; self @@ -155,7 +148,6 @@ impl<'a> UvBuildDispatchParams<'a> { self } - #[expect(unused)] pub fn with_preview_mode(mut self, preview: uv_preview::Preview) -> Self { self.preview = preview; self @@ -166,7 +158,6 @@ impl<'a> UvBuildDispatchParams<'a> { self } - #[expect(unused)] pub fn with_package_config_settings( mut self, package_config_settings: PackageConfigSettings, @@ -175,13 +166,36 @@ impl<'a> UvBuildDispatchParams<'a> { self } - #[expect(unused)] pub fn with_extra_build_requires(mut self, extra_build_requires: ExtraBuildRequires) -> Self { self.extra_build_requires = extra_build_requires; self } } +/// A conda prefix with a python interpreter, instantiated on demand when uv +/// needs to build a source distribution during a PyPI resolve. +pub struct ProvidedCondaPrefix { + /// The prefix the conda packages were installed into. + pub prefix: Prefix, + + /// The state of the python interpreter in the prefix. + pub python_status: PythonStatus, + + /// Environment variables (e.g. from activation of the prefix) exposed to + /// PEP 517 build backends. + pub env_vars: HashMap, +} + +/// Provides a conda prefix (and its activation environment) on demand. +/// +/// Instantiating a conda prefix is expensive and only required when a source +/// distribution actually has to be built, so [`LazyBuildDispatch`] defers the +/// call to [`CondaPrefixProvider::provide`] until uv first asks for a build. +/// Implementations are expected to memoize so repeated calls are cheap. +pub trait CondaPrefixProvider { + fn provide(&self) -> Pin> + '_>>; +} + /// Handles the lazy initialization of a build dispatch. /// /// A build dispatch is used to manage building Python packages from source, @@ -193,19 +207,12 @@ impl<'a> UvBuildDispatchParams<'a> { /// Both the [`BuildDispatch`] and the conda prefix are instantiated on demand. pub struct LazyBuildDispatch<'a> { pub params: UvBuildDispatchParams<'a>, - pub prefix_updater: CondaPrefixUpdater, - pub repodata_records: Cell>>>, - pub build_dispatch: AsyncCell>, - - // if we create a new conda prefix, we need to store the task result - // so that we can reuse it later - pub conda_task: Option, + /// Provides the conda prefix (python interpreter + activation env vars) + /// when a source build first requires one. + pub prefix_provider: &'a dyn CondaPrefixProvider, - // project environment variables - // this is used to get the activated environment variables - pub project_env_vars: HashMap, - pub environment: Environment<'a>, + pub build_dispatch: AsyncCell>, // what pkgs we dont need to activate pub no_build_isolation: NoBuildIsolation, @@ -218,8 +225,6 @@ pub struct LazyBuildDispatch<'a> { workspace_cache: WorkspaceCache, - pub ignore_packages: Option>, - /// Shared error holder for storing initialization errors that can be retrieved /// after the LazyBuildDispatch is consumed (e.g., in catch_unwind scenarios) pub last_error: Arc>, @@ -285,32 +290,22 @@ impl IsBuildBackendError for LazyBuildDispatchError { impl<'a> LazyBuildDispatch<'a> { /// Create a new `PixiBuildDispatch` instance. - #[allow(clippy::too_many_arguments)] pub fn new( params: UvBuildDispatchParams<'a>, - prefix_updater: CondaPrefixUpdater, - project_env_vars: HashMap, - environment: Environment<'a>, - repodata_records: miette::Result>, + prefix_provider: &'a dyn CondaPrefixProvider, no_build_isolation: NoBuildIsolation, lazy_deps: &'a LazyBuildDispatchDependencies, - ignore_packages: Option>, disallow_install_conda_prefix: bool, last_error: Arc>, ) -> Self { Self { params, - prefix_updater, - conda_task: None, - project_env_vars, - environment, - repodata_records: Cell::new(Some(repodata_records)), + prefix_provider, no_build_isolation, build_dispatch: AsyncCell::new(), lazy_deps, disallow_install_conda_prefix, workspace_cache: WorkspaceCache::default(), - ignore_packages, last_error, } } @@ -326,46 +321,22 @@ impl<'a> LazyBuildDispatch<'a> { if self.disallow_install_conda_prefix { return Err(LazyBuildDispatchError::InstallationRequiredButDisallowed); } - tracing::debug!( - "PyPI solve requires instantiation of conda prefix for '{}'", - self.prefix_updater.name().as_str() - ); - - let repodata_records = self - .repodata_records - .replace(None) - .expect("this function cannot be called twice") - .map_err(|err| LazyBuildDispatchError::InitializationError(err.into()))?; + tracing::debug!("PyPI solve requires instantiation of a conda prefix"); - let prefix = self - .prefix_updater - .update( - repodata_records.into_iter().map(Into::into).collect(), - None, - self.ignore_packages.clone(), - ) + let provided = self + .prefix_provider + .provide() .await .map_err(|err| LazyBuildDispatchError::InitializationError(err.into()))?; - // get the activation vars - let env_vars = get_activated_environment_variables( - &self.project_env_vars, - &self.environment, - CurrentEnvVarBehavior::Exclude, - None, - false, - false, - ) - .await - .map_err(|err| LazyBuildDispatchError::InitializationError(err.into()))?; - - let python_path = prefix + let python_path = provided .python_status .location() - .map(|path| prefix.prefix.root().join(path)) + .map(|path| provided.prefix.root().join(path)) .ok_or_else(|| LazyBuildDispatchError::PythonMissingError { - prefix: prefix.prefix.root().display().to_string(), + prefix: provided.prefix.root().display().to_string(), })?; + let env_vars = provided.env_vars; let interpreter = self .lazy_deps diff --git a/crates/pixi_core/src/lock_file/resolve/pypi.rs b/crates/pixi_install_pypi/src/resolve/mod.rs similarity index 93% rename from crates/pixi_core/src/lock_file/resolve/pypi.rs rename to crates/pixi_install_pypi/src/resolve/mod.rs index 9b57974da6..a8b1caf56b 100644 --- a/crates/pixi_core/src/lock_file/resolve/pypi.rs +++ b/crates/pixi_install_pypi/src/resolve/mod.rs @@ -1,3 +1,17 @@ +//! Resolution of PyPI dependencies against a conda environment. +//! +//! [`resolve_pypi`] drives uv's resolver to lock the PyPI side of an +//! environment. Conda-installed python packages override their PyPI +//! counterparts (see `CondaResolverProvider`), and when a source +//! distribution must be built to obtain metadata, a conda prefix with a +//! python interpreter is instantiated on demand through the caller-supplied +//! [`CondaPrefixProvider`]. + +pub mod build_dispatch; +mod resolver_provider; + +pub use build_dispatch::{CondaPrefixProvider, LazyBuildDispatchDependencies, ProvidedCondaPrefix}; + use std::{ cell::RefCell, collections::{HashMap, HashSet}, @@ -20,14 +34,9 @@ use itertools::{Either, Itertools}; use miette::{Context, IntoDiagnostic}; use ordermap::OrderSet; use pixi_consts::consts; -use pixi_install_pypi::{LockedPypiRecord, UnresolvedPypiRecord}; -use pixi_manifest::{ - EnvironmentName, HasWorkspaceManifest, PixiPlatform, PixiPlatformName, SolveStrategy, - pypi::pypi_options::PypiOptions, -}; +use pixi_manifest::{PixiPlatform, SolveStrategy, pypi::pypi_options::PypiOptions}; use pixi_pypi_spec::PixiPypiSpec; use pixi_record::{LockedGitUrl, PixiRecord}; -use pixi_reporters::{UvReporter, UvReporterOptions}; use pixi_uv_conversions::{ ConversionError, as_uv_req, configure_insecure_hosts_for_tls_bypass, convert_uv_requirements_to_pep508, into_pinned_git_spec, into_uv_git_reference, @@ -35,6 +44,7 @@ use pixi_uv_conversions::{ to_index_strategy, to_prerelease_mode, to_requirements, to_uv_normalize, to_uv_version, to_version_specifiers, }; +use pixi_uv_reporter::{UvReporter, UvReporterOptions}; use pypi_modifiers::{ pypi_marker_env::determine_marker_environment, pypi_tags::{get_pypi_tags, is_python_record}, @@ -67,24 +77,14 @@ use uv_resolver::{ use uv_types::EmptyInstalledPackages; use crate::{ - environment::CondaPrefixUpdated, - lock_file::{ - CondaPrefixUpdater, LockedPypiRecords, PixiRecordsByName, PypiPackageIdentifier, - outdated::PypiEnvironmentBuildCache, - records_by_name::HasNameVersion, - resolve::{ - build_dispatch::{LazyBuildDispatch, UvBuildDispatchParams}, - resolver_provider::CondaResolverProvider, - }, - }, - workspace::{ - Environment, EnvironmentVars, HasWorkspaceRef, PlatformOverrides, PlatformSource, - grouped_environment::GroupedEnvironment, + LockedPypiRecord, UnresolvedPypiRecord, + package_identifier::PypiPackageIdentifier, + resolve::{ + build_dispatch::{LazyBuildDispatch, UvBuildDispatchParams}, + resolver_provider::CondaResolverProvider, }, }; -use pixi_command_dispatcher::CommandDispatcher; use pixi_uv_context::UvResolutionContext; -use rattler_conda_types::GenericVirtualPackage; #[derive(Debug, thiserror::Error)] #[error("Invalid hash: {0} type: {1}")] @@ -295,19 +295,16 @@ pub async fn resolve_pypi( dependencies: IndexMap>, locked_pixi_records: &[PixiRecord], locked_pypi_packages: &[UnresolvedPypiRecord], - platform: PixiPlatformName, + platform: &PixiPlatform, pb: &ProgressBar, project_root: &Path, - command_dispatcher: CommandDispatcher, - repodata_building_records: miette::Result>, - project_env_vars: HashMap, - environment: Environment<'_>, + conda_prefix_provider: &dyn CondaPrefixProvider, disallow_install_conda_prefix: bool, exclude_newer: uv_resolver::ExcludeNewer, solve_strategy: SolveStrategy, - build_cache: Arc, + lazy_build_dispatch_deps: &LazyBuildDispatchDependencies, link_mode: LinkMode, -) -> miette::Result<(LockedPypiRecords, Option)> { +) -> miette::Result> { // Solve python packages pb.set_message("resolving pypi dependencies"); @@ -416,14 +413,7 @@ pub async fn resolve_pypi( })?; // Construct the marker environment for the target platform - let pixi_platform = environment - .workspace_manifest() - .workspace - .platform_by_name(&platform) - .ok_or_else(|| { - miette::miette!("workspace does not define a platform named '{platform}'") - })?; - let marker_environment = determine_marker_environment(pixi_platform, python_record.as_ref())?; + let marker_environment = determine_marker_environment(platform, python_record.as_ref())?; let requirements = dependencies .into_iter() @@ -436,7 +426,7 @@ pub async fn resolve_pypi( .into_diagnostic()?; // Determine the tags for this particular solve. - let tags = get_pypi_tags(pixi_platform, python_record.as_ref())?; + let tags = get_pypi_tags(platform, python_record.as_ref())?; // We need to setup both an interpreter and a requires_python specifier. // The interpreter is used to (potentially) build the wheel, and the @@ -446,13 +436,8 @@ pub async fn resolve_pypi( // A python-3.10.6-xxx.conda package record becomes a "==3.10.6.*" requires python specifier. let python_specifier = uv_pep440::VersionSpecifier::from_version( uv_pep440::Operator::EqualStar, - uv_pep440::Version::from_str( - &python_record - .version() - .expect("python record always has a version") - .as_str(), - ) - .into_diagnostic()?, + uv_pep440::Version::from_str(&python_record.package_record().version.as_str()) + .into_diagnostic()?, ) .into_diagnostic() .context("error creating version specifier for python version")?; @@ -595,55 +580,13 @@ pub async fn resolve_pypi( .with_concurrency(context.concurrency.clone()) .with_link_mode(link_mode); - // Use cached build dispatch dependencies - let lazy_build_dispatch_deps = &build_cache.lazy_build_dispatch_deps; - let last_error = Arc::new(OnceCell::new()); - // Use cached conda_prefix_updater if available, otherwise create new - let conda_prefix_updater = build_cache - .conda_prefix_updater - .get_or_try_init(|| { - // The conda prefix has to run on the current system; cross-platform - // pypi resolves still need a local Python to compute wheel tags. Fall - // back to a bare current-subdir platform when no declared workspace - // platform matches this machine. - let host_platform; - let prefix_platform: &PixiPlatform = match environment.best_declared_platform() { - Some(p) => p, - None => { - host_platform = environment.workspace().host_platform( - PlatformSource::Defaults, - PlatformOverrides::EnvironmentVariableOverrides, - ); - &host_platform - } - }; - let group = GroupedEnvironment::Environment(environment.clone()); - let virtual_packages = environment.virtual_packages(prefix_platform); - - CondaPrefixUpdater::builder( - group, - prefix_platform.clone(), - virtual_packages - .into_iter() - .map(GenericVirtualPackage::from) - .collect(), - command_dispatcher, - ) - .finish() - })? - .clone(); - let lazy_build_dispatch = LazyBuildDispatch::new( build_params, - conda_prefix_updater, - project_env_vars, - environment, - repodata_building_records.map(|r| r.records.clone()), + conda_prefix_provider, pypi_options.no_build_isolation.clone(), lazy_build_dispatch_deps, - None, disallow_install_conda_prefix, Arc::clone(&last_error), ); @@ -696,11 +639,11 @@ pub async fn resolve_pypi( let preferences = locked_pypi_packages .iter() .map(|record| { - let Some(version) = record.version() else { + let Some(version) = record.as_package_data().version() else { return Ok(None); }; let requirement = uv_pep508::Requirement { - name: to_uv_normalize(record.name())?, + name: to_uv_normalize(record.as_package_data().name())?, extras: Vec::new().into(), version_or_url: Some(uv_pep508::VersionOrUrl::VersionSpecifier( uv_pep440::VersionSpecifiers::from( @@ -869,13 +812,11 @@ pub async fn resolve_pypi( .await .map_err(|e| SolveError::Locking(e.into()))?; - let conda_task = lazy_build_dispatch.conda_task; - - Ok::<_, SolveError>((locked_packages, conda_task)) + Ok::<_, SolveError>(locked_packages) }); // We try to distinguish between build dispatch panics and any other panics that occur - let (locked_packages, conda_task) = match resolution_future.catch_unwind().await { + let locked_packages = match resolution_future.catch_unwind().await { Ok(result) => result?, Err(panic_payload) => { // Try to get the stored initialization error from the last_error holder @@ -902,7 +843,7 @@ pub async fn resolve_pypi( } }; - Ok((locked_packages, conda_task)) + Ok(locked_packages) } #[derive(Debug, thiserror::Error)] @@ -1029,7 +970,7 @@ async fn lock_pypi_packages( downloads_semaphore: Arc, abs_project_root: &Path, original_git_references: &HashMap, -) -> miette::Result { +) -> miette::Result> { let mut locked_packages = Vec::with_capacity(resolution.len()); let database = DistributionDatabase::new(registry_client, pixi_build_dispatch, downloads_semaphore); diff --git a/crates/pixi_core/src/lock_file/resolve/resolver_provider.rs b/crates/pixi_install_pypi/src/resolve/resolver_provider.rs similarity index 99% rename from crates/pixi_core/src/lock_file/resolve/resolver_provider.rs rename to crates/pixi_install_pypi/src/resolve/resolver_provider.rs index fbbb595b1f..da1be3ed86 100644 --- a/crates/pixi_core/src/lock_file/resolve/resolver_provider.rs +++ b/crates/pixi_install_pypi/src/resolve/resolver_provider.rs @@ -23,7 +23,7 @@ use uv_resolver::{ }; use uv_types::BuildContext; -use crate::lock_file::PypiPackageIdentifier; +use crate::package_identifier::PypiPackageIdentifier; pub(super) struct CondaResolverProvider<'a, Context: BuildContext> { pub(super) fallback: DefaultResolverProvider<'a, Context>, diff --git a/crates/pixi_reporters/Cargo.toml b/crates/pixi_reporters/Cargo.toml index f83d674028..0c6bc02015 100644 --- a/crates/pixi_reporters/Cargo.toml +++ b/crates/pixi_reporters/Cargo.toml @@ -13,7 +13,6 @@ version = "0.1.0" # Standard library futures = { workspace = true } indexmap = { workspace = true } -itertools = { workspace = true } parking_lot = { workspace = true } tokio = { workspace = true } url = { workspace = true } @@ -33,6 +32,7 @@ pixi_command_dispatcher = { workspace = true } pixi_compute_reporters = { workspace = true } pixi_git = { workspace = true } pixi_progress = { workspace = true } +pixi_uv_reporter = { workspace = true } # Rattler ecosystem rattler = { workspace = true } @@ -41,12 +41,6 @@ rattler_repodata_gateway = { workspace = true } # UV ecosystem uv-configuration = { workspace = true } -uv-distribution = { workspace = true } -uv-distribution-types = { workspace = true } -uv-installer = { workspace = true } -uv-normalize = { workspace = true } -uv-redacted = { workspace = true } -uv-resolver = { workspace = true } [dev-dependencies] # Add test dependencies if needed diff --git a/crates/pixi_reporters/src/lib.rs b/crates/pixi_reporters/src/lib.rs index 2491ae9276..80d40bc2dc 100644 --- a/crates/pixi_reporters/src/lib.rs +++ b/crates/pixi_reporters/src/lib.rs @@ -4,7 +4,6 @@ mod main_progress_bar; mod release_notes; mod repodata_reporter; mod sync_reporter; -pub mod uv_reporter; use std::{collections::HashMap, sync::Arc}; @@ -22,8 +21,9 @@ pub use release_notes::format_release_notes; use repodata_reporter::RepodataReporter; use sync_reporter::SyncReporter; use uv_configuration::initialize_rayon_once; -// Re-export the uv_reporter types for external use -pub use uv_reporter::{UvReporter, UvReporterOptions}; +// Re-export the uv reporter types for external use; they live in their own +// crate so that crates below the command dispatcher can use them too. +pub use pixi_uv_reporter::{UvReporter, UvReporterOptions}; /// Top-level progress reporter for `pixi`'s CLI. Use /// [`Self::register_with`] to wire it into a [`CommandDispatcherBuilder`]; diff --git a/crates/pixi_uv_reporter/Cargo.toml b/crates/pixi_uv_reporter/Cargo.toml new file mode 100644 index 0000000000..243bc54738 --- /dev/null +++ b/crates/pixi_uv_reporter/Cargo.toml @@ -0,0 +1,24 @@ +[package] +authors.workspace = true +description = "Progress reporting for uv operations in pixi" +edition.workspace = true +homepage.workspace = true +license.workspace = true +name = "pixi_uv_reporter" +readme.workspace = true +repository.workspace = true +version = "0.1.0" + +[dependencies] +indicatif = { workspace = true } +itertools = { workspace = true } + +pixi_git = { workspace = true } +pixi_progress = { workspace = true } + +uv-distribution = { workspace = true } +uv-distribution-types = { workspace = true } +uv-installer = { workspace = true } +uv-normalize = { workspace = true } +uv-redacted = { workspace = true } +uv-resolver = { workspace = true } diff --git a/crates/pixi_reporters/src/uv_reporter.rs b/crates/pixi_uv_reporter/src/lib.rs similarity index 98% rename from crates/pixi_reporters/src/uv_reporter.rs rename to crates/pixi_uv_reporter/src/lib.rs index f9c7be261a..75c7a379a7 100644 --- a/crates/pixi_reporters/src/uv_reporter.rs +++ b/crates/pixi_uv_reporter/src/lib.rs @@ -1,3 +1,6 @@ +//! Progress reporting for uv operations (resolution, preparation and +//! installation of PyPI distributions) driven by pixi. + use indicatif::ProgressBar; use itertools::Itertools; use pixi_git::GIT_SSH_CLONING_WARNING_MSG; diff --git a/docs/global_tools/manifest.md b/docs/global_tools/manifest.md index 2ab2751d6b..205a70962a 100644 --- a/docs/global_tools/manifest.md +++ b/docs/global_tools/manifest.md @@ -124,6 +124,33 @@ You can [`remove`](../reference/cli/pixi/global/remove.md) dependencies by runni pixi global remove --environment my-env package-a package-b ``` +## PyPI dependencies + +Next to conda packages, an environment can also contain packages from PyPI. +They are declared in the `pypi-dependencies` table of the environment and are +installed into the environment after the conda packages. +The environment must contain a `python` interpreter in its `dependencies` for +this to work. + +```toml +[envs.jupyter] +channels = ["conda-forge"] +dependencies = { python = "3.13.*" } +pypi-dependencies = { jupyterlab = "*" } +exposed = { jupyter = "jupyter" } +``` + +The next `pixi global sync` will resolve the PyPI packages against the +interpreter in the environment and install them into its `site-packages`. + +From the command line, PyPI packages are added with the `--pypi` flag, which +accepts PEP 508 requirements: + +```shell +pixi global install --environment jupyter python --pypi jupyterlab +pixi global add --environment jupyter --pypi "flask>=2" +pixi global remove --environment jupyter flask +``` ## Exposed executables diff --git a/docs/reference/cli/pixi/global/add.md b/docs/reference/cli/pixi/global/add.md index 3e0b5eaf5c..82b8eaa07d 100644 --- a/docs/reference/cli/pixi/global/add.md +++ b/docs/reference/cli/pixi/global/add.md @@ -27,6 +27,9 @@ pixi global add [OPTIONS] --environment [PACKAGE]... - `--expose ` : Add one or more mapping which describe which executables are exposed. The syntax is `exposed_name=executable_name`, so for example `python3.10=python`. Alternatively, you can input only an executable_name and `executable_name=executable_name` is assumed
May be provided more than once. +- `--pypi ` +: Add a PyPI package to the environment, in PEP 508 format. The environment must contain a python interpreter in its dependencies +
May be provided more than once. ## Config Options - `--auth-file ` @@ -79,6 +82,7 @@ Example: - `pixi global add --environment python numpy` - `pixi global add --environment my_env pytest pytest-cov --expose pytest=pytest` +- `pixi global add --environment python --pypi httpx` --8<-- "docs/reference/cli/pixi/global/add_extender:example" diff --git a/docs/reference/cli/pixi/global/install.md b/docs/reference/cli/pixi/global/install.md index f6e13a4569..e76230134b 100644 --- a/docs/reference/cli/pixi/global/install.md +++ b/docs/reference/cli/pixi/global/install.md @@ -34,6 +34,9 @@ pixi global install [OPTIONS] [PACKAGE]... - `--with ` : Add additional dependencies to the environment. Their executables will not be exposed
May be provided more than once. +- `--pypi ` +: Add a PyPI package to the environment, in PEP 508 format. The environment must contain a python interpreter in its dependencies +
May be provided more than once. - `--force-reinstall` : Specifies that the environment should be reinstalled - `--no-shortcuts` @@ -92,6 +95,7 @@ Example: - `pixi global install jupyter --with polars` - `pixi global install --expose python3.8=python python=3.8` - `pixi global install --environment science --expose jupyter --expose ipython jupyter ipython polars` +- `pixi global install python --pypi httpx --pypi "flask>=2"` --8<-- "docs/reference/cli/pixi/global/install_extender:example"