diff --git a/Cargo.lock b/Cargo.lock index 2d93f83ae7..b80bded9fc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5987,6 +5987,7 @@ dependencies = [ "pixi_utils", "pypi_mapping", "rattler_conda_types", + "rattler_digest", "rattler_lock", "rattler_package_streaming", "rattler_virtual_packages", @@ -6984,6 +6985,7 @@ dependencies = [ "dunce", "fancy_display", "fs-err", + "humantime", "indexmap 2.14.0", "insta", "itertools 0.14.0", @@ -7703,9 +7705,11 @@ dependencies = [ "astral-reqwest-retry", "async-once-cell", "dashmap", + "filetime", "fs-err", "futures", "http-cache-reqwest", + "humantime", "itertools 0.14.0", "miette 7.6.0", "pep440_rs", @@ -7716,6 +7720,7 @@ dependencies = [ "reqwest", "serde", "serde_json", + "tempfile", "thiserror 2.0.18", "tokio", "tracing", diff --git a/crates/pixi/Cargo.toml b/crates/pixi/Cargo.toml index b789fae9e9..226f85214d 100644 --- a/crates/pixi/Cargo.toml +++ b/crates/pixi/Cargo.toml @@ -62,6 +62,7 @@ pixi_test_utils = { workspace = true } pixi_utils = { workspace = true } pypi_mapping = { workspace = true } rattler_conda_types = { workspace = true } +rattler_digest = { workspace = true } rattler_lock = { workspace = true } rattler_package_streaming = { workspace = true } rattler_virtual_packages = { workspace = true } diff --git a/crates/pixi/tests/integration_rust/add_tests.rs b/crates/pixi/tests/integration_rust/add_tests.rs index f0dfc035d7..936ee3e85e 100644 --- a/crates/pixi/tests/integration_rust/add_tests.rs +++ b/crates/pixi/tests/integration_rust/add_tests.rs @@ -484,7 +484,7 @@ async fn add_pypi_extra_functionality() { name = "test-pypi-extras" channels = ["{channel_url}"] platforms = ["{platform}"] -conda-pypi-map = {{}} # disable mapping +conda-pypi-map = false # disable mapping [dependencies] python = "==3.12.0" diff --git a/crates/pixi/tests/integration_rust/conda_pypi_map_tests.rs b/crates/pixi/tests/integration_rust/conda_pypi_map_tests.rs new file mode 100644 index 0000000000..c3f607186a --- /dev/null +++ b/crates/pixi/tests/integration_rust/conda_pypi_map_tests.rs @@ -0,0 +1,1630 @@ +//! Tests for the conda↔PyPI name mapping: purl derivation through the +//! prefix.dev chain, project-defined `conda-pypi-map` overrides in their +//! extend/replace/disabled modes, and the `cache-ttl` mapping cache. + +use std::{ + collections::{BTreeSet, HashMap}, + path::Path, + str::FromStr, + sync::Arc, +}; + +use pypi_mapping::{ + self, ProjectDefinedChannelMapping, ProjectDefinedMapping, ProjectDefinedMappingLocation, + PurlDerivationMode, PurlDerivationSource, +}; +use rattler_conda_types::{PackageName, Platform, RepoDataRecord}; +use rattler_lock::DEFAULT_ENVIRONMENT_NAME; +use reqwest_middleware::ClientBuilder; +use tempfile::TempDir; +use url::Url; + +use crate::common::{ + LockFileExt, PixiControl, + builders::HasDependencyConfig, + client::OfflineMiddleware, + pypi_index::{Database as PyPIDatabase, PyPIPackage}, +}; +use crate::setup_tracing; +use pixi_test_utils::{MockRepoData, Package}; + +#[tokio::test(flavor = "multi_thread", worker_threads = 1)] +#[cfg_attr( + any(not(feature = "online_tests"), not(feature = "slow_integration_tests")), + ignore +)] +async fn test_purl_are_added_for_pypi() { + setup_tracing(); + + let pixi = PixiControl::new().unwrap(); + pixi.init().await.unwrap(); + // Add and update lock file with this version of python + pixi.add("boltons").await.unwrap(); + let lock_file = pixi.update_lock_file().await.unwrap(); + + // Check if boltons has a purl + let p = lock_file + .platform(&Platform::current().to_string()) + .unwrap(); + lock_file + .default_environment() + .unwrap() + .packages(p) + .unwrap() + .for_each(|dep| { + if dep.as_conda().unwrap().name() == &PackageName::from_str("boltons").unwrap() { + assert!(dep.as_conda().unwrap().record().unwrap().purls.is_none()); + } + }); + + // Add boltons from pypi + pixi.add("boltons") + .set_type(pixi_core::DependencyType::PypiDependency) + .await + .unwrap(); + + let lock_file = pixi.update_lock_file().await.unwrap(); + + // Check if boltons has a purl + let p = lock_file + .platform(&Platform::current().to_string()) + .unwrap(); + lock_file + .default_environment() + .unwrap() + .packages(p) + .unwrap() + .for_each(|dep| { + if dep.as_conda().unwrap().name() == &PackageName::from_str("boltons").unwrap() { + assert_eq!( + dep.as_conda() + .and_then(|c| c.as_binary()) + .and_then(|c| c.package_record.purls.as_ref()) + .unwrap() + .first() + .unwrap() + .qualifiers() + .get("source") + .unwrap(), + PurlDerivationSource::PrefixHashMapping.as_str() + ); + } + }); + + // Check if boltons exists only as conda dependency + assert!(lock_file.contains_match_spec( + DEFAULT_ENVIRONMENT_NAME, + Platform::current(), + "boltons" + )); + assert!(!lock_file.contains_pypi_package( + DEFAULT_ENVIRONMENT_NAME, + Platform::current(), + "boltons" + )); +} + +#[tokio::test] +#[cfg_attr(not(feature = "online_tests"), ignore)] +async fn test_purl_are_missing_for_non_conda_forge() { + setup_tracing(); + + let pixi = PixiControl::new().unwrap(); + pixi.init().await.unwrap(); + + let project = pixi.workspace().unwrap(); + let client = project.authenticated_client().unwrap(); + let foo_bar_package = Package::build("foo-bar-car", "2").finish(); + + let mut repo_data_record = RepoDataRecord { + identifier: foo_bar_package.identifier(), + package_record: foo_bar_package.package_record, + url: Url::parse("https://pypi.org/simple/boltons/").unwrap(), + channel: Some("dummy-channel".to_owned()), + }; + + let mapping_client = pypi_mapping::PurlDerivationClient::builder( + client.clone(), + project + .config() + .cache_dir_for(pixi_config::CacheKind::PypiMapping) + .unwrap(), + ) + .finish(); + mapping_client + .amend_purls( + &PurlDerivationMode::Prefix, + vec![&mut repo_data_record], + None, + ) + .await + .unwrap(); + + // Because foo-bar-car is not from conda-forge channel + // We verify that purls are missing for non-conda-forge packages + assert!( + repo_data_record + .package_record + .purls + .as_ref() + .and_then(BTreeSet::first) + .is_none() + ); +} + +#[tokio::test] +async fn test_purl_are_generated_using_custom_mapping() { + setup_tracing(); + + let pixi = PixiControl::new().unwrap(); + pixi.init().await.unwrap(); + + let project = pixi.workspace().unwrap(); + let client = project.authenticated_client().unwrap(); + let foo_bar_package = Package::build("foo-bar-car", "2").finish(); + + let mut repo_data_record = RepoDataRecord { + identifier: foo_bar_package.identifier(), + package_record: foo_bar_package.package_record, + url: Url::parse("https://pypi.org/simple/boltons/").unwrap(), + channel: Some("https://conda.anaconda.org/conda-forge/".to_owned()), + }; + + // We are using project-defined mapping + let compressed_mapping = + HashMap::from([("foo-bar-car".to_owned(), Some("my-test-name".to_owned()))]); + let source = HashMap::from([( + "https://conda.anaconda.org/conda-forge".to_owned(), + ProjectDefinedChannelMapping::replace(ProjectDefinedMappingLocation::InMemory( + compressed_mapping, + )), + )]); + + let mapping_client = pypi_mapping::PurlDerivationClient::builder( + client.clone(), + project + .config() + .cache_dir_for(pixi_config::CacheKind::PypiMapping) + .unwrap(), + ) + .finish(); + mapping_client + .amend_purls( + &PurlDerivationMode::ProjectDefined(Arc::new(ProjectDefinedMapping::new(source))), + vec![&mut repo_data_record], + None, + ) + .await + .unwrap(); + + let first_purl = repo_data_record + .package_record + .purls + .as_ref() + .and_then(BTreeSet::first) + .unwrap(); + + // We verify that `my-test-name` is used for `foo-bar-car` package + assert_eq!(first_purl.name(), "my-test-name") +} + +#[tokio::test] +#[cfg_attr(not(feature = "online_tests"), ignore)] +async fn test_compressed_mapping_catch_not_pandoc_not_a_python_package() { + setup_tracing(); + + let pixi = PixiControl::new().unwrap(); + pixi.init().await.unwrap(); + + let project = pixi.workspace().unwrap(); + let client = project.authenticated_client().unwrap(); + let foo_bar_package = Package::build("pandoc", "2").finish(); + + let mut repo_data_record = RepoDataRecord { + identifier: foo_bar_package.identifier(), + package_record: foo_bar_package.package_record, + url: Url::parse("https://haskell.org/pandoc/").unwrap(), + channel: Some("https://conda.anaconda.org/conda-forge/".to_owned()), + }; + + let packages = vec![&mut repo_data_record]; + + let mapping_client = pypi_mapping::PurlDerivationClient::builder( + client.clone(), + project + .config() + .cache_dir_for(pixi_config::CacheKind::PypiMapping) + .unwrap(), + ) + .finish(); + mapping_client + .amend_purls(&PurlDerivationMode::Prefix, packages, None) + .await + .unwrap(); + + // pandoc is not a python package + // so purls for it should be empty + assert!(repo_data_record.package_record.purls.unwrap().is_empty()) +} + +#[tokio::test] +#[cfg_attr(not(feature = "online_tests"), ignore)] +async fn test_dont_record_not_present_package_as_purl() { + setup_tracing(); + + let pixi = PixiControl::new().unwrap(); + pixi.init().await.unwrap(); + + let project = pixi.workspace().unwrap(); + let client = project.authenticated_client().unwrap(); + // We use one package that is present in our mapping: `boltons` + // and another one that is missing from conda and our mapping: + // `pixi-something-new-for-test` because `pixi-something-new-for-test` is + // from conda-forge channel we will anyway record a purl for it + // by assumption that it's a pypi package + let foo_bar_package = Package::build("pixi-something-new-for-test", "2").finish(); + // We use one package that is not present by hash + // but `boltons` name is still present in compressed mapping + // so we will record a purl for it + let boltons_package = Package::build("boltons", "99999").finish(); + + let mut repo_data_record = RepoDataRecord { + identifier: foo_bar_package.identifier(), + package_record: foo_bar_package.package_record, + url: Url::parse("https://pypi.org/simple/something-new/").unwrap(), + channel: Some("https://conda.anaconda.org/conda-forge/osx-arm64/brotli-python-1.1.0-py311ha891d26_1.conda".to_owned()), + }; + + let mut boltons_repo_data_record = RepoDataRecord { + identifier: boltons_package.identifier(), + package_record: boltons_package.package_record, + url: Url::parse("https://pypi.org/simple/boltons/").unwrap(), + channel: Some("https://conda.anaconda.org/conda-forge/".to_owned()), + }; + + let mapping_client = pypi_mapping::PurlDerivationClient::builder( + client.clone(), + project + .config() + .cache_dir_for(pixi_config::CacheKind::PypiMapping) + .unwrap(), + ) + .finish(); + mapping_client + .amend_purls( + project.pypi_name_derivation_mode().unwrap(), + vec![&mut repo_data_record, &mut boltons_repo_data_record], + None, + ) + .await + .unwrap(); + + mapping_client + .amend_purls( + project.pypi_name_derivation_mode().unwrap(), + vec![&mut repo_data_record, &mut boltons_repo_data_record], + None, + ) + .await + .unwrap(); + + let first_purl = repo_data_record + .package_record + .purls + .as_ref() + .and_then(BTreeSet::first) + .unwrap(); + + // we verify that even if this name is not present in our mapping + // we record a purl anyways. Because we make the assumption + // that it's a pypi package + assert_eq!(first_purl.name(), "pixi-something-new-for-test"); + + let boltons_purl = boltons_repo_data_record + .package_record + .purls + .as_ref() + .and_then(BTreeSet::first) + .unwrap(); + + // for boltons we have a mapping record + // so we test that we also record source=conda-forge-mapping qualifier + assert_eq!( + boltons_purl.qualifiers().get("source").unwrap(), + PurlDerivationSource::PrefixCompressedMapping.as_str() + ); +} + +fn absolute_custom_mapping_path() -> String { + dunce::simplified( + &Path::new(env!("CARGO_WORKSPACE_DIR")) + .join("tests/data/mapping_files/custom_mapping.json"), + ) + .display() + .to_string() + .replace("\\", "/") +} + +fn absolute_compressed_mapping_path() -> String { + dunce::simplified( + &Path::new(env!("CARGO_WORKSPACE_DIR")) + .join("tests/data/mapping_files/compressed_mapping.json"), + ) + .display() + .to_string() + .replace("\\", "/") +} + +#[tokio::test] +async fn test_we_record_not_present_package_as_purl_for_custom_mapping() { + setup_tracing(); + + let pixi = PixiControl::from_manifest(&format!( + r#" + [project] + name = "test-channel-change" + channels = ["conda-forge"] + platforms = ["linux-64"] + conda-pypi-map = {{ 'conda-forge' = {{ location = "{}", mode = "replace" }} }} + "#, + absolute_compressed_mapping_path() + )) + .unwrap(); + + let project = pixi.workspace().unwrap(); + + let client = project.authenticated_client().unwrap(); + + // We use one package that is present in our mapping: `boltons` + // and another one that is missing from conda and our mapping: + // `pixi-something-new-for-test`. Because the mapping uses + // `mode = "replace"` the mapping is exclusive: packages that are not in + // it must not get a purl, not even the conda-forge verbatim fallback. + let foo_bar_package = Package::build("pixi-something-new", "2").finish(); + let boltons_package = Package::build("boltons", "2").finish(); + + let repo_data_record = RepoDataRecord { + identifier: foo_bar_package.identifier(), + package_record: foo_bar_package.package_record, + url: Url::parse("https://pypi.org/simple/pixi-something-new-new/").unwrap(), + channel: Some("https://conda.anaconda.org/conda-forge/".to_owned()), + }; + + let boltons_repo_data_record = RepoDataRecord { + identifier: boltons_package.identifier(), + package_record: boltons_package.package_record, + url: Url::parse("https://pypi.org/simple/boltons/").unwrap(), + channel: Some("https://conda.anaconda.org/conda-forge/".to_owned()), + }; + + let mut packages = vec![repo_data_record, boltons_repo_data_record]; + + let mapping_client = pypi_mapping::PurlDerivationClient::builder( + client.clone(), + project + .config() + .cache_dir_for(pixi_config::CacheKind::PypiMapping) + .unwrap(), + ) + .finish(); + mapping_client + .amend_purls( + project.pypi_name_derivation_mode().unwrap(), + &mut packages, + None, + ) + .await + .unwrap(); + + let boltons_package = packages.pop().unwrap(); + + let boltons_first_purl = boltons_package + .package_record + .purls + .as_ref() + .and_then(BTreeSet::first) + .unwrap(); + + println!("{boltons_first_purl}"); + + // for boltons we have a mapping record + // so we test that we also record source=project-defined-mapping qualifier + assert_eq!(boltons_first_purl.name(), "boltons"); + assert_eq!( + boltons_first_purl.qualifiers().get("source").unwrap(), + PurlDerivationSource::ProjectDefinedMapping.as_str() + ); + + let package = packages.pop().unwrap(); + + // With a replace-mode project-defined mapping, packages not in the mapping + // should NOT get purls. This verifies that replace mode is exclusive - only + // packages explicitly mapped should be considered as pypi packages. + assert!( + package.package_record.purls.is_none() + || package.package_record.purls.as_ref().unwrap().is_empty(), + "pixi-something-new should not have purls when not in a replace-mode mapping" + ); +} + +#[tokio::test] +async fn test_custom_mapping_channel_with_suffix() { + setup_tracing(); + + let pixi = PixiControl::from_manifest(&format!( + r#" + [project] + name = "test-channel-change" + channels = ["conda-forge"] + platforms = ["linux-64"] + conda-pypi-map = {{ "https://conda.anaconda.org/conda-forge/" = "{}" }} + "#, + absolute_custom_mapping_path() + )) + .unwrap(); + + let project = pixi.workspace().unwrap(); + + let client = project.authenticated_client().unwrap(); + + let foo_bar_package = Package::build("pixi-something-new", "2").finish(); + + let repo_data_record = RepoDataRecord { + identifier: foo_bar_package.identifier(), + package_record: foo_bar_package.package_record, + url: Url::parse("https://pypi.org/simple/pixi-something-new-new/").unwrap(), + channel: Some("https://conda.anaconda.org/conda-forge".to_owned()), + }; + + let mut packages = vec![repo_data_record]; + + let mapping_client = pypi_mapping::PurlDerivationClient::builder( + client.clone(), + project + .config() + .cache_dir_for(pixi_config::CacheKind::PypiMapping) + .unwrap(), + ) + .finish(); + mapping_client + .amend_purls( + project.pypi_name_derivation_mode().unwrap(), + &mut packages, + None, + ) + .await + .unwrap(); + + let package = packages.pop().unwrap(); + + assert_eq!( + package + .package_record + .purls + .as_ref() + .and_then(BTreeSet::first) + .unwrap() + .qualifiers() + .get("source") + .unwrap(), + PurlDerivationSource::ProjectDefinedMapping.as_str() + ); +} + +#[tokio::test] +async fn test_repo_data_record_channel_with_suffix() { + setup_tracing(); + + let pixi = PixiControl::from_manifest(&format!( + r#" + [project] + name = "test-channel-change" + channels = ["conda-forge"] + platforms = ["linux-64"] + conda-pypi-map = {{ "https://conda.anaconda.org/conda-forge" = "{}" }} + "#, + absolute_custom_mapping_path(), + )) + .unwrap(); + + let project = pixi.workspace().unwrap(); + + let client = project.authenticated_client().unwrap(); + + let foo_bar_package = Package::build("pixi-something-new", "2").finish(); + + let repo_data_record = RepoDataRecord { + identifier: foo_bar_package.identifier(), + package_record: foo_bar_package.package_record, + url: Url::parse("https://pypi.org/simple/pixi-something-new-new/").unwrap(), + channel: Some("https://conda.anaconda.org/conda-forge/".to_owned()), + }; + + let mut packages = vec![repo_data_record]; + + let mapping_client = pypi_mapping::PurlDerivationClient::builder( + client.clone(), + project + .config() + .cache_dir_for(pixi_config::CacheKind::PypiMapping) + .unwrap(), + ) + .finish(); + mapping_client + .amend_purls( + project.pypi_name_derivation_mode().unwrap(), + &mut packages, + None, + ) + .await + .unwrap(); + + let package = packages.pop().unwrap(); + assert_eq!( + package + .package_record + .purls + .as_ref() + .and_then(BTreeSet::first) + .unwrap() + .qualifiers() + .get("source") + .unwrap(), + PurlDerivationSource::ProjectDefinedMapping.as_str() + ); +} + +#[tokio::test] +async fn test_path_channel() { + setup_tracing(); + + let pixi = PixiControl::from_manifest(&format!( + r#" + [project] + name = "test-channel-change" + channels = ["file:///home/user/staged-recipes/build_artifacts"] + platforms = ["linux-64"] + conda-pypi-map = {{"file:///home/user/staged-recipes/build_artifacts" = "{}" }} + "#, + absolute_custom_mapping_path() + )) + .unwrap(); + + let project = pixi.workspace().unwrap(); + + let client = project.authenticated_client().unwrap(); + + let foo_bar_package = Package::build("pixi-something-new", "2").finish(); + + let repo_data_record = RepoDataRecord { + identifier: foo_bar_package.identifier(), + package_record: foo_bar_package.package_record, + url: Url::parse("https://pypi.org/simple/pixi-something-new-new/").unwrap(), + channel: Some("file:///home/user/staged-recipes/build_artifacts".to_owned()), + }; + + let mut packages = vec![repo_data_record]; + + let mapping_client = pypi_mapping::PurlDerivationClient::builder( + client.clone(), + project + .config() + .cache_dir_for(pixi_config::CacheKind::PypiMapping) + .unwrap(), + ) + .finish(); + mapping_client + .amend_purls( + project.pypi_name_derivation_mode().unwrap(), + &mut packages, + None, + ) + .await + .unwrap(); + + let package = packages.pop().unwrap(); + + assert_eq!( + package + .package_record + .purls + .as_ref() + .and_then(BTreeSet::first) + .unwrap() + .qualifiers() + .get("source") + .unwrap(), + PurlDerivationSource::ProjectDefinedMapping.as_str() + ); +} + +#[tokio::test] +async fn test_file_url_as_mapping_location() { + setup_tracing(); + + let tmp_dir = tempfile::tempdir().unwrap(); + let mapping_file = tmp_dir.path().join("custom_mapping.json"); + + let _ = fs_err::write( + &mapping_file, + r#" + { + "pixi-something-new": "pixi-something-old" + } + "#, + ); + + let mapping_file_path_as_url = Url::from_file_path( + mapping_file, /* .canonicalize() + * .expect("should be canonicalized"), */ + ) + .unwrap(); + + let pixi = PixiControl::from_manifest( + format!( + r#" + [project] + name = "test-channel-change" + channels = ["conda-forge"] + platforms = ["linux-64"] + conda-pypi-map = {{"conda-forge" = "{}"}} + "#, + mapping_file_path_as_url.as_str() + ) + .as_str(), + ) + .unwrap(); + + let project = pixi.workspace().unwrap(); + + let client = project.authenticated_client().unwrap(); + + let foo_bar_package = Package::build("pixi-something-new", "2").finish(); + + let repo_data_record = RepoDataRecord { + identifier: foo_bar_package.identifier(), + package_record: foo_bar_package.package_record, + url: Url::parse("https://pypi.org/simple/pixi-something-new-new/").unwrap(), + channel: Some("https://conda.anaconda.org/conda-forge/".to_owned()), + }; + + let mut packages = vec![repo_data_record]; + + let mapping_client = pypi_mapping::PurlDerivationClient::builder( + client.clone(), + project + .config() + .cache_dir_for(pixi_config::CacheKind::PypiMapping) + .unwrap(), + ) + .finish(); + mapping_client + .amend_purls( + project.pypi_name_derivation_mode().unwrap(), + &mut packages, + None, + ) + .await + .unwrap(); + + let package = packages.pop().unwrap(); + + assert_eq!( + package + .package_record + .purls + .as_ref() + .and_then(BTreeSet::first) + .unwrap() + .qualifiers() + .get("source") + .unwrap(), + PurlDerivationSource::ProjectDefinedMapping.as_str() + ); +} + +/// Build a `PurlDerivationClient` whose http client refuses any network +/// request, backed by the given cache directory. +fn offline_mapping_client( + project: &pixi_core::Workspace, + cache_dir: std::path::PathBuf, +) -> pypi_mapping::PurlDerivationClient { + let client = project.authenticated_client().unwrap(); + let blocked_client = ClientBuilder::from_client(client.client().clone()) + .with(OfflineMiddleware) + .build(); + pypi_mapping::PurlDerivationClient::builder(blocked_client.into(), cache_dir).finish() +} + +fn conda_forge_record(name: &str) -> RepoDataRecord { + let package = Package::build(name, "2").finish(); + RepoDataRecord { + identifier: package.identifier(), + package_record: package.package_record, + url: Url::parse(&format!("https://pypi.org/simple/{name}/")).unwrap(), + channel: Some("https://conda.anaconda.org/conda-forge/".to_owned()), + } +} + +/// An inline mapping hit in the default (extend) mode is final and requires +/// no network access. +#[tokio::test] +async fn test_extend_mapping_inline_hit_without_network() { + setup_tracing(); + + let pixi = PixiControl::from_manifest( + r#" + [project] + name = "test-extend-inline" + channels = ["conda-forge"] + platforms = ["linux-64"] + conda-pypi-map = { conda-forge = { mapping = { pixi-something-new = "my-inline-name" } } } + "#, + ) + .unwrap(); + + let project = pixi.workspace().unwrap(); + let cache_dir = TempDir::new().unwrap(); + let mapping_client = offline_mapping_client(&project, cache_dir.path().to_path_buf()); + + let mut packages = vec![conda_forge_record("pixi-something-new")]; + mapping_client + .amend_purls( + project.pypi_name_derivation_mode().unwrap(), + &mut packages, + None, + ) + .await + .unwrap(); + + let package = packages.pop().unwrap(); + let purl = package + .package_record + .purls + .as_ref() + .and_then(BTreeSet::first) + .unwrap(); + assert_eq!(purl.name(), "my-inline-name"); + assert_eq!( + purl.qualifiers().get("source").unwrap(), + PurlDerivationSource::ProjectDefinedMapping.as_str() + ); +} + +/// An explicit `false` inline entry means "not a PyPI package": no purl is +/// derived and the conda-forge verbatim fallback does not kick in either. +#[tokio::test] +async fn test_extend_mapping_explicit_false_yields_no_purl() { + setup_tracing(); + + let pixi = PixiControl::from_manifest( + r#" + [project] + name = "test-extend-false" + channels = ["conda-forge"] + platforms = ["linux-64"] + conda-pypi-map = { conda-forge = { mapping = { pixi-something-new = false } } } + "#, + ) + .unwrap(); + + let project = pixi.workspace().unwrap(); + let cache_dir = TempDir::new().unwrap(); + let mapping_client = offline_mapping_client(&project, cache_dir.path().to_path_buf()); + + let mut packages = vec![conda_forge_record("pixi-something-new")]; + mapping_client + .amend_purls( + project.pypi_name_derivation_mode().unwrap(), + &mut packages, + None, + ) + .await + .unwrap(); + + let package = packages.pop().unwrap(); + assert!( + package + .package_record + .purls + .as_ref() + .is_none_or(|purls| purls.is_empty()), + "a package explicitly mapped to `false` must not get a purl" + ); +} + +/// ` = false` disables lookups for that channel; the offline +/// conda-forge verbatim fallback still applies. +#[tokio::test] +async fn test_channel_disabled_keeps_verbatim_fallback() { + setup_tracing(); + + let pixi = PixiControl::from_manifest( + r#" + [project] + name = "test-channel-disabled" + channels = ["conda-forge"] + platforms = ["linux-64"] + conda-pypi-map = { conda-forge = false } + "#, + ) + .unwrap(); + + let project = pixi.workspace().unwrap(); + let cache_dir = TempDir::new().unwrap(); + let mapping_client = offline_mapping_client(&project, cache_dir.path().to_path_buf()); + + let mut packages = vec![conda_forge_record("boltons")]; + mapping_client + .amend_purls( + project.pypi_name_derivation_mode().unwrap(), + &mut packages, + None, + ) + .await + .unwrap(); + + let package = packages.pop().unwrap(); + let purl = package + .package_record + .purls + .as_ref() + .and_then(BTreeSet::first) + .unwrap(); + // The verbatim fallback assumes the conda name is the pypi name and adds + // no source qualifier. + assert_eq!(purl.name(), "boltons"); + assert!(purl.qualifiers().is_empty()); +} + +/// Inline mapping keys are matched case-insensitively against the normalized +/// (lowercase) conda package names. +#[tokio::test] +async fn test_inline_mapping_keys_are_case_insensitive() { + setup_tracing(); + + let pixi = PixiControl::from_manifest( + r#" + [project] + name = "test-inline-case" + channels = ["conda-forge"] + platforms = ["linux-64"] + conda-pypi-map = { conda-forge = { mapping = { Pixi-Something-New = "mixed-case-win" } } } + "#, + ) + .unwrap(); + + let project = pixi.workspace().unwrap(); + let cache_dir = TempDir::new().unwrap(); + let mapping_client = offline_mapping_client(&project, cache_dir.path().to_path_buf()); + + let mut packages = vec![conda_forge_record("pixi-something-new")]; + mapping_client + .amend_purls( + project.pypi_name_derivation_mode().unwrap(), + &mut packages, + None, + ) + .await + .unwrap(); + + let package = packages.pop().unwrap(); + let purl = package + .package_record + .purls + .as_ref() + .and_then(BTreeSet::first) + .expect("a mixed-case inline key should match the lowercase record name"); + assert_eq!(purl.name(), "mixed-case-win"); +} + +/// The same channel spelled in two forms (by name and by URL) must be +/// rejected: the forms collapse to one channel after resolution and keeping +/// a nondeterministic winner would silently pick one of the two entries. +#[tokio::test] +async fn test_duplicate_channel_forms_are_rejected() { + setup_tracing(); + + let pixi = PixiControl::from_manifest( + r#" + [project] + name = "test-duplicate-channel" + channels = ["conda-forge"] + platforms = ["linux-64"] + + [project.conda-pypi-map] + conda-forge = false + "https://conda.anaconda.org/conda-forge" = { mapping = { a = "b" } } + "#, + ) + .unwrap(); + + let project = pixi.workspace().unwrap(); + let err = project + .pypi_name_derivation_mode() + .expect_err("duplicate channel forms should be rejected"); + assert!( + err.to_string().contains("more than once"), + "error should mention the duplicate, got: {err}" + ); +} + +/// `cache-ttl` combined with a local path location is rejected when the +/// derivation mode is built. +#[tokio::test] +async fn test_cache_ttl_on_local_path_is_rejected() { + setup_tracing(); + + let pixi = PixiControl::from_manifest( + r#" + [project] + name = "test-ttl-local-path" + channels = ["conda-forge"] + platforms = ["linux-64"] + conda-pypi-map = { conda-forge = { location = "mapping.json", cache-ttl = "24h" } } + "#, + ) + .unwrap(); + + let project = pixi.workspace().unwrap(); + let err = project + .pypi_name_derivation_mode() + .expect_err("cache-ttl on a local path should be rejected"); + assert!( + err.to_string().contains("cache-ttl"), + "error should mention cache-ttl, got: {err}" + ); +} + +/// A `file://` url in the table-form `location` is normalized to a local +/// path and works like one. +#[tokio::test] +async fn test_file_url_in_table_location() { + setup_tracing(); + + let tmp_dir = tempfile::tempdir().unwrap(); + let mapping_file = tmp_dir.path().join("mapping.json"); + fs_err::write( + &mapping_file, + r#"{ "pixi-something-new": "from-file-url" }"#, + ) + .unwrap(); + let mapping_url = Url::from_file_path(&mapping_file).unwrap(); + + let pixi = PixiControl::from_manifest(&format!( + r#" + [project] + name = "test-file-url-table" + channels = ["conda-forge"] + platforms = ["linux-64"] + conda-pypi-map = {{ conda-forge = {{ location = "{mapping_url}", mode = "extend" }} }} + "#, + )) + .unwrap(); + + let project = pixi.workspace().unwrap(); + let cache_dir = TempDir::new().unwrap(); + let mapping_client = offline_mapping_client(&project, cache_dir.path().to_path_buf()); + + let mut packages = vec![conda_forge_record("pixi-something-new")]; + mapping_client + .amend_purls( + project.pypi_name_derivation_mode().unwrap(), + &mut packages, + None, + ) + .await + .unwrap(); + + let package = packages.pop().unwrap(); + let purl = package + .package_record + .purls + .as_ref() + .and_then(BTreeSet::first) + .expect("a file:// location should resolve like a local path"); + assert_eq!(purl.name(), "from-file-url"); +} + +/// When an entry has both a `location` and inline `mapping` entries, the +/// inline entries override the ones from the location. +#[tokio::test] +async fn test_inline_mapping_overrides_location() { + setup_tracing(); + + // The custom mapping file maps `pixi-something-new` to itself; the inline + // entry must win. + let pixi = PixiControl::from_manifest(&format!( + r#" + [project] + name = "test-inline-overrides" + channels = ["conda-forge"] + platforms = ["linux-64"] + conda-pypi-map = {{ conda-forge = {{ location = "{}", mapping = {{ pixi-something-new = "inline-wins" }} }} }} + "#, + absolute_custom_mapping_path() + )) + .unwrap(); + + let project = pixi.workspace().unwrap(); + let cache_dir = TempDir::new().unwrap(); + let mapping_client = offline_mapping_client(&project, cache_dir.path().to_path_buf()); + + let mut packages = vec![conda_forge_record("pixi-something-new")]; + mapping_client + .amend_purls( + project.pypi_name_derivation_mode().unwrap(), + &mut packages, + None, + ) + .await + .unwrap(); + + let package = packages.pop().unwrap(); + let purl = package + .package_record + .purls + .as_ref() + .and_then(BTreeSet::first) + .unwrap(); + assert_eq!(purl.name(), "inline-wins"); +} + +/// In extend mode a miss in the project-defined mapping falls through to the +/// prefix.dev chain. +#[tokio::test] +#[cfg_attr(not(feature = "online_tests"), ignore)] +async fn test_extend_mapping_miss_falls_through_to_prefix() { + setup_tracing(); + + // The mapping contains an unrelated package, so `boltons` is a miss and + // must be resolved through the prefix.dev chain (the mock-built record's + // hash is unknown there, so the compressed name mapping answers). + let pixi = PixiControl::from_manifest( + r#" + [project] + name = "test-extend-miss" + channels = ["conda-forge"] + platforms = ["linux-64"] + conda-pypi-map = { conda-forge = { mapping = { some-other-package = "other" } } } + "#, + ) + .unwrap(); + + let project = pixi.workspace().unwrap(); + let client = project.authenticated_client().unwrap(); + let mapping_client = pypi_mapping::PurlDerivationClient::builder( + client.clone(), + project + .config() + .cache_dir_for(pixi_config::CacheKind::PypiMapping) + .unwrap(), + ) + .finish(); + + let mut packages = vec![conda_forge_record("boltons")]; + mapping_client + .amend_purls( + project.pypi_name_derivation_mode().unwrap(), + &mut packages, + None, + ) + .await + .unwrap(); + + let package = packages.pop().unwrap(); + let purl = package + .package_record + .purls + .as_ref() + .and_then(BTreeSet::first) + .unwrap(); + assert_eq!(purl.name(), "boltons"); + assert_eq!( + purl.qualifiers().get("source").unwrap(), + PurlDerivationSource::PrefixCompressedMapping.as_str() + ); +} + +/// A mapping for one channel must not affect records from other channels: +/// they go through the full default chain, including the conda-forge verbatim +/// fallback. +/// +/// This pins the per-record fallback behavior: previously, configuring any +/// `conda-pypi-map` suppressed the verbatim fallback globally, degrading purl +/// coverage even for channels that were not in the map. +#[tokio::test] +#[cfg_attr(not(feature = "online_tests"), ignore)] +async fn test_mapping_for_other_channel_keeps_verbatim_fallback() { + setup_tracing(); + + // A replace-mode mapping for robostack only; the record below comes from + // conda-forge and its name is unknown to both the prefix.dev hash and + // compressed mappings, so only the verbatim fallback can answer. + let pixi = PixiControl::from_manifest(&format!( + r#" + [project] + name = "test-unmapped-channel-verbatim" + channels = ["conda-forge", "robostack"] + platforms = ["linux-64"] + conda-pypi-map = {{ robostack = {{ location = "{}", mode = "replace" }} }} + "#, + absolute_custom_mapping_path() + )) + .unwrap(); + + let project = pixi.workspace().unwrap(); + let client = project.authenticated_client().unwrap(); + let mapping_client = pypi_mapping::PurlDerivationClient::builder( + client.clone(), + project + .config() + .cache_dir_for(pixi_config::CacheKind::PypiMapping) + .unwrap(), + ) + .finish(); + + let mut packages = vec![conda_forge_record("pixi-something-new")]; + mapping_client + .amend_purls( + project.pypi_name_derivation_mode().unwrap(), + &mut packages, + None, + ) + .await + .unwrap(); + + let package = packages.pop().unwrap(); + let purl = package + .package_record + .purls + .as_ref() + .and_then(BTreeSet::first) + .expect("a record from an unmapped channel should get the verbatim fallback purl"); + // The verbatim fallback assumes the conda name is the pypi name and adds + // no source qualifier. + assert_eq!(purl.name(), "pixi-something-new"); + assert!(purl.qualifiers().is_empty()); +} + +/// The on-disk path of the TTL cache for a mapping url, mirroring the layout +/// used by the project-defined mapping resolver. +fn ttl_cache_path_for(cache_dir: &Path, url: &str) -> std::path::PathBuf { + let hash = rattler_digest::compute_bytes_digest::(url.as_bytes()); + cache_dir + .join("project-defined") + .join(format!("{hash:x}.json")) +} + +fn manifest_with_ttl_mapping(url: &str, ttl: &str) -> String { + format!( + r#" + [project] + name = "test-cache-ttl" + channels = ["conda-forge"] + platforms = ["linux-64"] + conda-pypi-map = {{ conda-forge = {{ location = "{url}", cache-ttl = "{ttl}" }} }} + "# + ) +} + +/// A cached mapping younger than `cache-ttl` is used without any network +/// access. +/// +/// Note: with an offline client this test cannot distinguish the fresh-cache +/// path from the stale-fallback path (both serve the cached file). What it +/// pins is the on-disk cache layout (`project-defined/.json`) +/// and that a cached mapping is served without touching the network at all. +/// The fresh/expired age boundary itself is unit-tested in the +/// `pypi_mapping` crate (`read_ttl_cache`). +#[tokio::test] +async fn test_cache_ttl_fresh_cache_skips_network() { + setup_tracing(); + + let mapping_url = "https://example.invalid/mapping.json"; + let pixi = PixiControl::from_manifest(&manifest_with_ttl_mapping(mapping_url, "1h")).unwrap(); + + let project = pixi.workspace().unwrap(); + let cache_dir = TempDir::new().unwrap(); + + // Pre-populate the TTL cache; the url itself is unreachable and the + // client is offline, so a cache miss would fail the test. + let cache_file = ttl_cache_path_for(cache_dir.path(), mapping_url); + fs_err::create_dir_all(cache_file.parent().unwrap()).unwrap(); + fs_err::write(&cache_file, r#"{ "pixi-something-new": "from-the-cache" }"#).unwrap(); + + let mapping_client = offline_mapping_client(&project, cache_dir.path().to_path_buf()); + + let mut packages = vec![conda_forge_record("pixi-something-new")]; + mapping_client + .amend_purls( + project.pypi_name_derivation_mode().unwrap(), + &mut packages, + None, + ) + .await + .unwrap(); + + let package = packages.pop().unwrap(); + let purl = package + .package_record + .purls + .as_ref() + .and_then(BTreeSet::first) + .unwrap(); + assert_eq!(purl.name(), "from-the-cache"); +} + +/// When the cached mapping is expired and the refetch fails, the stale copy +/// is used so solves keep working offline. +#[tokio::test] +async fn test_cache_ttl_expired_falls_back_to_stale_copy() { + setup_tracing(); + + let mapping_url = "https://example.invalid/mapping.json"; + // A zero TTL means the cached copy is always considered expired. + let pixi = PixiControl::from_manifest(&manifest_with_ttl_mapping(mapping_url, "0s")).unwrap(); + + let project = pixi.workspace().unwrap(); + let cache_dir = TempDir::new().unwrap(); + + let cache_file = ttl_cache_path_for(cache_dir.path(), mapping_url); + fs_err::create_dir_all(cache_file.parent().unwrap()).unwrap(); + fs_err::write(&cache_file, r#"{ "pixi-something-new": "stale-but-used" }"#).unwrap(); + + let mapping_client = offline_mapping_client(&project, cache_dir.path().to_path_buf()); + + let mut packages = vec![conda_forge_record("pixi-something-new")]; + mapping_client + .amend_purls( + project.pypi_name_derivation_mode().unwrap(), + &mut packages, + None, + ) + .await + .unwrap(); + + let package = packages.pop().unwrap(); + let purl = package + .package_record + .purls + .as_ref() + .and_then(BTreeSet::first) + .unwrap(); + assert_eq!(purl.name(), "stale-but-used"); +} + +/// Without any cached copy, a failing fetch of a TTL-cached mapping is a hard +/// error. +#[tokio::test] +async fn test_cache_ttl_no_cache_and_fetch_failure_errors() { + setup_tracing(); + + let mapping_url = "https://example.invalid/mapping.json"; + let pixi = PixiControl::from_manifest(&manifest_with_ttl_mapping(mapping_url, "1h")).unwrap(); + + let project = pixi.workspace().unwrap(); + let cache_dir = TempDir::new().unwrap(); + let mapping_client = offline_mapping_client(&project, cache_dir.path().to_path_buf()); + + let mut packages = vec![conda_forge_record("pixi-something-new")]; + let result = mapping_client + .amend_purls( + project.pypi_name_derivation_mode().unwrap(), + &mut packages, + None, + ) + .await; + + assert!( + result.is_err(), + "an uncached TTL mapping with a failing fetch must error" + ); +} + +/// A failing prefix.dev lookup must point firewall-restricted users at the +/// manifest options that avoid the network. +#[tokio::test] +async fn test_prefix_fetch_failure_error_mentions_escape_hatches() { + setup_tracing(); + + // No `conda-pypi-map` -> the default prefix.dev chain, which needs the + // network to look up the record by hash. + let pixi = PixiControl::from_manifest( + r#" + [project] + name = "test-network-error" + channels = ["conda-forge"] + platforms = ["linux-64"] + "#, + ) + .unwrap(); + + let project = pixi.workspace().unwrap(); + let cache_dir = TempDir::new().unwrap(); + let mapping_client = offline_mapping_client(&project, cache_dir.path().to_path_buf()); + + let mut packages = vec![conda_forge_record("pixi-something-new")]; + let err = mapping_client + .amend_purls( + project.pypi_name_derivation_mode().unwrap(), + &mut packages, + None, + ) + .await + .expect_err("an offline prefix.dev lookup should fail"); + + let rendered = format!("{err:?}"); + // Strip all whitespace before matching: miette wraps the help text at + // arbitrary points, potentially splitting tokens across lines. + let collapsed: String = rendered.chars().filter(|c| !c.is_whitespace()).collect(); + assert!( + collapsed.contains("mode=\"replace\"") && collapsed.contains("conda-pypi-map=false"), + "the error should suggest the offline escape hatches, got: {rendered}" + ); +} + +/// `conda-pypi-map = {}` is a soft-deprecated alias for +/// `conda-pypi-map = false`; both disable all mapping lookups while keeping +/// the conda-forge verbatim fallback. +#[tokio::test] +async fn test_disabled_mapping() { + setup_tracing(); + + let pixi = PixiControl::from_manifest( + r#" + [project] + name = "test-channel-change" + channels = ["https://prefix.dev/conda-forge"] + platforms = ["linux-64"] + conda-pypi-map = { } + "#, + ) + .unwrap(); + + let project = pixi.workspace().unwrap(); + + let client = project.authenticated_client().unwrap(); + + let blocking_middleware = OfflineMiddleware; + + let blocked_client = ClientBuilder::from_client(client.client().clone()) + .with(blocking_middleware) + .build(); + + let boltons_package = Package::build("boltons", "2").finish(); + + let boltons_repo_data_record = RepoDataRecord { + identifier: boltons_package.identifier(), + package_record: boltons_package.package_record, + url: Url::parse("https://pypi.org/simple/boltons/").unwrap(), + channel: Some("https://conda.anaconda.org/conda-forge/".to_owned()), + }; + + let mut packages = vec![boltons_repo_data_record]; + + let mapping_client = pypi_mapping::PurlDerivationClient::builder( + blocked_client.into(), + project + .config() + .cache_dir_for(pixi_config::CacheKind::PypiMapping) + .unwrap(), + ) + .finish(); + mapping_client + .amend_purls( + project.pypi_name_derivation_mode().unwrap(), + &mut packages, + None, + ) + .await + .unwrap(); + + let boltons_package = packages.pop().unwrap(); + + let boltons_first_purl = boltons_package + .package_record + .purls + .as_ref() + .and_then(BTreeSet::first) + .unwrap(); + + // we verify that even if this name is not present in our mapping + // we record a purl anyways. Because we make the assumption + // that it's a pypi package + assert_eq!(boltons_first_purl.name(), "boltons"); + assert!(boltons_first_purl.qualifiers().is_empty()); +} + +/// `conda-pypi-map = false` is the canonical global disable: no lookups, but +/// the conda-forge verbatim fallback still applies. +#[tokio::test] +async fn test_disabled_mapping_via_false() { + setup_tracing(); + + let pixi = PixiControl::from_manifest( + r#" + [project] + name = "test-disable-false" + channels = ["https://prefix.dev/conda-forge"] + platforms = ["linux-64"] + conda-pypi-map = false + "#, + ) + .unwrap(); + + let project = pixi.workspace().unwrap(); + let cache_dir = TempDir::new().unwrap(); + let mapping_client = offline_mapping_client(&project, cache_dir.path().to_path_buf()); + + let mut packages = vec![conda_forge_record("boltons")]; + mapping_client + .amend_purls( + project.pypi_name_derivation_mode().unwrap(), + &mut packages, + None, + ) + .await + .unwrap(); + + let boltons_package = packages.pop().unwrap(); + let boltons_first_purl = boltons_package + .package_record + .purls + .as_ref() + .and_then(BTreeSet::first) + .unwrap(); + assert_eq!(boltons_first_purl.name(), "boltons"); + assert!(boltons_first_purl.qualifiers().is_empty()); +} + +#[tokio::test] +async fn test_custom_mapping_ignores_backwards_compatibility() { + setup_tracing(); + + // Create local conda channel with boltons and python packages + let mut package_database = MockRepoData::default(); + package_database.add_package( + Package::build("python", "3.12.0") + .with_subdir(Platform::Linux64) + .finish(), + ); + package_database.add_package( + Package::build("boltons", "24.0.0") + .with_subdir(Platform::Linux64) + .finish(), + ); + let channel = package_database.into_channel().await.unwrap(); + let channel_url = channel.url(); + + // Create local PyPI index with boltons package + let pypi_index = PyPIDatabase::new() + .with(PyPIPackage::new("boltons", "24.0.0")) + .into_simple_index() + .expect("failed to create local simple index"); + + // Create a project-defined mapping file that only includes specific packages + let temp_dir = TempDir::new().unwrap(); + let mapping_file = temp_dir.path().join("map.json"); + fs_err::write(&mapping_file, r#"{}"#).unwrap(); + + let pixi = PixiControl::from_manifest(&format!( + r#" + [workspace] + name = "test-custom-mapping" + channels = ["{channel_url}"] + platforms = ["linux-64"] + conda-pypi-map = {{ "{channel_url}" = {{ location = "{mapping_file}", mode = "replace" }} }} + + [dependencies] + python = "3.12.0" + boltons = "*" + + [pypi-dependencies] + boltons = "*" + + [pypi-options] + index-url = "{pypi_url}" + "#, + channel_url = channel_url, + mapping_file = mapping_file + .to_str() + .unwrap() + .to_string() + .replace("\\", "/"), + pypi_url = pypi_index.index_url(), + )) + .unwrap(); + + // Lock the project (this triggers the amend_purls logic) + pixi.lock().await.unwrap(); + + // Get the lock file + let lock = pixi.lock_file().await.unwrap(); + let p = lock.platform(&Platform::Linux64.to_string()).unwrap(); + let environment = lock.environment(DEFAULT_ENVIRONMENT_NAME).unwrap(); + let conda_packages = environment.conda_packages(p).unwrap(); + + // Collect conda packages to a vector so we can iterate over them + let conda_packages: Vec<_> = conda_packages.collect(); + + // Find boltons in conda packages + let boltons_package = conda_packages + .iter() + .find(|pkg| match pkg { + rattler_lock::CondaPackageData::Binary(binary) => { + binary.package_record.name.as_source() == "boltons" + } + _ => panic!("All packagees should be binary"), + }) + .expect("boltons should be present in conda packages"); + + // The issue: boltons should NOT have purls when using project-defined mapping + // because it's not specified in our project-defined mapping + // But due to backwards compatibility logic, it gets purls anyway + let purls = match boltons_package { + rattler_lock::CondaPackageData::Binary(binary) => &binary.package_record.purls, + _ => panic!("All packages should be binary"), + }; + + assert!( + purls.as_ref().is_none_or(|purls| purls.is_empty()), + "boltons should not have purls when not specified in custom conda-pypi-map" + ); +} + +#[tokio::test] +async fn test_missing_mapping_file_error_includes_path() { + setup_tracing(); + + let pixi = PixiControl::new().unwrap(); + pixi.init().await.unwrap(); + + let project = pixi.workspace().unwrap(); + let client = project.authenticated_client().unwrap(); + + // Use a non-existent file path for the project-defined mapping + let non_existent_path = Path::new("/this/path/does/not/exist/mapping.json"); + + let source = HashMap::from([( + "https://conda.anaconda.org/conda-forge".to_owned(), + ProjectDefinedChannelMapping::replace(ProjectDefinedMappingLocation::Path( + non_existent_path.to_path_buf(), + )), + )]); + + let foo_bar_package = Package::build("foo-bar-car", "2").finish(); + + let mut repo_data_record = RepoDataRecord { + identifier: foo_bar_package.identifier(), + package_record: foo_bar_package.package_record, + url: Url::parse("https://pypi.org/simple/boltons/").unwrap(), + channel: Some("https://conda.anaconda.org/conda-forge/".to_owned()), + }; + + let mapping_client = pypi_mapping::PurlDerivationClient::builder( + client.clone(), + project + .config() + .cache_dir_for(pixi_config::CacheKind::PypiMapping) + .unwrap(), + ) + .finish(); + let result = mapping_client + .amend_purls( + &PurlDerivationMode::ProjectDefined(Arc::new(ProjectDefinedMapping::new(source))), + vec![&mut repo_data_record], + None, + ) + .await; + + // The operation should fail because the mapping file doesn't exist + let err = result.expect_err("Expected an error when mapping file doesn't exist"); + insta::with_settings!({filters => vec![ + (r#"path: "([^"]+)""#, "[MAPPING_PATH]"), + (r#"message: "[^"]+""#, "[MAPPING_MESSAGE]"), + (r#"\bcode:\s*\d+\b"#, "[MAPPING_CODE]"), + ]}, { + insta::assert_debug_snapshot!(err); + }); +} diff --git a/crates/pixi/tests/integration_rust/main.rs b/crates/pixi/tests/integration_rust/main.rs index f6c46728bd..cfae77f832 100644 --- a/crates/pixi/tests/integration_rust/main.rs +++ b/crates/pixi/tests/integration_rust/main.rs @@ -2,6 +2,7 @@ use std::sync::Once; mod add_tests; mod common; +mod conda_pypi_map_tests; mod develop_dependencies_tests; mod global_tests; mod init_tests; diff --git a/crates/pixi/tests/integration_rust/pypi_tests.rs b/crates/pixi/tests/integration_rust/pypi_tests.rs index e90de8bb1d..455a820b82 100644 --- a/crates/pixi/tests/integration_rust/pypi_tests.rs +++ b/crates/pixi/tests/integration_rust/pypi_tests.rs @@ -65,7 +65,7 @@ test = ["recursive-optional-groups[np]", "pytest", {{include-group = "docs"}}] [tool.pixi.workspace] channels = ["{channel_url}"] platforms = ["{platform_str}"] -conda-pypi-map = {{}} +conda-pypi-map = false [tool.pixi.dependencies] python = "==3.11.0" @@ -154,7 +154,7 @@ dependencies = [ [tool.pixi.workspace] channels = ["conda-forge"] platforms = ["{platform_str}"] -conda-pypi-map = {{}} +conda-pypi-map = false [tool.pixi.dependencies] python = "==3.11.0" @@ -226,7 +226,7 @@ version = "1.0.0" [tool.pixi.workspace] channels = ["conda-forge"] platforms = ["{platform_str}"] -conda-pypi-map = {{}} +conda-pypi-map = false [tool.pixi.dependencies] python = "==3.11.0" @@ -417,7 +417,7 @@ version = "1.0.0" [tool.pixi.workspace] channels = ["conda-forge"] platforms = ["{platform_str}"] -conda-pypi-map = {{}} +conda-pypi-map = false [tool.pixi.dependencies] python = "==3.11.0" @@ -564,7 +564,7 @@ dependencies = [ [tool.pixi.workspace] channels = ["{channel_url}"] platforms = [{platform_str}] -conda-pypi-map = {{}} +conda-pypi-map = false [tool.pixi.dependencies] python = "==3.11.0" @@ -620,7 +620,7 @@ async fn test_flat_links_based_index_returns_path() { name = "pypi-flat-find-links" platforms = ["{platform}"] channels = ["{channel_url}"] - conda-pypi-map = {{}} + conda-pypi-map = false [dependencies] python = "==3.12.0" @@ -676,7 +676,7 @@ async fn test_file_based_index_returns_path() { name = "pypi-extra-index-url" platforms = ["{platform}"] channels = ["{channel_url}"] - conda-pypi-map = {{}} + conda-pypi-map = false [dependencies] python = "==3.12.0" @@ -741,7 +741,7 @@ async fn test_index_strategy() { name = "pypi-extra-index-url" platforms = ["{platform}"] channels = ["{channel_url}"] - conda-pypi-map = {{}} + conda-pypi-map = false [dependencies] python = "==3.12.0" @@ -832,7 +832,7 @@ async fn test_pinning_index() { name = "pypi-pinning-index" platforms = ["{platform}"] channels = ["{channel_url}"] - conda-pypi-map = {{}} + conda-pypi-map = false [dependencies] python = "==3.12.0" @@ -898,7 +898,7 @@ async fn test_exclude_newer_per_package_pypi_index_override() { platforms = ["{platform}"] channels = ["{channel_url}"] exclude-newer = "2015-12-02T02:07:43Z" - conda-pypi-map = {{}} + conda-pypi-map = false [dependencies] python = "==3.12.0" @@ -933,7 +933,7 @@ async fn test_exclude_newer_per_package_pypi_index_override() { platforms = ["{platform}"] channels = ["{channel_url}"] exclude-newer = "2015-12-02T02:07:43Z" - conda-pypi-map = {{}} + conda-pypi-map = false [dependencies] python = "==3.12.0" @@ -1010,7 +1010,7 @@ async fn test_exclude_newer_dependency_override_pypi_index_override() { platforms = ["{platform}"] channels = ["{channel_url}"] exclude-newer = "2015-12-02T02:07:43Z" - conda-pypi-map = {{}} + conda-pypi-map = false [dependencies] python = "==3.12.0" @@ -1048,7 +1048,7 @@ async fn test_exclude_newer_dependency_override_pypi_index_override() { platforms = ["{platform}"] channels = ["{channel_url}"] exclude-newer = "2015-12-02T02:07:43Z" - conda-pypi-map = {{}} + conda-pypi-map = false [dependencies] python = "==3.12.0" @@ -1118,7 +1118,7 @@ async fn pin_torch() { name = "pypi-pinning-index" platforms = [{platforms}] channels = ["{channel_url}"] - conda-pypi-map = {{}} + conda-pypi-map = false [dependencies] python = "==3.12.0" @@ -1170,7 +1170,7 @@ async fn test_exclude_newer_relative_pypi_rejects_unknown_timestamps() { name = "test-exclude-newer-relative-pypi-baseline" platforms = ["{platform}"] channels = ["{channel_url}"] - conda-pypi-map = {{}} + conda-pypi-map = false [dependencies] python = "==3.12.0" @@ -1201,7 +1201,7 @@ async fn test_exclude_newer_relative_pypi_rejects_unknown_timestamps() { platforms = ["{platform}"] channels = ["{channel_url}"] exclude-newer = "1d" - conda-pypi-map = {{}} + conda-pypi-map = false [dependencies] python = "==3.12.0" @@ -1257,7 +1257,7 @@ async fn test_allow_insecure_host() { name = "pypi-extra-index-url" platforms = ["{platform}"] channels = ["{channel_url}"] - conda-pypi-map = {{}} + conda-pypi-map = false [dependencies] python = "==3.12.0" @@ -1322,7 +1322,7 @@ async fn test_tls_no_verify_with_pypi_dependencies() { name = "pypi-tls-test" platforms = ["{platform}"] channels = ["{channel_url}"] - conda-pypi-map = {{}} + conda-pypi-map = false [dependencies] python = "==3.12.0" @@ -1406,7 +1406,7 @@ async fn test_tls_verify_still_fails_without_config() { name = "pypi-tls-verify-test" platforms = ["{platform}"] channels = ["{channel_url}"] - conda-pypi-map = {{}} + conda-pypi-map = false [dependencies] python = "==3.12.0" @@ -1657,7 +1657,7 @@ async fn test_cross_platform_resolve_with_no_build() { name = "pypi-extra-index-url" platforms = ["{platform}"] channels = ["{channel_url}"] - conda-pypi-map = {{}} + conda-pypi-map = false [dependencies] python = "==3.12.0" @@ -1724,7 +1724,7 @@ async fn test_pinned_help_message() { r#" [workspace] channels = ["{channel}"] - conda-pypi-map = {{}} + conda-pypi-map = false name = "local-pinned-help" platforms = ["{platform}"] version = "0.1.0" @@ -1806,7 +1806,7 @@ async fn test_uv_index_correctly_parsed() { [tool.pixi.workspace] channels = ["{channel_url}"] platforms = ["{platform}"] - conda-pypi-map = {{}} # Disable mapping + conda-pypi-map = false # Disable mapping [tool.pixi.pypi-dependencies] simple = {{ path = "." }} @@ -1865,7 +1865,7 @@ async fn test_prerelease_mode_allow() { name = "prerelease-test" platforms = ["{platform}"] channels = ["{channel_url}"] - conda-pypi-map = {{}} # Disable mapping + conda-pypi-map = false # Disable mapping [dependencies] python = "==3.12.0" @@ -1927,7 +1927,7 @@ async fn test_prerelease_mode_disallow() { name = "prerelease-test" platforms = ["{platform}"] channels = ["{channel_url}"] - conda-pypi-map = {{}} + conda-pypi-map = false [dependencies] python = "==3.12.0" @@ -2276,7 +2276,7 @@ requires-python = "{requires_python}" [tool.pixi.workspace] channels = ["{channel_url}"] platforms = ["{platform}"] -conda-pypi-map = {{}} +conda-pypi-map = false [tool.pixi.dependencies] python = "==3.10.6" @@ -2353,7 +2353,7 @@ async fn test_index_url_in_lock_file() { name = "index-url-test" platforms = ["{platform}"] channels = ["{channel_url}"] - conda-pypi-map = {{}} + conda-pypi-map = false [dependencies] python = "==3.12.0" @@ -2447,7 +2447,7 @@ async fn test_index_url_omitted_for_default_pypi() { name = "index-url-pypi-test" platforms = [{platforms}] channels = ["{channel_url}"] - conda-pypi-map = {{}} + conda-pypi-map = false [dependencies] python = "==3.12.0" diff --git a/crates/pixi/tests/integration_rust/snapshots/integration_rust__solve_group_tests__missing_mapping_file_error_includes_path.snap b/crates/pixi/tests/integration_rust/snapshots/integration_rust__conda_pypi_map_tests__missing_mapping_file_error_includes_path.snap similarity index 100% rename from crates/pixi/tests/integration_rust/snapshots/integration_rust__solve_group_tests__missing_mapping_file_error_includes_path.snap rename to crates/pixi/tests/integration_rust/snapshots/integration_rust__conda_pypi_map_tests__missing_mapping_file_error_includes_path.snap diff --git a/crates/pixi/tests/integration_rust/solve_group_tests.rs b/crates/pixi/tests/integration_rust/solve_group_tests.rs index 4c18de0dec..2c92f26507 100644 --- a/crates/pixi/tests/integration_rust/solve_group_tests.rs +++ b/crates/pixi/tests/integration_rust/solve_group_tests.rs @@ -1,26 +1,8 @@ -use std::{ - collections::{BTreeSet, HashMap}, - path::Path, - str::FromStr, - sync::Arc, -}; - -use pypi_mapping::{ - self, ProjectDefinedMapping, ProjectDefinedMappingLocation, PurlDerivationMode, - PurlDerivationSource, -}; -use rattler_conda_types::{PackageName, Platform, RepoDataRecord}; -use rattler_lock::DEFAULT_ENVIRONMENT_NAME; -use reqwest_middleware::ClientBuilder; +use rattler_conda_types::Platform; use tempfile::TempDir; use url::Url; -use crate::common::{ - LockFileExt, PixiControl, - builders::HasDependencyConfig, - client::OfflineMiddleware, - pypi_index::{Database as PyPIDatabase, PyPIPackage}, -}; +use crate::common::{LockFileExt, PixiControl}; use crate::setup_tracing; use pixi_test_utils::{MockRepoData, Package}; @@ -193,868 +175,6 @@ async fn conda_solve_group_heterogeneous_platforms() { ); } -#[tokio::test(flavor = "multi_thread", worker_threads = 1)] -#[cfg_attr( - any(not(feature = "online_tests"), not(feature = "slow_integration_tests")), - ignore -)] -async fn test_purl_are_added_for_pypi() { - setup_tracing(); - - let pixi = PixiControl::new().unwrap(); - pixi.init().await.unwrap(); - // Add and update lock file with this version of python - pixi.add("boltons").await.unwrap(); - let lock_file = pixi.update_lock_file().await.unwrap(); - - // Check if boltons has a purl - let p = lock_file - .platform(&Platform::current().to_string()) - .unwrap(); - lock_file - .default_environment() - .unwrap() - .packages(p) - .unwrap() - .for_each(|dep| { - if dep.as_conda().unwrap().name() == &PackageName::from_str("boltons").unwrap() { - assert!(dep.as_conda().unwrap().record().unwrap().purls.is_none()); - } - }); - - // Add boltons from pypi - pixi.add("boltons") - .set_type(pixi_core::DependencyType::PypiDependency) - .await - .unwrap(); - - let lock_file = pixi.update_lock_file().await.unwrap(); - - // Check if boltons has a purl - let p = lock_file - .platform(&Platform::current().to_string()) - .unwrap(); - lock_file - .default_environment() - .unwrap() - .packages(p) - .unwrap() - .for_each(|dep| { - if dep.as_conda().unwrap().name() == &PackageName::from_str("boltons").unwrap() { - assert_eq!( - dep.as_conda() - .and_then(|c| c.as_binary()) - .and_then(|c| c.package_record.purls.as_ref()) - .unwrap() - .first() - .unwrap() - .qualifiers() - .get("source") - .unwrap(), - PurlDerivationSource::PrefixHashMapping.as_str() - ); - } - }); - - // Check if boltons exists only as conda dependency - assert!(lock_file.contains_match_spec( - DEFAULT_ENVIRONMENT_NAME, - Platform::current(), - "boltons" - )); - assert!(!lock_file.contains_pypi_package( - DEFAULT_ENVIRONMENT_NAME, - Platform::current(), - "boltons" - )); -} - -#[tokio::test] -#[cfg_attr(not(feature = "online_tests"), ignore)] -async fn test_purl_are_missing_for_non_conda_forge() { - setup_tracing(); - - let pixi = PixiControl::new().unwrap(); - pixi.init().await.unwrap(); - - let project = pixi.workspace().unwrap(); - let client = project.authenticated_client().unwrap(); - let foo_bar_package = Package::build("foo-bar-car", "2").finish(); - - let mut repo_data_record = RepoDataRecord { - identifier: foo_bar_package.identifier(), - package_record: foo_bar_package.package_record, - url: Url::parse("https://pypi.org/simple/boltons/").unwrap(), - channel: Some("dummy-channel".to_owned()), - }; - - let mapping_client = pypi_mapping::PurlDerivationClient::builder( - client.clone(), - project - .config() - .cache_dir_for(pixi_config::CacheKind::PypiMapping) - .unwrap(), - ) - .finish(); - mapping_client - .amend_purls( - &PurlDerivationMode::Prefix, - vec![&mut repo_data_record], - None, - ) - .await - .unwrap(); - - // Because foo-bar-car is not from conda-forge channel - // We verify that purls are missing for non-conda-forge packages - assert!( - repo_data_record - .package_record - .purls - .as_ref() - .and_then(BTreeSet::first) - .is_none() - ); -} - -#[tokio::test] -async fn test_purl_are_generated_using_custom_mapping() { - setup_tracing(); - - let pixi = PixiControl::new().unwrap(); - pixi.init().await.unwrap(); - - let project = pixi.workspace().unwrap(); - let client = project.authenticated_client().unwrap(); - let foo_bar_package = Package::build("foo-bar-car", "2").finish(); - - let mut repo_data_record = RepoDataRecord { - identifier: foo_bar_package.identifier(), - package_record: foo_bar_package.package_record, - url: Url::parse("https://pypi.org/simple/boltons/").unwrap(), - channel: Some("https://conda.anaconda.org/conda-forge/".to_owned()), - }; - - // We are using project-defined mapping - let compressed_mapping = - HashMap::from([("foo-bar-car".to_owned(), Some("my-test-name".to_owned()))]); - let source = HashMap::from([( - "https://conda.anaconda.org/conda-forge".to_owned(), - ProjectDefinedMappingLocation::InMemory(compressed_mapping), - )]); - - let mapping_client = pypi_mapping::PurlDerivationClient::builder( - client.clone(), - project - .config() - .cache_dir_for(pixi_config::CacheKind::PypiMapping) - .unwrap(), - ) - .finish(); - mapping_client - .amend_purls( - &PurlDerivationMode::ProjectDefined(Arc::new(ProjectDefinedMapping::new(source))), - vec![&mut repo_data_record], - None, - ) - .await - .unwrap(); - - let first_purl = repo_data_record - .package_record - .purls - .as_ref() - .and_then(BTreeSet::first) - .unwrap(); - - // We verify that `my-test-name` is used for `foo-bar-car` package - assert_eq!(first_purl.name(), "my-test-name") -} - -#[tokio::test] -#[cfg_attr(not(feature = "online_tests"), ignore)] -async fn test_compressed_mapping_catch_not_pandoc_not_a_python_package() { - setup_tracing(); - - let pixi = PixiControl::new().unwrap(); - pixi.init().await.unwrap(); - - let project = pixi.workspace().unwrap(); - let client = project.authenticated_client().unwrap(); - let foo_bar_package = Package::build("pandoc", "2").finish(); - - let mut repo_data_record = RepoDataRecord { - identifier: foo_bar_package.identifier(), - package_record: foo_bar_package.package_record, - url: Url::parse("https://haskell.org/pandoc/").unwrap(), - channel: Some("https://conda.anaconda.org/conda-forge/".to_owned()), - }; - - let packages = vec![&mut repo_data_record]; - - let mapping_client = pypi_mapping::PurlDerivationClient::builder( - client.clone(), - project - .config() - .cache_dir_for(pixi_config::CacheKind::PypiMapping) - .unwrap(), - ) - .finish(); - mapping_client - .amend_purls(&PurlDerivationMode::Prefix, packages, None) - .await - .unwrap(); - - // pandoc is not a python package - // so purls for it should be empty - assert!(repo_data_record.package_record.purls.unwrap().is_empty()) -} - -#[tokio::test] -#[cfg_attr(not(feature = "online_tests"), ignore)] -async fn test_dont_record_not_present_package_as_purl() { - setup_tracing(); - - let pixi = PixiControl::new().unwrap(); - pixi.init().await.unwrap(); - - let project = pixi.workspace().unwrap(); - let client = project.authenticated_client().unwrap(); - // We use one package that is present in our mapping: `boltons` - // and another one that is missing from conda and our mapping: - // `pixi-something-new-for-test` because `pixi-something-new-for-test` is - // from conda-forge channel we will anyway record a purl for it - // by assumption that it's a pypi package - let foo_bar_package = Package::build("pixi-something-new-for-test", "2").finish(); - // We use one package that is not present by hash - // but `boltons` name is still present in compressed mapping - // so we will record a purl for it - let boltons_package = Package::build("boltons", "99999").finish(); - - let mut repo_data_record = RepoDataRecord { - identifier: foo_bar_package.identifier(), - package_record: foo_bar_package.package_record, - url: Url::parse("https://pypi.org/simple/something-new/").unwrap(), - channel: Some("https://conda.anaconda.org/conda-forge/osx-arm64/brotli-python-1.1.0-py311ha891d26_1.conda".to_owned()), - }; - - let mut boltons_repo_data_record = RepoDataRecord { - identifier: boltons_package.identifier(), - package_record: boltons_package.package_record, - url: Url::parse("https://pypi.org/simple/boltons/").unwrap(), - channel: Some("https://conda.anaconda.org/conda-forge/".to_owned()), - }; - - let mapping_client = pypi_mapping::PurlDerivationClient::builder( - client.clone(), - project - .config() - .cache_dir_for(pixi_config::CacheKind::PypiMapping) - .unwrap(), - ) - .finish(); - mapping_client - .amend_purls( - project.pypi_name_derivation_mode().unwrap(), - vec![&mut repo_data_record, &mut boltons_repo_data_record], - None, - ) - .await - .unwrap(); - - mapping_client - .amend_purls( - project.pypi_name_derivation_mode().unwrap(), - vec![&mut repo_data_record, &mut boltons_repo_data_record], - None, - ) - .await - .unwrap(); - - let first_purl = repo_data_record - .package_record - .purls - .as_ref() - .and_then(BTreeSet::first) - .unwrap(); - - // we verify that even if this name is not present in our mapping - // we record a purl anyways. Because we make the assumption - // that it's a pypi package - assert_eq!(first_purl.name(), "pixi-something-new-for-test"); - - let boltons_purl = boltons_repo_data_record - .package_record - .purls - .as_ref() - .and_then(BTreeSet::first) - .unwrap(); - - // for boltons we have a mapping record - // so we test that we also record source=conda-forge-mapping qualifier - assert_eq!( - boltons_purl.qualifiers().get("source").unwrap(), - PurlDerivationSource::PrefixCompressedMapping.as_str() - ); -} - -fn absolute_custom_mapping_path() -> String { - dunce::simplified( - &Path::new(env!("CARGO_WORKSPACE_DIR")) - .join("tests/data/mapping_files/custom_mapping.json"), - ) - .display() - .to_string() - .replace("\\", "/") -} - -fn absolute_compressed_mapping_path() -> String { - dunce::simplified( - &Path::new(env!("CARGO_WORKSPACE_DIR")) - .join("tests/data/mapping_files/compressed_mapping.json"), - ) - .display() - .to_string() - .replace("\\", "/") -} - -#[tokio::test] -async fn test_we_record_not_present_package_as_purl_for_custom_mapping() { - setup_tracing(); - - let pixi = PixiControl::from_manifest(&format!( - r#" - [project] - name = "test-channel-change" - channels = ["conda-forge"] - platforms = ["linux-64"] - conda-pypi-map = {{ 'conda-forge' = "{}" }} - "#, - absolute_compressed_mapping_path() - )) - .unwrap(); - - let project = pixi.workspace().unwrap(); - - let client = project.authenticated_client().unwrap(); - - // We use one package that is present in our mapping: `boltons` - // and another one that is missing from conda and our mapping: - // `pixi-something-new-for-test` because `pixi-something-new-for-test` is - // from conda-forge channel we will anyway record a purl for it - // by assumption that it's a pypi package - // also we are using some project-defined mapping - // so we will test for other purl qualifier comparing to - // `test_dont_record_not_present_package_as_purl` test - let foo_bar_package = Package::build("pixi-something-new", "2").finish(); - let boltons_package = Package::build("boltons", "2").finish(); - - let repo_data_record = RepoDataRecord { - identifier: foo_bar_package.identifier(), - package_record: foo_bar_package.package_record, - url: Url::parse("https://pypi.org/simple/pixi-something-new-new/").unwrap(), - channel: Some("https://conda.anaconda.org/conda-forge/".to_owned()), - }; - - let boltons_repo_data_record = RepoDataRecord { - identifier: boltons_package.identifier(), - package_record: boltons_package.package_record, - url: Url::parse("https://pypi.org/simple/boltons/").unwrap(), - channel: Some("https://conda.anaconda.org/conda-forge/".to_owned()), - }; - - let mut packages = vec![repo_data_record, boltons_repo_data_record]; - - let mapping_client = pypi_mapping::PurlDerivationClient::builder( - client.clone(), - project - .config() - .cache_dir_for(pixi_config::CacheKind::PypiMapping) - .unwrap(), - ) - .finish(); - mapping_client - .amend_purls( - project.pypi_name_derivation_mode().unwrap(), - &mut packages, - None, - ) - .await - .unwrap(); - - let boltons_package = packages.pop().unwrap(); - - let boltons_first_purl = boltons_package - .package_record - .purls - .as_ref() - .and_then(BTreeSet::first) - .unwrap(); - - println!("{boltons_first_purl}"); - - // for boltons we have a mapping record - // so we test that we also record source=project-defined-mapping qualifier - assert_eq!(boltons_first_purl.name(), "boltons"); - assert_eq!( - boltons_first_purl.qualifiers().get("source").unwrap(), - PurlDerivationSource::ProjectDefinedMapping.as_str() - ); - - let package = packages.pop().unwrap(); - - // With project-defined mapping, packages not in the mapping should NOT get purls - // This verifies that project-defined mapping is exclusive - only packages explicitly - // mapped should be considered as pypi packages - assert!( - package.package_record.purls.is_none() - || package.package_record.purls.as_ref().unwrap().is_empty(), - "pixi-something-new should not have purls when not in project-defined mapping" - ); -} - -#[tokio::test] -async fn test_custom_mapping_channel_with_suffix() { - setup_tracing(); - - let pixi = PixiControl::from_manifest(&format!( - r#" - [project] - name = "test-channel-change" - channels = ["conda-forge"] - platforms = ["linux-64"] - conda-pypi-map = {{ "https://conda.anaconda.org/conda-forge/" = "{}" }} - "#, - absolute_custom_mapping_path() - )) - .unwrap(); - - let project = pixi.workspace().unwrap(); - - let client = project.authenticated_client().unwrap(); - - let foo_bar_package = Package::build("pixi-something-new", "2").finish(); - - let repo_data_record = RepoDataRecord { - identifier: foo_bar_package.identifier(), - package_record: foo_bar_package.package_record, - url: Url::parse("https://pypi.org/simple/pixi-something-new-new/").unwrap(), - channel: Some("https://conda.anaconda.org/conda-forge".to_owned()), - }; - - let mut packages = vec![repo_data_record]; - - let mapping_client = pypi_mapping::PurlDerivationClient::builder( - client.clone(), - project - .config() - .cache_dir_for(pixi_config::CacheKind::PypiMapping) - .unwrap(), - ) - .finish(); - mapping_client - .amend_purls( - project.pypi_name_derivation_mode().unwrap(), - &mut packages, - None, - ) - .await - .unwrap(); - - let package = packages.pop().unwrap(); - - assert_eq!( - package - .package_record - .purls - .as_ref() - .and_then(BTreeSet::first) - .unwrap() - .qualifiers() - .get("source") - .unwrap(), - PurlDerivationSource::ProjectDefinedMapping.as_str() - ); -} - -#[tokio::test] -async fn test_repo_data_record_channel_with_suffix() { - setup_tracing(); - - let pixi = PixiControl::from_manifest(&format!( - r#" - [project] - name = "test-channel-change" - channels = ["conda-forge"] - platforms = ["linux-64"] - conda-pypi-map = {{ "https://conda.anaconda.org/conda-forge" = "{}" }} - "#, - absolute_custom_mapping_path(), - )) - .unwrap(); - - let project = pixi.workspace().unwrap(); - - let client = project.authenticated_client().unwrap(); - - let foo_bar_package = Package::build("pixi-something-new", "2").finish(); - - let repo_data_record = RepoDataRecord { - identifier: foo_bar_package.identifier(), - package_record: foo_bar_package.package_record, - url: Url::parse("https://pypi.org/simple/pixi-something-new-new/").unwrap(), - channel: Some("https://conda.anaconda.org/conda-forge/".to_owned()), - }; - - let mut packages = vec![repo_data_record]; - - let mapping_client = pypi_mapping::PurlDerivationClient::builder( - client.clone(), - project - .config() - .cache_dir_for(pixi_config::CacheKind::PypiMapping) - .unwrap(), - ) - .finish(); - mapping_client - .amend_purls( - project.pypi_name_derivation_mode().unwrap(), - &mut packages, - None, - ) - .await - .unwrap(); - - let package = packages.pop().unwrap(); - assert_eq!( - package - .package_record - .purls - .as_ref() - .and_then(BTreeSet::first) - .unwrap() - .qualifiers() - .get("source") - .unwrap(), - PurlDerivationSource::ProjectDefinedMapping.as_str() - ); -} - -#[tokio::test] -async fn test_path_channel() { - setup_tracing(); - - let pixi = PixiControl::from_manifest(&format!( - r#" - [project] - name = "test-channel-change" - channels = ["file:///home/user/staged-recipes/build_artifacts"] - platforms = ["linux-64"] - conda-pypi-map = {{"file:///home/user/staged-recipes/build_artifacts" = "{}" }} - "#, - absolute_custom_mapping_path() - )) - .unwrap(); - - let project = pixi.workspace().unwrap(); - - let client = project.authenticated_client().unwrap(); - - let foo_bar_package = Package::build("pixi-something-new", "2").finish(); - - let repo_data_record = RepoDataRecord { - identifier: foo_bar_package.identifier(), - package_record: foo_bar_package.package_record, - url: Url::parse("https://pypi.org/simple/pixi-something-new-new/").unwrap(), - channel: Some("file:///home/user/staged-recipes/build_artifacts".to_owned()), - }; - - let mut packages = vec![repo_data_record]; - - let mapping_client = pypi_mapping::PurlDerivationClient::builder( - client.clone(), - project - .config() - .cache_dir_for(pixi_config::CacheKind::PypiMapping) - .unwrap(), - ) - .finish(); - mapping_client - .amend_purls( - project.pypi_name_derivation_mode().unwrap(), - &mut packages, - None, - ) - .await - .unwrap(); - - let package = packages.pop().unwrap(); - - assert_eq!( - package - .package_record - .purls - .as_ref() - .and_then(BTreeSet::first) - .unwrap() - .qualifiers() - .get("source") - .unwrap(), - PurlDerivationSource::ProjectDefinedMapping.as_str() - ); -} - -#[tokio::test] -async fn test_file_url_as_mapping_location() { - setup_tracing(); - - let tmp_dir = tempfile::tempdir().unwrap(); - let mapping_file = tmp_dir.path().join("custom_mapping.json"); - - let _ = fs_err::write( - &mapping_file, - r#" - { - "pixi-something-new": "pixi-something-old" - } - "#, - ); - - let mapping_file_path_as_url = Url::from_file_path( - mapping_file, /* .canonicalize() - * .expect("should be canonicalized"), */ - ) - .unwrap(); - - let pixi = PixiControl::from_manifest( - format!( - r#" - [project] - name = "test-channel-change" - channels = ["conda-forge"] - platforms = ["linux-64"] - conda-pypi-map = {{"conda-forge" = "{}"}} - "#, - mapping_file_path_as_url.as_str() - ) - .as_str(), - ) - .unwrap(); - - let project = pixi.workspace().unwrap(); - - let client = project.authenticated_client().unwrap(); - - let foo_bar_package = Package::build("pixi-something-new", "2").finish(); - - let repo_data_record = RepoDataRecord { - identifier: foo_bar_package.identifier(), - package_record: foo_bar_package.package_record, - url: Url::parse("https://pypi.org/simple/pixi-something-new-new/").unwrap(), - channel: Some("https://conda.anaconda.org/conda-forge/".to_owned()), - }; - - let mut packages = vec![repo_data_record]; - - let mapping_client = pypi_mapping::PurlDerivationClient::builder( - client.clone(), - project - .config() - .cache_dir_for(pixi_config::CacheKind::PypiMapping) - .unwrap(), - ) - .finish(); - mapping_client - .amend_purls( - project.pypi_name_derivation_mode().unwrap(), - &mut packages, - None, - ) - .await - .unwrap(); - - let package = packages.pop().unwrap(); - - assert_eq!( - package - .package_record - .purls - .as_ref() - .and_then(BTreeSet::first) - .unwrap() - .qualifiers() - .get("source") - .unwrap(), - PurlDerivationSource::ProjectDefinedMapping.as_str() - ); -} - -#[tokio::test] -async fn test_disabled_mapping() { - setup_tracing(); - - let pixi = PixiControl::from_manifest( - r#" - [project] - name = "test-channel-change" - channels = ["https://prefix.dev/conda-forge"] - platforms = ["linux-64"] - conda-pypi-map = { } - "#, - ) - .unwrap(); - - let project = pixi.workspace().unwrap(); - - let client = project.authenticated_client().unwrap(); - - let blocking_middleware = OfflineMiddleware; - - let blocked_client = ClientBuilder::from_client(client.client().clone()) - .with(blocking_middleware) - .build(); - - let boltons_package = Package::build("boltons", "2").finish(); - - let boltons_repo_data_record = RepoDataRecord { - identifier: boltons_package.identifier(), - package_record: boltons_package.package_record, - url: Url::parse("https://pypi.org/simple/boltons/").unwrap(), - channel: Some("https://conda.anaconda.org/conda-forge/".to_owned()), - }; - - let mut packages = vec![boltons_repo_data_record]; - - let mapping_client = pypi_mapping::PurlDerivationClient::builder( - blocked_client.into(), - project - .config() - .cache_dir_for(pixi_config::CacheKind::PypiMapping) - .unwrap(), - ) - .finish(); - mapping_client - .amend_purls( - project.pypi_name_derivation_mode().unwrap(), - &mut packages, - None, - ) - .await - .unwrap(); - - let boltons_package = packages.pop().unwrap(); - - let boltons_first_purl = boltons_package - .package_record - .purls - .as_ref() - .and_then(BTreeSet::first) - .unwrap(); - - // we verify that even if this name is not present in our mapping - // we record a purl anyways. Because we make the assumption - // that it's a pypi package - assert_eq!(boltons_first_purl.name(), "boltons"); - assert!(boltons_first_purl.qualifiers().is_empty()); -} - -#[tokio::test] -async fn test_custom_mapping_ignores_backwards_compatibility() { - setup_tracing(); - - // Create local conda channel with boltons and python packages - let mut package_database = MockRepoData::default(); - package_database.add_package( - Package::build("python", "3.12.0") - .with_subdir(Platform::Linux64) - .finish(), - ); - package_database.add_package( - Package::build("boltons", "24.0.0") - .with_subdir(Platform::Linux64) - .finish(), - ); - let channel = package_database.into_channel().await.unwrap(); - let channel_url = channel.url(); - - // Create local PyPI index with boltons package - let pypi_index = PyPIDatabase::new() - .with(PyPIPackage::new("boltons", "24.0.0")) - .into_simple_index() - .expect("failed to create local simple index"); - - // Create a project-defined mapping file that only includes specific packages - let temp_dir = TempDir::new().unwrap(); - let mapping_file = temp_dir.path().join("map.json"); - fs_err::write(&mapping_file, r#"{}"#).unwrap(); - - let pixi = PixiControl::from_manifest(&format!( - r#" - [workspace] - name = "test-custom-mapping" - channels = ["{channel_url}"] - platforms = ["linux-64"] - conda-pypi-map = {{ "{channel_url}" = "{mapping_file}" }} - - [dependencies] - python = "3.12.0" - boltons = "*" - - [pypi-dependencies] - boltons = "*" - - [pypi-options] - index-url = "{pypi_url}" - "#, - channel_url = channel_url, - mapping_file = mapping_file - .to_str() - .unwrap() - .to_string() - .replace("\\", "/"), - pypi_url = pypi_index.index_url(), - )) - .unwrap(); - - // Lock the project (this triggers the amend_purls logic) - pixi.lock().await.unwrap(); - - // Get the lock file - let lock = pixi.lock_file().await.unwrap(); - let p = lock.platform(&Platform::Linux64.to_string()).unwrap(); - let environment = lock.environment(DEFAULT_ENVIRONMENT_NAME).unwrap(); - let conda_packages = environment.conda_packages(p).unwrap(); - - // Collect conda packages to a vector so we can iterate over them - let conda_packages: Vec<_> = conda_packages.collect(); - - // Find boltons in conda packages - let boltons_package = conda_packages - .iter() - .find(|pkg| match pkg { - rattler_lock::CondaPackageData::Binary(binary) => { - binary.package_record.name.as_source() == "boltons" - } - _ => panic!("All packagees should be binary"), - }) - .expect("boltons should be present in conda packages"); - - // The issue: boltons should NOT have purls when using project-defined mapping - // because it's not specified in our project-defined mapping - // But due to backwards compatibility logic, it gets purls anyway - let purls = match boltons_package { - rattler_lock::CondaPackageData::Binary(binary) => &binary.package_record.purls, - _ => panic!("All packages should be binary"), - }; - - if let Some(purls) = purls { - assert!( - purls.is_empty(), - "boltons should not have purls when not specified in custom conda-pypi-map" - ); - } -} - /// Test that environments in a solve-group can have different editability settings /// for the same path-based PyPI package. /// @@ -1089,7 +209,7 @@ async fn test_solve_group_per_environment_editability() { name = "test-editability" channels = ["{channel}"] platforms = ["{platform}"] -conda-pypi-map = {{}} # disable mapping +conda-pypi-map = false # disable mapping [dependencies] python = "*" @@ -1174,7 +294,7 @@ async fn test_transitive_uv_sources_editable_consistency() { name = "test-transitive-editable" channels = ["{channel}"] platforms = ["{platform}"] - conda-pypi-map = {{}} # disable mapping + conda-pypi-map = false # disable mapping [dependencies] python = "*" @@ -1241,57 +361,3 @@ core = { path = "../core", editable = true } "default environment should contain middle" ); } - -#[tokio::test] -async fn test_missing_mapping_file_error_includes_path() { - setup_tracing(); - - let pixi = PixiControl::new().unwrap(); - pixi.init().await.unwrap(); - - let project = pixi.workspace().unwrap(); - let client = project.authenticated_client().unwrap(); - - // Use a non-existent file path for the project-defined mapping - let non_existent_path = Path::new("/this/path/does/not/exist/mapping.json"); - - let source = HashMap::from([( - "https://conda.anaconda.org/conda-forge".to_owned(), - ProjectDefinedMappingLocation::Path(non_existent_path.to_path_buf()), - )]); - - let foo_bar_package = Package::build("foo-bar-car", "2").finish(); - - let mut repo_data_record = RepoDataRecord { - identifier: foo_bar_package.identifier(), - package_record: foo_bar_package.package_record, - url: Url::parse("https://pypi.org/simple/boltons/").unwrap(), - channel: Some("https://conda.anaconda.org/conda-forge/".to_owned()), - }; - - let mapping_client = pypi_mapping::PurlDerivationClient::builder( - client.clone(), - project - .config() - .cache_dir_for(pixi_config::CacheKind::PypiMapping) - .unwrap(), - ) - .finish(); - let result = mapping_client - .amend_purls( - &PurlDerivationMode::ProjectDefined(Arc::new(ProjectDefinedMapping::new(source))), - vec![&mut repo_data_record], - None, - ) - .await; - - // The operation should fail because the mapping file doesn't exist - let err = result.expect_err("Expected an error when mapping file doesn't exist"); - insta::with_settings!({filters => vec![ - (r#"path: "([^"]+)""#, "[MAPPING_PATH]"), - (r#"message: "[^"]+""#, "[MAPPING_MESSAGE]"), - (r#"\bcode:\s*\d+\b"#, "[MAPPING_CODE]"), - ]}, { - insta::assert_debug_snapshot!(err); - }); -} diff --git a/crates/pixi/tests/integration_rust/upgrade_tests.rs b/crates/pixi/tests/integration_rust/upgrade_tests.rs index 4e7961e13f..d1c53287a7 100644 --- a/crates/pixi/tests/integration_rust/upgrade_tests.rs +++ b/crates/pixi/tests/integration_rust/upgrade_tests.rs @@ -39,7 +39,7 @@ async fn pypi_dependency_index_preserved_on_upgrade() { [workspace] channels = ["{channel_url}"] platforms = ["{platform}"] - conda-pypi-map = {{}} + conda-pypi-map = false [pypi-dependencies] click = {{ version = "==8.2.0", index = "{pypi_index_url}" }} @@ -93,7 +93,7 @@ async fn pypi_dependency_index_preserved_on_upgrade() { [workspace] channels = ["[CHANNEL_URL]"] platforms = ["[PLATFORM]"] - conda-pypi-map = {} + conda-pypi-map = false [pypi-dependencies] click = { version = ">=8.3.1, <9", index = "[PYPI_INDEX_URL]" } @@ -326,7 +326,7 @@ async fn pypi_dependency_upgrade_uses_custom_index() { name = "pypi-upgrade-custom-index" platforms = ["{platform}"] channels = ["{channel}"] - conda-pypi-map = {{}} + conda-pypi-map = false [dependencies] python = "==3.12.0" diff --git a/crates/pixi_build_python/src/config.rs b/crates/pixi_build_python/src/config.rs index 3da5565189..9f4bbff2df 100644 --- a/crates/pixi_build_python/src/config.rs +++ b/crates/pixi_build_python/src/config.rs @@ -3,6 +3,64 @@ use pixi_build_backend::generated_recipe::BackendConfig; use serde::{Deserialize, Serialize}; use std::path::{Path, PathBuf}; +/// One value of the `pypi-conda-map` option: a conda package name, or `false` +/// to silently drop the dependency from the generated recipe. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum PypiCondaMapEntry { + /// Map the PyPI package to this conda package name. + CondaName(String), + /// Drop the dependency from the generated recipe. + Skip, +} + +impl Serialize for PypiCondaMapEntry { + fn serialize(&self, serializer: S) -> Result { + match self { + PypiCondaMapEntry::CondaName(name) => serializer.serialize_str(name), + PypiCondaMapEntry::Skip => serializer.serialize_bool(false), + } + } +} + +impl<'de> Deserialize<'de> for PypiCondaMapEntry { + fn deserialize>(deserializer: D) -> Result { + struct Visitor; + + impl<'de> serde::de::Visitor<'de> for Visitor { + type Value = PypiCondaMapEntry; + + fn expecting(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + f.write_str("a conda package name or `false`") + } + + fn visit_str(self, v: &str) -> Result { + Ok(PypiCondaMapEntry::CondaName(v.to_string())) + } + + fn visit_bool(self, v: bool) -> Result { + if v { + Err(E::custom( + "`true` is not supported; use a conda package name to map the \ + package, or `false` to drop the dependency", + )) + } else { + Ok(PypiCondaMapEntry::Skip) + } + } + + fn visit_unit(self) -> Result { + Ok(PypiCondaMapEntry::Skip) + } + + fn visit_none(self) -> Result { + Ok(PypiCondaMapEntry::Skip) + } + } + + deserializer.deserialize_any(Visitor) + } +} + /// Represents skip-pyc-compilation config: either `true` (skip all) or a list /// of glob patterns. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] @@ -63,6 +121,13 @@ pub struct PythonBackendConfig { /// Defaults to `true` (mapping disabled). #[serde(default)] pub ignore_pypi_mapping: Option, + /// User-defined overrides for the PyPI-to-conda name mapping, keyed by + /// PyPI package name. A string value maps the package to that conda name; + /// `false` drops the dependency from the generated recipe. Entries are + /// consulted before the remote mapping service. Only used when + /// `ignore-pypi-mapping = false`. + #[serde(default)] + pub pypi_conda_map: Option>, /// Whether the package uses the Python Stable ABI (abi3). /// When true, adds `python_abi` to host requirements. /// Only meaningful for packages with compiled extensions (non-noarch). @@ -137,6 +202,15 @@ impl BackendConfig for PythonBackendConfig { ignore_pypi_mapping: target_config .ignore_pypi_mapping .or(self.ignore_pypi_mapping), + pypi_conda_map: match (&self.pypi_conda_map, &target_config.pypi_conda_map) { + (None, None) => None, + (base, target) => { + // Per-key merge: target entries override or add to the base map. + let mut merged = base.clone().unwrap_or_default(); + merged.extend(target.clone().unwrap_or_default()); + Some(merged) + } + }, abi3: target_config.abi3.or(self.abi3), skip_pyc_compilation: if target_config.skip_pyc_compilation.is_none() { self.skip_pyc_compilation.clone() @@ -149,7 +223,7 @@ impl BackendConfig for PythonBackendConfig { #[cfg(test)] mod tests { - use super::{PythonBackendConfig, SkipPycCompilation}; + use super::{PypiCondaMapEntry, PythonBackendConfig, SkipPycCompilation}; use pixi_build_backend::generated_recipe::BackendConfig; use serde_json::json; use std::path::PathBuf; @@ -160,6 +234,71 @@ mod tests { serde_json::from_value::(json_data).unwrap(); } + #[test] + fn test_deserialize_pypi_conda_map() { + let json_data = json!({ + "pypi-conda-map": { + "torch": "pytorch", + "my-internal-pkg": false, + } + }); + let config = serde_json::from_value::(json_data).unwrap(); + let map = config.pypi_conda_map.unwrap(); + assert_eq!( + map.get("torch"), + Some(&PypiCondaMapEntry::CondaName("pytorch".to_string())) + ); + assert_eq!(map.get("my-internal-pkg"), Some(&PypiCondaMapEntry::Skip)); + } + + #[test] + fn test_deserialize_pypi_conda_map_rejects_true() { + let json_data = json!({ + "pypi-conda-map": { + "torch": true, + } + }); + let err = serde_json::from_value::(json_data).unwrap_err(); + assert!(err.to_string().contains("`true` is not supported")); + } + + #[test] + fn test_merge_pypi_conda_map_per_key() { + let base = PythonBackendConfig { + pypi_conda_map: Some(indexmap::indexmap! { + "torch".to_string() => PypiCondaMapEntry::CondaName("pytorch".to_string()), + "shared".to_string() => PypiCondaMapEntry::CondaName("base-name".to_string()), + }), + ..Default::default() + }; + let target = PythonBackendConfig { + pypi_conda_map: Some(indexmap::indexmap! { + "shared".to_string() => PypiCondaMapEntry::Skip, + "extra".to_string() => PypiCondaMapEntry::CondaName("extra-conda".to_string()), + }), + ..Default::default() + }; + + let merged = base.merge_with_target_config(&target).unwrap(); + let map = merged.pypi_conda_map.unwrap(); + // Base-only keys survive, target keys override or extend. + assert_eq!( + map.get("torch"), + Some(&PypiCondaMapEntry::CondaName("pytorch".to_string())) + ); + assert_eq!(map.get("shared"), Some(&PypiCondaMapEntry::Skip)); + assert_eq!( + map.get("extra"), + Some(&PypiCondaMapEntry::CondaName("extra-conda".to_string())) + ); + + // None + None stays None. + let merged = PythonBackendConfig::default() + .merge_with_target_config(&PythonBackendConfig::default()) + .unwrap(); + assert_eq!(merged.pypi_conda_map, None); + } + #[test] fn test_merge_with_target_config() { let mut base_env = indexmap::IndexMap::new(); @@ -175,6 +314,7 @@ mod tests { compilers: Some(vec!["c".to_string()]), ignore_pyproject_manifest: Some(true), ignore_pypi_mapping: Some(true), + pypi_conda_map: None, abi3: Some(true), skip_pyc_compilation: SkipPycCompilation::All(true), }; @@ -192,6 +332,7 @@ mod tests { compilers: Some(vec!["cxx".to_string(), "rust".to_string()]), ignore_pyproject_manifest: Some(false), ignore_pypi_mapping: Some(false), + pypi_conda_map: None, abi3: Some(false), skip_pyc_compilation: SkipPycCompilation::Globs(vec!["tests/**".to_string()]), }; @@ -252,6 +393,7 @@ mod tests { compilers: None, ignore_pyproject_manifest: Some(true), ignore_pypi_mapping: Some(true), + pypi_conda_map: None, abi3: None, skip_pyc_compilation: SkipPycCompilation::All(true), }; diff --git a/crates/pixi_build_python/src/main.rs b/crates/pixi_build_python/src/main.rs index c5f060c150..155296943f 100644 --- a/crates/pixi_build_python/src/main.rs +++ b/crates/pixi_build_python/src/main.rs @@ -276,11 +276,20 @@ impl GenerateRecipe for PythonGenerator { host_platform }; + // Warn when the user supplied mapping overrides that cannot take effect. + if config.pypi_conda_map.is_some() && config.ignore_pypi_mapping() { + tracing::warn!( + "`pypi-conda-map` is set but the PyPI-to-conda mapping is disabled; set \ + `ignore-pypi-mapping = false` to use it" + ); + } + // Map PyPI dependencies from pyproject.toml to conda dependencies if !config.ignore_pypi_mapping() { if let Some(pypi_deps) = pyproject_metadata_provider.project_dependencies()? { let mapped_deps = map_requirements_with_channels( pypi_deps, + config.pypi_conda_map.as_ref(), &channels, &cache_dir, "project", @@ -306,6 +315,7 @@ impl GenerateRecipe for PythonGenerator { if let Some(build_system_deps) = pyproject_metadata_provider.build_system_requires()? { let mapped_deps = map_requirements_with_channels( build_system_deps, + config.pypi_conda_map.as_ref(), &channels, &cache_dir, "build-system", diff --git a/crates/pixi_build_python/src/pypi_mapping.rs b/crates/pixi_build_python/src/pypi_mapping.rs index 912b9d5f6c..5a0f6289d0 100644 --- a/crates/pixi_build_python/src/pypi_mapping.rs +++ b/crates/pixi_build_python/src/pypi_mapping.rs @@ -21,6 +21,8 @@ use rattler_conda_types::{ use serde::{Deserialize, Serialize}; use thiserror::Error; +use crate::config::PypiCondaMapEntry; + /// Base URL for the PyPI to conda mapping API (without channel suffix). const MAPPING_BASE_URL: &str = "https://conda-mapping.prefix.dev/pypi-to-conda-v1"; @@ -452,26 +454,9 @@ impl PyPiToCondaMapper { let conda_name = PackageName::from_str(&conda_name_str) .map_err(|e| MappingError::InvalidPackageName(conda_name_str.clone(), e))?; - // Convert version specifiers - let version_spec = if let Some(ref version_or_url) = req.version_or_url { - match Self::convert_version_specifiers(version_or_url) { - Ok(spec) => spec, - Err(e) => { - tracing::warn!( - "Failed to convert version specifier for '{}': {}, using unconstrained version", - req.name, - e - ); - None - } - } - } else { - None - }; - mapped.push(MappedCondaDependency { name: conda_name, - version_spec, + version_spec: convert_requirement_version(req), }); } @@ -479,6 +464,26 @@ impl PyPiToCondaMapper { } } +/// Convert the version specifiers of a requirement to a conda version spec, +/// warning and falling back to an unconstrained version when the conversion +/// fails. +fn convert_requirement_version( + req: &pep508_rs::Requirement, +) -> Option { + let version_or_url = req.version_or_url.as_ref()?; + match PyPiToCondaMapper::convert_version_specifiers(version_or_url) { + Ok(spec) => spec, + Err(e) => { + tracing::warn!( + "Failed to convert version specifier for '{}': {}, using unconstrained version", + req.name, + e + ); + None + } + } +} + /// Filter mapped PyPI dependencies, returning only those not already specified /// in Pixi's run dependencies. /// @@ -507,32 +512,125 @@ pub fn extract_channel_name(channel: &ChannelUrl) -> Option<&str> { channel.as_str().trim_end_matches('/').rsplit('/').next() } -/// Map PyPI requirements to conda dependencies using the first channel that provides a valid mapping. +/// Resolve requirements against the user-defined `pypi-conda-map` overrides. /// -/// Tries each channel in order and returns the mapped dependencies from the first -/// channel that successfully maps at least one dependency. Returns an empty Vec -/// if no channel provides a mapping. +/// Returns the dependencies mapped (or dropped) by the user map together with +/// the requirements that are not covered by it and still need the remote +/// mapping service. Environment markers are evaluated for user-mapped +/// requirements exactly like for service-mapped ones. +fn apply_user_map( + requirements: &[pep508_rs::Requirement], + user_map: Option<&IndexMap>, + platform: Platform, +) -> ( + Vec, + Vec>, +) { + // Normalize the user-map keys so that e.g. `My_Pkg` matches `my-pkg`. + let mut normalized: IndexMap = IndexMap::new(); + for (name, entry) in user_map.into_iter().flatten() { + match pep508_rs::PackageName::from_str(name) { + Ok(normalized_name) => { + if normalized.insert(normalized_name, entry).is_some() { + tracing::warn!( + "multiple `pypi-conda-map` entries normalize to the same package name \ + as '{name}'; the last one wins" + ); + } + } + Err(err) => { + tracing::warn!( + "ignoring invalid PyPI package name '{name}' in `pypi-conda-map`: {err}" + ); + } + } + } + + let mut user_mapped = Vec::new(); + let mut remaining = Vec::new(); + for req in requirements { + let Some(entry) = normalized.get(&req.name) else { + remaining.push(req.clone()); + continue; + }; + + // Markers apply to user-mapped dependencies too. + if PyPiToCondaMapper::should_skip_requirement(req, platform) { + tracing::debug!( + "Skipping user-mapped dependency '{}' due to environment marker evaluation: {:?}", + req.name, + req.marker + ); + continue; + } + + match entry { + PypiCondaMapEntry::Skip => { + tracing::debug!( + "Dropping dependency '{}' because it is mapped to `false` in `pypi-conda-map`", + req.name + ); + } + PypiCondaMapEntry::CondaName(conda_name) => match PackageName::from_str(conda_name) { + Ok(name) => { + user_mapped.push(MappedCondaDependency { + name, + version_spec: convert_requirement_version(req), + }); + } + Err(err) => { + // An invalid override must not silently drop the + // dependency: warn and fall through to the mapping + // service. + tracing::warn!( + "ignoring `pypi-conda-map` entry for '{}': invalid conda package name '{conda_name}': {err}", + req.name + ); + remaining.push(req.clone()); + } + }, + } + } + (user_mapped, remaining) +} + +/// Map PyPI requirements to conda dependencies, consulting the user-defined +/// `pypi-conda-map` overrides first and the first channel that provides a +/// valid mapping for the rest. +/// +/// Tries each channel in order and returns the mapped dependencies from the +/// first channel that successfully maps at least one dependency. Requirements +/// resolved by the user map never reach the network and do not count towards +/// the channel selection. /// /// The `context` parameter is used for logging (e.g., "project dependencies" or /// "build-system requirements"). pub async fn map_requirements_with_channels( requirements: &[pep508_rs::Requirement], + user_map: Option<&IndexMap>, channels: &[ChannelUrl], cache_dir: &Option, context: &str, platform: Platform, ) -> Vec { + let (mut user_mapped, remaining) = apply_user_map(requirements, user_map, platform); + + if remaining.is_empty() { + return user_mapped; + } + for channel in channels { if let Some(channel_name) = extract_channel_name(channel) { let mapper = PyPiToCondaMapper::new(cache_dir.clone(), channel_name.to_string()); - match mapper.map_requirements(requirements, platform).await { + match mapper.map_requirements(&remaining, platform).await { Ok(deps) if !deps.is_empty() => { tracing::debug!( "Using PyPI-to-conda mapping for {} from channel '{}'", context, channel_name ); - return deps; + user_mapped.extend(deps); + return user_mapped; } Ok(_) => { tracing::warn!( @@ -552,7 +650,7 @@ pub async fn map_requirements_with_channels( } } } - Vec::new() + user_mapped } /// Build tools that require specific compilers. @@ -706,6 +804,170 @@ mod tests { } } + fn requirement(s: &str) -> pep508_rs::Requirement { + pep508_rs::Requirement::from_str(s).unwrap() + } + + #[test] + fn test_apply_user_map_override_and_skip() { + let user_map = IndexMap::from([ + ( + "torch".to_string(), + PypiCondaMapEntry::CondaName("pytorch".to_string()), + ), + ("my-internal-pkg".to_string(), PypiCondaMapEntry::Skip), + ]); + + let requirements = vec![ + requirement("torch>=2.0"), + requirement("my-internal-pkg"), + requirement("numpy"), + ]; + + let (mapped, remaining) = apply_user_map(&requirements, Some(&user_map), Platform::Linux64); + + // `torch` is mapped with its version spec, `my-internal-pkg` is + // silently dropped, `numpy` is left for the mapping service. + assert_eq!(mapped.len(), 1); + assert_eq!(mapped[0].name.as_normalized(), "pytorch"); + assert_eq!( + mapped[0].version_spec.as_ref().unwrap().to_string(), + ">=2.0" + ); + assert_eq!(remaining.len(), 1); + assert_eq!(remaining[0].name.as_ref(), "numpy"); + } + + #[test] + fn test_apply_user_map_normalizes_names() { + // `My_Pkg` in the config must match the normalized requirement `my-pkg`. + let user_map = IndexMap::from([( + "My_Pkg".to_string(), + PypiCondaMapEntry::CondaName("my-conda-pkg".to_string()), + )]); + + let requirements = vec![requirement("my-pkg")]; + let (mapped, remaining) = apply_user_map(&requirements, Some(&user_map), Platform::Linux64); + + assert_eq!(mapped.len(), 1); + assert_eq!(mapped[0].name.as_normalized(), "my-conda-pkg"); + assert!(remaining.is_empty()); + } + + #[test] + fn test_apply_user_map_respects_markers() { + let user_map = IndexMap::from([( + "torch".to_string(), + PypiCondaMapEntry::CondaName("pytorch".to_string()), + )]); + + // The marker does not apply to linux-64, so the user-mapped + // dependency is dropped entirely. + let requirements = vec![requirement("torch; sys_platform == 'win32'")]; + let (mapped, remaining) = apply_user_map(&requirements, Some(&user_map), Platform::Linux64); + assert!(mapped.is_empty()); + assert!(remaining.is_empty()); + + // On NoArch any marker-bearing dependency is dropped. + let (mapped, remaining) = apply_user_map(&requirements, Some(&user_map), Platform::NoArch); + assert!(mapped.is_empty()); + assert!(remaining.is_empty()); + } + + #[test] + fn test_apply_user_map_invalid_conda_name_falls_through() { + let user_map = IndexMap::from([( + "torch".to_string(), + PypiCondaMapEntry::CondaName("not a valid name!".to_string()), + )]); + + let requirements = vec![requirement("torch")]; + let (mapped, remaining) = apply_user_map(&requirements, Some(&user_map), Platform::Linux64); + // The invalid override is warned about and the dependency falls + // through to the mapping service instead of being silently dropped. + assert!(mapped.is_empty()); + assert_eq!(remaining.len(), 1); + assert_eq!(remaining[0].name.as_ref(), "torch"); + } + + #[test] + fn test_apply_user_map_skip_entry_respects_markers() { + let user_map = IndexMap::from([("torch".to_string(), PypiCondaMapEntry::Skip)]); + + // A marker-gated requirement that does not apply to the platform is + // dropped before the Skip entry matters: neither mapped nor remaining. + let requirements = vec![requirement("torch; sys_platform == 'win32'")]; + let (mapped, remaining) = apply_user_map(&requirements, Some(&user_map), Platform::Linux64); + assert!(mapped.is_empty()); + assert!(remaining.is_empty()); + + // On a matching platform the Skip entry drops it silently. + let (mapped, remaining) = apply_user_map(&requirements, Some(&user_map), Platform::Win64); + assert!(mapped.is_empty()); + assert!(remaining.is_empty()); + } + + #[test] + fn test_apply_user_map_colliding_keys_last_wins() { + // `My_Pkg` and `my-pkg` normalize to the same name; the last entry + // wins deterministically (and a warning is logged). + let user_map = IndexMap::from([ + ( + "My_Pkg".to_string(), + PypiCondaMapEntry::CondaName("first".to_string()), + ), + ( + "my-pkg".to_string(), + PypiCondaMapEntry::CondaName("second".to_string()), + ), + ]); + + let requirements = vec![requirement("my-pkg")]; + let (mapped, remaining) = apply_user_map(&requirements, Some(&user_map), Platform::Linux64); + assert!(remaining.is_empty()); + assert_eq!(mapped.len(), 1); + assert_eq!(mapped[0].name.as_normalized(), "second"); + } + + #[test] + fn test_apply_user_map_converts_pep440_operators() { + let user_map = IndexMap::from([( + "torch".to_string(), + PypiCondaMapEntry::CondaName("pytorch".to_string()), + )]); + + let requirements = vec![requirement("torch===2.0.0")]; + let (mapped, _) = apply_user_map(&requirements, Some(&user_map), Platform::Linux64); + assert_eq!( + mapped[0].version_spec.as_ref().unwrap().to_string(), + "==2.0.0" + ); + } + + #[tokio::test] + async fn test_map_requirements_with_channels_all_user_mapped() { + let user_map = IndexMap::from([( + "torch".to_string(), + PypiCondaMapEntry::CondaName("pytorch".to_string()), + )]); + + // All requirements are covered by the user map: no channel lookup + // happens (there are no channels to consult either). + let requirements = vec![requirement("torch>=2.0")]; + let mapped = map_requirements_with_channels( + &requirements, + Some(&user_map), + &[], + &None, + "test", + Platform::Linux64, + ) + .await; + + assert_eq!(mapped.len(), 1); + assert_eq!(mapped[0].name.as_normalized(), "pytorch"); + } + #[test] fn test_filter_mapped_pypi_deps_without_pixi_deps() { // When no Pixi deps are specified, all mapped deps should pass through diff --git a/crates/pixi_core/src/workspace/conda_pypi_map.rs b/crates/pixi_core/src/workspace/conda_pypi_map.rs new file mode 100644 index 0000000000..380356be88 --- /dev/null +++ b/crates/pixi_core/src/workspace/conda_pypi_map.rs @@ -0,0 +1,317 @@ +//! Converts the manifest `[workspace.conda-pypi-map]` configuration into the +//! per-channel mapping configuration used by the purl derivation client. + +use std::{ + collections::{HashMap, HashSet}, + path::PathBuf, + str::FromStr, +}; + +use itertools::Itertools; +use miette::{Context, IntoDiagnostic}; +use pixi_manifest::{ + CondaPypiMap, CondaPypiMapEntry, CondaPypiMapMode, CondaPypiMapSpec, MappingLocationSpec, + WorkspaceManifest, +}; +use pypi_mapping::{ + ChannelName, MappingMode, ProjectDefinedChannelMapping, ProjectDefinedMapping, + ProjectDefinedMappingLocation, PurlDerivationMode, +}; +use rattler_conda_types::{Channel, ChannelConfig}; +use rattler_lock::UrlOrPath; + +/// Determine the [`PurlDerivationMode`] for a workspace from its +/// `conda-pypi-map` configuration. +pub(crate) fn build_pypi_name_derivation_mode( + manifest: &WorkspaceManifest, + channel_config: &ChannelConfig, +) -> miette::Result { + let map = match &manifest.workspace.conda_pypi_map { + None => return Ok(PurlDerivationMode::Prefix), + Some(CondaPypiMap::Disabled) => return Ok(PurlDerivationMode::Disabled), + Some(CondaPypiMap::Map(map)) => map, + }; + + // An empty map is a soft-deprecated alias for `conda-pypi-map = false`; + // the deprecation warning is emitted when the manifest is parsed. + if map.is_empty() { + return Ok(PurlDerivationMode::Disabled); + } + + // The manifest map can spell the same channel in different forms (by + // name and by URL) that only collapse once resolved to a `Channel`. + // Collecting blindly would keep a nondeterministic winner, so reject + // duplicates instead. + let mut channel_to_entry_map: HashMap = + HashMap::with_capacity(map.len()); + for (key, value) in map { + let channel = key.clone().into_channel(channel_config).into_diagnostic()?; + let channel_name = channel + .name + .clone() + .unwrap_or_else(|| channel.base_url.to_string()); + if channel_to_entry_map.insert(channel, value).is_some() { + miette::bail!( + "the channel {} is configured more than once in `conda-pypi-map` \ + (e.g. both by name and by URL); keep a single entry per channel", + console::style(channel_name).bold(), + ); + } + } + + validate_mapped_channels_are_used(manifest, channel_config, channel_to_entry_map.keys())?; + + let mapping = channel_to_entry_map + .iter() + .map(|(channel, entry)| { + Ok(( + channel.canonical_name().trim_end_matches('/').into(), + convert_entry(entry, channel_config)?, + )) + }) + .collect::>>()?; + + Ok(PurlDerivationMode::ProjectDefined( + ProjectDefinedMapping::new(mapping).into(), + )) +} + +/// Every channel in `conda-pypi-map` must appear in the workspace or feature +/// channels; an entry for an unused channel is almost certainly a typo. +fn validate_mapped_channels_are_used<'a>( + manifest: &WorkspaceManifest, + channel_config: &ChannelConfig, + mapped_channels: impl Iterator, +) -> miette::Result<()> { + let project_channels: HashSet<_> = manifest + .workspace + .channels + .iter() + .map(|pc| pc.channel.clone().into_channel(channel_config)) + .try_collect() + .into_diagnostic()?; + + let feature_channels: HashSet<_> = manifest + .features + .values() + .flat_map(|feature| feature.channels.iter()) + .flatten() + .map(|pc| pc.channel.clone().into_channel(channel_config)) + .try_collect() + .into_diagnostic()?; + + let project_and_feature_channels: HashSet<_> = + project_channels.union(&feature_channels).collect(); + + for channel in mapped_channels { + if !project_and_feature_channels.contains(channel) { + let channels = project_and_feature_channels + .iter() + .map(|c| c.name.clone().unwrap_or_else(|| c.base_url.to_string())) + .sorted() + .collect::>() + .join(", "); + miette::bail!( + "conda-pypi-map is defined: the {} is missing from the channels array, which currently are: {}", + console::style( + channel + .name + .clone() + .unwrap_or_else(|| channel.base_url.to_string()) + ) + .bold(), + channels + ); + } + } + Ok(()) +} + +/// Convert a manifest entry to the per-channel mapping configuration used by +/// the purl derivation client. +fn convert_entry( + entry: &CondaPypiMapEntry, + channel_config: &ChannelConfig, +) -> miette::Result { + match entry { + CondaPypiMapEntry::Disabled => Ok(ProjectDefinedChannelMapping::disabled()), + CondaPypiMapEntry::Map(CondaPypiMapSpec { + location, + mapping, + mode, + }) => { + let mut sources = Vec::new(); + if let Some(location) = location { + sources.push(parse_mapping_location(location, channel_config)?); + } + // Inline entries come last so they override entries from the + // location. Keys are lowercased to match the normalized conda + // package names used for lookups. + if let Some(inline) = mapping { + sources.push(ProjectDefinedMappingLocation::InMemory( + inline + .iter() + .map(|(name, pypi_name)| (name.to_lowercase(), pypi_name.clone())) + .collect(), + )); + } + Ok(ProjectDefinedChannelMapping::new( + sources, + convert_mode(*mode), + )) + } + } +} + +/// Convert the manifest-level mode to the derivation-level [`MappingMode`]. +/// +/// The two enums are deliberately asymmetric: `MappingMode::Disabled` has no +/// manifest-level mode string because "disabled" is spelled ` = +/// false` in TOML (see [`CondaPypiMapEntry::Disabled`]), not `mode = +/// "disabled"`. This function and the `Disabled` arm in [`convert_entry`] are +/// the single place where the two representations meet. (A `From` impl cannot +/// encode this: neither `pixi_manifest` nor `pypi_mapping` depends on the +/// other, so the orphan rule forces the conversion to live here.) +fn convert_mode(mode: CondaPypiMapMode) -> MappingMode { + match mode { + CondaPypiMapMode::Extend => MappingMode::Extend, + CondaPypiMapMode::Replace => MappingMode::Replace, + } +} + +/// Classify a manifest location spec into a url or a path, resolving relative +/// paths against the workspace root. `file://` urls are normalized to paths. +fn parse_mapping_location( + spec: &MappingLocationSpec, + channel_config: &ChannelConfig, +) -> miette::Result { + let url_or_path = UrlOrPath::from_str(&spec.location) + .into_diagnostic() + .context(format!( + "Could not parse mapping location `{}`", + spec.location + ))?; + + match url_or_path { + UrlOrPath::Url(url) => { + if !matches!(url.scheme(), "http" | "https") { + miette::bail!( + "unsupported scheme `{}` in mapping location `{}`; only http(s) URLs and local paths are supported", + url.scheme(), + spec.location + ); + } + // A plaintext mapping URL can be tampered with on the network, + // and a tampered mapping changes which conda packages are + // considered to satisfy PyPI dependencies. + if url.scheme() == "http" { + tracing::warn!( + "the conda-pypi mapping location `{}` uses plain `http://`; the mapping can \ + be tampered with in transit. Prefer `https://` or a local file.", + spec.location + ); + } + Ok(ProjectDefinedMappingLocation::Url { + url, + cache_ttl: spec.cache_ttl, + }) + } + UrlOrPath::Path(path) => { + if spec.cache_ttl.is_some() { + miette::bail!( + "`cache-ttl` is only supported for http(s) mapping locations, but `{}` is a local file", + spec.location + ); + } + let path = PathBuf::from(path.as_str()); + let abs_path = if path.is_relative() { + channel_config.root_dir.join(path) + } else { + path + }; + Ok(ProjectDefinedMappingLocation::Path(abs_path)) + } + } +} + +#[cfg(test)] +mod test { + use std::time::Duration; + + use super::*; + + fn channel_config() -> ChannelConfig { + ChannelConfig::default_with_root_dir(PathBuf::from("/workspace")) + } + + fn location(location: &str, cache_ttl: Option) -> MappingLocationSpec { + MappingLocationSpec { + location: location.to_string(), + cache_ttl, + } + } + + #[test] + fn test_parse_mapping_location_http_url_with_ttl() { + let ttl = Some(Duration::from_secs(60)); + let parsed = parse_mapping_location( + &location("https://example.com/m.json", ttl), + &channel_config(), + ) + .unwrap(); + assert_eq!( + parsed, + ProjectDefinedMappingLocation::Url { + url: "https://example.com/m.json".parse().unwrap(), + cache_ttl: ttl, + } + ); + } + + #[test] + fn test_parse_mapping_location_relative_path() { + let parsed = + parse_mapping_location(&location("sub/m.json", None), &channel_config()).unwrap(); + assert_eq!( + parsed, + ProjectDefinedMappingLocation::Path(PathBuf::from("/workspace/sub/m.json")) + ); + } + + #[test] + fn test_parse_mapping_location_file_url_becomes_path() { + let parsed = + parse_mapping_location(&location("file:///abs/m.json", None), &channel_config()) + .unwrap(); + assert_eq!( + parsed, + ProjectDefinedMappingLocation::Path(PathBuf::from("/abs/m.json")) + ); + } + + #[test] + fn test_parse_mapping_location_rejects_ttl_on_path() { + let err = parse_mapping_location( + &location("sub/m.json", Some(Duration::from_secs(60))), + &channel_config(), + ) + .unwrap_err(); + assert!(err.to_string().contains("cache-ttl")); + } + + #[test] + fn test_parse_mapping_location_rejects_unsupported_scheme() { + let err = parse_mapping_location( + &location("ftp://example.com/m.json", None), + &channel_config(), + ) + .unwrap_err(); + assert!(err.to_string().contains("unsupported scheme")); + } + + #[test] + fn test_convert_entry_disabled() { + let converted = convert_entry(&CondaPypiMapEntry::Disabled, &channel_config()).unwrap(); + assert_eq!(converted, ProjectDefinedChannelMapping::disabled()); + } +} diff --git a/crates/pixi_core/src/workspace/mod.rs b/crates/pixi_core/src/workspace/mod.rs index ab08564323..1be5f605d6 100644 --- a/crates/pixi_core/src/workspace/mod.rs +++ b/crates/pixi_core/src/workspace/mod.rs @@ -1,3 +1,4 @@ +mod conda_pypi_map; mod discovery; mod environment; pub mod errors; @@ -30,8 +31,7 @@ pub use discovery::{DiscoveryStart, WorkspaceLocator, WorkspaceLocatorError}; pub use environment::Environment; pub use has_project_ref::HasWorkspaceRef; use indexmap::Equivalent; -use itertools::Itertools; -use miette::{Context, IntoDiagnostic}; +use miette::IntoDiagnostic; use once_cell::sync::OnceCell; use pep508_rs::Requirement; use pixi_build_frontend::BackendOverride; @@ -52,11 +52,9 @@ use pixi_utils::{ reqwest::LazyReqwestClient, variants::{VariantConfig, VariantValue}, }; -use pypi_mapping::{ - ChannelName, ProjectDefinedMapping, ProjectDefinedMappingLocation, PurlDerivationMode, -}; +use pypi_mapping::PurlDerivationMode; use rattler_conda_types::{ - Channel, ChannelConfig, GenericVirtualPackage, MatchSpec, PackageName, Platform, Version, + ChannelConfig, GenericVirtualPackage, MatchSpec, PackageName, Platform, Version, }; use rattler_lock::LockFile; @@ -69,7 +67,6 @@ use rattler_virtual_packages::{ pub use registry::{WorkspaceRegistry, WorkspaceRegistryError}; pub use solve_group::SolveGroup; use tokio::sync::Semaphore; -use url::Url; pub use workspace_mut::WorkspaceMut; use xxhash_rust::xxh3::xxh3_64; @@ -912,108 +909,11 @@ impl Workspace { /// It can use project-defined mappings in the format `conda_name: pypi_name`, /// or the self-hosted prefix.dev mappings. pub fn pypi_name_derivation_mode(&self) -> miette::Result<&PurlDerivationMode> { - fn build_pypi_name_derivation_mode( - manifest: &WorkspaceManifest, - channel_config: &ChannelConfig, - ) -> miette::Result { - match manifest.workspace.conda_pypi_map.clone() { - Some(map) => { - let channel_to_location_map = map - .into_iter() - .map(|(key, value)| { - let key = key.into_channel(channel_config).into_diagnostic()?; - Ok((key, value)) - }) - .collect::>>()?; - - // User can disable the mapping by providing an empty map - if channel_to_location_map.is_empty() { - return Ok(PurlDerivationMode::Disabled); - } - - let project_channels: HashSet<_> = manifest - .workspace - .channels - .iter() - .map(|pc| pc.channel.clone().into_channel(channel_config)) - .try_collect() - .into_diagnostic()?; - - let feature_channels: HashSet<_> = manifest - .features - .values() - .flat_map(|feature| feature.channels.iter()) - .flatten() - .map(|pc| pc.channel.clone().into_channel(channel_config)) - .try_collect() - .into_diagnostic()?; - - let project_and_feature_channels: HashSet<_> = - project_channels.union(&feature_channels).collect(); - - for channel in channel_to_location_map.keys() { - if !project_and_feature_channels.contains(channel) { - let channels = project_and_feature_channels - .iter() - .map(|c| c.name.clone().unwrap_or_else(|| c.base_url.to_string())) - .sorted() - .collect::>() - .join(", "); - miette::bail!( - "conda-pypi-map is defined: the {} is missing from the channels array, which currently are: {}", - console::style( - channel - .name - .clone() - .unwrap_or_else(|| channel.base_url.to_string()) - ) - .bold(), - channels - ); - } - } - - let mapping = channel_to_location_map - .iter() - .map(|(channel, mapping_location)| { - let url_or_path = if mapping_location.starts_with("https://") - || mapping_location.starts_with("http://") - || mapping_location.starts_with("file://") - { - match Url::parse(mapping_location) { - Ok(url) => ProjectDefinedMappingLocation::Url(url), - Err(err) => { - return Err(err).into_diagnostic().context(format!( - "Could not convert {mapping_location} to URL" - )); - } - } - } else { - let path = PathBuf::from(mapping_location); - let abs_path = if path.is_relative() { - channel_config.root_dir.join(path) - } else { - path - }; - ProjectDefinedMappingLocation::Path(abs_path) - }; - - Ok(( - channel.canonical_name().trim_end_matches('/').into(), - url_or_path, - )) - }) - .collect::>>()?; - - Ok(PurlDerivationMode::ProjectDefined( - ProjectDefinedMapping::new(mapping).into(), - )) - } - None => Ok(PurlDerivationMode::Prefix), - } - } self.derivation_mode.get_or_try_init(|| { - build_pypi_name_derivation_mode(&self.workspace.value, &self.channel_config()) + conda_pypi_map::build_pypi_name_derivation_mode( + &self.workspace.value, + &self.channel_config(), + ) }) } @@ -1193,7 +1093,9 @@ mod tests { use itertools::Itertools; use pixi_config::{Config, DetachedEnvironments}; use pixi_manifest::{FeatureName, FeaturesExt, HasWorkspaceManifest}; - use rattler_conda_types::{Platform, Version}; + use pypi_mapping::{ProjectDefinedChannelMapping, ProjectDefinedMappingLocation}; + use rattler_conda_types::{Channel, Platform, Version}; + use url::Url; use xxhash_rust::xxh3::xxh3_64; use super::*; @@ -1598,7 +1500,22 @@ mod tests { let canonical_channel_name = canonical_name.trim_end_matches('/'); - assert_eq!(mapping.project_defined().unwrap().mapping.get(canonical_channel_name).unwrap(), &ProjectDefinedMappingLocation::Url(Url::parse("https://github.com/prefix-dev/parselmouth/blob/main/files/compressed_mapping.json").unwrap())); + assert_eq!( + mapping + .project_defined() + .unwrap() + .mapping + .get(canonical_channel_name) + .unwrap(), + // Bare location strings use the additive (extend) mode. + &ProjectDefinedChannelMapping::extend(ProjectDefinedMappingLocation::Url { + url: Url::parse( + "https://github.com/prefix-dev/parselmouth/blob/main/files/compressed_mapping.json" + ) + .unwrap(), + cache_ttl: None, + }) + ); // Check url channel as map key let file_contents = r#" @@ -1626,12 +1543,12 @@ mod tests { .trim_end_matches('/') ) .unwrap(), - &ProjectDefinedMappingLocation::Path( + &ProjectDefinedChannelMapping::extend(ProjectDefinedMappingLocation::Path( workspace .channel_config() .root_dir .join(PathBuf::from("mapping.json")) - ) + )) ); } diff --git a/crates/pixi_manifest/Cargo.toml b/crates/pixi_manifest/Cargo.toml index f9656f01e5..219777d313 100644 --- a/crates/pixi_manifest/Cargo.toml +++ b/crates/pixi_manifest/Cargo.toml @@ -14,6 +14,7 @@ console = { workspace = true } dunce = { workspace = true } fancy_display = { workspace = true } fs-err = { workspace = true } +humantime = { workspace = true } indexmap = { workspace = true } itertools = { workspace = true } miette = { workspace = true, features = ["fancy-no-backtrace"] } diff --git a/crates/pixi_manifest/src/lib.rs b/crates/pixi_manifest/src/lib.rs index c5298283f8..4dc9fbd54a 100644 --- a/crates/pixi_manifest/src/lib.rs +++ b/crates/pixi_manifest/src/lib.rs @@ -62,7 +62,10 @@ pub use target::{PackageTarget, TargetSelector, Targets, WorkspaceTarget}; pub use task::{Task, TaskName}; use thiserror::Error; pub use warning::{Warning, WarningWithSource, WithWarnings}; -pub use workspace::{BuildVariantSource, ChannelPriority, SolveStrategy, Workspace}; +pub use workspace::{ + BuildVariantSource, ChannelPriority, CondaPypiMap, CondaPypiMapEntry, CondaPypiMapMode, + CondaPypiMapSpec, MappingLocationSpec, SolveStrategy, Workspace, +}; pub use crate::{ environments::Environments, diff --git a/crates/pixi_manifest/src/toml/conda_pypi_map.rs b/crates/pixi_manifest/src/toml/conda_pypi_map.rs new file mode 100644 index 0000000000..f1c10bc756 --- /dev/null +++ b/crates/pixi_manifest/src/toml/conda_pypi_map.rs @@ -0,0 +1,346 @@ +//! TOML deserialization for `[workspace.conda-pypi-map]`. +//! +//! The field accepts `false` (disable all lookups) or a per-channel table. +//! Each channel value is either a bare location string, `false` (disable +//! lookups for that channel) or a table with `location`, inline `mapping` +//! entries, `mode` and `cache-ttl`. + +use std::collections::HashMap; + +use pixi_toml::{TomlEnum, TomlHashMap, custom_error}; +use rattler_conda_types::NamedChannelOrUrl; +use toml_span::{ + DeserError, Value, + de_helpers::{TableHelper, expected}, + value::ValueInner, +}; + +use crate::workspace::{ + CondaPypiMap, CondaPypiMapEntry, CondaPypiMapMode, CondaPypiMapSpec, MappingLocationSpec, +}; + +impl<'de> toml_span::Deserialize<'de> for CondaPypiMap { + fn deserialize(value: &mut Value<'de>) -> Result { + match value.take() { + ValueInner::Boolean(false) => Ok(CondaPypiMap::Disabled), + ValueInner::Boolean(true) => Err(custom_error( + "`conda-pypi-map = true` is not supported; use `false` to disable the \ + mapping, or a table to configure it", + value.span, + ) + .into()), + inner @ ValueInner::Table(_) => { + let span = value.span; + let map = TomlHashMap::::deserialize( + &mut Value::with_span(inner, span), + )?; + Ok(CondaPypiMap::Map(map.into_inner())) + } + other => Err(expected("a table or `false`", other, value.span).into()), + } + } +} + +impl<'de> toml_span::Deserialize<'de> for CondaPypiMapEntry { + fn deserialize(value: &mut Value<'de>) -> Result { + match value.take() { + ValueInner::String(s) => Ok(CondaPypiMapEntry::from_location(s.into_owned())), + ValueInner::Boolean(false) => Ok(CondaPypiMapEntry::Disabled), + ValueInner::Boolean(true) => Err(custom_error( + "`true` is not supported; use `false` to disable the mapping for this \ + channel, or a string or table to configure it", + value.span, + ) + .into()), + inner @ ValueInner::Table(_) => { + let table_span = value.span; + let mut th = TableHelper::new(&mut Value::with_span(inner, table_span))?; + + let location: Option = th.optional("location"); + let mapping: Option>> = th + .optional::>("mapping") + .map(|map| { + map.into_inner() + .into_iter() + .map(|(name, value)| (name, value.0)) + .collect() + }); + let mode = th + .optional::>("mode") + .map(TomlEnum::into_inner) + .unwrap_or_default(); + let cache_ttl = match th.optional::>("cache-ttl") { + Some(spanned) => Some( + spanned + .value + .parse::() + .map_err(|e| { + custom_error( + format!( + "invalid `cache-ttl` duration: {e}; expected a duration \ + like \"24h\", \"7d\" or \"1h 30m\"" + ), + spanned.span, + ) + })? + .into(), + ), + None => None, + }; + + th.finalize(None)?; + + if location.is_none() && mapping.is_none() { + return Err(custom_error( + "expected at least one of `location` or `mapping`", + table_span, + ) + .into()); + } + + // `cache-ttl` is part of the location source; without a + // location it has nothing to apply to. + let location = match (location, cache_ttl) { + (Some(location), cache_ttl) => Some(MappingLocationSpec { + location, + cache_ttl, + }), + (None, Some(_)) => { + return Err(custom_error( + "`cache-ttl` requires a `location`; it is only effective for \ + http(s) URLs", + table_span, + ) + .into()); + } + (None, None) => None, + }; + + Ok(CondaPypiMapEntry::Map(CondaPypiMapSpec { + location, + mapping, + mode, + })) + } + other => Err(expected("a string, table or `false`", other, value.span).into()), + } + } +} + +/// The value of an inline mapping entry: a pypi name, or `false` to mark the +/// package as not available on PyPI. +pub(crate) struct TomlCondaPypiMapValue(pub(crate) Option); + +impl<'de> toml_span::Deserialize<'de> for TomlCondaPypiMapValue { + fn deserialize(value: &mut Value<'de>) -> Result { + match value.take() { + ValueInner::String(s) => Ok(Self(Some(s.into_owned()))), + ValueInner::Boolean(false) => Ok(Self(None)), + ValueInner::Boolean(true) => Err(custom_error( + "`true` is not supported; use a string to map the package to a PyPI name, \ + or `false` to mark it as not a PyPI package", + value.span, + ) + .into()), + other => Err(expected("a string or `false`", other, value.span).into()), + } + } +} + +#[cfg(test)] +mod test { + use std::time::Duration; + + use insta::assert_snapshot; + use rattler_conda_types::NamedChannelOrUrl; + + use super::*; + use crate::{ + toml::{FromTomlStr, TomlWorkspace}, + utils::test_utils::{expect_parse_failure, expect_parse_warnings}, + }; + + fn parse_map(conda_pypi_map: &str) -> CondaPypiMap { + let input = format!( + r#" + channels = [] + platforms = [] + conda-pypi-map = {conda_pypi_map} + "# + ); + TomlWorkspace::from_toml_str(&input) + .expect("parsing should succeed") + .conda_pypi_map + .expect("conda-pypi-map should be set") + } + + fn get_entry(map: &CondaPypiMap, channel: &str) -> CondaPypiMapEntry { + let CondaPypiMap::Map(map) = map else { + panic!("expected a per-channel map"); + }; + map.get(&NamedChannelOrUrl::Name(channel.to_string())) + .expect("channel should be present") + .clone() + } + + #[test] + fn test_bare_string_is_extend() { + let map = parse_map(r#"{ conda-forge = "mapping.json" }"#); + assert_eq!( + get_entry(&map, "conda-forge"), + CondaPypiMapEntry::Map(CondaPypiMapSpec { + location: Some(MappingLocationSpec { + location: "mapping.json".to_string(), + cache_ttl: None, + }), + mapping: None, + mode: CondaPypiMapMode::Extend, + }) + ); + } + + #[test] + fn test_table_with_location_mode_and_ttl() { + let map = parse_map( + r#"{ conda-forge = { location = "https://example.com/m.json", mode = "replace", cache-ttl = "24h" } }"#, + ); + assert_eq!( + get_entry(&map, "conda-forge"), + CondaPypiMapEntry::Map(CondaPypiMapSpec { + location: Some(MappingLocationSpec { + location: "https://example.com/m.json".to_string(), + cache_ttl: Some(Duration::from_secs(24 * 60 * 60)), + }), + mapping: None, + mode: CondaPypiMapMode::Replace, + }) + ); + } + + #[test] + fn test_inline_mapping_with_false_value() { + let map = parse_map( + r#"{ conda-forge = { mapping = { pytorch = "torch", not-on-pypi = false } } }"#, + ); + let CondaPypiMapEntry::Map(CondaPypiMapSpec { mapping, mode, .. }) = + get_entry(&map, "conda-forge") + else { + panic!("expected a mapping entry"); + }; + let mapping = mapping.expect("mapping should be set"); + assert_eq!(mode, CondaPypiMapMode::Extend); + assert_eq!(mapping["pytorch"], Some("torch".to_string())); + assert_eq!(mapping["not-on-pypi"], None); + } + + #[test] + fn test_channel_false_disables() { + let map = parse_map(r#"{ conda-forge = false }"#); + assert_eq!(get_entry(&map, "conda-forge"), CondaPypiMapEntry::Disabled); + } + + #[test] + fn test_top_level_false_disables() { + let map = parse_map("false"); + assert_eq!(map, CondaPypiMap::Disabled); + } + + #[test] + fn test_empty_map_parses_and_warns() { + let map = parse_map("{}"); + assert!(matches!(map, CondaPypiMap::Map(map) if map.is_empty())); + + assert_snapshot!(expect_parse_warnings( + r#" + [workspace] + channels = [] + platforms = [] + conda-pypi-map = {} + "# + )); + } + + #[test] + fn test_top_level_true_fails() { + assert_snapshot!(expect_parse_failure( + r#" + [workspace] + channels = [] + platforms = [] + conda-pypi-map = true + "# + )); + } + + #[test] + fn test_channel_true_fails() { + assert_snapshot!(expect_parse_failure( + r#" + [workspace] + channels = [] + platforms = [] + conda-pypi-map = { conda-forge = true } + "# + )); + } + + #[test] + fn test_inline_true_value_fails() { + assert_snapshot!(expect_parse_failure( + r#" + [workspace] + channels = [] + platforms = [] + conda-pypi-map = { conda-forge = { mapping = { pytorch = true } } } + "# + )); + } + + #[test] + fn test_empty_entry_table_fails() { + assert_snapshot!(expect_parse_failure( + r#" + [workspace] + channels = [] + platforms = [] + conda-pypi-map = { conda-forge = { mode = "extend" } } + "# + )); + } + + #[test] + fn test_bogus_mode_fails() { + assert_snapshot!(expect_parse_failure( + r#" + [workspace] + channels = [] + platforms = [] + conda-pypi-map = { conda-forge = { location = "m.json", mode = "bogus" } } + "# + )); + } + + #[test] + fn test_invalid_ttl_fails() { + assert_snapshot!(expect_parse_failure( + r#" + [workspace] + channels = [] + platforms = [] + conda-pypi-map = { conda-forge = { location = "https://example.com/m.json", cache-ttl = "bogus" } } + "# + )); + } + + #[test] + fn test_ttl_without_location_fails() { + assert_snapshot!(expect_parse_failure( + r#" + [workspace] + channels = [] + platforms = [] + conda-pypi-map = { conda-forge = { mapping = { a = "b" }, cache-ttl = "24h" } } + "# + )); + } +} diff --git a/crates/pixi_manifest/src/toml/mod.rs b/crates/pixi_manifest/src/toml/mod.rs index da4c745a91..89bac4164e 100644 --- a/crates/pixi_manifest/src/toml/mod.rs +++ b/crates/pixi_manifest/src/toml/mod.rs @@ -1,6 +1,7 @@ mod build_backend; mod build_target; mod channel; +mod conda_pypi_map; mod document; mod environment; mod feature; diff --git a/crates/pixi_manifest/src/toml/snapshots/pixi_manifest__toml__conda_pypi_map__test__bogus_mode_fails.snap b/crates/pixi_manifest/src/toml/snapshots/pixi_manifest__toml__conda_pypi_map__test__bogus_mode_fails.snap new file mode 100644 index 0000000000..1c81c6f9a1 --- /dev/null +++ b/crates/pixi_manifest/src/toml/snapshots/pixi_manifest__toml__conda_pypi_map__test__bogus_mode_fails.snap @@ -0,0 +1,11 @@ +--- +source: crates/pixi_manifest/src/toml/conda_pypi_map.rs +expression: "expect_parse_failure(r#\"\n [workspace]\n channels = []\n platforms = []\n conda-pypi-map = { conda-forge = { location = \"m.json\", mode = \"bogus\" } }\n \"#)" +--- + × Expected one of 'extend', 'replace' + ╭─[pixi.toml:5:77] + 4 │ platforms = [] + 5 │ conda-pypi-map = { conda-forge = { location = "m.json", mode = "bogus" } } + · ───── + 6 │ + ╰──── diff --git a/crates/pixi_manifest/src/toml/snapshots/pixi_manifest__toml__conda_pypi_map__test__channel_true_fails.snap b/crates/pixi_manifest/src/toml/snapshots/pixi_manifest__toml__conda_pypi_map__test__channel_true_fails.snap new file mode 100644 index 0000000000..7ba16035c2 --- /dev/null +++ b/crates/pixi_manifest/src/toml/snapshots/pixi_manifest__toml__conda_pypi_map__test__channel_true_fails.snap @@ -0,0 +1,11 @@ +--- +source: crates/pixi_manifest/src/toml/conda_pypi_map.rs +expression: "expect_parse_failure(r#\"\n [workspace]\n channels = []\n platforms = []\n conda-pypi-map = { conda-forge = true }\n \"#)" +--- + × `true` is not supported; use `false` to disable the mapping for this channel, or a string or table to configure it + ╭─[pixi.toml:5:46] + 4 │ platforms = [] + 5 │ conda-pypi-map = { conda-forge = true } + · ──── + 6 │ + ╰──── diff --git a/crates/pixi_manifest/src/toml/snapshots/pixi_manifest__toml__conda_pypi_map__test__empty_entry_table_fails.snap b/crates/pixi_manifest/src/toml/snapshots/pixi_manifest__toml__conda_pypi_map__test__empty_entry_table_fails.snap new file mode 100644 index 0000000000..4a831b152b --- /dev/null +++ b/crates/pixi_manifest/src/toml/snapshots/pixi_manifest__toml__conda_pypi_map__test__empty_entry_table_fails.snap @@ -0,0 +1,11 @@ +--- +source: crates/pixi_manifest/src/toml/conda_pypi_map.rs +expression: "expect_parse_failure(r#\"\n [workspace]\n channels = []\n platforms = []\n conda-pypi-map = { conda-forge = { mode = \"extend\" } }\n \"#)" +--- + × expected at least one of `location` or `mapping` + ╭─[pixi.toml:5:46] + 4 │ platforms = [] + 5 │ conda-pypi-map = { conda-forge = { mode = "extend" } } + · ─────────────────── + 6 │ + ╰──── diff --git a/crates/pixi_manifest/src/toml/snapshots/pixi_manifest__toml__conda_pypi_map__test__empty_map_parses_and_warns.snap b/crates/pixi_manifest/src/toml/snapshots/pixi_manifest__toml__conda_pypi_map__test__empty_map_parses_and_warns.snap new file mode 100644 index 0000000000..5ca8346e8a --- /dev/null +++ b/crates/pixi_manifest/src/toml/snapshots/pixi_manifest__toml__conda_pypi_map__test__empty_map_parses_and_warns.snap @@ -0,0 +1,6 @@ +--- +source: crates/pixi_manifest/src/toml/conda_pypi_map.rs +expression: "expect_parse_warnings(r#\"\n [workspace]\n channels = []\n platforms = []\n conda-pypi-map = {}\n \"#)" +--- + ⚠ `conda-pypi-map = {}` is deprecated + help: To disable all mapping lookups, write `conda-pypi-map = false` instead. diff --git a/crates/pixi_manifest/src/toml/snapshots/pixi_manifest__toml__conda_pypi_map__test__inline_true_value_fails.snap b/crates/pixi_manifest/src/toml/snapshots/pixi_manifest__toml__conda_pypi_map__test__inline_true_value_fails.snap new file mode 100644 index 0000000000..33010f3203 --- /dev/null +++ b/crates/pixi_manifest/src/toml/snapshots/pixi_manifest__toml__conda_pypi_map__test__inline_true_value_fails.snap @@ -0,0 +1,11 @@ +--- +source: crates/pixi_manifest/src/toml/conda_pypi_map.rs +expression: "expect_parse_failure(r#\"\n [workspace]\n channels = []\n platforms = []\n conda-pypi-map = { conda-forge = { mapping = { pytorch = true } } }\n \"#)" +--- + × `true` is not supported; use a string to map the package to a PyPI name, or `false` to mark it as not a PyPI package + ╭─[pixi.toml:5:70] + 4 │ platforms = [] + 5 │ conda-pypi-map = { conda-forge = { mapping = { pytorch = true } } } + · ──── + 6 │ + ╰──── diff --git a/crates/pixi_manifest/src/toml/snapshots/pixi_manifest__toml__conda_pypi_map__test__invalid_ttl_fails.snap b/crates/pixi_manifest/src/toml/snapshots/pixi_manifest__toml__conda_pypi_map__test__invalid_ttl_fails.snap new file mode 100644 index 0000000000..eed7f9610f --- /dev/null +++ b/crates/pixi_manifest/src/toml/snapshots/pixi_manifest__toml__conda_pypi_map__test__invalid_ttl_fails.snap @@ -0,0 +1,11 @@ +--- +source: crates/pixi_manifest/src/toml/conda_pypi_map.rs +expression: "expect_parse_failure(r#\"\n [workspace]\n channels = []\n platforms = []\n conda-pypi-map = { conda-forge = { location = \"https://example.com/m.json\", cache-ttl = \"bogus\" } }\n \"#)" +--- + × invalid `cache-ttl` duration: expected number at 0; expected a duration like "24h", "7d" or "1h 30m" + ╭─[pixi.toml:5:102] + 4 │ platforms = [] + 5 │ conda-pypi-map = { conda-forge = { location = "https://example.com/m.json", cache-ttl = "bogus" } } + · ───── + 6 │ + ╰──── diff --git a/crates/pixi_manifest/src/toml/snapshots/pixi_manifest__toml__conda_pypi_map__test__top_level_true_fails.snap b/crates/pixi_manifest/src/toml/snapshots/pixi_manifest__toml__conda_pypi_map__test__top_level_true_fails.snap new file mode 100644 index 0000000000..54de5ec994 --- /dev/null +++ b/crates/pixi_manifest/src/toml/snapshots/pixi_manifest__toml__conda_pypi_map__test__top_level_true_fails.snap @@ -0,0 +1,11 @@ +--- +source: crates/pixi_manifest/src/toml/conda_pypi_map.rs +expression: "expect_parse_failure(r#\"\n [workspace]\n channels = []\n platforms = []\n conda-pypi-map = true\n \"#)" +--- + × `conda-pypi-map = true` is not supported; use `false` to disable the mapping, or a table to configure it + ╭─[pixi.toml:5:30] + 4 │ platforms = [] + 5 │ conda-pypi-map = true + · ──── + 6 │ + ╰──── diff --git a/crates/pixi_manifest/src/toml/snapshots/pixi_manifest__toml__conda_pypi_map__test__ttl_without_location_fails.snap b/crates/pixi_manifest/src/toml/snapshots/pixi_manifest__toml__conda_pypi_map__test__ttl_without_location_fails.snap new file mode 100644 index 0000000000..77216ae0d9 --- /dev/null +++ b/crates/pixi_manifest/src/toml/snapshots/pixi_manifest__toml__conda_pypi_map__test__ttl_without_location_fails.snap @@ -0,0 +1,11 @@ +--- +source: crates/pixi_manifest/src/toml/conda_pypi_map.rs +expression: "expect_parse_failure(r#\"\n [workspace]\n channels = []\n platforms = []\n conda-pypi-map = { conda-forge = { mapping = { a = \"b\" }, cache-ttl = \"24h\" } }\n \"#)" +--- + × `cache-ttl` requires a `location`; it is only effective for http(s) URLs + ╭─[pixi.toml:5:46] + 4 │ platforms = [] + 5 │ conda-pypi-map = { conda-forge = { mapping = { a = "b" }, cache-ttl = "24h" } } + · ──────────────────────────────────────────── + 6 │ + ╰──── diff --git a/crates/pixi_manifest/src/toml/workspace.rs b/crates/pixi_manifest/src/toml/workspace.rs index 3381d1f434..0a04aad707 100644 --- a/crates/pixi_manifest/src/toml/workspace.rs +++ b/crates/pixi_manifest/src/toml/workspace.rs @@ -6,7 +6,7 @@ use std::{ use indexmap::{IndexMap, IndexSet}; use pixi_spec::{ExcludeNewer, TomlSpec, TomlVersionSpecStr}; use pixi_toml::{TomlFromStr, TomlHashMap, TomlIndexMap, TomlIndexSet, TomlWith}; -use rattler_conda_types::{NamedChannelOrUrl, PackageName, Version, VersionSpec}; +use rattler_conda_types::{PackageName, Version, VersionSpec}; use std::str::FromStr; use toml_span::{DeserError, Span, Spanned, Value, de_helpers::TableHelper, value::ValueInner}; use url::Url; @@ -20,7 +20,7 @@ use crate::{ manifest::ExternalWorkspaceProperties, platform::TomlPixiPlatform, preview::TomlPreview, }, utils::PixiSpanned, - workspace::{BuildVariantSource, ChannelPriority, SolveStrategy}, + workspace::{BuildVariantSource, ChannelPriority, CondaPypiMap, SolveStrategy}, }; /// Parses `[workspace.dependencies]` into an ordered `(name, TomlSpec)` map. @@ -110,7 +110,7 @@ pub struct TomlWorkspace { pub homepage: Option, pub repository: Option, pub documentation: Option, - pub conda_pypi_map: Option>, + pub conda_pypi_map: Option, pub pypi_options: Option, pub s3_options: Option>, pub preview: TomlPreview, @@ -177,7 +177,21 @@ impl TomlWorkspace { value: preview, } = self.preview.into_preview(); - let warnings = preview_warnings; + let mut warnings = preview_warnings; + + // An empty `conda-pypi-map = {}` is a soft-deprecated alias for + // `conda-pypi-map = false`. + if let Some(CondaPypiMap::Map(map)) = &self.conda_pypi_map + && map.is_empty() + { + warnings.push( + GenericError::new("`conda-pypi-map = {}` is deprecated") + .with_help( + "To disable all mapping lookups, write `conda-pypi-map = false` instead.", + ) + .into(), + ); + } let build_variant_files_default = convert_build_variant_files(self.build_variant_files, root_directory)?; @@ -354,9 +368,7 @@ impl<'de> toml_span::Deserialize<'de> for TomlWorkspace { let documentation = th .optional::>("documentation") .map(TomlFromStr::into_inner); - let conda_pypi_map = th - .optional::>("conda-pypi-map") - .map(TomlHashMap::into_inner); + let conda_pypi_map = th.optional("conda-pypi-map"); let pypi_options = th.optional("pypi-options"); let s3_options = th .optional::>("s3-options") diff --git a/crates/pixi_manifest/src/workspace.rs b/crates/pixi_manifest/src/workspace.rs index 8674232945..043264b22d 100644 --- a/crates/pixi_manifest/src/workspace.rs +++ b/crates/pixi_manifest/src/workspace.rs @@ -74,8 +74,8 @@ pub struct Workspace { /// URL of the project documentation pub documentation: Option, - /// URL or Path of the conda to pypi name mapping - pub conda_pypi_map: Option>, + /// The conda to pypi name mapping configuration. + pub conda_pypi_map: Option, /// The pypi options supported in the project pub pypi_options: Option, @@ -306,6 +306,94 @@ impl From for ChannelPriority { } } +/// The value of `[workspace.conda-pypi-map]`. +#[derive(Debug, Clone, PartialEq)] +pub enum CondaPypiMap { + /// `conda-pypi-map = false`: disable purl derivation lookups entirely. + /// + /// Note that the offline conda-forge verbatim fallback (assume the conda + /// name is the PyPI name) still applies; disabling only turns off the + /// project-defined and prefix.dev lookups. + Disabled, + /// Per-channel mapping configuration. An empty map is a soft-deprecated + /// alias for `Disabled`. + Map(HashMap), +} + +/// How a project-defined channel mapping interacts with the default +/// prefix.dev derivation chain. +#[derive( + Debug, + Copy, + Clone, + Default, + Eq, + PartialEq, + strum::Display, + strum::VariantNames, + strum::EnumString, +)] +#[strum(serialize_all = "kebab-case")] +pub enum CondaPypiMapMode { + /// The mapping overlays the defaults: a hit is final, a miss falls + /// through to the prefix.dev chain. + #[default] + Extend, + /// The mapping is exclusive: only packages in the mapping get purls. + Replace, +} + +/// The mapping configuration for one channel in `[workspace.conda-pypi-map]`. +#[derive(Debug, Clone, PartialEq)] +pub enum CondaPypiMapEntry { + /// ` = false`: no purl lookups for this channel. + /// + /// The offline conda-forge verbatim fallback (assume the conda name is + /// the PyPI name) still applies to records from this channel. + Disabled, + /// A mapping defined by a location (file or URL) and/or inline entries. + Map(CondaPypiMapSpec), +} + +/// A channel mapping built from up to two sources: an external location and +/// inline entries. Inline entries override entries from the location. +#[derive(Debug, Clone, PartialEq)] +pub struct CondaPypiMapSpec { + /// An external mapping JSON file (path or URL). + pub location: Option, + /// Inline conda-name to pypi-name entries. A `None` value (spelled + /// `false` in TOML) means the package is not a PyPI package. + pub mapping: Option>>, + pub mode: CondaPypiMapMode, +} + +/// An external mapping source: a file path or URL, with an optional cache +/// TTL for URL locations. +#[derive(Debug, Clone, PartialEq)] +pub struct MappingLocationSpec { + /// File path or URL of a mapping JSON file. Unresolved: relative paths + /// are resolved against the workspace root by the consumer. + pub location: String, + /// How long a mapping fetched from a URL may be reused before it is + /// re-fetched. Only valid for http(s) locations. + pub cache_ttl: Option, +} + +impl CondaPypiMapEntry { + /// Create an entry from a bare location string. Bare strings use the + /// default (extend) mode. + pub fn from_location(location: String) -> Self { + Self::Map(CondaPypiMapSpec { + location: Some(MappingLocationSpec { + location, + cache_ttl: None, + }), + mapping: None, + mode: CondaPypiMapMode::default(), + }) + } +} + #[derive( Debug, Copy, diff --git a/crates/pixi_toml/src/diagnostic.rs b/crates/pixi_toml/src/diagnostic.rs index 675cb4a956..591e44f248 100644 --- a/crates/pixi_toml/src/diagnostic.rs +++ b/crates/pixi_toml/src/diagnostic.rs @@ -20,6 +20,21 @@ fn split_custom_error_help(message: &str) -> (&str, Option<&str>) { .map_or((message, None), |(message, help)| (message, Some(help))) } +/// Construct a custom [`toml_span::Error`] with the given message and span. +/// +/// Shorthand for the common pattern of returning a hand-written validation +/// error from a [`toml_span::Deserialize`] implementation. +pub fn custom_error( + message: impl Into>, + span: toml_span::Span, +) -> toml_span::Error { + toml_span::Error { + kind: toml_span::ErrorKind::Custom(message.into()), + span, + line_info: None, + } +} + /// A wrapper around [`toml_span::Error`] that implements the `miette::Diagnostic` trait. #[derive(Debug)] pub struct TomlDiagnostic(pub toml_span::Error); diff --git a/crates/pixi_toml/src/lib.rs b/crates/pixi_toml/src/lib.rs index aec3f499e7..3993d340ee 100644 --- a/crates/pixi_toml/src/lib.rs +++ b/crates/pixi_toml/src/lib.rs @@ -21,7 +21,7 @@ mod with; use std::str::FromStr; pub use btree_set::TomlBTreeSet; -pub use diagnostic::{TomlDiagnostic, custom_error_message_with_help}; +pub use diagnostic::{TomlDiagnostic, custom_error, custom_error_message_with_help}; pub use digest::TomlDigest; pub use from_str::TomlFromStr; pub use hash_map::TomlHashMap; diff --git a/crates/pypi_mapping/Cargo.toml b/crates/pypi_mapping/Cargo.toml index 6bc3851d3a..db95be2347 100644 --- a/crates/pypi_mapping/Cargo.toml +++ b/crates/pypi_mapping/Cargo.toml @@ -15,6 +15,7 @@ dashmap = { workspace = true } fs-err = { workspace = true } futures = { workspace = true } http-cache-reqwest = { workspace = true } +humantime = { workspace = true } itertools = { workspace = true } miette = { workspace = true } pep440_rs = { workspace = true } @@ -27,7 +28,11 @@ reqwest-middleware = { workspace = true } reqwest-retry = { workspace = true } serde = { workspace = true, features = ["derive"] } serde_json = { workspace = true } +tempfile = { workspace = true } thiserror = { workspace = true } tokio = { workspace = true } tracing = { workspace = true } url = { workspace = true } + +[dev-dependencies] +filetime = { workspace = true } diff --git a/crates/pypi_mapping/src/derivation_mode.rs b/crates/pypi_mapping/src/derivation_mode.rs index f5f49708a1..8b6ce76467 100644 --- a/crates/pypi_mapping/src/derivation_mode.rs +++ b/crates/pypi_mapping/src/derivation_mode.rs @@ -1,20 +1,82 @@ -use std::{collections::HashMap, path::PathBuf, sync::Arc}; +use std::{collections::HashMap, path::PathBuf, sync::Arc, time::Duration}; use url::Url; use crate::{CompressedMapping, ProjectDefinedMapping}; pub type ChannelName = String; -pub type MappingMap = HashMap; -pub type MappingByChannel = HashMap; +pub type MappingMap = HashMap; +pub type MappingByChannel = HashMap; #[derive(Debug, Clone, PartialEq, Eq)] pub enum ProjectDefinedMappingLocation { Path(PathBuf), - Url(Url), + Url { + url: Url, + /// When set, the fetched mapping is cached on disk and only + /// re-fetched once the cached copy is older than this duration. + cache_ttl: Option, + }, InMemory(CompressedMapping), } +/// How a project-defined channel mapping interacts with the default +/// prefix.dev derivation chain. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub enum MappingMode { + /// The mapping overlays the defaults: a hit (including an explicit "not a + /// PyPI package" entry) is final, a miss falls through to the prefix.dev + /// chain (hash mapping, compressed mapping, conda-forge verbatim + /// fallback). + #[default] + Extend, + /// The mapping is exclusive: only packages in the mapping get purls. No + /// prefix.dev lookups and no conda-forge verbatim fallback happen for + /// records from this channel. + Replace, + /// No purls are looked up for records from this channel, neither + /// project-defined nor prefix.dev. The offline conda-forge verbatim + /// fallback still applies. + Disabled, +} + +/// The project-defined mapping configuration for a single channel. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ProjectDefinedChannelMapping { + /// The mapping sources, merged in order: entries from later sources + /// override entries from earlier ones. + pub sources: Vec, + pub mode: MappingMode, +} + +impl ProjectDefinedChannelMapping { + pub fn new(sources: Vec, mode: MappingMode) -> Self { + Self { sources, mode } + } + + /// A single-source mapping that overlays the prefix.dev defaults. + pub fn extend(source: ProjectDefinedMappingLocation) -> Self { + Self::new(vec![source], MappingMode::Extend) + } + + /// A single-source mapping that replaces the prefix.dev defaults. + pub fn replace(source: ProjectDefinedMappingLocation) -> Self { + Self::new(vec![source], MappingMode::Replace) + } + + /// Disable purl lookups for this channel. + pub fn disabled() -> Self { + Self::new(Vec::new(), MappingMode::Disabled) + } +} + +/// A channel mapping with all its sources fetched and merged. +#[derive(Debug, Clone)] +pub struct ResolvedChannelMapping { + pub mapping: CompressedMapping, + pub mode: MappingMode, +} + /// User-selected mapping mode. /// /// This controls which resolver family [`crate::PurlDerivationClient`] uses. It is not @@ -22,14 +84,18 @@ pub enum ProjectDefinedMappingLocation { /// concrete resolver that produced an individual purl. #[derive(Debug, Clone)] pub enum PurlDerivationMode { - /// Use only project-defined per-channel mappings. + /// Use project-defined per-channel mappings. Depending on each channel's + /// [`MappingMode`] the prefix.dev mappings may still be used as a + /// fallback. Records from channels without a project-defined mapping use + /// the prefix.dev mappings. ProjectDefined(Arc), /// Use prefix.dev mappings: hash mapping first, then compressed mapping. Prefix, /// Disable project-defined and prefix.dev mappings. /// - /// Note: the current resolver still allows the conda-forge verbatim fallback - /// in this mode. + /// The offline conda-forge verbatim fallback (assume the conda name is + /// the PyPI name) still applies in this mode; disabling only turns off + /// the lookups. Disabled, } diff --git a/crates/pypi_mapping/src/lib.rs b/crates/pypi_mapping/src/lib.rs index cf06a3b502..cf88d06ff7 100644 --- a/crates/pypi_mapping/src/lib.rs +++ b/crates/pypi_mapping/src/lib.rs @@ -12,6 +12,11 @@ //! 3. [`PurlDerivationSource::PrefixCompressedMapping`] — prefix.dev compressed name mapping. //! 4. [`PurlDerivationSource::CondaForgeVerbatimFallback`] — conda-forge fallback that assumes //! the conda package name is the PyPI package name. +//! +//! A project-defined mapping carries a per-channel [`MappingMode`] that +//! determines how it interacts with the prefix.dev chain: `Extend` overlays +//! it (a miss falls through to prefix.dev), `Replace` is exclusive, and +//! `Disabled` turns lookups for that channel off entirely. use std::{ collections::{BTreeSet, HashMap}, @@ -23,7 +28,6 @@ use std::{ use futures::{StreamExt, stream::FuturesUnordered}; use http_cache_reqwest::{CACacheManager, Cache, CacheMode, HttpCache, HttpCacheOptions}; use itertools::Itertools; -use miette::IntoDiagnostic; use rattler_conda_types::{PackageUrl, RepoDataRecord}; use rattler_networking::LazyClient; use reqwest_middleware::ClientBuilder; @@ -41,7 +45,8 @@ pub mod resolvers; pub use channel::{is_conda_forge_record, is_conda_forge_url}; pub use derivation_mode::{ - ChannelName, MappingByChannel, MappingMap, ProjectDefinedMappingLocation, PurlDerivationMode, + ChannelName, MappingByChannel, MappingMap, MappingMode, ProjectDefinedChannelMapping, + ProjectDefinedMappingLocation, PurlDerivationMode, ResolvedChannelMapping, }; pub use metrics::CacheMetrics; pub use purl::PurlDerivationSource; @@ -57,11 +62,22 @@ use crate::{ /// name. pub type CompressedMapping = HashMap>; +/// Help text shown when fetching a conda-pypi mapping over the network fails, +/// listing the manifest options that avoid the network lookup. +pub(crate) const MAPPING_OFFLINE_HELP: &str = "If this host cannot be reached (e.g. behind a firewall), you can avoid the network lookup: \ + point the channel's `conda-pypi-map` entry at your own mapping with `location` (optionally \ + cached with `cache-ttl`), make it exclusive with `mode = \"replace\"`, disable the channel \ + with ` = false`, or disable the mapping entirely with `conda-pypi-map = false`."; + /// The mapping client implements the logic to derive purls for conda packages. /// /// The resolver order depends on [`PurlDerivationMode`]: /// -/// - [`PurlDerivationMode::ProjectDefined`]: project-defined per-channel mapping only. +/// - [`PurlDerivationMode::ProjectDefined`]: project-defined per-channel mapping. How records +/// from a mapped channel interact with the prefix.dev chain depends on the channel's +/// [`MappingMode`]: `Extend` falls through to prefix.dev on a miss, `Replace` is exclusive +/// (no prefix.dev, no verbatim fallback), and `Disabled` skips all lookups. Records from +/// unmapped channels use the prefix.dev chain. /// - [`PurlDerivationMode::Prefix`]: prefix hash mapping, then prefix compressed mapping, /// then the conda-forge verbatim fallback. /// - [`PurlDerivationMode::Disabled`]: no project-defined or prefix mapping. The current behavior @@ -120,7 +136,7 @@ impl PurlDerivationClientBuilder { } } -#[derive(Debug, Error)] +#[derive(Debug, Error, miette::Diagnostic)] pub enum MappingError { #[error("failed to access conda-pypi mapping cache at '{path}'")] IoError { @@ -129,6 +145,7 @@ pub enum MappingError { path: PathBuf, }, #[error("failed to fetch conda-pypi mapping from remote source")] + #[diagnostic(help("{}", MAPPING_OFFLINE_HELP))] Reqwest(#[source] reqwest_middleware::Error), } @@ -207,7 +224,7 @@ impl PurlDerivationClient { if let PurlDerivationMode::ProjectDefined(mapping_url) = derivation_mode { Some(ProjectDefinedResolver::from( mapping_url - .fetch_project_defined_mapping(&self.client) + .fetch_project_defined_mapping(&self.client, &self.cache_path) .await?, )) } else { @@ -259,7 +276,9 @@ impl PurlDerivationClient { let mut amended_records = 0; let mut total_records = 0; while let Some(next) = amend_futures.next().await { - let (record, derived_purls) = next.into_diagnostic()?; + // Use `Report::new` instead of `into_diagnostic` to preserve the + // diagnostic help text on `MappingError`. + let (record, derived_purls) = next.map_err(miette::Report::new)?; if let Some(derived_purls) = derived_purls.into_purls() { amend_purls(record, derived_purls); @@ -300,30 +319,75 @@ impl PurlDerivationClient { record: &RepoDataRecord, cache_metrics: &CacheMetrics, ) -> Result { - let purls = if matches!(derivation_mode, PurlDerivationMode::Disabled) { - DerivationOutcome::NotApplicable - } else if let Some(project_defined_mappings) = - project_defined_mappings.filter(|mapping| mapping.is_mapping_for_record(record)) + /// What is consulted when the primary lookup does not apply to a record. + enum Fallback { + /// The prefix.dev chain, then the offline conda-forge verbatim + /// heuristic (assume the conda name is the PyPI name). + PrefixThenVerbatim, + /// Only the offline conda-forge verbatim heuristic. + Verbatim, + /// Nothing: a miss means the record gets no purls. Used for + /// `Replace` mappings, which are exclusive. + None, + } + + let project_defined_mode = project_defined_mappings + .as_ref() + .and_then(|mapping| mapping.mode_for_record(record)); + + // Consult the primary source for this record and determine which + // fallback applies when it has no answer. + let (mut outcome, fallback) = if matches!(derivation_mode, PurlDerivationMode::Disabled) { + (DerivationOutcome::NotApplicable, Fallback::Verbatim) + } else if let (Some(resolver), Some(mode)) = + (project_defined_mappings, project_defined_mode) { - project_defined_mappings - .derive_project_defined_purls(record, cache_metrics) - .await? + // A hit in the project-defined mapping (including an explicit + // "not a PyPI package" entry) is always final. + let project_outcome = match mode { + MappingMode::Disabled => DerivationOutcome::NotApplicable, + MappingMode::Replace | MappingMode::Extend => { + resolver + .derive_project_defined_purls(record, cache_metrics) + .await? + } + }; + let fallback = match mode { + MappingMode::Disabled => Fallback::Verbatim, + MappingMode::Replace => Fallback::None, + MappingMode::Extend => Fallback::PrefixThenVerbatim, + }; + (project_outcome, fallback) } else { - self.derive_purls_from_prefix(record, cache_metrics).await? + ( + DerivationOutcome::NotApplicable, + Fallback::PrefixThenVerbatim, + ) }; - // As a last resort use the verbatim conda-forge purls. - // But only if we're not using a project-defined mapping, since project-defined mapping - // should be exclusive - only packages explicitly in the mapping get purls. - if purls.is_not_applicable() - && !matches!(derivation_mode, PurlDerivationMode::ProjectDefined(_)) - { - return CondaForgeVerbatim - .derive_conda_forge_verbatim_purls(record, cache_metrics) - .await; + if outcome.is_not_applicable() { + outcome = match fallback { + Fallback::PrefixThenVerbatim => { + let prefix_outcome = + self.derive_purls_from_prefix(record, cache_metrics).await?; + if prefix_outcome.is_not_applicable() { + CondaForgeVerbatim + .derive_conda_forge_verbatim_purls(record, cache_metrics) + .await? + } else { + prefix_outcome + } + } + Fallback::Verbatim => { + CondaForgeVerbatim + .derive_conda_forge_verbatim_purls(record, cache_metrics) + .await? + } + Fallback::None => outcome, + }; } - Ok(purls) + Ok(outcome) } async fn derive_purls_from_prefix( diff --git a/crates/pypi_mapping/src/resolvers/project_defined_mapping.rs b/crates/pypi_mapping/src/resolvers/project_defined_mapping.rs index 3fb458f150..1e55d99714 100644 --- a/crates/pypi_mapping/src/resolvers/project_defined_mapping.rs +++ b/crates/pypi_mapping/src/resolvers/project_defined_mapping.rs @@ -2,19 +2,28 @@ use async_once_cell::OnceCell as AsyncCell; use miette::{IntoDiagnostic, WrapErr}; use rattler_conda_types::RepoDataRecord; use rattler_networking::LazyClient; -use std::path::Path; +use std::{ + path::{Path, PathBuf}, + time::{Duration, SystemTime}, +}; use url::Url; use crate::{ - CacheMetrics, CompressedMapping, MappingByChannel, MappingError, MappingMap, - ProjectDefinedMappingLocation, PurlDerivationSource, channel::normalize_channel, - derivation::DerivationOutcome, purl::pypi_purl, + CacheMetrics, CompressedMapping, MappingByChannel, MappingError, MappingMap, MappingMode, + ProjectDefinedMappingLocation, PurlDerivationSource, ResolvedChannelMapping, + channel::normalize_channel, derivation::DerivationOutcome, purl::pypi_purl, }; -/// Struct with a mapping of channel names to their respective mapping locations -/// location could be a remote url or local file. +/// Subdirectory of the conda-pypi mapping cache that holds TTL-cached +/// project-defined mappings. +const TTL_CACHE_SUBDIR: &str = "project-defined"; + +/// Struct with a mapping of channel names to their respective mapping +/// configuration: one or more sources (remote url, local file or in-memory +/// entries) and the mode that determines how the mapping interacts with the +/// default prefix.dev chain. /// -/// This struct caches the mapping internally. +/// This struct caches the fetched mapping internally. #[derive(Debug)] pub struct ProjectDefinedMapping { pub mapping: MappingMap, @@ -30,41 +39,51 @@ impl ProjectDefinedMapping { } } - /// Fetch the project-defined mapping from the server or load from the local + /// Fetch the project-defined mapping from the server or load from the + /// local filesystem. Each channel's sources are merged in order: entries + /// from later sources override entries from earlier ones. pub async fn fetch_project_defined_mapping( &self, client: &LazyClient, + cache_dir: &Path, ) -> miette::Result { self.mapping_value .get_or_try_init(async { let mut mapping_url_to_name: MappingByChannel = Default::default(); - for (name, url) in self.mapping.iter() { - // Fetch the mapping from the server or from the local - - match url { - ProjectDefinedMappingLocation::Url(url) => { - let mapping_by_name = match url.scheme() { - "file" => { - let file_path = url.to_file_path().map_err(|_| { - miette::miette!("{} is not a valid file url", url) - })?; - fetch_mapping_from_path(&file_path)? + for (name, channel_mapping) in self.mapping.iter() { + let mut merged = CompressedMapping::default(); + for source in &channel_mapping.sources { + let mapping_by_name = match source { + ProjectDefinedMappingLocation::Url { url, cache_ttl } => { + match (url.scheme(), cache_ttl) { + ("file", _) => { + let file_path = url.to_file_path().map_err(|_| { + miette::miette!("{} is not a valid file url", url) + })?; + fetch_mapping_from_path(&file_path)? + } + (_, Some(ttl)) => { + fetch_mapping_with_ttl(client, url, *ttl, cache_dir).await? + } + (_, None) => fetch_mapping_from_url(client, url).await?, } - _ => fetch_mapping_from_url(client, url).await?, - }; - - mapping_url_to_name.insert(name.to_string(), mapping_by_name); - } - ProjectDefinedMappingLocation::Path(path) => { - let mapping_by_name = fetch_mapping_from_path(path)?; - - mapping_url_to_name.insert(name.to_string(), mapping_by_name); - } - ProjectDefinedMappingLocation::InMemory(mapping) => { - mapping_url_to_name.insert(name.to_string(), mapping.clone()); - } + } + ProjectDefinedMappingLocation::Path(path) => { + fetch_mapping_from_path(path)? + } + ProjectDefinedMappingLocation::InMemory(mapping) => mapping.clone(), + }; + merged.extend(mapping_by_name); } + + mapping_url_to_name.insert( + name.to_string(), + ResolvedChannelMapping { + mapping: merged, + mode: channel_mapping.mode, + }, + ); } Ok(mapping_url_to_name) @@ -74,6 +93,13 @@ impl ProjectDefinedMapping { } } +/// Help text for failures to fetch a *user-configured* mapping location. +/// Unlike [`crate::MAPPING_OFFLINE_HELP`] this must not suggest "point at +/// your own mapping" — the user already did. +const LOCATION_FETCH_HELP: &str = "Check that the `location` URL in your `conda-pypi-map` entry is correct and reachable. \ + To tolerate temporary outages, add a `cache-ttl` so a previously fetched copy can be \ + reused, or use a local file instead."; + async fn fetch_mapping_from_url( client: &LazyClient, url: &Url, @@ -84,15 +110,18 @@ async fn fetch_mapping_from_url( .send() .await .into_diagnostic() - .context(format!( - "failed to download pypi mapping from {} location", + .wrap_err(miette::diagnostic!( + help = LOCATION_FETCH_HELP, + "failed to download conda-pypi mapping from {}", url.as_str() ))?; if !response.status().is_success() { return Err(miette::miette!( - "Could not request mapping located at {:?}", - url.as_str() + help = LOCATION_FETCH_HELP, + "fetching the conda-pypi mapping from {} returned status {}", + url.as_str(), + response.status() )); } @@ -103,6 +132,99 @@ async fn fetch_mapping_from_url( Ok(mapping_by_name) } +/// Fetch a mapping from a url, caching it on disk for `ttl`. +/// +/// A cached copy younger than `ttl` is used without touching the network. +/// When the refetch of an expired copy fails, the stale copy is used with a +/// warning so that solves keep working offline. +/// +/// This is a small mtime-based file cache (the same pattern as the reverse +/// pypi-to-conda mapping cache in pixi-build-python) rather than the +/// `http-cache` middleware that already wraps the client: the middleware's +/// freshness is driven by server cache headers, its `max_ttl` is client-global +/// while `cache-ttl` is configured per mapping entry, and it has no +/// use-stale-on-error behavior. +async fn fetch_mapping_with_ttl( + client: &LazyClient, + url: &Url, + ttl: Duration, + cache_dir: &Path, +) -> miette::Result { + let cache_path = ttl_cache_path(cache_dir, url); + + if let Some((mapping, age)) = read_ttl_cache(&cache_path) + && age < ttl + { + return Ok(mapping); + } + + match fetch_mapping_from_url(client, url).await { + Ok(mapping) => { + write_ttl_cache(&cache_path, &mapping); + Ok(mapping) + } + Err(err) => { + // Fall back to a stale cached copy if we have one. + if let Some((mapping, age)) = read_ttl_cache(&cache_path) { + tracing::warn!( + "could not refresh conda-pypi mapping from {url}; using a cached copy that is {} old", + humantime::format_duration(Duration::from_secs(age.as_secs())) + ); + Ok(mapping) + } else { + Err(err) + } + } + } +} + +/// The on-disk location of the TTL cache for a mapping url. +fn ttl_cache_path(cache_dir: &Path, url: &Url) -> PathBuf { + let hash = + rattler_digest::compute_bytes_digest::(url.as_str().as_bytes()); + cache_dir + .join(TTL_CACHE_SUBDIR) + .join(format!("{hash:x}.json")) +} + +/// Read a cached mapping and its age. Returns `None` if there is no cached +/// copy or it cannot be parsed. +fn read_ttl_cache(cache_path: &Path) -> Option<(CompressedMapping, Duration)> { + let metadata = fs_err::metadata(cache_path).ok()?; + // A modification time in the future (clock skew, NTP corrections) is + // treated as age zero; returning `None` here would make a perfectly good + // cached copy invisible to both the freshness check and the stale + // fallback. + let age = metadata.modified().ok().map(|modified| { + SystemTime::now() + .duration_since(modified) + .unwrap_or(Duration::ZERO) + })?; + let content = fs_err::read_to_string(cache_path).ok()?; + let mapping = serde_json::from_str(&content).ok()?; + Some((mapping, age)) +} + +/// Write a mapping to the TTL cache. Failures are ignored; the cache is an +/// optimization. +fn write_ttl_cache(cache_path: &Path, mapping: &CompressedMapping) { + let Some(parent) = cache_path.parent() else { + return; + }; + let _ = fs_err::create_dir_all(parent); + let Ok(content) = serde_json::to_string(mapping) else { + return; + }; + // Write via a temporary file and rename so a concurrent reader never + // observes a partially written cache file. + let Ok(temp_file) = tempfile::NamedTempFile::new_in(parent) else { + return; + }; + if fs_err::write(temp_file.path(), content).is_ok() { + let _ = temp_file.persist(cache_path); + } +} + fn fetch_mapping_from_path(path: &Path) -> miette::Result { let file = fs_err::File::open(path) .into_diagnostic() @@ -118,7 +240,7 @@ fn fetch_mapping_from_path(path: &Path) -> miette::Result { Ok(mapping_by_name) } -/// THis is a client that uses a project-defined in-memory mapping to derive purls. +/// This is a client that uses a project-defined in-memory mapping to derive purls. #[derive(Default)] pub(crate) struct ProjectDefinedResolver { mapping: MappingByChannel, @@ -126,16 +248,18 @@ pub(crate) struct ProjectDefinedResolver { impl ProjectDefinedResolver { /// Returns the mapping associated with a channel. - fn get_channel_mapping(&self, channel: &str) -> Option<&CompressedMapping> { + fn get_channel_mapping(&self, channel: &str) -> Option<&ResolvedChannelMapping> { self.mapping.get(normalize_channel(channel)) } - /// Returns true if this mapping applies to the given record. - pub fn is_mapping_for_record(&self, record: &RepoDataRecord) -> bool { + /// Returns the mapping mode that applies to the given record, or `None` + /// if no project-defined mapping covers the record's channel. + pub fn mode_for_record(&self, record: &RepoDataRecord) -> Option { record .channel .as_ref() - .is_some_and(|channel| self.get_channel_mapping(channel).is_some()) + .and_then(|channel| self.get_channel_mapping(channel)) + .map(|mapping| mapping.mode) } } @@ -161,7 +285,10 @@ impl ProjectDefinedResolver { }; // Find the mapping for this particular record - match project_defined_mapping.get(record.package_record.name.as_normalized()) { + match project_defined_mapping + .mapping + .get(record.package_record.name.as_normalized()) + { // The record is in the mapping, and it has a pypi name Some(Some(mapped_name)) => Ok(DerivationOutcome::Purls(vec![pypi_purl( mapped_name.to_string(), @@ -178,3 +305,58 @@ impl ProjectDefinedResolver { } } } + +#[cfg(test)] +mod test { + use std::time::{Duration, SystemTime}; + + use super::{read_ttl_cache, write_ttl_cache}; + + fn write_cache_with_mtime(dir: &std::path::Path, age: i64) -> std::path::PathBuf { + let path = dir.join("mapping.json"); + write_ttl_cache( + &path, + &[("foo".to_string(), Some("bar".to_string()))] + .into_iter() + .collect(), + ); + let mtime = filetime::FileTime::from_system_time(if age >= 0 { + SystemTime::now() - Duration::from_secs(age as u64) + } else { + SystemTime::now() + Duration::from_secs((-age) as u64) + }); + filetime::set_file_mtime(&path, mtime).unwrap(); + path + } + + #[test] + fn test_read_ttl_cache_reports_age() { + let dir = tempfile::tempdir().unwrap(); + let path = write_cache_with_mtime(dir.path(), 7200); + let (mapping, age) = read_ttl_cache(&path).expect("cache should be readable"); + assert_eq!(mapping["foo"], Some("bar".to_string())); + // Allow some slack for slow filesystems. + assert!(age >= Duration::from_secs(7100) && age < Duration::from_secs(7300)); + } + + #[test] + fn test_read_ttl_cache_future_mtime_is_age_zero() { + // A cache file with a modification time in the future (clock skew) + // must still be readable, with age zero, so that both the freshness + // check and the stale fallback can use it. + let dir = tempfile::tempdir().unwrap(); + let path = write_cache_with_mtime(dir.path(), -3600); + let (_, age) = read_ttl_cache(&path).expect("future-dated cache should be readable"); + assert_eq!(age, Duration::ZERO); + } + + #[test] + fn test_read_ttl_cache_missing_or_invalid() { + let dir = tempfile::tempdir().unwrap(); + assert!(read_ttl_cache(&dir.path().join("missing.json")).is_none()); + + let corrupt = dir.path().join("corrupt.json"); + fs_err::write(&corrupt, "not json").unwrap(); + assert!(read_ttl_cache(&corrupt).is_none()); + } +} diff --git a/docs/build/backends/pixi-build-python.md b/docs/build/backends/pixi-build-python.md index bca6456f33..c7fe599095 100644 --- a/docs/build/backends/pixi-build-python.md +++ b/docs/build/backends/pixi-build-python.md @@ -346,6 +346,36 @@ ignore-pypi-mapping = true # Disable mapping on Windows only # Result for win-64: true ``` +### `pypi-conda-map` + +- **Type**: `Map` +- **Default**: not set +- **Target Merge Behavior**: `Merge` - Platform-specific entries override or extend base entries per key + +User-defined overrides for the PyPI-to-conda name mapping, keyed by PyPI package name. +A string value maps the package to that conda name; `false` silently drops the dependency from the generated recipe. +Entries are consulted before the remote mapping service, so they never require network access; packages not in the map fall back to the service as usual. + +Only used when `ignore-pypi-mapping = false`; otherwise it has no effect and a warning is logged. +The overrides apply to both mapping passes: `project.dependencies` (run dependencies) and `build-system.requires` (host dependencies). + +```toml +[package.build.config] +ignore-pypi-mapping = false +pypi-conda-map = { torch = "pytorch", my-internal-pkg = false } +``` + +Per-platform entries merge with the base map key-by-key: + +```toml +[package.build.config] +pypi-conda-map = { torch = "pytorch" } + +[package.build.target.linux-64.config] +pypi-conda-map = { nvidia-cublas-cu12 = false } +# Result for linux-64: { torch = "pytorch", nvidia-cublas-cu12 = false } +``` + ## Automatic PyPI Dependency Mapping The Python backend can automatically map PyPI dependencies from your `pyproject.toml` to their corresponding conda packages. @@ -366,7 +396,7 @@ The backend reads dependencies from two sources in your `pyproject.toml`: 1. **`project.dependencies`** → Added to conda **run** dependencies 2. **`build-system.requires`** → Added to conda **host** dependencies -For each PyPI package, the backend queries a mapping service to find the corresponding conda-forge package name. The mapping is cached locally for 24 hours to improve performance. +For each PyPI package, the backend first consults the user-defined [`pypi-conda-map`](#pypi-conda-map) overrides, and then queries a mapping service to find the corresponding conda-forge package name. The mapping is cached locally for 24 hours to improve performance. ### Example @@ -408,7 +438,7 @@ This allows you to: - **Environment markers** (e.g., `requests>=2.28; python_version >= "3.8"`) are only partially supported. At the moment, only `platform_system`, `os_name`, `platform_machine` and `sys_platforms` are currently checked. - **URL-based dependencies** (e.g., `package @ https://...`) are skipped -- Packages without a conda-forge mapping are logged as warnings and skipped +- Packages without a conda-forge mapping are logged as warnings and skipped; use [`pypi-conda-map`](#pypi-conda-map) to map them explicitly or drop them silently with `false` ## Build Process diff --git a/docs/concepts/conda_pypi.md b/docs/concepts/conda_pypi.md index 8d4e71757f..87ac16318e 100644 --- a/docs/concepts/conda_pypi.md +++ b/docs/concepts/conda_pypi.md @@ -105,6 +105,43 @@ Then, since `numpy` is not specified as a conda dependency, Pixi will resolve th To override or change the mapping of conda packages to PyPI packages, you can use the [`conda-pypi-map`](../reference/pixi_manifest.md#conda-pypi-map-optional) field in the `pixi.toml` file. +### Overriding the name mapping + +`conda-pypi-map` layers your entries *on top of* the default mapping: for each package pixi first consults your entries, and only falls back to the default mapping (and finally the "conda-forge name equals PyPI name" heuristic) when your mapping does not mention the package. +This means fixing a single incorrectly mapped package is a one-liner: + +```toml title="pixi.toml" +[workspace.conda-pypi-map] +conda-forge = { mapping = { pytorch = "torch" } } +``` + +If you want your mapping to be *exclusive* — only packages you list are treated as PyPI packages — use `mode = "replace"`: + +```toml title="pixi.toml" +[workspace.conda-pypi-map] +conda-forge = { location = "full-mapping.json", mode = "replace" } +``` + +### Offline and firewall-restricted environments + +The default mapping is fetched from `conda-mapping.prefix.dev`. +If that host is unreachable in your environment, you have several options: + +- `conda-pypi-map = false` disables all mapping lookups. Conda-forge packages are still assumed to be the PyPI package of the same name, which requires no network access. +- ` = false` disables lookups for a single channel. +- `mode = "replace"` with your own mapping file avoids network lookups for that channel entirely. +- `cache-ttl = "24h"` on a URL location caches the fetched mapping on disk and re-fetches it at most once per TTL; if the re-fetch fails, the cached copy is used. Note that the cache must be populated once (one successful fetch) before this protects you offline. + +For example, you can pin the full conda-forge name mapping that `parselmouth` publishes (the same data the default mapping is built from) and refresh it at most once a day: + +```toml title="pixi.toml" +[workspace.conda-pypi-map] +conda-forge = { location = "https://raw.githubusercontent.com/prefix-dev/parselmouth/main/files/compressed_mapping.json", mode = "replace", cache-ttl = "24h" } +``` + +!!! note + Use the `raw.githubusercontent.com` URL — the regular `github.com/.../blob/...` page serves HTML, not JSON. + ### PyPI overrides vs conda constraints PyPI's [`pypi-options.dependency-overrides`](../advanced/override.md) diff --git a/docs/reference/pixi_manifest.md b/docs/reference/pixi_manifest.md index bbe0fa9e52..c1fa424b4d 100644 --- a/docs/reference/pixi_manifest.md +++ b/docs/reference/pixi_manifest.md @@ -196,32 +196,58 @@ URL of the workspace documentation. ### `conda-pypi-map` (optional) -Mapping of channel name or URL to location of mapping that can be URL/Path. -Mapping should be structured in `json` format where `conda_name`: `pypi_package_name`. -Example: +Per-channel overrides for the conda to PyPI name mapping that pixi uses to detect which conda packages satisfy PyPI dependencies. +Each entry maps a channel name or URL to either a mapping location (URL or path), a table, or `false`: + +```toml +[workspace.conda-pypi-map] +# Additive overlay from a file: your entries take priority, anything not +# listed falls back to the default mapping. +robostack = "local/robostack_mapping.json" +# Inline entries, no file needed. `false` means "not a PyPI package". +conda-forge = { mode = "extend", mapping = { pytorch = "torch", not-on-pypi = false } } +# Exclusive mapping: ONLY packages in this file get mapped, no default mapping. +my-mirror = { location = "https://example.com/mapping.json", mode = "replace" } +# Re-fetch a remote mapping at most once a day; a cached copy is used otherwise. +my-company = { location = "https://internal.example.com/map.json", cache-ttl = "24h" } +# Disable mapping lookups for this channel entirely. +internal = false +``` + +Mapping files are structured in `json` format with `conda_name: pypi_package_name` entries, where `null` marks a package as not available on PyPI: ```json title="local/robostack_mapping.json" { "jupyter-ros": "my-name-from-mapping", - "boltons": "boltons-pypi" + "boltons": "boltons-pypi", + "not-on-pypi": null } ``` -If `conda-forge` is not present in `conda-pypi-map` `pixi` will use `prefix.dev` mapping for it. +The table form accepts: -```toml -conda-pypi-map = { conda-forge = "https://example.com/mapping", "https://repo.prefix.dev/robostack" = "local/robostack_mapping.json"} -``` +- `location`: URL or path of a mapping `json` file. Relative paths are resolved against the workspace root. +- `mapping`: inline `conda_name = "pypi_name"` entries. A value of `false` marks the package as not available on PyPI. Inline entries override entries from `location`. +- `mode`: `"extend"` (default) or `"replace"`. With `extend`, your entries are consulted first and anything not listed falls back to the default [prefix.dev mapping](https://conda-mapping.prefix.dev/). With `replace`, only packages listed in your mapping are considered PyPI packages; no network lookups happen for that channel. +- `cache-ttl`: a duration like `"24h"` or `"7d"`. The mapping fetched from a `location` URL is cached on disk and only re-fetched once it is older than this. If a re-fetch fails (e.g. offline), the stale cached copy is used with a warning; if no cached copy exists yet, the failure is an error. Only valid for `http(s)` locations. -It is also possible to disable fetching external mpping by adding an empty map to the list +To disable the mapping, either per channel or entirely: ```toml -conda-pypi-map = { conda-forge = "map.json" } +[workspace] +conda-pypi-map = false # disable for all channels +# or +conda-pypi-map = { conda-forge = false } # disable for one channel ``` -```json title="map.json" -{} -``` +Even with the mapping disabled, conda-forge packages are still assumed to be the PyPI package of the same name (an offline heuristic that requires no network access). + +!!! warning "Behavior change" + Bare location strings (`conda-forge = "mapping.json"`) used to be *exclusive*: only packages in your file were mapped. + They are now *additive* (`mode = "extend"`). To restore the old behavior, use the table form with `mode = "replace"`. + The previous idiom of disabling the mapping with an empty map (`conda-pypi-map = {}`) is deprecated; use `conda-pypi-map = false` instead. + Additionally, configuring a mapping for one channel no longer suppresses the conda-forge name heuristic for channels that are *not* listed in `conda-pypi-map`; unlisted channels now behave exactly as if no mapping were configured. + To turn lookups off for a specific channel, add ` = false`. ### `channel-priority` (optional) diff --git a/schema/examples/invalid/bad_conda_pypi_map.toml b/schema/examples/invalid/bad_conda_pypi_map.toml new file mode 100644 index 0000000000..95d5f84398 --- /dev/null +++ b/schema/examples/invalid/bad_conda_pypi_map.toml @@ -0,0 +1,6 @@ +[project] +name = "bad-conda-pypi-map" +channels = ["conda-forge"] +platforms = ["linux-64"] +# `true` is not a valid channel entry; only a string, a table or `false` is. +conda-pypi-map = { conda-forge = true } diff --git a/schema/examples/invalid/bad_conda_pypi_map_mode.toml b/schema/examples/invalid/bad_conda_pypi_map_mode.toml new file mode 100644 index 0000000000..d57ceb39fa --- /dev/null +++ b/schema/examples/invalid/bad_conda_pypi_map_mode.toml @@ -0,0 +1,7 @@ +[project] +name = "bad-conda-pypi-map-mode" +channels = ["conda-forge"] +platforms = ["linux-64"] +# `mode` only accepts `extend` or `replace`, and inline mapping values must +# be a string or `false`. +conda-pypi-map = { conda-forge = { location = "mapping.json", mode = "bogus", mapping = { x = true } } } diff --git a/schema/examples/valid/full.toml b/schema/examples/valid/full.toml index 46e32c6aab..c15df03dd9 100644 --- a/schema/examples/valid/full.toml +++ b/schema/examples/valid/full.toml @@ -4,7 +4,7 @@ authors = ["Author "] channel-priority = "strict" channels = ["stable"] -conda-pypi-map = { "robostack" = "robostack_mapping.json", "conda-forge" = "https://repo.prefix.dev/conda-forge" } +conda-pypi-map = { "robostack" = "robostack_mapping.json", "conda-forge" = { location = "https://repo.prefix.dev/conda-forge", mode = "replace", cache-ttl = "24h" }, "pytorch" = { mode = "extend", mapping = { pytorch = "torch", not-on-pypi = false } }, "internal" = false } description = "A project" documentation = "https://docs.project.com" homepage = "https://project.com" diff --git a/schema/model.py b/schema/model.py index 868eeb33ef..bb55c30a52 100644 --- a/schema/model.py +++ b/schema/model.py @@ -200,6 +200,30 @@ class ChannelInlineTable(StrictBaseModel): Channel = ChannelName | ChannelInlineTable +class CondaPypiMapTable(StrictBaseModel): + """The mapping configuration for one channel in `conda-pypi-map`.""" + + location: AnyHttpUrl | NonEmptyStr | None = Field( + None, description="The URL or path to a mapping file with `conda_name: pypi_name` entries" + ) + mapping: dict[NonEmptyStr, NonEmptyStr | Literal[False]] | None = Field( + None, + description="Inline `conda_name: pypi_name` entries; `false` marks a package as not available on PyPI. Inline entries override entries from `location`.", + ) + mode: Literal["extend", "replace"] | None = Field( + None, + description="How the mapping interacts with the default mapping: `extend` (default) overlays it, `replace` makes it exclusive", + ) + cache_ttl: NonEmptyStr | None = Field( + None, + description='How long a mapping fetched from a URL may be reused before it is re-fetched (e.g. `"24h"`, `"7d"`). Only valid for http(s) locations.', + ) + + +CondaPypiMapEntry = AnyHttpUrl | NonEmptyStr | Literal[False] | CondaPypiMapTable +CondaPypiMap = dict[ChannelName, CondaPypiMapEntry] | Literal[False] + + class ChannelPriority(str, Enum): """The priority of the channel.""" @@ -296,8 +320,9 @@ class Workspace(StrictBaseModel): documentation: AnyHttpUrl | None = Field( None, description="The URL of the documentation of the project" ) - conda_pypi_map: dict[ChannelName, AnyHttpUrl | NonEmptyStr] | None = Field( - None, description="The `conda` to PyPI mapping configuration" + conda_pypi_map: CondaPypiMap | None = Field( + None, + description="The `conda` to PyPI mapping configuration; `false` disables the mapping entirely", ) pypi_options: PyPIOptions | None = Field( None, description="Options related to PyPI indexes for this project" diff --git a/schema/pyproject/partial-pixi.json b/schema/pyproject/partial-pixi.json index b2a487ecf2..f2fbc61512 100644 --- a/schema/pyproject/partial-pixi.json +++ b/schema/pyproject/partial-pixi.json @@ -657,6 +657,64 @@ "strict" ] }, + "CondaPypiMapTable": { + "title": "CondaPypiMapTable", + "description": "The mapping configuration for one channel in `conda-pypi-map`.", + "type": "object", + "additionalProperties": false, + "properties": { + "cache-ttl": { + "title": "Cache-Ttl", + "description": "How long a mapping fetched from a URL may be reused before it is re-fetched (e.g. `\"24h\"`, `\"7d\"`). Only valid for http(s) locations.", + "type": "string", + "minLength": 1 + }, + "location": { + "title": "Location", + "description": "The URL or path to a mapping file with `conda_name: pypi_name` entries", + "anyOf": [ + { + "type": "string", + "format": "uri", + "minLength": 1 + }, + { + "type": "string", + "minLength": 1 + } + ] + }, + "mapping": { + "title": "Mapping", + "description": "Inline `conda_name: pypi_name` entries; `false` marks a package as not available on PyPI. Inline entries override entries from `location`.", + "type": "object", + "additionalProperties": { + "anyOf": [ + { + "type": "string", + "minLength": 1 + }, + { + "type": "boolean", + "const": false + } + ] + }, + "propertyNames": { + "minLength": 1 + } + }, + "mode": { + "title": "Mode", + "description": "How the mapping interacts with the default mapping: `extend` (default) overlays it, `replace` makes it exclusive", + "type": "string", + "enum": [ + "extend", + "replace" + ] + } + } + }, "DependsOn": { "title": "DependsOn", "description": "The dependencies of a task.", @@ -3083,21 +3141,36 @@ }, "conda-pypi-map": { "title": "Conda-Pypi-Map", - "description": "The `conda` to PyPI mapping configuration", - "type": "object", - "additionalProperties": { - "anyOf": [ - { - "type": "string", - "format": "uri", - "minLength": 1 - }, - { - "type": "string", - "minLength": 1 + "description": "The `conda` to PyPI mapping configuration; `false` disables the mapping entirely", + "anyOf": [ + { + "type": "object", + "additionalProperties": { + "anyOf": [ + { + "type": "string", + "format": "uri", + "minLength": 1 + }, + { + "type": "string", + "minLength": 1 + }, + { + "type": "boolean", + "const": false + }, + { + "$ref": "#/$defs/CondaPypiMapTable" + } + ] } - ] - } + }, + { + "type": "boolean", + "const": false + } + ] }, "dependencies": { "title": "Dependencies", diff --git a/schema/pyproject/schema.json b/schema/pyproject/schema.json index d105b79001..59a12c0152 100644 --- a/schema/pyproject/schema.json +++ b/schema/pyproject/schema.json @@ -400,6 +400,64 @@ "strict" ] }, + "CondaPypiMapTable": { + "title": "CondaPypiMapTable", + "description": "The mapping configuration for one channel in `conda-pypi-map`.", + "type": "object", + "additionalProperties": false, + "properties": { + "cache-ttl": { + "title": "Cache-Ttl", + "description": "How long a mapping fetched from a URL may be reused before it is re-fetched (e.g. `\"24h\"`, `\"7d\"`). Only valid for http(s) locations.", + "type": "string", + "minLength": 1 + }, + "location": { + "title": "Location", + "description": "The URL or path to a mapping file with `conda_name: pypi_name` entries", + "anyOf": [ + { + "type": "string", + "format": "uri", + "minLength": 1 + }, + { + "type": "string", + "minLength": 1 + } + ] + }, + "mapping": { + "title": "Mapping", + "description": "Inline `conda_name: pypi_name` entries; `false` marks a package as not available on PyPI. Inline entries override entries from `location`.", + "type": "object", + "additionalProperties": { + "anyOf": [ + { + "type": "string", + "minLength": 1 + }, + { + "type": "boolean", + "const": false + } + ] + }, + "propertyNames": { + "minLength": 1 + } + }, + "mode": { + "title": "Mode", + "description": "How the mapping interacts with the default mapping: `extend` (default) overlays it, `replace` makes it exclusive", + "type": "string", + "enum": [ + "extend", + "replace" + ] + } + } + }, "DependsOn": { "title": "DependsOn", "description": "The dependencies of a task.", @@ -3121,21 +3179,36 @@ }, "conda-pypi-map": { "title": "Conda-Pypi-Map", - "description": "The `conda` to PyPI mapping configuration", - "type": "object", - "additionalProperties": { - "anyOf": [ - { - "type": "string", - "format": "uri", - "minLength": 1 - }, - { - "type": "string", - "minLength": 1 + "description": "The `conda` to PyPI mapping configuration; `false` disables the mapping entirely", + "anyOf": [ + { + "type": "object", + "additionalProperties": { + "anyOf": [ + { + "type": "string", + "format": "uri", + "minLength": 1 + }, + { + "type": "string", + "minLength": 1 + }, + { + "type": "boolean", + "const": false + }, + { + "$ref": "#/$defs/CondaPypiMapTable" + } + ] } - ] - } + }, + { + "type": "boolean", + "const": false + } + ] }, "dependencies": { "title": "Dependencies", diff --git a/schema/schema.json b/schema/schema.json index 07fe92b5c6..e756bfb6ab 100644 --- a/schema/schema.json +++ b/schema/schema.json @@ -681,6 +681,64 @@ "strict" ] }, + "CondaPypiMapTable": { + "title": "CondaPypiMapTable", + "description": "The mapping configuration for one channel in `conda-pypi-map`.", + "type": "object", + "additionalProperties": false, + "properties": { + "cache-ttl": { + "title": "Cache-Ttl", + "description": "How long a mapping fetched from a URL may be reused before it is re-fetched (e.g. `\"24h\"`, `\"7d\"`). Only valid for http(s) locations.", + "type": "string", + "minLength": 1 + }, + "location": { + "title": "Location", + "description": "The URL or path to a mapping file with `conda_name: pypi_name` entries", + "anyOf": [ + { + "type": "string", + "format": "uri", + "minLength": 1 + }, + { + "type": "string", + "minLength": 1 + } + ] + }, + "mapping": { + "title": "Mapping", + "description": "Inline `conda_name: pypi_name` entries; `false` marks a package as not available on PyPI. Inline entries override entries from `location`.", + "type": "object", + "additionalProperties": { + "anyOf": [ + { + "type": "string", + "minLength": 1 + }, + { + "type": "boolean", + "const": false + } + ] + }, + "propertyNames": { + "minLength": 1 + } + }, + "mode": { + "title": "Mode", + "description": "How the mapping interacts with the default mapping: `extend` (default) overlays it, `replace` makes it exclusive", + "type": "string", + "enum": [ + "extend", + "replace" + ] + } + } + }, "DependsOn": { "title": "DependsOn", "description": "The dependencies of a task.", @@ -3107,21 +3165,36 @@ }, "conda-pypi-map": { "title": "Conda-Pypi-Map", - "description": "The `conda` to PyPI mapping configuration", - "type": "object", - "additionalProperties": { - "anyOf": [ - { - "type": "string", - "format": "uri", - "minLength": 1 - }, - { - "type": "string", - "minLength": 1 + "description": "The `conda` to PyPI mapping configuration; `false` disables the mapping entirely", + "anyOf": [ + { + "type": "object", + "additionalProperties": { + "anyOf": [ + { + "type": "string", + "format": "uri", + "minLength": 1 + }, + { + "type": "string", + "minLength": 1 + }, + { + "type": "boolean", + "const": false + }, + { + "$ref": "#/$defs/CondaPypiMapTable" + } + ] } - ] - } + }, + { + "type": "boolean", + "const": false + } + ] }, "dependencies": { "title": "Dependencies",