From 4eb4cbcaa1f017cf55c7c698566a54b190eb0452 Mon Sep 17 00:00:00 2001 From: Tim de Jager Date: Wed, 10 Jun 2026 11:51:11 +0200 Subject: [PATCH 01/10] refactor(pypi_mapping): per-channel mapping modes with layered fallback and TTL cache --- Cargo.lock | 1 + .../integration_rust/solve_group_tests.rs | 11 +- crates/pixi_core/src/workspace/mod.rs | 32 +++- crates/pypi_mapping/Cargo.toml | 1 + crates/pypi_mapping/src/derivation_mode.rs | 75 +++++++- crates/pypi_mapping/src/lib.rs | 73 ++++++-- .../src/resolvers/project_defined_mapping.rs | 177 ++++++++++++++---- 7 files changed, 300 insertions(+), 70 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2d93f83ae7..575f1b3a0a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7706,6 +7706,7 @@ dependencies = [ "fs-err", "futures", "http-cache-reqwest", + "humantime", "itertools 0.14.0", "miette 7.6.0", "pep440_rs", diff --git a/crates/pixi/tests/integration_rust/solve_group_tests.rs b/crates/pixi/tests/integration_rust/solve_group_tests.rs index 4c18de0dec..380d1617c0 100644 --- a/crates/pixi/tests/integration_rust/solve_group_tests.rs +++ b/crates/pixi/tests/integration_rust/solve_group_tests.rs @@ -6,7 +6,8 @@ use std::{ }; use pypi_mapping::{ - self, ProjectDefinedMapping, ProjectDefinedMappingLocation, PurlDerivationMode, + self, ProjectDefinedChannelMapping, ProjectDefinedMapping, ProjectDefinedMappingLocation, + PurlDerivationMode, PurlDerivationSource, }; use rattler_conda_types::{PackageName, Platform, RepoDataRecord}; @@ -340,7 +341,9 @@ async fn test_purl_are_generated_using_custom_mapping() { HashMap::from([("foo-bar-car".to_owned(), Some("my-test-name".to_owned()))]); let source = HashMap::from([( "https://conda.anaconda.org/conda-forge".to_owned(), - ProjectDefinedMappingLocation::InMemory(compressed_mapping), + ProjectDefinedChannelMapping::replace(ProjectDefinedMappingLocation::InMemory( + compressed_mapping, + )), )]); let mapping_client = pypi_mapping::PurlDerivationClient::builder( @@ -1257,7 +1260,9 @@ async fn test_missing_mapping_file_error_includes_path() { let source = HashMap::from([( "https://conda.anaconda.org/conda-forge".to_owned(), - ProjectDefinedMappingLocation::Path(non_existent_path.to_path_buf()), + ProjectDefinedChannelMapping::replace(ProjectDefinedMappingLocation::Path( + non_existent_path.to_path_buf(), + )), )]); let foo_bar_package = Package::build("foo-bar-car", "2").finish(); diff --git a/crates/pixi_core/src/workspace/mod.rs b/crates/pixi_core/src/workspace/mod.rs index ab08564323..c352a541f5 100644 --- a/crates/pixi_core/src/workspace/mod.rs +++ b/crates/pixi_core/src/workspace/mod.rs @@ -53,7 +53,8 @@ use pixi_utils::{ variants::{VariantConfig, VariantValue}, }; use pypi_mapping::{ - ChannelName, ProjectDefinedMapping, ProjectDefinedMappingLocation, PurlDerivationMode, + ChannelName, ProjectDefinedChannelMapping, ProjectDefinedMapping, + ProjectDefinedMappingLocation, PurlDerivationMode, }; use rattler_conda_types::{ Channel, ChannelConfig, GenericVirtualPackage, MatchSpec, PackageName, Platform, Version, @@ -981,7 +982,10 @@ impl Workspace { || mapping_location.starts_with("file://") { match Url::parse(mapping_location) { - Ok(url) => ProjectDefinedMappingLocation::Url(url), + Ok(url) => ProjectDefinedMappingLocation::Url { + url, + cache_ttl: None, + }, Err(err) => { return Err(err).into_diagnostic().context(format!( "Could not convert {mapping_location} to URL" @@ -1000,10 +1004,10 @@ impl Workspace { Ok(( channel.canonical_name().trim_end_matches('/').into(), - url_or_path, + ProjectDefinedChannelMapping::replace(url_or_path), )) }) - .collect::>>()?; + .collect::>>()?; Ok(PurlDerivationMode::ProjectDefined( ProjectDefinedMapping::new(mapping).into(), @@ -1598,7 +1602,21 @@ mod tests { let canonical_channel_name = canonical_name.trim_end_matches('/'); - assert_eq!(mapping.project_defined().unwrap().mapping.get(canonical_channel_name).unwrap(), &ProjectDefinedMappingLocation::Url(Url::parse("https://github.com/prefix-dev/parselmouth/blob/main/files/compressed_mapping.json").unwrap())); + assert_eq!( + mapping + .project_defined() + .unwrap() + .mapping + .get(canonical_channel_name) + .unwrap(), + &ProjectDefinedChannelMapping::replace(ProjectDefinedMappingLocation::Url { + url: Url::parse( + "https://github.com/prefix-dev/parselmouth/blob/main/files/compressed_mapping.json" + ) + .unwrap(), + cache_ttl: None, + }) + ); // Check url channel as map key let file_contents = r#" @@ -1626,12 +1644,12 @@ mod tests { .trim_end_matches('/') ) .unwrap(), - &ProjectDefinedMappingLocation::Path( + &ProjectDefinedChannelMapping::replace(ProjectDefinedMappingLocation::Path( workspace .channel_config() .root_dir .join(PathBuf::from("mapping.json")) - ) + )) ); } diff --git a/crates/pypi_mapping/Cargo.toml b/crates/pypi_mapping/Cargo.toml index 6bc3851d3a..1cfdc28ab0 100644 --- a/crates/pypi_mapping/Cargo.toml +++ b/crates/pypi_mapping/Cargo.toml @@ -15,6 +15,7 @@ dashmap = { workspace = true } fs-err = { workspace = true } futures = { workspace = true } http-cache-reqwest = { workspace = true } +humantime = { workspace = true } itertools = { workspace = true } miette = { workspace = true } pep440_rs = { workspace = true } diff --git a/crates/pypi_mapping/src/derivation_mode.rs b/crates/pypi_mapping/src/derivation_mode.rs index f5f49708a1..3f159d86c2 100644 --- a/crates/pypi_mapping/src/derivation_mode.rs +++ b/crates/pypi_mapping/src/derivation_mode.rs @@ -1,20 +1,82 @@ -use std::{collections::HashMap, path::PathBuf, sync::Arc}; +use std::{collections::HashMap, path::PathBuf, sync::Arc, time::Duration}; use url::Url; use crate::{CompressedMapping, ProjectDefinedMapping}; pub type ChannelName = String; -pub type MappingMap = HashMap; -pub type MappingByChannel = HashMap; +pub type MappingMap = HashMap; +pub type MappingByChannel = HashMap; #[derive(Debug, Clone, PartialEq, Eq)] pub enum ProjectDefinedMappingLocation { Path(PathBuf), - Url(Url), + Url { + url: Url, + /// When set, the fetched mapping is cached on disk and only + /// re-fetched once the cached copy is older than this duration. + cache_ttl: Option, + }, InMemory(CompressedMapping), } +/// How a project-defined channel mapping interacts with the default +/// prefix.dev derivation chain. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub enum MappingMode { + /// The mapping overlays the defaults: a hit (including an explicit "not a + /// PyPI package" entry) is final, a miss falls through to the prefix.dev + /// chain (hash mapping, compressed mapping, conda-forge verbatim + /// fallback). + #[default] + Extend, + /// The mapping is exclusive: only packages in the mapping get purls. No + /// prefix.dev lookups and no conda-forge verbatim fallback happen for + /// records from this channel. + Replace, + /// No purls are looked up for records from this channel, neither + /// project-defined nor prefix.dev. The offline conda-forge verbatim + /// fallback still applies. + Disabled, +} + +/// The project-defined mapping configuration for a single channel. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ProjectDefinedChannelMapping { + /// The mapping sources, merged in order: entries from later sources + /// override entries from earlier ones. + pub sources: Vec, + pub mode: MappingMode, +} + +impl ProjectDefinedChannelMapping { + pub fn new(sources: Vec, mode: MappingMode) -> Self { + Self { sources, mode } + } + + /// A single-source mapping that overlays the prefix.dev defaults. + pub fn extend(source: ProjectDefinedMappingLocation) -> Self { + Self::new(vec![source], MappingMode::Extend) + } + + /// A single-source mapping that replaces the prefix.dev defaults. + pub fn replace(source: ProjectDefinedMappingLocation) -> Self { + Self::new(vec![source], MappingMode::Replace) + } + + /// Disable purl lookups for this channel. + pub fn disabled() -> Self { + Self::new(Vec::new(), MappingMode::Disabled) + } +} + +/// A channel mapping with all its sources fetched and merged. +#[derive(Debug, Clone)] +pub struct ResolvedChannelMapping { + pub mapping: CompressedMapping, + pub mode: MappingMode, +} + /// User-selected mapping mode. /// /// This controls which resolver family [`crate::PurlDerivationClient`] uses. It is not @@ -22,7 +84,10 @@ pub enum ProjectDefinedMappingLocation { /// concrete resolver that produced an individual purl. #[derive(Debug, Clone)] pub enum PurlDerivationMode { - /// Use only project-defined per-channel mappings. + /// Use project-defined per-channel mappings. Depending on each channel's + /// [`MappingMode`] the prefix.dev mappings may still be used as a + /// fallback. Records from channels without a project-defined mapping use + /// the prefix.dev mappings. ProjectDefined(Arc), /// Use prefix.dev mappings: hash mapping first, then compressed mapping. Prefix, diff --git a/crates/pypi_mapping/src/lib.rs b/crates/pypi_mapping/src/lib.rs index cf06a3b502..fceaba1907 100644 --- a/crates/pypi_mapping/src/lib.rs +++ b/crates/pypi_mapping/src/lib.rs @@ -12,6 +12,11 @@ //! 3. [`PurlDerivationSource::PrefixCompressedMapping`] — prefix.dev compressed name mapping. //! 4. [`PurlDerivationSource::CondaForgeVerbatimFallback`] — conda-forge fallback that assumes //! the conda package name is the PyPI package name. +//! +//! A project-defined mapping carries a per-channel [`MappingMode`] that +//! determines how it interacts with the prefix.dev chain: `Extend` overlays +//! it (a miss falls through to prefix.dev), `Replace` is exclusive, and +//! `Disabled` turns lookups for that channel off entirely. use std::{ collections::{BTreeSet, HashMap}, @@ -23,7 +28,6 @@ use std::{ use futures::{StreamExt, stream::FuturesUnordered}; use http_cache_reqwest::{CACacheManager, Cache, CacheMode, HttpCache, HttpCacheOptions}; use itertools::Itertools; -use miette::IntoDiagnostic; use rattler_conda_types::{PackageUrl, RepoDataRecord}; use rattler_networking::LazyClient; use reqwest_middleware::ClientBuilder; @@ -41,7 +45,8 @@ pub mod resolvers; pub use channel::{is_conda_forge_record, is_conda_forge_url}; pub use derivation_mode::{ - ChannelName, MappingByChannel, MappingMap, ProjectDefinedMappingLocation, PurlDerivationMode, + ChannelName, MappingByChannel, MappingMap, MappingMode, ProjectDefinedChannelMapping, + ProjectDefinedMappingLocation, PurlDerivationMode, ResolvedChannelMapping, }; pub use metrics::CacheMetrics; pub use purl::PurlDerivationSource; @@ -61,7 +66,11 @@ pub type CompressedMapping = HashMap>; /// /// The resolver order depends on [`PurlDerivationMode`]: /// -/// - [`PurlDerivationMode::ProjectDefined`]: project-defined per-channel mapping only. +/// - [`PurlDerivationMode::ProjectDefined`]: project-defined per-channel mapping. How records +/// from a mapped channel interact with the prefix.dev chain depends on the channel's +/// [`MappingMode`]: `Extend` falls through to prefix.dev on a miss, `Replace` is exclusive +/// (no prefix.dev, no verbatim fallback), and `Disabled` skips all lookups. Records from +/// unmapped channels use the prefix.dev chain. /// - [`PurlDerivationMode::Prefix`]: prefix hash mapping, then prefix compressed mapping, /// then the conda-forge verbatim fallback. /// - [`PurlDerivationMode::Disabled`]: no project-defined or prefix mapping. The current behavior @@ -120,7 +129,7 @@ impl PurlDerivationClientBuilder { } } -#[derive(Debug, Error)] +#[derive(Debug, Error, miette::Diagnostic)] pub enum MappingError { #[error("failed to access conda-pypi mapping cache at '{path}'")] IoError { @@ -129,6 +138,12 @@ pub enum MappingError { path: PathBuf, }, #[error("failed to fetch conda-pypi mapping from remote source")] + #[diagnostic(help( + "If this host cannot be reached (e.g. behind a firewall), you can avoid the network \ + lookup: use `mode = \"replace\"` on the `conda-pypi-map` entry for the channel, disable \ + the channel's mapping with ` = false`, or disable the mapping entirely with \ + `conda-pypi-map = false`." + ))] Reqwest(#[source] reqwest_middleware::Error), } @@ -207,7 +222,7 @@ impl PurlDerivationClient { if let PurlDerivationMode::ProjectDefined(mapping_url) = derivation_mode { Some(ProjectDefinedResolver::from( mapping_url - .fetch_project_defined_mapping(&self.client) + .fetch_project_defined_mapping(&self.client, &self.cache_path) .await?, )) } else { @@ -259,7 +274,9 @@ impl PurlDerivationClient { let mut amended_records = 0; let mut total_records = 0; while let Some(next) = amend_futures.next().await { - let (record, derived_purls) = next.into_diagnostic()?; + // Use `Report::new` instead of `into_diagnostic` to preserve the + // diagnostic help text on `MappingError`. + let (record, derived_purls) = next.map_err(miette::Report::new)?; if let Some(derived_purls) = derived_purls.into_purls() { amend_purls(record, derived_purls); @@ -300,24 +317,48 @@ impl PurlDerivationClient { record: &RepoDataRecord, cache_metrics: &CacheMetrics, ) -> Result { + // Whether the conda-forge verbatim fallback is suppressed for this record. Only + // a `Replace` mapping is exclusive: packages not explicitly in the mapping must + // not get purls. + let mut suppress_verbatim_fallback = false; + + let project_defined_mode = project_defined_mappings + .as_ref() + .and_then(|mapping| mapping.mode_for_record(record)); + let purls = if matches!(derivation_mode, PurlDerivationMode::Disabled) { DerivationOutcome::NotApplicable - } else if let Some(project_defined_mappings) = - project_defined_mappings.filter(|mapping| mapping.is_mapping_for_record(record)) + } else if let (Some(resolver), Some(mode)) = + (project_defined_mappings, project_defined_mode) { - project_defined_mappings - .derive_project_defined_purls(record, cache_metrics) - .await? + match mode { + MappingMode::Disabled => DerivationOutcome::NotApplicable, + MappingMode::Replace => { + suppress_verbatim_fallback = true; + resolver + .derive_project_defined_purls(record, cache_metrics) + .await? + } + MappingMode::Extend => { + // A hit in the project-defined mapping (including an explicit "not a + // PyPI package" entry) is final; only a miss falls through to the + // prefix.dev chain. + let outcome = resolver + .derive_project_defined_purls(record, cache_metrics) + .await?; + if outcome.is_not_applicable() { + self.derive_purls_from_prefix(record, cache_metrics).await? + } else { + outcome + } + } + } } else { self.derive_purls_from_prefix(record, cache_metrics).await? }; // As a last resort use the verbatim conda-forge purls. - // But only if we're not using a project-defined mapping, since project-defined mapping - // should be exclusive - only packages explicitly in the mapping get purls. - if purls.is_not_applicable() - && !matches!(derivation_mode, PurlDerivationMode::ProjectDefined(_)) - { + if purls.is_not_applicable() && !suppress_verbatim_fallback { return CondaForgeVerbatim .derive_conda_forge_verbatim_purls(record, cache_metrics) .await; diff --git a/crates/pypi_mapping/src/resolvers/project_defined_mapping.rs b/crates/pypi_mapping/src/resolvers/project_defined_mapping.rs index 3fb458f150..e1ab3c26c1 100644 --- a/crates/pypi_mapping/src/resolvers/project_defined_mapping.rs +++ b/crates/pypi_mapping/src/resolvers/project_defined_mapping.rs @@ -2,19 +2,28 @@ use async_once_cell::OnceCell as AsyncCell; use miette::{IntoDiagnostic, WrapErr}; use rattler_conda_types::RepoDataRecord; use rattler_networking::LazyClient; -use std::path::Path; +use std::{ + path::{Path, PathBuf}, + time::{Duration, SystemTime}, +}; use url::Url; use crate::{ - CacheMetrics, CompressedMapping, MappingByChannel, MappingError, MappingMap, - ProjectDefinedMappingLocation, PurlDerivationSource, channel::normalize_channel, - derivation::DerivationOutcome, purl::pypi_purl, + CacheMetrics, CompressedMapping, MappingByChannel, MappingError, MappingMap, MappingMode, + ProjectDefinedMappingLocation, PurlDerivationSource, ResolvedChannelMapping, + channel::normalize_channel, derivation::DerivationOutcome, purl::pypi_purl, }; -/// Struct with a mapping of channel names to their respective mapping locations -/// location could be a remote url or local file. +/// Subdirectory of the conda-pypi mapping cache that holds TTL-cached +/// project-defined mappings. +const TTL_CACHE_SUBDIR: &str = "project-defined"; + +/// Struct with a mapping of channel names to their respective mapping +/// configuration: one or more sources (remote url, local file or in-memory +/// entries) and the mode that determines how the mapping interacts with the +/// default prefix.dev chain. /// -/// This struct caches the mapping internally. +/// This struct caches the fetched mapping internally. #[derive(Debug)] pub struct ProjectDefinedMapping { pub mapping: MappingMap, @@ -30,41 +39,51 @@ impl ProjectDefinedMapping { } } - /// Fetch the project-defined mapping from the server or load from the local + /// Fetch the project-defined mapping from the server or load from the + /// local filesystem. Each channel's sources are merged in order: entries + /// from later sources override entries from earlier ones. pub async fn fetch_project_defined_mapping( &self, client: &LazyClient, + cache_dir: &Path, ) -> miette::Result { self.mapping_value .get_or_try_init(async { let mut mapping_url_to_name: MappingByChannel = Default::default(); - for (name, url) in self.mapping.iter() { - // Fetch the mapping from the server or from the local - - match url { - ProjectDefinedMappingLocation::Url(url) => { - let mapping_by_name = match url.scheme() { - "file" => { - let file_path = url.to_file_path().map_err(|_| { - miette::miette!("{} is not a valid file url", url) - })?; - fetch_mapping_from_path(&file_path)? + for (name, channel_mapping) in self.mapping.iter() { + let mut merged = CompressedMapping::default(); + for source in &channel_mapping.sources { + let mapping_by_name = match source { + ProjectDefinedMappingLocation::Url { url, cache_ttl } => { + match (url.scheme(), cache_ttl) { + ("file", _) => { + let file_path = url.to_file_path().map_err(|_| { + miette::miette!("{} is not a valid file url", url) + })?; + fetch_mapping_from_path(&file_path)? + } + (_, Some(ttl)) => { + fetch_mapping_with_ttl(client, url, *ttl, cache_dir).await? + } + (_, None) => fetch_mapping_from_url(client, url).await?, } - _ => fetch_mapping_from_url(client, url).await?, - }; - - mapping_url_to_name.insert(name.to_string(), mapping_by_name); - } - ProjectDefinedMappingLocation::Path(path) => { - let mapping_by_name = fetch_mapping_from_path(path)?; - - mapping_url_to_name.insert(name.to_string(), mapping_by_name); - } - ProjectDefinedMappingLocation::InMemory(mapping) => { - mapping_url_to_name.insert(name.to_string(), mapping.clone()); - } + } + ProjectDefinedMappingLocation::Path(path) => { + fetch_mapping_from_path(path)? + } + ProjectDefinedMappingLocation::InMemory(mapping) => mapping.clone(), + }; + merged.extend(mapping_by_name); } + + mapping_url_to_name.insert( + name.to_string(), + ResolvedChannelMapping { + mapping: merged, + mode: channel_mapping.mode, + }, + ); } Ok(mapping_url_to_name) @@ -84,7 +103,10 @@ async fn fetch_mapping_from_url( .send() .await .into_diagnostic() - .context(format!( + .wrap_err(miette::diagnostic!( + help = "If this host cannot be reached (e.g. behind a firewall), consider \ + caching the mapping with `cache-ttl`, using a local file, or disabling \ + the mapping for this channel with ` = false`.", "failed to download pypi mapping from {} location", url.as_str() ))?; @@ -103,6 +125,78 @@ async fn fetch_mapping_from_url( Ok(mapping_by_name) } +/// Fetch a mapping from a url, caching it on disk for `ttl`. +/// +/// A cached copy younger than `ttl` is used without touching the network. +/// When the refetch of an expired copy fails, the stale copy is used with a +/// warning so that solves keep working offline. +async fn fetch_mapping_with_ttl( + client: &LazyClient, + url: &Url, + ttl: Duration, + cache_dir: &Path, +) -> miette::Result { + let cache_path = ttl_cache_path(cache_dir, url); + + if let Some((mapping, age)) = read_ttl_cache(&cache_path) + && age < ttl + { + return Ok(mapping); + } + + match fetch_mapping_from_url(client, url).await { + Ok(mapping) => { + write_ttl_cache(&cache_path, &mapping); + Ok(mapping) + } + Err(err) => { + // Fall back to a stale cached copy if we have one. + if let Some((mapping, age)) = read_ttl_cache(&cache_path) { + tracing::warn!( + "could not refresh conda-pypi mapping from {url}; using a cached copy that is {} old", + humantime::format_duration(Duration::from_secs(age.as_secs())) + ); + Ok(mapping) + } else { + Err(err) + } + } + } +} + +/// The on-disk location of the TTL cache for a mapping url. +fn ttl_cache_path(cache_dir: &Path, url: &Url) -> PathBuf { + let hash = + rattler_digest::compute_bytes_digest::(url.as_str().as_bytes()); + cache_dir + .join(TTL_CACHE_SUBDIR) + .join(format!("{hash:x}.json")) +} + +/// Read a cached mapping and its age. Returns `None` if there is no cached +/// copy or it cannot be parsed. +fn read_ttl_cache(cache_path: &Path) -> Option<(CompressedMapping, Duration)> { + let metadata = fs_err::metadata(cache_path).ok()?; + let age = metadata + .modified() + .ok() + .and_then(|modified| SystemTime::now().duration_since(modified).ok())?; + let content = fs_err::read_to_string(cache_path).ok()?; + let mapping = serde_json::from_str(&content).ok()?; + Some((mapping, age)) +} + +/// Write a mapping to the TTL cache. Failures are ignored; the cache is an +/// optimization. +fn write_ttl_cache(cache_path: &Path, mapping: &CompressedMapping) { + if let Some(parent) = cache_path.parent() { + let _ = fs_err::create_dir_all(parent); + } + if let Ok(content) = serde_json::to_string(mapping) { + let _ = fs_err::write(cache_path, content); + } +} + fn fetch_mapping_from_path(path: &Path) -> miette::Result { let file = fs_err::File::open(path) .into_diagnostic() @@ -118,7 +212,7 @@ fn fetch_mapping_from_path(path: &Path) -> miette::Result { Ok(mapping_by_name) } -/// THis is a client that uses a project-defined in-memory mapping to derive purls. +/// This is a client that uses a project-defined in-memory mapping to derive purls. #[derive(Default)] pub(crate) struct ProjectDefinedResolver { mapping: MappingByChannel, @@ -126,16 +220,18 @@ pub(crate) struct ProjectDefinedResolver { impl ProjectDefinedResolver { /// Returns the mapping associated with a channel. - fn get_channel_mapping(&self, channel: &str) -> Option<&CompressedMapping> { + fn get_channel_mapping(&self, channel: &str) -> Option<&ResolvedChannelMapping> { self.mapping.get(normalize_channel(channel)) } - /// Returns true if this mapping applies to the given record. - pub fn is_mapping_for_record(&self, record: &RepoDataRecord) -> bool { + /// Returns the mapping mode that applies to the given record, or `None` + /// if no project-defined mapping covers the record's channel. + pub fn mode_for_record(&self, record: &RepoDataRecord) -> Option { record .channel .as_ref() - .is_some_and(|channel| self.get_channel_mapping(channel).is_some()) + .and_then(|channel| self.get_channel_mapping(channel)) + .map(|mapping| mapping.mode) } } @@ -161,7 +257,10 @@ impl ProjectDefinedResolver { }; // Find the mapping for this particular record - match project_defined_mapping.get(record.package_record.name.as_normalized()) { + match project_defined_mapping + .mapping + .get(record.package_record.name.as_normalized()) + { // The record is in the mapping, and it has a pypi name Some(Some(mapped_name)) => Ok(DerivationOutcome::Purls(vec![pypi_purl( mapped_name.to_string(), From e58b0deeba12c7a19566f6ca9d626c2509a239d3 Mon Sep 17 00:00:00 2001 From: Tim de Jager Date: Wed, 10 Jun 2026 11:58:08 +0200 Subject: [PATCH 02/10] feat(manifest): conda-pypi-map string/table/false forms with extend/replace modes, inline mappings and cache-ttl - conda-pypi-map now accepts false (global disable), and per-channel values can be a bare string, false, or a table with location, inline mapping entries, mode = "extend"|"replace" and cache-ttl. - BREAKING: bare location strings now use the additive extend mode (overlay over the prefix.dev chain) instead of the exclusive replace mode. The old behavior is available via mode = "replace". - conda-pypi-map = {} is soft-deprecated in favor of false and emits a deprecation warning. - pixi_core wires the manifest entries into the per-channel mapping configuration; inline keys are lowercased to match normalized conda names, and cache-ttl is validated to require an http(s) location. --- Cargo.lock | 1 + crates/pixi_core/src/workspace/mod.rs | 257 ++++++++----- crates/pixi_manifest/Cargo.toml | 1 + crates/pixi_manifest/src/lib.rs | 5 +- .../pixi_manifest/src/toml/conda_pypi_map.rs | 345 ++++++++++++++++++ crates/pixi_manifest/src/toml/mod.rs | 1 + ...onda_pypi_map__test__bogus_mode_fails.snap | 11 + ...da_pypi_map__test__channel_true_fails.snap | 11 + ...pi_map__test__empty_entry_table_fails.snap | 11 + ...map__test__empty_map_parses_and_warns.snap | 6 + ...pi_map__test__inline_true_value_fails.snap | 11 + ...nda_pypi_map__test__invalid_ttl_fails.snap | 11 + ..._pypi_map__test__top_level_true_fails.snap | 11 + ...map__test__ttl_without_location_fails.snap | 11 + crates/pixi_manifest/src/toml/workspace.rs | 27 +- crates/pixi_manifest/src/workspace.rs | 70 +++- 16 files changed, 681 insertions(+), 109 deletions(-) create mode 100644 crates/pixi_manifest/src/toml/conda_pypi_map.rs create mode 100644 crates/pixi_manifest/src/toml/snapshots/pixi_manifest__toml__conda_pypi_map__test__bogus_mode_fails.snap create mode 100644 crates/pixi_manifest/src/toml/snapshots/pixi_manifest__toml__conda_pypi_map__test__channel_true_fails.snap create mode 100644 crates/pixi_manifest/src/toml/snapshots/pixi_manifest__toml__conda_pypi_map__test__empty_entry_table_fails.snap create mode 100644 crates/pixi_manifest/src/toml/snapshots/pixi_manifest__toml__conda_pypi_map__test__empty_map_parses_and_warns.snap create mode 100644 crates/pixi_manifest/src/toml/snapshots/pixi_manifest__toml__conda_pypi_map__test__inline_true_value_fails.snap create mode 100644 crates/pixi_manifest/src/toml/snapshots/pixi_manifest__toml__conda_pypi_map__test__invalid_ttl_fails.snap create mode 100644 crates/pixi_manifest/src/toml/snapshots/pixi_manifest__toml__conda_pypi_map__test__top_level_true_fails.snap create mode 100644 crates/pixi_manifest/src/toml/snapshots/pixi_manifest__toml__conda_pypi_map__test__ttl_without_location_fails.snap diff --git a/Cargo.lock b/Cargo.lock index 575f1b3a0a..577e2b75e2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6984,6 +6984,7 @@ dependencies = [ "dunce", "fancy_display", "fs-err", + "humantime", "indexmap 2.14.0", "insta", "itertools 0.14.0", diff --git a/crates/pixi_core/src/workspace/mod.rs b/crates/pixi_core/src/workspace/mod.rs index c352a541f5..cee8e68798 100644 --- a/crates/pixi_core/src/workspace/mod.rs +++ b/crates/pixi_core/src/workspace/mod.rs @@ -40,9 +40,10 @@ use pixi_config::{Config, RunPostLinkScripts}; use pixi_consts::consts; use pixi_diff::LockFileDiff; use pixi_manifest::{ - AssociateProvenance, BuildVariantSource, EnvironmentName, Environments, HasWorkspaceManifest, - LoadManifestsError, ManifestProvenance, Manifests, PackageManifest, PixiPlatform, - PixiPlatformName, SpecType, WithProvenance, WithWarnings, WorkspaceManifest, + AssociateProvenance, BuildVariantSource, CondaPypiMap, CondaPypiMapEntry, CondaPypiMapMode, + EnvironmentName, Environments, HasWorkspaceManifest, LoadManifestsError, ManifestProvenance, + Manifests, PackageManifest, PixiPlatform, PixiPlatformName, SpecType, WithProvenance, + WithWarnings, WorkspaceManifest, }; use pixi_path::AbsPathBuf; use pixi_pypi_spec::{PixiPypiSpec, PypiPackageName}; @@ -53,7 +54,7 @@ use pixi_utils::{ variants::{VariantConfig, VariantValue}, }; use pypi_mapping::{ - ChannelName, ProjectDefinedChannelMapping, ProjectDefinedMapping, + ChannelName, MappingMode, ProjectDefinedChannelMapping, ProjectDefinedMapping, ProjectDefinedMappingLocation, PurlDerivationMode, }; use rattler_conda_types::{ @@ -913,108 +914,165 @@ impl Workspace { /// It can use project-defined mappings in the format `conda_name: pypi_name`, /// or the self-hosted prefix.dev mappings. pub fn pypi_name_derivation_mode(&self) -> miette::Result<&PurlDerivationMode> { + /// Classify a manifest location string into a url or a path, resolving + /// relative paths against the workspace root. + fn parse_mapping_location( + mapping_location: &str, + cache_ttl: Option, + channel_config: &ChannelConfig, + ) -> miette::Result { + if mapping_location.starts_with("https://") || mapping_location.starts_with("http://") { + let url = Url::parse(mapping_location) + .into_diagnostic() + .context(format!("Could not convert {mapping_location} to URL"))?; + Ok(ProjectDefinedMappingLocation::Url { url, cache_ttl }) + } else { + if cache_ttl.is_some() { + miette::bail!( + "`cache-ttl` is only supported for http(s) mapping locations, but `{mapping_location}` is a local file" + ); + } + if mapping_location.starts_with("file://") { + let url = Url::parse(mapping_location) + .into_diagnostic() + .context(format!("Could not convert {mapping_location} to URL"))?; + Ok(ProjectDefinedMappingLocation::Url { + url, + cache_ttl: None, + }) + } else { + let path = PathBuf::from(mapping_location); + let abs_path = if path.is_relative() { + channel_config.root_dir.join(path) + } else { + path + }; + Ok(ProjectDefinedMappingLocation::Path(abs_path)) + } + } + } + + /// Convert a manifest entry to the per-channel mapping configuration + /// used by the purl derivation client. + fn convert_entry( + entry: &CondaPypiMapEntry, + channel_config: &ChannelConfig, + ) -> miette::Result { + match entry { + CondaPypiMapEntry::Disabled => Ok(ProjectDefinedChannelMapping::disabled()), + CondaPypiMapEntry::Map { + location, + mapping, + mode, + cache_ttl, + } => { + let mut sources = Vec::new(); + if let Some(location) = location { + sources.push(parse_mapping_location( + location, + *cache_ttl, + channel_config, + )?); + } + // Inline entries come last so they override entries from + // the location. Keys are lowercased to match the + // normalized conda package names used for lookups. + if let Some(inline) = mapping { + sources.push(ProjectDefinedMappingLocation::InMemory( + inline + .iter() + .map(|(name, pypi_name)| (name.to_lowercase(), pypi_name.clone())) + .collect(), + )); + } + let mode = match mode { + CondaPypiMapMode::Extend => MappingMode::Extend, + CondaPypiMapMode::Replace => MappingMode::Replace, + }; + Ok(ProjectDefinedChannelMapping::new(sources, mode)) + } + } + } + fn build_pypi_name_derivation_mode( manifest: &WorkspaceManifest, channel_config: &ChannelConfig, ) -> miette::Result { - match manifest.workspace.conda_pypi_map.clone() { - Some(map) => { - let channel_to_location_map = map - .into_iter() - .map(|(key, value)| { - let key = key.into_channel(channel_config).into_diagnostic()?; - Ok((key, value)) - }) - .collect::>>()?; - - // User can disable the mapping by providing an empty map - if channel_to_location_map.is_empty() { - return Ok(PurlDerivationMode::Disabled); - } + let map = match &manifest.workspace.conda_pypi_map { + None => return Ok(PurlDerivationMode::Prefix), + Some(CondaPypiMap::Disabled) => return Ok(PurlDerivationMode::Disabled), + Some(CondaPypiMap::Map(map)) => map, + }; - let project_channels: HashSet<_> = manifest - .workspace - .channels - .iter() - .map(|pc| pc.channel.clone().into_channel(channel_config)) - .try_collect() - .into_diagnostic()?; - - let feature_channels: HashSet<_> = manifest - .features - .values() - .flat_map(|feature| feature.channels.iter()) - .flatten() - .map(|pc| pc.channel.clone().into_channel(channel_config)) - .try_collect() - .into_diagnostic()?; - - let project_and_feature_channels: HashSet<_> = - project_channels.union(&feature_channels).collect(); - - for channel in channel_to_location_map.keys() { - if !project_and_feature_channels.contains(channel) { - let channels = project_and_feature_channels - .iter() - .map(|c| c.name.clone().unwrap_or_else(|| c.base_url.to_string())) - .sorted() - .collect::>() - .join(", "); - miette::bail!( - "conda-pypi-map is defined: the {} is missing from the channels array, which currently are: {}", - console::style( - channel - .name - .clone() - .unwrap_or_else(|| channel.base_url.to_string()) - ) - .bold(), - channels - ); - } - } + // An empty map is a soft-deprecated alias for `conda-pypi-map = false`; + // the deprecation warning is emitted when the manifest is parsed. + if map.is_empty() { + return Ok(PurlDerivationMode::Disabled); + } - let mapping = channel_to_location_map + let channel_to_entry_map = map + .iter() + .map(|(key, value)| { + let key = key.clone().into_channel(channel_config).into_diagnostic()?; + Ok((key, value)) + }) + .collect::>>()?; + + let project_channels: HashSet<_> = manifest + .workspace + .channels + .iter() + .map(|pc| pc.channel.clone().into_channel(channel_config)) + .try_collect() + .into_diagnostic()?; + + let feature_channels: HashSet<_> = manifest + .features + .values() + .flat_map(|feature| feature.channels.iter()) + .flatten() + .map(|pc| pc.channel.clone().into_channel(channel_config)) + .try_collect() + .into_diagnostic()?; + + let project_and_feature_channels: HashSet<_> = + project_channels.union(&feature_channels).collect(); + + for channel in channel_to_entry_map.keys() { + if !project_and_feature_channels.contains(channel) { + let channels = project_and_feature_channels .iter() - .map(|(channel, mapping_location)| { - let url_or_path = if mapping_location.starts_with("https://") - || mapping_location.starts_with("http://") - || mapping_location.starts_with("file://") - { - match Url::parse(mapping_location) { - Ok(url) => ProjectDefinedMappingLocation::Url { - url, - cache_ttl: None, - }, - Err(err) => { - return Err(err).into_diagnostic().context(format!( - "Could not convert {mapping_location} to URL" - )); - } - } - } else { - let path = PathBuf::from(mapping_location); - let abs_path = if path.is_relative() { - channel_config.root_dir.join(path) - } else { - path - }; - ProjectDefinedMappingLocation::Path(abs_path) - }; - - Ok(( - channel.canonical_name().trim_end_matches('/').into(), - ProjectDefinedChannelMapping::replace(url_or_path), - )) - }) - .collect::>>()?; - - Ok(PurlDerivationMode::ProjectDefined( - ProjectDefinedMapping::new(mapping).into(), - )) + .map(|c| c.name.clone().unwrap_or_else(|| c.base_url.to_string())) + .sorted() + .collect::>() + .join(", "); + miette::bail!( + "conda-pypi-map is defined: the {} is missing from the channels array, which currently are: {}", + console::style( + channel + .name + .clone() + .unwrap_or_else(|| channel.base_url.to_string()) + ) + .bold(), + channels + ); } - None => Ok(PurlDerivationMode::Prefix), } + + let mapping = channel_to_entry_map + .iter() + .map(|(channel, entry)| { + Ok(( + channel.canonical_name().trim_end_matches('/').into(), + convert_entry(entry, channel_config)?, + )) + }) + .collect::>>()?; + + Ok(PurlDerivationMode::ProjectDefined( + ProjectDefinedMapping::new(mapping).into(), + )) } self.derivation_mode.get_or_try_init(|| { build_pypi_name_derivation_mode(&self.workspace.value, &self.channel_config()) @@ -1609,7 +1667,8 @@ mod tests { .mapping .get(canonical_channel_name) .unwrap(), - &ProjectDefinedChannelMapping::replace(ProjectDefinedMappingLocation::Url { + // Bare location strings use the additive (extend) mode. + &ProjectDefinedChannelMapping::extend(ProjectDefinedMappingLocation::Url { url: Url::parse( "https://github.com/prefix-dev/parselmouth/blob/main/files/compressed_mapping.json" ) @@ -1644,7 +1703,7 @@ mod tests { .trim_end_matches('/') ) .unwrap(), - &ProjectDefinedChannelMapping::replace(ProjectDefinedMappingLocation::Path( + &ProjectDefinedChannelMapping::extend(ProjectDefinedMappingLocation::Path( workspace .channel_config() .root_dir diff --git a/crates/pixi_manifest/Cargo.toml b/crates/pixi_manifest/Cargo.toml index f9656f01e5..219777d313 100644 --- a/crates/pixi_manifest/Cargo.toml +++ b/crates/pixi_manifest/Cargo.toml @@ -14,6 +14,7 @@ console = { workspace = true } dunce = { workspace = true } fancy_display = { workspace = true } fs-err = { workspace = true } +humantime = { workspace = true } indexmap = { workspace = true } itertools = { workspace = true } miette = { workspace = true, features = ["fancy-no-backtrace"] } diff --git a/crates/pixi_manifest/src/lib.rs b/crates/pixi_manifest/src/lib.rs index c5298283f8..54989d477c 100644 --- a/crates/pixi_manifest/src/lib.rs +++ b/crates/pixi_manifest/src/lib.rs @@ -62,7 +62,10 @@ pub use target::{PackageTarget, TargetSelector, Targets, WorkspaceTarget}; pub use task::{Task, TaskName}; use thiserror::Error; pub use warning::{Warning, WarningWithSource, WithWarnings}; -pub use workspace::{BuildVariantSource, ChannelPriority, SolveStrategy, Workspace}; +pub use workspace::{ + BuildVariantSource, ChannelPriority, CondaPypiMap, CondaPypiMapEntry, CondaPypiMapMode, + SolveStrategy, Workspace, +}; pub use crate::{ environments::Environments, diff --git a/crates/pixi_manifest/src/toml/conda_pypi_map.rs b/crates/pixi_manifest/src/toml/conda_pypi_map.rs new file mode 100644 index 0000000000..18e6b972dc --- /dev/null +++ b/crates/pixi_manifest/src/toml/conda_pypi_map.rs @@ -0,0 +1,345 @@ +//! TOML deserialization for `[workspace.conda-pypi-map]`. +//! +//! The field accepts `false` (disable all lookups) or a per-channel table. +//! Each channel value is either a bare location string, `false` (disable +//! lookups for that channel) or a table with `location`, inline `mapping` +//! entries, `mode` and `cache-ttl`. + +use std::collections::HashMap; + +use pixi_toml::{TomlEnum, TomlHashMap}; +use rattler_conda_types::NamedChannelOrUrl; +use toml_span::{ + DeserError, Error, ErrorKind, Value, + de_helpers::{TableHelper, expected}, + value::ValueInner, +}; + +use crate::workspace::{CondaPypiMap, CondaPypiMapEntry, CondaPypiMapMode}; + +impl<'de> toml_span::Deserialize<'de> for CondaPypiMap { + fn deserialize(value: &mut Value<'de>) -> Result { + match value.take() { + ValueInner::Boolean(false) => Ok(CondaPypiMap::Disabled), + ValueInner::Boolean(true) => Err(Error { + kind: ErrorKind::Custom( + "`conda-pypi-map = true` is not supported; use `false` to disable the \ + mapping, or a table to configure it" + .into(), + ), + span: value.span, + line_info: None, + } + .into()), + inner @ ValueInner::Table(_) => { + let span = value.span; + let map = TomlHashMap::::deserialize( + &mut Value::with_span(inner, span), + )?; + Ok(CondaPypiMap::Map(map.into_inner())) + } + other => Err(expected("a table or `false`", other, value.span).into()), + } + } +} + +impl<'de> toml_span::Deserialize<'de> for CondaPypiMapEntry { + fn deserialize(value: &mut Value<'de>) -> Result { + match value.take() { + ValueInner::String(s) => Ok(CondaPypiMapEntry::from_location(s.into_owned())), + ValueInner::Boolean(false) => Ok(CondaPypiMapEntry::Disabled), + ValueInner::Boolean(true) => Err(Error { + kind: ErrorKind::Custom( + "`true` is not supported; use `false` to disable the mapping for this \ + channel, or a string or table to configure it" + .into(), + ), + span: value.span, + line_info: None, + } + .into()), + inner @ ValueInner::Table(_) => { + let table_span = value.span; + let mut th = TableHelper::new(&mut Value::with_span(inner, table_span))?; + + let location: Option = th.optional("location"); + let mapping: Option>> = th + .optional::>("mapping") + .map(|map| { + map.into_inner() + .into_iter() + .map(|(name, value)| (name, value.0)) + .collect() + }); + let mode = th + .optional::>("mode") + .map(TomlEnum::into_inner) + .unwrap_or_default(); + let cache_ttl = match th.optional::>("cache-ttl") { + Some(spanned) => Some( + spanned + .value + .parse::() + .map_err(|e| Error { + kind: ErrorKind::Custom( + format!("invalid `cache-ttl` duration: {e}").into(), + ), + span: spanned.span, + line_info: None, + })? + .into(), + ), + None => None, + }; + + th.finalize(None)?; + + if location.is_none() && mapping.is_none() { + return Err(Error { + kind: ErrorKind::Custom( + "expected at least one of `location` or `mapping`".into(), + ), + span: table_span, + line_info: None, + } + .into()); + } + + if cache_ttl.is_some() && location.is_none() { + return Err(Error { + kind: ErrorKind::Custom( + "`cache-ttl` requires a `location` that is a URL".into(), + ), + span: table_span, + line_info: None, + } + .into()); + } + + Ok(CondaPypiMapEntry::Map { + location, + mapping, + mode, + cache_ttl, + }) + } + other => Err(expected("a string, table or `false`", other, value.span).into()), + } + } +} + +/// The value of an inline mapping entry: a pypi name, or `false` to mark the +/// package as not available on PyPI. +pub(crate) struct TomlCondaPypiMapValue(pub(crate) Option); + +impl<'de> toml_span::Deserialize<'de> for TomlCondaPypiMapValue { + fn deserialize(value: &mut Value<'de>) -> Result { + match value.take() { + ValueInner::String(s) => Ok(Self(Some(s.into_owned()))), + ValueInner::Boolean(false) => Ok(Self(None)), + ValueInner::Boolean(true) => Err(Error { + kind: ErrorKind::Custom( + "`true` is not supported; use a string to map the package to a PyPI name, \ + or `false` to mark it as not a PyPI package" + .into(), + ), + span: value.span, + line_info: None, + } + .into()), + other => Err(expected("a string or `false`", other, value.span).into()), + } + } +} + +#[cfg(test)] +mod test { + use std::time::Duration; + + use insta::assert_snapshot; + use rattler_conda_types::NamedChannelOrUrl; + + use super::*; + use crate::{ + toml::{FromTomlStr, TomlWorkspace}, + utils::test_utils::{expect_parse_failure, expect_parse_warnings}, + }; + + fn parse_map(conda_pypi_map: &str) -> CondaPypiMap { + let input = format!( + r#" + channels = [] + platforms = [] + conda-pypi-map = {conda_pypi_map} + "# + ); + TomlWorkspace::from_toml_str(&input) + .expect("parsing should succeed") + .conda_pypi_map + .expect("conda-pypi-map should be set") + } + + fn get_entry(map: &CondaPypiMap, channel: &str) -> CondaPypiMapEntry { + let CondaPypiMap::Map(map) = map else { + panic!("expected a per-channel map"); + }; + map.get(&NamedChannelOrUrl::Name(channel.to_string())) + .expect("channel should be present") + .clone() + } + + #[test] + fn test_bare_string_is_extend() { + let map = parse_map(r#"{ conda-forge = "mapping.json" }"#); + assert_eq!( + get_entry(&map, "conda-forge"), + CondaPypiMapEntry::Map { + location: Some("mapping.json".to_string()), + mapping: None, + mode: CondaPypiMapMode::Extend, + cache_ttl: None, + } + ); + } + + #[test] + fn test_table_with_location_mode_and_ttl() { + let map = parse_map( + r#"{ conda-forge = { location = "https://example.com/m.json", mode = "replace", cache-ttl = "24h" } }"#, + ); + assert_eq!( + get_entry(&map, "conda-forge"), + CondaPypiMapEntry::Map { + location: Some("https://example.com/m.json".to_string()), + mapping: None, + mode: CondaPypiMapMode::Replace, + cache_ttl: Some(Duration::from_secs(24 * 60 * 60)), + } + ); + } + + #[test] + fn test_inline_mapping_with_false_value() { + let map = parse_map( + r#"{ conda-forge = { mapping = { pytorch = "torch", not-on-pypi = false } } }"#, + ); + let CondaPypiMapEntry::Map { mapping, mode, .. } = get_entry(&map, "conda-forge") else { + panic!("expected a mapping entry"); + }; + let mapping = mapping.expect("mapping should be set"); + assert_eq!(mode, CondaPypiMapMode::Extend); + assert_eq!(mapping["pytorch"], Some("torch".to_string())); + assert_eq!(mapping["not-on-pypi"], None); + } + + #[test] + fn test_channel_false_disables() { + let map = parse_map(r#"{ conda-forge = false }"#); + assert_eq!(get_entry(&map, "conda-forge"), CondaPypiMapEntry::Disabled); + } + + #[test] + fn test_top_level_false_disables() { + let map = parse_map("false"); + assert_eq!(map, CondaPypiMap::Disabled); + } + + #[test] + fn test_empty_map_parses_and_warns() { + let map = parse_map("{}"); + assert!(matches!(map, CondaPypiMap::Map(map) if map.is_empty())); + + assert_snapshot!(expect_parse_warnings( + r#" + [workspace] + channels = [] + platforms = [] + conda-pypi-map = {} + "# + )); + } + + #[test] + fn test_top_level_true_fails() { + assert_snapshot!(expect_parse_failure( + r#" + [workspace] + channels = [] + platforms = [] + conda-pypi-map = true + "# + )); + } + + #[test] + fn test_channel_true_fails() { + assert_snapshot!(expect_parse_failure( + r#" + [workspace] + channels = [] + platforms = [] + conda-pypi-map = { conda-forge = true } + "# + )); + } + + #[test] + fn test_inline_true_value_fails() { + assert_snapshot!(expect_parse_failure( + r#" + [workspace] + channels = [] + platforms = [] + conda-pypi-map = { conda-forge = { mapping = { pytorch = true } } } + "# + )); + } + + #[test] + fn test_empty_entry_table_fails() { + assert_snapshot!(expect_parse_failure( + r#" + [workspace] + channels = [] + platforms = [] + conda-pypi-map = { conda-forge = { mode = "extend" } } + "# + )); + } + + #[test] + fn test_bogus_mode_fails() { + assert_snapshot!(expect_parse_failure( + r#" + [workspace] + channels = [] + platforms = [] + conda-pypi-map = { conda-forge = { location = "m.json", mode = "bogus" } } + "# + )); + } + + #[test] + fn test_invalid_ttl_fails() { + assert_snapshot!(expect_parse_failure( + r#" + [workspace] + channels = [] + platforms = [] + conda-pypi-map = { conda-forge = { location = "https://example.com/m.json", cache-ttl = "bogus" } } + "# + )); + } + + #[test] + fn test_ttl_without_location_fails() { + assert_snapshot!(expect_parse_failure( + r#" + [workspace] + channels = [] + platforms = [] + conda-pypi-map = { conda-forge = { mapping = { a = "b" }, cache-ttl = "24h" } } + "# + )); + } +} diff --git a/crates/pixi_manifest/src/toml/mod.rs b/crates/pixi_manifest/src/toml/mod.rs index da4c745a91..89bac4164e 100644 --- a/crates/pixi_manifest/src/toml/mod.rs +++ b/crates/pixi_manifest/src/toml/mod.rs @@ -1,6 +1,7 @@ mod build_backend; mod build_target; mod channel; +mod conda_pypi_map; mod document; mod environment; mod feature; diff --git a/crates/pixi_manifest/src/toml/snapshots/pixi_manifest__toml__conda_pypi_map__test__bogus_mode_fails.snap b/crates/pixi_manifest/src/toml/snapshots/pixi_manifest__toml__conda_pypi_map__test__bogus_mode_fails.snap new file mode 100644 index 0000000000..1c81c6f9a1 --- /dev/null +++ b/crates/pixi_manifest/src/toml/snapshots/pixi_manifest__toml__conda_pypi_map__test__bogus_mode_fails.snap @@ -0,0 +1,11 @@ +--- +source: crates/pixi_manifest/src/toml/conda_pypi_map.rs +expression: "expect_parse_failure(r#\"\n [workspace]\n channels = []\n platforms = []\n conda-pypi-map = { conda-forge = { location = \"m.json\", mode = \"bogus\" } }\n \"#)" +--- + × Expected one of 'extend', 'replace' + ╭─[pixi.toml:5:77] + 4 │ platforms = [] + 5 │ conda-pypi-map = { conda-forge = { location = "m.json", mode = "bogus" } } + · ───── + 6 │ + ╰──── diff --git a/crates/pixi_manifest/src/toml/snapshots/pixi_manifest__toml__conda_pypi_map__test__channel_true_fails.snap b/crates/pixi_manifest/src/toml/snapshots/pixi_manifest__toml__conda_pypi_map__test__channel_true_fails.snap new file mode 100644 index 0000000000..7ba16035c2 --- /dev/null +++ b/crates/pixi_manifest/src/toml/snapshots/pixi_manifest__toml__conda_pypi_map__test__channel_true_fails.snap @@ -0,0 +1,11 @@ +--- +source: crates/pixi_manifest/src/toml/conda_pypi_map.rs +expression: "expect_parse_failure(r#\"\n [workspace]\n channels = []\n platforms = []\n conda-pypi-map = { conda-forge = true }\n \"#)" +--- + × `true` is not supported; use `false` to disable the mapping for this channel, or a string or table to configure it + ╭─[pixi.toml:5:46] + 4 │ platforms = [] + 5 │ conda-pypi-map = { conda-forge = true } + · ──── + 6 │ + ╰──── diff --git a/crates/pixi_manifest/src/toml/snapshots/pixi_manifest__toml__conda_pypi_map__test__empty_entry_table_fails.snap b/crates/pixi_manifest/src/toml/snapshots/pixi_manifest__toml__conda_pypi_map__test__empty_entry_table_fails.snap new file mode 100644 index 0000000000..4a831b152b --- /dev/null +++ b/crates/pixi_manifest/src/toml/snapshots/pixi_manifest__toml__conda_pypi_map__test__empty_entry_table_fails.snap @@ -0,0 +1,11 @@ +--- +source: crates/pixi_manifest/src/toml/conda_pypi_map.rs +expression: "expect_parse_failure(r#\"\n [workspace]\n channels = []\n platforms = []\n conda-pypi-map = { conda-forge = { mode = \"extend\" } }\n \"#)" +--- + × expected at least one of `location` or `mapping` + ╭─[pixi.toml:5:46] + 4 │ platforms = [] + 5 │ conda-pypi-map = { conda-forge = { mode = "extend" } } + · ─────────────────── + 6 │ + ╰──── diff --git a/crates/pixi_manifest/src/toml/snapshots/pixi_manifest__toml__conda_pypi_map__test__empty_map_parses_and_warns.snap b/crates/pixi_manifest/src/toml/snapshots/pixi_manifest__toml__conda_pypi_map__test__empty_map_parses_and_warns.snap new file mode 100644 index 0000000000..38fd9cd533 --- /dev/null +++ b/crates/pixi_manifest/src/toml/snapshots/pixi_manifest__toml__conda_pypi_map__test__empty_map_parses_and_warns.snap @@ -0,0 +1,6 @@ +--- +source: crates/pixi_manifest/src/toml/conda_pypi_map.rs +expression: "expect_parse_warnings(r#\"\n [workspace]\n channels = []\n platforms = []\n conda-pypi-map = {}\n \"#)" +--- + ⚠ `conda-pypi-map = {}` is deprecated + help: To disable the conda-pypi mapping, write `conda-pypi-map = false` instead. diff --git a/crates/pixi_manifest/src/toml/snapshots/pixi_manifest__toml__conda_pypi_map__test__inline_true_value_fails.snap b/crates/pixi_manifest/src/toml/snapshots/pixi_manifest__toml__conda_pypi_map__test__inline_true_value_fails.snap new file mode 100644 index 0000000000..33010f3203 --- /dev/null +++ b/crates/pixi_manifest/src/toml/snapshots/pixi_manifest__toml__conda_pypi_map__test__inline_true_value_fails.snap @@ -0,0 +1,11 @@ +--- +source: crates/pixi_manifest/src/toml/conda_pypi_map.rs +expression: "expect_parse_failure(r#\"\n [workspace]\n channels = []\n platforms = []\n conda-pypi-map = { conda-forge = { mapping = { pytorch = true } } }\n \"#)" +--- + × `true` is not supported; use a string to map the package to a PyPI name, or `false` to mark it as not a PyPI package + ╭─[pixi.toml:5:70] + 4 │ platforms = [] + 5 │ conda-pypi-map = { conda-forge = { mapping = { pytorch = true } } } + · ──── + 6 │ + ╰──── diff --git a/crates/pixi_manifest/src/toml/snapshots/pixi_manifest__toml__conda_pypi_map__test__invalid_ttl_fails.snap b/crates/pixi_manifest/src/toml/snapshots/pixi_manifest__toml__conda_pypi_map__test__invalid_ttl_fails.snap new file mode 100644 index 0000000000..aeff496873 --- /dev/null +++ b/crates/pixi_manifest/src/toml/snapshots/pixi_manifest__toml__conda_pypi_map__test__invalid_ttl_fails.snap @@ -0,0 +1,11 @@ +--- +source: crates/pixi_manifest/src/toml/conda_pypi_map.rs +expression: "expect_parse_failure(r#\"\n [workspace]\n channels = []\n platforms = []\n conda-pypi-map = { conda-forge = { location = \"https://example.com/m.json\", cache-ttl = \"bogus\" } }\n \"#)" +--- + × invalid `cache-ttl` duration: expected number at 0 + ╭─[pixi.toml:5:102] + 4 │ platforms = [] + 5 │ conda-pypi-map = { conda-forge = { location = "https://example.com/m.json", cache-ttl = "bogus" } } + · ───── + 6 │ + ╰──── diff --git a/crates/pixi_manifest/src/toml/snapshots/pixi_manifest__toml__conda_pypi_map__test__top_level_true_fails.snap b/crates/pixi_manifest/src/toml/snapshots/pixi_manifest__toml__conda_pypi_map__test__top_level_true_fails.snap new file mode 100644 index 0000000000..54de5ec994 --- /dev/null +++ b/crates/pixi_manifest/src/toml/snapshots/pixi_manifest__toml__conda_pypi_map__test__top_level_true_fails.snap @@ -0,0 +1,11 @@ +--- +source: crates/pixi_manifest/src/toml/conda_pypi_map.rs +expression: "expect_parse_failure(r#\"\n [workspace]\n channels = []\n platforms = []\n conda-pypi-map = true\n \"#)" +--- + × `conda-pypi-map = true` is not supported; use `false` to disable the mapping, or a table to configure it + ╭─[pixi.toml:5:30] + 4 │ platforms = [] + 5 │ conda-pypi-map = true + · ──── + 6 │ + ╰──── diff --git a/crates/pixi_manifest/src/toml/snapshots/pixi_manifest__toml__conda_pypi_map__test__ttl_without_location_fails.snap b/crates/pixi_manifest/src/toml/snapshots/pixi_manifest__toml__conda_pypi_map__test__ttl_without_location_fails.snap new file mode 100644 index 0000000000..1876202544 --- /dev/null +++ b/crates/pixi_manifest/src/toml/snapshots/pixi_manifest__toml__conda_pypi_map__test__ttl_without_location_fails.snap @@ -0,0 +1,11 @@ +--- +source: crates/pixi_manifest/src/toml/conda_pypi_map.rs +expression: "expect_parse_failure(r#\"\n [workspace]\n channels = []\n platforms = []\n conda-pypi-map = { conda-forge = { mapping = { a = \"b\" }, cache-ttl = \"24h\" } }\n \"#)" +--- + × `cache-ttl` requires a `location` that is a URL + ╭─[pixi.toml:5:46] + 4 │ platforms = [] + 5 │ conda-pypi-map = { conda-forge = { mapping = { a = "b" }, cache-ttl = "24h" } } + · ──────────────────────────────────────────── + 6 │ + ╰──── diff --git a/crates/pixi_manifest/src/toml/workspace.rs b/crates/pixi_manifest/src/toml/workspace.rs index 3381d1f434..a46e745b03 100644 --- a/crates/pixi_manifest/src/toml/workspace.rs +++ b/crates/pixi_manifest/src/toml/workspace.rs @@ -6,7 +6,7 @@ use std::{ use indexmap::{IndexMap, IndexSet}; use pixi_spec::{ExcludeNewer, TomlSpec, TomlVersionSpecStr}; use pixi_toml::{TomlFromStr, TomlHashMap, TomlIndexMap, TomlIndexSet, TomlWith}; -use rattler_conda_types::{NamedChannelOrUrl, PackageName, Version, VersionSpec}; +use rattler_conda_types::{PackageName, Version, VersionSpec}; use std::str::FromStr; use toml_span::{DeserError, Span, Spanned, Value, de_helpers::TableHelper, value::ValueInner}; use url::Url; @@ -20,7 +20,7 @@ use crate::{ manifest::ExternalWorkspaceProperties, platform::TomlPixiPlatform, preview::TomlPreview, }, utils::PixiSpanned, - workspace::{BuildVariantSource, ChannelPriority, SolveStrategy}, + workspace::{BuildVariantSource, ChannelPriority, CondaPypiMap, SolveStrategy}, }; /// Parses `[workspace.dependencies]` into an ordered `(name, TomlSpec)` map. @@ -110,7 +110,7 @@ pub struct TomlWorkspace { pub homepage: Option, pub repository: Option, pub documentation: Option, - pub conda_pypi_map: Option>, + pub conda_pypi_map: Option, pub pypi_options: Option, pub s3_options: Option>, pub preview: TomlPreview, @@ -177,7 +177,22 @@ impl TomlWorkspace { value: preview, } = self.preview.into_preview(); - let warnings = preview_warnings; + let mut warnings = preview_warnings; + + // An empty `conda-pypi-map = {}` is a soft-deprecated alias for + // `conda-pypi-map = false`. + if let Some(CondaPypiMap::Map(map)) = &self.conda_pypi_map + && map.is_empty() + { + warnings.push( + GenericError::new("`conda-pypi-map = {}` is deprecated") + .with_help( + "To disable the conda-pypi mapping, write `conda-pypi-map = false` \ + instead.", + ) + .into(), + ); + } let build_variant_files_default = convert_build_variant_files(self.build_variant_files, root_directory)?; @@ -354,9 +369,7 @@ impl<'de> toml_span::Deserialize<'de> for TomlWorkspace { let documentation = th .optional::>("documentation") .map(TomlFromStr::into_inner); - let conda_pypi_map = th - .optional::>("conda-pypi-map") - .map(TomlHashMap::into_inner); + let conda_pypi_map = th.optional("conda-pypi-map"); let pypi_options = th.optional("pypi-options"); let s3_options = th .optional::>("s3-options") diff --git a/crates/pixi_manifest/src/workspace.rs b/crates/pixi_manifest/src/workspace.rs index 8674232945..5be276a073 100644 --- a/crates/pixi_manifest/src/workspace.rs +++ b/crates/pixi_manifest/src/workspace.rs @@ -74,8 +74,8 @@ pub struct Workspace { /// URL of the project documentation pub documentation: Option, - /// URL or Path of the conda to pypi name mapping - pub conda_pypi_map: Option>, + /// The conda to pypi name mapping configuration. + pub conda_pypi_map: Option, /// The pypi options supported in the project pub pypi_options: Option, @@ -306,6 +306,72 @@ impl From for ChannelPriority { } } +/// The value of `[workspace.conda-pypi-map]`. +#[derive(Debug, Clone, PartialEq)] +pub enum CondaPypiMap { + /// `conda-pypi-map = false`: disable purl derivation lookups entirely. + Disabled, + /// Per-channel mapping configuration. An empty map is a soft-deprecated + /// alias for `Disabled`. + Map(HashMap), +} + +/// How a project-defined channel mapping interacts with the default +/// prefix.dev derivation chain. +#[derive( + Debug, + Copy, + Clone, + Default, + Eq, + PartialEq, + strum::Display, + strum::VariantNames, + strum::EnumString, +)] +#[strum(serialize_all = "kebab-case")] +pub enum CondaPypiMapMode { + /// The mapping overlays the defaults: a hit is final, a miss falls + /// through to the prefix.dev chain. + #[default] + Extend, + /// The mapping is exclusive: only packages in the mapping get purls. + Replace, +} + +/// The mapping configuration for one channel in `[workspace.conda-pypi-map]`. +#[derive(Debug, Clone, PartialEq)] +pub enum CondaPypiMapEntry { + /// ` = false`: no purl lookups for this channel. + Disabled, + /// A mapping defined by a location (file or URL) and/or inline entries. + Map { + /// File path or URL of a mapping JSON file. Unresolved: relative + /// paths are resolved against the workspace root by the consumer. + location: Option, + /// Inline conda-name to pypi-name entries. A `None` value (spelled + /// `false` in TOML) means the package is not a PyPI package. + mapping: Option>>, + mode: CondaPypiMapMode, + /// How long a mapping fetched from a URL may be reused before it is + /// re-fetched. Only valid for http(s) locations. + cache_ttl: Option, + }, +} + +impl CondaPypiMapEntry { + /// Create an entry from a bare location string. Bare strings use the + /// default (extend) mode. + pub fn from_location(location: String) -> Self { + Self::Map { + location: Some(location), + mapping: None, + mode: CondaPypiMapMode::default(), + cache_ttl: None, + } + } +} + #[derive( Debug, Copy, From 6631465de5dd756656987bd8c7fe704300fefff5 Mon Sep 17 00:00:00 2001 From: Tim de Jager Date: Wed, 10 Jun 2026 12:08:38 +0200 Subject: [PATCH 03/10] test: cover extend/replace/disabled conda-pypi-map modes and migrate {} disable to false --- Cargo.lock | 1 + crates/pixi/Cargo.toml | 1 + .../pixi/tests/integration_rust/add_tests.rs | 2 +- .../pixi/tests/integration_rust/pypi_tests.rs | 54 +- .../integration_rust/solve_group_tests.rs | 494 +++++++++++++++++- .../tests/integration_rust/upgrade_tests.rs | 6 +- 6 files changed, 511 insertions(+), 47 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 577e2b75e2..481d791d15 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5987,6 +5987,7 @@ dependencies = [ "pixi_utils", "pypi_mapping", "rattler_conda_types", + "rattler_digest", "rattler_lock", "rattler_package_streaming", "rattler_virtual_packages", diff --git a/crates/pixi/Cargo.toml b/crates/pixi/Cargo.toml index b789fae9e9..226f85214d 100644 --- a/crates/pixi/Cargo.toml +++ b/crates/pixi/Cargo.toml @@ -62,6 +62,7 @@ pixi_test_utils = { workspace = true } pixi_utils = { workspace = true } pypi_mapping = { workspace = true } rattler_conda_types = { workspace = true } +rattler_digest = { workspace = true } rattler_lock = { workspace = true } rattler_package_streaming = { workspace = true } rattler_virtual_packages = { workspace = true } diff --git a/crates/pixi/tests/integration_rust/add_tests.rs b/crates/pixi/tests/integration_rust/add_tests.rs index f0dfc035d7..936ee3e85e 100644 --- a/crates/pixi/tests/integration_rust/add_tests.rs +++ b/crates/pixi/tests/integration_rust/add_tests.rs @@ -484,7 +484,7 @@ async fn add_pypi_extra_functionality() { name = "test-pypi-extras" channels = ["{channel_url}"] platforms = ["{platform}"] -conda-pypi-map = {{}} # disable mapping +conda-pypi-map = false # disable mapping [dependencies] python = "==3.12.0" diff --git a/crates/pixi/tests/integration_rust/pypi_tests.rs b/crates/pixi/tests/integration_rust/pypi_tests.rs index e90de8bb1d..455a820b82 100644 --- a/crates/pixi/tests/integration_rust/pypi_tests.rs +++ b/crates/pixi/tests/integration_rust/pypi_tests.rs @@ -65,7 +65,7 @@ test = ["recursive-optional-groups[np]", "pytest", {{include-group = "docs"}}] [tool.pixi.workspace] channels = ["{channel_url}"] platforms = ["{platform_str}"] -conda-pypi-map = {{}} +conda-pypi-map = false [tool.pixi.dependencies] python = "==3.11.0" @@ -154,7 +154,7 @@ dependencies = [ [tool.pixi.workspace] channels = ["conda-forge"] platforms = ["{platform_str}"] -conda-pypi-map = {{}} +conda-pypi-map = false [tool.pixi.dependencies] python = "==3.11.0" @@ -226,7 +226,7 @@ version = "1.0.0" [tool.pixi.workspace] channels = ["conda-forge"] platforms = ["{platform_str}"] -conda-pypi-map = {{}} +conda-pypi-map = false [tool.pixi.dependencies] python = "==3.11.0" @@ -417,7 +417,7 @@ version = "1.0.0" [tool.pixi.workspace] channels = ["conda-forge"] platforms = ["{platform_str}"] -conda-pypi-map = {{}} +conda-pypi-map = false [tool.pixi.dependencies] python = "==3.11.0" @@ -564,7 +564,7 @@ dependencies = [ [tool.pixi.workspace] channels = ["{channel_url}"] platforms = [{platform_str}] -conda-pypi-map = {{}} +conda-pypi-map = false [tool.pixi.dependencies] python = "==3.11.0" @@ -620,7 +620,7 @@ async fn test_flat_links_based_index_returns_path() { name = "pypi-flat-find-links" platforms = ["{platform}"] channels = ["{channel_url}"] - conda-pypi-map = {{}} + conda-pypi-map = false [dependencies] python = "==3.12.0" @@ -676,7 +676,7 @@ async fn test_file_based_index_returns_path() { name = "pypi-extra-index-url" platforms = ["{platform}"] channels = ["{channel_url}"] - conda-pypi-map = {{}} + conda-pypi-map = false [dependencies] python = "==3.12.0" @@ -741,7 +741,7 @@ async fn test_index_strategy() { name = "pypi-extra-index-url" platforms = ["{platform}"] channels = ["{channel_url}"] - conda-pypi-map = {{}} + conda-pypi-map = false [dependencies] python = "==3.12.0" @@ -832,7 +832,7 @@ async fn test_pinning_index() { name = "pypi-pinning-index" platforms = ["{platform}"] channels = ["{channel_url}"] - conda-pypi-map = {{}} + conda-pypi-map = false [dependencies] python = "==3.12.0" @@ -898,7 +898,7 @@ async fn test_exclude_newer_per_package_pypi_index_override() { platforms = ["{platform}"] channels = ["{channel_url}"] exclude-newer = "2015-12-02T02:07:43Z" - conda-pypi-map = {{}} + conda-pypi-map = false [dependencies] python = "==3.12.0" @@ -933,7 +933,7 @@ async fn test_exclude_newer_per_package_pypi_index_override() { platforms = ["{platform}"] channels = ["{channel_url}"] exclude-newer = "2015-12-02T02:07:43Z" - conda-pypi-map = {{}} + conda-pypi-map = false [dependencies] python = "==3.12.0" @@ -1010,7 +1010,7 @@ async fn test_exclude_newer_dependency_override_pypi_index_override() { platforms = ["{platform}"] channels = ["{channel_url}"] exclude-newer = "2015-12-02T02:07:43Z" - conda-pypi-map = {{}} + conda-pypi-map = false [dependencies] python = "==3.12.0" @@ -1048,7 +1048,7 @@ async fn test_exclude_newer_dependency_override_pypi_index_override() { platforms = ["{platform}"] channels = ["{channel_url}"] exclude-newer = "2015-12-02T02:07:43Z" - conda-pypi-map = {{}} + conda-pypi-map = false [dependencies] python = "==3.12.0" @@ -1118,7 +1118,7 @@ async fn pin_torch() { name = "pypi-pinning-index" platforms = [{platforms}] channels = ["{channel_url}"] - conda-pypi-map = {{}} + conda-pypi-map = false [dependencies] python = "==3.12.0" @@ -1170,7 +1170,7 @@ async fn test_exclude_newer_relative_pypi_rejects_unknown_timestamps() { name = "test-exclude-newer-relative-pypi-baseline" platforms = ["{platform}"] channels = ["{channel_url}"] - conda-pypi-map = {{}} + conda-pypi-map = false [dependencies] python = "==3.12.0" @@ -1201,7 +1201,7 @@ async fn test_exclude_newer_relative_pypi_rejects_unknown_timestamps() { platforms = ["{platform}"] channels = ["{channel_url}"] exclude-newer = "1d" - conda-pypi-map = {{}} + conda-pypi-map = false [dependencies] python = "==3.12.0" @@ -1257,7 +1257,7 @@ async fn test_allow_insecure_host() { name = "pypi-extra-index-url" platforms = ["{platform}"] channels = ["{channel_url}"] - conda-pypi-map = {{}} + conda-pypi-map = false [dependencies] python = "==3.12.0" @@ -1322,7 +1322,7 @@ async fn test_tls_no_verify_with_pypi_dependencies() { name = "pypi-tls-test" platforms = ["{platform}"] channels = ["{channel_url}"] - conda-pypi-map = {{}} + conda-pypi-map = false [dependencies] python = "==3.12.0" @@ -1406,7 +1406,7 @@ async fn test_tls_verify_still_fails_without_config() { name = "pypi-tls-verify-test" platforms = ["{platform}"] channels = ["{channel_url}"] - conda-pypi-map = {{}} + conda-pypi-map = false [dependencies] python = "==3.12.0" @@ -1657,7 +1657,7 @@ async fn test_cross_platform_resolve_with_no_build() { name = "pypi-extra-index-url" platforms = ["{platform}"] channels = ["{channel_url}"] - conda-pypi-map = {{}} + conda-pypi-map = false [dependencies] python = "==3.12.0" @@ -1724,7 +1724,7 @@ async fn test_pinned_help_message() { r#" [workspace] channels = ["{channel}"] - conda-pypi-map = {{}} + conda-pypi-map = false name = "local-pinned-help" platforms = ["{platform}"] version = "0.1.0" @@ -1806,7 +1806,7 @@ async fn test_uv_index_correctly_parsed() { [tool.pixi.workspace] channels = ["{channel_url}"] platforms = ["{platform}"] - conda-pypi-map = {{}} # Disable mapping + conda-pypi-map = false # Disable mapping [tool.pixi.pypi-dependencies] simple = {{ path = "." }} @@ -1865,7 +1865,7 @@ async fn test_prerelease_mode_allow() { name = "prerelease-test" platforms = ["{platform}"] channels = ["{channel_url}"] - conda-pypi-map = {{}} # Disable mapping + conda-pypi-map = false # Disable mapping [dependencies] python = "==3.12.0" @@ -1927,7 +1927,7 @@ async fn test_prerelease_mode_disallow() { name = "prerelease-test" platforms = ["{platform}"] channels = ["{channel_url}"] - conda-pypi-map = {{}} + conda-pypi-map = false [dependencies] python = "==3.12.0" @@ -2276,7 +2276,7 @@ requires-python = "{requires_python}" [tool.pixi.workspace] channels = ["{channel_url}"] platforms = ["{platform}"] -conda-pypi-map = {{}} +conda-pypi-map = false [tool.pixi.dependencies] python = "==3.10.6" @@ -2353,7 +2353,7 @@ async fn test_index_url_in_lock_file() { name = "index-url-test" platforms = ["{platform}"] channels = ["{channel_url}"] - conda-pypi-map = {{}} + conda-pypi-map = false [dependencies] python = "==3.12.0" @@ -2447,7 +2447,7 @@ async fn test_index_url_omitted_for_default_pypi() { name = "index-url-pypi-test" platforms = [{platforms}] channels = ["{channel_url}"] - conda-pypi-map = {{}} + conda-pypi-map = false [dependencies] python = "==3.12.0" diff --git a/crates/pixi/tests/integration_rust/solve_group_tests.rs b/crates/pixi/tests/integration_rust/solve_group_tests.rs index 380d1617c0..b7c374ca4e 100644 --- a/crates/pixi/tests/integration_rust/solve_group_tests.rs +++ b/crates/pixi/tests/integration_rust/solve_group_tests.rs @@ -7,8 +7,7 @@ use std::{ use pypi_mapping::{ self, ProjectDefinedChannelMapping, ProjectDefinedMapping, ProjectDefinedMappingLocation, - PurlDerivationMode, - PurlDerivationSource, + PurlDerivationMode, PurlDerivationSource, }; use rattler_conda_types::{PackageName, Platform, RepoDataRecord}; use rattler_lock::DEFAULT_ENVIRONMENT_NAME; @@ -531,7 +530,7 @@ async fn test_we_record_not_present_package_as_purl_for_custom_mapping() { name = "test-channel-change" channels = ["conda-forge"] platforms = ["linux-64"] - conda-pypi-map = {{ 'conda-forge' = "{}" }} + conda-pypi-map = {{ 'conda-forge' = {{ location = "{}", mode = "replace" }} }} "#, absolute_compressed_mapping_path() )) @@ -543,12 +542,9 @@ async fn test_we_record_not_present_package_as_purl_for_custom_mapping() { // We use one package that is present in our mapping: `boltons` // and another one that is missing from conda and our mapping: - // `pixi-something-new-for-test` because `pixi-something-new-for-test` is - // from conda-forge channel we will anyway record a purl for it - // by assumption that it's a pypi package - // also we are using some project-defined mapping - // so we will test for other purl qualifier comparing to - // `test_dont_record_not_present_package_as_purl` test + // `pixi-something-new-for-test`. Because the mapping uses + // `mode = "replace"` the mapping is exclusive: packages that are not in + // it must not get a purl, not even the conda-forge verbatim fallback. let foo_bar_package = Package::build("pixi-something-new", "2").finish(); let boltons_package = Package::build("boltons", "2").finish(); @@ -606,13 +602,13 @@ async fn test_we_record_not_present_package_as_purl_for_custom_mapping() { let package = packages.pop().unwrap(); - // With project-defined mapping, packages not in the mapping should NOT get purls - // This verifies that project-defined mapping is exclusive - only packages explicitly - // mapped should be considered as pypi packages + // With a replace-mode project-defined mapping, packages not in the mapping + // should NOT get purls. This verifies that replace mode is exclusive - only + // packages explicitly mapped should be considered as pypi packages. assert!( package.package_record.purls.is_none() || package.package_record.purls.as_ref().unwrap().is_empty(), - "pixi-something-new should not have purls when not in project-defined mapping" + "pixi-something-new should not have purls when not in a replace-mode mapping" ); } @@ -892,6 +888,430 @@ async fn test_file_url_as_mapping_location() { ); } +/// Build a `PurlDerivationClient` whose http client refuses any network +/// request, backed by the given cache directory. +fn offline_mapping_client( + project: &pixi_core::Workspace, + cache_dir: std::path::PathBuf, +) -> pypi_mapping::PurlDerivationClient { + let client = project.authenticated_client().unwrap(); + let blocked_client = ClientBuilder::from_client(client.client().clone()) + .with(OfflineMiddleware) + .build(); + pypi_mapping::PurlDerivationClient::builder(blocked_client.into(), cache_dir).finish() +} + +fn conda_forge_record(name: &str) -> RepoDataRecord { + let package = Package::build(name, "2").finish(); + RepoDataRecord { + identifier: package.identifier(), + package_record: package.package_record, + url: Url::parse(&format!("https://pypi.org/simple/{name}/")).unwrap(), + channel: Some("https://conda.anaconda.org/conda-forge/".to_owned()), + } +} + +/// An inline mapping hit in the default (extend) mode is final and requires +/// no network access. +#[tokio::test] +async fn test_extend_mapping_inline_hit_without_network() { + setup_tracing(); + + let pixi = PixiControl::from_manifest( + r#" + [project] + name = "test-extend-inline" + channels = ["conda-forge"] + platforms = ["linux-64"] + conda-pypi-map = { conda-forge = { mapping = { pixi-something-new = "my-inline-name" } } } + "#, + ) + .unwrap(); + + let project = pixi.workspace().unwrap(); + let cache_dir = TempDir::new().unwrap(); + let mapping_client = offline_mapping_client(&project, cache_dir.path().to_path_buf()); + + let mut packages = vec![conda_forge_record("pixi-something-new")]; + mapping_client + .amend_purls( + project.pypi_name_derivation_mode().unwrap(), + &mut packages, + None, + ) + .await + .unwrap(); + + let package = packages.pop().unwrap(); + let purl = package + .package_record + .purls + .as_ref() + .and_then(BTreeSet::first) + .unwrap(); + assert_eq!(purl.name(), "my-inline-name"); + assert_eq!( + purl.qualifiers().get("source").unwrap(), + PurlDerivationSource::ProjectDefinedMapping.as_str() + ); +} + +/// An explicit `false` inline entry means "not a PyPI package": no purl is +/// derived and the conda-forge verbatim fallback does not kick in either. +#[tokio::test] +async fn test_extend_mapping_explicit_false_yields_no_purl() { + setup_tracing(); + + let pixi = PixiControl::from_manifest( + r#" + [project] + name = "test-extend-false" + channels = ["conda-forge"] + platforms = ["linux-64"] + conda-pypi-map = { conda-forge = { mapping = { pixi-something-new = false } } } + "#, + ) + .unwrap(); + + let project = pixi.workspace().unwrap(); + let cache_dir = TempDir::new().unwrap(); + let mapping_client = offline_mapping_client(&project, cache_dir.path().to_path_buf()); + + let mut packages = vec![conda_forge_record("pixi-something-new")]; + mapping_client + .amend_purls( + project.pypi_name_derivation_mode().unwrap(), + &mut packages, + None, + ) + .await + .unwrap(); + + let package = packages.pop().unwrap(); + assert!( + package + .package_record + .purls + .as_ref() + .is_none_or(|purls| purls.is_empty()), + "a package explicitly mapped to `false` must not get a purl" + ); +} + +/// ` = false` disables lookups for that channel; the offline +/// conda-forge verbatim fallback still applies. +#[tokio::test] +async fn test_channel_disabled_keeps_verbatim_fallback() { + setup_tracing(); + + let pixi = PixiControl::from_manifest( + r#" + [project] + name = "test-channel-disabled" + channels = ["conda-forge"] + platforms = ["linux-64"] + conda-pypi-map = { conda-forge = false } + "#, + ) + .unwrap(); + + let project = pixi.workspace().unwrap(); + let cache_dir = TempDir::new().unwrap(); + let mapping_client = offline_mapping_client(&project, cache_dir.path().to_path_buf()); + + let mut packages = vec![conda_forge_record("boltons")]; + mapping_client + .amend_purls( + project.pypi_name_derivation_mode().unwrap(), + &mut packages, + None, + ) + .await + .unwrap(); + + let package = packages.pop().unwrap(); + let purl = package + .package_record + .purls + .as_ref() + .and_then(BTreeSet::first) + .unwrap(); + // The verbatim fallback assumes the conda name is the pypi name and adds + // no source qualifier. + assert_eq!(purl.name(), "boltons"); + assert!(purl.qualifiers().is_empty()); +} + +/// When an entry has both a `location` and inline `mapping` entries, the +/// inline entries override the ones from the location. +#[tokio::test] +async fn test_inline_mapping_overrides_location() { + setup_tracing(); + + // The custom mapping file maps `pixi-something-new` to itself; the inline + // entry must win. + let pixi = PixiControl::from_manifest(&format!( + r#" + [project] + name = "test-inline-overrides" + channels = ["conda-forge"] + platforms = ["linux-64"] + conda-pypi-map = {{ conda-forge = {{ location = "{}", mapping = {{ pixi-something-new = "inline-wins" }} }} }} + "#, + absolute_custom_mapping_path() + )) + .unwrap(); + + let project = pixi.workspace().unwrap(); + let cache_dir = TempDir::new().unwrap(); + let mapping_client = offline_mapping_client(&project, cache_dir.path().to_path_buf()); + + let mut packages = vec![conda_forge_record("pixi-something-new")]; + mapping_client + .amend_purls( + project.pypi_name_derivation_mode().unwrap(), + &mut packages, + None, + ) + .await + .unwrap(); + + let package = packages.pop().unwrap(); + let purl = package + .package_record + .purls + .as_ref() + .and_then(BTreeSet::first) + .unwrap(); + assert_eq!(purl.name(), "inline-wins"); +} + +/// In extend mode a miss in the project-defined mapping falls through to the +/// prefix.dev chain. +#[tokio::test] +#[cfg_attr(not(feature = "online_tests"), ignore)] +async fn test_extend_mapping_miss_falls_through_to_prefix() { + setup_tracing(); + + // The mapping contains an unrelated package, so `boltons` is a miss and + // must be resolved through the prefix.dev chain (the mock-built record's + // hash is unknown there, so the compressed name mapping answers). + let pixi = PixiControl::from_manifest( + r#" + [project] + name = "test-extend-miss" + channels = ["conda-forge"] + platforms = ["linux-64"] + conda-pypi-map = { conda-forge = { mapping = { some-other-package = "other" } } } + "#, + ) + .unwrap(); + + let project = pixi.workspace().unwrap(); + let client = project.authenticated_client().unwrap(); + let mapping_client = pypi_mapping::PurlDerivationClient::builder( + client.clone(), + project + .config() + .cache_dir_for(pixi_config::CacheKind::PypiMapping) + .unwrap(), + ) + .finish(); + + let mut packages = vec![conda_forge_record("boltons")]; + mapping_client + .amend_purls( + project.pypi_name_derivation_mode().unwrap(), + &mut packages, + None, + ) + .await + .unwrap(); + + let package = packages.pop().unwrap(); + let purl = package + .package_record + .purls + .as_ref() + .and_then(BTreeSet::first) + .unwrap(); + assert_eq!(purl.name(), "boltons"); + assert_eq!( + purl.qualifiers().get("source").unwrap(), + PurlDerivationSource::PrefixCompressedMapping.as_str() + ); +} + +/// The on-disk path of the TTL cache for a mapping url, mirroring the layout +/// used by the project-defined mapping resolver. +fn ttl_cache_path_for(cache_dir: &Path, url: &str) -> std::path::PathBuf { + let hash = rattler_digest::compute_bytes_digest::(url.as_bytes()); + cache_dir + .join("project-defined") + .join(format!("{hash:x}.json")) +} + +fn manifest_with_ttl_mapping(url: &str, ttl: &str) -> String { + format!( + r#" + [project] + name = "test-cache-ttl" + channels = ["conda-forge"] + platforms = ["linux-64"] + conda-pypi-map = {{ conda-forge = {{ location = "{url}", cache-ttl = "{ttl}" }} }} + "# + ) +} + +/// A cached mapping younger than `cache-ttl` is used without any network +/// access. +#[tokio::test] +async fn test_cache_ttl_fresh_cache_skips_network() { + setup_tracing(); + + let mapping_url = "https://example.invalid/mapping.json"; + let pixi = PixiControl::from_manifest(&manifest_with_ttl_mapping(mapping_url, "1h")).unwrap(); + + let project = pixi.workspace().unwrap(); + let cache_dir = TempDir::new().unwrap(); + + // Pre-populate the TTL cache; the url itself is unreachable and the + // client is offline, so a cache miss would fail the test. + let cache_file = ttl_cache_path_for(cache_dir.path(), mapping_url); + fs_err::create_dir_all(cache_file.parent().unwrap()).unwrap(); + fs_err::write(&cache_file, r#"{ "pixi-something-new": "from-the-cache" }"#).unwrap(); + + let mapping_client = offline_mapping_client(&project, cache_dir.path().to_path_buf()); + + let mut packages = vec![conda_forge_record("pixi-something-new")]; + mapping_client + .amend_purls( + project.pypi_name_derivation_mode().unwrap(), + &mut packages, + None, + ) + .await + .unwrap(); + + let package = packages.pop().unwrap(); + let purl = package + .package_record + .purls + .as_ref() + .and_then(BTreeSet::first) + .unwrap(); + assert_eq!(purl.name(), "from-the-cache"); +} + +/// When the cached mapping is expired and the refetch fails, the stale copy +/// is used so solves keep working offline. +#[tokio::test] +async fn test_cache_ttl_expired_falls_back_to_stale_copy() { + setup_tracing(); + + let mapping_url = "https://example.invalid/mapping.json"; + // A zero TTL means the cached copy is always considered expired. + let pixi = PixiControl::from_manifest(&manifest_with_ttl_mapping(mapping_url, "0s")).unwrap(); + + let project = pixi.workspace().unwrap(); + let cache_dir = TempDir::new().unwrap(); + + let cache_file = ttl_cache_path_for(cache_dir.path(), mapping_url); + fs_err::create_dir_all(cache_file.parent().unwrap()).unwrap(); + fs_err::write(&cache_file, r#"{ "pixi-something-new": "stale-but-used" }"#).unwrap(); + + let mapping_client = offline_mapping_client(&project, cache_dir.path().to_path_buf()); + + let mut packages = vec![conda_forge_record("pixi-something-new")]; + mapping_client + .amend_purls( + project.pypi_name_derivation_mode().unwrap(), + &mut packages, + None, + ) + .await + .unwrap(); + + let package = packages.pop().unwrap(); + let purl = package + .package_record + .purls + .as_ref() + .and_then(BTreeSet::first) + .unwrap(); + assert_eq!(purl.name(), "stale-but-used"); +} + +/// Without any cached copy, a failing fetch of a TTL-cached mapping is a hard +/// error. +#[tokio::test] +async fn test_cache_ttl_no_cache_and_fetch_failure_errors() { + setup_tracing(); + + let mapping_url = "https://example.invalid/mapping.json"; + let pixi = PixiControl::from_manifest(&manifest_with_ttl_mapping(mapping_url, "1h")).unwrap(); + + let project = pixi.workspace().unwrap(); + let cache_dir = TempDir::new().unwrap(); + let mapping_client = offline_mapping_client(&project, cache_dir.path().to_path_buf()); + + let mut packages = vec![conda_forge_record("pixi-something-new")]; + let result = mapping_client + .amend_purls( + project.pypi_name_derivation_mode().unwrap(), + &mut packages, + None, + ) + .await; + + assert!( + result.is_err(), + "an uncached TTL mapping with a failing fetch must error" + ); +} + +/// A failing prefix.dev lookup must point firewall-restricted users at the +/// manifest options that avoid the network. +#[tokio::test] +async fn test_prefix_fetch_failure_error_mentions_escape_hatches() { + setup_tracing(); + + // No `conda-pypi-map` -> the default prefix.dev chain, which needs the + // network to look up the record by hash. + let pixi = PixiControl::from_manifest( + r#" + [project] + name = "test-network-error" + channels = ["conda-forge"] + platforms = ["linux-64"] + "#, + ) + .unwrap(); + + let project = pixi.workspace().unwrap(); + let cache_dir = TempDir::new().unwrap(); + let mapping_client = offline_mapping_client(&project, cache_dir.path().to_path_buf()); + + let mut packages = vec![conda_forge_record("pixi-something-new")]; + let err = mapping_client + .amend_purls( + project.pypi_name_derivation_mode().unwrap(), + &mut packages, + None, + ) + .await + .expect_err("an offline prefix.dev lookup should fail"); + + let rendered = format!("{err:?}"); + assert!( + rendered.contains("mode = \"replace\"") && rendered.contains("conda-pypi-map = false"), + "the error should suggest the offline escape hatches, got: {rendered}" + ); +} + +/// `conda-pypi-map = {}` is a soft-deprecated alias for +/// `conda-pypi-map = false`; both disable all mapping lookups while keeping +/// the conda-forge verbatim fallback. #[tokio::test] async fn test_disabled_mapping() { setup_tracing(); @@ -961,6 +1381,48 @@ async fn test_disabled_mapping() { assert!(boltons_first_purl.qualifiers().is_empty()); } +/// `conda-pypi-map = false` is the canonical global disable: no lookups, but +/// the conda-forge verbatim fallback still applies. +#[tokio::test] +async fn test_disabled_mapping_via_false() { + setup_tracing(); + + let pixi = PixiControl::from_manifest( + r#" + [project] + name = "test-disable-false" + channels = ["https://prefix.dev/conda-forge"] + platforms = ["linux-64"] + conda-pypi-map = false + "#, + ) + .unwrap(); + + let project = pixi.workspace().unwrap(); + let cache_dir = TempDir::new().unwrap(); + let mapping_client = offline_mapping_client(&project, cache_dir.path().to_path_buf()); + + let mut packages = vec![conda_forge_record("boltons")]; + mapping_client + .amend_purls( + project.pypi_name_derivation_mode().unwrap(), + &mut packages, + None, + ) + .await + .unwrap(); + + let boltons_package = packages.pop().unwrap(); + let boltons_first_purl = boltons_package + .package_record + .purls + .as_ref() + .and_then(BTreeSet::first) + .unwrap(); + assert_eq!(boltons_first_purl.name(), "boltons"); + assert!(boltons_first_purl.qualifiers().is_empty()); +} + #[tokio::test] async fn test_custom_mapping_ignores_backwards_compatibility() { setup_tracing(); @@ -997,7 +1459,7 @@ async fn test_custom_mapping_ignores_backwards_compatibility() { name = "test-custom-mapping" channels = ["{channel_url}"] platforms = ["linux-64"] - conda-pypi-map = {{ "{channel_url}" = "{mapping_file}" }} + conda-pypi-map = {{ "{channel_url}" = {{ location = "{mapping_file}", mode = "replace" }} }} [dependencies] python = "3.12.0" @@ -1092,7 +1554,7 @@ async fn test_solve_group_per_environment_editability() { name = "test-editability" channels = ["{channel}"] platforms = ["{platform}"] -conda-pypi-map = {{}} # disable mapping +conda-pypi-map = false # disable mapping [dependencies] python = "*" @@ -1177,7 +1639,7 @@ async fn test_transitive_uv_sources_editable_consistency() { name = "test-transitive-editable" channels = ["{channel}"] platforms = ["{platform}"] - conda-pypi-map = {{}} # disable mapping + conda-pypi-map = false # disable mapping [dependencies] python = "*" diff --git a/crates/pixi/tests/integration_rust/upgrade_tests.rs b/crates/pixi/tests/integration_rust/upgrade_tests.rs index 4e7961e13f..d1c53287a7 100644 --- a/crates/pixi/tests/integration_rust/upgrade_tests.rs +++ b/crates/pixi/tests/integration_rust/upgrade_tests.rs @@ -39,7 +39,7 @@ async fn pypi_dependency_index_preserved_on_upgrade() { [workspace] channels = ["{channel_url}"] platforms = ["{platform}"] - conda-pypi-map = {{}} + conda-pypi-map = false [pypi-dependencies] click = {{ version = "==8.2.0", index = "{pypi_index_url}" }} @@ -93,7 +93,7 @@ async fn pypi_dependency_index_preserved_on_upgrade() { [workspace] channels = ["[CHANNEL_URL]"] platforms = ["[PLATFORM]"] - conda-pypi-map = {} + conda-pypi-map = false [pypi-dependencies] click = { version = ">=8.3.1, <9", index = "[PYPI_INDEX_URL]" } @@ -326,7 +326,7 @@ async fn pypi_dependency_upgrade_uses_custom_index() { name = "pypi-upgrade-custom-index" platforms = ["{platform}"] channels = ["{channel}"] - conda-pypi-map = {{}} + conda-pypi-map = false [dependencies] python = "==3.12.0" From ca3ab776af4c31838b5eac1b1b041b00b5bb5b0e Mon Sep 17 00:00:00 2001 From: Tim de Jager Date: Wed, 10 Jun 2026 12:15:45 +0200 Subject: [PATCH 04/10] feat(pixi-build-python): pypi-conda-map user overrides for the pypi-to-conda mapping --- crates/pixi_build_python/src/config.rs | 144 ++++++++++- crates/pixi_build_python/src/main.rs | 10 + crates/pixi_build_python/src/pypi_mapping.rs | 236 ++++++++++++++++++- 3 files changed, 382 insertions(+), 8 deletions(-) diff --git a/crates/pixi_build_python/src/config.rs b/crates/pixi_build_python/src/config.rs index 3da5565189..9f4bbff2df 100644 --- a/crates/pixi_build_python/src/config.rs +++ b/crates/pixi_build_python/src/config.rs @@ -3,6 +3,64 @@ use pixi_build_backend::generated_recipe::BackendConfig; use serde::{Deserialize, Serialize}; use std::path::{Path, PathBuf}; +/// One value of the `pypi-conda-map` option: a conda package name, or `false` +/// to silently drop the dependency from the generated recipe. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum PypiCondaMapEntry { + /// Map the PyPI package to this conda package name. + CondaName(String), + /// Drop the dependency from the generated recipe. + Skip, +} + +impl Serialize for PypiCondaMapEntry { + fn serialize(&self, serializer: S) -> Result { + match self { + PypiCondaMapEntry::CondaName(name) => serializer.serialize_str(name), + PypiCondaMapEntry::Skip => serializer.serialize_bool(false), + } + } +} + +impl<'de> Deserialize<'de> for PypiCondaMapEntry { + fn deserialize>(deserializer: D) -> Result { + struct Visitor; + + impl<'de> serde::de::Visitor<'de> for Visitor { + type Value = PypiCondaMapEntry; + + fn expecting(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + f.write_str("a conda package name or `false`") + } + + fn visit_str(self, v: &str) -> Result { + Ok(PypiCondaMapEntry::CondaName(v.to_string())) + } + + fn visit_bool(self, v: bool) -> Result { + if v { + Err(E::custom( + "`true` is not supported; use a conda package name to map the \ + package, or `false` to drop the dependency", + )) + } else { + Ok(PypiCondaMapEntry::Skip) + } + } + + fn visit_unit(self) -> Result { + Ok(PypiCondaMapEntry::Skip) + } + + fn visit_none(self) -> Result { + Ok(PypiCondaMapEntry::Skip) + } + } + + deserializer.deserialize_any(Visitor) + } +} + /// Represents skip-pyc-compilation config: either `true` (skip all) or a list /// of glob patterns. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] @@ -63,6 +121,13 @@ pub struct PythonBackendConfig { /// Defaults to `true` (mapping disabled). #[serde(default)] pub ignore_pypi_mapping: Option, + /// User-defined overrides for the PyPI-to-conda name mapping, keyed by + /// PyPI package name. A string value maps the package to that conda name; + /// `false` drops the dependency from the generated recipe. Entries are + /// consulted before the remote mapping service. Only used when + /// `ignore-pypi-mapping = false`. + #[serde(default)] + pub pypi_conda_map: Option>, /// Whether the package uses the Python Stable ABI (abi3). /// When true, adds `python_abi` to host requirements. /// Only meaningful for packages with compiled extensions (non-noarch). @@ -137,6 +202,15 @@ impl BackendConfig for PythonBackendConfig { ignore_pypi_mapping: target_config .ignore_pypi_mapping .or(self.ignore_pypi_mapping), + pypi_conda_map: match (&self.pypi_conda_map, &target_config.pypi_conda_map) { + (None, None) => None, + (base, target) => { + // Per-key merge: target entries override or add to the base map. + let mut merged = base.clone().unwrap_or_default(); + merged.extend(target.clone().unwrap_or_default()); + Some(merged) + } + }, abi3: target_config.abi3.or(self.abi3), skip_pyc_compilation: if target_config.skip_pyc_compilation.is_none() { self.skip_pyc_compilation.clone() @@ -149,7 +223,7 @@ impl BackendConfig for PythonBackendConfig { #[cfg(test)] mod tests { - use super::{PythonBackendConfig, SkipPycCompilation}; + use super::{PypiCondaMapEntry, PythonBackendConfig, SkipPycCompilation}; use pixi_build_backend::generated_recipe::BackendConfig; use serde_json::json; use std::path::PathBuf; @@ -160,6 +234,71 @@ mod tests { serde_json::from_value::(json_data).unwrap(); } + #[test] + fn test_deserialize_pypi_conda_map() { + let json_data = json!({ + "pypi-conda-map": { + "torch": "pytorch", + "my-internal-pkg": false, + } + }); + let config = serde_json::from_value::(json_data).unwrap(); + let map = config.pypi_conda_map.unwrap(); + assert_eq!( + map.get("torch"), + Some(&PypiCondaMapEntry::CondaName("pytorch".to_string())) + ); + assert_eq!(map.get("my-internal-pkg"), Some(&PypiCondaMapEntry::Skip)); + } + + #[test] + fn test_deserialize_pypi_conda_map_rejects_true() { + let json_data = json!({ + "pypi-conda-map": { + "torch": true, + } + }); + let err = serde_json::from_value::(json_data).unwrap_err(); + assert!(err.to_string().contains("`true` is not supported")); + } + + #[test] + fn test_merge_pypi_conda_map_per_key() { + let base = PythonBackendConfig { + pypi_conda_map: Some(indexmap::indexmap! { + "torch".to_string() => PypiCondaMapEntry::CondaName("pytorch".to_string()), + "shared".to_string() => PypiCondaMapEntry::CondaName("base-name".to_string()), + }), + ..Default::default() + }; + let target = PythonBackendConfig { + pypi_conda_map: Some(indexmap::indexmap! { + "shared".to_string() => PypiCondaMapEntry::Skip, + "extra".to_string() => PypiCondaMapEntry::CondaName("extra-conda".to_string()), + }), + ..Default::default() + }; + + let merged = base.merge_with_target_config(&target).unwrap(); + let map = merged.pypi_conda_map.unwrap(); + // Base-only keys survive, target keys override or extend. + assert_eq!( + map.get("torch"), + Some(&PypiCondaMapEntry::CondaName("pytorch".to_string())) + ); + assert_eq!(map.get("shared"), Some(&PypiCondaMapEntry::Skip)); + assert_eq!( + map.get("extra"), + Some(&PypiCondaMapEntry::CondaName("extra-conda".to_string())) + ); + + // None + None stays None. + let merged = PythonBackendConfig::default() + .merge_with_target_config(&PythonBackendConfig::default()) + .unwrap(); + assert_eq!(merged.pypi_conda_map, None); + } + #[test] fn test_merge_with_target_config() { let mut base_env = indexmap::IndexMap::new(); @@ -175,6 +314,7 @@ mod tests { compilers: Some(vec!["c".to_string()]), ignore_pyproject_manifest: Some(true), ignore_pypi_mapping: Some(true), + pypi_conda_map: None, abi3: Some(true), skip_pyc_compilation: SkipPycCompilation::All(true), }; @@ -192,6 +332,7 @@ mod tests { compilers: Some(vec!["cxx".to_string(), "rust".to_string()]), ignore_pyproject_manifest: Some(false), ignore_pypi_mapping: Some(false), + pypi_conda_map: None, abi3: Some(false), skip_pyc_compilation: SkipPycCompilation::Globs(vec!["tests/**".to_string()]), }; @@ -252,6 +393,7 @@ mod tests { compilers: None, ignore_pyproject_manifest: Some(true), ignore_pypi_mapping: Some(true), + pypi_conda_map: None, abi3: None, skip_pyc_compilation: SkipPycCompilation::All(true), }; diff --git a/crates/pixi_build_python/src/main.rs b/crates/pixi_build_python/src/main.rs index c5f060c150..155296943f 100644 --- a/crates/pixi_build_python/src/main.rs +++ b/crates/pixi_build_python/src/main.rs @@ -276,11 +276,20 @@ impl GenerateRecipe for PythonGenerator { host_platform }; + // Warn when the user supplied mapping overrides that cannot take effect. + if config.pypi_conda_map.is_some() && config.ignore_pypi_mapping() { + tracing::warn!( + "`pypi-conda-map` is set but the PyPI-to-conda mapping is disabled; set \ + `ignore-pypi-mapping = false` to use it" + ); + } + // Map PyPI dependencies from pyproject.toml to conda dependencies if !config.ignore_pypi_mapping() { if let Some(pypi_deps) = pyproject_metadata_provider.project_dependencies()? { let mapped_deps = map_requirements_with_channels( pypi_deps, + config.pypi_conda_map.as_ref(), &channels, &cache_dir, "project", @@ -306,6 +315,7 @@ impl GenerateRecipe for PythonGenerator { if let Some(build_system_deps) = pyproject_metadata_provider.build_system_requires()? { let mapped_deps = map_requirements_with_channels( build_system_deps, + config.pypi_conda_map.as_ref(), &channels, &cache_dir, "build-system", diff --git a/crates/pixi_build_python/src/pypi_mapping.rs b/crates/pixi_build_python/src/pypi_mapping.rs index 912b9d5f6c..1b3cd6d1cc 100644 --- a/crates/pixi_build_python/src/pypi_mapping.rs +++ b/crates/pixi_build_python/src/pypi_mapping.rs @@ -21,6 +21,8 @@ use rattler_conda_types::{ use serde::{Deserialize, Serialize}; use thiserror::Error; +use crate::config::PypiCondaMapEntry; + /// Base URL for the PyPI to conda mapping API (without channel suffix). const MAPPING_BASE_URL: &str = "https://conda-mapping.prefix.dev/pypi-to-conda-v1"; @@ -507,32 +509,128 @@ pub fn extract_channel_name(channel: &ChannelUrl) -> Option<&str> { channel.as_str().trim_end_matches('/').rsplit('/').next() } -/// Map PyPI requirements to conda dependencies using the first channel that provides a valid mapping. +/// Resolve requirements against the user-defined `pypi-conda-map` overrides. /// -/// Tries each channel in order and returns the mapped dependencies from the first -/// channel that successfully maps at least one dependency. Returns an empty Vec -/// if no channel provides a mapping. +/// Returns the dependencies mapped (or dropped) by the user map together with +/// the requirements that are not covered by it and still need the remote +/// mapping service. Environment markers are evaluated for user-mapped +/// requirements exactly like for service-mapped ones. +fn apply_user_map( + requirements: &[pep508_rs::Requirement], + user_map: Option<&IndexMap>, + platform: Platform, +) -> ( + Vec, + Vec>, +) { + // Normalize the user-map keys so that e.g. `My_Pkg` matches `my-pkg`. + let normalized: IndexMap = user_map + .map(|map| { + map.iter() + .filter_map(|(name, entry)| match pep508_rs::PackageName::from_str(name) { + Ok(name) => Some((name, entry)), + Err(err) => { + tracing::warn!( + "ignoring invalid PyPI package name '{name}' in `pypi-conda-map`: {err}" + ); + None + } + }) + .collect() + }) + .unwrap_or_default(); + + let mut user_mapped = Vec::new(); + let mut remaining = Vec::new(); + for req in requirements { + let Some(entry) = normalized.get(&req.name) else { + remaining.push(req.clone()); + continue; + }; + + // Markers apply to user-mapped dependencies too. + if PyPiToCondaMapper::should_skip_requirement(req, platform) { + tracing::debug!( + "Skipping user-mapped dependency '{}' due to environment marker evaluation: {:?}", + req.name, + req.marker + ); + continue; + } + + match entry { + PypiCondaMapEntry::Skip => { + tracing::debug!( + "Dropping dependency '{}' because it is mapped to `false` in `pypi-conda-map`", + req.name + ); + } + PypiCondaMapEntry::CondaName(conda_name) => match PackageName::from_str(conda_name) { + Ok(name) => { + let version_spec = req.version_or_url.as_ref().and_then(|version_or_url| { + match PyPiToCondaMapper::convert_version_specifiers(version_or_url) { + Ok(spec) => spec, + Err(err) => { + tracing::warn!( + "Failed to convert version specifier for '{}': {}, using unconstrained version", + req.name, + err + ); + None + } + } + }); + user_mapped.push(MappedCondaDependency { name, version_spec }); + } + Err(err) => { + tracing::warn!( + "ignoring `pypi-conda-map` entry for '{}': invalid conda package name '{conda_name}': {err}", + req.name + ); + } + }, + } + } + (user_mapped, remaining) +} + +/// Map PyPI requirements to conda dependencies, consulting the user-defined +/// `pypi-conda-map` overrides first and the first channel that provides a +/// valid mapping for the rest. +/// +/// Tries each channel in order and returns the mapped dependencies from the +/// first channel that successfully maps at least one dependency. Requirements +/// resolved by the user map never reach the network and do not count towards +/// the channel selection. /// /// The `context` parameter is used for logging (e.g., "project dependencies" or /// "build-system requirements"). pub async fn map_requirements_with_channels( requirements: &[pep508_rs::Requirement], + user_map: Option<&IndexMap>, channels: &[ChannelUrl], cache_dir: &Option, context: &str, platform: Platform, ) -> Vec { + let (mut user_mapped, remaining) = apply_user_map(requirements, user_map, platform); + + if remaining.is_empty() { + return user_mapped; + } + for channel in channels { if let Some(channel_name) = extract_channel_name(channel) { let mapper = PyPiToCondaMapper::new(cache_dir.clone(), channel_name.to_string()); - match mapper.map_requirements(requirements, platform).await { + match mapper.map_requirements(&remaining, platform).await { Ok(deps) if !deps.is_empty() => { tracing::debug!( "Using PyPI-to-conda mapping for {} from channel '{}'", context, channel_name ); - return deps; + user_mapped.extend(deps); + return user_mapped; } Ok(_) => { tracing::warn!( @@ -552,7 +650,7 @@ pub async fn map_requirements_with_channels( } } } - Vec::new() + user_mapped } /// Build tools that require specific compilers. @@ -706,6 +804,130 @@ mod tests { } } + fn requirement(s: &str) -> pep508_rs::Requirement { + pep508_rs::Requirement::from_str(s).unwrap() + } + + #[test] + fn test_apply_user_map_override_and_skip() { + let user_map = IndexMap::from([ + ( + "torch".to_string(), + PypiCondaMapEntry::CondaName("pytorch".to_string()), + ), + ("my-internal-pkg".to_string(), PypiCondaMapEntry::Skip), + ]); + + let requirements = vec![ + requirement("torch>=2.0"), + requirement("my-internal-pkg"), + requirement("numpy"), + ]; + + let (mapped, remaining) = apply_user_map(&requirements, Some(&user_map), Platform::Linux64); + + // `torch` is mapped with its version spec, `my-internal-pkg` is + // silently dropped, `numpy` is left for the mapping service. + assert_eq!(mapped.len(), 1); + assert_eq!(mapped[0].name.as_normalized(), "pytorch"); + assert_eq!( + mapped[0].version_spec.as_ref().unwrap().to_string(), + ">=2.0" + ); + assert_eq!(remaining.len(), 1); + assert_eq!(remaining[0].name.as_ref(), "numpy"); + } + + #[test] + fn test_apply_user_map_normalizes_names() { + // `My_Pkg` in the config must match the normalized requirement `my-pkg`. + let user_map = IndexMap::from([( + "My_Pkg".to_string(), + PypiCondaMapEntry::CondaName("my-conda-pkg".to_string()), + )]); + + let requirements = vec![requirement("my-pkg")]; + let (mapped, remaining) = apply_user_map(&requirements, Some(&user_map), Platform::Linux64); + + assert_eq!(mapped.len(), 1); + assert_eq!(mapped[0].name.as_normalized(), "my-conda-pkg"); + assert!(remaining.is_empty()); + } + + #[test] + fn test_apply_user_map_respects_markers() { + let user_map = IndexMap::from([( + "torch".to_string(), + PypiCondaMapEntry::CondaName("pytorch".to_string()), + )]); + + // The marker does not apply to linux-64, so the user-mapped + // dependency is dropped entirely. + let requirements = vec![requirement("torch; sys_platform == 'win32'")]; + let (mapped, remaining) = apply_user_map(&requirements, Some(&user_map), Platform::Linux64); + assert!(mapped.is_empty()); + assert!(remaining.is_empty()); + + // On NoArch any marker-bearing dependency is dropped. + let (mapped, remaining) = apply_user_map(&requirements, Some(&user_map), Platform::NoArch); + assert!(mapped.is_empty()); + assert!(remaining.is_empty()); + } + + #[test] + fn test_apply_user_map_invalid_conda_name_is_skipped() { + let user_map = IndexMap::from([( + "torch".to_string(), + PypiCondaMapEntry::CondaName("not a valid name!".to_string()), + )]); + + let requirements = vec![requirement("torch")]; + let (mapped, remaining) = apply_user_map(&requirements, Some(&user_map), Platform::Linux64); + // The override is consumed (warned about) rather than handed to the + // mapping service. + assert!(mapped.is_empty()); + assert!(remaining.is_empty()); + } + + #[test] + fn test_apply_user_map_converts_pep440_operators() { + let user_map = IndexMap::from([( + "torch".to_string(), + PypiCondaMapEntry::CondaName("pytorch".to_string()), + )]); + + let requirements = vec![requirement("torch===2.0.0")]; + let (mapped, _) = apply_user_map(&requirements, Some(&user_map), Platform::Linux64); + assert_eq!( + mapped[0].version_spec.as_ref().unwrap().to_string(), + "==2.0.0" + ); + } + + #[tokio::test] + async fn test_map_requirements_with_channels_all_user_mapped() { + let user_map = IndexMap::from([( + "torch".to_string(), + PypiCondaMapEntry::CondaName("pytorch".to_string()), + )]); + + // All requirements are covered by the user map: no channel lookup + // happens (there are no channels to consult either). + let requirements = vec![requirement("torch>=2.0")]; + let mapped = map_requirements_with_channels( + &requirements, + Some(&user_map), + &[], + &None, + "test", + Platform::Linux64, + ) + .await; + + assert_eq!(mapped.len(), 1); + assert_eq!(mapped[0].name.as_normalized(), "pytorch"); + } + #[test] fn test_filter_mapped_pypi_deps_without_pixi_deps() { // When no Pixi deps are specified, all mapped deps should pass through From bb09d3c7f0299f4a0f0c97b86bd8822957f307ad Mon Sep 17 00:00:00 2001 From: Tim de Jager Date: Wed, 10 Jun 2026 12:21:06 +0200 Subject: [PATCH 05/10] docs+schema: additive conda-pypi-map forms, pypi-conda-map backend option and breaking-change notes --- docs/build/backends/pixi-build-python.md | 34 +++++- docs/concepts/conda_pypi.md | 27 +++++ docs/reference/pixi_manifest.md | 49 ++++++--- .../examples/invalid/bad_conda_pypi_map.toml | 6 ++ .../invalid/bad_conda_pypi_map_mode.toml | 7 ++ schema/examples/valid/full.toml | 2 +- schema/model.py | 33 +++++- schema/pyproject/partial-pixi.json | 101 +++++++++++++++--- schema/pyproject/schema.json | 101 +++++++++++++++--- schema/schema.json | 101 +++++++++++++++--- 10 files changed, 401 insertions(+), 60 deletions(-) create mode 100644 schema/examples/invalid/bad_conda_pypi_map.toml create mode 100644 schema/examples/invalid/bad_conda_pypi_map_mode.toml diff --git a/docs/build/backends/pixi-build-python.md b/docs/build/backends/pixi-build-python.md index bca6456f33..c7fe599095 100644 --- a/docs/build/backends/pixi-build-python.md +++ b/docs/build/backends/pixi-build-python.md @@ -346,6 +346,36 @@ ignore-pypi-mapping = true # Disable mapping on Windows only # Result for win-64: true ``` +### `pypi-conda-map` + +- **Type**: `Map` +- **Default**: not set +- **Target Merge Behavior**: `Merge` - Platform-specific entries override or extend base entries per key + +User-defined overrides for the PyPI-to-conda name mapping, keyed by PyPI package name. +A string value maps the package to that conda name; `false` silently drops the dependency from the generated recipe. +Entries are consulted before the remote mapping service, so they never require network access; packages not in the map fall back to the service as usual. + +Only used when `ignore-pypi-mapping = false`; otherwise it has no effect and a warning is logged. +The overrides apply to both mapping passes: `project.dependencies` (run dependencies) and `build-system.requires` (host dependencies). + +```toml +[package.build.config] +ignore-pypi-mapping = false +pypi-conda-map = { torch = "pytorch", my-internal-pkg = false } +``` + +Per-platform entries merge with the base map key-by-key: + +```toml +[package.build.config] +pypi-conda-map = { torch = "pytorch" } + +[package.build.target.linux-64.config] +pypi-conda-map = { nvidia-cublas-cu12 = false } +# Result for linux-64: { torch = "pytorch", nvidia-cublas-cu12 = false } +``` + ## Automatic PyPI Dependency Mapping The Python backend can automatically map PyPI dependencies from your `pyproject.toml` to their corresponding conda packages. @@ -366,7 +396,7 @@ The backend reads dependencies from two sources in your `pyproject.toml`: 1. **`project.dependencies`** → Added to conda **run** dependencies 2. **`build-system.requires`** → Added to conda **host** dependencies -For each PyPI package, the backend queries a mapping service to find the corresponding conda-forge package name. The mapping is cached locally for 24 hours to improve performance. +For each PyPI package, the backend first consults the user-defined [`pypi-conda-map`](#pypi-conda-map) overrides, and then queries a mapping service to find the corresponding conda-forge package name. The mapping is cached locally for 24 hours to improve performance. ### Example @@ -408,7 +438,7 @@ This allows you to: - **Environment markers** (e.g., `requests>=2.28; python_version >= "3.8"`) are only partially supported. At the moment, only `platform_system`, `os_name`, `platform_machine` and `sys_platforms` are currently checked. - **URL-based dependencies** (e.g., `package @ https://...`) are skipped -- Packages without a conda-forge mapping are logged as warnings and skipped +- Packages without a conda-forge mapping are logged as warnings and skipped; use [`pypi-conda-map`](#pypi-conda-map) to map them explicitly or drop them silently with `false` ## Build Process diff --git a/docs/concepts/conda_pypi.md b/docs/concepts/conda_pypi.md index 8d4e71757f..5a6c22f643 100644 --- a/docs/concepts/conda_pypi.md +++ b/docs/concepts/conda_pypi.md @@ -105,6 +105,33 @@ Then, since `numpy` is not specified as a conda dependency, Pixi will resolve th To override or change the mapping of conda packages to PyPI packages, you can use the [`conda-pypi-map`](../reference/pixi_manifest.md#conda-pypi-map-optional) field in the `pixi.toml` file. +### Overriding the name mapping + +`conda-pypi-map` layers your entries *on top of* the default mapping: for each package pixi first consults your entries, and only falls back to the default mapping (and finally the "conda-forge name equals PyPI name" heuristic) when your mapping does not mention the package. +This means fixing a single mis-mapped package is a one-liner: + +```toml title="pixi.toml" +[workspace.conda-pypi-map] +conda-forge = { mapping = { pytorch = "torch" } } +``` + +If you want your mapping to be *exclusive* — only packages you list are treated as PyPI packages — use `mode = "replace"`: + +```toml title="pixi.toml" +[workspace.conda-pypi-map] +conda-forge = { location = "full-mapping.json", mode = "replace" } +``` + +### Offline and firewall-restricted environments + +The default mapping is fetched from `conda-mapping.prefix.dev`. +If that host is unreachable in your environment, you have several options: + +- `conda-pypi-map = false` disables all mapping lookups. Conda-forge packages are still assumed to be the PyPI package of the same name, which requires no network access. +- ` = false` disables lookups for a single channel. +- `mode = "replace"` with your own mapping file avoids network lookups for that channel entirely. +- `cache-ttl = "24h"` on a URL location caches the fetched mapping on disk and re-fetches it at most once per TTL; if the re-fetch fails, the cached copy is used. + ### PyPI overrides vs conda constraints PyPI's [`pypi-options.dependency-overrides`](../advanced/override.md) diff --git a/docs/reference/pixi_manifest.md b/docs/reference/pixi_manifest.md index bbe0fa9e52..64c3597376 100644 --- a/docs/reference/pixi_manifest.md +++ b/docs/reference/pixi_manifest.md @@ -196,32 +196,55 @@ URL of the workspace documentation. ### `conda-pypi-map` (optional) -Mapping of channel name or URL to location of mapping that can be URL/Path. -Mapping should be structured in `json` format where `conda_name`: `pypi_package_name`. -Example: +Per-channel overrides for the conda to PyPI name mapping that pixi uses to detect which conda packages satisfy PyPI dependencies. +Each entry maps a channel name or URL to either a mapping location (URL or path), a table, or `false`: + +```toml +[workspace.conda-pypi-map] +# Additive overlay from a file: entries win, everything else uses the default mapping. +robostack = "local/robostack_mapping.json" +# Inline entries, no file needed. `false` means "not a PyPI package". +pytorch = { mode = "extend", mapping = { pytorch = "torch", not-on-pypi = false } } +# Exclusive mapping: ONLY packages in this file get mapped, no default mapping. +conda-forge = { location = "https://example.com/mapping.json", mode = "replace" } +# Re-fetch a remote mapping at most once a day; a cached copy is used otherwise. +my-company = { location = "https://internal.example.com/map.json", cache-ttl = "24h" } +# Disable mapping lookups for this channel entirely. +internal = false +``` + +Mapping files are structured in `json` format with `conda_name: pypi_package_name` entries, where `null` marks a package as not available on PyPI: ```json title="local/robostack_mapping.json" { "jupyter-ros": "my-name-from-mapping", - "boltons": "boltons-pypi" + "boltons": "boltons-pypi", + "not-on-pypi": null } ``` -If `conda-forge` is not present in `conda-pypi-map` `pixi` will use `prefix.dev` mapping for it. +The table form accepts: -```toml -conda-pypi-map = { conda-forge = "https://example.com/mapping", "https://repo.prefix.dev/robostack" = "local/robostack_mapping.json"} -``` +- `location`: URL or path of a mapping `json` file. Relative paths are resolved against the workspace root. +- `mapping`: inline `conda_name = "pypi_name"` entries. A value of `false` marks the package as not available on PyPI. Inline entries override entries from `location`. +- `mode`: `"extend"` (default) or `"replace"`. With `extend`, your entries are consulted first and anything not listed falls back to the default [prefix.dev mapping](https://conda-mapping.prefix.dev/). With `replace`, only packages listed in your mapping are considered PyPI packages; no network lookups happen for that channel. +- `cache-ttl`: a duration like `"24h"` or `"7d"`. The mapping fetched from a `location` URL is cached on disk and only re-fetched once it is older than this. If a re-fetch fails (e.g. offline), the stale cached copy is used with a warning. Only valid for `http(s)` locations. -It is also possible to disable fetching external mpping by adding an empty map to the list +To disable the mapping, either per channel or entirely: ```toml -conda-pypi-map = { conda-forge = "map.json" } +[workspace] +conda-pypi-map = false # disable for all channels +# or +conda-pypi-map = { conda-forge = false } # disable for one channel ``` -```json title="map.json" -{} -``` +Even with the mapping disabled, conda-forge packages are still assumed to be the PyPI package of the same name (an offline heuristic that requires no network access). + +!!! warning "Behavior change" + Bare location strings (`conda-forge = "mapping.json"`) used to be *exclusive*: only packages in your file were mapped. + They are now *additive* (`mode = "extend"`). To restore the old behavior, use the table form with `mode = "replace"`. + The previous idiom of disabling the mapping with an empty map (`conda-pypi-map = {}`) is deprecated; use `conda-pypi-map = false` instead. ### `channel-priority` (optional) diff --git a/schema/examples/invalid/bad_conda_pypi_map.toml b/schema/examples/invalid/bad_conda_pypi_map.toml new file mode 100644 index 0000000000..95d5f84398 --- /dev/null +++ b/schema/examples/invalid/bad_conda_pypi_map.toml @@ -0,0 +1,6 @@ +[project] +name = "bad-conda-pypi-map" +channels = ["conda-forge"] +platforms = ["linux-64"] +# `true` is not a valid channel entry; only a string, a table or `false` is. +conda-pypi-map = { conda-forge = true } diff --git a/schema/examples/invalid/bad_conda_pypi_map_mode.toml b/schema/examples/invalid/bad_conda_pypi_map_mode.toml new file mode 100644 index 0000000000..d57ceb39fa --- /dev/null +++ b/schema/examples/invalid/bad_conda_pypi_map_mode.toml @@ -0,0 +1,7 @@ +[project] +name = "bad-conda-pypi-map-mode" +channels = ["conda-forge"] +platforms = ["linux-64"] +# `mode` only accepts `extend` or `replace`, and inline mapping values must +# be a string or `false`. +conda-pypi-map = { conda-forge = { location = "mapping.json", mode = "bogus", mapping = { x = true } } } diff --git a/schema/examples/valid/full.toml b/schema/examples/valid/full.toml index 46e32c6aab..c15df03dd9 100644 --- a/schema/examples/valid/full.toml +++ b/schema/examples/valid/full.toml @@ -4,7 +4,7 @@ authors = ["Author "] channel-priority = "strict" channels = ["stable"] -conda-pypi-map = { "robostack" = "robostack_mapping.json", "conda-forge" = "https://repo.prefix.dev/conda-forge" } +conda-pypi-map = { "robostack" = "robostack_mapping.json", "conda-forge" = { location = "https://repo.prefix.dev/conda-forge", mode = "replace", cache-ttl = "24h" }, "pytorch" = { mode = "extend", mapping = { pytorch = "torch", not-on-pypi = false } }, "internal" = false } description = "A project" documentation = "https://docs.project.com" homepage = "https://project.com" diff --git a/schema/model.py b/schema/model.py index 868eeb33ef..f1be645715 100644 --- a/schema/model.py +++ b/schema/model.py @@ -200,6 +200,33 @@ class ChannelInlineTable(StrictBaseModel): Channel = ChannelName | ChannelInlineTable +class CondaPypiMapTable(StrictBaseModel): + """The mapping configuration for one channel in `conda-pypi-map`.""" + + location: AnyHttpUrl | NonEmptyStr | None = Field( + None, description="The URL or path to a mapping file with `conda_name: pypi_name` entries" + ) + mapping: dict[NonEmptyStr, NonEmptyStr | Literal[False]] | None = Field( + None, + description="Inline `conda_name: pypi_name` entries; `false` marks a package as not " + "available on PyPI. Inline entries override entries from `location`.", + ) + mode: Literal["extend", "replace"] | None = Field( + None, + description="How the mapping interacts with the default mapping: `extend` (default) " + "overlays it, `replace` makes it exclusive", + ) + cache_ttl: NonEmptyStr | None = Field( + None, + description="How long a mapping fetched from a URL may be reused before it is " + 're-fetched (e.g. `"24h"`, `"7d"`). Only valid for http(s) locations.', + ) + + +CondaPypiMapEntry = AnyHttpUrl | NonEmptyStr | Literal[False] | CondaPypiMapTable +CondaPypiMap = dict[ChannelName, CondaPypiMapEntry] | Literal[False] + + class ChannelPriority(str, Enum): """The priority of the channel.""" @@ -296,8 +323,10 @@ class Workspace(StrictBaseModel): documentation: AnyHttpUrl | None = Field( None, description="The URL of the documentation of the project" ) - conda_pypi_map: dict[ChannelName, AnyHttpUrl | NonEmptyStr] | None = Field( - None, description="The `conda` to PyPI mapping configuration" + conda_pypi_map: CondaPypiMap | None = Field( + None, + description="The `conda` to PyPI mapping configuration; `false` disables the mapping " + "entirely", ) pypi_options: PyPIOptions | None = Field( None, description="Options related to PyPI indexes for this project" diff --git a/schema/pyproject/partial-pixi.json b/schema/pyproject/partial-pixi.json index b2a487ecf2..f2fbc61512 100644 --- a/schema/pyproject/partial-pixi.json +++ b/schema/pyproject/partial-pixi.json @@ -657,6 +657,64 @@ "strict" ] }, + "CondaPypiMapTable": { + "title": "CondaPypiMapTable", + "description": "The mapping configuration for one channel in `conda-pypi-map`.", + "type": "object", + "additionalProperties": false, + "properties": { + "cache-ttl": { + "title": "Cache-Ttl", + "description": "How long a mapping fetched from a URL may be reused before it is re-fetched (e.g. `\"24h\"`, `\"7d\"`). Only valid for http(s) locations.", + "type": "string", + "minLength": 1 + }, + "location": { + "title": "Location", + "description": "The URL or path to a mapping file with `conda_name: pypi_name` entries", + "anyOf": [ + { + "type": "string", + "format": "uri", + "minLength": 1 + }, + { + "type": "string", + "minLength": 1 + } + ] + }, + "mapping": { + "title": "Mapping", + "description": "Inline `conda_name: pypi_name` entries; `false` marks a package as not available on PyPI. Inline entries override entries from `location`.", + "type": "object", + "additionalProperties": { + "anyOf": [ + { + "type": "string", + "minLength": 1 + }, + { + "type": "boolean", + "const": false + } + ] + }, + "propertyNames": { + "minLength": 1 + } + }, + "mode": { + "title": "Mode", + "description": "How the mapping interacts with the default mapping: `extend` (default) overlays it, `replace` makes it exclusive", + "type": "string", + "enum": [ + "extend", + "replace" + ] + } + } + }, "DependsOn": { "title": "DependsOn", "description": "The dependencies of a task.", @@ -3083,21 +3141,36 @@ }, "conda-pypi-map": { "title": "Conda-Pypi-Map", - "description": "The `conda` to PyPI mapping configuration", - "type": "object", - "additionalProperties": { - "anyOf": [ - { - "type": "string", - "format": "uri", - "minLength": 1 - }, - { - "type": "string", - "minLength": 1 + "description": "The `conda` to PyPI mapping configuration; `false` disables the mapping entirely", + "anyOf": [ + { + "type": "object", + "additionalProperties": { + "anyOf": [ + { + "type": "string", + "format": "uri", + "minLength": 1 + }, + { + "type": "string", + "minLength": 1 + }, + { + "type": "boolean", + "const": false + }, + { + "$ref": "#/$defs/CondaPypiMapTable" + } + ] } - ] - } + }, + { + "type": "boolean", + "const": false + } + ] }, "dependencies": { "title": "Dependencies", diff --git a/schema/pyproject/schema.json b/schema/pyproject/schema.json index d105b79001..59a12c0152 100644 --- a/schema/pyproject/schema.json +++ b/schema/pyproject/schema.json @@ -400,6 +400,64 @@ "strict" ] }, + "CondaPypiMapTable": { + "title": "CondaPypiMapTable", + "description": "The mapping configuration for one channel in `conda-pypi-map`.", + "type": "object", + "additionalProperties": false, + "properties": { + "cache-ttl": { + "title": "Cache-Ttl", + "description": "How long a mapping fetched from a URL may be reused before it is re-fetched (e.g. `\"24h\"`, `\"7d\"`). Only valid for http(s) locations.", + "type": "string", + "minLength": 1 + }, + "location": { + "title": "Location", + "description": "The URL or path to a mapping file with `conda_name: pypi_name` entries", + "anyOf": [ + { + "type": "string", + "format": "uri", + "minLength": 1 + }, + { + "type": "string", + "minLength": 1 + } + ] + }, + "mapping": { + "title": "Mapping", + "description": "Inline `conda_name: pypi_name` entries; `false` marks a package as not available on PyPI. Inline entries override entries from `location`.", + "type": "object", + "additionalProperties": { + "anyOf": [ + { + "type": "string", + "minLength": 1 + }, + { + "type": "boolean", + "const": false + } + ] + }, + "propertyNames": { + "minLength": 1 + } + }, + "mode": { + "title": "Mode", + "description": "How the mapping interacts with the default mapping: `extend` (default) overlays it, `replace` makes it exclusive", + "type": "string", + "enum": [ + "extend", + "replace" + ] + } + } + }, "DependsOn": { "title": "DependsOn", "description": "The dependencies of a task.", @@ -3121,21 +3179,36 @@ }, "conda-pypi-map": { "title": "Conda-Pypi-Map", - "description": "The `conda` to PyPI mapping configuration", - "type": "object", - "additionalProperties": { - "anyOf": [ - { - "type": "string", - "format": "uri", - "minLength": 1 - }, - { - "type": "string", - "minLength": 1 + "description": "The `conda` to PyPI mapping configuration; `false` disables the mapping entirely", + "anyOf": [ + { + "type": "object", + "additionalProperties": { + "anyOf": [ + { + "type": "string", + "format": "uri", + "minLength": 1 + }, + { + "type": "string", + "minLength": 1 + }, + { + "type": "boolean", + "const": false + }, + { + "$ref": "#/$defs/CondaPypiMapTable" + } + ] } - ] - } + }, + { + "type": "boolean", + "const": false + } + ] }, "dependencies": { "title": "Dependencies", diff --git a/schema/schema.json b/schema/schema.json index 07fe92b5c6..e756bfb6ab 100644 --- a/schema/schema.json +++ b/schema/schema.json @@ -681,6 +681,64 @@ "strict" ] }, + "CondaPypiMapTable": { + "title": "CondaPypiMapTable", + "description": "The mapping configuration for one channel in `conda-pypi-map`.", + "type": "object", + "additionalProperties": false, + "properties": { + "cache-ttl": { + "title": "Cache-Ttl", + "description": "How long a mapping fetched from a URL may be reused before it is re-fetched (e.g. `\"24h\"`, `\"7d\"`). Only valid for http(s) locations.", + "type": "string", + "minLength": 1 + }, + "location": { + "title": "Location", + "description": "The URL or path to a mapping file with `conda_name: pypi_name` entries", + "anyOf": [ + { + "type": "string", + "format": "uri", + "minLength": 1 + }, + { + "type": "string", + "minLength": 1 + } + ] + }, + "mapping": { + "title": "Mapping", + "description": "Inline `conda_name: pypi_name` entries; `false` marks a package as not available on PyPI. Inline entries override entries from `location`.", + "type": "object", + "additionalProperties": { + "anyOf": [ + { + "type": "string", + "minLength": 1 + }, + { + "type": "boolean", + "const": false + } + ] + }, + "propertyNames": { + "minLength": 1 + } + }, + "mode": { + "title": "Mode", + "description": "How the mapping interacts with the default mapping: `extend` (default) overlays it, `replace` makes it exclusive", + "type": "string", + "enum": [ + "extend", + "replace" + ] + } + } + }, "DependsOn": { "title": "DependsOn", "description": "The dependencies of a task.", @@ -3107,21 +3165,36 @@ }, "conda-pypi-map": { "title": "Conda-Pypi-Map", - "description": "The `conda` to PyPI mapping configuration", - "type": "object", - "additionalProperties": { - "anyOf": [ - { - "type": "string", - "format": "uri", - "minLength": 1 - }, - { - "type": "string", - "minLength": 1 + "description": "The `conda` to PyPI mapping configuration; `false` disables the mapping entirely", + "anyOf": [ + { + "type": "object", + "additionalProperties": { + "anyOf": [ + { + "type": "string", + "format": "uri", + "minLength": 1 + }, + { + "type": "string", + "minLength": 1 + }, + { + "type": "boolean", + "const": false + }, + { + "$ref": "#/$defs/CondaPypiMapTable" + } + ] } - ] - } + }, + { + "type": "boolean", + "const": false + } + ] }, "dependencies": { "title": "Dependencies", From cf0209d56d4601423452f42afd470fedc9411423 Mon Sep 17 00:00:00 2001 From: Tim de Jager Date: Wed, 10 Jun 2026 14:28:54 +0200 Subject: [PATCH 06/10] refactor: address review feedback on conda-pypi-map - Move all mapping/purl tests from solve_group_tests.rs into a new conda_pypi_map_tests.rs integration module. - Split CondaPypiMapEntry::Map into CondaPypiMapSpec with a dedicated MappingLocationSpec { location, cache_ttl } so the TTL is structurally tied to the location source it applies to. - Clarify in the Disabled doc comments that the offline conda-forge verbatim fallback still applies when lookups are disabled. - Deduplicate the offline help text into a shared MAPPING_OFFLINE_HELP const used by both the prefix.dev and project-defined fetch errors, and mention pointing at a custom mapping location (with cache-ttl) as an escape hatch. - Document why the TTL cache cannot reuse the http-cache middleware (header-driven freshness, client-global max_ttl, no stale-on-error). - Docs: add a parselmouth raw-URL pinning recipe (and a note that blob URLs serve HTML). --- .../integration_rust/conda_pypi_map_tests.rs | 1415 +++++++++++++++++ crates/pixi/tests/integration_rust/main.rs | 1 + ...ing_mapping_file_error_includes_path.snap} | 0 .../integration_rust/solve_group_tests.rs | 1405 +--------------- crates/pixi_core/src/workspace/mod.rs | 24 +- crates/pixi_manifest/src/lib.rs | 2 +- .../pixi_manifest/src/toml/conda_pypi_map.rs | 60 +- crates/pixi_manifest/src/workspace.rs | 54 +- crates/pypi_mapping/src/lib.rs | 14 +- .../src/resolvers/project_defined_mapping.rs | 11 +- docs/concepts/conda_pypi.md | 10 + 11 files changed, 1532 insertions(+), 1464 deletions(-) create mode 100644 crates/pixi/tests/integration_rust/conda_pypi_map_tests.rs rename crates/pixi/tests/integration_rust/snapshots/{integration_rust__solve_group_tests__missing_mapping_file_error_includes_path.snap => integration_rust__conda_pypi_map_tests__missing_mapping_file_error_includes_path.snap} (100%) diff --git a/crates/pixi/tests/integration_rust/conda_pypi_map_tests.rs b/crates/pixi/tests/integration_rust/conda_pypi_map_tests.rs new file mode 100644 index 0000000000..18ab481e6e --- /dev/null +++ b/crates/pixi/tests/integration_rust/conda_pypi_map_tests.rs @@ -0,0 +1,1415 @@ +//! Tests for the conda↔PyPI name mapping: purl derivation through the +//! prefix.dev chain, project-defined `conda-pypi-map` overrides in their +//! extend/replace/disabled modes, and the `cache-ttl` mapping cache. + +use std::{ + collections::{BTreeSet, HashMap}, + path::Path, + str::FromStr, + sync::Arc, +}; + +use pypi_mapping::{ + self, ProjectDefinedChannelMapping, ProjectDefinedMapping, ProjectDefinedMappingLocation, + PurlDerivationMode, PurlDerivationSource, +}; +use rattler_conda_types::{PackageName, Platform, RepoDataRecord}; +use rattler_lock::DEFAULT_ENVIRONMENT_NAME; +use reqwest_middleware::ClientBuilder; +use tempfile::TempDir; +use url::Url; + +use crate::common::{ + LockFileExt, PixiControl, + builders::HasDependencyConfig, + client::OfflineMiddleware, + pypi_index::{Database as PyPIDatabase, PyPIPackage}, +}; +use crate::setup_tracing; +use pixi_test_utils::{MockRepoData, Package}; + +#[tokio::test(flavor = "multi_thread", worker_threads = 1)] +#[cfg_attr( + any(not(feature = "online_tests"), not(feature = "slow_integration_tests")), + ignore +)] +async fn test_purl_are_added_for_pypi() { + setup_tracing(); + + let pixi = PixiControl::new().unwrap(); + pixi.init().await.unwrap(); + // Add and update lock file with this version of python + pixi.add("boltons").await.unwrap(); + let lock_file = pixi.update_lock_file().await.unwrap(); + + // Check if boltons has a purl + let p = lock_file + .platform(&Platform::current().to_string()) + .unwrap(); + lock_file + .default_environment() + .unwrap() + .packages(p) + .unwrap() + .for_each(|dep| { + if dep.as_conda().unwrap().name() == &PackageName::from_str("boltons").unwrap() { + assert!(dep.as_conda().unwrap().record().unwrap().purls.is_none()); + } + }); + + // Add boltons from pypi + pixi.add("boltons") + .set_type(pixi_core::DependencyType::PypiDependency) + .await + .unwrap(); + + let lock_file = pixi.update_lock_file().await.unwrap(); + + // Check if boltons has a purl + let p = lock_file + .platform(&Platform::current().to_string()) + .unwrap(); + lock_file + .default_environment() + .unwrap() + .packages(p) + .unwrap() + .for_each(|dep| { + if dep.as_conda().unwrap().name() == &PackageName::from_str("boltons").unwrap() { + assert_eq!( + dep.as_conda() + .and_then(|c| c.as_binary()) + .and_then(|c| c.package_record.purls.as_ref()) + .unwrap() + .first() + .unwrap() + .qualifiers() + .get("source") + .unwrap(), + PurlDerivationSource::PrefixHashMapping.as_str() + ); + } + }); + + // Check if boltons exists only as conda dependency + assert!(lock_file.contains_match_spec( + DEFAULT_ENVIRONMENT_NAME, + Platform::current(), + "boltons" + )); + assert!(!lock_file.contains_pypi_package( + DEFAULT_ENVIRONMENT_NAME, + Platform::current(), + "boltons" + )); +} + +#[tokio::test] +#[cfg_attr(not(feature = "online_tests"), ignore)] +async fn test_purl_are_missing_for_non_conda_forge() { + setup_tracing(); + + let pixi = PixiControl::new().unwrap(); + pixi.init().await.unwrap(); + + let project = pixi.workspace().unwrap(); + let client = project.authenticated_client().unwrap(); + let foo_bar_package = Package::build("foo-bar-car", "2").finish(); + + let mut repo_data_record = RepoDataRecord { + identifier: foo_bar_package.identifier(), + package_record: foo_bar_package.package_record, + url: Url::parse("https://pypi.org/simple/boltons/").unwrap(), + channel: Some("dummy-channel".to_owned()), + }; + + let mapping_client = pypi_mapping::PurlDerivationClient::builder( + client.clone(), + project + .config() + .cache_dir_for(pixi_config::CacheKind::PypiMapping) + .unwrap(), + ) + .finish(); + mapping_client + .amend_purls( + &PurlDerivationMode::Prefix, + vec![&mut repo_data_record], + None, + ) + .await + .unwrap(); + + // Because foo-bar-car is not from conda-forge channel + // We verify that purls are missing for non-conda-forge packages + assert!( + repo_data_record + .package_record + .purls + .as_ref() + .and_then(BTreeSet::first) + .is_none() + ); +} + +#[tokio::test] +async fn test_purl_are_generated_using_custom_mapping() { + setup_tracing(); + + let pixi = PixiControl::new().unwrap(); + pixi.init().await.unwrap(); + + let project = pixi.workspace().unwrap(); + let client = project.authenticated_client().unwrap(); + let foo_bar_package = Package::build("foo-bar-car", "2").finish(); + + let mut repo_data_record = RepoDataRecord { + identifier: foo_bar_package.identifier(), + package_record: foo_bar_package.package_record, + url: Url::parse("https://pypi.org/simple/boltons/").unwrap(), + channel: Some("https://conda.anaconda.org/conda-forge/".to_owned()), + }; + + // We are using project-defined mapping + let compressed_mapping = + HashMap::from([("foo-bar-car".to_owned(), Some("my-test-name".to_owned()))]); + let source = HashMap::from([( + "https://conda.anaconda.org/conda-forge".to_owned(), + ProjectDefinedChannelMapping::replace(ProjectDefinedMappingLocation::InMemory( + compressed_mapping, + )), + )]); + + let mapping_client = pypi_mapping::PurlDerivationClient::builder( + client.clone(), + project + .config() + .cache_dir_for(pixi_config::CacheKind::PypiMapping) + .unwrap(), + ) + .finish(); + mapping_client + .amend_purls( + &PurlDerivationMode::ProjectDefined(Arc::new(ProjectDefinedMapping::new(source))), + vec![&mut repo_data_record], + None, + ) + .await + .unwrap(); + + let first_purl = repo_data_record + .package_record + .purls + .as_ref() + .and_then(BTreeSet::first) + .unwrap(); + + // We verify that `my-test-name` is used for `foo-bar-car` package + assert_eq!(first_purl.name(), "my-test-name") +} + +#[tokio::test] +#[cfg_attr(not(feature = "online_tests"), ignore)] +async fn test_compressed_mapping_catch_not_pandoc_not_a_python_package() { + setup_tracing(); + + let pixi = PixiControl::new().unwrap(); + pixi.init().await.unwrap(); + + let project = pixi.workspace().unwrap(); + let client = project.authenticated_client().unwrap(); + let foo_bar_package = Package::build("pandoc", "2").finish(); + + let mut repo_data_record = RepoDataRecord { + identifier: foo_bar_package.identifier(), + package_record: foo_bar_package.package_record, + url: Url::parse("https://haskell.org/pandoc/").unwrap(), + channel: Some("https://conda.anaconda.org/conda-forge/".to_owned()), + }; + + let packages = vec![&mut repo_data_record]; + + let mapping_client = pypi_mapping::PurlDerivationClient::builder( + client.clone(), + project + .config() + .cache_dir_for(pixi_config::CacheKind::PypiMapping) + .unwrap(), + ) + .finish(); + mapping_client + .amend_purls(&PurlDerivationMode::Prefix, packages, None) + .await + .unwrap(); + + // pandoc is not a python package + // so purls for it should be empty + assert!(repo_data_record.package_record.purls.unwrap().is_empty()) +} + +#[tokio::test] +#[cfg_attr(not(feature = "online_tests"), ignore)] +async fn test_dont_record_not_present_package_as_purl() { + setup_tracing(); + + let pixi = PixiControl::new().unwrap(); + pixi.init().await.unwrap(); + + let project = pixi.workspace().unwrap(); + let client = project.authenticated_client().unwrap(); + // We use one package that is present in our mapping: `boltons` + // and another one that is missing from conda and our mapping: + // `pixi-something-new-for-test` because `pixi-something-new-for-test` is + // from conda-forge channel we will anyway record a purl for it + // by assumption that it's a pypi package + let foo_bar_package = Package::build("pixi-something-new-for-test", "2").finish(); + // We use one package that is not present by hash + // but `boltons` name is still present in compressed mapping + // so we will record a purl for it + let boltons_package = Package::build("boltons", "99999").finish(); + + let mut repo_data_record = RepoDataRecord { + identifier: foo_bar_package.identifier(), + package_record: foo_bar_package.package_record, + url: Url::parse("https://pypi.org/simple/something-new/").unwrap(), + channel: Some("https://conda.anaconda.org/conda-forge/osx-arm64/brotli-python-1.1.0-py311ha891d26_1.conda".to_owned()), + }; + + let mut boltons_repo_data_record = RepoDataRecord { + identifier: boltons_package.identifier(), + package_record: boltons_package.package_record, + url: Url::parse("https://pypi.org/simple/boltons/").unwrap(), + channel: Some("https://conda.anaconda.org/conda-forge/".to_owned()), + }; + + let mapping_client = pypi_mapping::PurlDerivationClient::builder( + client.clone(), + project + .config() + .cache_dir_for(pixi_config::CacheKind::PypiMapping) + .unwrap(), + ) + .finish(); + mapping_client + .amend_purls( + project.pypi_name_derivation_mode().unwrap(), + vec![&mut repo_data_record, &mut boltons_repo_data_record], + None, + ) + .await + .unwrap(); + + mapping_client + .amend_purls( + project.pypi_name_derivation_mode().unwrap(), + vec![&mut repo_data_record, &mut boltons_repo_data_record], + None, + ) + .await + .unwrap(); + + let first_purl = repo_data_record + .package_record + .purls + .as_ref() + .and_then(BTreeSet::first) + .unwrap(); + + // we verify that even if this name is not present in our mapping + // we record a purl anyways. Because we make the assumption + // that it's a pypi package + assert_eq!(first_purl.name(), "pixi-something-new-for-test"); + + let boltons_purl = boltons_repo_data_record + .package_record + .purls + .as_ref() + .and_then(BTreeSet::first) + .unwrap(); + + // for boltons we have a mapping record + // so we test that we also record source=conda-forge-mapping qualifier + assert_eq!( + boltons_purl.qualifiers().get("source").unwrap(), + PurlDerivationSource::PrefixCompressedMapping.as_str() + ); +} + +fn absolute_custom_mapping_path() -> String { + dunce::simplified( + &Path::new(env!("CARGO_WORKSPACE_DIR")) + .join("tests/data/mapping_files/custom_mapping.json"), + ) + .display() + .to_string() + .replace("\\", "/") +} + +fn absolute_compressed_mapping_path() -> String { + dunce::simplified( + &Path::new(env!("CARGO_WORKSPACE_DIR")) + .join("tests/data/mapping_files/compressed_mapping.json"), + ) + .display() + .to_string() + .replace("\\", "/") +} + +#[tokio::test] +async fn test_we_record_not_present_package_as_purl_for_custom_mapping() { + setup_tracing(); + + let pixi = PixiControl::from_manifest(&format!( + r#" + [project] + name = "test-channel-change" + channels = ["conda-forge"] + platforms = ["linux-64"] + conda-pypi-map = {{ 'conda-forge' = {{ location = "{}", mode = "replace" }} }} + "#, + absolute_compressed_mapping_path() + )) + .unwrap(); + + let project = pixi.workspace().unwrap(); + + let client = project.authenticated_client().unwrap(); + + // We use one package that is present in our mapping: `boltons` + // and another one that is missing from conda and our mapping: + // `pixi-something-new-for-test`. Because the mapping uses + // `mode = "replace"` the mapping is exclusive: packages that are not in + // it must not get a purl, not even the conda-forge verbatim fallback. + let foo_bar_package = Package::build("pixi-something-new", "2").finish(); + let boltons_package = Package::build("boltons", "2").finish(); + + let repo_data_record = RepoDataRecord { + identifier: foo_bar_package.identifier(), + package_record: foo_bar_package.package_record, + url: Url::parse("https://pypi.org/simple/pixi-something-new-new/").unwrap(), + channel: Some("https://conda.anaconda.org/conda-forge/".to_owned()), + }; + + let boltons_repo_data_record = RepoDataRecord { + identifier: boltons_package.identifier(), + package_record: boltons_package.package_record, + url: Url::parse("https://pypi.org/simple/boltons/").unwrap(), + channel: Some("https://conda.anaconda.org/conda-forge/".to_owned()), + }; + + let mut packages = vec![repo_data_record, boltons_repo_data_record]; + + let mapping_client = pypi_mapping::PurlDerivationClient::builder( + client.clone(), + project + .config() + .cache_dir_for(pixi_config::CacheKind::PypiMapping) + .unwrap(), + ) + .finish(); + mapping_client + .amend_purls( + project.pypi_name_derivation_mode().unwrap(), + &mut packages, + None, + ) + .await + .unwrap(); + + let boltons_package = packages.pop().unwrap(); + + let boltons_first_purl = boltons_package + .package_record + .purls + .as_ref() + .and_then(BTreeSet::first) + .unwrap(); + + println!("{boltons_first_purl}"); + + // for boltons we have a mapping record + // so we test that we also record source=project-defined-mapping qualifier + assert_eq!(boltons_first_purl.name(), "boltons"); + assert_eq!( + boltons_first_purl.qualifiers().get("source").unwrap(), + PurlDerivationSource::ProjectDefinedMapping.as_str() + ); + + let package = packages.pop().unwrap(); + + // With a replace-mode project-defined mapping, packages not in the mapping + // should NOT get purls. This verifies that replace mode is exclusive - only + // packages explicitly mapped should be considered as pypi packages. + assert!( + package.package_record.purls.is_none() + || package.package_record.purls.as_ref().unwrap().is_empty(), + "pixi-something-new should not have purls when not in a replace-mode mapping" + ); +} + +#[tokio::test] +async fn test_custom_mapping_channel_with_suffix() { + setup_tracing(); + + let pixi = PixiControl::from_manifest(&format!( + r#" + [project] + name = "test-channel-change" + channels = ["conda-forge"] + platforms = ["linux-64"] + conda-pypi-map = {{ "https://conda.anaconda.org/conda-forge/" = "{}" }} + "#, + absolute_custom_mapping_path() + )) + .unwrap(); + + let project = pixi.workspace().unwrap(); + + let client = project.authenticated_client().unwrap(); + + let foo_bar_package = Package::build("pixi-something-new", "2").finish(); + + let repo_data_record = RepoDataRecord { + identifier: foo_bar_package.identifier(), + package_record: foo_bar_package.package_record, + url: Url::parse("https://pypi.org/simple/pixi-something-new-new/").unwrap(), + channel: Some("https://conda.anaconda.org/conda-forge".to_owned()), + }; + + let mut packages = vec![repo_data_record]; + + let mapping_client = pypi_mapping::PurlDerivationClient::builder( + client.clone(), + project + .config() + .cache_dir_for(pixi_config::CacheKind::PypiMapping) + .unwrap(), + ) + .finish(); + mapping_client + .amend_purls( + project.pypi_name_derivation_mode().unwrap(), + &mut packages, + None, + ) + .await + .unwrap(); + + let package = packages.pop().unwrap(); + + assert_eq!( + package + .package_record + .purls + .as_ref() + .and_then(BTreeSet::first) + .unwrap() + .qualifiers() + .get("source") + .unwrap(), + PurlDerivationSource::ProjectDefinedMapping.as_str() + ); +} + +#[tokio::test] +async fn test_repo_data_record_channel_with_suffix() { + setup_tracing(); + + let pixi = PixiControl::from_manifest(&format!( + r#" + [project] + name = "test-channel-change" + channels = ["conda-forge"] + platforms = ["linux-64"] + conda-pypi-map = {{ "https://conda.anaconda.org/conda-forge" = "{}" }} + "#, + absolute_custom_mapping_path(), + )) + .unwrap(); + + let project = pixi.workspace().unwrap(); + + let client = project.authenticated_client().unwrap(); + + let foo_bar_package = Package::build("pixi-something-new", "2").finish(); + + let repo_data_record = RepoDataRecord { + identifier: foo_bar_package.identifier(), + package_record: foo_bar_package.package_record, + url: Url::parse("https://pypi.org/simple/pixi-something-new-new/").unwrap(), + channel: Some("https://conda.anaconda.org/conda-forge/".to_owned()), + }; + + let mut packages = vec![repo_data_record]; + + let mapping_client = pypi_mapping::PurlDerivationClient::builder( + client.clone(), + project + .config() + .cache_dir_for(pixi_config::CacheKind::PypiMapping) + .unwrap(), + ) + .finish(); + mapping_client + .amend_purls( + project.pypi_name_derivation_mode().unwrap(), + &mut packages, + None, + ) + .await + .unwrap(); + + let package = packages.pop().unwrap(); + assert_eq!( + package + .package_record + .purls + .as_ref() + .and_then(BTreeSet::first) + .unwrap() + .qualifiers() + .get("source") + .unwrap(), + PurlDerivationSource::ProjectDefinedMapping.as_str() + ); +} + +#[tokio::test] +async fn test_path_channel() { + setup_tracing(); + + let pixi = PixiControl::from_manifest(&format!( + r#" + [project] + name = "test-channel-change" + channels = ["file:///home/user/staged-recipes/build_artifacts"] + platforms = ["linux-64"] + conda-pypi-map = {{"file:///home/user/staged-recipes/build_artifacts" = "{}" }} + "#, + absolute_custom_mapping_path() + )) + .unwrap(); + + let project = pixi.workspace().unwrap(); + + let client = project.authenticated_client().unwrap(); + + let foo_bar_package = Package::build("pixi-something-new", "2").finish(); + + let repo_data_record = RepoDataRecord { + identifier: foo_bar_package.identifier(), + package_record: foo_bar_package.package_record, + url: Url::parse("https://pypi.org/simple/pixi-something-new-new/").unwrap(), + channel: Some("file:///home/user/staged-recipes/build_artifacts".to_owned()), + }; + + let mut packages = vec![repo_data_record]; + + let mapping_client = pypi_mapping::PurlDerivationClient::builder( + client.clone(), + project + .config() + .cache_dir_for(pixi_config::CacheKind::PypiMapping) + .unwrap(), + ) + .finish(); + mapping_client + .amend_purls( + project.pypi_name_derivation_mode().unwrap(), + &mut packages, + None, + ) + .await + .unwrap(); + + let package = packages.pop().unwrap(); + + assert_eq!( + package + .package_record + .purls + .as_ref() + .and_then(BTreeSet::first) + .unwrap() + .qualifiers() + .get("source") + .unwrap(), + PurlDerivationSource::ProjectDefinedMapping.as_str() + ); +} + +#[tokio::test] +async fn test_file_url_as_mapping_location() { + setup_tracing(); + + let tmp_dir = tempfile::tempdir().unwrap(); + let mapping_file = tmp_dir.path().join("custom_mapping.json"); + + let _ = fs_err::write( + &mapping_file, + r#" + { + "pixi-something-new": "pixi-something-old" + } + "#, + ); + + let mapping_file_path_as_url = Url::from_file_path( + mapping_file, /* .canonicalize() + * .expect("should be canonicalized"), */ + ) + .unwrap(); + + let pixi = PixiControl::from_manifest( + format!( + r#" + [project] + name = "test-channel-change" + channels = ["conda-forge"] + platforms = ["linux-64"] + conda-pypi-map = {{"conda-forge" = "{}"}} + "#, + mapping_file_path_as_url.as_str() + ) + .as_str(), + ) + .unwrap(); + + let project = pixi.workspace().unwrap(); + + let client = project.authenticated_client().unwrap(); + + let foo_bar_package = Package::build("pixi-something-new", "2").finish(); + + let repo_data_record = RepoDataRecord { + identifier: foo_bar_package.identifier(), + package_record: foo_bar_package.package_record, + url: Url::parse("https://pypi.org/simple/pixi-something-new-new/").unwrap(), + channel: Some("https://conda.anaconda.org/conda-forge/".to_owned()), + }; + + let mut packages = vec![repo_data_record]; + + let mapping_client = pypi_mapping::PurlDerivationClient::builder( + client.clone(), + project + .config() + .cache_dir_for(pixi_config::CacheKind::PypiMapping) + .unwrap(), + ) + .finish(); + mapping_client + .amend_purls( + project.pypi_name_derivation_mode().unwrap(), + &mut packages, + None, + ) + .await + .unwrap(); + + let package = packages.pop().unwrap(); + + assert_eq!( + package + .package_record + .purls + .as_ref() + .and_then(BTreeSet::first) + .unwrap() + .qualifiers() + .get("source") + .unwrap(), + PurlDerivationSource::ProjectDefinedMapping.as_str() + ); +} + +/// Build a `PurlDerivationClient` whose http client refuses any network +/// request, backed by the given cache directory. +fn offline_mapping_client( + project: &pixi_core::Workspace, + cache_dir: std::path::PathBuf, +) -> pypi_mapping::PurlDerivationClient { + let client = project.authenticated_client().unwrap(); + let blocked_client = ClientBuilder::from_client(client.client().clone()) + .with(OfflineMiddleware) + .build(); + pypi_mapping::PurlDerivationClient::builder(blocked_client.into(), cache_dir).finish() +} + +fn conda_forge_record(name: &str) -> RepoDataRecord { + let package = Package::build(name, "2").finish(); + RepoDataRecord { + identifier: package.identifier(), + package_record: package.package_record, + url: Url::parse(&format!("https://pypi.org/simple/{name}/")).unwrap(), + channel: Some("https://conda.anaconda.org/conda-forge/".to_owned()), + } +} + +/// An inline mapping hit in the default (extend) mode is final and requires +/// no network access. +#[tokio::test] +async fn test_extend_mapping_inline_hit_without_network() { + setup_tracing(); + + let pixi = PixiControl::from_manifest( + r#" + [project] + name = "test-extend-inline" + channels = ["conda-forge"] + platforms = ["linux-64"] + conda-pypi-map = { conda-forge = { mapping = { pixi-something-new = "my-inline-name" } } } + "#, + ) + .unwrap(); + + let project = pixi.workspace().unwrap(); + let cache_dir = TempDir::new().unwrap(); + let mapping_client = offline_mapping_client(&project, cache_dir.path().to_path_buf()); + + let mut packages = vec![conda_forge_record("pixi-something-new")]; + mapping_client + .amend_purls( + project.pypi_name_derivation_mode().unwrap(), + &mut packages, + None, + ) + .await + .unwrap(); + + let package = packages.pop().unwrap(); + let purl = package + .package_record + .purls + .as_ref() + .and_then(BTreeSet::first) + .unwrap(); + assert_eq!(purl.name(), "my-inline-name"); + assert_eq!( + purl.qualifiers().get("source").unwrap(), + PurlDerivationSource::ProjectDefinedMapping.as_str() + ); +} + +/// An explicit `false` inline entry means "not a PyPI package": no purl is +/// derived and the conda-forge verbatim fallback does not kick in either. +#[tokio::test] +async fn test_extend_mapping_explicit_false_yields_no_purl() { + setup_tracing(); + + let pixi = PixiControl::from_manifest( + r#" + [project] + name = "test-extend-false" + channels = ["conda-forge"] + platforms = ["linux-64"] + conda-pypi-map = { conda-forge = { mapping = { pixi-something-new = false } } } + "#, + ) + .unwrap(); + + let project = pixi.workspace().unwrap(); + let cache_dir = TempDir::new().unwrap(); + let mapping_client = offline_mapping_client(&project, cache_dir.path().to_path_buf()); + + let mut packages = vec![conda_forge_record("pixi-something-new")]; + mapping_client + .amend_purls( + project.pypi_name_derivation_mode().unwrap(), + &mut packages, + None, + ) + .await + .unwrap(); + + let package = packages.pop().unwrap(); + assert!( + package + .package_record + .purls + .as_ref() + .is_none_or(|purls| purls.is_empty()), + "a package explicitly mapped to `false` must not get a purl" + ); +} + +/// ` = false` disables lookups for that channel; the offline +/// conda-forge verbatim fallback still applies. +#[tokio::test] +async fn test_channel_disabled_keeps_verbatim_fallback() { + setup_tracing(); + + let pixi = PixiControl::from_manifest( + r#" + [project] + name = "test-channel-disabled" + channels = ["conda-forge"] + platforms = ["linux-64"] + conda-pypi-map = { conda-forge = false } + "#, + ) + .unwrap(); + + let project = pixi.workspace().unwrap(); + let cache_dir = TempDir::new().unwrap(); + let mapping_client = offline_mapping_client(&project, cache_dir.path().to_path_buf()); + + let mut packages = vec![conda_forge_record("boltons")]; + mapping_client + .amend_purls( + project.pypi_name_derivation_mode().unwrap(), + &mut packages, + None, + ) + .await + .unwrap(); + + let package = packages.pop().unwrap(); + let purl = package + .package_record + .purls + .as_ref() + .and_then(BTreeSet::first) + .unwrap(); + // The verbatim fallback assumes the conda name is the pypi name and adds + // no source qualifier. + assert_eq!(purl.name(), "boltons"); + assert!(purl.qualifiers().is_empty()); +} + +/// When an entry has both a `location` and inline `mapping` entries, the +/// inline entries override the ones from the location. +#[tokio::test] +async fn test_inline_mapping_overrides_location() { + setup_tracing(); + + // The custom mapping file maps `pixi-something-new` to itself; the inline + // entry must win. + let pixi = PixiControl::from_manifest(&format!( + r#" + [project] + name = "test-inline-overrides" + channels = ["conda-forge"] + platforms = ["linux-64"] + conda-pypi-map = {{ conda-forge = {{ location = "{}", mapping = {{ pixi-something-new = "inline-wins" }} }} }} + "#, + absolute_custom_mapping_path() + )) + .unwrap(); + + let project = pixi.workspace().unwrap(); + let cache_dir = TempDir::new().unwrap(); + let mapping_client = offline_mapping_client(&project, cache_dir.path().to_path_buf()); + + let mut packages = vec![conda_forge_record("pixi-something-new")]; + mapping_client + .amend_purls( + project.pypi_name_derivation_mode().unwrap(), + &mut packages, + None, + ) + .await + .unwrap(); + + let package = packages.pop().unwrap(); + let purl = package + .package_record + .purls + .as_ref() + .and_then(BTreeSet::first) + .unwrap(); + assert_eq!(purl.name(), "inline-wins"); +} + +/// In extend mode a miss in the project-defined mapping falls through to the +/// prefix.dev chain. +#[tokio::test] +#[cfg_attr(not(feature = "online_tests"), ignore)] +async fn test_extend_mapping_miss_falls_through_to_prefix() { + setup_tracing(); + + // The mapping contains an unrelated package, so `boltons` is a miss and + // must be resolved through the prefix.dev chain (the mock-built record's + // hash is unknown there, so the compressed name mapping answers). + let pixi = PixiControl::from_manifest( + r#" + [project] + name = "test-extend-miss" + channels = ["conda-forge"] + platforms = ["linux-64"] + conda-pypi-map = { conda-forge = { mapping = { some-other-package = "other" } } } + "#, + ) + .unwrap(); + + let project = pixi.workspace().unwrap(); + let client = project.authenticated_client().unwrap(); + let mapping_client = pypi_mapping::PurlDerivationClient::builder( + client.clone(), + project + .config() + .cache_dir_for(pixi_config::CacheKind::PypiMapping) + .unwrap(), + ) + .finish(); + + let mut packages = vec![conda_forge_record("boltons")]; + mapping_client + .amend_purls( + project.pypi_name_derivation_mode().unwrap(), + &mut packages, + None, + ) + .await + .unwrap(); + + let package = packages.pop().unwrap(); + let purl = package + .package_record + .purls + .as_ref() + .and_then(BTreeSet::first) + .unwrap(); + assert_eq!(purl.name(), "boltons"); + assert_eq!( + purl.qualifiers().get("source").unwrap(), + PurlDerivationSource::PrefixCompressedMapping.as_str() + ); +} + +/// The on-disk path of the TTL cache for a mapping url, mirroring the layout +/// used by the project-defined mapping resolver. +fn ttl_cache_path_for(cache_dir: &Path, url: &str) -> std::path::PathBuf { + let hash = rattler_digest::compute_bytes_digest::(url.as_bytes()); + cache_dir + .join("project-defined") + .join(format!("{hash:x}.json")) +} + +fn manifest_with_ttl_mapping(url: &str, ttl: &str) -> String { + format!( + r#" + [project] + name = "test-cache-ttl" + channels = ["conda-forge"] + platforms = ["linux-64"] + conda-pypi-map = {{ conda-forge = {{ location = "{url}", cache-ttl = "{ttl}" }} }} + "# + ) +} + +/// A cached mapping younger than `cache-ttl` is used without any network +/// access. +#[tokio::test] +async fn test_cache_ttl_fresh_cache_skips_network() { + setup_tracing(); + + let mapping_url = "https://example.invalid/mapping.json"; + let pixi = PixiControl::from_manifest(&manifest_with_ttl_mapping(mapping_url, "1h")).unwrap(); + + let project = pixi.workspace().unwrap(); + let cache_dir = TempDir::new().unwrap(); + + // Pre-populate the TTL cache; the url itself is unreachable and the + // client is offline, so a cache miss would fail the test. + let cache_file = ttl_cache_path_for(cache_dir.path(), mapping_url); + fs_err::create_dir_all(cache_file.parent().unwrap()).unwrap(); + fs_err::write(&cache_file, r#"{ "pixi-something-new": "from-the-cache" }"#).unwrap(); + + let mapping_client = offline_mapping_client(&project, cache_dir.path().to_path_buf()); + + let mut packages = vec![conda_forge_record("pixi-something-new")]; + mapping_client + .amend_purls( + project.pypi_name_derivation_mode().unwrap(), + &mut packages, + None, + ) + .await + .unwrap(); + + let package = packages.pop().unwrap(); + let purl = package + .package_record + .purls + .as_ref() + .and_then(BTreeSet::first) + .unwrap(); + assert_eq!(purl.name(), "from-the-cache"); +} + +/// When the cached mapping is expired and the refetch fails, the stale copy +/// is used so solves keep working offline. +#[tokio::test] +async fn test_cache_ttl_expired_falls_back_to_stale_copy() { + setup_tracing(); + + let mapping_url = "https://example.invalid/mapping.json"; + // A zero TTL means the cached copy is always considered expired. + let pixi = PixiControl::from_manifest(&manifest_with_ttl_mapping(mapping_url, "0s")).unwrap(); + + let project = pixi.workspace().unwrap(); + let cache_dir = TempDir::new().unwrap(); + + let cache_file = ttl_cache_path_for(cache_dir.path(), mapping_url); + fs_err::create_dir_all(cache_file.parent().unwrap()).unwrap(); + fs_err::write(&cache_file, r#"{ "pixi-something-new": "stale-but-used" }"#).unwrap(); + + let mapping_client = offline_mapping_client(&project, cache_dir.path().to_path_buf()); + + let mut packages = vec![conda_forge_record("pixi-something-new")]; + mapping_client + .amend_purls( + project.pypi_name_derivation_mode().unwrap(), + &mut packages, + None, + ) + .await + .unwrap(); + + let package = packages.pop().unwrap(); + let purl = package + .package_record + .purls + .as_ref() + .and_then(BTreeSet::first) + .unwrap(); + assert_eq!(purl.name(), "stale-but-used"); +} + +/// Without any cached copy, a failing fetch of a TTL-cached mapping is a hard +/// error. +#[tokio::test] +async fn test_cache_ttl_no_cache_and_fetch_failure_errors() { + setup_tracing(); + + let mapping_url = "https://example.invalid/mapping.json"; + let pixi = PixiControl::from_manifest(&manifest_with_ttl_mapping(mapping_url, "1h")).unwrap(); + + let project = pixi.workspace().unwrap(); + let cache_dir = TempDir::new().unwrap(); + let mapping_client = offline_mapping_client(&project, cache_dir.path().to_path_buf()); + + let mut packages = vec![conda_forge_record("pixi-something-new")]; + let result = mapping_client + .amend_purls( + project.pypi_name_derivation_mode().unwrap(), + &mut packages, + None, + ) + .await; + + assert!( + result.is_err(), + "an uncached TTL mapping with a failing fetch must error" + ); +} + +/// A failing prefix.dev lookup must point firewall-restricted users at the +/// manifest options that avoid the network. +#[tokio::test] +async fn test_prefix_fetch_failure_error_mentions_escape_hatches() { + setup_tracing(); + + // No `conda-pypi-map` -> the default prefix.dev chain, which needs the + // network to look up the record by hash. + let pixi = PixiControl::from_manifest( + r#" + [project] + name = "test-network-error" + channels = ["conda-forge"] + platforms = ["linux-64"] + "#, + ) + .unwrap(); + + let project = pixi.workspace().unwrap(); + let cache_dir = TempDir::new().unwrap(); + let mapping_client = offline_mapping_client(&project, cache_dir.path().to_path_buf()); + + let mut packages = vec![conda_forge_record("pixi-something-new")]; + let err = mapping_client + .amend_purls( + project.pypi_name_derivation_mode().unwrap(), + &mut packages, + None, + ) + .await + .expect_err("an offline prefix.dev lookup should fail"); + + let rendered = format!("{err:?}"); + // Strip all whitespace before matching: miette wraps the help text at + // arbitrary points, potentially splitting tokens across lines. + let collapsed: String = rendered.chars().filter(|c| !c.is_whitespace()).collect(); + assert!( + collapsed.contains("mode=\"replace\"") && collapsed.contains("conda-pypi-map=false"), + "the error should suggest the offline escape hatches, got: {rendered}" + ); +} + +/// `conda-pypi-map = {}` is a soft-deprecated alias for +/// `conda-pypi-map = false`; both disable all mapping lookups while keeping +/// the conda-forge verbatim fallback. +#[tokio::test] +async fn test_disabled_mapping() { + setup_tracing(); + + let pixi = PixiControl::from_manifest( + r#" + [project] + name = "test-channel-change" + channels = ["https://prefix.dev/conda-forge"] + platforms = ["linux-64"] + conda-pypi-map = { } + "#, + ) + .unwrap(); + + let project = pixi.workspace().unwrap(); + + let client = project.authenticated_client().unwrap(); + + let blocking_middleware = OfflineMiddleware; + + let blocked_client = ClientBuilder::from_client(client.client().clone()) + .with(blocking_middleware) + .build(); + + let boltons_package = Package::build("boltons", "2").finish(); + + let boltons_repo_data_record = RepoDataRecord { + identifier: boltons_package.identifier(), + package_record: boltons_package.package_record, + url: Url::parse("https://pypi.org/simple/boltons/").unwrap(), + channel: Some("https://conda.anaconda.org/conda-forge/".to_owned()), + }; + + let mut packages = vec![boltons_repo_data_record]; + + let mapping_client = pypi_mapping::PurlDerivationClient::builder( + blocked_client.into(), + project + .config() + .cache_dir_for(pixi_config::CacheKind::PypiMapping) + .unwrap(), + ) + .finish(); + mapping_client + .amend_purls( + project.pypi_name_derivation_mode().unwrap(), + &mut packages, + None, + ) + .await + .unwrap(); + + let boltons_package = packages.pop().unwrap(); + + let boltons_first_purl = boltons_package + .package_record + .purls + .as_ref() + .and_then(BTreeSet::first) + .unwrap(); + + // we verify that even if this name is not present in our mapping + // we record a purl anyways. Because we make the assumption + // that it's a pypi package + assert_eq!(boltons_first_purl.name(), "boltons"); + assert!(boltons_first_purl.qualifiers().is_empty()); +} + +/// `conda-pypi-map = false` is the canonical global disable: no lookups, but +/// the conda-forge verbatim fallback still applies. +#[tokio::test] +async fn test_disabled_mapping_via_false() { + setup_tracing(); + + let pixi = PixiControl::from_manifest( + r#" + [project] + name = "test-disable-false" + channels = ["https://prefix.dev/conda-forge"] + platforms = ["linux-64"] + conda-pypi-map = false + "#, + ) + .unwrap(); + + let project = pixi.workspace().unwrap(); + let cache_dir = TempDir::new().unwrap(); + let mapping_client = offline_mapping_client(&project, cache_dir.path().to_path_buf()); + + let mut packages = vec![conda_forge_record("boltons")]; + mapping_client + .amend_purls( + project.pypi_name_derivation_mode().unwrap(), + &mut packages, + None, + ) + .await + .unwrap(); + + let boltons_package = packages.pop().unwrap(); + let boltons_first_purl = boltons_package + .package_record + .purls + .as_ref() + .and_then(BTreeSet::first) + .unwrap(); + assert_eq!(boltons_first_purl.name(), "boltons"); + assert!(boltons_first_purl.qualifiers().is_empty()); +} + +#[tokio::test] +async fn test_custom_mapping_ignores_backwards_compatibility() { + setup_tracing(); + + // Create local conda channel with boltons and python packages + let mut package_database = MockRepoData::default(); + package_database.add_package( + Package::build("python", "3.12.0") + .with_subdir(Platform::Linux64) + .finish(), + ); + package_database.add_package( + Package::build("boltons", "24.0.0") + .with_subdir(Platform::Linux64) + .finish(), + ); + let channel = package_database.into_channel().await.unwrap(); + let channel_url = channel.url(); + + // Create local PyPI index with boltons package + let pypi_index = PyPIDatabase::new() + .with(PyPIPackage::new("boltons", "24.0.0")) + .into_simple_index() + .expect("failed to create local simple index"); + + // Create a project-defined mapping file that only includes specific packages + let temp_dir = TempDir::new().unwrap(); + let mapping_file = temp_dir.path().join("map.json"); + fs_err::write(&mapping_file, r#"{}"#).unwrap(); + + let pixi = PixiControl::from_manifest(&format!( + r#" + [workspace] + name = "test-custom-mapping" + channels = ["{channel_url}"] + platforms = ["linux-64"] + conda-pypi-map = {{ "{channel_url}" = {{ location = "{mapping_file}", mode = "replace" }} }} + + [dependencies] + python = "3.12.0" + boltons = "*" + + [pypi-dependencies] + boltons = "*" + + [pypi-options] + index-url = "{pypi_url}" + "#, + channel_url = channel_url, + mapping_file = mapping_file + .to_str() + .unwrap() + .to_string() + .replace("\\", "/"), + pypi_url = pypi_index.index_url(), + )) + .unwrap(); + + // Lock the project (this triggers the amend_purls logic) + pixi.lock().await.unwrap(); + + // Get the lock file + let lock = pixi.lock_file().await.unwrap(); + let p = lock.platform(&Platform::Linux64.to_string()).unwrap(); + let environment = lock.environment(DEFAULT_ENVIRONMENT_NAME).unwrap(); + let conda_packages = environment.conda_packages(p).unwrap(); + + // Collect conda packages to a vector so we can iterate over them + let conda_packages: Vec<_> = conda_packages.collect(); + + // Find boltons in conda packages + let boltons_package = conda_packages + .iter() + .find(|pkg| match pkg { + rattler_lock::CondaPackageData::Binary(binary) => { + binary.package_record.name.as_source() == "boltons" + } + _ => panic!("All packagees should be binary"), + }) + .expect("boltons should be present in conda packages"); + + // The issue: boltons should NOT have purls when using project-defined mapping + // because it's not specified in our project-defined mapping + // But due to backwards compatibility logic, it gets purls anyway + let purls = match boltons_package { + rattler_lock::CondaPackageData::Binary(binary) => &binary.package_record.purls, + _ => panic!("All packages should be binary"), + }; + + if let Some(purls) = purls { + assert!( + purls.is_empty(), + "boltons should not have purls when not specified in custom conda-pypi-map" + ); + } +} + +#[tokio::test] +async fn test_missing_mapping_file_error_includes_path() { + setup_tracing(); + + let pixi = PixiControl::new().unwrap(); + pixi.init().await.unwrap(); + + let project = pixi.workspace().unwrap(); + let client = project.authenticated_client().unwrap(); + + // Use a non-existent file path for the project-defined mapping + let non_existent_path = Path::new("/this/path/does/not/exist/mapping.json"); + + let source = HashMap::from([( + "https://conda.anaconda.org/conda-forge".to_owned(), + ProjectDefinedChannelMapping::replace(ProjectDefinedMappingLocation::Path( + non_existent_path.to_path_buf(), + )), + )]); + + let foo_bar_package = Package::build("foo-bar-car", "2").finish(); + + let mut repo_data_record = RepoDataRecord { + identifier: foo_bar_package.identifier(), + package_record: foo_bar_package.package_record, + url: Url::parse("https://pypi.org/simple/boltons/").unwrap(), + channel: Some("https://conda.anaconda.org/conda-forge/".to_owned()), + }; + + let mapping_client = pypi_mapping::PurlDerivationClient::builder( + client.clone(), + project + .config() + .cache_dir_for(pixi_config::CacheKind::PypiMapping) + .unwrap(), + ) + .finish(); + let result = mapping_client + .amend_purls( + &PurlDerivationMode::ProjectDefined(Arc::new(ProjectDefinedMapping::new(source))), + vec![&mut repo_data_record], + None, + ) + .await; + + // The operation should fail because the mapping file doesn't exist + let err = result.expect_err("Expected an error when mapping file doesn't exist"); + insta::with_settings!({filters => vec![ + (r#"path: "([^"]+)""#, "[MAPPING_PATH]"), + (r#"message: "[^"]+""#, "[MAPPING_MESSAGE]"), + (r#"\bcode:\s*\d+\b"#, "[MAPPING_CODE]"), + ]}, { + insta::assert_debug_snapshot!(err); + }); +} diff --git a/crates/pixi/tests/integration_rust/main.rs b/crates/pixi/tests/integration_rust/main.rs index f6c46728bd..cfae77f832 100644 --- a/crates/pixi/tests/integration_rust/main.rs +++ b/crates/pixi/tests/integration_rust/main.rs @@ -2,6 +2,7 @@ use std::sync::Once; mod add_tests; mod common; +mod conda_pypi_map_tests; mod develop_dependencies_tests; mod global_tests; mod init_tests; diff --git a/crates/pixi/tests/integration_rust/snapshots/integration_rust__solve_group_tests__missing_mapping_file_error_includes_path.snap b/crates/pixi/tests/integration_rust/snapshots/integration_rust__conda_pypi_map_tests__missing_mapping_file_error_includes_path.snap similarity index 100% rename from crates/pixi/tests/integration_rust/snapshots/integration_rust__solve_group_tests__missing_mapping_file_error_includes_path.snap rename to crates/pixi/tests/integration_rust/snapshots/integration_rust__conda_pypi_map_tests__missing_mapping_file_error_includes_path.snap diff --git a/crates/pixi/tests/integration_rust/solve_group_tests.rs b/crates/pixi/tests/integration_rust/solve_group_tests.rs index b7c374ca4e..2c92f26507 100644 --- a/crates/pixi/tests/integration_rust/solve_group_tests.rs +++ b/crates/pixi/tests/integration_rust/solve_group_tests.rs @@ -1,26 +1,8 @@ -use std::{ - collections::{BTreeSet, HashMap}, - path::Path, - str::FromStr, - sync::Arc, -}; - -use pypi_mapping::{ - self, ProjectDefinedChannelMapping, ProjectDefinedMapping, ProjectDefinedMappingLocation, - PurlDerivationMode, PurlDerivationSource, -}; -use rattler_conda_types::{PackageName, Platform, RepoDataRecord}; -use rattler_lock::DEFAULT_ENVIRONMENT_NAME; -use reqwest_middleware::ClientBuilder; +use rattler_conda_types::Platform; use tempfile::TempDir; use url::Url; -use crate::common::{ - LockFileExt, PixiControl, - builders::HasDependencyConfig, - client::OfflineMiddleware, - pypi_index::{Database as PyPIDatabase, PyPIPackage}, -}; +use crate::common::{LockFileExt, PixiControl}; use crate::setup_tracing; use pixi_test_utils::{MockRepoData, Package}; @@ -193,1333 +175,6 @@ async fn conda_solve_group_heterogeneous_platforms() { ); } -#[tokio::test(flavor = "multi_thread", worker_threads = 1)] -#[cfg_attr( - any(not(feature = "online_tests"), not(feature = "slow_integration_tests")), - ignore -)] -async fn test_purl_are_added_for_pypi() { - setup_tracing(); - - let pixi = PixiControl::new().unwrap(); - pixi.init().await.unwrap(); - // Add and update lock file with this version of python - pixi.add("boltons").await.unwrap(); - let lock_file = pixi.update_lock_file().await.unwrap(); - - // Check if boltons has a purl - let p = lock_file - .platform(&Platform::current().to_string()) - .unwrap(); - lock_file - .default_environment() - .unwrap() - .packages(p) - .unwrap() - .for_each(|dep| { - if dep.as_conda().unwrap().name() == &PackageName::from_str("boltons").unwrap() { - assert!(dep.as_conda().unwrap().record().unwrap().purls.is_none()); - } - }); - - // Add boltons from pypi - pixi.add("boltons") - .set_type(pixi_core::DependencyType::PypiDependency) - .await - .unwrap(); - - let lock_file = pixi.update_lock_file().await.unwrap(); - - // Check if boltons has a purl - let p = lock_file - .platform(&Platform::current().to_string()) - .unwrap(); - lock_file - .default_environment() - .unwrap() - .packages(p) - .unwrap() - .for_each(|dep| { - if dep.as_conda().unwrap().name() == &PackageName::from_str("boltons").unwrap() { - assert_eq!( - dep.as_conda() - .and_then(|c| c.as_binary()) - .and_then(|c| c.package_record.purls.as_ref()) - .unwrap() - .first() - .unwrap() - .qualifiers() - .get("source") - .unwrap(), - PurlDerivationSource::PrefixHashMapping.as_str() - ); - } - }); - - // Check if boltons exists only as conda dependency - assert!(lock_file.contains_match_spec( - DEFAULT_ENVIRONMENT_NAME, - Platform::current(), - "boltons" - )); - assert!(!lock_file.contains_pypi_package( - DEFAULT_ENVIRONMENT_NAME, - Platform::current(), - "boltons" - )); -} - -#[tokio::test] -#[cfg_attr(not(feature = "online_tests"), ignore)] -async fn test_purl_are_missing_for_non_conda_forge() { - setup_tracing(); - - let pixi = PixiControl::new().unwrap(); - pixi.init().await.unwrap(); - - let project = pixi.workspace().unwrap(); - let client = project.authenticated_client().unwrap(); - let foo_bar_package = Package::build("foo-bar-car", "2").finish(); - - let mut repo_data_record = RepoDataRecord { - identifier: foo_bar_package.identifier(), - package_record: foo_bar_package.package_record, - url: Url::parse("https://pypi.org/simple/boltons/").unwrap(), - channel: Some("dummy-channel".to_owned()), - }; - - let mapping_client = pypi_mapping::PurlDerivationClient::builder( - client.clone(), - project - .config() - .cache_dir_for(pixi_config::CacheKind::PypiMapping) - .unwrap(), - ) - .finish(); - mapping_client - .amend_purls( - &PurlDerivationMode::Prefix, - vec![&mut repo_data_record], - None, - ) - .await - .unwrap(); - - // Because foo-bar-car is not from conda-forge channel - // We verify that purls are missing for non-conda-forge packages - assert!( - repo_data_record - .package_record - .purls - .as_ref() - .and_then(BTreeSet::first) - .is_none() - ); -} - -#[tokio::test] -async fn test_purl_are_generated_using_custom_mapping() { - setup_tracing(); - - let pixi = PixiControl::new().unwrap(); - pixi.init().await.unwrap(); - - let project = pixi.workspace().unwrap(); - let client = project.authenticated_client().unwrap(); - let foo_bar_package = Package::build("foo-bar-car", "2").finish(); - - let mut repo_data_record = RepoDataRecord { - identifier: foo_bar_package.identifier(), - package_record: foo_bar_package.package_record, - url: Url::parse("https://pypi.org/simple/boltons/").unwrap(), - channel: Some("https://conda.anaconda.org/conda-forge/".to_owned()), - }; - - // We are using project-defined mapping - let compressed_mapping = - HashMap::from([("foo-bar-car".to_owned(), Some("my-test-name".to_owned()))]); - let source = HashMap::from([( - "https://conda.anaconda.org/conda-forge".to_owned(), - ProjectDefinedChannelMapping::replace(ProjectDefinedMappingLocation::InMemory( - compressed_mapping, - )), - )]); - - let mapping_client = pypi_mapping::PurlDerivationClient::builder( - client.clone(), - project - .config() - .cache_dir_for(pixi_config::CacheKind::PypiMapping) - .unwrap(), - ) - .finish(); - mapping_client - .amend_purls( - &PurlDerivationMode::ProjectDefined(Arc::new(ProjectDefinedMapping::new(source))), - vec![&mut repo_data_record], - None, - ) - .await - .unwrap(); - - let first_purl = repo_data_record - .package_record - .purls - .as_ref() - .and_then(BTreeSet::first) - .unwrap(); - - // We verify that `my-test-name` is used for `foo-bar-car` package - assert_eq!(first_purl.name(), "my-test-name") -} - -#[tokio::test] -#[cfg_attr(not(feature = "online_tests"), ignore)] -async fn test_compressed_mapping_catch_not_pandoc_not_a_python_package() { - setup_tracing(); - - let pixi = PixiControl::new().unwrap(); - pixi.init().await.unwrap(); - - let project = pixi.workspace().unwrap(); - let client = project.authenticated_client().unwrap(); - let foo_bar_package = Package::build("pandoc", "2").finish(); - - let mut repo_data_record = RepoDataRecord { - identifier: foo_bar_package.identifier(), - package_record: foo_bar_package.package_record, - url: Url::parse("https://haskell.org/pandoc/").unwrap(), - channel: Some("https://conda.anaconda.org/conda-forge/".to_owned()), - }; - - let packages = vec![&mut repo_data_record]; - - let mapping_client = pypi_mapping::PurlDerivationClient::builder( - client.clone(), - project - .config() - .cache_dir_for(pixi_config::CacheKind::PypiMapping) - .unwrap(), - ) - .finish(); - mapping_client - .amend_purls(&PurlDerivationMode::Prefix, packages, None) - .await - .unwrap(); - - // pandoc is not a python package - // so purls for it should be empty - assert!(repo_data_record.package_record.purls.unwrap().is_empty()) -} - -#[tokio::test] -#[cfg_attr(not(feature = "online_tests"), ignore)] -async fn test_dont_record_not_present_package_as_purl() { - setup_tracing(); - - let pixi = PixiControl::new().unwrap(); - pixi.init().await.unwrap(); - - let project = pixi.workspace().unwrap(); - let client = project.authenticated_client().unwrap(); - // We use one package that is present in our mapping: `boltons` - // and another one that is missing from conda and our mapping: - // `pixi-something-new-for-test` because `pixi-something-new-for-test` is - // from conda-forge channel we will anyway record a purl for it - // by assumption that it's a pypi package - let foo_bar_package = Package::build("pixi-something-new-for-test", "2").finish(); - // We use one package that is not present by hash - // but `boltons` name is still present in compressed mapping - // so we will record a purl for it - let boltons_package = Package::build("boltons", "99999").finish(); - - let mut repo_data_record = RepoDataRecord { - identifier: foo_bar_package.identifier(), - package_record: foo_bar_package.package_record, - url: Url::parse("https://pypi.org/simple/something-new/").unwrap(), - channel: Some("https://conda.anaconda.org/conda-forge/osx-arm64/brotli-python-1.1.0-py311ha891d26_1.conda".to_owned()), - }; - - let mut boltons_repo_data_record = RepoDataRecord { - identifier: boltons_package.identifier(), - package_record: boltons_package.package_record, - url: Url::parse("https://pypi.org/simple/boltons/").unwrap(), - channel: Some("https://conda.anaconda.org/conda-forge/".to_owned()), - }; - - let mapping_client = pypi_mapping::PurlDerivationClient::builder( - client.clone(), - project - .config() - .cache_dir_for(pixi_config::CacheKind::PypiMapping) - .unwrap(), - ) - .finish(); - mapping_client - .amend_purls( - project.pypi_name_derivation_mode().unwrap(), - vec![&mut repo_data_record, &mut boltons_repo_data_record], - None, - ) - .await - .unwrap(); - - mapping_client - .amend_purls( - project.pypi_name_derivation_mode().unwrap(), - vec![&mut repo_data_record, &mut boltons_repo_data_record], - None, - ) - .await - .unwrap(); - - let first_purl = repo_data_record - .package_record - .purls - .as_ref() - .and_then(BTreeSet::first) - .unwrap(); - - // we verify that even if this name is not present in our mapping - // we record a purl anyways. Because we make the assumption - // that it's a pypi package - assert_eq!(first_purl.name(), "pixi-something-new-for-test"); - - let boltons_purl = boltons_repo_data_record - .package_record - .purls - .as_ref() - .and_then(BTreeSet::first) - .unwrap(); - - // for boltons we have a mapping record - // so we test that we also record source=conda-forge-mapping qualifier - assert_eq!( - boltons_purl.qualifiers().get("source").unwrap(), - PurlDerivationSource::PrefixCompressedMapping.as_str() - ); -} - -fn absolute_custom_mapping_path() -> String { - dunce::simplified( - &Path::new(env!("CARGO_WORKSPACE_DIR")) - .join("tests/data/mapping_files/custom_mapping.json"), - ) - .display() - .to_string() - .replace("\\", "/") -} - -fn absolute_compressed_mapping_path() -> String { - dunce::simplified( - &Path::new(env!("CARGO_WORKSPACE_DIR")) - .join("tests/data/mapping_files/compressed_mapping.json"), - ) - .display() - .to_string() - .replace("\\", "/") -} - -#[tokio::test] -async fn test_we_record_not_present_package_as_purl_for_custom_mapping() { - setup_tracing(); - - let pixi = PixiControl::from_manifest(&format!( - r#" - [project] - name = "test-channel-change" - channels = ["conda-forge"] - platforms = ["linux-64"] - conda-pypi-map = {{ 'conda-forge' = {{ location = "{}", mode = "replace" }} }} - "#, - absolute_compressed_mapping_path() - )) - .unwrap(); - - let project = pixi.workspace().unwrap(); - - let client = project.authenticated_client().unwrap(); - - // We use one package that is present in our mapping: `boltons` - // and another one that is missing from conda and our mapping: - // `pixi-something-new-for-test`. Because the mapping uses - // `mode = "replace"` the mapping is exclusive: packages that are not in - // it must not get a purl, not even the conda-forge verbatim fallback. - let foo_bar_package = Package::build("pixi-something-new", "2").finish(); - let boltons_package = Package::build("boltons", "2").finish(); - - let repo_data_record = RepoDataRecord { - identifier: foo_bar_package.identifier(), - package_record: foo_bar_package.package_record, - url: Url::parse("https://pypi.org/simple/pixi-something-new-new/").unwrap(), - channel: Some("https://conda.anaconda.org/conda-forge/".to_owned()), - }; - - let boltons_repo_data_record = RepoDataRecord { - identifier: boltons_package.identifier(), - package_record: boltons_package.package_record, - url: Url::parse("https://pypi.org/simple/boltons/").unwrap(), - channel: Some("https://conda.anaconda.org/conda-forge/".to_owned()), - }; - - let mut packages = vec![repo_data_record, boltons_repo_data_record]; - - let mapping_client = pypi_mapping::PurlDerivationClient::builder( - client.clone(), - project - .config() - .cache_dir_for(pixi_config::CacheKind::PypiMapping) - .unwrap(), - ) - .finish(); - mapping_client - .amend_purls( - project.pypi_name_derivation_mode().unwrap(), - &mut packages, - None, - ) - .await - .unwrap(); - - let boltons_package = packages.pop().unwrap(); - - let boltons_first_purl = boltons_package - .package_record - .purls - .as_ref() - .and_then(BTreeSet::first) - .unwrap(); - - println!("{boltons_first_purl}"); - - // for boltons we have a mapping record - // so we test that we also record source=project-defined-mapping qualifier - assert_eq!(boltons_first_purl.name(), "boltons"); - assert_eq!( - boltons_first_purl.qualifiers().get("source").unwrap(), - PurlDerivationSource::ProjectDefinedMapping.as_str() - ); - - let package = packages.pop().unwrap(); - - // With a replace-mode project-defined mapping, packages not in the mapping - // should NOT get purls. This verifies that replace mode is exclusive - only - // packages explicitly mapped should be considered as pypi packages. - assert!( - package.package_record.purls.is_none() - || package.package_record.purls.as_ref().unwrap().is_empty(), - "pixi-something-new should not have purls when not in a replace-mode mapping" - ); -} - -#[tokio::test] -async fn test_custom_mapping_channel_with_suffix() { - setup_tracing(); - - let pixi = PixiControl::from_manifest(&format!( - r#" - [project] - name = "test-channel-change" - channels = ["conda-forge"] - platforms = ["linux-64"] - conda-pypi-map = {{ "https://conda.anaconda.org/conda-forge/" = "{}" }} - "#, - absolute_custom_mapping_path() - )) - .unwrap(); - - let project = pixi.workspace().unwrap(); - - let client = project.authenticated_client().unwrap(); - - let foo_bar_package = Package::build("pixi-something-new", "2").finish(); - - let repo_data_record = RepoDataRecord { - identifier: foo_bar_package.identifier(), - package_record: foo_bar_package.package_record, - url: Url::parse("https://pypi.org/simple/pixi-something-new-new/").unwrap(), - channel: Some("https://conda.anaconda.org/conda-forge".to_owned()), - }; - - let mut packages = vec![repo_data_record]; - - let mapping_client = pypi_mapping::PurlDerivationClient::builder( - client.clone(), - project - .config() - .cache_dir_for(pixi_config::CacheKind::PypiMapping) - .unwrap(), - ) - .finish(); - mapping_client - .amend_purls( - project.pypi_name_derivation_mode().unwrap(), - &mut packages, - None, - ) - .await - .unwrap(); - - let package = packages.pop().unwrap(); - - assert_eq!( - package - .package_record - .purls - .as_ref() - .and_then(BTreeSet::first) - .unwrap() - .qualifiers() - .get("source") - .unwrap(), - PurlDerivationSource::ProjectDefinedMapping.as_str() - ); -} - -#[tokio::test] -async fn test_repo_data_record_channel_with_suffix() { - setup_tracing(); - - let pixi = PixiControl::from_manifest(&format!( - r#" - [project] - name = "test-channel-change" - channels = ["conda-forge"] - platforms = ["linux-64"] - conda-pypi-map = {{ "https://conda.anaconda.org/conda-forge" = "{}" }} - "#, - absolute_custom_mapping_path(), - )) - .unwrap(); - - let project = pixi.workspace().unwrap(); - - let client = project.authenticated_client().unwrap(); - - let foo_bar_package = Package::build("pixi-something-new", "2").finish(); - - let repo_data_record = RepoDataRecord { - identifier: foo_bar_package.identifier(), - package_record: foo_bar_package.package_record, - url: Url::parse("https://pypi.org/simple/pixi-something-new-new/").unwrap(), - channel: Some("https://conda.anaconda.org/conda-forge/".to_owned()), - }; - - let mut packages = vec![repo_data_record]; - - let mapping_client = pypi_mapping::PurlDerivationClient::builder( - client.clone(), - project - .config() - .cache_dir_for(pixi_config::CacheKind::PypiMapping) - .unwrap(), - ) - .finish(); - mapping_client - .amend_purls( - project.pypi_name_derivation_mode().unwrap(), - &mut packages, - None, - ) - .await - .unwrap(); - - let package = packages.pop().unwrap(); - assert_eq!( - package - .package_record - .purls - .as_ref() - .and_then(BTreeSet::first) - .unwrap() - .qualifiers() - .get("source") - .unwrap(), - PurlDerivationSource::ProjectDefinedMapping.as_str() - ); -} - -#[tokio::test] -async fn test_path_channel() { - setup_tracing(); - - let pixi = PixiControl::from_manifest(&format!( - r#" - [project] - name = "test-channel-change" - channels = ["file:///home/user/staged-recipes/build_artifacts"] - platforms = ["linux-64"] - conda-pypi-map = {{"file:///home/user/staged-recipes/build_artifacts" = "{}" }} - "#, - absolute_custom_mapping_path() - )) - .unwrap(); - - let project = pixi.workspace().unwrap(); - - let client = project.authenticated_client().unwrap(); - - let foo_bar_package = Package::build("pixi-something-new", "2").finish(); - - let repo_data_record = RepoDataRecord { - identifier: foo_bar_package.identifier(), - package_record: foo_bar_package.package_record, - url: Url::parse("https://pypi.org/simple/pixi-something-new-new/").unwrap(), - channel: Some("file:///home/user/staged-recipes/build_artifacts".to_owned()), - }; - - let mut packages = vec![repo_data_record]; - - let mapping_client = pypi_mapping::PurlDerivationClient::builder( - client.clone(), - project - .config() - .cache_dir_for(pixi_config::CacheKind::PypiMapping) - .unwrap(), - ) - .finish(); - mapping_client - .amend_purls( - project.pypi_name_derivation_mode().unwrap(), - &mut packages, - None, - ) - .await - .unwrap(); - - let package = packages.pop().unwrap(); - - assert_eq!( - package - .package_record - .purls - .as_ref() - .and_then(BTreeSet::first) - .unwrap() - .qualifiers() - .get("source") - .unwrap(), - PurlDerivationSource::ProjectDefinedMapping.as_str() - ); -} - -#[tokio::test] -async fn test_file_url_as_mapping_location() { - setup_tracing(); - - let tmp_dir = tempfile::tempdir().unwrap(); - let mapping_file = tmp_dir.path().join("custom_mapping.json"); - - let _ = fs_err::write( - &mapping_file, - r#" - { - "pixi-something-new": "pixi-something-old" - } - "#, - ); - - let mapping_file_path_as_url = Url::from_file_path( - mapping_file, /* .canonicalize() - * .expect("should be canonicalized"), */ - ) - .unwrap(); - - let pixi = PixiControl::from_manifest( - format!( - r#" - [project] - name = "test-channel-change" - channels = ["conda-forge"] - platforms = ["linux-64"] - conda-pypi-map = {{"conda-forge" = "{}"}} - "#, - mapping_file_path_as_url.as_str() - ) - .as_str(), - ) - .unwrap(); - - let project = pixi.workspace().unwrap(); - - let client = project.authenticated_client().unwrap(); - - let foo_bar_package = Package::build("pixi-something-new", "2").finish(); - - let repo_data_record = RepoDataRecord { - identifier: foo_bar_package.identifier(), - package_record: foo_bar_package.package_record, - url: Url::parse("https://pypi.org/simple/pixi-something-new-new/").unwrap(), - channel: Some("https://conda.anaconda.org/conda-forge/".to_owned()), - }; - - let mut packages = vec![repo_data_record]; - - let mapping_client = pypi_mapping::PurlDerivationClient::builder( - client.clone(), - project - .config() - .cache_dir_for(pixi_config::CacheKind::PypiMapping) - .unwrap(), - ) - .finish(); - mapping_client - .amend_purls( - project.pypi_name_derivation_mode().unwrap(), - &mut packages, - None, - ) - .await - .unwrap(); - - let package = packages.pop().unwrap(); - - assert_eq!( - package - .package_record - .purls - .as_ref() - .and_then(BTreeSet::first) - .unwrap() - .qualifiers() - .get("source") - .unwrap(), - PurlDerivationSource::ProjectDefinedMapping.as_str() - ); -} - -/// Build a `PurlDerivationClient` whose http client refuses any network -/// request, backed by the given cache directory. -fn offline_mapping_client( - project: &pixi_core::Workspace, - cache_dir: std::path::PathBuf, -) -> pypi_mapping::PurlDerivationClient { - let client = project.authenticated_client().unwrap(); - let blocked_client = ClientBuilder::from_client(client.client().clone()) - .with(OfflineMiddleware) - .build(); - pypi_mapping::PurlDerivationClient::builder(blocked_client.into(), cache_dir).finish() -} - -fn conda_forge_record(name: &str) -> RepoDataRecord { - let package = Package::build(name, "2").finish(); - RepoDataRecord { - identifier: package.identifier(), - package_record: package.package_record, - url: Url::parse(&format!("https://pypi.org/simple/{name}/")).unwrap(), - channel: Some("https://conda.anaconda.org/conda-forge/".to_owned()), - } -} - -/// An inline mapping hit in the default (extend) mode is final and requires -/// no network access. -#[tokio::test] -async fn test_extend_mapping_inline_hit_without_network() { - setup_tracing(); - - let pixi = PixiControl::from_manifest( - r#" - [project] - name = "test-extend-inline" - channels = ["conda-forge"] - platforms = ["linux-64"] - conda-pypi-map = { conda-forge = { mapping = { pixi-something-new = "my-inline-name" } } } - "#, - ) - .unwrap(); - - let project = pixi.workspace().unwrap(); - let cache_dir = TempDir::new().unwrap(); - let mapping_client = offline_mapping_client(&project, cache_dir.path().to_path_buf()); - - let mut packages = vec![conda_forge_record("pixi-something-new")]; - mapping_client - .amend_purls( - project.pypi_name_derivation_mode().unwrap(), - &mut packages, - None, - ) - .await - .unwrap(); - - let package = packages.pop().unwrap(); - let purl = package - .package_record - .purls - .as_ref() - .and_then(BTreeSet::first) - .unwrap(); - assert_eq!(purl.name(), "my-inline-name"); - assert_eq!( - purl.qualifiers().get("source").unwrap(), - PurlDerivationSource::ProjectDefinedMapping.as_str() - ); -} - -/// An explicit `false` inline entry means "not a PyPI package": no purl is -/// derived and the conda-forge verbatim fallback does not kick in either. -#[tokio::test] -async fn test_extend_mapping_explicit_false_yields_no_purl() { - setup_tracing(); - - let pixi = PixiControl::from_manifest( - r#" - [project] - name = "test-extend-false" - channels = ["conda-forge"] - platforms = ["linux-64"] - conda-pypi-map = { conda-forge = { mapping = { pixi-something-new = false } } } - "#, - ) - .unwrap(); - - let project = pixi.workspace().unwrap(); - let cache_dir = TempDir::new().unwrap(); - let mapping_client = offline_mapping_client(&project, cache_dir.path().to_path_buf()); - - let mut packages = vec![conda_forge_record("pixi-something-new")]; - mapping_client - .amend_purls( - project.pypi_name_derivation_mode().unwrap(), - &mut packages, - None, - ) - .await - .unwrap(); - - let package = packages.pop().unwrap(); - assert!( - package - .package_record - .purls - .as_ref() - .is_none_or(|purls| purls.is_empty()), - "a package explicitly mapped to `false` must not get a purl" - ); -} - -/// ` = false` disables lookups for that channel; the offline -/// conda-forge verbatim fallback still applies. -#[tokio::test] -async fn test_channel_disabled_keeps_verbatim_fallback() { - setup_tracing(); - - let pixi = PixiControl::from_manifest( - r#" - [project] - name = "test-channel-disabled" - channels = ["conda-forge"] - platforms = ["linux-64"] - conda-pypi-map = { conda-forge = false } - "#, - ) - .unwrap(); - - let project = pixi.workspace().unwrap(); - let cache_dir = TempDir::new().unwrap(); - let mapping_client = offline_mapping_client(&project, cache_dir.path().to_path_buf()); - - let mut packages = vec![conda_forge_record("boltons")]; - mapping_client - .amend_purls( - project.pypi_name_derivation_mode().unwrap(), - &mut packages, - None, - ) - .await - .unwrap(); - - let package = packages.pop().unwrap(); - let purl = package - .package_record - .purls - .as_ref() - .and_then(BTreeSet::first) - .unwrap(); - // The verbatim fallback assumes the conda name is the pypi name and adds - // no source qualifier. - assert_eq!(purl.name(), "boltons"); - assert!(purl.qualifiers().is_empty()); -} - -/// When an entry has both a `location` and inline `mapping` entries, the -/// inline entries override the ones from the location. -#[tokio::test] -async fn test_inline_mapping_overrides_location() { - setup_tracing(); - - // The custom mapping file maps `pixi-something-new` to itself; the inline - // entry must win. - let pixi = PixiControl::from_manifest(&format!( - r#" - [project] - name = "test-inline-overrides" - channels = ["conda-forge"] - platforms = ["linux-64"] - conda-pypi-map = {{ conda-forge = {{ location = "{}", mapping = {{ pixi-something-new = "inline-wins" }} }} }} - "#, - absolute_custom_mapping_path() - )) - .unwrap(); - - let project = pixi.workspace().unwrap(); - let cache_dir = TempDir::new().unwrap(); - let mapping_client = offline_mapping_client(&project, cache_dir.path().to_path_buf()); - - let mut packages = vec![conda_forge_record("pixi-something-new")]; - mapping_client - .amend_purls( - project.pypi_name_derivation_mode().unwrap(), - &mut packages, - None, - ) - .await - .unwrap(); - - let package = packages.pop().unwrap(); - let purl = package - .package_record - .purls - .as_ref() - .and_then(BTreeSet::first) - .unwrap(); - assert_eq!(purl.name(), "inline-wins"); -} - -/// In extend mode a miss in the project-defined mapping falls through to the -/// prefix.dev chain. -#[tokio::test] -#[cfg_attr(not(feature = "online_tests"), ignore)] -async fn test_extend_mapping_miss_falls_through_to_prefix() { - setup_tracing(); - - // The mapping contains an unrelated package, so `boltons` is a miss and - // must be resolved through the prefix.dev chain (the mock-built record's - // hash is unknown there, so the compressed name mapping answers). - let pixi = PixiControl::from_manifest( - r#" - [project] - name = "test-extend-miss" - channels = ["conda-forge"] - platforms = ["linux-64"] - conda-pypi-map = { conda-forge = { mapping = { some-other-package = "other" } } } - "#, - ) - .unwrap(); - - let project = pixi.workspace().unwrap(); - let client = project.authenticated_client().unwrap(); - let mapping_client = pypi_mapping::PurlDerivationClient::builder( - client.clone(), - project - .config() - .cache_dir_for(pixi_config::CacheKind::PypiMapping) - .unwrap(), - ) - .finish(); - - let mut packages = vec![conda_forge_record("boltons")]; - mapping_client - .amend_purls( - project.pypi_name_derivation_mode().unwrap(), - &mut packages, - None, - ) - .await - .unwrap(); - - let package = packages.pop().unwrap(); - let purl = package - .package_record - .purls - .as_ref() - .and_then(BTreeSet::first) - .unwrap(); - assert_eq!(purl.name(), "boltons"); - assert_eq!( - purl.qualifiers().get("source").unwrap(), - PurlDerivationSource::PrefixCompressedMapping.as_str() - ); -} - -/// The on-disk path of the TTL cache for a mapping url, mirroring the layout -/// used by the project-defined mapping resolver. -fn ttl_cache_path_for(cache_dir: &Path, url: &str) -> std::path::PathBuf { - let hash = rattler_digest::compute_bytes_digest::(url.as_bytes()); - cache_dir - .join("project-defined") - .join(format!("{hash:x}.json")) -} - -fn manifest_with_ttl_mapping(url: &str, ttl: &str) -> String { - format!( - r#" - [project] - name = "test-cache-ttl" - channels = ["conda-forge"] - platforms = ["linux-64"] - conda-pypi-map = {{ conda-forge = {{ location = "{url}", cache-ttl = "{ttl}" }} }} - "# - ) -} - -/// A cached mapping younger than `cache-ttl` is used without any network -/// access. -#[tokio::test] -async fn test_cache_ttl_fresh_cache_skips_network() { - setup_tracing(); - - let mapping_url = "https://example.invalid/mapping.json"; - let pixi = PixiControl::from_manifest(&manifest_with_ttl_mapping(mapping_url, "1h")).unwrap(); - - let project = pixi.workspace().unwrap(); - let cache_dir = TempDir::new().unwrap(); - - // Pre-populate the TTL cache; the url itself is unreachable and the - // client is offline, so a cache miss would fail the test. - let cache_file = ttl_cache_path_for(cache_dir.path(), mapping_url); - fs_err::create_dir_all(cache_file.parent().unwrap()).unwrap(); - fs_err::write(&cache_file, r#"{ "pixi-something-new": "from-the-cache" }"#).unwrap(); - - let mapping_client = offline_mapping_client(&project, cache_dir.path().to_path_buf()); - - let mut packages = vec![conda_forge_record("pixi-something-new")]; - mapping_client - .amend_purls( - project.pypi_name_derivation_mode().unwrap(), - &mut packages, - None, - ) - .await - .unwrap(); - - let package = packages.pop().unwrap(); - let purl = package - .package_record - .purls - .as_ref() - .and_then(BTreeSet::first) - .unwrap(); - assert_eq!(purl.name(), "from-the-cache"); -} - -/// When the cached mapping is expired and the refetch fails, the stale copy -/// is used so solves keep working offline. -#[tokio::test] -async fn test_cache_ttl_expired_falls_back_to_stale_copy() { - setup_tracing(); - - let mapping_url = "https://example.invalid/mapping.json"; - // A zero TTL means the cached copy is always considered expired. - let pixi = PixiControl::from_manifest(&manifest_with_ttl_mapping(mapping_url, "0s")).unwrap(); - - let project = pixi.workspace().unwrap(); - let cache_dir = TempDir::new().unwrap(); - - let cache_file = ttl_cache_path_for(cache_dir.path(), mapping_url); - fs_err::create_dir_all(cache_file.parent().unwrap()).unwrap(); - fs_err::write(&cache_file, r#"{ "pixi-something-new": "stale-but-used" }"#).unwrap(); - - let mapping_client = offline_mapping_client(&project, cache_dir.path().to_path_buf()); - - let mut packages = vec![conda_forge_record("pixi-something-new")]; - mapping_client - .amend_purls( - project.pypi_name_derivation_mode().unwrap(), - &mut packages, - None, - ) - .await - .unwrap(); - - let package = packages.pop().unwrap(); - let purl = package - .package_record - .purls - .as_ref() - .and_then(BTreeSet::first) - .unwrap(); - assert_eq!(purl.name(), "stale-but-used"); -} - -/// Without any cached copy, a failing fetch of a TTL-cached mapping is a hard -/// error. -#[tokio::test] -async fn test_cache_ttl_no_cache_and_fetch_failure_errors() { - setup_tracing(); - - let mapping_url = "https://example.invalid/mapping.json"; - let pixi = PixiControl::from_manifest(&manifest_with_ttl_mapping(mapping_url, "1h")).unwrap(); - - let project = pixi.workspace().unwrap(); - let cache_dir = TempDir::new().unwrap(); - let mapping_client = offline_mapping_client(&project, cache_dir.path().to_path_buf()); - - let mut packages = vec![conda_forge_record("pixi-something-new")]; - let result = mapping_client - .amend_purls( - project.pypi_name_derivation_mode().unwrap(), - &mut packages, - None, - ) - .await; - - assert!( - result.is_err(), - "an uncached TTL mapping with a failing fetch must error" - ); -} - -/// A failing prefix.dev lookup must point firewall-restricted users at the -/// manifest options that avoid the network. -#[tokio::test] -async fn test_prefix_fetch_failure_error_mentions_escape_hatches() { - setup_tracing(); - - // No `conda-pypi-map` -> the default prefix.dev chain, which needs the - // network to look up the record by hash. - let pixi = PixiControl::from_manifest( - r#" - [project] - name = "test-network-error" - channels = ["conda-forge"] - platforms = ["linux-64"] - "#, - ) - .unwrap(); - - let project = pixi.workspace().unwrap(); - let cache_dir = TempDir::new().unwrap(); - let mapping_client = offline_mapping_client(&project, cache_dir.path().to_path_buf()); - - let mut packages = vec![conda_forge_record("pixi-something-new")]; - let err = mapping_client - .amend_purls( - project.pypi_name_derivation_mode().unwrap(), - &mut packages, - None, - ) - .await - .expect_err("an offline prefix.dev lookup should fail"); - - let rendered = format!("{err:?}"); - assert!( - rendered.contains("mode = \"replace\"") && rendered.contains("conda-pypi-map = false"), - "the error should suggest the offline escape hatches, got: {rendered}" - ); -} - -/// `conda-pypi-map = {}` is a soft-deprecated alias for -/// `conda-pypi-map = false`; both disable all mapping lookups while keeping -/// the conda-forge verbatim fallback. -#[tokio::test] -async fn test_disabled_mapping() { - setup_tracing(); - - let pixi = PixiControl::from_manifest( - r#" - [project] - name = "test-channel-change" - channels = ["https://prefix.dev/conda-forge"] - platforms = ["linux-64"] - conda-pypi-map = { } - "#, - ) - .unwrap(); - - let project = pixi.workspace().unwrap(); - - let client = project.authenticated_client().unwrap(); - - let blocking_middleware = OfflineMiddleware; - - let blocked_client = ClientBuilder::from_client(client.client().clone()) - .with(blocking_middleware) - .build(); - - let boltons_package = Package::build("boltons", "2").finish(); - - let boltons_repo_data_record = RepoDataRecord { - identifier: boltons_package.identifier(), - package_record: boltons_package.package_record, - url: Url::parse("https://pypi.org/simple/boltons/").unwrap(), - channel: Some("https://conda.anaconda.org/conda-forge/".to_owned()), - }; - - let mut packages = vec![boltons_repo_data_record]; - - let mapping_client = pypi_mapping::PurlDerivationClient::builder( - blocked_client.into(), - project - .config() - .cache_dir_for(pixi_config::CacheKind::PypiMapping) - .unwrap(), - ) - .finish(); - mapping_client - .amend_purls( - project.pypi_name_derivation_mode().unwrap(), - &mut packages, - None, - ) - .await - .unwrap(); - - let boltons_package = packages.pop().unwrap(); - - let boltons_first_purl = boltons_package - .package_record - .purls - .as_ref() - .and_then(BTreeSet::first) - .unwrap(); - - // we verify that even if this name is not present in our mapping - // we record a purl anyways. Because we make the assumption - // that it's a pypi package - assert_eq!(boltons_first_purl.name(), "boltons"); - assert!(boltons_first_purl.qualifiers().is_empty()); -} - -/// `conda-pypi-map = false` is the canonical global disable: no lookups, but -/// the conda-forge verbatim fallback still applies. -#[tokio::test] -async fn test_disabled_mapping_via_false() { - setup_tracing(); - - let pixi = PixiControl::from_manifest( - r#" - [project] - name = "test-disable-false" - channels = ["https://prefix.dev/conda-forge"] - platforms = ["linux-64"] - conda-pypi-map = false - "#, - ) - .unwrap(); - - let project = pixi.workspace().unwrap(); - let cache_dir = TempDir::new().unwrap(); - let mapping_client = offline_mapping_client(&project, cache_dir.path().to_path_buf()); - - let mut packages = vec![conda_forge_record("boltons")]; - mapping_client - .amend_purls( - project.pypi_name_derivation_mode().unwrap(), - &mut packages, - None, - ) - .await - .unwrap(); - - let boltons_package = packages.pop().unwrap(); - let boltons_first_purl = boltons_package - .package_record - .purls - .as_ref() - .and_then(BTreeSet::first) - .unwrap(); - assert_eq!(boltons_first_purl.name(), "boltons"); - assert!(boltons_first_purl.qualifiers().is_empty()); -} - -#[tokio::test] -async fn test_custom_mapping_ignores_backwards_compatibility() { - setup_tracing(); - - // Create local conda channel with boltons and python packages - let mut package_database = MockRepoData::default(); - package_database.add_package( - Package::build("python", "3.12.0") - .with_subdir(Platform::Linux64) - .finish(), - ); - package_database.add_package( - Package::build("boltons", "24.0.0") - .with_subdir(Platform::Linux64) - .finish(), - ); - let channel = package_database.into_channel().await.unwrap(); - let channel_url = channel.url(); - - // Create local PyPI index with boltons package - let pypi_index = PyPIDatabase::new() - .with(PyPIPackage::new("boltons", "24.0.0")) - .into_simple_index() - .expect("failed to create local simple index"); - - // Create a project-defined mapping file that only includes specific packages - let temp_dir = TempDir::new().unwrap(); - let mapping_file = temp_dir.path().join("map.json"); - fs_err::write(&mapping_file, r#"{}"#).unwrap(); - - let pixi = PixiControl::from_manifest(&format!( - r#" - [workspace] - name = "test-custom-mapping" - channels = ["{channel_url}"] - platforms = ["linux-64"] - conda-pypi-map = {{ "{channel_url}" = {{ location = "{mapping_file}", mode = "replace" }} }} - - [dependencies] - python = "3.12.0" - boltons = "*" - - [pypi-dependencies] - boltons = "*" - - [pypi-options] - index-url = "{pypi_url}" - "#, - channel_url = channel_url, - mapping_file = mapping_file - .to_str() - .unwrap() - .to_string() - .replace("\\", "/"), - pypi_url = pypi_index.index_url(), - )) - .unwrap(); - - // Lock the project (this triggers the amend_purls logic) - pixi.lock().await.unwrap(); - - // Get the lock file - let lock = pixi.lock_file().await.unwrap(); - let p = lock.platform(&Platform::Linux64.to_string()).unwrap(); - let environment = lock.environment(DEFAULT_ENVIRONMENT_NAME).unwrap(); - let conda_packages = environment.conda_packages(p).unwrap(); - - // Collect conda packages to a vector so we can iterate over them - let conda_packages: Vec<_> = conda_packages.collect(); - - // Find boltons in conda packages - let boltons_package = conda_packages - .iter() - .find(|pkg| match pkg { - rattler_lock::CondaPackageData::Binary(binary) => { - binary.package_record.name.as_source() == "boltons" - } - _ => panic!("All packagees should be binary"), - }) - .expect("boltons should be present in conda packages"); - - // The issue: boltons should NOT have purls when using project-defined mapping - // because it's not specified in our project-defined mapping - // But due to backwards compatibility logic, it gets purls anyway - let purls = match boltons_package { - rattler_lock::CondaPackageData::Binary(binary) => &binary.package_record.purls, - _ => panic!("All packages should be binary"), - }; - - if let Some(purls) = purls { - assert!( - purls.is_empty(), - "boltons should not have purls when not specified in custom conda-pypi-map" - ); - } -} - /// Test that environments in a solve-group can have different editability settings /// for the same path-based PyPI package. /// @@ -1706,59 +361,3 @@ core = { path = "../core", editable = true } "default environment should contain middle" ); } - -#[tokio::test] -async fn test_missing_mapping_file_error_includes_path() { - setup_tracing(); - - let pixi = PixiControl::new().unwrap(); - pixi.init().await.unwrap(); - - let project = pixi.workspace().unwrap(); - let client = project.authenticated_client().unwrap(); - - // Use a non-existent file path for the project-defined mapping - let non_existent_path = Path::new("/this/path/does/not/exist/mapping.json"); - - let source = HashMap::from([( - "https://conda.anaconda.org/conda-forge".to_owned(), - ProjectDefinedChannelMapping::replace(ProjectDefinedMappingLocation::Path( - non_existent_path.to_path_buf(), - )), - )]); - - let foo_bar_package = Package::build("foo-bar-car", "2").finish(); - - let mut repo_data_record = RepoDataRecord { - identifier: foo_bar_package.identifier(), - package_record: foo_bar_package.package_record, - url: Url::parse("https://pypi.org/simple/boltons/").unwrap(), - channel: Some("https://conda.anaconda.org/conda-forge/".to_owned()), - }; - - let mapping_client = pypi_mapping::PurlDerivationClient::builder( - client.clone(), - project - .config() - .cache_dir_for(pixi_config::CacheKind::PypiMapping) - .unwrap(), - ) - .finish(); - let result = mapping_client - .amend_purls( - &PurlDerivationMode::ProjectDefined(Arc::new(ProjectDefinedMapping::new(source))), - vec![&mut repo_data_record], - None, - ) - .await; - - // The operation should fail because the mapping file doesn't exist - let err = result.expect_err("Expected an error when mapping file doesn't exist"); - insta::with_settings!({filters => vec![ - (r#"path: "([^"]+)""#, "[MAPPING_PATH]"), - (r#"message: "[^"]+""#, "[MAPPING_MESSAGE]"), - (r#"\bcode:\s*\d+\b"#, "[MAPPING_CODE]"), - ]}, { - insta::assert_debug_snapshot!(err); - }); -} diff --git a/crates/pixi_core/src/workspace/mod.rs b/crates/pixi_core/src/workspace/mod.rs index cee8e68798..32d8687125 100644 --- a/crates/pixi_core/src/workspace/mod.rs +++ b/crates/pixi_core/src/workspace/mod.rs @@ -914,20 +914,23 @@ impl Workspace { /// It can use project-defined mappings in the format `conda_name: pypi_name`, /// or the self-hosted prefix.dev mappings. pub fn pypi_name_derivation_mode(&self) -> miette::Result<&PurlDerivationMode> { - /// Classify a manifest location string into a url or a path, resolving + /// Classify a manifest location spec into a url or a path, resolving /// relative paths against the workspace root. fn parse_mapping_location( - mapping_location: &str, - cache_ttl: Option, + spec: &pixi_manifest::MappingLocationSpec, channel_config: &ChannelConfig, ) -> miette::Result { + let mapping_location = spec.location.as_str(); if mapping_location.starts_with("https://") || mapping_location.starts_with("http://") { let url = Url::parse(mapping_location) .into_diagnostic() .context(format!("Could not convert {mapping_location} to URL"))?; - Ok(ProjectDefinedMappingLocation::Url { url, cache_ttl }) + Ok(ProjectDefinedMappingLocation::Url { + url, + cache_ttl: spec.cache_ttl, + }) } else { - if cache_ttl.is_some() { + if spec.cache_ttl.is_some() { miette::bail!( "`cache-ttl` is only supported for http(s) mapping locations, but `{mapping_location}` is a local file" ); @@ -960,19 +963,14 @@ impl Workspace { ) -> miette::Result { match entry { CondaPypiMapEntry::Disabled => Ok(ProjectDefinedChannelMapping::disabled()), - CondaPypiMapEntry::Map { + CondaPypiMapEntry::Map(pixi_manifest::CondaPypiMapSpec { location, mapping, mode, - cache_ttl, - } => { + }) => { let mut sources = Vec::new(); if let Some(location) = location { - sources.push(parse_mapping_location( - location, - *cache_ttl, - channel_config, - )?); + sources.push(parse_mapping_location(location, channel_config)?); } // Inline entries come last so they override entries from // the location. Keys are lowercased to match the diff --git a/crates/pixi_manifest/src/lib.rs b/crates/pixi_manifest/src/lib.rs index 54989d477c..4dc9fbd54a 100644 --- a/crates/pixi_manifest/src/lib.rs +++ b/crates/pixi_manifest/src/lib.rs @@ -64,7 +64,7 @@ use thiserror::Error; pub use warning::{Warning, WarningWithSource, WithWarnings}; pub use workspace::{ BuildVariantSource, ChannelPriority, CondaPypiMap, CondaPypiMapEntry, CondaPypiMapMode, - SolveStrategy, Workspace, + CondaPypiMapSpec, MappingLocationSpec, SolveStrategy, Workspace, }; pub use crate::{ diff --git a/crates/pixi_manifest/src/toml/conda_pypi_map.rs b/crates/pixi_manifest/src/toml/conda_pypi_map.rs index 18e6b972dc..43a7ebbd4a 100644 --- a/crates/pixi_manifest/src/toml/conda_pypi_map.rs +++ b/crates/pixi_manifest/src/toml/conda_pypi_map.rs @@ -15,7 +15,9 @@ use toml_span::{ value::ValueInner, }; -use crate::workspace::{CondaPypiMap, CondaPypiMapEntry, CondaPypiMapMode}; +use crate::workspace::{ + CondaPypiMap, CondaPypiMapEntry, CondaPypiMapMode, CondaPypiMapSpec, MappingLocationSpec, +}; impl<'de> toml_span::Deserialize<'de> for CondaPypiMap { fn deserialize(value: &mut Value<'de>) -> Result { @@ -105,23 +107,31 @@ impl<'de> toml_span::Deserialize<'de> for CondaPypiMapEntry { .into()); } - if cache_ttl.is_some() && location.is_none() { - return Err(Error { - kind: ErrorKind::Custom( - "`cache-ttl` requires a `location` that is a URL".into(), - ), - span: table_span, - line_info: None, + // `cache-ttl` is part of the location source; without a + // location it has nothing to apply to. + let location = match (location, cache_ttl) { + (Some(location), cache_ttl) => Some(MappingLocationSpec { + location, + cache_ttl, + }), + (None, Some(_)) => { + return Err(Error { + kind: ErrorKind::Custom( + "`cache-ttl` requires a `location` that is a URL".into(), + ), + span: table_span, + line_info: None, + } + .into()); } - .into()); - } + (None, None) => None, + }; - Ok(CondaPypiMapEntry::Map { + Ok(CondaPypiMapEntry::Map(CondaPypiMapSpec { location, mapping, mode, - cache_ttl, - }) + })) } other => Err(expected("a string, table or `false`", other, value.span).into()), } @@ -193,12 +203,14 @@ mod test { let map = parse_map(r#"{ conda-forge = "mapping.json" }"#); assert_eq!( get_entry(&map, "conda-forge"), - CondaPypiMapEntry::Map { - location: Some("mapping.json".to_string()), + CondaPypiMapEntry::Map(CondaPypiMapSpec { + location: Some(MappingLocationSpec { + location: "mapping.json".to_string(), + cache_ttl: None, + }), mapping: None, mode: CondaPypiMapMode::Extend, - cache_ttl: None, - } + }) ); } @@ -209,12 +221,14 @@ mod test { ); assert_eq!( get_entry(&map, "conda-forge"), - CondaPypiMapEntry::Map { - location: Some("https://example.com/m.json".to_string()), + CondaPypiMapEntry::Map(CondaPypiMapSpec { + location: Some(MappingLocationSpec { + location: "https://example.com/m.json".to_string(), + cache_ttl: Some(Duration::from_secs(24 * 60 * 60)), + }), mapping: None, mode: CondaPypiMapMode::Replace, - cache_ttl: Some(Duration::from_secs(24 * 60 * 60)), - } + }) ); } @@ -223,7 +237,9 @@ mod test { let map = parse_map( r#"{ conda-forge = { mapping = { pytorch = "torch", not-on-pypi = false } } }"#, ); - let CondaPypiMapEntry::Map { mapping, mode, .. } = get_entry(&map, "conda-forge") else { + let CondaPypiMapEntry::Map(CondaPypiMapSpec { mapping, mode, .. }) = + get_entry(&map, "conda-forge") + else { panic!("expected a mapping entry"); }; let mapping = mapping.expect("mapping should be set"); diff --git a/crates/pixi_manifest/src/workspace.rs b/crates/pixi_manifest/src/workspace.rs index 5be276a073..043264b22d 100644 --- a/crates/pixi_manifest/src/workspace.rs +++ b/crates/pixi_manifest/src/workspace.rs @@ -310,6 +310,10 @@ impl From for ChannelPriority { #[derive(Debug, Clone, PartialEq)] pub enum CondaPypiMap { /// `conda-pypi-map = false`: disable purl derivation lookups entirely. + /// + /// Note that the offline conda-forge verbatim fallback (assume the conda + /// name is the PyPI name) still applies; disabling only turns off the + /// project-defined and prefix.dev lookups. Disabled, /// Per-channel mapping configuration. An empty map is a soft-deprecated /// alias for `Disabled`. @@ -343,32 +347,50 @@ pub enum CondaPypiMapMode { #[derive(Debug, Clone, PartialEq)] pub enum CondaPypiMapEntry { /// ` = false`: no purl lookups for this channel. + /// + /// The offline conda-forge verbatim fallback (assume the conda name is + /// the PyPI name) still applies to records from this channel. Disabled, /// A mapping defined by a location (file or URL) and/or inline entries. - Map { - /// File path or URL of a mapping JSON file. Unresolved: relative - /// paths are resolved against the workspace root by the consumer. - location: Option, - /// Inline conda-name to pypi-name entries. A `None` value (spelled - /// `false` in TOML) means the package is not a PyPI package. - mapping: Option>>, - mode: CondaPypiMapMode, - /// How long a mapping fetched from a URL may be reused before it is - /// re-fetched. Only valid for http(s) locations. - cache_ttl: Option, - }, + Map(CondaPypiMapSpec), +} + +/// A channel mapping built from up to two sources: an external location and +/// inline entries. Inline entries override entries from the location. +#[derive(Debug, Clone, PartialEq)] +pub struct CondaPypiMapSpec { + /// An external mapping JSON file (path or URL). + pub location: Option, + /// Inline conda-name to pypi-name entries. A `None` value (spelled + /// `false` in TOML) means the package is not a PyPI package. + pub mapping: Option>>, + pub mode: CondaPypiMapMode, +} + +/// An external mapping source: a file path or URL, with an optional cache +/// TTL for URL locations. +#[derive(Debug, Clone, PartialEq)] +pub struct MappingLocationSpec { + /// File path or URL of a mapping JSON file. Unresolved: relative paths + /// are resolved against the workspace root by the consumer. + pub location: String, + /// How long a mapping fetched from a URL may be reused before it is + /// re-fetched. Only valid for http(s) locations. + pub cache_ttl: Option, } impl CondaPypiMapEntry { /// Create an entry from a bare location string. Bare strings use the /// default (extend) mode. pub fn from_location(location: String) -> Self { - Self::Map { - location: Some(location), + Self::Map(CondaPypiMapSpec { + location: Some(MappingLocationSpec { + location, + cache_ttl: None, + }), mapping: None, mode: CondaPypiMapMode::default(), - cache_ttl: None, - } + }) } } diff --git a/crates/pypi_mapping/src/lib.rs b/crates/pypi_mapping/src/lib.rs index fceaba1907..e087205dd3 100644 --- a/crates/pypi_mapping/src/lib.rs +++ b/crates/pypi_mapping/src/lib.rs @@ -62,6 +62,13 @@ use crate::{ /// name. pub type CompressedMapping = HashMap>; +/// Help text shown when fetching a conda-pypi mapping over the network fails, +/// listing the manifest options that avoid the network lookup. +pub(crate) const MAPPING_OFFLINE_HELP: &str = "If this host cannot be reached (e.g. behind a firewall), you can avoid the network lookup: \ + point the channel's `conda-pypi-map` entry at your own mapping with `location` (optionally \ + cached with `cache-ttl`), make it exclusive with `mode = \"replace\"`, disable the channel \ + with ` = false`, or disable the mapping entirely with `conda-pypi-map = false`."; + /// The mapping client implements the logic to derive purls for conda packages. /// /// The resolver order depends on [`PurlDerivationMode`]: @@ -138,12 +145,7 @@ pub enum MappingError { path: PathBuf, }, #[error("failed to fetch conda-pypi mapping from remote source")] - #[diagnostic(help( - "If this host cannot be reached (e.g. behind a firewall), you can avoid the network \ - lookup: use `mode = \"replace\"` on the `conda-pypi-map` entry for the channel, disable \ - the channel's mapping with ` = false`, or disable the mapping entirely with \ - `conda-pypi-map = false`." - ))] + #[diagnostic(help("{}", MAPPING_OFFLINE_HELP))] Reqwest(#[source] reqwest_middleware::Error), } diff --git a/crates/pypi_mapping/src/resolvers/project_defined_mapping.rs b/crates/pypi_mapping/src/resolvers/project_defined_mapping.rs index e1ab3c26c1..f58ddf7238 100644 --- a/crates/pypi_mapping/src/resolvers/project_defined_mapping.rs +++ b/crates/pypi_mapping/src/resolvers/project_defined_mapping.rs @@ -104,9 +104,7 @@ async fn fetch_mapping_from_url( .await .into_diagnostic() .wrap_err(miette::diagnostic!( - help = "If this host cannot be reached (e.g. behind a firewall), consider \ - caching the mapping with `cache-ttl`, using a local file, or disabling \ - the mapping for this channel with ` = false`.", + help = crate::MAPPING_OFFLINE_HELP, "failed to download pypi mapping from {} location", url.as_str() ))?; @@ -130,6 +128,13 @@ async fn fetch_mapping_from_url( /// A cached copy younger than `ttl` is used without touching the network. /// When the refetch of an expired copy fails, the stale copy is used with a /// warning so that solves keep working offline. +/// +/// This is a small mtime-based file cache (the same pattern as the reverse +/// pypi-to-conda mapping cache in pixi-build-python) rather than the +/// `http-cache` middleware that already wraps the client: the middleware's +/// freshness is driven by server cache headers, its `max_ttl` is client-global +/// while `cache-ttl` is configured per mapping entry, and it has no +/// use-stale-on-error behavior. async fn fetch_mapping_with_ttl( client: &LazyClient, url: &Url, diff --git a/docs/concepts/conda_pypi.md b/docs/concepts/conda_pypi.md index 5a6c22f643..876c6c099d 100644 --- a/docs/concepts/conda_pypi.md +++ b/docs/concepts/conda_pypi.md @@ -132,6 +132,16 @@ If that host is unreachable in your environment, you have several options: - `mode = "replace"` with your own mapping file avoids network lookups for that channel entirely. - `cache-ttl = "24h"` on a URL location caches the fetched mapping on disk and re-fetches it at most once per TTL; if the re-fetch fails, the cached copy is used. +For example, you can pin the full conda-forge name mapping that `parselmouth` publishes (the same data the default mapping is built from) and refresh it at most once a day: + +```toml title="pixi.toml" +[workspace.conda-pypi-map] +conda-forge = { location = "https://raw.githubusercontent.com/prefix-dev/parselmouth/main/files/compressed_mapping.json", mode = "replace", cache-ttl = "24h" } +``` + +!!! note + Use the `raw.githubusercontent.com` URL — the regular `github.com/.../blob/...` page serves HTML, not JSON. + ### PyPI overrides vs conda constraints PyPI's [`pypi-options.dependency-overrides`](../advanced/override.md) From ac9d12f2023731eb6bf6052071688d4335edec7d Mon Sep 17 00:00:00 2001 From: Tim de Jager Date: Wed, 10 Jun 2026 14:42:21 +0200 Subject: [PATCH 07/10] refactor: abstraction cleanups from review - pixi_toml: add a custom_error(message, span) constructor and use it for the conda-pypi-map validation errors. - pixi_core: extract the conda-pypi-map manifest conversion out of workspace/mod.rs into a workspace::conda_pypi_map module with named, unit-testable functions (incl. the channel-membership validation). - pixi_core: classify mapping locations with rattler_lock::UrlOrPath instead of hand-rolled starts_with checks; file:// urls normalize to paths and non-http(s) remote schemes are rejected with a clear error. - pypi_mapping: make the per-record fallback policy explicit with a Fallback enum (PrefixThenVerbatim | Verbatim | None) instead of a mutable suppression flag. - pixi-build-python: dedupe the requirement version conversion into convert_requirement_version, shared by the user-map and service paths. - test: pin that a mapping for one channel no longer suppresses the verbatim fallback for records from other, unmapped channels (online). --- .../integration_rust/conda_pypi_map_tests.rs | 61 ++++++ crates/pixi_build_python/src/pypi_mapping.rs | 55 +++-- .../pixi_core/src/workspace/conda_pypi_map.rs | 197 ++++++++++++++++++ crates/pixi_core/src/workspace/mod.rs | 189 ++--------------- .../pixi_manifest/src/toml/conda_pypi_map.rs | 79 +++---- crates/pixi_toml/src/diagnostic.rs | 15 ++ crates/pixi_toml/src/lib.rs | 2 +- crates/pypi_mapping/src/lib.rs | 81 ++++--- 8 files changed, 396 insertions(+), 283 deletions(-) create mode 100644 crates/pixi_core/src/workspace/conda_pypi_map.rs diff --git a/crates/pixi/tests/integration_rust/conda_pypi_map_tests.rs b/crates/pixi/tests/integration_rust/conda_pypi_map_tests.rs index 18ab481e6e..6ff1093102 100644 --- a/crates/pixi/tests/integration_rust/conda_pypi_map_tests.rs +++ b/crates/pixi/tests/integration_rust/conda_pypi_map_tests.rs @@ -977,6 +977,67 @@ async fn test_extend_mapping_miss_falls_through_to_prefix() { ); } +/// A mapping for one channel must not affect records from other channels: +/// they go through the full default chain, including the conda-forge verbatim +/// fallback. +/// +/// This pins the per-record fallback behavior: previously, configuring any +/// `conda-pypi-map` suppressed the verbatim fallback globally, degrading purl +/// coverage even for channels that were not in the map. +#[tokio::test] +#[cfg_attr(not(feature = "online_tests"), ignore)] +async fn test_mapping_for_other_channel_keeps_verbatim_fallback() { + setup_tracing(); + + // A replace-mode mapping for robostack only; the record below comes from + // conda-forge and its name is unknown to both the prefix.dev hash and + // compressed mappings, so only the verbatim fallback can answer. + let pixi = PixiControl::from_manifest(&format!( + r#" + [project] + name = "test-unmapped-channel-verbatim" + channels = ["conda-forge", "robostack"] + platforms = ["linux-64"] + conda-pypi-map = {{ robostack = {{ location = "{}", mode = "replace" }} }} + "#, + absolute_custom_mapping_path() + )) + .unwrap(); + + let project = pixi.workspace().unwrap(); + let client = project.authenticated_client().unwrap(); + let mapping_client = pypi_mapping::PurlDerivationClient::builder( + client.clone(), + project + .config() + .cache_dir_for(pixi_config::CacheKind::PypiMapping) + .unwrap(), + ) + .finish(); + + let mut packages = vec![conda_forge_record("pixi-something-new")]; + mapping_client + .amend_purls( + project.pypi_name_derivation_mode().unwrap(), + &mut packages, + None, + ) + .await + .unwrap(); + + let package = packages.pop().unwrap(); + let purl = package + .package_record + .purls + .as_ref() + .and_then(BTreeSet::first) + .expect("a record from an unmapped channel should get the verbatim fallback purl"); + // The verbatim fallback assumes the conda name is the pypi name and adds + // no source qualifier. + assert_eq!(purl.name(), "pixi-something-new"); + assert!(purl.qualifiers().is_empty()); +} + /// The on-disk path of the TTL cache for a mapping url, mirroring the layout /// used by the project-defined mapping resolver. fn ttl_cache_path_for(cache_dir: &Path, url: &str) -> std::path::PathBuf { diff --git a/crates/pixi_build_python/src/pypi_mapping.rs b/crates/pixi_build_python/src/pypi_mapping.rs index 1b3cd6d1cc..146125d775 100644 --- a/crates/pixi_build_python/src/pypi_mapping.rs +++ b/crates/pixi_build_python/src/pypi_mapping.rs @@ -454,26 +454,9 @@ impl PyPiToCondaMapper { let conda_name = PackageName::from_str(&conda_name_str) .map_err(|e| MappingError::InvalidPackageName(conda_name_str.clone(), e))?; - // Convert version specifiers - let version_spec = if let Some(ref version_or_url) = req.version_or_url { - match Self::convert_version_specifiers(version_or_url) { - Ok(spec) => spec, - Err(e) => { - tracing::warn!( - "Failed to convert version specifier for '{}': {}, using unconstrained version", - req.name, - e - ); - None - } - } - } else { - None - }; - mapped.push(MappedCondaDependency { name: conda_name, - version_spec, + version_spec: convert_requirement_version(req), }); } @@ -481,6 +464,26 @@ impl PyPiToCondaMapper { } } +/// Convert the version specifiers of a requirement to a conda version spec, +/// warning and falling back to an unconstrained version when the conversion +/// fails. +fn convert_requirement_version( + req: &pep508_rs::Requirement, +) -> Option { + let version_or_url = req.version_or_url.as_ref()?; + match PyPiToCondaMapper::convert_version_specifiers(version_or_url) { + Ok(spec) => spec, + Err(e) => { + tracing::warn!( + "Failed to convert version specifier for '{}': {}, using unconstrained version", + req.name, + e + ); + None + } + } +} + /// Filter mapped PyPI dependencies, returning only those not already specified /// in Pixi's run dependencies. /// @@ -567,20 +570,10 @@ fn apply_user_map( } PypiCondaMapEntry::CondaName(conda_name) => match PackageName::from_str(conda_name) { Ok(name) => { - let version_spec = req.version_or_url.as_ref().and_then(|version_or_url| { - match PyPiToCondaMapper::convert_version_specifiers(version_or_url) { - Ok(spec) => spec, - Err(err) => { - tracing::warn!( - "Failed to convert version specifier for '{}': {}, using unconstrained version", - req.name, - err - ); - None - } - } + user_mapped.push(MappedCondaDependency { + name, + version_spec: convert_requirement_version(req), }); - user_mapped.push(MappedCondaDependency { name, version_spec }); } Err(err) => { tracing::warn!( diff --git a/crates/pixi_core/src/workspace/conda_pypi_map.rs b/crates/pixi_core/src/workspace/conda_pypi_map.rs new file mode 100644 index 0000000000..5b334cd5a8 --- /dev/null +++ b/crates/pixi_core/src/workspace/conda_pypi_map.rs @@ -0,0 +1,197 @@ +//! Converts the manifest `[workspace.conda-pypi-map]` configuration into the +//! per-channel mapping configuration used by the purl derivation client. + +use std::{ + collections::{HashMap, HashSet}, + path::PathBuf, + str::FromStr, +}; + +use itertools::Itertools; +use miette::{Context, IntoDiagnostic}; +use pixi_manifest::{ + CondaPypiMap, CondaPypiMapEntry, CondaPypiMapMode, CondaPypiMapSpec, MappingLocationSpec, + WorkspaceManifest, +}; +use pypi_mapping::{ + ChannelName, MappingMode, ProjectDefinedChannelMapping, ProjectDefinedMapping, + ProjectDefinedMappingLocation, PurlDerivationMode, +}; +use rattler_conda_types::{Channel, ChannelConfig}; +use rattler_lock::UrlOrPath; + +/// Determine the [`PurlDerivationMode`] for a workspace from its +/// `conda-pypi-map` configuration. +pub(crate) fn build_pypi_name_derivation_mode( + manifest: &WorkspaceManifest, + channel_config: &ChannelConfig, +) -> miette::Result { + let map = match &manifest.workspace.conda_pypi_map { + None => return Ok(PurlDerivationMode::Prefix), + Some(CondaPypiMap::Disabled) => return Ok(PurlDerivationMode::Disabled), + Some(CondaPypiMap::Map(map)) => map, + }; + + // An empty map is a soft-deprecated alias for `conda-pypi-map = false`; + // the deprecation warning is emitted when the manifest is parsed. + if map.is_empty() { + return Ok(PurlDerivationMode::Disabled); + } + + let channel_to_entry_map = map + .iter() + .map(|(key, value)| { + let key = key.clone().into_channel(channel_config).into_diagnostic()?; + Ok((key, value)) + }) + .collect::>>()?; + + validate_mapped_channels_are_used(manifest, channel_config, channel_to_entry_map.keys())?; + + let mapping = channel_to_entry_map + .iter() + .map(|(channel, entry)| { + Ok(( + channel.canonical_name().trim_end_matches('/').into(), + convert_entry(entry, channel_config)?, + )) + }) + .collect::>>()?; + + Ok(PurlDerivationMode::ProjectDefined( + ProjectDefinedMapping::new(mapping).into(), + )) +} + +/// Every channel in `conda-pypi-map` must appear in the workspace or feature +/// channels; an entry for an unused channel is almost certainly a typo. +fn validate_mapped_channels_are_used<'a>( + manifest: &WorkspaceManifest, + channel_config: &ChannelConfig, + mapped_channels: impl Iterator, +) -> miette::Result<()> { + let project_channels: HashSet<_> = manifest + .workspace + .channels + .iter() + .map(|pc| pc.channel.clone().into_channel(channel_config)) + .try_collect() + .into_diagnostic()?; + + let feature_channels: HashSet<_> = manifest + .features + .values() + .flat_map(|feature| feature.channels.iter()) + .flatten() + .map(|pc| pc.channel.clone().into_channel(channel_config)) + .try_collect() + .into_diagnostic()?; + + let project_and_feature_channels: HashSet<_> = + project_channels.union(&feature_channels).collect(); + + for channel in mapped_channels { + if !project_and_feature_channels.contains(channel) { + let channels = project_and_feature_channels + .iter() + .map(|c| c.name.clone().unwrap_or_else(|| c.base_url.to_string())) + .sorted() + .collect::>() + .join(", "); + miette::bail!( + "conda-pypi-map is defined: the {} is missing from the channels array, which currently are: {}", + console::style( + channel + .name + .clone() + .unwrap_or_else(|| channel.base_url.to_string()) + ) + .bold(), + channels + ); + } + } + Ok(()) +} + +/// Convert a manifest entry to the per-channel mapping configuration used by +/// the purl derivation client. +fn convert_entry( + entry: &CondaPypiMapEntry, + channel_config: &ChannelConfig, +) -> miette::Result { + match entry { + CondaPypiMapEntry::Disabled => Ok(ProjectDefinedChannelMapping::disabled()), + CondaPypiMapEntry::Map(CondaPypiMapSpec { + location, + mapping, + mode, + }) => { + let mut sources = Vec::new(); + if let Some(location) = location { + sources.push(parse_mapping_location(location, channel_config)?); + } + // Inline entries come last so they override entries from the + // location. Keys are lowercased to match the normalized conda + // package names used for lookups. + if let Some(inline) = mapping { + sources.push(ProjectDefinedMappingLocation::InMemory( + inline + .iter() + .map(|(name, pypi_name)| (name.to_lowercase(), pypi_name.clone())) + .collect(), + )); + } + let mode = match mode { + CondaPypiMapMode::Extend => MappingMode::Extend, + CondaPypiMapMode::Replace => MappingMode::Replace, + }; + Ok(ProjectDefinedChannelMapping::new(sources, mode)) + } + } +} + +/// Classify a manifest location spec into a url or a path, resolving relative +/// paths against the workspace root. `file://` urls are normalized to paths. +fn parse_mapping_location( + spec: &MappingLocationSpec, + channel_config: &ChannelConfig, +) -> miette::Result { + let url_or_path = UrlOrPath::from_str(&spec.location) + .into_diagnostic() + .context(format!( + "Could not parse mapping location `{}`", + spec.location + ))?; + + match url_or_path { + UrlOrPath::Url(url) => { + if !matches!(url.scheme(), "http" | "https") { + miette::bail!( + "unsupported scheme `{}` in mapping location `{}`; only http(s) URLs and local paths are supported", + url.scheme(), + spec.location + ); + } + Ok(ProjectDefinedMappingLocation::Url { + url, + cache_ttl: spec.cache_ttl, + }) + } + UrlOrPath::Path(path) => { + if spec.cache_ttl.is_some() { + miette::bail!( + "`cache-ttl` is only supported for http(s) mapping locations, but `{}` is a local file", + spec.location + ); + } + let path = PathBuf::from(path.as_str()); + let abs_path = if path.is_relative() { + channel_config.root_dir.join(path) + } else { + path + }; + Ok(ProjectDefinedMappingLocation::Path(abs_path)) + } + } +} diff --git a/crates/pixi_core/src/workspace/mod.rs b/crates/pixi_core/src/workspace/mod.rs index 32d8687125..ebcf3cc95e 100644 --- a/crates/pixi_core/src/workspace/mod.rs +++ b/crates/pixi_core/src/workspace/mod.rs @@ -1,3 +1,4 @@ +mod conda_pypi_map; mod discovery; mod environment; pub mod errors; @@ -30,8 +31,7 @@ pub use discovery::{DiscoveryStart, WorkspaceLocator, WorkspaceLocatorError}; pub use environment::Environment; pub use has_project_ref::HasWorkspaceRef; use indexmap::Equivalent; -use itertools::Itertools; -use miette::{Context, IntoDiagnostic}; +use miette::IntoDiagnostic; use once_cell::sync::OnceCell; use pep508_rs::Requirement; use pixi_build_frontend::BackendOverride; @@ -40,10 +40,9 @@ use pixi_config::{Config, RunPostLinkScripts}; use pixi_consts::consts; use pixi_diff::LockFileDiff; use pixi_manifest::{ - AssociateProvenance, BuildVariantSource, CondaPypiMap, CondaPypiMapEntry, CondaPypiMapMode, - EnvironmentName, Environments, HasWorkspaceManifest, LoadManifestsError, ManifestProvenance, - Manifests, PackageManifest, PixiPlatform, PixiPlatformName, SpecType, WithProvenance, - WithWarnings, WorkspaceManifest, + AssociateProvenance, BuildVariantSource, EnvironmentName, Environments, HasWorkspaceManifest, + LoadManifestsError, ManifestProvenance, Manifests, PackageManifest, PixiPlatform, + PixiPlatformName, SpecType, WithProvenance, WithWarnings, WorkspaceManifest, }; use pixi_path::AbsPathBuf; use pixi_pypi_spec::{PixiPypiSpec, PypiPackageName}; @@ -53,12 +52,9 @@ use pixi_utils::{ reqwest::LazyReqwestClient, variants::{VariantConfig, VariantValue}, }; -use pypi_mapping::{ - ChannelName, MappingMode, ProjectDefinedChannelMapping, ProjectDefinedMapping, - ProjectDefinedMappingLocation, PurlDerivationMode, -}; +use pypi_mapping::PurlDerivationMode; use rattler_conda_types::{ - Channel, ChannelConfig, GenericVirtualPackage, MatchSpec, PackageName, Platform, Version, + ChannelConfig, GenericVirtualPackage, MatchSpec, PackageName, Platform, Version, }; use rattler_lock::LockFile; @@ -71,7 +67,6 @@ use rattler_virtual_packages::{ pub use registry::{WorkspaceRegistry, WorkspaceRegistryError}; pub use solve_group::SolveGroup; use tokio::sync::Semaphore; -use url::Url; pub use workspace_mut::WorkspaceMut; use xxhash_rust::xxh3::xxh3_64; @@ -910,170 +905,18 @@ impl Workspace { self.pixi_dir().join(consts::ACTIVATION_ENV_CACHE_DIR) } + /// Returns which PyPI purl derivation mode we should use. + /// It can use project-defined mappings in the format `conda_name: pypi_name`, + /// or the self-hosted prefix.dev mappings. /// Returns which PyPI purl derivation mode we should use. /// It can use project-defined mappings in the format `conda_name: pypi_name`, /// or the self-hosted prefix.dev mappings. pub fn pypi_name_derivation_mode(&self) -> miette::Result<&PurlDerivationMode> { - /// Classify a manifest location spec into a url or a path, resolving - /// relative paths against the workspace root. - fn parse_mapping_location( - spec: &pixi_manifest::MappingLocationSpec, - channel_config: &ChannelConfig, - ) -> miette::Result { - let mapping_location = spec.location.as_str(); - if mapping_location.starts_with("https://") || mapping_location.starts_with("http://") { - let url = Url::parse(mapping_location) - .into_diagnostic() - .context(format!("Could not convert {mapping_location} to URL"))?; - Ok(ProjectDefinedMappingLocation::Url { - url, - cache_ttl: spec.cache_ttl, - }) - } else { - if spec.cache_ttl.is_some() { - miette::bail!( - "`cache-ttl` is only supported for http(s) mapping locations, but `{mapping_location}` is a local file" - ); - } - if mapping_location.starts_with("file://") { - let url = Url::parse(mapping_location) - .into_diagnostic() - .context(format!("Could not convert {mapping_location} to URL"))?; - Ok(ProjectDefinedMappingLocation::Url { - url, - cache_ttl: None, - }) - } else { - let path = PathBuf::from(mapping_location); - let abs_path = if path.is_relative() { - channel_config.root_dir.join(path) - } else { - path - }; - Ok(ProjectDefinedMappingLocation::Path(abs_path)) - } - } - } - - /// Convert a manifest entry to the per-channel mapping configuration - /// used by the purl derivation client. - fn convert_entry( - entry: &CondaPypiMapEntry, - channel_config: &ChannelConfig, - ) -> miette::Result { - match entry { - CondaPypiMapEntry::Disabled => Ok(ProjectDefinedChannelMapping::disabled()), - CondaPypiMapEntry::Map(pixi_manifest::CondaPypiMapSpec { - location, - mapping, - mode, - }) => { - let mut sources = Vec::new(); - if let Some(location) = location { - sources.push(parse_mapping_location(location, channel_config)?); - } - // Inline entries come last so they override entries from - // the location. Keys are lowercased to match the - // normalized conda package names used for lookups. - if let Some(inline) = mapping { - sources.push(ProjectDefinedMappingLocation::InMemory( - inline - .iter() - .map(|(name, pypi_name)| (name.to_lowercase(), pypi_name.clone())) - .collect(), - )); - } - let mode = match mode { - CondaPypiMapMode::Extend => MappingMode::Extend, - CondaPypiMapMode::Replace => MappingMode::Replace, - }; - Ok(ProjectDefinedChannelMapping::new(sources, mode)) - } - } - } - - fn build_pypi_name_derivation_mode( - manifest: &WorkspaceManifest, - channel_config: &ChannelConfig, - ) -> miette::Result { - let map = match &manifest.workspace.conda_pypi_map { - None => return Ok(PurlDerivationMode::Prefix), - Some(CondaPypiMap::Disabled) => return Ok(PurlDerivationMode::Disabled), - Some(CondaPypiMap::Map(map)) => map, - }; - - // An empty map is a soft-deprecated alias for `conda-pypi-map = false`; - // the deprecation warning is emitted when the manifest is parsed. - if map.is_empty() { - return Ok(PurlDerivationMode::Disabled); - } - - let channel_to_entry_map = map - .iter() - .map(|(key, value)| { - let key = key.clone().into_channel(channel_config).into_diagnostic()?; - Ok((key, value)) - }) - .collect::>>()?; - - let project_channels: HashSet<_> = manifest - .workspace - .channels - .iter() - .map(|pc| pc.channel.clone().into_channel(channel_config)) - .try_collect() - .into_diagnostic()?; - - let feature_channels: HashSet<_> = manifest - .features - .values() - .flat_map(|feature| feature.channels.iter()) - .flatten() - .map(|pc| pc.channel.clone().into_channel(channel_config)) - .try_collect() - .into_diagnostic()?; - - let project_and_feature_channels: HashSet<_> = - project_channels.union(&feature_channels).collect(); - - for channel in channel_to_entry_map.keys() { - if !project_and_feature_channels.contains(channel) { - let channels = project_and_feature_channels - .iter() - .map(|c| c.name.clone().unwrap_or_else(|| c.base_url.to_string())) - .sorted() - .collect::>() - .join(", "); - miette::bail!( - "conda-pypi-map is defined: the {} is missing from the channels array, which currently are: {}", - console::style( - channel - .name - .clone() - .unwrap_or_else(|| channel.base_url.to_string()) - ) - .bold(), - channels - ); - } - } - - let mapping = channel_to_entry_map - .iter() - .map(|(channel, entry)| { - Ok(( - channel.canonical_name().trim_end_matches('/').into(), - convert_entry(entry, channel_config)?, - )) - }) - .collect::>>()?; - - Ok(PurlDerivationMode::ProjectDefined( - ProjectDefinedMapping::new(mapping).into(), - )) - } self.derivation_mode.get_or_try_init(|| { - build_pypi_name_derivation_mode(&self.workspace.value, &self.channel_config()) + conda_pypi_map::build_pypi_name_derivation_mode( + &self.workspace.value, + &self.channel_config(), + ) }) } @@ -1253,7 +1096,9 @@ mod tests { use itertools::Itertools; use pixi_config::{Config, DetachedEnvironments}; use pixi_manifest::{FeatureName, FeaturesExt, HasWorkspaceManifest}; - use rattler_conda_types::{Platform, Version}; + use pypi_mapping::{ProjectDefinedChannelMapping, ProjectDefinedMappingLocation}; + use rattler_conda_types::{Channel, Platform, Version}; + use url::Url; use xxhash_rust::xxh3::xxh3_64; use super::*; diff --git a/crates/pixi_manifest/src/toml/conda_pypi_map.rs b/crates/pixi_manifest/src/toml/conda_pypi_map.rs index 43a7ebbd4a..5274deb5a9 100644 --- a/crates/pixi_manifest/src/toml/conda_pypi_map.rs +++ b/crates/pixi_manifest/src/toml/conda_pypi_map.rs @@ -7,10 +7,10 @@ use std::collections::HashMap; -use pixi_toml::{TomlEnum, TomlHashMap}; +use pixi_toml::{TomlEnum, TomlHashMap, custom_error}; use rattler_conda_types::NamedChannelOrUrl; use toml_span::{ - DeserError, Error, ErrorKind, Value, + DeserError, Value, de_helpers::{TableHelper, expected}, value::ValueInner, }; @@ -23,15 +23,11 @@ impl<'de> toml_span::Deserialize<'de> for CondaPypiMap { fn deserialize(value: &mut Value<'de>) -> Result { match value.take() { ValueInner::Boolean(false) => Ok(CondaPypiMap::Disabled), - ValueInner::Boolean(true) => Err(Error { - kind: ErrorKind::Custom( - "`conda-pypi-map = true` is not supported; use `false` to disable the \ - mapping, or a table to configure it" - .into(), - ), - span: value.span, - line_info: None, - } + ValueInner::Boolean(true) => Err(custom_error( + "`conda-pypi-map = true` is not supported; use `false` to disable the \ + mapping, or a table to configure it", + value.span, + ) .into()), inner @ ValueInner::Table(_) => { let span = value.span; @@ -50,15 +46,11 @@ impl<'de> toml_span::Deserialize<'de> for CondaPypiMapEntry { match value.take() { ValueInner::String(s) => Ok(CondaPypiMapEntry::from_location(s.into_owned())), ValueInner::Boolean(false) => Ok(CondaPypiMapEntry::Disabled), - ValueInner::Boolean(true) => Err(Error { - kind: ErrorKind::Custom( - "`true` is not supported; use `false` to disable the mapping for this \ - channel, or a string or table to configure it" - .into(), - ), - span: value.span, - line_info: None, - } + ValueInner::Boolean(true) => Err(custom_error( + "`true` is not supported; use `false` to disable the mapping for this \ + channel, or a string or table to configure it", + value.span, + ) .into()), inner @ ValueInner::Table(_) => { let table_span = value.span; @@ -82,12 +74,11 @@ impl<'de> toml_span::Deserialize<'de> for CondaPypiMapEntry { spanned .value .parse::() - .map_err(|e| Error { - kind: ErrorKind::Custom( - format!("invalid `cache-ttl` duration: {e}").into(), - ), - span: spanned.span, - line_info: None, + .map_err(|e| { + custom_error( + format!("invalid `cache-ttl` duration: {e}"), + spanned.span, + ) })? .into(), ), @@ -97,13 +88,10 @@ impl<'de> toml_span::Deserialize<'de> for CondaPypiMapEntry { th.finalize(None)?; if location.is_none() && mapping.is_none() { - return Err(Error { - kind: ErrorKind::Custom( - "expected at least one of `location` or `mapping`".into(), - ), - span: table_span, - line_info: None, - } + return Err(custom_error( + "expected at least one of `location` or `mapping`", + table_span, + ) .into()); } @@ -115,13 +103,10 @@ impl<'de> toml_span::Deserialize<'de> for CondaPypiMapEntry { cache_ttl, }), (None, Some(_)) => { - return Err(Error { - kind: ErrorKind::Custom( - "`cache-ttl` requires a `location` that is a URL".into(), - ), - span: table_span, - line_info: None, - } + return Err(custom_error( + "`cache-ttl` requires a `location` that is a URL", + table_span, + ) .into()); } (None, None) => None, @@ -147,15 +132,11 @@ impl<'de> toml_span::Deserialize<'de> for TomlCondaPypiMapValue { match value.take() { ValueInner::String(s) => Ok(Self(Some(s.into_owned()))), ValueInner::Boolean(false) => Ok(Self(None)), - ValueInner::Boolean(true) => Err(Error { - kind: ErrorKind::Custom( - "`true` is not supported; use a string to map the package to a PyPI name, \ - or `false` to mark it as not a PyPI package" - .into(), - ), - span: value.span, - line_info: None, - } + ValueInner::Boolean(true) => Err(custom_error( + "`true` is not supported; use a string to map the package to a PyPI name, \ + or `false` to mark it as not a PyPI package", + value.span, + ) .into()), other => Err(expected("a string or `false`", other, value.span).into()), } diff --git a/crates/pixi_toml/src/diagnostic.rs b/crates/pixi_toml/src/diagnostic.rs index 675cb4a956..591e44f248 100644 --- a/crates/pixi_toml/src/diagnostic.rs +++ b/crates/pixi_toml/src/diagnostic.rs @@ -20,6 +20,21 @@ fn split_custom_error_help(message: &str) -> (&str, Option<&str>) { .map_or((message, None), |(message, help)| (message, Some(help))) } +/// Construct a custom [`toml_span::Error`] with the given message and span. +/// +/// Shorthand for the common pattern of returning a hand-written validation +/// error from a [`toml_span::Deserialize`] implementation. +pub fn custom_error( + message: impl Into>, + span: toml_span::Span, +) -> toml_span::Error { + toml_span::Error { + kind: toml_span::ErrorKind::Custom(message.into()), + span, + line_info: None, + } +} + /// A wrapper around [`toml_span::Error`] that implements the `miette::Diagnostic` trait. #[derive(Debug)] pub struct TomlDiagnostic(pub toml_span::Error); diff --git a/crates/pixi_toml/src/lib.rs b/crates/pixi_toml/src/lib.rs index aec3f499e7..3993d340ee 100644 --- a/crates/pixi_toml/src/lib.rs +++ b/crates/pixi_toml/src/lib.rs @@ -21,7 +21,7 @@ mod with; use std::str::FromStr; pub use btree_set::TomlBTreeSet; -pub use diagnostic::{TomlDiagnostic, custom_error_message_with_help}; +pub use diagnostic::{TomlDiagnostic, custom_error, custom_error_message_with_help}; pub use digest::TomlDigest; pub use from_str::TomlFromStr; pub use hash_map::TomlHashMap; diff --git a/crates/pypi_mapping/src/lib.rs b/crates/pypi_mapping/src/lib.rs index e087205dd3..cf88d06ff7 100644 --- a/crates/pypi_mapping/src/lib.rs +++ b/crates/pypi_mapping/src/lib.rs @@ -319,54 +319,75 @@ impl PurlDerivationClient { record: &RepoDataRecord, cache_metrics: &CacheMetrics, ) -> Result { - // Whether the conda-forge verbatim fallback is suppressed for this record. Only - // a `Replace` mapping is exclusive: packages not explicitly in the mapping must - // not get purls. - let mut suppress_verbatim_fallback = false; + /// What is consulted when the primary lookup does not apply to a record. + enum Fallback { + /// The prefix.dev chain, then the offline conda-forge verbatim + /// heuristic (assume the conda name is the PyPI name). + PrefixThenVerbatim, + /// Only the offline conda-forge verbatim heuristic. + Verbatim, + /// Nothing: a miss means the record gets no purls. Used for + /// `Replace` mappings, which are exclusive. + None, + } let project_defined_mode = project_defined_mappings .as_ref() .and_then(|mapping| mapping.mode_for_record(record)); - let purls = if matches!(derivation_mode, PurlDerivationMode::Disabled) { - DerivationOutcome::NotApplicable + // Consult the primary source for this record and determine which + // fallback applies when it has no answer. + let (mut outcome, fallback) = if matches!(derivation_mode, PurlDerivationMode::Disabled) { + (DerivationOutcome::NotApplicable, Fallback::Verbatim) } else if let (Some(resolver), Some(mode)) = (project_defined_mappings, project_defined_mode) { - match mode { + // A hit in the project-defined mapping (including an explicit + // "not a PyPI package" entry) is always final. + let project_outcome = match mode { MappingMode::Disabled => DerivationOutcome::NotApplicable, - MappingMode::Replace => { - suppress_verbatim_fallback = true; + MappingMode::Replace | MappingMode::Extend => { resolver .derive_project_defined_purls(record, cache_metrics) .await? } - MappingMode::Extend => { - // A hit in the project-defined mapping (including an explicit "not a - // PyPI package" entry) is final; only a miss falls through to the - // prefix.dev chain. - let outcome = resolver - .derive_project_defined_purls(record, cache_metrics) - .await?; - if outcome.is_not_applicable() { - self.derive_purls_from_prefix(record, cache_metrics).await? - } else { - outcome - } - } - } + }; + let fallback = match mode { + MappingMode::Disabled => Fallback::Verbatim, + MappingMode::Replace => Fallback::None, + MappingMode::Extend => Fallback::PrefixThenVerbatim, + }; + (project_outcome, fallback) } else { - self.derive_purls_from_prefix(record, cache_metrics).await? + ( + DerivationOutcome::NotApplicable, + Fallback::PrefixThenVerbatim, + ) }; - // As a last resort use the verbatim conda-forge purls. - if purls.is_not_applicable() && !suppress_verbatim_fallback { - return CondaForgeVerbatim - .derive_conda_forge_verbatim_purls(record, cache_metrics) - .await; + if outcome.is_not_applicable() { + outcome = match fallback { + Fallback::PrefixThenVerbatim => { + let prefix_outcome = + self.derive_purls_from_prefix(record, cache_metrics).await?; + if prefix_outcome.is_not_applicable() { + CondaForgeVerbatim + .derive_conda_forge_verbatim_purls(record, cache_metrics) + .await? + } else { + prefix_outcome + } + } + Fallback::Verbatim => { + CondaForgeVerbatim + .derive_conda_forge_verbatim_purls(record, cache_metrics) + .await? + } + Fallback::None => outcome, + }; } - Ok(purls) + Ok(outcome) } async fn derive_purls_from_prefix( From c4a1019706e2171406c1336c472db3e09bb8ab7d Mon Sep 17 00:00:00 2001 From: Tim de Jager Date: Wed, 10 Jun 2026 15:26:19 +0200 Subject: [PATCH 08/10] fix: apply final-review findings - TTL cache: treat a future mtime (clock skew) as age zero instead of making the cached copy invisible to the freshness check and the stale fallback; write cache files atomically via tempfile + persist; unit tests for the age computation. - pypi-conda-map: an invalid conda name in an override now falls through to the mapping service instead of silently dropping the dependency. - Split the offline help text: failures fetching a user-configured location now suggest checking the URL / adding cache-ttl instead of the firewall-framed prefix.dev advice; clearer HTTP status error. - Warn when a mapping location uses plain http://, since a tampered mapping influences dependency resolution. - Encode the manifest-mode to MappingMode conversion in a documented convert_mode function (a From impl is impossible: neither crate depends on the other, so the orphan rule forces it into pixi_core). - Error wording: cache-ttl duration errors show example values; cache-ttl-without-location message no longer implies location must be a URL; {} deprecation help reworded; stale Disabled doc hedge fixed; duplicated doc comment removed. - Docs: warning box now also covers the verbatim-fallback scope change for unmapped channels; cache-ttl docs state the no-cache hard-failure; inline-mapping example no longer reuses 'pytorch' as both channel and package name. - New tests: mixed-case inline keys, cache-ttl on a local path rejected, file:// table-form location works (pins UrlOrPath normalization), Skip entries with markers, vacuous purls assertion fixed, unit tests for parse_mapping_location/convert_entry; re-documented what the fresh-cache TTL test actually pins (cache layout + no network). --- Cargo.lock | 2 + .../integration_rust/conda_pypi_map_tests.rs | 135 +++++++++++++++++- crates/pixi_build_python/src/pypi_mapping.rs | 28 +++- .../pixi_core/src/workspace/conda_pypi_map.rs | 117 ++++++++++++++- crates/pixi_core/src/workspace/mod.rs | 3 - .../pixi_manifest/src/toml/conda_pypi_map.rs | 8 +- ...map__test__empty_map_parses_and_warns.snap | 2 +- ...nda_pypi_map__test__invalid_ttl_fails.snap | 2 +- ...map__test__ttl_without_location_fails.snap | 2 +- crates/pixi_manifest/src/toml/workspace.rs | 3 +- crates/pypi_mapping/Cargo.toml | 4 + crates/pypi_mapping/src/derivation_mode.rs | 5 +- .../src/resolvers/project_defined_mapping.rs | 104 ++++++++++++-- docs/concepts/conda_pypi.md | 2 +- docs/reference/pixi_manifest.md | 11 +- 15 files changed, 384 insertions(+), 44 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 481d791d15..b80bded9fc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7705,6 +7705,7 @@ dependencies = [ "astral-reqwest-retry", "async-once-cell", "dashmap", + "filetime", "fs-err", "futures", "http-cache-reqwest", @@ -7719,6 +7720,7 @@ dependencies = [ "reqwest", "serde", "serde_json", + "tempfile", "thiserror 2.0.18", "tokio", "tracing", diff --git a/crates/pixi/tests/integration_rust/conda_pypi_map_tests.rs b/crates/pixi/tests/integration_rust/conda_pypi_map_tests.rs index 6ff1093102..cba3f3f4e7 100644 --- a/crates/pixi/tests/integration_rust/conda_pypi_map_tests.rs +++ b/crates/pixi/tests/integration_rust/conda_pypi_map_tests.rs @@ -877,6 +877,124 @@ async fn test_channel_disabled_keeps_verbatim_fallback() { assert!(purl.qualifiers().is_empty()); } +/// Inline mapping keys are matched case-insensitively against the normalized +/// (lowercase) conda package names. +#[tokio::test] +async fn test_inline_mapping_keys_are_case_insensitive() { + setup_tracing(); + + let pixi = PixiControl::from_manifest( + r#" + [project] + name = "test-inline-case" + channels = ["conda-forge"] + platforms = ["linux-64"] + conda-pypi-map = { conda-forge = { mapping = { Pixi-Something-New = "mixed-case-win" } } } + "#, + ) + .unwrap(); + + let project = pixi.workspace().unwrap(); + let cache_dir = TempDir::new().unwrap(); + let mapping_client = offline_mapping_client(&project, cache_dir.path().to_path_buf()); + + let mut packages = vec![conda_forge_record("pixi-something-new")]; + mapping_client + .amend_purls( + project.pypi_name_derivation_mode().unwrap(), + &mut packages, + None, + ) + .await + .unwrap(); + + let package = packages.pop().unwrap(); + let purl = package + .package_record + .purls + .as_ref() + .and_then(BTreeSet::first) + .expect("a mixed-case inline key should match the lowercase record name"); + assert_eq!(purl.name(), "mixed-case-win"); +} + +/// `cache-ttl` combined with a local path location is rejected when the +/// derivation mode is built. +#[tokio::test] +async fn test_cache_ttl_on_local_path_is_rejected() { + setup_tracing(); + + let pixi = PixiControl::from_manifest( + r#" + [project] + name = "test-ttl-local-path" + channels = ["conda-forge"] + platforms = ["linux-64"] + conda-pypi-map = { conda-forge = { location = "mapping.json", cache-ttl = "24h" } } + "#, + ) + .unwrap(); + + let project = pixi.workspace().unwrap(); + let err = project + .pypi_name_derivation_mode() + .expect_err("cache-ttl on a local path should be rejected"); + assert!( + err.to_string().contains("cache-ttl"), + "error should mention cache-ttl, got: {err}" + ); +} + +/// A `file://` url in the table-form `location` is normalized to a local +/// path and works like one. +#[tokio::test] +async fn test_file_url_in_table_location() { + setup_tracing(); + + let tmp_dir = tempfile::tempdir().unwrap(); + let mapping_file = tmp_dir.path().join("mapping.json"); + fs_err::write( + &mapping_file, + r#"{ "pixi-something-new": "from-file-url" }"#, + ) + .unwrap(); + let mapping_url = Url::from_file_path(&mapping_file).unwrap(); + + let pixi = PixiControl::from_manifest(&format!( + r#" + [project] + name = "test-file-url-table" + channels = ["conda-forge"] + platforms = ["linux-64"] + conda-pypi-map = {{ conda-forge = {{ location = "{mapping_url}", mode = "extend" }} }} + "#, + )) + .unwrap(); + + let project = pixi.workspace().unwrap(); + let cache_dir = TempDir::new().unwrap(); + let mapping_client = offline_mapping_client(&project, cache_dir.path().to_path_buf()); + + let mut packages = vec![conda_forge_record("pixi-something-new")]; + mapping_client + .amend_purls( + project.pypi_name_derivation_mode().unwrap(), + &mut packages, + None, + ) + .await + .unwrap(); + + let package = packages.pop().unwrap(); + let purl = package + .package_record + .purls + .as_ref() + .and_then(BTreeSet::first) + .expect("a file:// location should resolve like a local path"); + assert_eq!(purl.name(), "from-file-url"); +} + /// When an entry has both a `location` and inline `mapping` entries, the /// inline entries override the ones from the location. #[tokio::test] @@ -1061,6 +1179,13 @@ fn manifest_with_ttl_mapping(url: &str, ttl: &str) -> String { /// A cached mapping younger than `cache-ttl` is used without any network /// access. +/// +/// Note: with an offline client this test cannot distinguish the fresh-cache +/// path from the stale-fallback path (both serve the cached file). What it +/// pins is the on-disk cache layout (`project-defined/.json`) +/// and that a cached mapping is served without touching the network at all. +/// The fresh/expired age boundary itself is unit-tested in the +/// `pypi_mapping` crate (`read_ttl_cache`). #[tokio::test] async fn test_cache_ttl_fresh_cache_skips_network() { setup_tracing(); @@ -1411,12 +1536,10 @@ async fn test_custom_mapping_ignores_backwards_compatibility() { _ => panic!("All packages should be binary"), }; - if let Some(purls) = purls { - assert!( - purls.is_empty(), - "boltons should not have purls when not specified in custom conda-pypi-map" - ); - } + assert!( + purls.as_ref().is_none_or(|purls| purls.is_empty()), + "boltons should not have purls when not specified in custom conda-pypi-map" + ); } #[tokio::test] diff --git a/crates/pixi_build_python/src/pypi_mapping.rs b/crates/pixi_build_python/src/pypi_mapping.rs index 146125d775..59f942396d 100644 --- a/crates/pixi_build_python/src/pypi_mapping.rs +++ b/crates/pixi_build_python/src/pypi_mapping.rs @@ -576,10 +576,14 @@ fn apply_user_map( }); } Err(err) => { + // An invalid override must not silently drop the + // dependency: warn and fall through to the mapping + // service. tracing::warn!( "ignoring `pypi-conda-map` entry for '{}': invalid conda package name '{conda_name}': {err}", req.name ); + remaining.push(req.clone()); } }, } @@ -868,7 +872,7 @@ mod tests { } #[test] - fn test_apply_user_map_invalid_conda_name_is_skipped() { + fn test_apply_user_map_invalid_conda_name_falls_through() { let user_map = IndexMap::from([( "torch".to_string(), PypiCondaMapEntry::CondaName("not a valid name!".to_string()), @@ -876,8 +880,26 @@ mod tests { let requirements = vec![requirement("torch")]; let (mapped, remaining) = apply_user_map(&requirements, Some(&user_map), Platform::Linux64); - // The override is consumed (warned about) rather than handed to the - // mapping service. + // The invalid override is warned about and the dependency falls + // through to the mapping service instead of being silently dropped. + assert!(mapped.is_empty()); + assert_eq!(remaining.len(), 1); + assert_eq!(remaining[0].name.as_ref(), "torch"); + } + + #[test] + fn test_apply_user_map_skip_entry_respects_markers() { + let user_map = IndexMap::from([("torch".to_string(), PypiCondaMapEntry::Skip)]); + + // A marker-gated requirement that does not apply to the platform is + // dropped before the Skip entry matters: neither mapped nor remaining. + let requirements = vec![requirement("torch; sys_platform == 'win32'")]; + let (mapped, remaining) = apply_user_map(&requirements, Some(&user_map), Platform::Linux64); + assert!(mapped.is_empty()); + assert!(remaining.is_empty()); + + // On a matching platform the Skip entry drops it silently. + let (mapped, remaining) = apply_user_map(&requirements, Some(&user_map), Platform::Win64); assert!(mapped.is_empty()); assert!(remaining.is_empty()); } diff --git a/crates/pixi_core/src/workspace/conda_pypi_map.rs b/crates/pixi_core/src/workspace/conda_pypi_map.rs index 5b334cd5a8..776b02e6ff 100644 --- a/crates/pixi_core/src/workspace/conda_pypi_map.rs +++ b/crates/pixi_core/src/workspace/conda_pypi_map.rs @@ -142,15 +142,30 @@ fn convert_entry( .collect(), )); } - let mode = match mode { - CondaPypiMapMode::Extend => MappingMode::Extend, - CondaPypiMapMode::Replace => MappingMode::Replace, - }; - Ok(ProjectDefinedChannelMapping::new(sources, mode)) + Ok(ProjectDefinedChannelMapping::new( + sources, + convert_mode(*mode), + )) } } } +/// Convert the manifest-level mode to the derivation-level [`MappingMode`]. +/// +/// The two enums are deliberately asymmetric: `MappingMode::Disabled` has no +/// manifest-level mode string because "disabled" is spelled ` = +/// false` in TOML (see [`CondaPypiMapEntry::Disabled`]), not `mode = +/// "disabled"`. This function and the `Disabled` arm in [`convert_entry`] are +/// the single place where the two representations meet. (A `From` impl cannot +/// encode this: neither `pixi_manifest` nor `pypi_mapping` depends on the +/// other, so the orphan rule forces the conversion to live here.) +fn convert_mode(mode: CondaPypiMapMode) -> MappingMode { + match mode { + CondaPypiMapMode::Extend => MappingMode::Extend, + CondaPypiMapMode::Replace => MappingMode::Replace, + } +} + /// Classify a manifest location spec into a url or a path, resolving relative /// paths against the workspace root. `file://` urls are normalized to paths. fn parse_mapping_location( @@ -173,6 +188,16 @@ fn parse_mapping_location( spec.location ); } + // A plaintext mapping URL can be tampered with on the network, + // and a tampered mapping changes which conda packages are + // considered to satisfy PyPI dependencies. + if url.scheme() == "http" { + tracing::warn!( + "the conda-pypi mapping location `{}` uses plain `http://`; the mapping can \ + be tampered with in transit. Prefer `https://` or a local file.", + spec.location + ); + } Ok(ProjectDefinedMappingLocation::Url { url, cache_ttl: spec.cache_ttl, @@ -195,3 +220,85 @@ fn parse_mapping_location( } } } + +#[cfg(test)] +mod test { + use std::time::Duration; + + use super::*; + + fn channel_config() -> ChannelConfig { + ChannelConfig::default_with_root_dir(PathBuf::from("/workspace")) + } + + fn location(location: &str, cache_ttl: Option) -> MappingLocationSpec { + MappingLocationSpec { + location: location.to_string(), + cache_ttl, + } + } + + #[test] + fn test_parse_mapping_location_http_url_with_ttl() { + let ttl = Some(Duration::from_secs(60)); + let parsed = parse_mapping_location( + &location("https://example.com/m.json", ttl), + &channel_config(), + ) + .unwrap(); + assert_eq!( + parsed, + ProjectDefinedMappingLocation::Url { + url: "https://example.com/m.json".parse().unwrap(), + cache_ttl: ttl, + } + ); + } + + #[test] + fn test_parse_mapping_location_relative_path() { + let parsed = + parse_mapping_location(&location("sub/m.json", None), &channel_config()).unwrap(); + assert_eq!( + parsed, + ProjectDefinedMappingLocation::Path(PathBuf::from("/workspace/sub/m.json")) + ); + } + + #[test] + fn test_parse_mapping_location_file_url_becomes_path() { + let parsed = + parse_mapping_location(&location("file:///abs/m.json", None), &channel_config()) + .unwrap(); + assert_eq!( + parsed, + ProjectDefinedMappingLocation::Path(PathBuf::from("/abs/m.json")) + ); + } + + #[test] + fn test_parse_mapping_location_rejects_ttl_on_path() { + let err = parse_mapping_location( + &location("sub/m.json", Some(Duration::from_secs(60))), + &channel_config(), + ) + .unwrap_err(); + assert!(err.to_string().contains("cache-ttl")); + } + + #[test] + fn test_parse_mapping_location_rejects_unsupported_scheme() { + let err = parse_mapping_location( + &location("ftp://example.com/m.json", None), + &channel_config(), + ) + .unwrap_err(); + assert!(err.to_string().contains("unsupported scheme")); + } + + #[test] + fn test_convert_entry_disabled() { + let converted = convert_entry(&CondaPypiMapEntry::Disabled, &channel_config()).unwrap(); + assert_eq!(converted, ProjectDefinedChannelMapping::disabled()); + } +} diff --git a/crates/pixi_core/src/workspace/mod.rs b/crates/pixi_core/src/workspace/mod.rs index ebcf3cc95e..1be5f605d6 100644 --- a/crates/pixi_core/src/workspace/mod.rs +++ b/crates/pixi_core/src/workspace/mod.rs @@ -905,9 +905,6 @@ impl Workspace { self.pixi_dir().join(consts::ACTIVATION_ENV_CACHE_DIR) } - /// Returns which PyPI purl derivation mode we should use. - /// It can use project-defined mappings in the format `conda_name: pypi_name`, - /// or the self-hosted prefix.dev mappings. /// Returns which PyPI purl derivation mode we should use. /// It can use project-defined mappings in the format `conda_name: pypi_name`, /// or the self-hosted prefix.dev mappings. diff --git a/crates/pixi_manifest/src/toml/conda_pypi_map.rs b/crates/pixi_manifest/src/toml/conda_pypi_map.rs index 5274deb5a9..f1c10bc756 100644 --- a/crates/pixi_manifest/src/toml/conda_pypi_map.rs +++ b/crates/pixi_manifest/src/toml/conda_pypi_map.rs @@ -76,7 +76,10 @@ impl<'de> toml_span::Deserialize<'de> for CondaPypiMapEntry { .parse::() .map_err(|e| { custom_error( - format!("invalid `cache-ttl` duration: {e}"), + format!( + "invalid `cache-ttl` duration: {e}; expected a duration \ + like \"24h\", \"7d\" or \"1h 30m\"" + ), spanned.span, ) })? @@ -104,7 +107,8 @@ impl<'de> toml_span::Deserialize<'de> for CondaPypiMapEntry { }), (None, Some(_)) => { return Err(custom_error( - "`cache-ttl` requires a `location` that is a URL", + "`cache-ttl` requires a `location`; it is only effective for \ + http(s) URLs", table_span, ) .into()); diff --git a/crates/pixi_manifest/src/toml/snapshots/pixi_manifest__toml__conda_pypi_map__test__empty_map_parses_and_warns.snap b/crates/pixi_manifest/src/toml/snapshots/pixi_manifest__toml__conda_pypi_map__test__empty_map_parses_and_warns.snap index 38fd9cd533..5ca8346e8a 100644 --- a/crates/pixi_manifest/src/toml/snapshots/pixi_manifest__toml__conda_pypi_map__test__empty_map_parses_and_warns.snap +++ b/crates/pixi_manifest/src/toml/snapshots/pixi_manifest__toml__conda_pypi_map__test__empty_map_parses_and_warns.snap @@ -3,4 +3,4 @@ source: crates/pixi_manifest/src/toml/conda_pypi_map.rs expression: "expect_parse_warnings(r#\"\n [workspace]\n channels = []\n platforms = []\n conda-pypi-map = {}\n \"#)" --- ⚠ `conda-pypi-map = {}` is deprecated - help: To disable the conda-pypi mapping, write `conda-pypi-map = false` instead. + help: To disable all mapping lookups, write `conda-pypi-map = false` instead. diff --git a/crates/pixi_manifest/src/toml/snapshots/pixi_manifest__toml__conda_pypi_map__test__invalid_ttl_fails.snap b/crates/pixi_manifest/src/toml/snapshots/pixi_manifest__toml__conda_pypi_map__test__invalid_ttl_fails.snap index aeff496873..eed7f9610f 100644 --- a/crates/pixi_manifest/src/toml/snapshots/pixi_manifest__toml__conda_pypi_map__test__invalid_ttl_fails.snap +++ b/crates/pixi_manifest/src/toml/snapshots/pixi_manifest__toml__conda_pypi_map__test__invalid_ttl_fails.snap @@ -2,7 +2,7 @@ source: crates/pixi_manifest/src/toml/conda_pypi_map.rs expression: "expect_parse_failure(r#\"\n [workspace]\n channels = []\n platforms = []\n conda-pypi-map = { conda-forge = { location = \"https://example.com/m.json\", cache-ttl = \"bogus\" } }\n \"#)" --- - × invalid `cache-ttl` duration: expected number at 0 + × invalid `cache-ttl` duration: expected number at 0; expected a duration like "24h", "7d" or "1h 30m" ╭─[pixi.toml:5:102] 4 │ platforms = [] 5 │ conda-pypi-map = { conda-forge = { location = "https://example.com/m.json", cache-ttl = "bogus" } } diff --git a/crates/pixi_manifest/src/toml/snapshots/pixi_manifest__toml__conda_pypi_map__test__ttl_without_location_fails.snap b/crates/pixi_manifest/src/toml/snapshots/pixi_manifest__toml__conda_pypi_map__test__ttl_without_location_fails.snap index 1876202544..77216ae0d9 100644 --- a/crates/pixi_manifest/src/toml/snapshots/pixi_manifest__toml__conda_pypi_map__test__ttl_without_location_fails.snap +++ b/crates/pixi_manifest/src/toml/snapshots/pixi_manifest__toml__conda_pypi_map__test__ttl_without_location_fails.snap @@ -2,7 +2,7 @@ source: crates/pixi_manifest/src/toml/conda_pypi_map.rs expression: "expect_parse_failure(r#\"\n [workspace]\n channels = []\n platforms = []\n conda-pypi-map = { conda-forge = { mapping = { a = \"b\" }, cache-ttl = \"24h\" } }\n \"#)" --- - × `cache-ttl` requires a `location` that is a URL + × `cache-ttl` requires a `location`; it is only effective for http(s) URLs ╭─[pixi.toml:5:46] 4 │ platforms = [] 5 │ conda-pypi-map = { conda-forge = { mapping = { a = "b" }, cache-ttl = "24h" } } diff --git a/crates/pixi_manifest/src/toml/workspace.rs b/crates/pixi_manifest/src/toml/workspace.rs index a46e745b03..0a04aad707 100644 --- a/crates/pixi_manifest/src/toml/workspace.rs +++ b/crates/pixi_manifest/src/toml/workspace.rs @@ -187,8 +187,7 @@ impl TomlWorkspace { warnings.push( GenericError::new("`conda-pypi-map = {}` is deprecated") .with_help( - "To disable the conda-pypi mapping, write `conda-pypi-map = false` \ - instead.", + "To disable all mapping lookups, write `conda-pypi-map = false` instead.", ) .into(), ); diff --git a/crates/pypi_mapping/Cargo.toml b/crates/pypi_mapping/Cargo.toml index 1cfdc28ab0..db95be2347 100644 --- a/crates/pypi_mapping/Cargo.toml +++ b/crates/pypi_mapping/Cargo.toml @@ -28,7 +28,11 @@ reqwest-middleware = { workspace = true } reqwest-retry = { workspace = true } serde = { workspace = true, features = ["derive"] } serde_json = { workspace = true } +tempfile = { workspace = true } thiserror = { workspace = true } tokio = { workspace = true } tracing = { workspace = true } url = { workspace = true } + +[dev-dependencies] +filetime = { workspace = true } diff --git a/crates/pypi_mapping/src/derivation_mode.rs b/crates/pypi_mapping/src/derivation_mode.rs index 3f159d86c2..8b6ce76467 100644 --- a/crates/pypi_mapping/src/derivation_mode.rs +++ b/crates/pypi_mapping/src/derivation_mode.rs @@ -93,8 +93,9 @@ pub enum PurlDerivationMode { Prefix, /// Disable project-defined and prefix.dev mappings. /// - /// Note: the current resolver still allows the conda-forge verbatim fallback - /// in this mode. + /// The offline conda-forge verbatim fallback (assume the conda name is + /// the PyPI name) still applies in this mode; disabling only turns off + /// the lookups. Disabled, } diff --git a/crates/pypi_mapping/src/resolvers/project_defined_mapping.rs b/crates/pypi_mapping/src/resolvers/project_defined_mapping.rs index f58ddf7238..1e55d99714 100644 --- a/crates/pypi_mapping/src/resolvers/project_defined_mapping.rs +++ b/crates/pypi_mapping/src/resolvers/project_defined_mapping.rs @@ -93,6 +93,13 @@ impl ProjectDefinedMapping { } } +/// Help text for failures to fetch a *user-configured* mapping location. +/// Unlike [`crate::MAPPING_OFFLINE_HELP`] this must not suggest "point at +/// your own mapping" — the user already did. +const LOCATION_FETCH_HELP: &str = "Check that the `location` URL in your `conda-pypi-map` entry is correct and reachable. \ + To tolerate temporary outages, add a `cache-ttl` so a previously fetched copy can be \ + reused, or use a local file instead."; + async fn fetch_mapping_from_url( client: &LazyClient, url: &Url, @@ -104,15 +111,17 @@ async fn fetch_mapping_from_url( .await .into_diagnostic() .wrap_err(miette::diagnostic!( - help = crate::MAPPING_OFFLINE_HELP, - "failed to download pypi mapping from {} location", + help = LOCATION_FETCH_HELP, + "failed to download conda-pypi mapping from {}", url.as_str() ))?; if !response.status().is_success() { return Err(miette::miette!( - "Could not request mapping located at {:?}", - url.as_str() + help = LOCATION_FETCH_HELP, + "fetching the conda-pypi mapping from {} returned status {}", + url.as_str(), + response.status() )); } @@ -182,10 +191,15 @@ fn ttl_cache_path(cache_dir: &Path, url: &Url) -> PathBuf { /// copy or it cannot be parsed. fn read_ttl_cache(cache_path: &Path) -> Option<(CompressedMapping, Duration)> { let metadata = fs_err::metadata(cache_path).ok()?; - let age = metadata - .modified() - .ok() - .and_then(|modified| SystemTime::now().duration_since(modified).ok())?; + // A modification time in the future (clock skew, NTP corrections) is + // treated as age zero; returning `None` here would make a perfectly good + // cached copy invisible to both the freshness check and the stale + // fallback. + let age = metadata.modified().ok().map(|modified| { + SystemTime::now() + .duration_since(modified) + .unwrap_or(Duration::ZERO) + })?; let content = fs_err::read_to_string(cache_path).ok()?; let mapping = serde_json::from_str(&content).ok()?; Some((mapping, age)) @@ -194,11 +208,20 @@ fn read_ttl_cache(cache_path: &Path) -> Option<(CompressedMapping, Duration)> { /// Write a mapping to the TTL cache. Failures are ignored; the cache is an /// optimization. fn write_ttl_cache(cache_path: &Path, mapping: &CompressedMapping) { - if let Some(parent) = cache_path.parent() { - let _ = fs_err::create_dir_all(parent); - } - if let Ok(content) = serde_json::to_string(mapping) { - let _ = fs_err::write(cache_path, content); + let Some(parent) = cache_path.parent() else { + return; + }; + let _ = fs_err::create_dir_all(parent); + let Ok(content) = serde_json::to_string(mapping) else { + return; + }; + // Write via a temporary file and rename so a concurrent reader never + // observes a partially written cache file. + let Ok(temp_file) = tempfile::NamedTempFile::new_in(parent) else { + return; + }; + if fs_err::write(temp_file.path(), content).is_ok() { + let _ = temp_file.persist(cache_path); } } @@ -282,3 +305,58 @@ impl ProjectDefinedResolver { } } } + +#[cfg(test)] +mod test { + use std::time::{Duration, SystemTime}; + + use super::{read_ttl_cache, write_ttl_cache}; + + fn write_cache_with_mtime(dir: &std::path::Path, age: i64) -> std::path::PathBuf { + let path = dir.join("mapping.json"); + write_ttl_cache( + &path, + &[("foo".to_string(), Some("bar".to_string()))] + .into_iter() + .collect(), + ); + let mtime = filetime::FileTime::from_system_time(if age >= 0 { + SystemTime::now() - Duration::from_secs(age as u64) + } else { + SystemTime::now() + Duration::from_secs((-age) as u64) + }); + filetime::set_file_mtime(&path, mtime).unwrap(); + path + } + + #[test] + fn test_read_ttl_cache_reports_age() { + let dir = tempfile::tempdir().unwrap(); + let path = write_cache_with_mtime(dir.path(), 7200); + let (mapping, age) = read_ttl_cache(&path).expect("cache should be readable"); + assert_eq!(mapping["foo"], Some("bar".to_string())); + // Allow some slack for slow filesystems. + assert!(age >= Duration::from_secs(7100) && age < Duration::from_secs(7300)); + } + + #[test] + fn test_read_ttl_cache_future_mtime_is_age_zero() { + // A cache file with a modification time in the future (clock skew) + // must still be readable, with age zero, so that both the freshness + // check and the stale fallback can use it. + let dir = tempfile::tempdir().unwrap(); + let path = write_cache_with_mtime(dir.path(), -3600); + let (_, age) = read_ttl_cache(&path).expect("future-dated cache should be readable"); + assert_eq!(age, Duration::ZERO); + } + + #[test] + fn test_read_ttl_cache_missing_or_invalid() { + let dir = tempfile::tempdir().unwrap(); + assert!(read_ttl_cache(&dir.path().join("missing.json")).is_none()); + + let corrupt = dir.path().join("corrupt.json"); + fs_err::write(&corrupt, "not json").unwrap(); + assert!(read_ttl_cache(&corrupt).is_none()); + } +} diff --git a/docs/concepts/conda_pypi.md b/docs/concepts/conda_pypi.md index 876c6c099d..f49a0f863e 100644 --- a/docs/concepts/conda_pypi.md +++ b/docs/concepts/conda_pypi.md @@ -130,7 +130,7 @@ If that host is unreachable in your environment, you have several options: - `conda-pypi-map = false` disables all mapping lookups. Conda-forge packages are still assumed to be the PyPI package of the same name, which requires no network access. - ` = false` disables lookups for a single channel. - `mode = "replace"` with your own mapping file avoids network lookups for that channel entirely. -- `cache-ttl = "24h"` on a URL location caches the fetched mapping on disk and re-fetches it at most once per TTL; if the re-fetch fails, the cached copy is used. +- `cache-ttl = "24h"` on a URL location caches the fetched mapping on disk and re-fetches it at most once per TTL; if the re-fetch fails, the cached copy is used. Note that the cache must be populated once (one successful fetch) before this protects you offline. For example, you can pin the full conda-forge name mapping that `parselmouth` publishes (the same data the default mapping is built from) and refresh it at most once a day: diff --git a/docs/reference/pixi_manifest.md b/docs/reference/pixi_manifest.md index 64c3597376..c1fa424b4d 100644 --- a/docs/reference/pixi_manifest.md +++ b/docs/reference/pixi_manifest.md @@ -201,12 +201,13 @@ Each entry maps a channel name or URL to either a mapping location (URL or path) ```toml [workspace.conda-pypi-map] -# Additive overlay from a file: entries win, everything else uses the default mapping. +# Additive overlay from a file: your entries take priority, anything not +# listed falls back to the default mapping. robostack = "local/robostack_mapping.json" # Inline entries, no file needed. `false` means "not a PyPI package". -pytorch = { mode = "extend", mapping = { pytorch = "torch", not-on-pypi = false } } +conda-forge = { mode = "extend", mapping = { pytorch = "torch", not-on-pypi = false } } # Exclusive mapping: ONLY packages in this file get mapped, no default mapping. -conda-forge = { location = "https://example.com/mapping.json", mode = "replace" } +my-mirror = { location = "https://example.com/mapping.json", mode = "replace" } # Re-fetch a remote mapping at most once a day; a cached copy is used otherwise. my-company = { location = "https://internal.example.com/map.json", cache-ttl = "24h" } # Disable mapping lookups for this channel entirely. @@ -228,7 +229,7 @@ The table form accepts: - `location`: URL or path of a mapping `json` file. Relative paths are resolved against the workspace root. - `mapping`: inline `conda_name = "pypi_name"` entries. A value of `false` marks the package as not available on PyPI. Inline entries override entries from `location`. - `mode`: `"extend"` (default) or `"replace"`. With `extend`, your entries are consulted first and anything not listed falls back to the default [prefix.dev mapping](https://conda-mapping.prefix.dev/). With `replace`, only packages listed in your mapping are considered PyPI packages; no network lookups happen for that channel. -- `cache-ttl`: a duration like `"24h"` or `"7d"`. The mapping fetched from a `location` URL is cached on disk and only re-fetched once it is older than this. If a re-fetch fails (e.g. offline), the stale cached copy is used with a warning. Only valid for `http(s)` locations. +- `cache-ttl`: a duration like `"24h"` or `"7d"`. The mapping fetched from a `location` URL is cached on disk and only re-fetched once it is older than this. If a re-fetch fails (e.g. offline), the stale cached copy is used with a warning; if no cached copy exists yet, the failure is an error. Only valid for `http(s)` locations. To disable the mapping, either per channel or entirely: @@ -245,6 +246,8 @@ Even with the mapping disabled, conda-forge packages are still assumed to be the Bare location strings (`conda-forge = "mapping.json"`) used to be *exclusive*: only packages in your file were mapped. They are now *additive* (`mode = "extend"`). To restore the old behavior, use the table form with `mode = "replace"`. The previous idiom of disabling the mapping with an empty map (`conda-pypi-map = {}`) is deprecated; use `conda-pypi-map = false` instead. + Additionally, configuring a mapping for one channel no longer suppresses the conda-forge name heuristic for channels that are *not* listed in `conda-pypi-map`; unlisted channels now behave exactly as if no mapping were configured. + To turn lookups off for a specific channel, add ` = false`. ### `channel-priority` (optional) From afebe070af0849e6354090a90694d7249263fb01 Mon Sep 17 00:00:00 2001 From: Tim de Jager Date: Wed, 10 Jun 2026 16:01:54 +0200 Subject: [PATCH 09/10] fix: reject duplicate conda-pypi-map entries for the same channel --- .../integration_rust/conda_pypi_map_tests.rs | 31 +++++++++++ crates/pixi_build_python/src/pypi_mapping.rs | 55 ++++++++++++++----- .../pixi_core/src/workspace/conda_pypi_map.rs | 27 ++++++--- 3 files changed, 91 insertions(+), 22 deletions(-) diff --git a/crates/pixi/tests/integration_rust/conda_pypi_map_tests.rs b/crates/pixi/tests/integration_rust/conda_pypi_map_tests.rs index cba3f3f4e7..c3f607186a 100644 --- a/crates/pixi/tests/integration_rust/conda_pypi_map_tests.rs +++ b/crates/pixi/tests/integration_rust/conda_pypi_map_tests.rs @@ -918,6 +918,37 @@ async fn test_inline_mapping_keys_are_case_insensitive() { assert_eq!(purl.name(), "mixed-case-win"); } +/// The same channel spelled in two forms (by name and by URL) must be +/// rejected: the forms collapse to one channel after resolution and keeping +/// a nondeterministic winner would silently pick one of the two entries. +#[tokio::test] +async fn test_duplicate_channel_forms_are_rejected() { + setup_tracing(); + + let pixi = PixiControl::from_manifest( + r#" + [project] + name = "test-duplicate-channel" + channels = ["conda-forge"] + platforms = ["linux-64"] + + [project.conda-pypi-map] + conda-forge = false + "https://conda.anaconda.org/conda-forge" = { mapping = { a = "b" } } + "#, + ) + .unwrap(); + + let project = pixi.workspace().unwrap(); + let err = project + .pypi_name_derivation_mode() + .expect_err("duplicate channel forms should be rejected"); + assert!( + err.to_string().contains("more than once"), + "error should mention the duplicate, got: {err}" + ); +} + /// `cache-ttl` combined with a local path location is rejected when the /// derivation mode is built. #[tokio::test] diff --git a/crates/pixi_build_python/src/pypi_mapping.rs b/crates/pixi_build_python/src/pypi_mapping.rs index 59f942396d..5a0f6289d0 100644 --- a/crates/pixi_build_python/src/pypi_mapping.rs +++ b/crates/pixi_build_python/src/pypi_mapping.rs @@ -527,21 +527,24 @@ fn apply_user_map( Vec>, ) { // Normalize the user-map keys so that e.g. `My_Pkg` matches `my-pkg`. - let normalized: IndexMap = user_map - .map(|map| { - map.iter() - .filter_map(|(name, entry)| match pep508_rs::PackageName::from_str(name) { - Ok(name) => Some((name, entry)), - Err(err) => { - tracing::warn!( - "ignoring invalid PyPI package name '{name}' in `pypi-conda-map`: {err}" - ); - None - } - }) - .collect() - }) - .unwrap_or_default(); + let mut normalized: IndexMap = IndexMap::new(); + for (name, entry) in user_map.into_iter().flatten() { + match pep508_rs::PackageName::from_str(name) { + Ok(normalized_name) => { + if normalized.insert(normalized_name, entry).is_some() { + tracing::warn!( + "multiple `pypi-conda-map` entries normalize to the same package name \ + as '{name}'; the last one wins" + ); + } + } + Err(err) => { + tracing::warn!( + "ignoring invalid PyPI package name '{name}' in `pypi-conda-map`: {err}" + ); + } + } + } let mut user_mapped = Vec::new(); let mut remaining = Vec::new(); @@ -904,6 +907,28 @@ mod tests { assert!(remaining.is_empty()); } + #[test] + fn test_apply_user_map_colliding_keys_last_wins() { + // `My_Pkg` and `my-pkg` normalize to the same name; the last entry + // wins deterministically (and a warning is logged). + let user_map = IndexMap::from([ + ( + "My_Pkg".to_string(), + PypiCondaMapEntry::CondaName("first".to_string()), + ), + ( + "my-pkg".to_string(), + PypiCondaMapEntry::CondaName("second".to_string()), + ), + ]); + + let requirements = vec![requirement("my-pkg")]; + let (mapped, remaining) = apply_user_map(&requirements, Some(&user_map), Platform::Linux64); + assert!(remaining.is_empty()); + assert_eq!(mapped.len(), 1); + assert_eq!(mapped[0].name.as_normalized(), "second"); + } + #[test] fn test_apply_user_map_converts_pep440_operators() { let user_map = IndexMap::from([( diff --git a/crates/pixi_core/src/workspace/conda_pypi_map.rs b/crates/pixi_core/src/workspace/conda_pypi_map.rs index 776b02e6ff..380356be88 100644 --- a/crates/pixi_core/src/workspace/conda_pypi_map.rs +++ b/crates/pixi_core/src/workspace/conda_pypi_map.rs @@ -38,13 +38,26 @@ pub(crate) fn build_pypi_name_derivation_mode( return Ok(PurlDerivationMode::Disabled); } - let channel_to_entry_map = map - .iter() - .map(|(key, value)| { - let key = key.clone().into_channel(channel_config).into_diagnostic()?; - Ok((key, value)) - }) - .collect::>>()?; + // The manifest map can spell the same channel in different forms (by + // name and by URL) that only collapse once resolved to a `Channel`. + // Collecting blindly would keep a nondeterministic winner, so reject + // duplicates instead. + let mut channel_to_entry_map: HashMap = + HashMap::with_capacity(map.len()); + for (key, value) in map { + let channel = key.clone().into_channel(channel_config).into_diagnostic()?; + let channel_name = channel + .name + .clone() + .unwrap_or_else(|| channel.base_url.to_string()); + if channel_to_entry_map.insert(channel, value).is_some() { + miette::bail!( + "the channel {} is configured more than once in `conda-pypi-map` \ + (e.g. both by name and by URL); keep a single entry per channel", + console::style(channel_name).bold(), + ); + } + } validate_mapped_channels_are_used(manifest, channel_config, channel_to_entry_map.keys())?; From 50cdea7fee992010fcfb45b984d3f4a60db3367e Mon Sep 17 00:00:00 2001 From: Tim de Jager Date: Wed, 10 Jun 2026 17:15:30 +0200 Subject: [PATCH 10/10] fix: lint findings - typos: reword 'mis-mapped' in the conda/PyPI concepts page. - basedpyright: no implicit string concatenation in the new schema/model.py field descriptions (schema output unchanged). --- docs/concepts/conda_pypi.md | 2 +- schema/model.py | 12 ++++-------- 2 files changed, 5 insertions(+), 9 deletions(-) diff --git a/docs/concepts/conda_pypi.md b/docs/concepts/conda_pypi.md index f49a0f863e..87ac16318e 100644 --- a/docs/concepts/conda_pypi.md +++ b/docs/concepts/conda_pypi.md @@ -108,7 +108,7 @@ To override or change the mapping of conda packages to PyPI packages, you can us ### Overriding the name mapping `conda-pypi-map` layers your entries *on top of* the default mapping: for each package pixi first consults your entries, and only falls back to the default mapping (and finally the "conda-forge name equals PyPI name" heuristic) when your mapping does not mention the package. -This means fixing a single mis-mapped package is a one-liner: +This means fixing a single incorrectly mapped package is a one-liner: ```toml title="pixi.toml" [workspace.conda-pypi-map] diff --git a/schema/model.py b/schema/model.py index f1be645715..bb55c30a52 100644 --- a/schema/model.py +++ b/schema/model.py @@ -208,18 +208,15 @@ class CondaPypiMapTable(StrictBaseModel): ) mapping: dict[NonEmptyStr, NonEmptyStr | Literal[False]] | None = Field( None, - description="Inline `conda_name: pypi_name` entries; `false` marks a package as not " - "available on PyPI. Inline entries override entries from `location`.", + description="Inline `conda_name: pypi_name` entries; `false` marks a package as not available on PyPI. Inline entries override entries from `location`.", ) mode: Literal["extend", "replace"] | None = Field( None, - description="How the mapping interacts with the default mapping: `extend` (default) " - "overlays it, `replace` makes it exclusive", + description="How the mapping interacts with the default mapping: `extend` (default) overlays it, `replace` makes it exclusive", ) cache_ttl: NonEmptyStr | None = Field( None, - description="How long a mapping fetched from a URL may be reused before it is " - 're-fetched (e.g. `"24h"`, `"7d"`). Only valid for http(s) locations.', + description='How long a mapping fetched from a URL may be reused before it is re-fetched (e.g. `"24h"`, `"7d"`). Only valid for http(s) locations.', ) @@ -325,8 +322,7 @@ class Workspace(StrictBaseModel): ) conda_pypi_map: CondaPypiMap | None = Field( None, - description="The `conda` to PyPI mapping configuration; `false` disables the mapping " - "entirely", + description="The `conda` to PyPI mapping configuration; `false` disables the mapping entirely", ) pypi_options: PyPIOptions | None = Field( None, description="Options related to PyPI indexes for this project"