diff --git a/crates/pixi_cli/src/global/install.rs b/crates/pixi_cli/src/global/install.rs index a77ca9a5a2..2f7ccba16f 100644 --- a/crates/pixi_cli/src/global/install.rs +++ b/crates/pixi_cli/src/global/install.rs @@ -203,6 +203,16 @@ async fn setup_environment( project.manifest.set_platform(env_name, platform)?; } + // Record any `CONDA_OVERRIDE_*` set during install as virtual packages on + // the environment's platform, so later solves (`pixi global update`/`sync`) + // reuse the same values without the variable having to be set again. + let env_overrides = pixi_core::workspace::virtual_packages_from_env_overrides(); + if !env_overrides.is_empty() { + project + .manifest + .add_platform_virtual_packages(env_name, env_overrides)?; + } + let converted_with_inclusions = args .with .iter() diff --git a/crates/pixi_core/src/workspace/mod.rs b/crates/pixi_core/src/workspace/mod.rs index ab08564323..f49e569888 100644 --- a/crates/pixi_core/src/workspace/mod.rs +++ b/crates/pixi_core/src/workspace/mod.rs @@ -306,6 +306,20 @@ fn apply_environment_variable_overrides(packages: &mut Vec Vec { + let mut packages = Vec::new(); + apply_environment_variable_overrides(&mut packages); + packages +} + /// Apply `CONDA_OVERRIDE_GLIBC` (rattler's only libc slot) to `packages`. The /// glibc env var governs glibc alone: unset leaves libc packages untouched, an /// empty value removes `__glibc`, and a concrete version pins diff --git a/crates/pixi_global/src/list.rs b/crates/pixi_global/src/list.rs index 3829a9cf2d..6127b23304 100644 --- a/crates/pixi_global/src/list.rs +++ b/crates/pixi_global/src/list.rs @@ -107,7 +107,7 @@ pub async fn list_global_environments_json( name: env_name.as_str().to_string(), dependencies, exposed, - platform: env.platform.map(|p| p.to_string()), + platform: env.platform.as_ref().map(|p| p.subdir().to_string()), }); } @@ -200,12 +200,12 @@ fn print_meta_info(environment: &ParsedEnvironment) -> miette::Result<()> { } // Print platform - if let Some(platform) = environment.platform { + if let Some(platform) = &environment.platform { writeln!( std::io::stdout(), "{} {}", console::style("Platform:").bold().cyan(), - platform + platform.subdir() ) .inspect_err(|e| { if e.kind() == std::io::ErrorKind::BrokenPipe { diff --git a/crates/pixi_global/src/project/manifest.rs b/crates/pixi_global/src/project/manifest.rs index 46a5dd3e90..cc70433986 100644 --- a/crates/pixi_global/src/project/manifest.rs +++ b/crates/pixi_global/src/project/manifest.rs @@ -9,10 +9,13 @@ use indexmap::IndexSet; use miette::IntoDiagnostic; use pixi_config::Config; use pixi_consts::consts; -use pixi_manifest::{PrioritizedChannel, toml::TomlDocument}; +use pixi_manifest::{ + PixiPlatform, PlatformEdit, PrioritizedChannel, + toml::{TomlDocument, pixi_platform_to_toml_value}, +}; use pixi_toml::TomlIndexMap; use pixi_utils::{executable_from_path, strip_executable_extension}; -use rattler_conda_types::{NamedChannelOrUrl, PackageName, Platform}; +use rattler_conda_types::{GenericVirtualPackage, NamedChannelOrUrl, PackageName, Platform}; use toml_edit::{DocumentMut, Item}; use toml_span::{DeserError, Value}; @@ -205,7 +208,12 @@ impl Manifest { Ok(name.clone()) } - /// Sets the platform of a specific environment in the manifest + /// Sets the platform of a specific environment in the manifest. + /// + /// This sets the bare subdir, discarding any virtual packages previously + /// recorded for the environment -- an explicit `--platform` selects the + /// platform afresh. Use [`Self::add_platform_virtual_packages`] to record + /// virtual packages on top. pub fn set_platform( &mut self, env_name: &EnvironmentName, @@ -216,6 +224,16 @@ impl Manifest { miette::bail!("Environment {} doesn't exist", env_name.fancy_display()); } + let pixi_platform = PixiPlatform::from_subdir(platform); + + // Update self.document + self.document + .get_or_insert_nested_table(&["envs", env_name.as_str()])? + .insert( + "platform", + Item::Value(pixi_platform_to_toml_value(&pixi_platform)), + ); + // Update self.parsed self.parsed .envs @@ -223,21 +241,71 @@ impl Manifest { .ok_or_else(|| { miette::miette!("Can't find environment {} yet", env_name.fancy_display()) })? + .platform = Some(pixi_platform); + + tracing::debug!( + "Set platform {} for environment {} in toml document", + platform, + env_name + ); + Ok(()) + } + + /// Records virtual packages on an environment's platform, e.g. a + /// `CONDA_OVERRIDE_*` set during `pixi global install`. The recorded values + /// are then used whenever the environment is solved, including by + /// `pixi global update`/`sync`, without the variable having to be set again. + /// + /// A virtual package of the same name replaces an earlier recording. The + /// subdir is preserved, defaulting to the current platform when the + /// environment had no platform recorded yet. + pub fn add_platform_virtual_packages( + &mut self, + env_name: &EnvironmentName, + virtual_packages: Vec, + ) -> miette::Result<()> { + if virtual_packages.is_empty() { + return Ok(()); + } + + let mut platform = self + .parsed + .envs + .get(env_name) + .ok_or_else(|| { + miette::miette!("Environment {} doesn't exist", env_name.fancy_display()) + })? .platform - .replace(platform); + .clone() + .unwrap_or_else(|| PixiPlatform::from_subdir(Platform::current())); + + platform + .apply_edit(PlatformEdit { + set_subdir: None, + clear_virtual_packages: false, + insert_or_update_virtual_packages: virtual_packages, + remove_virtual_packages: Vec::new(), + }) + .into_diagnostic()?; // Update self.document self.document .get_or_insert_nested_table(&["envs", env_name.as_str()])? .insert( "platform", - Item::Value(toml_edit::Value::from(platform.to_string())), + Item::Value(pixi_platform_to_toml_value(&platform)), ); + // Update self.parsed + self.parsed + .envs + .get_mut(env_name) + .expect("environment existence checked above") + .platform = Some(platform); + tracing::debug!( - "Set platform {} for environment {} in toml document", - platform, - env_name + "Recorded virtual packages on platform for environment {} in toml document", + env_name.fancy_display() ); Ok(()) } @@ -1088,8 +1156,59 @@ mod tests { .get(&env_name) .unwrap() .platform + .as_ref() + .unwrap(); + assert_eq!(actual_platform.subdir(), platform); + } + + #[test] + fn test_add_platform_virtual_packages() { + let mut manifest = Manifest::default(); + let env_name = EnvironmentName::from_str("cuda-tool").unwrap(); + manifest.add_environment(&env_name, None).unwrap(); + + let cuda = GenericVirtualPackage { + name: PackageName::from_str("__cuda").unwrap(), + version: "12.0".parse().unwrap(), + build_string: String::new(), + }; + manifest + .add_platform_virtual_packages(&env_name, vec![cuda]) + .unwrap(); + + let declares_cuda = |platform: &PixiPlatform| { + platform + .declared_virtual_packages() + .iter() + .any(|vp| vp.name.as_normalized() == "__cuda" && vp.version.to_string() == "12.0") + }; + + // The override is recorded on the environment's platform, on the + // current subdir since none was set. + let platform = manifest + .parsed + .envs + .get(&env_name) + .unwrap() + .platform + .as_ref() + .unwrap(); + assert_eq!(platform.subdir(), Platform::current()); + assert!(declares_cuda(platform)); + + // And it survives a round-trip through the document. + let reparsed = + Manifest::from_str(Path::new("pixi-global.toml"), manifest.document.to_string()) + .unwrap(); + let reparsed_platform = reparsed + .parsed + .envs + .get(&env_name) + .unwrap() + .platform + .as_ref() .unwrap(); - assert_eq!(actual_platform, platform); + assert!(declares_cuda(reparsed_platform)); } #[test] diff --git a/crates/pixi_global/src/project/mod.rs b/crates/pixi_global/src/project/mod.rs index 879320b4a1..67e2035d64 100644 --- a/crates/pixi_global/src/project/mod.rs +++ b/crates/pixi_global/src/project/mod.rs @@ -29,7 +29,7 @@ use pixi_command_dispatcher::{ use pixi_config::{Config, RunPostLinkScripts, default_channel_config, pixi_home}; use pixi_consts::consts::{self}; use pixi_core::repodata::Repodata; -use pixi_manifest::PrioritizedChannel; +use pixi_manifest::{PixiPlatform, PrioritizedChannel, is_subdir_default}; use pixi_path::AbsPathBuf; use pixi_reporters::TopLevelProgress; use pixi_spec::{BinarySpec, PathBinarySpec}; @@ -46,10 +46,7 @@ use rattler_conda_types::{ }; use rattler_networking::LazyClient; use rattler_repodata_gateway::Gateway; -// Removed unused rattler_solve imports -use rattler_virtual_packages::{ - DetectVirtualPackageError, VirtualPackage, VirtualPackageOverrides, -}; +use rattler_virtual_packages::DetectVirtualPackageError; use tokio::sync::Semaphore; use toml_edit::DocumentMut; @@ -561,25 +558,37 @@ impl Project { self.config.global_channel_config() } - /// Check if the platform matches the current platform (OS) - /// We only need to detect virtual packages if the platform is the current - /// one. Otherwise, we use an empty list - pub(crate) fn virtual_packages_for( - platform: &Platform, + /// The platform recorded for an environment in the manifest (subdir plus + /// any declared virtual packages, e.g. a recorded `__cuda` override), or + /// the current platform when none was recorded. + fn solve_platform(environment: &ParsedEnvironment) -> PixiPlatform { + environment + .platform + .clone() + .unwrap_or_else(|| PixiPlatform::from_subdir(Platform::current())) + } + + /// The virtual packages to solve an environment against: the machine's + /// auto-detected virtual packages, with the platform's recorded overrides + /// (its declared virtual packages that differ from the subdir defaults) + /// layered on top. This keeps the old behaviour of detecting the machine + /// while honouring recorded `CONDA_OVERRIDE_*` values, instead of freezing + /// the detected packages to the subdir defaults. + fn solve_virtual_packages( + platform: &PixiPlatform, ) -> Result, DetectVirtualPackageError> { - if platform - .only_platform() - .map(|p| p == Platform::current().only_platform().unwrap_or("")) - .unwrap_or(false) - { - Ok(VirtualPackage::detect(&VirtualPackageOverrides::default())? - .iter() - .cloned() - .map(GenericVirtualPackage::from) - .collect()) - } else { - Ok(vec![]) - } + let overrides: Vec = platform + .declared_virtual_packages() + .iter() + .filter(|vp| !is_subdir_default(vp, platform.subdir())) + .cloned() + .collect(); + Ok( + PixiPlatform::from_required_virtual_packages(platform.subdir(), overrides) + .virtual_packages()? + .into_generic_virtual_packages() + .collect(), + ) } pub async fn install_environment( @@ -608,7 +617,10 @@ impl Project { .collect::, _>>() .into_diagnostic()?; - let platform = environment.platform.unwrap_or_else(Platform::current); + let env_platform = Self::solve_platform(environment); + let platform = env_platform.subdir(); + let solve_virtual_packages = + Self::solve_virtual_packages(&env_platform).into_diagnostic()?; // Convert dependency specs to binary specs for CommandDispatcher let mut pixi_specs = DependencyMap::default(); @@ -626,10 +638,7 @@ impl Project { .map(|channel| channel.base_url.clone()) .collect::>(); - let build_environment = BuildEnvironment::simple( - platform, - Self::virtual_packages_for(&platform).into_diagnostic()?, - ); + let build_environment = BuildEnvironment::simple(platform, solve_virtual_packages); // Create solve spec (compute-engine keys path). let solve_spec = SolvePixiEnvironmentSpec { dependencies: pixi_specs, @@ -993,7 +1002,7 @@ impl Project { &prefix_records, &specs, &source_package_names, - environment.platform, + environment.platform.as_ref().map(PixiPlatform::subdir), ) .await?; if !specs_in_sync { @@ -1290,7 +1299,11 @@ impl Project { rattler_menuinst::install_menuitems_for_record( prefix.root(), &record, - environment.platform.unwrap_or(Platform::current()), + environment + .platform + .as_ref() + .map(PixiPlatform::subdir) + .unwrap_or_else(Platform::current), MenuMode::User, ) .into_diagnostic()?; @@ -1880,4 +1893,51 @@ mod tests { ); assert_eq!(package, "python".parse().unwrap()); } + + #[test] + fn test_solve_virtual_packages_honors_override_without_freezing() { + let subdir = Platform::current(); + + // A bare subdir platform contributes no overrides; solving falls back to + // pure machine detection (the old behaviour). + let bare = PixiPlatform::from_subdir(subdir); + let detected = Project::solve_virtual_packages(&bare).unwrap(); + assert!( + !detected + .iter() + .any(|vp| vp.name.as_normalized() == "__cuda"), + "a bare platform must not invent a __cuda the machine lacks" + ); + + // Recording a `__cuda` override (as `pixi global install` would) makes + // it appear, while the rest stays detected rather than frozen. + let mut rich = PixiPlatform::from_subdir(subdir); + rich.apply_edit(pixi_manifest::PlatformEdit { + set_subdir: None, + clear_virtual_packages: false, + insert_or_update_virtual_packages: vec![GenericVirtualPackage { + name: "__cuda".parse().unwrap(), + version: "12.0".parse().unwrap(), + build_string: String::new(), + }], + remove_virtual_packages: Vec::new(), + }) + .unwrap(); + let with_cuda = Project::solve_virtual_packages(&rich).unwrap(); + assert!( + with_cuda + .iter() + .any(|vp| vp.name.as_normalized() == "__cuda" && vp.version.to_string() == "12.0"), + "the recorded __cuda override must be honored" + ); + // Everything the bare platform detected is still present -- the override + // adds to detection rather than replacing it with subdir defaults. + for detected_vp in &detected { + assert!( + with_cuda.iter().any(|vp| vp.name == detected_vp.name), + "detected virtual package {} was dropped when recording an override", + detected_vp.name.as_normalized() + ); + } + } } diff --git a/crates/pixi_global/src/project/parsed_manifest.rs b/crates/pixi_global/src/project/parsed_manifest.rs index 40fe65768f..89a6f4eeb7 100644 --- a/crates/pixi_global/src/project/parsed_manifest.rs +++ b/crates/pixi_global/src/project/parsed_manifest.rs @@ -6,10 +6,12 @@ use indexmap::{IndexMap, IndexSet}; use itertools::{Either, Itertools}; use miette::{Context, Diagnostic, IntoDiagnostic, LabeledSpan, NamedSource, Report}; use pixi_consts::consts; -use pixi_manifest::{PrioritizedChannel, toml::TomlPlatform, utils::package_map::UniquePackageMap}; +use pixi_manifest::{ + PixiPlatform, PrioritizedChannel, toml::TomlPixiPlatform, utils::package_map::UniquePackageMap, +}; use pixi_spec::PixiSpec; use pixi_toml::{TomlFromStr, TomlIndexMap, TomlIndexSet, TomlWith}; -use rattler_conda_types::{NamedChannelOrUrl, PackageName, Platform}; +use rattler_conda_types::{NamedChannelOrUrl, PackageName}; use serde::{Serialize, Serializer, ser::SerializeMap}; use serde_derive::Deserialize; use thiserror::Error; @@ -258,7 +260,7 @@ where } = data; let parsed_environment = envs.entry(env_name).or_default(); parsed_environment.channels.extend(channels); - parsed_environment.platform = platform; + parsed_environment.platform = platform.map(PixiPlatform::from_subdir); parsed_environment .dependencies .specs @@ -290,12 +292,31 @@ where map.end() } +/// Serialize a rich platform the same way the workspace manifest does: a bare +/// subdir string when it carries no extra virtual packages, an inline table +/// otherwise. +fn serialize_platform(platform: &Option, serializer: S) -> Result +where + S: Serializer, +{ + match platform { + Some(platform) => TomlPixiPlatform(platform.clone()).serialize(serializer), + None => serializer.serialize_none(), + } +} + #[derive(Serialize, Debug, Clone, Default)] #[serde(deny_unknown_fields, rename_all = "kebab-case")] pub struct ParsedEnvironment { pub channels: IndexSet, - /// Platform used by the environment. - pub platform: Option, + /// Platform used by the environment, optionally extended with the virtual + /// packages it guarantees (e.g. a recorded `__cuda` override). + #[serde( + default, + skip_serializing_if = "Option::is_none", + serialize_with = "serialize_platform" + )] + pub platform: Option, pub dependencies: UniquePackageMap, #[serde(default, serialize_with = "serialize_expose_mappings")] pub exposed: IndexSet, @@ -310,7 +331,9 @@ impl<'de> toml_span::Deserialize<'de> for ParsedEnvironment { .optional::>("channels") .map(TomlIndexSet::into_inner) .unwrap_or_default(); - let platform = th.optional::("platform").map(Platform::from); + let platform = th + .optional::("platform") + .map(TomlPixiPlatform::into_inner); let dependencies = th.optional("dependencies").unwrap_or_default(); let exposed = th .optional::("exposed") @@ -414,9 +437,43 @@ impl AsRef for ExposedName { #[cfg(test)] mod tests { use insta::assert_snapshot; + use rattler_conda_types::Platform; use super::ParsedManifest; + #[test] + fn test_rich_platform_deserialization() { + let contents = r#" + [envs.cuda-tool] + channels = ["conda-forge"] + platform = { platform = "linux-64", cuda = "12.0" } + [envs.cuda-tool.dependencies] + cuda-tool = "*" + "#; + let manifest = ParsedManifest::from_toml_str(contents).unwrap(); + let env = manifest.envs.values().next().unwrap(); + let platform = env.platform.as_ref().unwrap(); + assert_eq!(platform.subdir(), Platform::Linux64); + assert!( + platform.declared_virtual_packages().iter().any(|vp| { + vp.name.as_normalized() == "__cuda" && vp.version.to_string() == "12.0" + }), + "expected the recorded __cuda override to be declared" + ); + } + + #[test] + fn test_bare_platform_deserialization() { + let contents = r#" + [envs.tool] + channels = ["conda-forge"] + platform = "osx-64" + "#; + let manifest = ParsedManifest::from_toml_str(contents).unwrap(); + let platform = manifest.envs.values().next().unwrap().platform.as_ref(); + assert_eq!(platform.unwrap().subdir(), Platform::Osx64); + } + #[test] fn test_invalid_key() { let examples = [ diff --git a/crates/pixi_manifest/src/lib.rs b/crates/pixi_manifest/src/lib.rs index c5298283f8..bb849f1c64 100644 --- a/crates/pixi_manifest/src/lib.rs +++ b/crates/pixi_manifest/src/lib.rs @@ -51,6 +51,7 @@ use miette::Diagnostic; pub use package::Package; pub use platform::{ PixiPlatform, PixiPlatformError, PixiPlatformName, PixiPlatformNameError, PlatformEdit, + is_subdir_default, }; pub use preview::{KnownPreviewFeature, Preview}; pub use s3::S3Options; diff --git a/crates/pixi_manifest/src/toml/mod.rs b/crates/pixi_manifest/src/toml/mod.rs index da4c745a91..da2cf4e598 100644 --- a/crates/pixi_manifest/src/toml/mod.rs +++ b/crates/pixi_manifest/src/toml/mod.rs @@ -29,7 +29,10 @@ pub use manifest::ExternalWorkspaceProperties; pub use manifest::TomlManifest; use miette::LabeledSpan; pub use package::{PackageDefaults, PackageError, TomlPackage, WorkspacePackageProperties}; -pub use platform::{InlineVirtualPackage, TomlPlatform, inline_virtual_package_specs}; +pub use platform::{ + InlineVirtualPackage, TomlPixiPlatform, TomlPlatform, inline_virtual_package_specs, + pixi_platform_to_toml_value, +}; pub use preview::TomlPreview; pub use pyproject::PyProjectToml; pub use target::TomlTarget; diff --git a/crates/pixi_manifest/src/toml/platform.rs b/crates/pixi_manifest/src/toml/platform.rs index 9e7a0003ee..2f53b791b8 100644 --- a/crates/pixi_manifest/src/toml/platform.rs +++ b/crates/pixi_manifest/src/toml/platform.rs @@ -729,7 +729,7 @@ fn classify_virtual_packages( /// bare-string vs inline-table shape as the serde `Serialize` impl above. /// This lets the document-editor rewrite the `platforms` array without /// going through serde. -pub(crate) fn pixi_platform_to_toml_value(platform: &PixiPlatform) -> toml_edit::Value { +pub fn pixi_platform_to_toml_value(platform: &PixiPlatform) -> toml_edit::Value { let name = platform.name().as_str(); let subdir_str = platform.subdir().to_string(); let declared = platform.declared_virtual_packages(); diff --git a/docs/global_tools/manifest.md b/docs/global_tools/manifest.md index 2ab2751d6b..c1ed582470 100644 --- a/docs/global_tools/manifest.md +++ b/docs/global_tools/manifest.md @@ -125,6 +125,41 @@ pixi global remove --environment my-env package-a package-b ``` +## Platform + +Each environment is solved for a single platform, defaulting to your current one. +Set it explicitly with `--platform` or by editing the `platform` key: + +```toml +[envs.vim] +channels = ["conda-forge"] +platform = "osx-64" +dependencies = { vim = "*" } +``` + +Some packages are constrained by [virtual packages](https://docs.conda.io/projects/conda/en/latest/user-guide/tasks/manage-virtual.html) like `__cuda`. +These are detected on your machine when an environment is solved, and can be overridden with the `CONDA_OVERRIDE_*` environment variables, for example `CONDA_OVERRIDE_CUDA=12.0`. + +Overrides that are set while running `pixi global install` are recorded on the environment's platform: + +```shell +CONDA_OVERRIDE_CUDA=12.0 pixi global install cuda-tool +``` + +results in: + +```toml +[envs.cuda-tool] +channels = ["conda-forge"] +dependencies = { cuda-tool = "*" } +platform = { platform = "linux-64", cuda = "12.0" } +exposed = { cuda-tool = "cuda-tool" } +``` + +This way commands that solve the environment again later, like `pixi global update` or `pixi global sync`, keep using the same values without the variable having to be set again. +The table can also be edited by hand and accepts the same fields as the workspace [platforms](../workspace/multi_platform_configuration.md): `cuda`, `linux`, `macos` (alias `osx`), `glibc`, `windows` and `archspec`. +To change a recorded value, install again with a different override; to drop it, set the variable to an empty string (e.g. `CONDA_OVERRIDE_CUDA=`). + ## Exposed executables One can instruct `pixi global install`, under which name it will expose executables: