diff --git a/.github/.cspell/project-dictionary.txt b/.github/.cspell/project-dictionary.txt index 24f5b767..1be720c6 100644 --- a/.github/.cspell/project-dictionary.txt +++ b/.github/.cspell/project-dictionary.txt @@ -47,6 +47,7 @@ devkitm dlfcn dlsym ecall +Espressif espup exynos fild @@ -140,6 +141,7 @@ picolibc prctl prefetcher PRIMASK +PSRAM pstq quadword rcpc diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b08630f9..1512a895 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1013,6 +1013,8 @@ jobs: # if: matrix.rust == 'stable' - run: tools/no-std.sh +esp xtensa-esp32s2-none-elf if: matrix.rust == 'stable' + - run: tools/no-std.sh +esp xtensa-esp32s3-none-elf + if: matrix.rust == 'stable' miri: needs: tidy diff --git a/CHANGELOG.md b/CHANGELOG.md index 17773c8d..6b1d1491 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,8 @@ Note: In this file, do not use the hard wrap in the middle of a sentence for com ## [Unreleased] +- Xtensa: route atomic read-modify-write operations on ESP32 / ESP32-S3 PSRAM addresses through a critical section. `Atomic*::IS_ALWAYS_LOCK_FREE` becomes `false` on these CPUs. + ## [1.13.1] - 2026-01-31 - Update to stabilized [PowerPC64](https://github.com/rust-lang/rust/pull/147996) inline assembly. ([92b02f8a](https://github.com/taiki-e/portable-atomic/commit/92b02f8a279327a1780cbe127d9effb2baae9b2f)) diff --git a/build.rs b/build.rs index a278a6f5..c788f646 100644 --- a/build.rs +++ b/build.rs @@ -56,13 +56,16 @@ fn main() { // Custom cfgs set by build script. Not public API. // grep -F 'cargo:rustc-cfg=' build.rs | grep -Ev '^ *//' | sed -E 's/^.*cargo:rustc-cfg=//; s/(=\\)?".*$//' | LC_ALL=C sort -u | tr '\n' ',' | sed -E 's/,$/\n/' println!( - "cargo:rustc-check-cfg=cfg(atomic_maybe_uninit_no_asm,portable_atomic_atomic_intrinsics,portable_atomic_disable_fiq,portable_atomic_force_amo,portable_atomic_ll_sc_rmw,portable_atomic_no_asm,portable_atomic_no_asm_maybe_uninit,portable_atomic_no_asm_syscall,portable_atomic_no_atomic_64,portable_atomic_no_atomic_cas,portable_atomic_no_atomic_load_store,portable_atomic_no_atomic_min_max,portable_atomic_no_cfg_target_has_atomic,portable_atomic_no_cmpxchg16b_intrinsic,portable_atomic_no_cmpxchg16b_target_feature,portable_atomic_no_const_mut_refs,portable_atomic_no_const_raw_ptr_deref,portable_atomic_no_const_transmute,portable_atomic_no_core_unwind_safe,portable_atomic_no_diagnostic_namespace,portable_atomic_no_strict_provenance,portable_atomic_no_strict_provenance_atomic_ptr,portable_atomic_no_stronger_failure_ordering,portable_atomic_no_track_caller,portable_atomic_no_unsafe_op_in_unsafe_fn,portable_atomic_pre_llvm_15,portable_atomic_pre_llvm_16,portable_atomic_pre_llvm_18,portable_atomic_pre_llvm_20,portable_atomic_s_mode,portable_atomic_sanitize_thread,portable_atomic_target_feature,portable_atomic_unsafe_assume_privileged,portable_atomic_unsafe_assume_single_core,portable_atomic_unstable_asm,portable_atomic_unstable_asm_experimental_arch,portable_atomic_unstable_cfg_target_has_atomic,portable_atomic_unstable_isa_attribute)" + "cargo:rustc-check-cfg=cfg(atomic_maybe_uninit_no_asm,portable_atomic_atomic_intrinsics,portable_atomic_disable_fiq,portable_atomic_force_amo,portable_atomic_ll_sc_rmw,portable_atomic_no_asm,portable_atomic_no_asm_maybe_uninit,portable_atomic_no_asm_syscall,portable_atomic_no_atomic_64,portable_atomic_no_atomic_cas,portable_atomic_no_atomic_load_store,portable_atomic_no_atomic_min_max,portable_atomic_no_cfg_target_has_atomic,portable_atomic_no_cmpxchg16b_intrinsic,portable_atomic_no_cmpxchg16b_target_feature,portable_atomic_no_const_mut_refs,portable_atomic_no_const_raw_ptr_deref,portable_atomic_no_const_transmute,portable_atomic_no_core_unwind_safe,portable_atomic_no_diagnostic_namespace,portable_atomic_no_strict_provenance,portable_atomic_no_strict_provenance_atomic_ptr,portable_atomic_no_stronger_failure_ordering,portable_atomic_no_track_caller,portable_atomic_no_unsafe_op_in_unsafe_fn,portable_atomic_pre_llvm_15,portable_atomic_pre_llvm_16,portable_atomic_pre_llvm_18,portable_atomic_pre_llvm_20,portable_atomic_s_mode,portable_atomic_sanitize_thread,portable_atomic_target_cpu,portable_atomic_target_feature,portable_atomic_unsafe_assume_privileged,portable_atomic_unsafe_assume_single_core,portable_atomic_unstable_asm,portable_atomic_unstable_asm_experimental_arch,portable_atomic_unstable_cfg_target_has_atomic,portable_atomic_unstable_isa_attribute)" ); // TODO: handle multi-line target_feature_fallback // grep -F 'target_feature_fallback("' build.rs | grep -Ev '^ *//' | sed -E 's/^.*target_feature_fallback\(//; s/",.*$/"/' | LC_ALL=C sort -u | tr '\n' ',' | sed -E 's/,$/\n/' println!( r#"cargo:rustc-check-cfg=cfg(portable_atomic_target_feature,values("cmpxchg16b","distinct-ops","fast-serialization","load-store-on-cond","lse","lse128","lse2","lsfe","mclass","miscellaneous-extensions-3","quadword-atomics","rcpc3","rmw","v6","v7","zaamo","zabha","zacas"))"# ); + println!( + r#"cargo:rustc-check-cfg=cfg(portable_atomic_target_cpu,values("esp32","esp32s3"))"# + ); } // https://github.com/rust-lang/rust/pull/123745 (includes https://github.com/rust-lang/cargo/pull/13560) merged in Rust 1.79 (nightly-2024-04-11). @@ -510,6 +513,33 @@ fn main() { target_feature_fallback("rmw", xmegau); } } + "xtensa" => { + // ESP32 / ESP32-S3 advertise atomic CAS in core atomics, but LLVM's + // atomic instructions do not behave atomically on addresses backed by + // external (PSRAM / data bus) memory. The `src/imp/xtensa.rs` wrapper + // dispatches each operation based on the address at runtime; to + // select it we need to know which of the two affected CPUs we are + // targeting. Infer from `target-cpu`, or from the known Espressif + // target triples when no target-cpu is specified. + // ESP32-S2 has no atomic CAS at all and is handled by the generic + // no-CAS path, so it is intentionally absent here. + if let Some(cpu) = target_cpu() { + if cpu == "esp32" || cpu == "esp32s3" { + target_cpu_fallback(&cpu); + } + } else { + let esp_cpu = match target { + "xtensa-esp32-none-elf" | "xtensa-esp32-espidf" => Some("esp32"), + "xtensa-esp32s3-none-elf" | "xtensa-esp32s3-espidf" => Some("esp32s3"), + // ESP32-S2 does not have atomic CAS, so it is not affected by the issue the same way. + // For other Xtensa CPUs, assume they are not affected. + _ => None, + }; + if let Some(cpu) = esp_cpu { + target_cpu_fallback(cpu); + } + } + } _ => {} } } @@ -568,6 +598,11 @@ fn target_cpu() -> Option { cpu.map(str::to_owned) } +// `target_cpu` is not a valid cfg option. Where there is absolutely no other option, inject a cfg fallback. +fn target_cpu_fallback(cpu: &str) { + println!("cargo:rustc-cfg=portable_atomic_target_cpu=\"{}\"", cpu); +} + fn is_allowed_feature(name: &str) -> bool { // https://github.com/dtolnay/thiserror/pull/248 if env::var_os("RUSTC_STAGE").is_some() { diff --git a/src/imp/mod.rs b/src/imp/mod.rs index 33fcf837..58adfa4a 100644 --- a/src/imp/mod.rs +++ b/src/imp/mod.rs @@ -35,6 +35,13 @@ not(target_has_atomic = "ptr"), ))) )] +// ESP32 and ESP32-S3 have native atomic CAS on internal memory but not on +// PSRAM. Redirect this module to a wrapper, which exposes the same public API +// and dispatches each operation based on the address at runtime. +#[cfg_attr( + any(portable_atomic_target_cpu = "esp32", portable_atomic_target_cpu = "esp32s3"), + path = "xtensa.rs" +)] mod core_atomic; // AVR diff --git a/src/imp/xtensa.rs b/src/imp/xtensa.rs new file mode 100644 index 00000000..718e0b22 --- /dev/null +++ b/src/imp/xtensa.rs @@ -0,0 +1,598 @@ +// SPDX-License-Identifier: Apache-2.0 OR MIT + +/* +PSRAM-aware atomics for ESP32 / ESP32-S3. + +These chips advertise native atomic CAS (Xtensa `S32C1I`) which works for +internal SRAM, but the instruction does NOT behave atomically on addresses +backed by the external data bus (PSRAM). Every operation therefore checks +the target address at runtime and dispatches to either +`core::sync::atomic::*` (fast path, when the atomic lives in internal +memory) or a critical section (when it lives in PSRAM). + +Address ranges: +- ESP32: 0x3F80_0000..0x3FC0_0000 +- ESP32-S3: 0x3C00_0000..0x3E00_0000 + +This module is selected in place of `src/imp/core_atomic.rs` by a `#[path]` +attribute in `src/imp/mod.rs`, so it exposes exactly the same public API. +*/ + +use core::{cell::UnsafeCell, marker::PhantomData, sync::atomic::Ordering}; + +// See `src/imp/core_atomic.rs` for why this marker is necessary — it +// prevents the `#[repr(transparent)]` wrapper below from inheriting +// `RefUnwindSafe` via the std atomic auto-impl, so that portable-atomic's +// public types carry `RefUnwindSafe` uniformly on every backend. +struct NotRefUnwindSafe(UnsafeCell<()>); +// SAFETY: marker type; the UnsafeCell value is never accessed. +unsafe impl Sync for NotRefUnwindSafe {} + +// The external (PSRAM / data bus) address ranges that do not support +// atomic RMW instructions on these CPUs. + +// https://documentation.espressif.com/esp32_technical_reference_manual_en.pdf, Table 3.3-4. External Memory Address Mapping +#[cfg(portable_atomic_target_cpu = "esp32")] +const PSRAM: core::ops::Range = 0x3F80_0000..0x3FC0_0000; + +// https://documentation.espressif.com/esp32-s3_technical_reference_manual_en.pdf, Table 4.3-2. External Memory Address Mapping +#[cfg(portable_atomic_target_cpu = "esp32s3")] +const PSRAM: core::ops::Range = 0x3C00_0000..0x3E00_0000; + +#[inline(always)] +fn in_psram(ptr: *const T) -> bool { + let addr = ptr as usize; + addr >= PSRAM.start && addr < PSRAM.end +} + +#[cfg(not(feature = "critical-section"))] +#[cold] +#[inline(never)] +#[track_caller] +fn psram_rmw_without_cs() -> ! { + panic!( + "portable-atomic: atomic read-modify-write on PSRAM requires the \ + `critical-section` feature on ESP32 / ESP32-S3" + ); +} + +// Run an RMW closure either natively (addr in internal memory) or under a +// critical section (addr in PSRAM). When the `critical-section` feature is +// disabled, the PSRAM path panics. +// +// `$ptr` is the backing raw pointer; the CS-path closure is an +// `unsafe` block that uses `read_volatile`/`write_volatile` on it. +macro_rules! rmw { + ($self:ident, $native:expr, |$ptr:ident| $cs:block) => {{ + let $ptr: *mut _ = $self.as_ptr(); + if in_psram($ptr) { + #[cfg(feature = "critical-section")] + { + critical_section::with(|_cs| { + // SAFETY: inside a critical section, we have exclusive + // access to `$ptr` and the pointer is valid because + // we got it from `&self`. The accesses use volatile + // reads/writes and so are not reordered with respect + // to each other or with the CS entry/exit barriers. + unsafe { $cs } + }) + } + #[cfg(not(feature = "critical-section"))] + { + let _ = $ptr; // suppress unused warning for move-only closures + psram_rmw_without_cs() + } + } else { + $native + } + }}; +} + +// --------------------------------------------------------------------------- +// AtomicPtr + +#[repr(transparent)] +pub(crate) struct AtomicPtr { + inner: core::sync::atomic::AtomicPtr, + _not_ref_unwind_safe: PhantomData, +} +impl AtomicPtr { + #[inline] + pub(crate) const fn new(v: *mut T) -> Self { + Self { inner: core::sync::atomic::AtomicPtr::new(v), _not_ref_unwind_safe: PhantomData } + } + #[inline] + pub(crate) fn is_lock_free() -> bool { + Self::IS_ALWAYS_LOCK_FREE + } + // Not lock-free: if the atomic happens to live in PSRAM, RMWs take a + // critical section. We can only give a conservative compile-time answer. + pub(crate) const IS_ALWAYS_LOCK_FREE: bool = false; + + #[inline] + #[cfg_attr(any(debug_assertions, miri), track_caller)] + pub(crate) fn load(&self, order: Ordering) -> *mut T { + crate::utils::assert_load_ordering(order); + self.inner.load(order) + } + #[inline] + #[cfg_attr(any(debug_assertions, miri), track_caller)] + pub(crate) fn store(&self, ptr: *mut T, order: Ordering) { + crate::utils::assert_store_ordering(order); + self.inner.store(ptr, order); + } + + #[inline] + #[cfg_attr(miri, track_caller)] + pub(crate) fn swap(&self, ptr: *mut T, order: Ordering) -> *mut T { + rmw!(self, self.inner.swap(ptr, order), |p| { + let prev = core::ptr::read_volatile(p); + core::ptr::write_volatile(p, ptr); + prev + }) + } + + #[inline] + #[cfg_attr(any(debug_assertions, miri), track_caller)] + pub(crate) fn compare_exchange( + &self, + current: *mut T, + new: *mut T, + success: Ordering, + failure: Ordering, + ) -> Result<*mut T, *mut T> { + crate::utils::assert_compare_exchange_ordering(success, failure); + rmw!(self, self.inner.compare_exchange(current, new, success, failure), |p| { + let prev = core::ptr::read_volatile(p); + if prev == current { + core::ptr::write_volatile(p, new); + Ok(prev) + } else { + Err(prev) + } + }) + } + #[inline] + #[cfg_attr(any(debug_assertions, miri), track_caller)] + pub(crate) fn compare_exchange_weak( + &self, + current: *mut T, + new: *mut T, + success: Ordering, + failure: Ordering, + ) -> Result<*mut T, *mut T> { + crate::utils::assert_compare_exchange_ordering(success, failure); + rmw!(self, self.inner.compare_exchange_weak(current, new, success, failure), |p| { + let prev = core::ptr::read_volatile(p); + if prev == current { + core::ptr::write_volatile(p, new); + Ok(prev) + } else { + Err(prev) + } + }) + } + + #[inline] + #[cfg_attr(miri, track_caller)] + pub(crate) fn fetch_byte_add(&self, val: usize, order: Ordering) -> *mut T { + #[cfg(portable_atomic_no_strict_provenance)] + use crate::utils::ptr::PtrExt as _; + rmw!( + self, + { + #[cfg(not(portable_atomic_no_strict_provenance_atomic_ptr))] + { + self.inner.fetch_byte_add(val, order) + } + #[cfg(portable_atomic_no_strict_provenance_atomic_ptr)] + { + crate::utils::ptr::with_exposed_provenance_mut( + self.as_atomic_usize().fetch_add(val, order), + ) + } + }, + |p| { + let prev = core::ptr::read_volatile(p); + let next = prev.with_addr(prev.addr().wrapping_add(val)); + core::ptr::write_volatile(p, next); + prev + } + ) + } + #[inline] + #[cfg_attr(miri, track_caller)] + pub(crate) fn fetch_byte_sub(&self, val: usize, order: Ordering) -> *mut T { + #[cfg(portable_atomic_no_strict_provenance)] + use crate::utils::ptr::PtrExt as _; + rmw!( + self, + { + #[cfg(not(portable_atomic_no_strict_provenance_atomic_ptr))] + { + self.inner.fetch_byte_sub(val, order) + } + #[cfg(portable_atomic_no_strict_provenance_atomic_ptr)] + { + crate::utils::ptr::with_exposed_provenance_mut( + self.as_atomic_usize().fetch_sub(val, order), + ) + } + }, + |p| { + let prev = core::ptr::read_volatile(p); + let next = prev.with_addr(prev.addr().wrapping_sub(val)); + core::ptr::write_volatile(p, next); + prev + } + ) + } + #[inline] + #[cfg_attr(miri, track_caller)] + pub(crate) fn fetch_or(&self, val: usize, order: Ordering) -> *mut T { + #[cfg(portable_atomic_no_strict_provenance)] + use crate::utils::ptr::PtrExt as _; + rmw!( + self, + { + #[cfg(not(portable_atomic_no_strict_provenance_atomic_ptr))] + { + self.inner.fetch_or(val, order) + } + #[cfg(portable_atomic_no_strict_provenance_atomic_ptr)] + { + crate::utils::ptr::with_exposed_provenance_mut( + self.as_atomic_usize().fetch_or(val, order), + ) + } + }, + |p| { + let prev = core::ptr::read_volatile(p); + let next = prev.with_addr(prev.addr() | val); + core::ptr::write_volatile(p, next); + prev + } + ) + } + #[inline] + #[cfg_attr(miri, track_caller)] + pub(crate) fn fetch_and(&self, val: usize, order: Ordering) -> *mut T { + #[cfg(portable_atomic_no_strict_provenance)] + use crate::utils::ptr::PtrExt as _; + rmw!( + self, + { + #[cfg(not(portable_atomic_no_strict_provenance_atomic_ptr))] + { + self.inner.fetch_and(val, order) + } + #[cfg(portable_atomic_no_strict_provenance_atomic_ptr)] + { + crate::utils::ptr::with_exposed_provenance_mut( + self.as_atomic_usize().fetch_and(val, order), + ) + } + }, + |p| { + let prev = core::ptr::read_volatile(p); + let next = prev.with_addr(prev.addr() & val); + core::ptr::write_volatile(p, next); + prev + } + ) + } + #[inline] + #[cfg_attr(miri, track_caller)] + pub(crate) fn fetch_xor(&self, val: usize, order: Ordering) -> *mut T { + #[cfg(portable_atomic_no_strict_provenance)] + use crate::utils::ptr::PtrExt as _; + rmw!( + self, + { + #[cfg(not(portable_atomic_no_strict_provenance_atomic_ptr))] + { + self.inner.fetch_xor(val, order) + } + #[cfg(portable_atomic_no_strict_provenance_atomic_ptr)] + { + crate::utils::ptr::with_exposed_provenance_mut( + self.as_atomic_usize().fetch_xor(val, order), + ) + } + }, + |p| { + let prev = core::ptr::read_volatile(p); + let next = prev.with_addr(prev.addr() ^ val); + core::ptr::write_volatile(p, next); + prev + } + ) + } + + #[cfg(portable_atomic_no_strict_provenance_atomic_ptr)] + #[inline(always)] + fn as_atomic_usize(&self) -> &AtomicUsize { + static_assert!( + core::mem::size_of::>() == core::mem::size_of::() + ); + static_assert!( + core::mem::align_of::>() == core::mem::align_of::() + ); + // SAFETY: AtomicPtr and AtomicUsize have the same layout, and both + // route through the same PSRAM dispatch above. + unsafe { &*(self as *const Self as *const AtomicUsize) } + } + + #[inline] + pub(crate) const fn as_ptr(&self) -> *mut *mut T { + // SAFETY: Self is #[repr(transparent)] and internally + // UnsafeCell<*mut T>. See also the equivalent trick in + // src/imp/core_atomic.rs. + unsafe { (*(self as *const Self as *const UnsafeCell<*mut T>)).get() } + } +} +impl_default_bit_opts!(AtomicPtr, usize); + +// --------------------------------------------------------------------------- +// AtomicInt + +macro_rules! atomic_int { + ($atomic_type:ident, $int_type:ident, $align:literal) => { + #[repr(transparent)] + pub(crate) struct $atomic_type { + inner: core::sync::atomic::$atomic_type, + _not_ref_unwind_safe: PhantomData, + } + impl $atomic_type { + #[inline] + pub(crate) const fn new(v: $int_type) -> Self { + Self { + inner: core::sync::atomic::$atomic_type::new(v), + _not_ref_unwind_safe: PhantomData, + } + } + #[inline] + pub(crate) fn is_lock_free() -> bool { + Self::IS_ALWAYS_LOCK_FREE + } + // We cannot promise lock-freedom without knowing the address + // at compile time; RMWs on PSRAM take a critical section. + pub(crate) const IS_ALWAYS_LOCK_FREE: bool = false; + + #[inline] + #[cfg_attr(any(debug_assertions, miri), track_caller)] + pub(crate) fn load(&self, order: Ordering) -> $int_type { + crate::utils::assert_load_ordering(order); + self.inner.load(order) + } + #[inline] + #[cfg_attr(any(debug_assertions, miri), track_caller)] + pub(crate) fn store(&self, val: $int_type, order: Ordering) { + crate::utils::assert_store_ordering(order); + self.inner.store(val, order); + } + + #[inline] + #[cfg_attr(miri, track_caller)] + pub(crate) fn swap(&self, val: $int_type, order: Ordering) -> $int_type { + rmw!(self, self.inner.swap(val, order), |p| { + let prev = core::ptr::read_volatile(p); + core::ptr::write_volatile(p, val); + prev + }) + } + + #[inline] + #[cfg_attr(any(debug_assertions, miri), track_caller)] + pub(crate) fn compare_exchange( + &self, + current: $int_type, + new: $int_type, + success: Ordering, + failure: Ordering, + ) -> Result<$int_type, $int_type> { + crate::utils::assert_compare_exchange_ordering(success, failure); + rmw!(self, self.inner.compare_exchange(current, new, success, failure), |p| { + let prev = core::ptr::read_volatile(p); + if prev == current { + core::ptr::write_volatile(p, new); + Ok(prev) + } else { + Err(prev) + } + }) + } + #[inline] + #[cfg_attr(any(debug_assertions, miri), track_caller)] + pub(crate) fn compare_exchange_weak( + &self, + current: $int_type, + new: $int_type, + success: Ordering, + failure: Ordering, + ) -> Result<$int_type, $int_type> { + crate::utils::assert_compare_exchange_ordering(success, failure); + rmw!(self, self.inner.compare_exchange_weak(current, new, success, failure), |p| { + let prev = core::ptr::read_volatile(p); + if prev == current { + core::ptr::write_volatile(p, new); + Ok(prev) + } else { + Err(prev) + } + }) + } + + #[allow(dead_code)] + #[inline] + #[cfg_attr(miri, track_caller)] + fn fetch_update_(&self, order: Ordering, mut f: F) -> $int_type + where + F: FnMut($int_type) -> $int_type, + { + let mut prev = self.load(Ordering::Relaxed); + loop { + let next = f(prev); + match self.compare_exchange_weak(prev, next, order, Ordering::Relaxed) { + Ok(x) => return x, + Err(next_prev) => prev = next_prev, + } + } + } + + // Arithmetic RMWs ------------------------------------------------ + #[inline] + #[cfg_attr(miri, track_caller)] + pub(crate) fn fetch_add(&self, val: $int_type, order: Ordering) -> $int_type { + rmw!(self, self.inner.fetch_add(val, order), |p| { + let prev = core::ptr::read_volatile(p); + core::ptr::write_volatile(p, prev.wrapping_add(val)); + prev + }) + } + #[inline] + #[cfg_attr(miri, track_caller)] + pub(crate) fn fetch_sub(&self, val: $int_type, order: Ordering) -> $int_type { + rmw!(self, self.inner.fetch_sub(val, order), |p| { + let prev = core::ptr::read_volatile(p); + core::ptr::write_volatile(p, prev.wrapping_sub(val)); + prev + }) + } + + // Bitwise RMWs --------------------------------------------------- + #[inline] + #[cfg_attr(miri, track_caller)] + pub(crate) fn fetch_and(&self, val: $int_type, order: Ordering) -> $int_type { + rmw!(self, self.inner.fetch_and(val, order), |p| { + let prev = core::ptr::read_volatile(p); + core::ptr::write_volatile(p, prev & val); + prev + }) + } + #[inline] + #[cfg_attr(miri, track_caller)] + pub(crate) fn fetch_nand(&self, val: $int_type, order: Ordering) -> $int_type { + rmw!(self, self.inner.fetch_nand(val, order), |p| { + let prev = core::ptr::read_volatile(p); + core::ptr::write_volatile(p, !(prev & val)); + prev + }) + } + #[inline] + #[cfg_attr(miri, track_caller)] + pub(crate) fn fetch_or(&self, val: $int_type, order: Ordering) -> $int_type { + rmw!(self, self.inner.fetch_or(val, order), |p| { + let prev = core::ptr::read_volatile(p); + core::ptr::write_volatile(p, prev | val); + prev + }) + } + #[inline] + #[cfg_attr(miri, track_caller)] + pub(crate) fn fetch_xor(&self, val: $int_type, order: Ordering) -> $int_type { + rmw!(self, self.inner.fetch_xor(val, order), |p| { + let prev = core::ptr::read_volatile(p); + core::ptr::write_volatile(p, prev ^ val); + prev + }) + } + + // fetch_max / fetch_min ----------------------------------------- + // + // The PSRAM arm performs the whole read-compute-write inside a + // single critical section. std's native min/max atomics are + // always available on xtensa (stabilized in 1.45; xtensa MSRV + // is 1.81+). + #[inline] + #[cfg_attr(miri, track_caller)] + pub(crate) fn fetch_max(&self, val: $int_type, order: Ordering) -> $int_type { + rmw!(self, self.inner.fetch_max(val, order), |p| { + let prev = core::ptr::read_volatile(p); + core::ptr::write_volatile(p, core::cmp::max(prev, val)); + prev + }) + } + #[inline] + #[cfg_attr(miri, track_caller)] + pub(crate) fn fetch_min(&self, val: $int_type, order: Ordering) -> $int_type { + rmw!(self, self.inner.fetch_min(val, order), |p| { + let prev = core::ptr::read_volatile(p); + core::ptr::write_volatile(p, core::cmp::min(prev, val)); + prev + }) + } + + // Unary RMWs ---------------------------------------------------- + #[inline] + #[cfg_attr(miri, track_caller)] + pub(crate) fn fetch_not(&self, order: Ordering) -> $int_type { + self.fetch_xor(!0, order) + } + #[inline] + #[cfg_attr(miri, track_caller)] + pub(crate) fn fetch_neg(&self, order: Ordering) -> $int_type { + // `core::sync::atomic::*` has no native fetch_neg, so the + // SRAM path is a CAS loop. The PSRAM path does the negate + // inside a single critical section. + rmw!(self, self.fetch_update_(order, $int_type::wrapping_neg), |p| { + let prev = core::ptr::read_volatile(p); + core::ptr::write_volatile(p, prev.wrapping_neg()); + prev + }) + } + #[inline] + #[cfg_attr(miri, track_caller)] + pub(crate) fn not(&self, order: Ordering) { + self.fetch_not(order); + } + #[inline] + #[cfg_attr(miri, track_caller)] + pub(crate) fn neg(&self, order: Ordering) { + self.fetch_neg(order); + } + + #[inline] + pub(crate) const fn as_ptr(&self) -> *mut $int_type { + // SAFETY: Self is #[repr(transparent)] and internally + // UnsafeCell<$int_type>. + unsafe { (*(self as *const Self as *const UnsafeCell<$int_type>)).get() } + } + } + impl_default_no_fetch_ops!($atomic_type, $int_type); + impl_default_bit_opts!($atomic_type, $int_type); + + // Force the requested alignment just like `core::sync::atomic::*`. + // #[repr(transparent)] inherits the inner alignment; the asserts + // document the expected layout. + #[allow(dead_code)] + const _: () = { + if core::mem::align_of::<$atomic_type>() < $align { + panic!(concat!(stringify!($atomic_type), " has smaller alignment than expected")); + } + }; + }; +} + +atomic_int!(AtomicIsize, isize, 4); +atomic_int!(AtomicUsize, usize, 4); +atomic_int!(AtomicI8, i8, 1); +atomic_int!(AtomicU8, u8, 1); +atomic_int!(AtomicI16, i16, 2); +atomic_int!(AtomicU16, u16, 2); +atomic_int!(AtomicI32, i32, 4); +atomic_int!(AtomicU32, u32, 4); + +// espidf Xtensa targets expose 64-bit atomics via libatomic; wrap those too. +// On bare-metal xtensa targets `target_has_atomic = "64"` is false and these +// types are not emitted. `cfg(target_has_atomic = ...)` stabilized in Rust +// 1.60; xtensa MSRV is 1.81+ so `portable_atomic_no_cfg_target_has_atomic` +// is never set in this module. +#[cfg(any( + target_has_atomic = "64", + not(any(target_pointer_width = "16", target_pointer_width = "32")), +))] +items!({ + atomic_int!(AtomicI64, i64, 8); + atomic_int!(AtomicU64, u64, 8); +}); diff --git a/tools/build.sh b/tools/build.sh index 63915f9a..2e9b0855 100755 --- a/tools/build.sh +++ b/tools/build.sh @@ -414,6 +414,9 @@ build() { armv[4-5]t* | thumbv[4-5]t* | thumbv6m* | riscv??[ie]-*-none* | riscv??[ie]m-*-none* | riscv??[ie]mc-*-none* | xtensa-esp32s2-*) target_rustflags+=" --cfg portable_atomic_unsafe_assume_single_core" ;; + xtensa-esp32*) + # At this time, these chips require critical-section to be enabled, which is incompatible with the single-core and privileged assumptions. + ;; esac fi # TODO: handle SIGILL and ERR