Skip to content

Commit

Permalink
detect/x86_64: Reduce scope of allow(dead_code)
Browse files Browse the repository at this point in the history
  • Loading branch information
@taiki-e
taiki-e committed Mar 9, 2024
1 parent 8b73709 commit 5b5ecb8
Show file tree
Hide file tree
Showing 3 changed files with 39 additions and 31 deletions.
2 changes: 2 additions & 0 deletions src/imp/atomic128/detect/common.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -89,6 +89,7 @@ impl CpuInfo {
/// Whether CMPXCHG16B is available
const HAS_CMPXCHG16B: u32 = 1;
/// Whether VMOVDQA is atomic
#[cfg(target_feature = "sse")]
const HAS_VMOVDQA_ATOMIC: u32 = 2;

#[cfg(any(
Expand All @@ -99,6 +100,7 @@ impl CpuInfo {
pub(crate) fn has_cmpxchg16b(self) -> bool {
self.test(CpuInfo::HAS_CMPXCHG16B)
}
#[cfg(target_feature = "sse")]
#[inline]
pub(crate) fn has_vmovdqa_atomic(self) -> bool {
self.test(CpuInfo::HAS_VMOVDQA_ATOMIC)
Expand Down
53 changes: 29 additions & 24 deletions src/imp/atomic128/detect/x86_64.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@

// Adapted from https://github.com/rust-lang/stdarch.

#![cfg_attr(any(not(target_feature = "sse"), portable_atomic_sanitize_thread), allow(dead_code))]
#![cfg_attr(portable_atomic_sanitize_thread, allow(dead_code))]

// Miri doesn't support inline assembly used in __cpuid: https://github.com/rust-lang/miri/issues/932
// SGX doesn't support CPUID: https://github.com/rust-lang/stdarch/blob/a0c30f3e3c75adcd6ee7efc94014ebcead61c507/crates/core_arch/src/x86/cpuid.rs#L102-L105
Expand All @@ -13,7 +13,7 @@ include!("common.rs");

#[cfg(not(portable_atomic_no_asm))]
use core::arch::asm;
use core::arch::x86_64::{CpuidResult, _xgetbv};
use core::arch::x86_64::CpuidResult;

// Workaround for https://github.com/rust-lang/rust/issues/101346
// It is not clear if our use cases are affected, but we implement this just in case.
Expand Down Expand Up @@ -45,8 +45,8 @@ unsafe fn __cpuid(leaf: u32) -> CpuidResult {
}

// https://en.wikipedia.org/wiki/CPUID
const VENDOR_ID_INTEL: [u8; 12] = *b"GenuineIntel";
const VENDOR_ID_AMD: [u8; 12] = *b"AuthenticAMD";
const _VENDOR_ID_INTEL: [u8; 12] = *b"GenuineIntel";
const _VENDOR_ID_AMD: [u8; 12] = *b"AuthenticAMD";

unsafe fn _vendor_id() -> [u8; 12] {
// https://github.com/rust-lang/stdarch/blob/a0c30f3e3c75adcd6ee7efc94014ebcead61c507/crates/std_detect/src/detect/os/x86.rs#L40-L59
Expand All @@ -59,9 +59,6 @@ unsafe fn _vendor_id() -> [u8; 12] {

#[cold]
fn _detect(info: &mut CpuInfo) {
// SAFETY: Calling `_vendor_id`` is safe because the CPU has `cpuid` support.
let vendor_id = unsafe { _vendor_id() };

// SAFETY: Calling `__cpuid`` is safe because the CPU has `cpuid` support.
let proc_info_ecx = unsafe { __cpuid(0x0000_0001_u32).ecx };

Expand All @@ -70,20 +67,29 @@ fn _detect(info: &mut CpuInfo) {
info.set(CpuInfo::HAS_CMPXCHG16B);
}

// VMOVDQA is atomic on Intel and AMD CPUs with AVX.
// See https://gcc.gnu.org/bugzilla/show_bug.cgi?id=104688 for details.
if vendor_id == VENDOR_ID_INTEL || vendor_id == VENDOR_ID_AMD {
// https://github.com/rust-lang/stdarch/blob/a0c30f3e3c75adcd6ee7efc94014ebcead61c507/crates/std_detect/src/detect/os/x86.rs#L131-L224
let cpu_xsave = test(proc_info_ecx, 26);
if cpu_xsave {
let cpu_osxsave = test(proc_info_ecx, 27);
if cpu_osxsave {
// SAFETY: Calling `_xgetbv`` is safe because the CPU has `xsave` support
// and OS has set `osxsave`.
let xcr0 = unsafe { _xgetbv(0) };
let os_avx_support = xcr0 & 6 == 6;
if os_avx_support && test(proc_info_ecx, 28) {
info.set(CpuInfo::HAS_VMOVDQA_ATOMIC);
// We only use VMOVDQA when SSE is enabled. See atomic_load_vmovdqa() in atomic128/x86_64.rs for more.
#[cfg(target_feature = "sse")]
{
use core::arch::x86_64::_xgetbv;

// SAFETY: Calling `vendor_id`` is safe because the CPU has `cpuid` support.
let vendor_id = unsafe { _vendor_id() };

// VMOVDQA is atomic on Intel and AMD CPUs with AVX.
// See https://gcc.gnu.org/bugzilla/show_bug.cgi?id=104688 for details.
if vendor_id == _VENDOR_ID_INTEL || vendor_id == _VENDOR_ID_AMD {
// https://github.com/rust-lang/stdarch/blob/a0c30f3e3c75adcd6ee7efc94014ebcead61c507/crates/std_detect/src/detect/os/x86.rs#L131-L224
let cpu_xsave = test(proc_info_ecx, 26);
if cpu_xsave {
let cpu_osxsave = test(proc_info_ecx, 27);
if cpu_osxsave {
// SAFETY: Calling `_xgetbv`` is safe because the CPU has `xsave` support
// and OS has set `osxsave`.
let xcr0 = unsafe { _xgetbv(0) };
let os_avx_support = xcr0 & 6 == 6;
if os_avx_support && test(proc_info_ecx, 28) {
info.set(CpuInfo::HAS_VMOVDQA_ATOMIC);
}
}
}
}
Expand All @@ -99,15 +105,14 @@ fn _detect(info: &mut CpuInfo) {
)]
#[cfg(test)]
mod tests {
#[cfg(not(portable_atomic_test_outline_atomics_detect_false))]
use super::*;

#[cfg(not(portable_atomic_test_outline_atomics_detect_false))]
#[test]
#[cfg_attr(portable_atomic_test_outline_atomics_detect_false, ignore)]
fn test_cpuid() {
assert_eq!(std::is_x86_feature_detected!("cmpxchg16b"), detect().has_cmpxchg16b());
let vendor_id = unsafe { _vendor_id() };
if vendor_id == VENDOR_ID_INTEL || vendor_id == VENDOR_ID_AMD {
if vendor_id == _VENDOR_ID_INTEL || vendor_id == _VENDOR_ID_AMD {
assert_eq!(std::is_x86_feature_detected!("avx"), detect().has_vmovdqa_atomic());
} else {
assert!(!detect().has_vmovdqa_atomic());
Expand Down
15 changes: 8 additions & 7 deletions src/imp/atomic128/x86_64.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -131,8 +131,11 @@ unsafe fn cmpxchg16b(dst: *mut u128, old: u128, new: u128) -> (u128, bool) {
//
// Refs: https://www.felixcloutier.com/x86/movdqa:vmovdqa32:vmovdqa64
//
// Do not use vector registers on targets such as x86_64-unknown-none unless SSE is explicitly enabled.
// https://doc.rust-lang.org/nightly/rustc/platform-support/x86_64-unknown-none.html
// Use cfg(target_feature = "sse") here -- SSE is included in the x86_64
// baseline and is always available, but the SSE target feature is disabled for
// use cases such as kernels and firmware that should not use vector registers.
// So, do not use vector registers unless SSE target feature is enabled.
// See also https://doc.rust-lang.org/nightly/rustc/platform-support/x86_64-unknown-none.html.
#[cfg(not(any(portable_atomic_no_outline_atomics, target_env = "sgx")))]
#[cfg(target_feature = "sse")]
#[target_feature(enable = "avx")]
Expand Down Expand Up @@ -208,7 +211,7 @@ macro_rules! load_store_detect {
{
// Check CMPXCHG16B first to prevent mixing atomic and non-atomic access.
if cpuid.has_cmpxchg16b() {
// We do not use vector registers on targets such as x86_64-unknown-none unless SSE is explicitly enabled.
// We only use VMOVDQA when SSE is enabled. See atomic_load_vmovdqa() for more.
#[cfg(target_feature = "sse")]
{
if cpuid.has_vmovdqa_atomic() {
Expand Down Expand Up @@ -238,8 +241,7 @@ macro_rules! load_store_detect {

#[inline]
unsafe fn atomic_load(src: *mut u128, _order: Ordering) -> u128 {
// Do not use vector registers on targets such as x86_64-unknown-none unless SSE is explicitly enabled.
// https://doc.rust-lang.org/nightly/rustc/platform-support/x86_64-unknown-none.html
// We only use VMOVDQA when SSE is enabled. See atomic_load_vmovdqa() for more.
// SGX doesn't support CPUID.
#[cfg(all(
any(target_feature = "cmpxchg16b", portable_atomic_target_feature = "cmpxchg16b"),
Expand Down Expand Up @@ -317,8 +319,7 @@ unsafe fn atomic_load_cmpxchg16b(src: *mut u128) -> u128 {

#[inline]
unsafe fn atomic_store(dst: *mut u128, val: u128, order: Ordering) {
// Do not use vector registers on targets such as x86_64-unknown-none unless SSE is explicitly enabled.
// https://doc.rust-lang.org/nightly/rustc/platform-support/x86_64-unknown-none.html
// We only use VMOVDQA when SSE is enabled. See atomic_load_vmovdqa() for more.
// SGX doesn't support CPUID.
#[cfg(all(
any(target_feature = "cmpxchg16b", portable_atomic_target_feature = "cmpxchg16b"),
Expand Down

0 comments on commit 5b5ecb8

Please sign in to comment.