Skip to content

use intrinsics::simd for interleaving store #2032

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Amanieu merged 4 commits into from
Feb 21, 2026
+164 −491
Merged

use intrinsics::simd for interleaving store #2032

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
567e428
use `intrinsics::simd` for interleaving store
folkertdev Feb 18, 2026
04451c8
use `intrinsics::simd` for interleaving store of `int64x1`
folkertdev Feb 18, 2026
0c452e4
use `intrinsics::simd` for interleaving store of f16
folkertdev Feb 18, 2026
281d367
use `intrinsics::simd` for aarch64 interleaving `st`
folkertdev Feb 18, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
83 changes: 10 additions & 73 deletions crates/core_arch/src/aarch64/neon/generated.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25039,16 +25039,9 @@ pub unsafe fn vst1q_lane_f64<const LANE: i32>(a: *mut f64, b: float64x2_t) {
#[inline(always)]
#[target_feature(enable = "neon")]
#[stable(feature = "neon_intrinsics", since = "1.59.0")]
#[cfg_attr(test, assert_instr(st1))]
#[cfg_attr(test, assert_instr(stp))]
pub unsafe fn vst2_f64(a: *mut f64, b: float64x1x2_t) {
unsafe extern "unadjusted" {
#[cfg_attr(
any(target_arch = "aarch64", target_arch = "arm64ec"),
link_name = "llvm.aarch64.neon.st2.v1f64.p0"
)]
fn _vst2_f64(a: float64x1_t, b: float64x1_t, ptr: *mut i8);
}
_vst2_f64(b.0, b.1, a as _)
core::ptr::write_unaligned(a.cast(), b)
}
#[doc = "Store multiple 2-element structures from two registers"]
#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst2_lane_f64)"]
Expand Down Expand Up @@ -25125,14 +25118,7 @@ pub unsafe fn vst2_lane_u64<const LANE: i32>(a: *mut u64, b: uint64x1x2_t) {
#[stable(feature = "neon_intrinsics", since = "1.59.0")]
#[cfg_attr(test, assert_instr(st2))]
pub unsafe fn vst2q_f64(a: *mut f64, b: float64x2x2_t) {
unsafe extern "unadjusted" {
#[cfg_attr(
any(target_arch = "aarch64", target_arch = "arm64ec"),
link_name = "llvm.aarch64.neon.st2.v2f64.p0"
)]
fn _vst2q_f64(a: float64x2_t, b: float64x2_t, ptr: *mut i8);
}
_vst2q_f64(b.0, b.1, a as _)
crate::core_arch::macros::interleaving_store!(f64, 2, 2, a, b)
}
#[doc = "Store multiple 2-element structures from two registers"]
#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst2q_s64)"]
Expand All @@ -25143,14 +25129,7 @@ pub unsafe fn vst2q_f64(a: *mut f64, b: float64x2x2_t) {
#[stable(feature = "neon_intrinsics", since = "1.59.0")]
#[cfg_attr(test, assert_instr(st2))]
pub unsafe fn vst2q_s64(a: *mut i64, b: int64x2x2_t) {
unsafe extern "unadjusted" {
#[cfg_attr(
any(target_arch = "aarch64", target_arch = "arm64ec"),
link_name = "llvm.aarch64.neon.st2.v2i64.p0"
)]
fn _vst2q_s64(a: int64x2_t, b: int64x2_t, ptr: *mut i8);
}
_vst2q_s64(b.0, b.1, a as _)
crate::core_arch::macros::interleaving_store!(i64, 2, 2, a, b)
}
#[doc = "Store multiple 2-element structures from two registers"]
#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst2q_lane_f64)"]
Expand Down Expand Up @@ -25295,14 +25274,7 @@ pub unsafe fn vst2q_u64(a: *mut u64, b: uint64x2x2_t) {
#[stable(feature = "neon_intrinsics", since = "1.59.0")]
#[cfg_attr(test, assert_instr(nop))]
pub unsafe fn vst3_f64(a: *mut f64, b: float64x1x3_t) {
unsafe extern "unadjusted" {
#[cfg_attr(
any(target_arch = "aarch64", target_arch = "arm64ec"),
link_name = "llvm.aarch64.neon.st3.v1f64.p0"
)]
fn _vst3_f64(a: float64x1_t, b: float64x1_t, c: float64x1_t, ptr: *mut i8);
}
_vst3_f64(b.0, b.1, b.2, a as _)
core::ptr::write_unaligned(a.cast(), b)
}
#[doc = "Store multiple 3-element structures from three registers"]
#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst3_lane_f64)"]
Expand Down Expand Up @@ -25379,14 +25351,7 @@ pub unsafe fn vst3_lane_u64<const LANE: i32>(a: *mut u64, b: uint64x1x3_t) {
#[stable(feature = "neon_intrinsics", since = "1.59.0")]
#[cfg_attr(test, assert_instr(st3))]
pub unsafe fn vst3q_f64(a: *mut f64, b: float64x2x3_t) {
unsafe extern "unadjusted" {
#[cfg_attr(
any(target_arch = "aarch64", target_arch = "arm64ec"),
link_name = "llvm.aarch64.neon.st3.v2f64.p0"
)]
fn _vst3q_f64(a: float64x2_t, b: float64x2_t, c: float64x2_t, ptr: *mut i8);
}
_vst3q_f64(b.0, b.1, b.2, a as _)
crate::core_arch::macros::interleaving_store!(f64, 2, 3, a, b)
}
#[doc = "Store multiple 3-element structures from three registers"]
#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst3q_s64)"]
Expand All @@ -25397,14 +25362,7 @@ pub unsafe fn vst3q_f64(a: *mut f64, b: float64x2x3_t) {
#[stable(feature = "neon_intrinsics", since = "1.59.0")]
#[cfg_attr(test, assert_instr(st3))]
pub unsafe fn vst3q_s64(a: *mut i64, b: int64x2x3_t) {
unsafe extern "unadjusted" {
#[cfg_attr(
any(target_arch = "aarch64", target_arch = "arm64ec"),
link_name = "llvm.aarch64.neon.st3.v2i64.p0"
)]
fn _vst3q_s64(a: int64x2_t, b: int64x2_t, c: int64x2_t, ptr: *mut i8);
}
_vst3q_s64(b.0, b.1, b.2, a as _)
crate::core_arch::macros::interleaving_store!(i64, 2, 3, a, b)
}
#[doc = "Store multiple 3-element structures from three registers"]
#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst3q_lane_f64)"]
Expand Down Expand Up @@ -25549,14 +25507,7 @@ pub unsafe fn vst3q_u64(a: *mut u64, b: uint64x2x3_t) {
#[stable(feature = "neon_intrinsics", since = "1.59.0")]
#[cfg_attr(test, assert_instr(nop))]
pub unsafe fn vst4_f64(a: *mut f64, b: float64x1x4_t) {
unsafe extern "unadjusted" {
#[cfg_attr(
any(target_arch = "aarch64", target_arch = "arm64ec"),
link_name = "llvm.aarch64.neon.st4.v1f64.p0"
)]
fn _vst4_f64(a: float64x1_t, b: float64x1_t, c: float64x1_t, d: float64x1_t, ptr: *mut i8);
}
_vst4_f64(b.0, b.1, b.2, b.3, a as _)
core::ptr::write_unaligned(a.cast(), b)
}
#[doc = "Store multiple 4-element structures from four registers"]
#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst4_lane_f64)"]
Expand Down Expand Up @@ -25647,14 +25598,7 @@ pub unsafe fn vst4_lane_u64<const LANE: i32>(a: *mut u64, b: uint64x1x4_t) {
#[stable(feature = "neon_intrinsics", since = "1.59.0")]
#[cfg_attr(test, assert_instr(st4))]
pub unsafe fn vst4q_f64(a: *mut f64, b: float64x2x4_t) {
unsafe extern "unadjusted" {
#[cfg_attr(
any(target_arch = "aarch64", target_arch = "arm64ec"),
link_name = "llvm.aarch64.neon.st4.v2f64.p0"
)]
fn _vst4q_f64(a: float64x2_t, b: float64x2_t, c: float64x2_t, d: float64x2_t, ptr: *mut i8);
}
_vst4q_f64(b.0, b.1, b.2, b.3, a as _)
crate::core_arch::macros::interleaving_store!(f64, 2, 4, a, b)
}
#[doc = "Store multiple 4-element structures from four registers"]
#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst4q_s64)"]
Expand All @@ -25665,14 +25609,7 @@ pub unsafe fn vst4q_f64(a: *mut f64, b: float64x2x4_t) {
#[stable(feature = "neon_intrinsics", since = "1.59.0")]
#[cfg_attr(test, assert_instr(st4))]
pub unsafe fn vst4q_s64(a: *mut i64, b: int64x2x4_t) {
unsafe extern "unadjusted" {
#[cfg_attr(
any(target_arch = "aarch64", target_arch = "arm64ec"),
link_name = "llvm.aarch64.neon.st4.v2i64.p0"
)]
fn _vst4q_s64(a: int64x2_t, b: int64x2_t, c: int64x2_t, d: int64x2_t, ptr: *mut i8);
}
_vst4q_s64(b.0, b.1, b.2, b.3, a as _)
crate::core_arch::macros::interleaving_store!(i64, 2, 4, a, b)
}
#[doc = "Store multiple 4-element structures from four registers"]
#[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst4q_lane_f64)"]
Expand Down
8 changes: 8 additions & 0 deletions crates/core_arch/src/aarch64/neon/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1050,6 +1050,14 @@ mod tests {
test_vld1q_f16_x2(f16, 16, float16x8x2_t, vst1q_f16_x2, vld1q_f16_x2);
test_vld1q_f16_x3(f16, 24, float16x8x3_t, vst1q_f16_x3, vld1q_f16_x3);
test_vld1q_f16_x4(f16, 32, float16x8x4_t, vst1q_f16_x4, vld1q_f16_x4);

test_vld2_f16_x2(f16, 8, float16x4x2_t, vst2_f16, vld2_f16);
test_vld2_f16_x3(f16, 12, float16x4x3_t, vst3_f16, vld3_f16);
test_vld2_f16_x4(f16, 16, float16x4x4_t, vst4_f16, vld4_f16);

test_vld2q_f16_x2(f16, 16, float16x8x2_t, vst2q_f16, vld2q_f16);
test_vld3q_f16_x3(f16, 24, float16x8x3_t, vst3q_f16, vld3q_f16);
test_vld4q_f16_x4(f16, 32, float16x8x4_t, vst4q_f16, vld4q_f16);
}

macro_rules! wide_store_load_roundtrip_aes {
Expand Down
Loading