Add "slide" operation

Author: valadaptive

fearless_simd/src/generated/avx2.rs

@@ -42,3 +42,24 @@ pub(crate) fn simd_debug_impl<Element: core::fmt::Debug>(
         .field("simd", token)
         .finish()
 }
+
+/// Selects the input operands to be used for `slignr`/`vext`/etc. when computing a single output block for cross-block
+/// "slide" operations. Extracts from [a : b].
+#[inline(always)]
+#[allow(clippy::allow_attributes, reason = "Only needed in some cfgs.")]
+#[allow(dead_code, reason = "Only used in some cfgs.")]
+pub(crate) fn cross_block_slide_blocks_at<const N: usize, Block: Copy>(
+    a: &[Block; N],
+    b: &[Block; N],
+    out_idx: usize,
+    shift_bytes: usize,
+) -> [Block; 2] {
+    const BLOCK_BYTES: usize = 16;
+    let out_byte_start = out_idx * BLOCK_BYTES + shift_bytes;
+    let lo_idx = out_byte_start.div_euclid(BLOCK_BYTES);
+    let hi_idx = lo_idx + 1;
+    // Concatenation is [a : b], so indices 0..N are from a, indices N..2N are from b
+    let lo_block = if lo_idx < N { a[lo_idx] } else { b[lo_idx - N] };
+    let hi_block = if hi_idx < N { a[hi_idx] } else { b[hi_idx - N] };
+    [lo_block, hi_block]
+}

fearless_simd/src/generated/fallback.rs

@@ -80,7 +80,10 @@ pub fn simd_test(_: TokenStream, item: TokenStream) -> TokenStream {
         fn #sse4_name() {
             if std::arch::is_x86_feature_detected!("sse4.2") {
                 let sse4 = unsafe { fearless_simd::x86::Sse4_2::new_unchecked() };
-                #input_fn_name(sse4);
+                sse4.vectorize(
+                    #[inline(always)]
+                    || #input_fn_name(sse4)
+                );
             }
         }
     };
@@ -94,7 +97,10 @@ pub fn simd_test(_: TokenStream, item: TokenStream) -> TokenStream {
                 && std::arch::is_x86_feature_detected!("fma")
             {
                 let avx2 = unsafe { fearless_simd::x86::Avx2::new_unchecked() };
-                #input_fn_name(avx2);
+                avx2.vectorize(
+                    #[inline(always)]
+                    || #input_fn_name(avx2)
+                );
             }
         }
     };
@@ -110,6 +116,7 @@ pub fn simd_test(_: TokenStream, item: TokenStream) -> TokenStream {
     };
 
     quote! {
+        #[inline(always)]
         #input_fn
 
         #fallback_snippet

fearless_simd/src/generated/neon.rs

@@ -5,7 +5,7 @@ use proc_macro2::{Ident, Span, TokenStream};
 use quote::{format_ident, quote};
 
 use crate::{
-    ops::{OpSig, RefKind},
+    ops::{OpSig, RefKind, SlideGranularity},
     types::{SIMD_TYPES, ScalarType, VecType},
 };
 
@@ -234,6 +234,32 @@ pub(crate) fn generic_op(op: &str, sig: OpSig, ty: &VecType) -> TokenStream {
         }
         OpSig::FromBytes => generic_from_bytes(method_sig, ty),
         OpSig::ToBytes => generic_to_bytes(method_sig, ty),
+        OpSig::Slide { granularity, .. } => {
+            match (granularity, ty.n_bits()) {
+                (SlideGranularity::WithinBlocks, 128) => {
+                    // If this operation is done on a 128-bit vector type, the "within blocks" method is identical to the
+                    // non-within-blocks one, so just defer to that.
+                    let non_blockwise = generic_op_name("slide", ty);
+                    quote! {
+                        #method_sig {
+                            self.#non_blockwise::<SHIFT>(a, b)
+                        }
+                    }
+                }
+                (SlideGranularity::WithinBlocks, _) => {
+                    quote! {
+                        #method_sig {
+                            let (a0, a1) = self.#split(a);
+                            let (b0, b1) = self.#split(b);
+                            self.#combine(self.#do_half::<SHIFT>(a0, b0), self.#do_half::<SHIFT>(a1, b1))
+                        }
+                    }
+                }
+                _ => {
+                    panic!("Item-wise shifts across blocks cannot be done via split/combine");
+                }
+            }
+        }
     }
 }