Init foo_input_joc
build / windows (push) Has been cancelled

This commit is contained in:
TheM14 committed 2026-09-25 17:07:04 +08:00
commit ad62a914e6
127 files changed
+38545

No files matched your search

+101
View File
@@ -0,0 +1,101 @@
// CPU feature probe; see cpu_probe.h.
//
// Nothing here is ISA-specific code: the x86 path uses CPUID / XGETBV (MSVC) or
// the compiler's own runtime probe (GCC/Clang, so no inline assembly), and the
// AArch64 path reads the aux vector. That is what lets this unit stay baseline
// and run before the dispatcher has decided anything.
#include "simd/cpu_probe.h"
#if defined(_M_X64)
#include <immintrin.h>
#include <intrin.h>
#elif defined(__x86_64__)
#include <cpuid.h>
#endif
#if defined(__linux__) && (defined(__aarch64__) || defined(_M_ARM64))
#include <sys/auxv.h>
#endif
namespace joc::simd {
namespace {
// ------------------------------------------------------------------- x86-64 --
#if defined(_M_X64) || defined(__x86_64__)
#if defined(_M_X64)
// CPUID tells us what the silicon can do; XCR0 tells us whether the OS saves the
// state the wider registers need. Both have to agree, or the first AVX
// instruction after a context switch corrupts another thread's registers.
CpuFeatures probe_x86() noexcept {
CpuFeatures features;
int regs[4] = {0, 0, 0, 0};
__cpuid(regs, 0);
const int max_leaf = regs[0];
__cpuid(regs, 1);
features.sse2 = (regs[3] & (1 << 26)) != 0;
const bool osxsave = (regs[2] & (1 << 27)) != 0;
const bool avx = (regs[2] & (1 << 28)) != 0;
if (!osxsave || !avx || max_leaf < 7) {
return features;
}
const unsigned long long xcr0 = _xgetbv(0);
const bool ymm_state = (xcr0 & 0x06ull) == 0x06ull; // XMM + YMM saved
const bool zmm_state = (xcr0 & 0xe6ull) == 0xe6ull; // + opmask / ZMM / hi16
if (!ymm_state) {
return features;
}
__cpuidex(regs, 7, 0);
features.avx2 = (regs[1] & (1 << 5)) != 0;
features.avx512 = zmm_state && (regs[1] & (1 << 16)) != 0; // AVX512F
return features;
}
#else // GCC/Clang on x86-64
// The compiler runtime performs the same CPUID + XGETBV probe (libgcc's cpuinfo
// checks XCR0 before it reports AVX), which keeps this file free of inline
// assembly.
CpuFeatures probe_x86() noexcept {
CpuFeatures features;
features.sse2 = __builtin_cpu_supports("sse2") != 0;
features.avx2 = __builtin_cpu_supports("avx2") != 0;
features.avx512 = __builtin_cpu_supports("avx512f") != 0;
return features;
}
#endif
CpuFeatures probe() noexcept { return probe_x86(); }
// ----------------------------------------------------------------- AArch64 --
#elif defined(__aarch64__) || defined(_M_ARM64)
CpuFeatures probe() noexcept {
CpuFeatures features;
#if defined(__linux__)
// ASIMD is architectural for AArch64; the aux vector only confirms it.
features.neon = (getauxval(AT_HWCAP) & (1u << 1)) != 0u;
#else
features.neon = true;
#endif
return features;
}
// ------------------------------------------------------------ anything else --
#else
CpuFeatures probe() noexcept { return CpuFeatures{}; }
#endif
} // namespace
const CpuFeatures& cpu_features() noexcept {
static const CpuFeatures features = probe();
return features;
}
} // namespace joc::simd
+30
View File
@@ -0,0 +1,30 @@
#pragma once
// "Can this CPU and this operating system actually execute ISA X?"
//
// Its own unit and free of policy: it reads CPUID / XGETBV / HWCAP, caches the
// answer, and knows nothing about JOC_SIMD, the kernel table or which
// implementation is preferred. This is the one place that can fault the process
// if it is wrong, so it stays small enough to review on its own.
//
// Compiled with the baseline ISA like every other non-intrinsic unit (see
// CMakeLists.txt): it runs before any ISA unit is reached, so it must not execute
// a wide instruction itself.
namespace joc::simd {
// One flag per ISA the dispatcher can ask about. A flag means "usable here":
// both the silicon and the OS state-management agree, not merely that CPUID
// advertises the feature.
struct CpuFeatures {
bool sse2 = false;
bool avx2 = false;
bool avx512 = false;
bool neon = false;
};
// Probed once, on first use. Never throws and never reads the environment (the
// JOC_SIMD override is the dispatcher's business).
const CpuFeatures& cpu_features() noexcept;
} // namespace joc::simd
+293
View File
@@ -0,0 +1,293 @@
// Kernel dispatch: CPU features in, kernel table out.
//
// This file holds policy only -- which ISA is asked for, which ladder is walked,
// which slot ends up with which implementation. The question "can this machine
// run ISA X at all" belongs to cpu_probe.cpp, and the implementations themselves
// belong to the one ISA unit per ISA (kernels_scalar.cpp / kernels_intrin_*.cpp).
//
// JOC_SIMD=auto|scalar|sse2|avx2|avx512|neon. A forced ISA that this build or
// this machine cannot provide is reported on stderr and then ignored -- forcing a
// path the CPU lacks would fault instead of verifying anything.
//
// This unit is compiled with the baseline ISA: it runs before anything else in
// any binary that links the kernels, so it must not execute an ISA-specific
// instruction itself.
#include "simd/simd.h"
#include "simd/cpu_probe.h"
#include <cstdio>
#include <cstdlib>
#include <cstring>
// Which ISA units the build system put into this binary. An ISA that was not
// compiled in has no entry point to reference at all.
#if !defined(JOC_SIMD_HAVE_SSE2)
#define JOC_SIMD_HAVE_SSE2 0
#endif
#if !defined(JOC_SIMD_HAVE_AVX2)
#define JOC_SIMD_HAVE_AVX2 0
#endif
#if !defined(JOC_SIMD_HAVE_AVX512)
#define JOC_SIMD_HAVE_AVX512 0
#endif
#if !defined(JOC_SIMD_HAVE_NEON)
#define JOC_SIMD_HAVE_NEON 0
#endif
namespace joc::simd {
// Implemented by the per-ISA units; referenced only when compiled in. The file
// name carries the ISA (FLAC's `*_intrin_<isa>.c` convention), and the CMake list
// gives exactly those units their wider /arch or -m flag.
const Kernels& kernels_scalar() noexcept;
#if JOC_SIMD_HAVE_AVX2
const Kernels& kernels_intrin_avx2() noexcept;
#endif
#if JOC_SIMD_HAVE_AVX512
const Kernels& kernels_intrin_avx512() noexcept;
#endif
#if JOC_SIMD_HAVE_NEON
const Kernels& kernels_intrin_neon() noexcept;
#endif
namespace {
constexpr std::size_t kKernelCount = static_cast<std::size_t>(Kernel::count);
// The probe itself lives in cpu_probe.cpp; all this file needs from it is the
// cached answer, turned into the two questions the ladder asks: "was it compiled
// into this binary" (JOC_SIMD_HAVE_*) and "can this machine run it".
struct Resolution {
Kernels kernels;
Isa active[kKernelCount] = {};
Isa selected = Isa::scalar;
bool forced = false;
};
int isa_rank(Isa isa) noexcept {
switch (isa) {
case Isa::scalar: return 0;
case Isa::sse2: return 1;
case Isa::neon: return 1;
case Isa::avx2: return 2;
case Isa::avx512: return 3;
}
return 0;
}
bool isa_compiled(Isa isa) noexcept {
switch (isa) {
case Isa::scalar: return true;
// The x86-64 baseline *is* SSE2: the baseline units are compiled for it
// and there is no separate unit, because a 128-bit SSE2 register is
// exactly the register a scalar double already uses -- SSE2 cannot widen
// a double-precision operation, so "sse2" selects the reference kernels.
case Isa::sse2: return JOC_SIMD_HAVE_SSE2 != 0;
case Isa::avx2: return JOC_SIMD_HAVE_AVX2 != 0;
case Isa::avx512: return JOC_SIMD_HAVE_AVX512 != 0;
case Isa::neon: return JOC_SIMD_HAVE_NEON != 0;
}
return false;
}
const Kernels& kernels_of(Isa isa) noexcept {
switch (isa) {
case Isa::scalar:
case Isa::sse2: return kernels_scalar();
#if JOC_SIMD_HAVE_AVX2
case Isa::avx2: return kernels_intrin_avx2();
#endif
#if JOC_SIMD_HAVE_AVX512
case Isa::avx512: return kernels_intrin_avx512();
#endif
#if JOC_SIMD_HAVE_NEON
case Isa::neon: return kernels_intrin_neon();
#endif
default: break;
}
return kernels_scalar();
}
bool parse_isa(const char* text, Isa* out) noexcept {
struct Entry {
const char* name;
Isa isa;
};
static const Entry kEntries[] = {
{"scalar", Isa::scalar}, {"sse2", Isa::sse2}, {"avx2", Isa::avx2},
{"avx512", Isa::avx512}, {"neon", Isa::neon},
};
for (const Entry& entry : kEntries) {
if (std::strcmp(text, entry.name) == 0) {
*out = entry.isa;
return true;
}
}
return false;
}
Resolution resolve() noexcept {
Resolution resolution;
Isa best = Isa::scalar;
bool forced = false;
if (const char* requested = std::getenv("JOC_SIMD");
requested != nullptr && requested[0] != '\0' && std::strcmp(requested, "auto") != 0) {
Isa wanted = Isa::scalar;
if (!parse_isa(requested, &wanted)) {
std::fprintf(stderr,
"joc: JOC_SIMD=%s is not one of auto|scalar|sse2|avx2|avx512|neon; "
"using auto\n",
requested);
} else if (!isa_compiled(wanted)) {
std::fprintf(stderr, "joc: JOC_SIMD=%s is not part of this build; using auto\n",
requested);
} else if (!isa_supported(wanted)) {
std::fprintf(stderr, "joc: JOC_SIMD=%s is not supported by this CPU or OS; using auto\n",
requested);
} else {
best = wanted;
forced = true;
}
}
if (!forced) {
// Widest runnable ISA, in descending order.
static const Isa kLadder[] = {Isa::avx512, Isa::avx2, Isa::sse2, Isa::neon};
for (const Isa candidate : kLadder) {
if (isa_compiled(candidate) && isa_supported(candidate)) {
best = candidate;
break;
}
}
}
resolution.selected = best;
resolution.forced = forced;
// Per kernel: widest compiled + runnable implementation at or below the
// selected ISA, falling back to the scalar reference, which is always there.
static const Isa kCandidates[] = {Isa::avx512, Isa::avx2, Isa::sse2, Isa::neon, Isa::scalar};
for (std::size_t index = 0u; index < kKernelCount; ++index) {
const Kernel kernel = static_cast<Kernel>(index);
resolution.active[index] = Isa::scalar;
for (const Isa candidate : kCandidates) {
if (isa_rank(candidate) > isa_rank(best) || !isa_compiled(candidate)
|| !isa_supported(candidate)) {
continue;
}
const Kernels& set = kernels_of(candidate);
bool resolved = false;
switch (kernel) {
case Kernel::fft_butterflies:
if (set.fft_butterflies != nullptr) {
resolution.kernels.fft_butterflies = set.fft_butterflies;
resolved = true;
}
break;
case Kernel::qmf_synthesis_basis:
if (set.qmf_synthesis_basis != nullptr) {
resolution.kernels.qmf_synthesis_basis = set.qmf_synthesis_basis;
resolved = true;
}
break;
case Kernel::hybrid_low_join:
if (set.hybrid_low_join != nullptr) {
resolution.kernels.hybrid_low_join = set.hybrid_low_join;
resolved = true;
}
break;
case Kernel::render_hybrid_path:
if (set.render_hybrid_path != nullptr) {
resolution.kernels.render_hybrid_path = set.render_hybrid_path;
resolved = true;
}
break;
case Kernel::complex_axpy:
if (set.complex_axpy != nullptr) {
resolution.kernels.complex_axpy = set.complex_axpy;
resolved = true;
}
break;
case Kernel::qmf_analysis_taps:
if (set.qmf_analysis_taps != nullptr) {
resolution.kernels.qmf_analysis_taps = set.qmf_analysis_taps;
resolved = true;
}
break;
case Kernel::complex_multiply:
if (set.complex_multiply != nullptr) {
resolution.kernels.complex_multiply = set.complex_multiply;
resolved = true;
}
break;
case Kernel::count: break;
}
if (resolved) {
resolution.active[index] = candidate;
break;
}
}
}
if (std::getenv("JOC_SIMD_LOG") != nullptr) {
std::fprintf(stderr, "joc: simd selected=%s forced=%d\n", isa_name(resolution.selected),
resolution.forced ? 1 : 0);
for (std::size_t index = 0u; index < kKernelCount; ++index) {
std::fprintf(stderr, "joc: simd kernel[%u]=%s\n", static_cast<unsigned>(index),
isa_name(resolution.active[index]));
}
}
return resolution;
}
const Resolution& resolution() noexcept {
static const Resolution resolved = resolve();
return resolved;
}
} // namespace
const char* isa_name(Isa isa) noexcept {
switch (isa) {
case Isa::scalar: return "scalar";
case Isa::sse2: return "sse2";
case Isa::avx2: return "avx2";
case Isa::avx512: return "avx512";
case Isa::neon: return "neon";
}
return "?";
}
bool isa_supported(Isa isa) noexcept {
const CpuFeatures& cpu = cpu_features();
switch (isa) {
case Isa::scalar:
return true;
case Isa::sse2:
return cpu.sse2;
case Isa::avx2:
return cpu.avx2;
case Isa::avx512:
return cpu.avx512;
case Isa::neon:
return cpu.neon;
}
return false;
}
Isa selected_isa() noexcept { return resolution().selected; }
Isa active_isa(Kernel kernel) noexcept {
const std::size_t index = static_cast<std::size_t>(kernel);
if (index >= kKernelCount) {
return Isa::scalar;
}
return resolution().active[index];
}
const Kernels& kernels() noexcept { return resolution().kernels; }
} // namespace joc::simd
+321
View File
@@ -0,0 +1,321 @@
// AVX2 implementation of the dispatched kernels. Compiled with /arch:AVX2 (or
// -mavx2) and only ever called after the dispatcher has confirmed that this CPU
// and the OS state it saves can execute AVX2 code.
//
// CMake gives exactly the `kernels_intrin_*.cpp` units a wider flag, so no
// baseline unit can inherit one by accident.
//
// Nothing in this file may have dynamic initialisation: it is linked into the
// same image as the baseline code and would run before the dispatcher.
#include "simd/simd.h"
#include <immintrin.h>
#include <cstddef>
namespace joc::simd {
namespace {
// Two interleaved complexes per vector.
//
// The scalar kernel computes, for one butterfly,
// odd.re = o.re * w.re - o.im * w.im
// odd.im = o.re * w.im + o.im * w.re
// with separate multiplies and one rounding per operation. In a 256-bit vector
// the even element of every 128-bit half is a real part, so duplicating the even
// elements of the twiddle broadcasts w.re onto both components of a complex and
// permuting the odd data elements in lines up `o.im`, `o.re` against `w.im`.
// addsub then subtracts on the real lanes and adds on the imaginary ones, which
// is the same two roundings: `o.im * w.re + o.re * w.im` is the scalar's
// `o.re * w.im + o.im * w.re` with the summands exchanged, and floating-point
// addition is commutative.
inline __m256d mul_pair(__m256d odd, __m256d twiddle) noexcept {
const __m256d real = _mm256_mul_pd(odd, _mm256_movedup_pd(twiddle));
const __m256d cross = _mm256_mul_pd(_mm256_permute_pd(odd, 0x05), _mm256_permute_pd(twiddle, 0x0f));
return _mm256_addsub_pd(real, cross);
}
// The radix-2 DIT cascade over a bit-reversed buffer of any power-of-two size at
// or above kMinVectorFftSize.
//
// Stage s pairs element (start + offset) with (start + offset + 2^s) for every
// group of 2^(s+1) elements, and the butterflies of a stage are mutually
// independent: none of them reads a slot another one writes. They are therefore
// free to share a vector, and each one still accumulates nothing -- every output
// is a single add or subtract of two products, in lanes whose operations are
// independent of each other.
//
// Stage 0 is the one case whose group holds a single butterfly, so the pair is
// packed from two neighbouring groups instead: their factors are identical
// (offset is always 0), and a permute2f128 moves the even and odd halves into
// two vector registers.
void fft_butterflies_avx2(double* data, std::size_t size, const double* twiddle,
const std::size_t* stage_begin) noexcept {
{
const auto* pair = reinterpret_cast<const __m128d*>(twiddle + 2u * stage_begin[0]);
const __m256d factor = _mm256_broadcast_pd(pair);
for (std::size_t index = 0u; index < 2u * size; index += 8u) {
const __m256d low = _mm256_loadu_pd(data + index);
const __m256d high = _mm256_loadu_pd(data + index + 4u);
const __m256d even = _mm256_permute2f128_pd(low, high, 0x20);
const __m256d odd = _mm256_permute2f128_pd(low, high, 0x31);
const __m256d rotated = mul_pair(odd, factor);
const __m256d sum = _mm256_add_pd(even, rotated);
const __m256d difference = _mm256_sub_pd(even, rotated);
_mm256_storeu_pd(data + index, _mm256_permute2f128_pd(sum, difference, 0x20));
_mm256_storeu_pd(data + index + 4u, _mm256_permute2f128_pd(sum, difference, 0x31));
}
}
std::size_t stage = 1u;
for (std::size_t half = 2u; half < size; half <<= 1u, ++stage) {
const std::size_t length = half << 1u;
const double* table = twiddle + 2u * stage_begin[stage];
for (std::size_t start = 0u; start < size; start += length) {
for (std::size_t offset = 0u; offset < half; offset += 2u) {
const std::size_t even_index = (start + offset) * 2u;
const std::size_t odd_index = even_index + half * 2u;
const std::size_t factor_index = offset * 2u;
const __m256d even = _mm256_loadu_pd(data + even_index);
const __m256d odd = _mm256_loadu_pd(data + odd_index);
const __m256d factor = _mm256_loadu_pd(table + factor_index);
const __m256d rotated = mul_pair(odd, factor);
_mm256_storeu_pd(data + even_index, _mm256_add_pd(even, rotated));
_mm256_storeu_pd(data + odd_index, _mm256_sub_pd(even, rotated));
}
}
}
}
// QMF synthesis basis, four ranks of one band per vector.
//
// The four ranks are four dot products over the same 128 taps, so they are four
// independent accumulators and nothing has to be reassociated to fill a vector.
// Each lane performs exactly the caller's `sum += values[tap] * weight`, in tap
// order, with a separate multiply and add, and the rank-minor table makes the
// four weights one contiguous load.
//
// A band's chain is 128 dependent adds, so the loop is latency-bound long before
// it is throughput-bound; the cure is more independent chains, not a shorter
// chain (that would reassociate). `Rows` output rows share one weight load and
// run their chains side by side -- the caller's rows are consecutive output
// channels, which do read the same weights.
template <int Rows>
void synthesize_rows(const double* values, const double* basis, double* out) noexcept {
constexpr std::size_t kRanks = kSynthesisRanks;
constexpr std::size_t kTaps = kSynthesisTaps;
constexpr std::size_t kRowOut = kSynthesisBands * kRanks;
for (std::size_t band = 0u; band < kSynthesisBands; ++band) {
const double* table = basis + band * kTaps * kRanks;
__m256d sum[Rows];
for (int row = 0; row < Rows; ++row) {
sum[row] = _mm256_setzero_pd();
}
for (std::size_t tap = 0u; tap < kTaps; ++tap) {
const __m256d weight = _mm256_loadu_pd(table + tap * kRanks);
for (int row = 0; row < Rows; ++row) {
sum[row] = _mm256_add_pd(
sum[row],
_mm256_mul_pd(_mm256_broadcast_sd(values + row * kTaps + tap), weight));
}
}
for (int row = 0; row < Rows; ++row) {
_mm256_storeu_pd(out + row * kRowOut + band * kRanks, sum[row]);
}
}
}
void qmf_synthesis_basis_avx2(const double* values, const double* basis, double* out,
std::size_t rows) noexcept {
constexpr std::size_t kTaps = kSynthesisTaps;
constexpr std::size_t kRowOut = kSynthesisBands * kSynthesisRanks;
std::size_t row = 0u;
for (; row + 4u <= rows; row += 4u) {
synthesize_rows<4>(values + row * kTaps, basis, out + row * kRowOut);
}
for (; row < rows; ++row) {
synthesize_rows<1>(values + row * kTaps, basis, out + row * kRowOut);
}
}
// Hybrid analysis low join, one term against all 32 outputs.
//
// The 32 outputs of a row are independent accumulations over the same 78 terms,
// so they are what fills the four vectors: one broadcast of the term's value,
// then a multiply and an add per vector. Every lane keeps the term order and the
// two roundings of the caller's `out += value * weight`. Terms that are exactly
// zero are skipped, which is what the caller does and what leaves its sums
// unchanged (a lane's running sum is never a negative zero).
void hybrid_low_join_avx2(const double* values, const double* kernel, double* out,
std::size_t rows) noexcept {
constexpr std::size_t kTerms = kHybridTerms;
constexpr std::size_t kOutputs = kHybridOutputs;
static_assert(kOutputs == 32u, "eight 256-bit accumulators cover one row's outputs");
for (std::size_t row = 0u; row < rows; ++row) {
const double* source = values + row * kTerms;
double* destination = out + row * kOutputs;
__m256d sum0 = _mm256_setzero_pd();
__m256d sum1 = _mm256_setzero_pd();
__m256d sum2 = _mm256_setzero_pd();
__m256d sum3 = _mm256_setzero_pd();
__m256d sum4 = _mm256_setzero_pd();
__m256d sum5 = _mm256_setzero_pd();
__m256d sum6 = _mm256_setzero_pd();
__m256d sum7 = _mm256_setzero_pd();
for (std::size_t term = 0u; term < kTerms; ++term) {
const double value = source[term];
if (value == 0.0) {
continue;
}
const double* weights = kernel + term * kOutputs;
const __m256d factor = _mm256_broadcast_sd(&value);
sum0 = _mm256_add_pd(sum0, _mm256_mul_pd(factor, _mm256_loadu_pd(weights)));
sum1 = _mm256_add_pd(sum1, _mm256_mul_pd(factor, _mm256_loadu_pd(weights + 4u)));
sum2 = _mm256_add_pd(sum2, _mm256_mul_pd(factor, _mm256_loadu_pd(weights + 8u)));
sum3 = _mm256_add_pd(sum3, _mm256_mul_pd(factor, _mm256_loadu_pd(weights + 12u)));
sum4 = _mm256_add_pd(sum4, _mm256_mul_pd(factor, _mm256_loadu_pd(weights + 16u)));
sum5 = _mm256_add_pd(sum5, _mm256_mul_pd(factor, _mm256_loadu_pd(weights + 20u)));
sum6 = _mm256_add_pd(sum6, _mm256_mul_pd(factor, _mm256_loadu_pd(weights + 24u)));
sum7 = _mm256_add_pd(sum7, _mm256_mul_pd(factor, _mm256_loadu_pd(weights + 28u)));
}
_mm256_storeu_pd(destination, sum0);
_mm256_storeu_pd(destination + 4u, sum1);
_mm256_storeu_pd(destination + 8u, sum2);
_mm256_storeu_pd(destination + 12u, sum3);
_mm256_storeu_pd(destination + 16u, sum4);
_mm256_storeu_pd(destination + 20u, sum5);
_mm256_storeu_pd(destination + 24u, sum6);
_mm256_storeu_pd(destination + 28u, sum7);
}
}
// One rendered path, two bands per vector.
//
// A band's real and imaginary parts are adjacent, so a vector holds two whole
// bands and the cross terms are one swap away -- the same shape the FFT kernel
// uses. Two products are needed per component here (`h * t` and `h * swapped(t)`),
// each scaled on its own, and the real part is their difference while the
// imaginary part is their sum: addsub produces both pairings and a blend keeps
// one lane of each. Bands are independent accumulations into independent
// outputs, so this is still nothing but cross-band parallelism.
void render_hybrid_path_avx2(double* out, const double* transfer, const double* history0,
const double* history1, double scale) noexcept {
const __m256d factor = _mm256_set1_pd(scale);
const double* histories[2] = {history0, history1};
for (std::size_t ear = 0u; ear < 2u; ++ear) {
const double* source = histories[ear];
const double* taps = transfer + ear * kPathBands * 2u;
double* destination = out + ear * kPathBands * 2u;
std::size_t band = 0u;
for (; band + 2u <= kPathBands; band += 2u) {
const __m256d history = _mm256_loadu_pd(source + band * 2u);
const __m256d tap = _mm256_loadu_pd(taps + band * 2u);
const __m256d product = _mm256_mul_pd(_mm256_mul_pd(history, tap), factor);
const __m256d crossed = _mm256_mul_pd(
_mm256_mul_pd(history, _mm256_permute_pd(tap, 0x05)), factor);
const __m256d difference =
_mm256_addsub_pd(product, _mm256_permute_pd(product, 0x05));
const __m256d sum = _mm256_add_pd(crossed, _mm256_permute_pd(crossed, 0x05));
// `difference` carries the real parts in lanes 0 and 2, `sum` the
// imaginary ones in lanes 1 and 3 (four elements, so the blend takes a
// four-bit selector).
const __m256d result = _mm256_blend_pd(difference, sum, 0xa);
_mm256_storeu_pd(destination + band * 2u,
_mm256_add_pd(_mm256_loadu_pd(destination + band * 2u), result));
}
for (; band < kPathBands; ++band) {
const double source_real = source[band * 2u];
const double source_imag = source[band * 2u + 1u];
const double tap_real = taps[band * 2u];
const double tap_imag = taps[band * 2u + 1u];
destination[band * 2u] +=
source_real * tap_real * scale - source_imag * tap_imag * scale;
destination[band * 2u + 1u] +=
source_real * tap_imag * scale + source_imag * tap_real * scale;
}
}
}
// Scalar times a run of interleaved complexes, accumulated in place. The elements
// are independent, so two complexes (one 256-bit vector) go together and each
// component keeps the caller's own multiply-then-add.
void complex_axpy_avx2(double* out, const double* field, double scale,
std::size_t count) noexcept {
const __m256d factor = _mm256_set1_pd(scale);
std::size_t index = 0u;
for (; index + 2u <= count; index += 2u) {
const std::size_t offset = index * 2u;
_mm256_storeu_pd(out + offset,
_mm256_add_pd(_mm256_loadu_pd(out + offset),
_mm256_mul_pd(_mm256_loadu_pd(field + offset), factor)));
}
for (; index < count; ++index) {
out[index * 2u] += field[index * 2u] * scale;
out[index * 2u + 1u] += field[index * 2u + 1u] * scale;
}
}
// QMF analysis polyphase accumulate: the 64 bands of a row are 64 independent
// accumulations of one product each, and the coefficient row is reused by every
// row, so the rows stream past a first-level-cache-resident set of coefficients.
void qmf_analysis_taps_avx2(double* out, const double* source, const double* coefficients,
std::size_t rows) noexcept {
for (std::size_t row = 0u; row < rows; ++row) {
double* destination = out + row * kQmfAnalysisBands;
const double* values = source + row * kQmfAnalysisBands;
for (std::size_t band = 0u; band < kQmfAnalysisBands; band += 4u) {
_mm256_storeu_pd(destination + band,
_mm256_add_pd(_mm256_loadu_pd(destination + band),
_mm256_mul_pd(_mm256_loadu_pd(values + band),
_mm256_loadu_pd(coefficients + band))));
}
}
}
// Element-wise complex product of two runs, two complexes per vector. Both
// operands take part in every lane, so both products of the scalar formula are
// formed and addsub pairs the real difference with the imaginary sum: the same
// four multiplies and the same two roundings per component as `a * b`.
void complex_multiply_avx2(double* out, const double* left, const double* right,
std::size_t count) noexcept {
std::size_t index = 0u;
for (; index + 2u <= count; index += 2u) {
const std::size_t offset = index * 2u;
const __m256d first = _mm256_loadu_pd(left + offset);
const __m256d second = _mm256_loadu_pd(right + offset);
// (re*re, im*im) and (im*re, re*im): the first gives the real difference,
// the second the imaginary sum.
const __m256d product = _mm256_mul_pd(first, second);
const __m256d crossed = _mm256_mul_pd(_mm256_permute_pd(first, 0x05), second);
const __m256d difference =
_mm256_addsub_pd(product, _mm256_permute_pd(product, 0x05));
const __m256d total = _mm256_add_pd(crossed, _mm256_permute_pd(crossed, 0x05));
_mm256_storeu_pd(out + offset, _mm256_blend_pd(difference, total, 0xa));
}
for (; index < count; ++index) {
const std::size_t offset = index * 2u;
const double left_real = left[offset];
const double left_imag = left[offset + 1u];
const double right_real = right[offset];
const double right_imag = right[offset + 1u];
out[offset] = left_real * right_real - left_imag * right_imag;
out[offset + 1u] = left_real * right_imag + left_imag * right_real;
}
}
const Kernels kAvx2{
&fft_butterflies_avx2,
&qmf_synthesis_basis_avx2,
&hybrid_low_join_avx2,
&render_hybrid_path_avx2,
&complex_axpy_avx2,
&qmf_analysis_taps_avx2,
&complex_multiply_avx2,
};
} // namespace
const Kernels& kernels_intrin_avx2() noexcept { return kAvx2; }
} // namespace joc::simd
+272
View File
@@ -0,0 +1,272 @@
// AVX-512F implementation of the dispatched kernels. Compiled with /arch:AVX512
// (or -mavx512f) and only ever called after the dispatcher has confirmed AVX-512F
// *and* that the OS saves the opmask/ZMM state.
//
// CMake gives exactly the `kernels_intrin_*.cpp` units a wider flag, so no
// baseline unit can inherit one by accident.
//
// Nothing in this file may have dynamic initialisation: it is linked into the
// same image as the baseline code and would run before the dispatcher.
#include "simd/simd.h"
#include <immintrin.h>
#include <cstddef>
namespace joc::simd {
namespace {
// Four interleaved complexes per vector: a 512-bit register is four 128-bit
// lanes, each holding one (re, im) pair, so the AVX2 argument applies one lane
// wider -- movedup/permute broadcast the twiddle components inside every lane.
// AVX-512 never widened VADDSUBPD to 512 bits, so the real lanes take the
// difference and the imaginary ones the sum through merge masking: that is still
// one exact subtract and one exact add per output, with the same two roundings
// per component and no reassociation.
inline __m512d mul_quad(__m512d odd, __m512d twiddle) noexcept {
const __m512d real = _mm512_mul_pd(odd, _mm512_movedup_pd(twiddle));
const __m512d cross =
_mm512_mul_pd(_mm512_permute_pd(odd, 0x55), _mm512_permute_pd(twiddle, 0xff));
const __m512d upper = _mm512_mask_add_pd(real, 0xaa, real, cross);
return _mm512_mask_sub_pd(upper, 0x55, real, cross);
}
// Stages 0 and 1 hold fewer than four complexes in a group, so their butterflies
// are packed from neighbouring groups instead; the factor depends only on the
// offset inside a group, so one broadcast serves every packed lane.
//
// VSHUFF64X2 takes a two-bit selector per output lane, but its first two output
// lanes can only name lanes of its first operand and its last two only lanes of
// its second (verified against the instruction). These are the immediates the
// packings need, written as the four lane selections they spell out.
constexpr int kHalvesEven = 0x88; // (low.l0, low.l2, high.l0, high.l2) = x0,x2,x4,x6
constexpr int kHalvesOdd = 0xdd; // (low.l1, low.l3, high.l1, high.l3) = x1,x3,x5,x7
constexpr int kGroupsEven = 0x44; // (low.l0, low.l1, high.l0, high.l1)
constexpr int kGroupsOdd = 0xee; // (low.l2, low.l3, high.l2, high.l3)
constexpr int kSwapMiddle = 0xd8; // (a.l0, a.l2, a.l1, a.l3)
void pack_halves(double* buffer, __m512d factor) noexcept {
const __m512d low = _mm512_loadu_pd(buffer);
const __m512d high = _mm512_loadu_pd(buffer + 8u);
const __m512d even = _mm512_shuffle_f64x2(low, high, kHalvesEven);
const __m512d odd = _mm512_shuffle_f64x2(low, high, kHalvesOdd);
const __m512d rotated = mul_quad(odd, factor);
const __m512d sum = _mm512_add_pd(even, rotated);
const __m512d difference = _mm512_sub_pd(even, rotated);
// `sum` holds the results for the even positions of the four packed groups and
// `difference` those for the odd ones, and the buffer wants them interleaved
// again. One shuffle cannot do that, so each store pairs the two halves and
// then swaps the two middle lanes of the pair.
const __m512d pairs_low = _mm512_shuffle_f64x2(sum, difference, kGroupsEven);
const __m512d pairs_high = _mm512_shuffle_f64x2(sum, difference, kGroupsOdd);
_mm512_storeu_pd(buffer, _mm512_shuffle_f64x2(pairs_low, pairs_low, kSwapMiddle));
_mm512_storeu_pd(buffer + 8u, _mm512_shuffle_f64x2(pairs_high, pairs_high, kSwapMiddle));
}
void pack_groups(double* buffer, __m512d factor) noexcept {
const __m512d low = _mm512_loadu_pd(buffer);
const __m512d high = _mm512_loadu_pd(buffer + 8u);
const __m512d even = _mm512_shuffle_f64x2(low, high, kGroupsEven);
const __m512d odd = _mm512_shuffle_f64x2(low, high, kGroupsOdd);
const __m512d rotated = mul_quad(odd, factor);
const __m512d sum = _mm512_add_pd(even, rotated);
const __m512d difference = _mm512_sub_pd(even, rotated);
_mm512_storeu_pd(buffer, _mm512_shuffle_f64x2(sum, difference, kGroupsEven));
_mm512_storeu_pd(buffer + 8u, _mm512_shuffle_f64x2(sum, difference, kGroupsOdd));
}
void fft_butterflies_avx512(double* buffer, std::size_t size, const double* twiddle,
const std::size_t* stage_begin) noexcept {
// Stage 0: length 2, one butterfly per group, one factor for all of them.
{
const __m128d pair = _mm_loadu_pd(twiddle + 2u * stage_begin[0]);
const __m512d factor = _mm512_broadcast_f64x4(_mm256_broadcast_pd(&pair));
for (std::size_t index = 0u; index < 2u * size; index += 16u) {
pack_halves(buffer + index, factor);
}
}
// Stage 1: length 4, two butterflies per group and two factors.
{
const __m512d factor =
_mm512_broadcast_f64x4(_mm256_loadu_pd(twiddle + 2u * stage_begin[1]));
for (std::size_t index = 0u; index < 2u * size; index += 16u) {
pack_groups(buffer + index, factor);
}
}
std::size_t stage = 2u;
for (std::size_t half = 4u; half < size; half <<= 1u, ++stage) {
const std::size_t length = half << 1u;
const double* table = twiddle + 2u * stage_begin[stage];
for (std::size_t start = 0u; start < size; start += length) {
for (std::size_t offset = 0u; offset < half; offset += 4u) {
const std::size_t even_index = (start + offset) * 2u;
const std::size_t odd_index = even_index + half * 2u;
const std::size_t factor_index = offset * 2u;
const __m512d even = _mm512_loadu_pd(buffer + even_index);
const __m512d odd = _mm512_loadu_pd(buffer + odd_index);
const __m512d factor = _mm512_loadu_pd(table + factor_index);
const __m512d rotated = mul_quad(odd, factor);
_mm512_storeu_pd(buffer + even_index, _mm512_add_pd(even, rotated));
_mm512_storeu_pd(buffer + odd_index, _mm512_sub_pd(even, rotated));
}
}
}
}
// Scalar times a run of interleaved complexes: four complexes (eight doubles) per
// vector, every element an independent accumulation.
void complex_axpy_avx512(double* out, const double* field, double scale,
std::size_t count) noexcept {
const __m512d factor = _mm512_set1_pd(scale);
std::size_t index = 0u;
for (; index + 4u <= count; index += 4u) {
const std::size_t offset = index * 2u;
_mm512_storeu_pd(out + offset,
_mm512_add_pd(_mm512_loadu_pd(out + offset),
_mm512_mul_pd(_mm512_loadu_pd(field + offset), factor)));
}
for (; index < count; ++index) {
out[index * 2u] += field[index * 2u] * scale;
out[index * 2u + 1u] += field[index * 2u + 1u] * scale;
}
}
// QMF analysis polyphase accumulate, eight bands per vector.
void qmf_analysis_taps_avx512(double* out, const double* source, const double* coefficients,
std::size_t rows) noexcept {
for (std::size_t row = 0u; row < rows; ++row) {
double* destination = out + row * kQmfAnalysisBands;
const double* values = source + row * kQmfAnalysisBands;
for (std::size_t band = 0u; band < kQmfAnalysisBands; band += 8u) {
_mm512_storeu_pd(destination + band,
_mm512_add_pd(_mm512_loadu_pd(destination + band),
_mm512_mul_pd(_mm512_loadu_pd(values + band),
_mm512_loadu_pd(coefficients + band))));
}
}
}
// Hybrid analysis low join: 32 outputs are exactly four 512-bit vectors, so one
// broadcast of the term's value feeds four multiplies and four adds.
void hybrid_low_join_avx512(const double* values, const double* kernel, double* out,
std::size_t rows) noexcept {
constexpr std::size_t kTerms = kHybridTerms;
constexpr std::size_t kOutputs = kHybridOutputs;
static_assert(kOutputs == 32u, "four 512-bit accumulators cover one row");
for (std::size_t row = 0u; row < rows; ++row) {
const double* source = values + row * kTerms;
double* destination = out + row * kOutputs;
__m512d sum0 = _mm512_setzero_pd();
__m512d sum1 = _mm512_setzero_pd();
__m512d sum2 = _mm512_setzero_pd();
__m512d sum3 = _mm512_setzero_pd();
for (std::size_t term = 0u; term < kTerms; ++term) {
const double value = source[term];
if (value == 0.0) {
continue;
}
const double* weights = kernel + term * kOutputs;
const __m512d factor = _mm512_set1_pd(value);
sum0 = _mm512_add_pd(sum0, _mm512_mul_pd(factor, _mm512_loadu_pd(weights)));
sum1 = _mm512_add_pd(sum1, _mm512_mul_pd(factor, _mm512_loadu_pd(weights + 8u)));
sum2 = _mm512_add_pd(sum2, _mm512_mul_pd(factor, _mm512_loadu_pd(weights + 16u)));
sum3 = _mm512_add_pd(sum3, _mm512_mul_pd(factor, _mm512_loadu_pd(weights + 24u)));
}
_mm512_storeu_pd(destination, sum0);
_mm512_storeu_pd(destination + 8u, sum1);
_mm512_storeu_pd(destination + 16u, sum2);
_mm512_storeu_pd(destination + 24u, sum3);
}
}
// One rendered path, four bands per vector. AVX-512 has no 512-bit addsub, so the
// two pairings are built with merge masking -- still one exact subtract and one
// exact add per output, and the same separate scale multiply per product.
void render_hybrid_path_avx512(double* out, const double* transfer, const double* history0,
const double* history1, double scale) noexcept {
const __m512d factor = _mm512_set1_pd(scale);
const double* histories[2] = {history0, history1};
for (std::size_t ear = 0u; ear < 2u; ++ear) {
const double* source = histories[ear];
const double* taps = transfer + ear * kPathBands * 2u;
double* destination = out + ear * kPathBands * 2u;
std::size_t band = 0u;
for (; band + 4u <= kPathBands; band += 4u) {
const __m512d history = _mm512_loadu_pd(source + band * 2u);
const __m512d tap = _mm512_loadu_pd(taps + band * 2u);
const __m512d product = _mm512_mul_pd(
_mm512_mul_pd(history, tap), factor);
const __m512d crossed = _mm512_mul_pd(
_mm512_mul_pd(history, _mm512_permute_pd(tap, 0x55)), factor);
const __m512d swapped_product = _mm512_permute_pd(product, 0x55);
const __m512d swapped_crossed = _mm512_permute_pd(crossed, 0x55);
// The real component is the difference of the two products and sits in
// the even element of each complex, the imaginary one is the sum of the
// crossed pair and sits in the odd element -- so the subtract is masked
// to the even lanes and the blend keeps the sum on the odd ones.
const __m512d difference =
_mm512_mask_sub_pd(product, 0x55, product, swapped_product);
const __m512d total = _mm512_add_pd(crossed, swapped_crossed);
const __m512d result = _mm512_mask_blend_pd(0xaa, difference, total);
_mm512_storeu_pd(destination + band * 2u,
_mm512_add_pd(_mm512_loadu_pd(destination + band * 2u), result));
}
for (; band < kPathBands; ++band) {
const double source_real = source[band * 2u];
const double source_imag = source[band * 2u + 1u];
const double tap_real = taps[band * 2u];
const double tap_imag = taps[band * 2u + 1u];
destination[band * 2u] +=
source_real * tap_real * scale - source_imag * tap_imag * scale;
destination[band * 2u + 1u] +=
source_real * tap_imag * scale + source_imag * tap_real * scale;
}
}
}
// Element-wise complex product, four complexes per vector. AVX-512 has no 512-bit
// addsub, so the real difference is masked into the even lanes and the imaginary
// sum into the odd ones.
void complex_multiply_avx512(double* out, const double* left, const double* right,
std::size_t count) noexcept {
std::size_t index = 0u;
for (; index + 4u <= count; index += 4u) {
const std::size_t offset = index * 2u;
const __m512d first = _mm512_loadu_pd(left + offset);
const __m512d second = _mm512_loadu_pd(right + offset);
const __m512d product = _mm512_mul_pd(first, second);
const __m512d crossed = _mm512_mul_pd(_mm512_permute_pd(first, 0x55), second);
const __m512d difference =
_mm512_mask_sub_pd(product, 0x55, product, _mm512_permute_pd(product, 0x55));
const __m512d total = _mm512_add_pd(crossed, _mm512_permute_pd(crossed, 0x55));
_mm512_storeu_pd(out + offset, _mm512_mask_blend_pd(0xaa, difference, total));
}
for (; index < count; ++index) {
const std::size_t offset = index * 2u;
const double left_real = left[offset];
const double left_imag = left[offset + 1u];
const double right_real = right[offset];
const double right_imag = right[offset + 1u];
out[offset] = left_real * right_real - left_imag * right_imag;
out[offset + 1u] = left_real * right_imag + left_imag * right_real;
}
}
const Kernels kAvx512{
&fft_butterflies_avx512,
nullptr, // qmf_synthesis_basis: four ranks do not fill a 512-bit vector
&hybrid_low_join_avx512,
&render_hybrid_path_avx512,
&complex_axpy_avx512,
&qmf_analysis_taps_avx512,
&complex_multiply_avx512,
};
} // namespace
const Kernels& kernels_intrin_avx512() noexcept { return kAvx512; }
} // namespace joc::simd
+311
View File
@@ -0,0 +1,311 @@
// AArch64 NEON (ASIMD) implementation of the dispatched kernels.
//
// CMake gives exactly the `kernels_intrin_*.cpp` units their ISA flag, so no
// baseline unit can inherit one by accident.
//
// Unlike the x86 units this one needs no feature probe -- ASIMD is architectural
// for AArch64 -- but it is still only reached through the dispatcher, so
// JOC_SIMD=scalar, sse2 or neon all stay available on the same binary.
//
// A NEON register holds two doubles, so a vector carries two lanes where an AVX2
// vector carries four. The lanes are the same independent work items as in the
// AVX2 unit, and each lane performs exactly the scalar reference's operations in
// the scalar reference's order: separate multiplies and adds, never a fused
// multiply-add, and never a reassociated sum.
//
// Nothing in this file may have dynamic initialisation (it is linked into the
// same image as the baseline code and would run before the dispatcher).
//
// NOTE: the unit tests do not exercise this unit. It is built by the AArch64 CI
// job, and its bit-exactness is argued from the shared kernel contract (simd.h)
// rather than measured here.
#include "simd/simd.h"
#include <arm_neon.h>
#include <cstddef>
#include "foundation/fft.h"
namespace joc::simd {
namespace {
constexpr std::size_t kFftSize = dsp::kQmfFftSize;
static_assert(kFftSize == 128u, "the QMF transform is the 128-point case");
// ------------------------------------------------------------ radix-2 FFT ----
//
// Two butterflies per iteration, held as separate real/imaginary vectors: vld2q
// loads two interleaved complexes and deinterleaves them, vst2q puts them back.
// `sum = even + odd * w` and `difference = even - odd * w` are then two add/sub
// pairs, and the complex product is four multiplies and two add/subs, which is
// what the scalar kernel writes.
inline float64x2x2_t complex_mul(float64x2_t odd_real, float64x2_t odd_imag,
float64x2_t weight_real, float64x2_t weight_imag) {
float64x2x2_t result;
result.val[0] = vsubq_f64(vmulq_f64(odd_real, weight_real), vmulq_f64(odd_imag, weight_imag));
result.val[1] = vaddq_f64(vmulq_f64(odd_real, weight_imag), vmulq_f64(odd_imag, weight_real));
return result;
}
// Stages 0 and 1 hold fewer than two complexes per group, so their butterflies are
// packed from neighbouring groups; the factor depends only on the offset inside a
// group, so the two lanes share it.
void fft_butterflies_neon(double* buffer, std::size_t size, const double* twiddle,
const std::size_t* stage_begin) noexcept {
// Stage 0: length 2, one butterfly per group. The two lanes take the two
// halves of two neighbouring groups, which the transposes select.
{
const double* table = twiddle + 2u * stage_begin[0];
const float64x2_t weight_real = vdupq_n_f64(table[0]);
const float64x2_t weight_imag = vdupq_n_f64(table[1]);
for (std::size_t index = 0u; index < 2u * size; index += 8u) {
const float64x2_t first = vld1q_f64(buffer + index);
const float64x2_t second = vld1q_f64(buffer + index + 2u);
const float64x2_t third = vld1q_f64(buffer + index + 4u);
const float64x2_t fourth = vld1q_f64(buffer + index + 6u);
const float64x2_t even_real = vtrn1q_f64(first, third);
const float64x2_t even_imag = vtrn2q_f64(first, third);
const float64x2_t odd_real = vtrn1q_f64(second, fourth);
const float64x2_t odd_imag = vtrn2q_f64(second, fourth);
const float64x2x2_t rotated =
complex_mul(odd_real, odd_imag, weight_real, weight_imag);
const float64x2_t sum_real = vaddq_f64(even_real, rotated.val[0]);
const float64x2_t sum_imag = vaddq_f64(even_imag, rotated.val[1]);
const float64x2_t difference_real = vsubq_f64(even_real, rotated.val[0]);
const float64x2_t difference_imag = vsubq_f64(even_imag, rotated.val[1]);
// sum.* are the two groups' even-position results and difference.* their
// odd-position ones; a transpose of the two component vectors puts each
// complex back together, in the order the groups appear.
vst1q_f64(buffer + index, vtrn1q_f64(sum_real, sum_imag));
vst1q_f64(buffer + index + 2u, vtrn1q_f64(difference_real, difference_imag));
vst1q_f64(buffer + index + 4u, vtrn2q_f64(sum_real, sum_imag));
vst1q_f64(buffer + index + 6u, vtrn2q_f64(difference_real, difference_imag));
}
}
std::size_t stage = 1u;
for (std::size_t half = 2u; half < size; half <<= 1u, ++stage) {
const std::size_t length = half << 1u;
const double* table = twiddle + 2u * stage_begin[stage];
for (std::size_t start = 0u; start < size; start += length) {
for (std::size_t offset = 0u; offset < half; offset += 2u) {
double* even_values = buffer + (start + offset) * 2u;
double* odd_values = even_values + half * 2u;
const float64x2x2_t even = vld2q_f64(even_values);
const float64x2x2_t odd = vld2q_f64(odd_values);
const float64x2x2_t weight = vld2q_f64(table + offset * 2u);
const float64x2x2_t rotated =
complex_mul(odd.val[0], odd.val[1], weight.val[0], weight.val[1]);
float64x2x2_t sum;
float64x2x2_t difference;
sum.val[0] = vaddq_f64(even.val[0], rotated.val[0]);
sum.val[1] = vaddq_f64(even.val[1], rotated.val[1]);
difference.val[0] = vsubq_f64(even.val[0], rotated.val[0]);
difference.val[1] = vsubq_f64(even.val[1], rotated.val[1]);
vst2q_f64(even_values, sum);
vst2q_f64(odd_values, difference);
}
}
}
}
// --------------------------------------------------- QMF synthesis basis -----
//
// The four ranks of a band are four independent dot products over the same row, so
// two registers carry four lanes. `Rows` rows run side by side because a band's
// chain is 128 dependent adds: the cure is more independent chains, not a shorter
// chain.
template <int Rows>
void synthesize_rows_neon(const double* values, const double* basis, double* out) noexcept {
constexpr std::size_t kRanks = kSynthesisRanks;
constexpr std::size_t kTaps = kSynthesisTaps;
constexpr std::size_t kRowOut = kSynthesisBands * kRanks;
for (std::size_t band = 0u; band < kSynthesisBands; ++band) {
const double* table = basis + band * kTaps * kRanks;
float64x2_t low[Rows];
float64x2_t high[Rows];
for (int row = 0; row < Rows; ++row) {
low[row] = vdupq_n_f64(0.0);
high[row] = vdupq_n_f64(0.0);
}
for (std::size_t tap = 0u; tap < kTaps; ++tap) {
const float64x2_t weight_low = vld1q_f64(table + tap * kRanks);
const float64x2_t weight_high = vld1q_f64(table + tap * kRanks + 2u);
for (int row = 0; row < Rows; ++row) {
const float64x2_t factor = vdupq_n_f64(values[row * kTaps + tap]);
low[row] = vaddq_f64(low[row], vmulq_f64(factor, weight_low));
high[row] = vaddq_f64(high[row], vmulq_f64(factor, weight_high));
}
}
for (int row = 0; row < Rows; ++row) {
vst1q_f64(out + row * kRowOut + band * kRanks, low[row]);
vst1q_f64(out + row * kRowOut + band * kRanks + 2u, high[row]);
}
}
}
void qmf_synthesis_basis_neon(const double* values, const double* basis, double* out,
std::size_t rows) noexcept {
constexpr std::size_t kTaps = kSynthesisTaps;
constexpr std::size_t kRowOut = kSynthesisBands * kSynthesisRanks;
std::size_t row = 0u;
for (; row + 4u <= rows; row += 4u) {
synthesize_rows_neon<4>(values + row * kTaps, basis, out + row * kRowOut);
}
for (; row < rows; ++row) {
synthesize_rows_neon<1>(values + row * kTaps, basis, out + row * kRowOut);
}
}
// ------------------------------------------------------ Hybrid low join -----
//
// One term against all 32 outputs, two outputs per register: the outputs are
// independent accumulations over the same terms, which is what the lanes carry.
void hybrid_low_join_neon(const double* values, const double* kernel, double* out,
std::size_t rows) noexcept {
constexpr std::size_t kTerms = kHybridTerms;
constexpr std::size_t kOutputs = kHybridOutputs;
static_assert(kOutputs == 32u, "sixteen 128-bit accumulators cover one row");
for (std::size_t row = 0u; row < rows; ++row) {
const double* source = values + row * kTerms;
double* destination = out + row * kOutputs;
float64x2_t partial[16];
for (int chunk = 0; chunk < 16; ++chunk) {
partial[chunk] = vdupq_n_f64(0.0);
}
for (std::size_t term = 0u; term < kTerms; ++term) {
const double value = source[term];
if (value == 0.0) {
continue;
}
const double* weights = kernel + term * kOutputs;
const float64x2_t factor = vdupq_n_f64(value);
for (int chunk = 0; chunk < 16; ++chunk) {
partial[chunk] = vaddq_f64(
partial[chunk], vmulq_f64(factor, vld1q_f64(weights + chunk * 2)));
}
}
for (int chunk = 0; chunk < 16; ++chunk) {
vst1q_f64(destination + chunk * 2, partial[chunk]);
}
}
}
// --------------------------------------------------- Rendered hybrid path ---
//
// Two bands per iteration, real and imaginary held apart and reloaded from memory
// in the caller's own layout: `out += h * t * scale` for the real part and
// `out += h * swapped(t) * scale` for the imaginary one, with the scale applied to
// each product separately.
void render_hybrid_path_neon(double* out, const double* transfer, const double* history0,
const double* history1, double scale) noexcept {
const float64x2_t factor = vdupq_n_f64(scale);
const double* histories[2] = {history0, history1};
for (std::size_t ear = 0u; ear < 2u; ++ear) {
const double* source = histories[ear];
const double* taps = transfer + ear * kPathBands * 2u;
double* destination = out + ear * kPathBands * 2u;
std::size_t band = 0u;
for (; band + 2u <= kPathBands; band += 2u) {
const float64x2x2_t history = vld2q_f64(source + band * 2u);
const float64x2x2_t tap = vld2q_f64(taps + band * 2u);
float64x2x2_t output = vld2q_f64(destination + band * 2u);
// The caller scales each product on its own before combining, so the
// scale is a separate multiply of every product -- not a multiply of the
// difference and the sum, which would round differently.
const float64x2_t real = vsubq_f64(
vmulq_f64(vmulq_f64(history.val[0], tap.val[0]), factor),
vmulq_f64(vmulq_f64(history.val[1], tap.val[1]), factor));
const float64x2_t imaginary = vaddq_f64(
vmulq_f64(vmulq_f64(history.val[0], tap.val[1]), factor),
vmulq_f64(vmulq_f64(history.val[1], tap.val[0]), factor));
output.val[0] = vaddq_f64(output.val[0], real);
output.val[1] = vaddq_f64(output.val[1], imaginary);
vst2q_f64(destination + band * 2u, output);
}
for (; band < kPathBands; ++band) {
const double source_real = source[band * 2u];
const double source_imag = source[band * 2u + 1u];
const double tap_real = taps[band * 2u];
const double tap_imag = taps[band * 2u + 1u];
destination[band * 2u] +=
source_real * tap_real * scale - source_imag * tap_imag * scale;
destination[band * 2u + 1u] +=
source_real * tap_imag * scale + source_imag * tap_real * scale;
}
}
}
// Scalar times a run of interleaved complexes, two complexes per iteration.
void complex_axpy_neon(double* out, const double* field, double scale,
std::size_t count) noexcept {
const float64x2_t factor = vdupq_n_f64(scale);
std::size_t index = 0u;
for (; index + 2u <= count; index += 2u) {
const float64x2_t values = vld1q_f64(field + index * 2u);
const float64x2_t accumulated = vld1q_f64(out + index * 2u);
vst1q_f64(out + index * 2u, vaddq_f64(accumulated, vmulq_f64(values, factor)));
}
for (; index < count; ++index) {
out[index * 2u] += field[index * 2u] * scale;
out[index * 2u + 1u] += field[index * 2u + 1u] * scale;
}
}
// QMF analysis polyphase accumulate, two bands per iteration.
void qmf_analysis_taps_neon(double* out, const double* source, const double* coefficients,
std::size_t rows) noexcept {
for (std::size_t row = 0u; row < rows; ++row) {
double* destination = out + row * kQmfAnalysisBands;
const double* values = source + row * kQmfAnalysisBands;
for (std::size_t band = 0u; band < kQmfAnalysisBands; band += 2u) {
vst1q_f64(destination + band,
vaddq_f64(vld1q_f64(destination + band),
vmulq_f64(vld1q_f64(values + band),
vld1q_f64(coefficients + band))));
}
}
}
// Element-wise complex product of two runs, two complexes per iteration.
void complex_multiply_neon(double* out, const double* left, const double* right,
std::size_t count) noexcept {
std::size_t index = 0u;
for (; index + 2u <= count; index += 2u) {
const float64x2x2_t first = vld2q_f64(left + index * 2u);
const float64x2x2_t second = vld2q_f64(right + index * 2u);
float64x2x2_t result;
result.val[0] = vsubq_f64(vmulq_f64(first.val[0], second.val[0]),
vmulq_f64(first.val[1], second.val[1]));
result.val[1] = vaddq_f64(vmulq_f64(first.val[0], second.val[1]),
vmulq_f64(first.val[1], second.val[0]));
vst2q_f64(out + index * 2u, result);
}
for (; index < count; ++index) {
const std::size_t offset = index * 2u;
const double left_real = left[offset];
const double left_imag = left[offset + 1u];
const double right_real = right[offset];
const double right_imag = right[offset + 1u];
out[offset] = left_real * right_real - left_imag * right_imag;
out[offset + 1u] = left_real * right_imag + left_imag * right_real;
}
}
const Kernels kNeon{
&fft_butterflies_neon,
&qmf_synthesis_basis_neon,
&hybrid_low_join_neon,
&render_hybrid_path_neon,
&complex_axpy_neon,
&qmf_analysis_taps_neon,
&complex_multiply_neon,
};
} // namespace
const Kernels& kernels_intrin_neon() noexcept { return kNeon; }
} // namespace joc::simd
+156
View File
@@ -0,0 +1,156 @@
// Scalar reference implementations of the dispatched DSP kernels.
//
// This unit is compiled with the baseline ISA of the target (no /arch flag on
// MSVC, no -m flag elsewhere) and is the ground truth every vector path is
// compared against: JOC_SIMD=scalar selects exactly these functions, and their
// output must be bit-identical to any ISA path's.
//
// The arithmetic below is deliberately written with std::complex<double>, which
// on MSVC expands to the plain `re * rr - im * ri` / `re * ri + im * rr` pair
// (no helper call, no scaling trick), so the vector paths have an unambiguous
// two-roundings-per-component target to reproduce.
#include "simd/simd.h"
#include <complex>
#include "foundation/fft.h"
namespace joc::simd {
namespace {
using Complex = dsp::Complex;
// Exactly the stage loop of FftPlan::apply, over a table this kernel does not own,
// for any power-of-two size. The bit-reversal permutation stays with the caller:
// both call sites already write their input in permuted order.
void fft_butterflies_scalar(double* raw_data, std::size_t size, const double* raw_twiddle,
const std::size_t* stage_begin) noexcept {
auto* data = reinterpret_cast<Complex*>(raw_data);
const auto* twiddle = reinterpret_cast<const Complex*>(raw_twiddle);
std::size_t stage = 0u;
for (std::size_t length = 2u; length <= size; length <<= 1u, ++stage) {
const Complex* table = twiddle + stage_begin[stage];
for (std::size_t start = 0u; start < size; start += length) {
for (std::size_t offset = 0u; offset < length / 2u; ++offset) {
const Complex even = data[start + offset];
const Complex odd = data[start + offset + length / 2u] * table[offset];
data[start + offset] = even + odd;
data[start + offset + length / 2u] = even - odd;
}
}
}
}
// The reference for the synthesis basis: one accumulator per output, taps in
// increasing order, one rounding per multiply and per add. This is the shape
// both callers write, with the basis rows reordered rank-minor -- the products
// and their order are the caller's.
void qmf_synthesis_basis_scalar(const double* values, const double* basis, double* out,
std::size_t rows) noexcept {
for (std::size_t row = 0u; row < rows; ++row) {
const double* source = values + row * kSynthesisTaps;
double* destination = out + row * kSynthesisBands * kSynthesisRanks;
for (std::size_t band = 0u; band < kSynthesisBands; ++band) {
const double* table = basis + band * kSynthesisTaps * kSynthesisRanks;
for (std::size_t rank = 0u; rank < kSynthesisRanks; ++rank) {
double sum = 0.0;
for (std::size_t tap = 0u; tap < kSynthesisTaps; ++tap) {
sum += source[tap] * table[tap * kSynthesisRanks + rank];
}
destination[band * kSynthesisRanks + rank] = sum;
}
}
}
}
// The reference for the hybrid low join: one accumulator per output, terms in the
// caller's order, one rounding per multiply and per add, zero terms skipped.
void hybrid_low_join_scalar(const double* values, const double* kernel, double* out,
std::size_t rows) noexcept {
for (std::size_t row = 0u; row < rows; ++row) {
const double* source = values + row * kHybridTerms;
double* destination = out + row * kHybridOutputs;
for (std::size_t output = 0u; output < kHybridOutputs; ++output) {
destination[output] = 0.0;
}
for (std::size_t term = 0u; term < kHybridTerms; ++term) {
const double value = source[term];
if (value == 0.0) {
continue;
}
const double* weights = kernel + term * kHybridOutputs;
for (std::size_t output = 0u; output < kHybridOutputs; ++output) {
destination[output] += value * weights[output];
}
}
}
}
// The reference for one rendered path: each band of each ear accumulates its own
// value, in the caller's order, with the scale applied to each product.
void render_hybrid_path_scalar(double* out, const double* transfer, const double* history0,
const double* history1, double scale) noexcept {
const double* histories[2] = {history0, history1};
for (std::size_t ear = 0u; ear < 2u; ++ear) {
const double* source = histories[ear];
const double* taps = transfer + ear * kPathBands * 2u;
double* destination = out + ear * kPathBands * 2u;
for (std::size_t band = 0u; band < kPathBands; ++band) {
const double source_real = source[band * 2u];
const double source_imag = source[band * 2u + 1u];
const double tap_real = taps[band * 2u];
const double tap_imag = taps[band * 2u + 1u];
destination[band * 2u] +=
source_real * tap_real * scale - source_imag * tap_imag * scale;
destination[band * 2u + 1u] +=
source_real * tap_imag * scale + source_imag * tap_real * scale;
}
}
}
// Neighbouring complexes and neighbouring bands are independent outputs, so the
// references for the two accumulates are plain element-wise loops.
void complex_axpy_scalar(double* out, const double* field, double scale,
std::size_t count) noexcept {
for (std::size_t index = 0u; index < count * 2u; ++index) {
out[index] += field[index] * scale;
}
}
void qmf_analysis_taps_scalar(double* out, const double* source, const double* coefficients,
std::size_t rows) noexcept {
for (std::size_t row = 0u; row < rows; ++row) {
double* destination = out + row * kQmfAnalysisBands;
const double* values = source + row * kQmfAnalysisBands;
for (std::size_t band = 0u; band < kQmfAnalysisBands; ++band) {
destination[band] += values[band] * coefficients[band];
}
}
}
void complex_multiply_scalar(double* out, const double* left, const double* right,
std::size_t count) noexcept {
auto* destination = reinterpret_cast<Complex*>(out);
const auto* first = reinterpret_cast<const Complex*>(left);
const auto* second = reinterpret_cast<const Complex*>(right);
for (std::size_t index = 0u; index < count; ++index) {
destination[index] = first[index] * second[index];
}
}
const Kernels kScalar{
&fft_butterflies_scalar,
&qmf_synthesis_basis_scalar,
&hybrid_low_join_scalar,
&render_hybrid_path_scalar,
&complex_axpy_scalar,
&qmf_analysis_taps_scalar,
&complex_multiply_scalar,
};
} // namespace
const Kernels& kernels_scalar() noexcept { return kScalar; }
} // namespace joc::simd
+209
View File
@@ -0,0 +1,209 @@
#pragma once
#include <cstddef>
// Runtime-dispatched SIMD kernels for the DSP core.
//
// Module layout (the ISA lives in the file name, never in a subdirectory):
//
// simd.h this contract -- the only header a caller includes
// cpu_probe.{h,cpp} "can this CPU and OS run ISA X", no policy
// dispatch.cpp JOC_SIMD + the ladder + the kernel table
// kernels_scalar.cpp the reference implementation, Isa::scalar
// kernels_intrin_avx2.cpp /arch:AVX2 -mavx2 -- exactly the
// kernels_intrin_avx512.cpp /arch:AVX512 -mavx512f -- `kernels_intrin_*`
// kernels_intrin_neon.cpp AArch64 default -- units get a wider flag
//
// The scalar reference in kernels_scalar.cpp fixes both the data layout and the
// arithmetic: a vector path may only compute *independent* outputs in parallel
// lanes. It may never reassociate an accumulation, contract a multiply into an
// FMA, or change the order in which one output's terms are summed, so every ISA
// path reproduces the scalar bits exactly and JOC_SIMD=scalar is a valid
// reference for any of them. Kernel-level layout changes (AoS -> SoA inside a
// kernel, for instance) are fine as long as each individual output value is
// still produced by the same sequence of roundings.
//
// MSVC has no function-level ISA attribute, so each ISA lives in its own
// translation unit compiled with its own /arch (or -m) flag and the dispatcher
// picks one at run time: CPUID + XGETBV on x86-64, the architectural AArch64
// baseline (NEON/ASIMD needs no probing) elsewhere.
//
// The ISA units are linked into the same binaries as the baseline code, so they
// must stay free of objects with dynamic initialisation: a global constructor
// would execute ISA-specific instructions before the dispatcher has had a chance
// to look at the CPU. Constant tables are fine -- they live in .rdata.
namespace joc::simd {
enum class Isa {
scalar = 0,
sse2,
avx2,
avx512,
neon,
};
// One entry per dispatched kernel.
enum class Kernel {
// Power-of-two radix-2 DIT butterfly cascade, in place, over `size` interleaved
// (re, im) pairs already in bit-reversed order. `twiddle` is the caller's own
// factor table, read as interleaved pairs: stage s (length 2^(s+1)) takes its
// 2^s factors starting at stage_begin[s], a *complex* index into `twiddle`.
fft_butterflies = 0,
// QMF synthesis basis application, see kSynthesis* below.
qmf_synthesis_basis,
// Hybrid analysis low join, see kHybrid* below.
hybrid_low_join,
// Hybrid-domain path rendering, see kPathBands below.
render_hybrid_path,
// Scalar-times-vector accumulate over interleaved complexes.
complex_axpy,
// QMF analysis polyphase accumulate, see kQmfAnalysisBands below.
qmf_analysis_taps,
// Element-wise complex product of two whole runs.
complex_multiply,
count,
};
// Both filterbanks apply the same map: for every output row and every one of the
// 64 bands, four ranks each accumulate 128 taps.
inline constexpr std::size_t kSynthesisBands = 64;
inline constexpr std::size_t kSynthesisRanks = 4;
inline constexpr std::size_t kSynthesisTaps = 128;
// ... and both derive the same 16 low hybrid bands from 78 terms: 3 parents x 2
// components x 13 lags.
inline constexpr std::size_t kHybridTerms = 78;
inline constexpr std::size_t kHybridOutputs = 32;
// Rows the callers stage at a time: enough to keep the kernel busy, small enough
// that the staged values stay in the first-level cache.
inline constexpr std::size_t kHybridJoinBlock = 32;
// Bands in one ear of a rendered path (the SOFA renderer's 77-band hybrid domain).
inline constexpr std::size_t kPathBands = 77;
// The cascade packs whole groups into a vector, so it needs at least one full
// packing: a size of 8 covers a 512-bit stage. Smaller transforms are rare enough
// (and short enough) that the caller's portable loop is the right answer there.
inline constexpr std::size_t kMinVectorFftSize = 8;
// Bands in one QMF analysis row (the polyphase accumulator of both filterbanks).
inline constexpr std::size_t kQmfAnalysisBands = 64;
const char* isa_name(Isa isa) noexcept;
// Whether this CPU *and* the OS state it has to save can execute the ISA. The
// answer never changes, so it is computed once.
bool isa_supported(Isa isa) noexcept;
// The ISA the dispatcher settled on, after applying a JOC_SIMD override. This
// is the widest ISA considered, not necessarily the one that serves every
// kernel: active_isa() answers that per kernel.
Isa selected_isa() noexcept;
// Which ISA actually implements `kernel` after the per-kernel fallback.
Isa active_isa(Kernel kernel) noexcept;
// The dispatched kernels; a slot is never null.
struct Kernels {
void (*fft_butterflies)(double* data, std::size_t size, const double* twiddle,
const std::size_t* stage_begin) noexcept = nullptr;
// values: [rows][kSynthesisTaps] real/imaginary of the 64 bands of one row.
// basis: the same weights the caller holds, reordered rank-minor, so that
// (band, tap) addresses its kSynthesisRanks weights contiguously:
// basis[(band * kSynthesisTaps + tap) * kSynthesisRanks + rank].
// out: [rows][kSynthesisBands][kSynthesisRanks], rank-minor as well.
//
// Each of the four ranks is an independent dot product over the row, so the
// four of them are what shares a vector; every lane keeps the tap order and
// the two roundings of the caller's `sum += values[tap] * weight`.
void (*qmf_synthesis_basis)(const double* values, const double* basis, double* out,
std::size_t rows) noexcept = nullptr;
// Hybrid analysis low join. Every output row accumulates kHybridTerms values
// into kHybridOutputs outputs (16 bands x a real/imaginary pair, component
// minor):
// out[row][output] = sum over term of values[row][term] * kernel[term][output]
// values is [rows][kHybridTerms] in the caller's own term order and kernel is
// the caller's taps regrouped to that same order, so that one term's 32
// weights are contiguous. Terms whose value is exactly zero are skipped, as
// both callers do: adding a zero product to a lane can only leave it alone
// (no lane's running sum can be a negative zero, since it starts at +0 and
// sums without ever producing one).
void (*hybrid_low_join)(const double* values, const double* kernel, double* out,
std::size_t rows) noexcept = nullptr;
// One rendered path. For each ear e and band b, with the ear's history sample
// h and the path's transfer t:
// out[e][b].re += h.re * t[e][b].re * scale - h.im * t[e][b].im * scale
// out[e][b].im += h.re * t[e][b].im * scale + h.im * t[e][b].re * scale
// `history0` and `history1` are the two ears' [kPathBands] interleaved complex
// rows (they come from different places in the history ring); `transfer` and
// `out` are [2][kPathBands] interleaved complexes. The scale multiplies each
// product separately, exactly as the caller writes it, so the bands -- which
// are independent accumulations into independent outputs -- are what the lanes
// carry.
void (*render_hybrid_path)(double* out, const double* transfer, const double* history0,
const double* history1, double scale) noexcept = nullptr;
// `count` interleaved complexes, accumulated in place:
// out[i] += field[i] * scale
// Every element is an independent accumulation of one product, so the lanes
// carry neighbouring elements and each one keeps the caller's multiply-then-add.
void (*complex_axpy)(double* out, const double* field, double scale,
std::size_t count) noexcept = nullptr;
// QMF analysis polyphase accumulate: `rows` rows of kQmfAnalysisBands bands,
// out[row][band] += source[row][band] * coefficients[band]
// Neighbouring bands are neighbouring outputs, so they are what fills a vector;
// each band accumulates its own product once, in the caller's order.
void (*qmf_analysis_taps)(double* out, const double* source, const double* coefficients,
std::size_t rows) noexcept = nullptr;
// `count` interleaved complexes, multiplied element by element:
// out[i] = left[i] * right[i]
// with the caller's `re * re - im * im` and `re * im + im * re`, two roundings
// per component. Neighbouring complexes are independent products.
void (*complex_multiply)(double* out, const double* left, const double* right,
std::size_t count) noexcept = nullptr;
};
const Kernels& kernels() noexcept;
// Convenience wrappers.
inline void fft_butterflies(double* data, std::size_t size, const double* twiddle,
const std::size_t* stage_begin) noexcept {
kernels().fft_butterflies(data, size, twiddle, stage_begin);
}
inline void qmf_synthesis_basis(const double* values, const double* basis, double* out,
std::size_t rows) noexcept {
kernels().qmf_synthesis_basis(values, basis, out, rows);
}
inline void hybrid_low_join(const double* values, const double* kernel, double* out,
std::size_t rows) noexcept {
kernels().hybrid_low_join(values, kernel, out, rows);
}
inline void render_hybrid_path(double* out, const double* transfer, const double* history0,
const double* history1, double scale) noexcept {
kernels().render_hybrid_path(out, transfer, history0, history1, scale);
}
inline void complex_axpy(double* out, const double* field, double scale,
std::size_t count) noexcept {
kernels().complex_axpy(out, field, scale, count);
}
inline void qmf_analysis_taps(double* out, const double* source, const double* coefficients,
std::size_t rows) noexcept {
kernels().qmf_analysis_taps(out, source, coefficients, rows);
}
inline void complex_multiply(double* out, const double* left, const double* right,
std::size_t count) noexcept {
kernels().complex_multiply(out, left, right, count);
}
} // namespace joc::simd