commit
ad62a914e6
127 files changed
+38545
No files matched your search
@@ -0,0 +1,101 @@
|
||||
// CPU feature probe; see cpu_probe.h.
|
||||
//
|
||||
// Nothing here is ISA-specific code: the x86 path uses CPUID / XGETBV (MSVC) or
|
||||
// the compiler's own runtime probe (GCC/Clang, so no inline assembly), and the
|
||||
// AArch64 path reads the aux vector. That is what lets this unit stay baseline
|
||||
// and run before the dispatcher has decided anything.
|
||||
|
||||
#include "simd/cpu_probe.h"
|
||||
|
||||
#if defined(_M_X64)
|
||||
#include <immintrin.h>
|
||||
#include <intrin.h>
|
||||
#elif defined(__x86_64__)
|
||||
#include <cpuid.h>
|
||||
#endif
|
||||
|
||||
#if defined(__linux__) && (defined(__aarch64__) || defined(_M_ARM64))
|
||||
#include <sys/auxv.h>
|
||||
#endif
|
||||
|
||||
namespace joc::simd {
|
||||
namespace {
|
||||
|
||||
// ------------------------------------------------------------------- x86-64 --
|
||||
#if defined(_M_X64) || defined(__x86_64__)
|
||||
|
||||
#if defined(_M_X64)
|
||||
|
||||
// CPUID tells us what the silicon can do; XCR0 tells us whether the OS saves the
|
||||
// state the wider registers need. Both have to agree, or the first AVX
|
||||
// instruction after a context switch corrupts another thread's registers.
|
||||
CpuFeatures probe_x86() noexcept {
|
||||
CpuFeatures features;
|
||||
int regs[4] = {0, 0, 0, 0};
|
||||
__cpuid(regs, 0);
|
||||
const int max_leaf = regs[0];
|
||||
__cpuid(regs, 1);
|
||||
features.sse2 = (regs[3] & (1 << 26)) != 0;
|
||||
const bool osxsave = (regs[2] & (1 << 27)) != 0;
|
||||
const bool avx = (regs[2] & (1 << 28)) != 0;
|
||||
if (!osxsave || !avx || max_leaf < 7) {
|
||||
return features;
|
||||
}
|
||||
const unsigned long long xcr0 = _xgetbv(0);
|
||||
const bool ymm_state = (xcr0 & 0x06ull) == 0x06ull; // XMM + YMM saved
|
||||
const bool zmm_state = (xcr0 & 0xe6ull) == 0xe6ull; // + opmask / ZMM / hi16
|
||||
if (!ymm_state) {
|
||||
return features;
|
||||
}
|
||||
__cpuidex(regs, 7, 0);
|
||||
features.avx2 = (regs[1] & (1 << 5)) != 0;
|
||||
features.avx512 = zmm_state && (regs[1] & (1 << 16)) != 0; // AVX512F
|
||||
return features;
|
||||
}
|
||||
|
||||
#else // GCC/Clang on x86-64
|
||||
|
||||
// The compiler runtime performs the same CPUID + XGETBV probe (libgcc's cpuinfo
|
||||
// checks XCR0 before it reports AVX), which keeps this file free of inline
|
||||
// assembly.
|
||||
CpuFeatures probe_x86() noexcept {
|
||||
CpuFeatures features;
|
||||
features.sse2 = __builtin_cpu_supports("sse2") != 0;
|
||||
features.avx2 = __builtin_cpu_supports("avx2") != 0;
|
||||
features.avx512 = __builtin_cpu_supports("avx512f") != 0;
|
||||
return features;
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
CpuFeatures probe() noexcept { return probe_x86(); }
|
||||
|
||||
// ----------------------------------------------------------------- AArch64 --
|
||||
#elif defined(__aarch64__) || defined(_M_ARM64)
|
||||
|
||||
CpuFeatures probe() noexcept {
|
||||
CpuFeatures features;
|
||||
#if defined(__linux__)
|
||||
// ASIMD is architectural for AArch64; the aux vector only confirms it.
|
||||
features.neon = (getauxval(AT_HWCAP) & (1u << 1)) != 0u;
|
||||
#else
|
||||
features.neon = true;
|
||||
#endif
|
||||
return features;
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------ anything else --
|
||||
#else
|
||||
|
||||
CpuFeatures probe() noexcept { return CpuFeatures{}; }
|
||||
|
||||
#endif
|
||||
|
||||
} // namespace
|
||||
|
||||
const CpuFeatures& cpu_features() noexcept {
|
||||
static const CpuFeatures features = probe();
|
||||
return features;
|
||||
}
|
||||
|
||||
} // namespace joc::simd
|
||||
@@ -0,0 +1,30 @@
|
||||
#pragma once
|
||||
|
||||
// "Can this CPU and this operating system actually execute ISA X?"
|
||||
//
|
||||
// Its own unit and free of policy: it reads CPUID / XGETBV / HWCAP, caches the
|
||||
// answer, and knows nothing about JOC_SIMD, the kernel table or which
|
||||
// implementation is preferred. This is the one place that can fault the process
|
||||
// if it is wrong, so it stays small enough to review on its own.
|
||||
//
|
||||
// Compiled with the baseline ISA like every other non-intrinsic unit (see
|
||||
// CMakeLists.txt): it runs before any ISA unit is reached, so it must not execute
|
||||
// a wide instruction itself.
|
||||
|
||||
namespace joc::simd {
|
||||
|
||||
// One flag per ISA the dispatcher can ask about. A flag means "usable here":
|
||||
// both the silicon and the OS state-management agree, not merely that CPUID
|
||||
// advertises the feature.
|
||||
struct CpuFeatures {
|
||||
bool sse2 = false;
|
||||
bool avx2 = false;
|
||||
bool avx512 = false;
|
||||
bool neon = false;
|
||||
};
|
||||
|
||||
// Probed once, on first use. Never throws and never reads the environment (the
|
||||
// JOC_SIMD override is the dispatcher's business).
|
||||
const CpuFeatures& cpu_features() noexcept;
|
||||
|
||||
} // namespace joc::simd
|
||||
@@ -0,0 +1,293 @@
|
||||
// Kernel dispatch: CPU features in, kernel table out.
|
||||
//
|
||||
// This file holds policy only -- which ISA is asked for, which ladder is walked,
|
||||
// which slot ends up with which implementation. The question "can this machine
|
||||
// run ISA X at all" belongs to cpu_probe.cpp, and the implementations themselves
|
||||
// belong to the one ISA unit per ISA (kernels_scalar.cpp / kernels_intrin_*.cpp).
|
||||
//
|
||||
// JOC_SIMD=auto|scalar|sse2|avx2|avx512|neon. A forced ISA that this build or
|
||||
// this machine cannot provide is reported on stderr and then ignored -- forcing a
|
||||
// path the CPU lacks would fault instead of verifying anything.
|
||||
//
|
||||
// This unit is compiled with the baseline ISA: it runs before anything else in
|
||||
// any binary that links the kernels, so it must not execute an ISA-specific
|
||||
// instruction itself.
|
||||
|
||||
#include "simd/simd.h"
|
||||
|
||||
#include "simd/cpu_probe.h"
|
||||
|
||||
#include <cstdio>
|
||||
#include <cstdlib>
|
||||
#include <cstring>
|
||||
|
||||
// Which ISA units the build system put into this binary. An ISA that was not
|
||||
// compiled in has no entry point to reference at all.
|
||||
#if !defined(JOC_SIMD_HAVE_SSE2)
|
||||
#define JOC_SIMD_HAVE_SSE2 0
|
||||
#endif
|
||||
#if !defined(JOC_SIMD_HAVE_AVX2)
|
||||
#define JOC_SIMD_HAVE_AVX2 0
|
||||
#endif
|
||||
#if !defined(JOC_SIMD_HAVE_AVX512)
|
||||
#define JOC_SIMD_HAVE_AVX512 0
|
||||
#endif
|
||||
#if !defined(JOC_SIMD_HAVE_NEON)
|
||||
#define JOC_SIMD_HAVE_NEON 0
|
||||
#endif
|
||||
|
||||
namespace joc::simd {
|
||||
|
||||
// Implemented by the per-ISA units; referenced only when compiled in. The file
|
||||
// name carries the ISA (FLAC's `*_intrin_<isa>.c` convention), and the CMake list
|
||||
// gives exactly those units their wider /arch or -m flag.
|
||||
const Kernels& kernels_scalar() noexcept;
|
||||
#if JOC_SIMD_HAVE_AVX2
|
||||
const Kernels& kernels_intrin_avx2() noexcept;
|
||||
#endif
|
||||
#if JOC_SIMD_HAVE_AVX512
|
||||
const Kernels& kernels_intrin_avx512() noexcept;
|
||||
#endif
|
||||
#if JOC_SIMD_HAVE_NEON
|
||||
const Kernels& kernels_intrin_neon() noexcept;
|
||||
#endif
|
||||
|
||||
namespace {
|
||||
|
||||
constexpr std::size_t kKernelCount = static_cast<std::size_t>(Kernel::count);
|
||||
|
||||
// The probe itself lives in cpu_probe.cpp; all this file needs from it is the
|
||||
// cached answer, turned into the two questions the ladder asks: "was it compiled
|
||||
// into this binary" (JOC_SIMD_HAVE_*) and "can this machine run it".
|
||||
|
||||
struct Resolution {
|
||||
Kernels kernels;
|
||||
Isa active[kKernelCount] = {};
|
||||
Isa selected = Isa::scalar;
|
||||
bool forced = false;
|
||||
};
|
||||
|
||||
int isa_rank(Isa isa) noexcept {
|
||||
switch (isa) {
|
||||
case Isa::scalar: return 0;
|
||||
case Isa::sse2: return 1;
|
||||
case Isa::neon: return 1;
|
||||
case Isa::avx2: return 2;
|
||||
case Isa::avx512: return 3;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
bool isa_compiled(Isa isa) noexcept {
|
||||
switch (isa) {
|
||||
case Isa::scalar: return true;
|
||||
// The x86-64 baseline *is* SSE2: the baseline units are compiled for it
|
||||
// and there is no separate unit, because a 128-bit SSE2 register is
|
||||
// exactly the register a scalar double already uses -- SSE2 cannot widen
|
||||
// a double-precision operation, so "sse2" selects the reference kernels.
|
||||
case Isa::sse2: return JOC_SIMD_HAVE_SSE2 != 0;
|
||||
case Isa::avx2: return JOC_SIMD_HAVE_AVX2 != 0;
|
||||
case Isa::avx512: return JOC_SIMD_HAVE_AVX512 != 0;
|
||||
case Isa::neon: return JOC_SIMD_HAVE_NEON != 0;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
const Kernels& kernels_of(Isa isa) noexcept {
|
||||
switch (isa) {
|
||||
case Isa::scalar:
|
||||
case Isa::sse2: return kernels_scalar();
|
||||
#if JOC_SIMD_HAVE_AVX2
|
||||
case Isa::avx2: return kernels_intrin_avx2();
|
||||
#endif
|
||||
#if JOC_SIMD_HAVE_AVX512
|
||||
case Isa::avx512: return kernels_intrin_avx512();
|
||||
#endif
|
||||
#if JOC_SIMD_HAVE_NEON
|
||||
case Isa::neon: return kernels_intrin_neon();
|
||||
#endif
|
||||
default: break;
|
||||
}
|
||||
return kernels_scalar();
|
||||
}
|
||||
|
||||
bool parse_isa(const char* text, Isa* out) noexcept {
|
||||
struct Entry {
|
||||
const char* name;
|
||||
Isa isa;
|
||||
};
|
||||
static const Entry kEntries[] = {
|
||||
{"scalar", Isa::scalar}, {"sse2", Isa::sse2}, {"avx2", Isa::avx2},
|
||||
{"avx512", Isa::avx512}, {"neon", Isa::neon},
|
||||
};
|
||||
for (const Entry& entry : kEntries) {
|
||||
if (std::strcmp(text, entry.name) == 0) {
|
||||
*out = entry.isa;
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
Resolution resolve() noexcept {
|
||||
Resolution resolution;
|
||||
Isa best = Isa::scalar;
|
||||
bool forced = false;
|
||||
|
||||
if (const char* requested = std::getenv("JOC_SIMD");
|
||||
requested != nullptr && requested[0] != '\0' && std::strcmp(requested, "auto") != 0) {
|
||||
Isa wanted = Isa::scalar;
|
||||
if (!parse_isa(requested, &wanted)) {
|
||||
std::fprintf(stderr,
|
||||
"joc: JOC_SIMD=%s is not one of auto|scalar|sse2|avx2|avx512|neon; "
|
||||
"using auto\n",
|
||||
requested);
|
||||
} else if (!isa_compiled(wanted)) {
|
||||
std::fprintf(stderr, "joc: JOC_SIMD=%s is not part of this build; using auto\n",
|
||||
requested);
|
||||
} else if (!isa_supported(wanted)) {
|
||||
std::fprintf(stderr, "joc: JOC_SIMD=%s is not supported by this CPU or OS; using auto\n",
|
||||
requested);
|
||||
} else {
|
||||
best = wanted;
|
||||
forced = true;
|
||||
}
|
||||
}
|
||||
|
||||
if (!forced) {
|
||||
// Widest runnable ISA, in descending order.
|
||||
static const Isa kLadder[] = {Isa::avx512, Isa::avx2, Isa::sse2, Isa::neon};
|
||||
for (const Isa candidate : kLadder) {
|
||||
if (isa_compiled(candidate) && isa_supported(candidate)) {
|
||||
best = candidate;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
resolution.selected = best;
|
||||
resolution.forced = forced;
|
||||
|
||||
// Per kernel: widest compiled + runnable implementation at or below the
|
||||
// selected ISA, falling back to the scalar reference, which is always there.
|
||||
static const Isa kCandidates[] = {Isa::avx512, Isa::avx2, Isa::sse2, Isa::neon, Isa::scalar};
|
||||
for (std::size_t index = 0u; index < kKernelCount; ++index) {
|
||||
const Kernel kernel = static_cast<Kernel>(index);
|
||||
resolution.active[index] = Isa::scalar;
|
||||
for (const Isa candidate : kCandidates) {
|
||||
if (isa_rank(candidate) > isa_rank(best) || !isa_compiled(candidate)
|
||||
|| !isa_supported(candidate)) {
|
||||
continue;
|
||||
}
|
||||
const Kernels& set = kernels_of(candidate);
|
||||
bool resolved = false;
|
||||
switch (kernel) {
|
||||
case Kernel::fft_butterflies:
|
||||
if (set.fft_butterflies != nullptr) {
|
||||
resolution.kernels.fft_butterflies = set.fft_butterflies;
|
||||
resolved = true;
|
||||
}
|
||||
break;
|
||||
case Kernel::qmf_synthesis_basis:
|
||||
if (set.qmf_synthesis_basis != nullptr) {
|
||||
resolution.kernels.qmf_synthesis_basis = set.qmf_synthesis_basis;
|
||||
resolved = true;
|
||||
}
|
||||
break;
|
||||
case Kernel::hybrid_low_join:
|
||||
if (set.hybrid_low_join != nullptr) {
|
||||
resolution.kernels.hybrid_low_join = set.hybrid_low_join;
|
||||
resolved = true;
|
||||
}
|
||||
break;
|
||||
case Kernel::render_hybrid_path:
|
||||
if (set.render_hybrid_path != nullptr) {
|
||||
resolution.kernels.render_hybrid_path = set.render_hybrid_path;
|
||||
resolved = true;
|
||||
}
|
||||
break;
|
||||
case Kernel::complex_axpy:
|
||||
if (set.complex_axpy != nullptr) {
|
||||
resolution.kernels.complex_axpy = set.complex_axpy;
|
||||
resolved = true;
|
||||
}
|
||||
break;
|
||||
case Kernel::qmf_analysis_taps:
|
||||
if (set.qmf_analysis_taps != nullptr) {
|
||||
resolution.kernels.qmf_analysis_taps = set.qmf_analysis_taps;
|
||||
resolved = true;
|
||||
}
|
||||
break;
|
||||
case Kernel::complex_multiply:
|
||||
if (set.complex_multiply != nullptr) {
|
||||
resolution.kernels.complex_multiply = set.complex_multiply;
|
||||
resolved = true;
|
||||
}
|
||||
break;
|
||||
case Kernel::count: break;
|
||||
}
|
||||
if (resolved) {
|
||||
resolution.active[index] = candidate;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (std::getenv("JOC_SIMD_LOG") != nullptr) {
|
||||
std::fprintf(stderr, "joc: simd selected=%s forced=%d\n", isa_name(resolution.selected),
|
||||
resolution.forced ? 1 : 0);
|
||||
for (std::size_t index = 0u; index < kKernelCount; ++index) {
|
||||
std::fprintf(stderr, "joc: simd kernel[%u]=%s\n", static_cast<unsigned>(index),
|
||||
isa_name(resolution.active[index]));
|
||||
}
|
||||
}
|
||||
return resolution;
|
||||
}
|
||||
|
||||
const Resolution& resolution() noexcept {
|
||||
static const Resolution resolved = resolve();
|
||||
return resolved;
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
const char* isa_name(Isa isa) noexcept {
|
||||
switch (isa) {
|
||||
case Isa::scalar: return "scalar";
|
||||
case Isa::sse2: return "sse2";
|
||||
case Isa::avx2: return "avx2";
|
||||
case Isa::avx512: return "avx512";
|
||||
case Isa::neon: return "neon";
|
||||
}
|
||||
return "?";
|
||||
}
|
||||
|
||||
bool isa_supported(Isa isa) noexcept {
|
||||
const CpuFeatures& cpu = cpu_features();
|
||||
switch (isa) {
|
||||
case Isa::scalar:
|
||||
return true;
|
||||
case Isa::sse2:
|
||||
return cpu.sse2;
|
||||
case Isa::avx2:
|
||||
return cpu.avx2;
|
||||
case Isa::avx512:
|
||||
return cpu.avx512;
|
||||
case Isa::neon:
|
||||
return cpu.neon;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
Isa selected_isa() noexcept { return resolution().selected; }
|
||||
|
||||
Isa active_isa(Kernel kernel) noexcept {
|
||||
const std::size_t index = static_cast<std::size_t>(kernel);
|
||||
if (index >= kKernelCount) {
|
||||
return Isa::scalar;
|
||||
}
|
||||
return resolution().active[index];
|
||||
}
|
||||
|
||||
const Kernels& kernels() noexcept { return resolution().kernels; }
|
||||
|
||||
} // namespace joc::simd
|
||||
@@ -0,0 +1,321 @@
|
||||
// AVX2 implementation of the dispatched kernels. Compiled with /arch:AVX2 (or
|
||||
// -mavx2) and only ever called after the dispatcher has confirmed that this CPU
|
||||
// and the OS state it saves can execute AVX2 code.
|
||||
//
|
||||
// CMake gives exactly the `kernels_intrin_*.cpp` units a wider flag, so no
|
||||
// baseline unit can inherit one by accident.
|
||||
//
|
||||
// Nothing in this file may have dynamic initialisation: it is linked into the
|
||||
// same image as the baseline code and would run before the dispatcher.
|
||||
|
||||
#include "simd/simd.h"
|
||||
|
||||
#include <immintrin.h>
|
||||
|
||||
#include <cstddef>
|
||||
|
||||
namespace joc::simd {
|
||||
namespace {
|
||||
|
||||
// Two interleaved complexes per vector.
|
||||
//
|
||||
// The scalar kernel computes, for one butterfly,
|
||||
// odd.re = o.re * w.re - o.im * w.im
|
||||
// odd.im = o.re * w.im + o.im * w.re
|
||||
// with separate multiplies and one rounding per operation. In a 256-bit vector
|
||||
// the even element of every 128-bit half is a real part, so duplicating the even
|
||||
// elements of the twiddle broadcasts w.re onto both components of a complex and
|
||||
// permuting the odd data elements in lines up `o.im`, `o.re` against `w.im`.
|
||||
// addsub then subtracts on the real lanes and adds on the imaginary ones, which
|
||||
// is the same two roundings: `o.im * w.re + o.re * w.im` is the scalar's
|
||||
// `o.re * w.im + o.im * w.re` with the summands exchanged, and floating-point
|
||||
// addition is commutative.
|
||||
inline __m256d mul_pair(__m256d odd, __m256d twiddle) noexcept {
|
||||
const __m256d real = _mm256_mul_pd(odd, _mm256_movedup_pd(twiddle));
|
||||
const __m256d cross = _mm256_mul_pd(_mm256_permute_pd(odd, 0x05), _mm256_permute_pd(twiddle, 0x0f));
|
||||
return _mm256_addsub_pd(real, cross);
|
||||
}
|
||||
|
||||
// The radix-2 DIT cascade over a bit-reversed buffer of any power-of-two size at
|
||||
// or above kMinVectorFftSize.
|
||||
//
|
||||
// Stage s pairs element (start + offset) with (start + offset + 2^s) for every
|
||||
// group of 2^(s+1) elements, and the butterflies of a stage are mutually
|
||||
// independent: none of them reads a slot another one writes. They are therefore
|
||||
// free to share a vector, and each one still accumulates nothing -- every output
|
||||
// is a single add or subtract of two products, in lanes whose operations are
|
||||
// independent of each other.
|
||||
//
|
||||
// Stage 0 is the one case whose group holds a single butterfly, so the pair is
|
||||
// packed from two neighbouring groups instead: their factors are identical
|
||||
// (offset is always 0), and a permute2f128 moves the even and odd halves into
|
||||
// two vector registers.
|
||||
void fft_butterflies_avx2(double* data, std::size_t size, const double* twiddle,
|
||||
const std::size_t* stage_begin) noexcept {
|
||||
{
|
||||
const auto* pair = reinterpret_cast<const __m128d*>(twiddle + 2u * stage_begin[0]);
|
||||
const __m256d factor = _mm256_broadcast_pd(pair);
|
||||
for (std::size_t index = 0u; index < 2u * size; index += 8u) {
|
||||
const __m256d low = _mm256_loadu_pd(data + index);
|
||||
const __m256d high = _mm256_loadu_pd(data + index + 4u);
|
||||
const __m256d even = _mm256_permute2f128_pd(low, high, 0x20);
|
||||
const __m256d odd = _mm256_permute2f128_pd(low, high, 0x31);
|
||||
const __m256d rotated = mul_pair(odd, factor);
|
||||
const __m256d sum = _mm256_add_pd(even, rotated);
|
||||
const __m256d difference = _mm256_sub_pd(even, rotated);
|
||||
_mm256_storeu_pd(data + index, _mm256_permute2f128_pd(sum, difference, 0x20));
|
||||
_mm256_storeu_pd(data + index + 4u, _mm256_permute2f128_pd(sum, difference, 0x31));
|
||||
}
|
||||
}
|
||||
|
||||
std::size_t stage = 1u;
|
||||
for (std::size_t half = 2u; half < size; half <<= 1u, ++stage) {
|
||||
const std::size_t length = half << 1u;
|
||||
const double* table = twiddle + 2u * stage_begin[stage];
|
||||
for (std::size_t start = 0u; start < size; start += length) {
|
||||
for (std::size_t offset = 0u; offset < half; offset += 2u) {
|
||||
const std::size_t even_index = (start + offset) * 2u;
|
||||
const std::size_t odd_index = even_index + half * 2u;
|
||||
const std::size_t factor_index = offset * 2u;
|
||||
const __m256d even = _mm256_loadu_pd(data + even_index);
|
||||
const __m256d odd = _mm256_loadu_pd(data + odd_index);
|
||||
const __m256d factor = _mm256_loadu_pd(table + factor_index);
|
||||
const __m256d rotated = mul_pair(odd, factor);
|
||||
_mm256_storeu_pd(data + even_index, _mm256_add_pd(even, rotated));
|
||||
_mm256_storeu_pd(data + odd_index, _mm256_sub_pd(even, rotated));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// QMF synthesis basis, four ranks of one band per vector.
|
||||
//
|
||||
// The four ranks are four dot products over the same 128 taps, so they are four
|
||||
// independent accumulators and nothing has to be reassociated to fill a vector.
|
||||
// Each lane performs exactly the caller's `sum += values[tap] * weight`, in tap
|
||||
// order, with a separate multiply and add, and the rank-minor table makes the
|
||||
// four weights one contiguous load.
|
||||
//
|
||||
// A band's chain is 128 dependent adds, so the loop is latency-bound long before
|
||||
// it is throughput-bound; the cure is more independent chains, not a shorter
|
||||
// chain (that would reassociate). `Rows` output rows share one weight load and
|
||||
// run their chains side by side -- the caller's rows are consecutive output
|
||||
// channels, which do read the same weights.
|
||||
template <int Rows>
|
||||
void synthesize_rows(const double* values, const double* basis, double* out) noexcept {
|
||||
constexpr std::size_t kRanks = kSynthesisRanks;
|
||||
constexpr std::size_t kTaps = kSynthesisTaps;
|
||||
constexpr std::size_t kRowOut = kSynthesisBands * kRanks;
|
||||
for (std::size_t band = 0u; band < kSynthesisBands; ++band) {
|
||||
const double* table = basis + band * kTaps * kRanks;
|
||||
__m256d sum[Rows];
|
||||
for (int row = 0; row < Rows; ++row) {
|
||||
sum[row] = _mm256_setzero_pd();
|
||||
}
|
||||
for (std::size_t tap = 0u; tap < kTaps; ++tap) {
|
||||
const __m256d weight = _mm256_loadu_pd(table + tap * kRanks);
|
||||
for (int row = 0; row < Rows; ++row) {
|
||||
sum[row] = _mm256_add_pd(
|
||||
sum[row],
|
||||
_mm256_mul_pd(_mm256_broadcast_sd(values + row * kTaps + tap), weight));
|
||||
}
|
||||
}
|
||||
for (int row = 0; row < Rows; ++row) {
|
||||
_mm256_storeu_pd(out + row * kRowOut + band * kRanks, sum[row]);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void qmf_synthesis_basis_avx2(const double* values, const double* basis, double* out,
|
||||
std::size_t rows) noexcept {
|
||||
constexpr std::size_t kTaps = kSynthesisTaps;
|
||||
constexpr std::size_t kRowOut = kSynthesisBands * kSynthesisRanks;
|
||||
std::size_t row = 0u;
|
||||
for (; row + 4u <= rows; row += 4u) {
|
||||
synthesize_rows<4>(values + row * kTaps, basis, out + row * kRowOut);
|
||||
}
|
||||
for (; row < rows; ++row) {
|
||||
synthesize_rows<1>(values + row * kTaps, basis, out + row * kRowOut);
|
||||
}
|
||||
}
|
||||
|
||||
// Hybrid analysis low join, one term against all 32 outputs.
|
||||
//
|
||||
// The 32 outputs of a row are independent accumulations over the same 78 terms,
|
||||
// so they are what fills the four vectors: one broadcast of the term's value,
|
||||
// then a multiply and an add per vector. Every lane keeps the term order and the
|
||||
// two roundings of the caller's `out += value * weight`. Terms that are exactly
|
||||
// zero are skipped, which is what the caller does and what leaves its sums
|
||||
// unchanged (a lane's running sum is never a negative zero).
|
||||
void hybrid_low_join_avx2(const double* values, const double* kernel, double* out,
|
||||
std::size_t rows) noexcept {
|
||||
constexpr std::size_t kTerms = kHybridTerms;
|
||||
constexpr std::size_t kOutputs = kHybridOutputs;
|
||||
static_assert(kOutputs == 32u, "eight 256-bit accumulators cover one row's outputs");
|
||||
for (std::size_t row = 0u; row < rows; ++row) {
|
||||
const double* source = values + row * kTerms;
|
||||
double* destination = out + row * kOutputs;
|
||||
__m256d sum0 = _mm256_setzero_pd();
|
||||
__m256d sum1 = _mm256_setzero_pd();
|
||||
__m256d sum2 = _mm256_setzero_pd();
|
||||
__m256d sum3 = _mm256_setzero_pd();
|
||||
__m256d sum4 = _mm256_setzero_pd();
|
||||
__m256d sum5 = _mm256_setzero_pd();
|
||||
__m256d sum6 = _mm256_setzero_pd();
|
||||
__m256d sum7 = _mm256_setzero_pd();
|
||||
for (std::size_t term = 0u; term < kTerms; ++term) {
|
||||
const double value = source[term];
|
||||
if (value == 0.0) {
|
||||
continue;
|
||||
}
|
||||
const double* weights = kernel + term * kOutputs;
|
||||
const __m256d factor = _mm256_broadcast_sd(&value);
|
||||
sum0 = _mm256_add_pd(sum0, _mm256_mul_pd(factor, _mm256_loadu_pd(weights)));
|
||||
sum1 = _mm256_add_pd(sum1, _mm256_mul_pd(factor, _mm256_loadu_pd(weights + 4u)));
|
||||
sum2 = _mm256_add_pd(sum2, _mm256_mul_pd(factor, _mm256_loadu_pd(weights + 8u)));
|
||||
sum3 = _mm256_add_pd(sum3, _mm256_mul_pd(factor, _mm256_loadu_pd(weights + 12u)));
|
||||
sum4 = _mm256_add_pd(sum4, _mm256_mul_pd(factor, _mm256_loadu_pd(weights + 16u)));
|
||||
sum5 = _mm256_add_pd(sum5, _mm256_mul_pd(factor, _mm256_loadu_pd(weights + 20u)));
|
||||
sum6 = _mm256_add_pd(sum6, _mm256_mul_pd(factor, _mm256_loadu_pd(weights + 24u)));
|
||||
sum7 = _mm256_add_pd(sum7, _mm256_mul_pd(factor, _mm256_loadu_pd(weights + 28u)));
|
||||
}
|
||||
_mm256_storeu_pd(destination, sum0);
|
||||
_mm256_storeu_pd(destination + 4u, sum1);
|
||||
_mm256_storeu_pd(destination + 8u, sum2);
|
||||
_mm256_storeu_pd(destination + 12u, sum3);
|
||||
_mm256_storeu_pd(destination + 16u, sum4);
|
||||
_mm256_storeu_pd(destination + 20u, sum5);
|
||||
_mm256_storeu_pd(destination + 24u, sum6);
|
||||
_mm256_storeu_pd(destination + 28u, sum7);
|
||||
}
|
||||
}
|
||||
|
||||
// One rendered path, two bands per vector.
|
||||
//
|
||||
// A band's real and imaginary parts are adjacent, so a vector holds two whole
|
||||
// bands and the cross terms are one swap away -- the same shape the FFT kernel
|
||||
// uses. Two products are needed per component here (`h * t` and `h * swapped(t)`),
|
||||
// each scaled on its own, and the real part is their difference while the
|
||||
// imaginary part is their sum: addsub produces both pairings and a blend keeps
|
||||
// one lane of each. Bands are independent accumulations into independent
|
||||
// outputs, so this is still nothing but cross-band parallelism.
|
||||
void render_hybrid_path_avx2(double* out, const double* transfer, const double* history0,
|
||||
const double* history1, double scale) noexcept {
|
||||
const __m256d factor = _mm256_set1_pd(scale);
|
||||
const double* histories[2] = {history0, history1};
|
||||
for (std::size_t ear = 0u; ear < 2u; ++ear) {
|
||||
const double* source = histories[ear];
|
||||
const double* taps = transfer + ear * kPathBands * 2u;
|
||||
double* destination = out + ear * kPathBands * 2u;
|
||||
std::size_t band = 0u;
|
||||
for (; band + 2u <= kPathBands; band += 2u) {
|
||||
const __m256d history = _mm256_loadu_pd(source + band * 2u);
|
||||
const __m256d tap = _mm256_loadu_pd(taps + band * 2u);
|
||||
const __m256d product = _mm256_mul_pd(_mm256_mul_pd(history, tap), factor);
|
||||
const __m256d crossed = _mm256_mul_pd(
|
||||
_mm256_mul_pd(history, _mm256_permute_pd(tap, 0x05)), factor);
|
||||
const __m256d difference =
|
||||
_mm256_addsub_pd(product, _mm256_permute_pd(product, 0x05));
|
||||
const __m256d sum = _mm256_add_pd(crossed, _mm256_permute_pd(crossed, 0x05));
|
||||
// `difference` carries the real parts in lanes 0 and 2, `sum` the
|
||||
// imaginary ones in lanes 1 and 3 (four elements, so the blend takes a
|
||||
// four-bit selector).
|
||||
const __m256d result = _mm256_blend_pd(difference, sum, 0xa);
|
||||
_mm256_storeu_pd(destination + band * 2u,
|
||||
_mm256_add_pd(_mm256_loadu_pd(destination + band * 2u), result));
|
||||
}
|
||||
for (; band < kPathBands; ++band) {
|
||||
const double source_real = source[band * 2u];
|
||||
const double source_imag = source[band * 2u + 1u];
|
||||
const double tap_real = taps[band * 2u];
|
||||
const double tap_imag = taps[band * 2u + 1u];
|
||||
destination[band * 2u] +=
|
||||
source_real * tap_real * scale - source_imag * tap_imag * scale;
|
||||
destination[band * 2u + 1u] +=
|
||||
source_real * tap_imag * scale + source_imag * tap_real * scale;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Scalar times a run of interleaved complexes, accumulated in place. The elements
|
||||
// are independent, so two complexes (one 256-bit vector) go together and each
|
||||
// component keeps the caller's own multiply-then-add.
|
||||
void complex_axpy_avx2(double* out, const double* field, double scale,
|
||||
std::size_t count) noexcept {
|
||||
const __m256d factor = _mm256_set1_pd(scale);
|
||||
std::size_t index = 0u;
|
||||
for (; index + 2u <= count; index += 2u) {
|
||||
const std::size_t offset = index * 2u;
|
||||
_mm256_storeu_pd(out + offset,
|
||||
_mm256_add_pd(_mm256_loadu_pd(out + offset),
|
||||
_mm256_mul_pd(_mm256_loadu_pd(field + offset), factor)));
|
||||
}
|
||||
for (; index < count; ++index) {
|
||||
out[index * 2u] += field[index * 2u] * scale;
|
||||
out[index * 2u + 1u] += field[index * 2u + 1u] * scale;
|
||||
}
|
||||
}
|
||||
|
||||
// QMF analysis polyphase accumulate: the 64 bands of a row are 64 independent
|
||||
// accumulations of one product each, and the coefficient row is reused by every
|
||||
// row, so the rows stream past a first-level-cache-resident set of coefficients.
|
||||
void qmf_analysis_taps_avx2(double* out, const double* source, const double* coefficients,
|
||||
std::size_t rows) noexcept {
|
||||
for (std::size_t row = 0u; row < rows; ++row) {
|
||||
double* destination = out + row * kQmfAnalysisBands;
|
||||
const double* values = source + row * kQmfAnalysisBands;
|
||||
for (std::size_t band = 0u; band < kQmfAnalysisBands; band += 4u) {
|
||||
_mm256_storeu_pd(destination + band,
|
||||
_mm256_add_pd(_mm256_loadu_pd(destination + band),
|
||||
_mm256_mul_pd(_mm256_loadu_pd(values + band),
|
||||
_mm256_loadu_pd(coefficients + band))));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Element-wise complex product of two runs, two complexes per vector. Both
|
||||
// operands take part in every lane, so both products of the scalar formula are
|
||||
// formed and addsub pairs the real difference with the imaginary sum: the same
|
||||
// four multiplies and the same two roundings per component as `a * b`.
|
||||
void complex_multiply_avx2(double* out, const double* left, const double* right,
|
||||
std::size_t count) noexcept {
|
||||
std::size_t index = 0u;
|
||||
for (; index + 2u <= count; index += 2u) {
|
||||
const std::size_t offset = index * 2u;
|
||||
const __m256d first = _mm256_loadu_pd(left + offset);
|
||||
const __m256d second = _mm256_loadu_pd(right + offset);
|
||||
// (re*re, im*im) and (im*re, re*im): the first gives the real difference,
|
||||
// the second the imaginary sum.
|
||||
const __m256d product = _mm256_mul_pd(first, second);
|
||||
const __m256d crossed = _mm256_mul_pd(_mm256_permute_pd(first, 0x05), second);
|
||||
const __m256d difference =
|
||||
_mm256_addsub_pd(product, _mm256_permute_pd(product, 0x05));
|
||||
const __m256d total = _mm256_add_pd(crossed, _mm256_permute_pd(crossed, 0x05));
|
||||
_mm256_storeu_pd(out + offset, _mm256_blend_pd(difference, total, 0xa));
|
||||
}
|
||||
for (; index < count; ++index) {
|
||||
const std::size_t offset = index * 2u;
|
||||
const double left_real = left[offset];
|
||||
const double left_imag = left[offset + 1u];
|
||||
const double right_real = right[offset];
|
||||
const double right_imag = right[offset + 1u];
|
||||
out[offset] = left_real * right_real - left_imag * right_imag;
|
||||
out[offset + 1u] = left_real * right_imag + left_imag * right_real;
|
||||
}
|
||||
}
|
||||
|
||||
const Kernels kAvx2{
|
||||
&fft_butterflies_avx2,
|
||||
&qmf_synthesis_basis_avx2,
|
||||
&hybrid_low_join_avx2,
|
||||
&render_hybrid_path_avx2,
|
||||
&complex_axpy_avx2,
|
||||
&qmf_analysis_taps_avx2,
|
||||
&complex_multiply_avx2,
|
||||
};
|
||||
|
||||
} // namespace
|
||||
|
||||
const Kernels& kernels_intrin_avx2() noexcept { return kAvx2; }
|
||||
|
||||
} // namespace joc::simd
|
||||
@@ -0,0 +1,272 @@
|
||||
// AVX-512F implementation of the dispatched kernels. Compiled with /arch:AVX512
|
||||
// (or -mavx512f) and only ever called after the dispatcher has confirmed AVX-512F
|
||||
// *and* that the OS saves the opmask/ZMM state.
|
||||
//
|
||||
// CMake gives exactly the `kernels_intrin_*.cpp` units a wider flag, so no
|
||||
// baseline unit can inherit one by accident.
|
||||
//
|
||||
// Nothing in this file may have dynamic initialisation: it is linked into the
|
||||
// same image as the baseline code and would run before the dispatcher.
|
||||
|
||||
#include "simd/simd.h"
|
||||
|
||||
#include <immintrin.h>
|
||||
|
||||
#include <cstddef>
|
||||
|
||||
namespace joc::simd {
|
||||
namespace {
|
||||
|
||||
// Four interleaved complexes per vector: a 512-bit register is four 128-bit
|
||||
// lanes, each holding one (re, im) pair, so the AVX2 argument applies one lane
|
||||
// wider -- movedup/permute broadcast the twiddle components inside every lane.
|
||||
// AVX-512 never widened VADDSUBPD to 512 bits, so the real lanes take the
|
||||
// difference and the imaginary ones the sum through merge masking: that is still
|
||||
// one exact subtract and one exact add per output, with the same two roundings
|
||||
// per component and no reassociation.
|
||||
inline __m512d mul_quad(__m512d odd, __m512d twiddle) noexcept {
|
||||
const __m512d real = _mm512_mul_pd(odd, _mm512_movedup_pd(twiddle));
|
||||
const __m512d cross =
|
||||
_mm512_mul_pd(_mm512_permute_pd(odd, 0x55), _mm512_permute_pd(twiddle, 0xff));
|
||||
const __m512d upper = _mm512_mask_add_pd(real, 0xaa, real, cross);
|
||||
return _mm512_mask_sub_pd(upper, 0x55, real, cross);
|
||||
}
|
||||
|
||||
// Stages 0 and 1 hold fewer than four complexes in a group, so their butterflies
|
||||
// are packed from neighbouring groups instead; the factor depends only on the
|
||||
// offset inside a group, so one broadcast serves every packed lane.
|
||||
//
|
||||
// VSHUFF64X2 takes a two-bit selector per output lane, but its first two output
|
||||
// lanes can only name lanes of its first operand and its last two only lanes of
|
||||
// its second (verified against the instruction). These are the immediates the
|
||||
// packings need, written as the four lane selections they spell out.
|
||||
constexpr int kHalvesEven = 0x88; // (low.l0, low.l2, high.l0, high.l2) = x0,x2,x4,x6
|
||||
constexpr int kHalvesOdd = 0xdd; // (low.l1, low.l3, high.l1, high.l3) = x1,x3,x5,x7
|
||||
constexpr int kGroupsEven = 0x44; // (low.l0, low.l1, high.l0, high.l1)
|
||||
constexpr int kGroupsOdd = 0xee; // (low.l2, low.l3, high.l2, high.l3)
|
||||
constexpr int kSwapMiddle = 0xd8; // (a.l0, a.l2, a.l1, a.l3)
|
||||
|
||||
void pack_halves(double* buffer, __m512d factor) noexcept {
|
||||
const __m512d low = _mm512_loadu_pd(buffer);
|
||||
const __m512d high = _mm512_loadu_pd(buffer + 8u);
|
||||
const __m512d even = _mm512_shuffle_f64x2(low, high, kHalvesEven);
|
||||
const __m512d odd = _mm512_shuffle_f64x2(low, high, kHalvesOdd);
|
||||
const __m512d rotated = mul_quad(odd, factor);
|
||||
const __m512d sum = _mm512_add_pd(even, rotated);
|
||||
const __m512d difference = _mm512_sub_pd(even, rotated);
|
||||
// `sum` holds the results for the even positions of the four packed groups and
|
||||
// `difference` those for the odd ones, and the buffer wants them interleaved
|
||||
// again. One shuffle cannot do that, so each store pairs the two halves and
|
||||
// then swaps the two middle lanes of the pair.
|
||||
const __m512d pairs_low = _mm512_shuffle_f64x2(sum, difference, kGroupsEven);
|
||||
const __m512d pairs_high = _mm512_shuffle_f64x2(sum, difference, kGroupsOdd);
|
||||
_mm512_storeu_pd(buffer, _mm512_shuffle_f64x2(pairs_low, pairs_low, kSwapMiddle));
|
||||
_mm512_storeu_pd(buffer + 8u, _mm512_shuffle_f64x2(pairs_high, pairs_high, kSwapMiddle));
|
||||
}
|
||||
|
||||
void pack_groups(double* buffer, __m512d factor) noexcept {
|
||||
const __m512d low = _mm512_loadu_pd(buffer);
|
||||
const __m512d high = _mm512_loadu_pd(buffer + 8u);
|
||||
const __m512d even = _mm512_shuffle_f64x2(low, high, kGroupsEven);
|
||||
const __m512d odd = _mm512_shuffle_f64x2(low, high, kGroupsOdd);
|
||||
const __m512d rotated = mul_quad(odd, factor);
|
||||
const __m512d sum = _mm512_add_pd(even, rotated);
|
||||
const __m512d difference = _mm512_sub_pd(even, rotated);
|
||||
_mm512_storeu_pd(buffer, _mm512_shuffle_f64x2(sum, difference, kGroupsEven));
|
||||
_mm512_storeu_pd(buffer + 8u, _mm512_shuffle_f64x2(sum, difference, kGroupsOdd));
|
||||
}
|
||||
|
||||
void fft_butterflies_avx512(double* buffer, std::size_t size, const double* twiddle,
|
||||
const std::size_t* stage_begin) noexcept {
|
||||
// Stage 0: length 2, one butterfly per group, one factor for all of them.
|
||||
{
|
||||
const __m128d pair = _mm_loadu_pd(twiddle + 2u * stage_begin[0]);
|
||||
const __m512d factor = _mm512_broadcast_f64x4(_mm256_broadcast_pd(&pair));
|
||||
for (std::size_t index = 0u; index < 2u * size; index += 16u) {
|
||||
pack_halves(buffer + index, factor);
|
||||
}
|
||||
}
|
||||
|
||||
// Stage 1: length 4, two butterflies per group and two factors.
|
||||
{
|
||||
const __m512d factor =
|
||||
_mm512_broadcast_f64x4(_mm256_loadu_pd(twiddle + 2u * stage_begin[1]));
|
||||
for (std::size_t index = 0u; index < 2u * size; index += 16u) {
|
||||
pack_groups(buffer + index, factor);
|
||||
}
|
||||
}
|
||||
|
||||
std::size_t stage = 2u;
|
||||
for (std::size_t half = 4u; half < size; half <<= 1u, ++stage) {
|
||||
const std::size_t length = half << 1u;
|
||||
const double* table = twiddle + 2u * stage_begin[stage];
|
||||
for (std::size_t start = 0u; start < size; start += length) {
|
||||
for (std::size_t offset = 0u; offset < half; offset += 4u) {
|
||||
const std::size_t even_index = (start + offset) * 2u;
|
||||
const std::size_t odd_index = even_index + half * 2u;
|
||||
const std::size_t factor_index = offset * 2u;
|
||||
const __m512d even = _mm512_loadu_pd(buffer + even_index);
|
||||
const __m512d odd = _mm512_loadu_pd(buffer + odd_index);
|
||||
const __m512d factor = _mm512_loadu_pd(table + factor_index);
|
||||
const __m512d rotated = mul_quad(odd, factor);
|
||||
_mm512_storeu_pd(buffer + even_index, _mm512_add_pd(even, rotated));
|
||||
_mm512_storeu_pd(buffer + odd_index, _mm512_sub_pd(even, rotated));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Scalar times a run of interleaved complexes: four complexes (eight doubles) per
|
||||
// vector, every element an independent accumulation.
|
||||
void complex_axpy_avx512(double* out, const double* field, double scale,
|
||||
std::size_t count) noexcept {
|
||||
const __m512d factor = _mm512_set1_pd(scale);
|
||||
std::size_t index = 0u;
|
||||
for (; index + 4u <= count; index += 4u) {
|
||||
const std::size_t offset = index * 2u;
|
||||
_mm512_storeu_pd(out + offset,
|
||||
_mm512_add_pd(_mm512_loadu_pd(out + offset),
|
||||
_mm512_mul_pd(_mm512_loadu_pd(field + offset), factor)));
|
||||
}
|
||||
for (; index < count; ++index) {
|
||||
out[index * 2u] += field[index * 2u] * scale;
|
||||
out[index * 2u + 1u] += field[index * 2u + 1u] * scale;
|
||||
}
|
||||
}
|
||||
|
||||
// QMF analysis polyphase accumulate, eight bands per vector.
|
||||
void qmf_analysis_taps_avx512(double* out, const double* source, const double* coefficients,
|
||||
std::size_t rows) noexcept {
|
||||
for (std::size_t row = 0u; row < rows; ++row) {
|
||||
double* destination = out + row * kQmfAnalysisBands;
|
||||
const double* values = source + row * kQmfAnalysisBands;
|
||||
for (std::size_t band = 0u; band < kQmfAnalysisBands; band += 8u) {
|
||||
_mm512_storeu_pd(destination + band,
|
||||
_mm512_add_pd(_mm512_loadu_pd(destination + band),
|
||||
_mm512_mul_pd(_mm512_loadu_pd(values + band),
|
||||
_mm512_loadu_pd(coefficients + band))));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Hybrid analysis low join: 32 outputs are exactly four 512-bit vectors, so one
|
||||
// broadcast of the term's value feeds four multiplies and four adds.
|
||||
void hybrid_low_join_avx512(const double* values, const double* kernel, double* out,
|
||||
std::size_t rows) noexcept {
|
||||
constexpr std::size_t kTerms = kHybridTerms;
|
||||
constexpr std::size_t kOutputs = kHybridOutputs;
|
||||
static_assert(kOutputs == 32u, "four 512-bit accumulators cover one row");
|
||||
for (std::size_t row = 0u; row < rows; ++row) {
|
||||
const double* source = values + row * kTerms;
|
||||
double* destination = out + row * kOutputs;
|
||||
__m512d sum0 = _mm512_setzero_pd();
|
||||
__m512d sum1 = _mm512_setzero_pd();
|
||||
__m512d sum2 = _mm512_setzero_pd();
|
||||
__m512d sum3 = _mm512_setzero_pd();
|
||||
for (std::size_t term = 0u; term < kTerms; ++term) {
|
||||
const double value = source[term];
|
||||
if (value == 0.0) {
|
||||
continue;
|
||||
}
|
||||
const double* weights = kernel + term * kOutputs;
|
||||
const __m512d factor = _mm512_set1_pd(value);
|
||||
sum0 = _mm512_add_pd(sum0, _mm512_mul_pd(factor, _mm512_loadu_pd(weights)));
|
||||
sum1 = _mm512_add_pd(sum1, _mm512_mul_pd(factor, _mm512_loadu_pd(weights + 8u)));
|
||||
sum2 = _mm512_add_pd(sum2, _mm512_mul_pd(factor, _mm512_loadu_pd(weights + 16u)));
|
||||
sum3 = _mm512_add_pd(sum3, _mm512_mul_pd(factor, _mm512_loadu_pd(weights + 24u)));
|
||||
}
|
||||
_mm512_storeu_pd(destination, sum0);
|
||||
_mm512_storeu_pd(destination + 8u, sum1);
|
||||
_mm512_storeu_pd(destination + 16u, sum2);
|
||||
_mm512_storeu_pd(destination + 24u, sum3);
|
||||
}
|
||||
}
|
||||
|
||||
// One rendered path, four bands per vector. AVX-512 has no 512-bit addsub, so the
|
||||
// two pairings are built with merge masking -- still one exact subtract and one
|
||||
// exact add per output, and the same separate scale multiply per product.
|
||||
void render_hybrid_path_avx512(double* out, const double* transfer, const double* history0,
|
||||
const double* history1, double scale) noexcept {
|
||||
const __m512d factor = _mm512_set1_pd(scale);
|
||||
const double* histories[2] = {history0, history1};
|
||||
for (std::size_t ear = 0u; ear < 2u; ++ear) {
|
||||
const double* source = histories[ear];
|
||||
const double* taps = transfer + ear * kPathBands * 2u;
|
||||
double* destination = out + ear * kPathBands * 2u;
|
||||
std::size_t band = 0u;
|
||||
for (; band + 4u <= kPathBands; band += 4u) {
|
||||
const __m512d history = _mm512_loadu_pd(source + band * 2u);
|
||||
const __m512d tap = _mm512_loadu_pd(taps + band * 2u);
|
||||
const __m512d product = _mm512_mul_pd(
|
||||
_mm512_mul_pd(history, tap), factor);
|
||||
const __m512d crossed = _mm512_mul_pd(
|
||||
_mm512_mul_pd(history, _mm512_permute_pd(tap, 0x55)), factor);
|
||||
const __m512d swapped_product = _mm512_permute_pd(product, 0x55);
|
||||
const __m512d swapped_crossed = _mm512_permute_pd(crossed, 0x55);
|
||||
// The real component is the difference of the two products and sits in
|
||||
// the even element of each complex, the imaginary one is the sum of the
|
||||
// crossed pair and sits in the odd element -- so the subtract is masked
|
||||
// to the even lanes and the blend keeps the sum on the odd ones.
|
||||
const __m512d difference =
|
||||
_mm512_mask_sub_pd(product, 0x55, product, swapped_product);
|
||||
const __m512d total = _mm512_add_pd(crossed, swapped_crossed);
|
||||
const __m512d result = _mm512_mask_blend_pd(0xaa, difference, total);
|
||||
_mm512_storeu_pd(destination + band * 2u,
|
||||
_mm512_add_pd(_mm512_loadu_pd(destination + band * 2u), result));
|
||||
}
|
||||
for (; band < kPathBands; ++band) {
|
||||
const double source_real = source[band * 2u];
|
||||
const double source_imag = source[band * 2u + 1u];
|
||||
const double tap_real = taps[band * 2u];
|
||||
const double tap_imag = taps[band * 2u + 1u];
|
||||
destination[band * 2u] +=
|
||||
source_real * tap_real * scale - source_imag * tap_imag * scale;
|
||||
destination[band * 2u + 1u] +=
|
||||
source_real * tap_imag * scale + source_imag * tap_real * scale;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Element-wise complex product, four complexes per vector. AVX-512 has no 512-bit
|
||||
// addsub, so the real difference is masked into the even lanes and the imaginary
|
||||
// sum into the odd ones.
|
||||
void complex_multiply_avx512(double* out, const double* left, const double* right,
|
||||
std::size_t count) noexcept {
|
||||
std::size_t index = 0u;
|
||||
for (; index + 4u <= count; index += 4u) {
|
||||
const std::size_t offset = index * 2u;
|
||||
const __m512d first = _mm512_loadu_pd(left + offset);
|
||||
const __m512d second = _mm512_loadu_pd(right + offset);
|
||||
const __m512d product = _mm512_mul_pd(first, second);
|
||||
const __m512d crossed = _mm512_mul_pd(_mm512_permute_pd(first, 0x55), second);
|
||||
const __m512d difference =
|
||||
_mm512_mask_sub_pd(product, 0x55, product, _mm512_permute_pd(product, 0x55));
|
||||
const __m512d total = _mm512_add_pd(crossed, _mm512_permute_pd(crossed, 0x55));
|
||||
_mm512_storeu_pd(out + offset, _mm512_mask_blend_pd(0xaa, difference, total));
|
||||
}
|
||||
for (; index < count; ++index) {
|
||||
const std::size_t offset = index * 2u;
|
||||
const double left_real = left[offset];
|
||||
const double left_imag = left[offset + 1u];
|
||||
const double right_real = right[offset];
|
||||
const double right_imag = right[offset + 1u];
|
||||
out[offset] = left_real * right_real - left_imag * right_imag;
|
||||
out[offset + 1u] = left_real * right_imag + left_imag * right_real;
|
||||
}
|
||||
}
|
||||
|
||||
const Kernels kAvx512{
|
||||
&fft_butterflies_avx512,
|
||||
nullptr, // qmf_synthesis_basis: four ranks do not fill a 512-bit vector
|
||||
&hybrid_low_join_avx512,
|
||||
&render_hybrid_path_avx512,
|
||||
&complex_axpy_avx512,
|
||||
&qmf_analysis_taps_avx512,
|
||||
&complex_multiply_avx512,
|
||||
};
|
||||
|
||||
} // namespace
|
||||
|
||||
const Kernels& kernels_intrin_avx512() noexcept { return kAvx512; }
|
||||
|
||||
} // namespace joc::simd
|
||||
@@ -0,0 +1,311 @@
|
||||
// AArch64 NEON (ASIMD) implementation of the dispatched kernels.
|
||||
//
|
||||
// CMake gives exactly the `kernels_intrin_*.cpp` units their ISA flag, so no
|
||||
// baseline unit can inherit one by accident.
|
||||
//
|
||||
// Unlike the x86 units this one needs no feature probe -- ASIMD is architectural
|
||||
// for AArch64 -- but it is still only reached through the dispatcher, so
|
||||
// JOC_SIMD=scalar, sse2 or neon all stay available on the same binary.
|
||||
//
|
||||
// A NEON register holds two doubles, so a vector carries two lanes where an AVX2
|
||||
// vector carries four. The lanes are the same independent work items as in the
|
||||
// AVX2 unit, and each lane performs exactly the scalar reference's operations in
|
||||
// the scalar reference's order: separate multiplies and adds, never a fused
|
||||
// multiply-add, and never a reassociated sum.
|
||||
//
|
||||
// Nothing in this file may have dynamic initialisation (it is linked into the
|
||||
// same image as the baseline code and would run before the dispatcher).
|
||||
//
|
||||
// NOTE: the unit tests do not exercise this unit. It is built by the AArch64 CI
|
||||
// job, and its bit-exactness is argued from the shared kernel contract (simd.h)
|
||||
// rather than measured here.
|
||||
|
||||
#include "simd/simd.h"
|
||||
|
||||
#include <arm_neon.h>
|
||||
|
||||
#include <cstddef>
|
||||
|
||||
#include "foundation/fft.h"
|
||||
|
||||
namespace joc::simd {
|
||||
namespace {
|
||||
|
||||
constexpr std::size_t kFftSize = dsp::kQmfFftSize;
|
||||
static_assert(kFftSize == 128u, "the QMF transform is the 128-point case");
|
||||
|
||||
// ------------------------------------------------------------ radix-2 FFT ----
|
||||
//
|
||||
// Two butterflies per iteration, held as separate real/imaginary vectors: vld2q
|
||||
// loads two interleaved complexes and deinterleaves them, vst2q puts them back.
|
||||
// `sum = even + odd * w` and `difference = even - odd * w` are then two add/sub
|
||||
// pairs, and the complex product is four multiplies and two add/subs, which is
|
||||
// what the scalar kernel writes.
|
||||
inline float64x2x2_t complex_mul(float64x2_t odd_real, float64x2_t odd_imag,
|
||||
float64x2_t weight_real, float64x2_t weight_imag) {
|
||||
float64x2x2_t result;
|
||||
result.val[0] = vsubq_f64(vmulq_f64(odd_real, weight_real), vmulq_f64(odd_imag, weight_imag));
|
||||
result.val[1] = vaddq_f64(vmulq_f64(odd_real, weight_imag), vmulq_f64(odd_imag, weight_real));
|
||||
return result;
|
||||
}
|
||||
|
||||
// Stages 0 and 1 hold fewer than two complexes per group, so their butterflies are
|
||||
// packed from neighbouring groups; the factor depends only on the offset inside a
|
||||
// group, so the two lanes share it.
|
||||
void fft_butterflies_neon(double* buffer, std::size_t size, const double* twiddle,
|
||||
const std::size_t* stage_begin) noexcept {
|
||||
// Stage 0: length 2, one butterfly per group. The two lanes take the two
|
||||
// halves of two neighbouring groups, which the transposes select.
|
||||
{
|
||||
const double* table = twiddle + 2u * stage_begin[0];
|
||||
const float64x2_t weight_real = vdupq_n_f64(table[0]);
|
||||
const float64x2_t weight_imag = vdupq_n_f64(table[1]);
|
||||
for (std::size_t index = 0u; index < 2u * size; index += 8u) {
|
||||
const float64x2_t first = vld1q_f64(buffer + index);
|
||||
const float64x2_t second = vld1q_f64(buffer + index + 2u);
|
||||
const float64x2_t third = vld1q_f64(buffer + index + 4u);
|
||||
const float64x2_t fourth = vld1q_f64(buffer + index + 6u);
|
||||
const float64x2_t even_real = vtrn1q_f64(first, third);
|
||||
const float64x2_t even_imag = vtrn2q_f64(first, third);
|
||||
const float64x2_t odd_real = vtrn1q_f64(second, fourth);
|
||||
const float64x2_t odd_imag = vtrn2q_f64(second, fourth);
|
||||
const float64x2x2_t rotated =
|
||||
complex_mul(odd_real, odd_imag, weight_real, weight_imag);
|
||||
const float64x2_t sum_real = vaddq_f64(even_real, rotated.val[0]);
|
||||
const float64x2_t sum_imag = vaddq_f64(even_imag, rotated.val[1]);
|
||||
const float64x2_t difference_real = vsubq_f64(even_real, rotated.val[0]);
|
||||
const float64x2_t difference_imag = vsubq_f64(even_imag, rotated.val[1]);
|
||||
// sum.* are the two groups' even-position results and difference.* their
|
||||
// odd-position ones; a transpose of the two component vectors puts each
|
||||
// complex back together, in the order the groups appear.
|
||||
vst1q_f64(buffer + index, vtrn1q_f64(sum_real, sum_imag));
|
||||
vst1q_f64(buffer + index + 2u, vtrn1q_f64(difference_real, difference_imag));
|
||||
vst1q_f64(buffer + index + 4u, vtrn2q_f64(sum_real, sum_imag));
|
||||
vst1q_f64(buffer + index + 6u, vtrn2q_f64(difference_real, difference_imag));
|
||||
}
|
||||
}
|
||||
|
||||
std::size_t stage = 1u;
|
||||
for (std::size_t half = 2u; half < size; half <<= 1u, ++stage) {
|
||||
const std::size_t length = half << 1u;
|
||||
const double* table = twiddle + 2u * stage_begin[stage];
|
||||
for (std::size_t start = 0u; start < size; start += length) {
|
||||
for (std::size_t offset = 0u; offset < half; offset += 2u) {
|
||||
double* even_values = buffer + (start + offset) * 2u;
|
||||
double* odd_values = even_values + half * 2u;
|
||||
const float64x2x2_t even = vld2q_f64(even_values);
|
||||
const float64x2x2_t odd = vld2q_f64(odd_values);
|
||||
const float64x2x2_t weight = vld2q_f64(table + offset * 2u);
|
||||
const float64x2x2_t rotated =
|
||||
complex_mul(odd.val[0], odd.val[1], weight.val[0], weight.val[1]);
|
||||
float64x2x2_t sum;
|
||||
float64x2x2_t difference;
|
||||
sum.val[0] = vaddq_f64(even.val[0], rotated.val[0]);
|
||||
sum.val[1] = vaddq_f64(even.val[1], rotated.val[1]);
|
||||
difference.val[0] = vsubq_f64(even.val[0], rotated.val[0]);
|
||||
difference.val[1] = vsubq_f64(even.val[1], rotated.val[1]);
|
||||
vst2q_f64(even_values, sum);
|
||||
vst2q_f64(odd_values, difference);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// --------------------------------------------------- QMF synthesis basis -----
|
||||
//
|
||||
// The four ranks of a band are four independent dot products over the same row, so
|
||||
// two registers carry four lanes. `Rows` rows run side by side because a band's
|
||||
// chain is 128 dependent adds: the cure is more independent chains, not a shorter
|
||||
// chain.
|
||||
template <int Rows>
|
||||
void synthesize_rows_neon(const double* values, const double* basis, double* out) noexcept {
|
||||
constexpr std::size_t kRanks = kSynthesisRanks;
|
||||
constexpr std::size_t kTaps = kSynthesisTaps;
|
||||
constexpr std::size_t kRowOut = kSynthesisBands * kRanks;
|
||||
for (std::size_t band = 0u; band < kSynthesisBands; ++band) {
|
||||
const double* table = basis + band * kTaps * kRanks;
|
||||
float64x2_t low[Rows];
|
||||
float64x2_t high[Rows];
|
||||
for (int row = 0; row < Rows; ++row) {
|
||||
low[row] = vdupq_n_f64(0.0);
|
||||
high[row] = vdupq_n_f64(0.0);
|
||||
}
|
||||
for (std::size_t tap = 0u; tap < kTaps; ++tap) {
|
||||
const float64x2_t weight_low = vld1q_f64(table + tap * kRanks);
|
||||
const float64x2_t weight_high = vld1q_f64(table + tap * kRanks + 2u);
|
||||
for (int row = 0; row < Rows; ++row) {
|
||||
const float64x2_t factor = vdupq_n_f64(values[row * kTaps + tap]);
|
||||
low[row] = vaddq_f64(low[row], vmulq_f64(factor, weight_low));
|
||||
high[row] = vaddq_f64(high[row], vmulq_f64(factor, weight_high));
|
||||
}
|
||||
}
|
||||
for (int row = 0; row < Rows; ++row) {
|
||||
vst1q_f64(out + row * kRowOut + band * kRanks, low[row]);
|
||||
vst1q_f64(out + row * kRowOut + band * kRanks + 2u, high[row]);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void qmf_synthesis_basis_neon(const double* values, const double* basis, double* out,
|
||||
std::size_t rows) noexcept {
|
||||
constexpr std::size_t kTaps = kSynthesisTaps;
|
||||
constexpr std::size_t kRowOut = kSynthesisBands * kSynthesisRanks;
|
||||
std::size_t row = 0u;
|
||||
for (; row + 4u <= rows; row += 4u) {
|
||||
synthesize_rows_neon<4>(values + row * kTaps, basis, out + row * kRowOut);
|
||||
}
|
||||
for (; row < rows; ++row) {
|
||||
synthesize_rows_neon<1>(values + row * kTaps, basis, out + row * kRowOut);
|
||||
}
|
||||
}
|
||||
|
||||
// ------------------------------------------------------ Hybrid low join -----
|
||||
//
|
||||
// One term against all 32 outputs, two outputs per register: the outputs are
|
||||
// independent accumulations over the same terms, which is what the lanes carry.
|
||||
void hybrid_low_join_neon(const double* values, const double* kernel, double* out,
|
||||
std::size_t rows) noexcept {
|
||||
constexpr std::size_t kTerms = kHybridTerms;
|
||||
constexpr std::size_t kOutputs = kHybridOutputs;
|
||||
static_assert(kOutputs == 32u, "sixteen 128-bit accumulators cover one row");
|
||||
for (std::size_t row = 0u; row < rows; ++row) {
|
||||
const double* source = values + row * kTerms;
|
||||
double* destination = out + row * kOutputs;
|
||||
float64x2_t partial[16];
|
||||
for (int chunk = 0; chunk < 16; ++chunk) {
|
||||
partial[chunk] = vdupq_n_f64(0.0);
|
||||
}
|
||||
for (std::size_t term = 0u; term < kTerms; ++term) {
|
||||
const double value = source[term];
|
||||
if (value == 0.0) {
|
||||
continue;
|
||||
}
|
||||
const double* weights = kernel + term * kOutputs;
|
||||
const float64x2_t factor = vdupq_n_f64(value);
|
||||
for (int chunk = 0; chunk < 16; ++chunk) {
|
||||
partial[chunk] = vaddq_f64(
|
||||
partial[chunk], vmulq_f64(factor, vld1q_f64(weights + chunk * 2)));
|
||||
}
|
||||
}
|
||||
for (int chunk = 0; chunk < 16; ++chunk) {
|
||||
vst1q_f64(destination + chunk * 2, partial[chunk]);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// --------------------------------------------------- Rendered hybrid path ---
|
||||
//
|
||||
// Two bands per iteration, real and imaginary held apart and reloaded from memory
|
||||
// in the caller's own layout: `out += h * t * scale` for the real part and
|
||||
// `out += h * swapped(t) * scale` for the imaginary one, with the scale applied to
|
||||
// each product separately.
|
||||
void render_hybrid_path_neon(double* out, const double* transfer, const double* history0,
|
||||
const double* history1, double scale) noexcept {
|
||||
const float64x2_t factor = vdupq_n_f64(scale);
|
||||
const double* histories[2] = {history0, history1};
|
||||
for (std::size_t ear = 0u; ear < 2u; ++ear) {
|
||||
const double* source = histories[ear];
|
||||
const double* taps = transfer + ear * kPathBands * 2u;
|
||||
double* destination = out + ear * kPathBands * 2u;
|
||||
std::size_t band = 0u;
|
||||
for (; band + 2u <= kPathBands; band += 2u) {
|
||||
const float64x2x2_t history = vld2q_f64(source + band * 2u);
|
||||
const float64x2x2_t tap = vld2q_f64(taps + band * 2u);
|
||||
float64x2x2_t output = vld2q_f64(destination + band * 2u);
|
||||
// The caller scales each product on its own before combining, so the
|
||||
// scale is a separate multiply of every product -- not a multiply of the
|
||||
// difference and the sum, which would round differently.
|
||||
const float64x2_t real = vsubq_f64(
|
||||
vmulq_f64(vmulq_f64(history.val[0], tap.val[0]), factor),
|
||||
vmulq_f64(vmulq_f64(history.val[1], tap.val[1]), factor));
|
||||
const float64x2_t imaginary = vaddq_f64(
|
||||
vmulq_f64(vmulq_f64(history.val[0], tap.val[1]), factor),
|
||||
vmulq_f64(vmulq_f64(history.val[1], tap.val[0]), factor));
|
||||
output.val[0] = vaddq_f64(output.val[0], real);
|
||||
output.val[1] = vaddq_f64(output.val[1], imaginary);
|
||||
vst2q_f64(destination + band * 2u, output);
|
||||
}
|
||||
for (; band < kPathBands; ++band) {
|
||||
const double source_real = source[band * 2u];
|
||||
const double source_imag = source[band * 2u + 1u];
|
||||
const double tap_real = taps[band * 2u];
|
||||
const double tap_imag = taps[band * 2u + 1u];
|
||||
destination[band * 2u] +=
|
||||
source_real * tap_real * scale - source_imag * tap_imag * scale;
|
||||
destination[band * 2u + 1u] +=
|
||||
source_real * tap_imag * scale + source_imag * tap_real * scale;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Scalar times a run of interleaved complexes, two complexes per iteration.
|
||||
void complex_axpy_neon(double* out, const double* field, double scale,
|
||||
std::size_t count) noexcept {
|
||||
const float64x2_t factor = vdupq_n_f64(scale);
|
||||
std::size_t index = 0u;
|
||||
for (; index + 2u <= count; index += 2u) {
|
||||
const float64x2_t values = vld1q_f64(field + index * 2u);
|
||||
const float64x2_t accumulated = vld1q_f64(out + index * 2u);
|
||||
vst1q_f64(out + index * 2u, vaddq_f64(accumulated, vmulq_f64(values, factor)));
|
||||
}
|
||||
for (; index < count; ++index) {
|
||||
out[index * 2u] += field[index * 2u] * scale;
|
||||
out[index * 2u + 1u] += field[index * 2u + 1u] * scale;
|
||||
}
|
||||
}
|
||||
|
||||
// QMF analysis polyphase accumulate, two bands per iteration.
|
||||
void qmf_analysis_taps_neon(double* out, const double* source, const double* coefficients,
|
||||
std::size_t rows) noexcept {
|
||||
for (std::size_t row = 0u; row < rows; ++row) {
|
||||
double* destination = out + row * kQmfAnalysisBands;
|
||||
const double* values = source + row * kQmfAnalysisBands;
|
||||
for (std::size_t band = 0u; band < kQmfAnalysisBands; band += 2u) {
|
||||
vst1q_f64(destination + band,
|
||||
vaddq_f64(vld1q_f64(destination + band),
|
||||
vmulq_f64(vld1q_f64(values + band),
|
||||
vld1q_f64(coefficients + band))));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Element-wise complex product of two runs, two complexes per iteration.
|
||||
void complex_multiply_neon(double* out, const double* left, const double* right,
|
||||
std::size_t count) noexcept {
|
||||
std::size_t index = 0u;
|
||||
for (; index + 2u <= count; index += 2u) {
|
||||
const float64x2x2_t first = vld2q_f64(left + index * 2u);
|
||||
const float64x2x2_t second = vld2q_f64(right + index * 2u);
|
||||
float64x2x2_t result;
|
||||
result.val[0] = vsubq_f64(vmulq_f64(first.val[0], second.val[0]),
|
||||
vmulq_f64(first.val[1], second.val[1]));
|
||||
result.val[1] = vaddq_f64(vmulq_f64(first.val[0], second.val[1]),
|
||||
vmulq_f64(first.val[1], second.val[0]));
|
||||
vst2q_f64(out + index * 2u, result);
|
||||
}
|
||||
for (; index < count; ++index) {
|
||||
const std::size_t offset = index * 2u;
|
||||
const double left_real = left[offset];
|
||||
const double left_imag = left[offset + 1u];
|
||||
const double right_real = right[offset];
|
||||
const double right_imag = right[offset + 1u];
|
||||
out[offset] = left_real * right_real - left_imag * right_imag;
|
||||
out[offset + 1u] = left_real * right_imag + left_imag * right_real;
|
||||
}
|
||||
}
|
||||
|
||||
const Kernels kNeon{
|
||||
&fft_butterflies_neon,
|
||||
&qmf_synthesis_basis_neon,
|
||||
&hybrid_low_join_neon,
|
||||
&render_hybrid_path_neon,
|
||||
&complex_axpy_neon,
|
||||
&qmf_analysis_taps_neon,
|
||||
&complex_multiply_neon,
|
||||
};
|
||||
|
||||
} // namespace
|
||||
|
||||
const Kernels& kernels_intrin_neon() noexcept { return kNeon; }
|
||||
|
||||
} // namespace joc::simd
|
||||
@@ -0,0 +1,156 @@
|
||||
// Scalar reference implementations of the dispatched DSP kernels.
|
||||
//
|
||||
// This unit is compiled with the baseline ISA of the target (no /arch flag on
|
||||
// MSVC, no -m flag elsewhere) and is the ground truth every vector path is
|
||||
// compared against: JOC_SIMD=scalar selects exactly these functions, and their
|
||||
// output must be bit-identical to any ISA path's.
|
||||
//
|
||||
// The arithmetic below is deliberately written with std::complex<double>, which
|
||||
// on MSVC expands to the plain `re * rr - im * ri` / `re * ri + im * rr` pair
|
||||
// (no helper call, no scaling trick), so the vector paths have an unambiguous
|
||||
// two-roundings-per-component target to reproduce.
|
||||
|
||||
#include "simd/simd.h"
|
||||
|
||||
#include <complex>
|
||||
|
||||
#include "foundation/fft.h"
|
||||
|
||||
namespace joc::simd {
|
||||
namespace {
|
||||
|
||||
using Complex = dsp::Complex;
|
||||
|
||||
// Exactly the stage loop of FftPlan::apply, over a table this kernel does not own,
|
||||
// for any power-of-two size. The bit-reversal permutation stays with the caller:
|
||||
// both call sites already write their input in permuted order.
|
||||
void fft_butterflies_scalar(double* raw_data, std::size_t size, const double* raw_twiddle,
|
||||
const std::size_t* stage_begin) noexcept {
|
||||
auto* data = reinterpret_cast<Complex*>(raw_data);
|
||||
const auto* twiddle = reinterpret_cast<const Complex*>(raw_twiddle);
|
||||
std::size_t stage = 0u;
|
||||
for (std::size_t length = 2u; length <= size; length <<= 1u, ++stage) {
|
||||
const Complex* table = twiddle + stage_begin[stage];
|
||||
for (std::size_t start = 0u; start < size; start += length) {
|
||||
for (std::size_t offset = 0u; offset < length / 2u; ++offset) {
|
||||
const Complex even = data[start + offset];
|
||||
const Complex odd = data[start + offset + length / 2u] * table[offset];
|
||||
data[start + offset] = even + odd;
|
||||
data[start + offset + length / 2u] = even - odd;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The reference for the synthesis basis: one accumulator per output, taps in
|
||||
// increasing order, one rounding per multiply and per add. This is the shape
|
||||
// both callers write, with the basis rows reordered rank-minor -- the products
|
||||
// and their order are the caller's.
|
||||
void qmf_synthesis_basis_scalar(const double* values, const double* basis, double* out,
|
||||
std::size_t rows) noexcept {
|
||||
for (std::size_t row = 0u; row < rows; ++row) {
|
||||
const double* source = values + row * kSynthesisTaps;
|
||||
double* destination = out + row * kSynthesisBands * kSynthesisRanks;
|
||||
for (std::size_t band = 0u; band < kSynthesisBands; ++band) {
|
||||
const double* table = basis + band * kSynthesisTaps * kSynthesisRanks;
|
||||
for (std::size_t rank = 0u; rank < kSynthesisRanks; ++rank) {
|
||||
double sum = 0.0;
|
||||
for (std::size_t tap = 0u; tap < kSynthesisTaps; ++tap) {
|
||||
sum += source[tap] * table[tap * kSynthesisRanks + rank];
|
||||
}
|
||||
destination[band * kSynthesisRanks + rank] = sum;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The reference for the hybrid low join: one accumulator per output, terms in the
|
||||
// caller's order, one rounding per multiply and per add, zero terms skipped.
|
||||
void hybrid_low_join_scalar(const double* values, const double* kernel, double* out,
|
||||
std::size_t rows) noexcept {
|
||||
for (std::size_t row = 0u; row < rows; ++row) {
|
||||
const double* source = values + row * kHybridTerms;
|
||||
double* destination = out + row * kHybridOutputs;
|
||||
for (std::size_t output = 0u; output < kHybridOutputs; ++output) {
|
||||
destination[output] = 0.0;
|
||||
}
|
||||
for (std::size_t term = 0u; term < kHybridTerms; ++term) {
|
||||
const double value = source[term];
|
||||
if (value == 0.0) {
|
||||
continue;
|
||||
}
|
||||
const double* weights = kernel + term * kHybridOutputs;
|
||||
for (std::size_t output = 0u; output < kHybridOutputs; ++output) {
|
||||
destination[output] += value * weights[output];
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The reference for one rendered path: each band of each ear accumulates its own
|
||||
// value, in the caller's order, with the scale applied to each product.
|
||||
void render_hybrid_path_scalar(double* out, const double* transfer, const double* history0,
|
||||
const double* history1, double scale) noexcept {
|
||||
const double* histories[2] = {history0, history1};
|
||||
for (std::size_t ear = 0u; ear < 2u; ++ear) {
|
||||
const double* source = histories[ear];
|
||||
const double* taps = transfer + ear * kPathBands * 2u;
|
||||
double* destination = out + ear * kPathBands * 2u;
|
||||
for (std::size_t band = 0u; band < kPathBands; ++band) {
|
||||
const double source_real = source[band * 2u];
|
||||
const double source_imag = source[band * 2u + 1u];
|
||||
const double tap_real = taps[band * 2u];
|
||||
const double tap_imag = taps[band * 2u + 1u];
|
||||
destination[band * 2u] +=
|
||||
source_real * tap_real * scale - source_imag * tap_imag * scale;
|
||||
destination[band * 2u + 1u] +=
|
||||
source_real * tap_imag * scale + source_imag * tap_real * scale;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Neighbouring complexes and neighbouring bands are independent outputs, so the
|
||||
// references for the two accumulates are plain element-wise loops.
|
||||
void complex_axpy_scalar(double* out, const double* field, double scale,
|
||||
std::size_t count) noexcept {
|
||||
for (std::size_t index = 0u; index < count * 2u; ++index) {
|
||||
out[index] += field[index] * scale;
|
||||
}
|
||||
}
|
||||
|
||||
void qmf_analysis_taps_scalar(double* out, const double* source, const double* coefficients,
|
||||
std::size_t rows) noexcept {
|
||||
for (std::size_t row = 0u; row < rows; ++row) {
|
||||
double* destination = out + row * kQmfAnalysisBands;
|
||||
const double* values = source + row * kQmfAnalysisBands;
|
||||
for (std::size_t band = 0u; band < kQmfAnalysisBands; ++band) {
|
||||
destination[band] += values[band] * coefficients[band];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void complex_multiply_scalar(double* out, const double* left, const double* right,
|
||||
std::size_t count) noexcept {
|
||||
auto* destination = reinterpret_cast<Complex*>(out);
|
||||
const auto* first = reinterpret_cast<const Complex*>(left);
|
||||
const auto* second = reinterpret_cast<const Complex*>(right);
|
||||
for (std::size_t index = 0u; index < count; ++index) {
|
||||
destination[index] = first[index] * second[index];
|
||||
}
|
||||
}
|
||||
|
||||
const Kernels kScalar{
|
||||
&fft_butterflies_scalar,
|
||||
&qmf_synthesis_basis_scalar,
|
||||
&hybrid_low_join_scalar,
|
||||
&render_hybrid_path_scalar,
|
||||
&complex_axpy_scalar,
|
||||
&qmf_analysis_taps_scalar,
|
||||
&complex_multiply_scalar,
|
||||
};
|
||||
|
||||
} // namespace
|
||||
|
||||
const Kernels& kernels_scalar() noexcept { return kScalar; }
|
||||
|
||||
} // namespace joc::simd
|
||||
@@ -0,0 +1,209 @@
|
||||
#pragma once
|
||||
|
||||
#include <cstddef>
|
||||
|
||||
// Runtime-dispatched SIMD kernels for the DSP core.
|
||||
//
|
||||
// Module layout (the ISA lives in the file name, never in a subdirectory):
|
||||
//
|
||||
// simd.h this contract -- the only header a caller includes
|
||||
// cpu_probe.{h,cpp} "can this CPU and OS run ISA X", no policy
|
||||
// dispatch.cpp JOC_SIMD + the ladder + the kernel table
|
||||
// kernels_scalar.cpp the reference implementation, Isa::scalar
|
||||
// kernels_intrin_avx2.cpp /arch:AVX2 -mavx2 -- exactly the
|
||||
// kernels_intrin_avx512.cpp /arch:AVX512 -mavx512f -- `kernels_intrin_*`
|
||||
// kernels_intrin_neon.cpp AArch64 default -- units get a wider flag
|
||||
//
|
||||
// The scalar reference in kernels_scalar.cpp fixes both the data layout and the
|
||||
// arithmetic: a vector path may only compute *independent* outputs in parallel
|
||||
// lanes. It may never reassociate an accumulation, contract a multiply into an
|
||||
// FMA, or change the order in which one output's terms are summed, so every ISA
|
||||
// path reproduces the scalar bits exactly and JOC_SIMD=scalar is a valid
|
||||
// reference for any of them. Kernel-level layout changes (AoS -> SoA inside a
|
||||
// kernel, for instance) are fine as long as each individual output value is
|
||||
// still produced by the same sequence of roundings.
|
||||
//
|
||||
// MSVC has no function-level ISA attribute, so each ISA lives in its own
|
||||
// translation unit compiled with its own /arch (or -m) flag and the dispatcher
|
||||
// picks one at run time: CPUID + XGETBV on x86-64, the architectural AArch64
|
||||
// baseline (NEON/ASIMD needs no probing) elsewhere.
|
||||
//
|
||||
// The ISA units are linked into the same binaries as the baseline code, so they
|
||||
// must stay free of objects with dynamic initialisation: a global constructor
|
||||
// would execute ISA-specific instructions before the dispatcher has had a chance
|
||||
// to look at the CPU. Constant tables are fine -- they live in .rdata.
|
||||
namespace joc::simd {
|
||||
|
||||
enum class Isa {
|
||||
scalar = 0,
|
||||
sse2,
|
||||
avx2,
|
||||
avx512,
|
||||
neon,
|
||||
};
|
||||
|
||||
// One entry per dispatched kernel.
|
||||
enum class Kernel {
|
||||
// Power-of-two radix-2 DIT butterfly cascade, in place, over `size` interleaved
|
||||
// (re, im) pairs already in bit-reversed order. `twiddle` is the caller's own
|
||||
// factor table, read as interleaved pairs: stage s (length 2^(s+1)) takes its
|
||||
// 2^s factors starting at stage_begin[s], a *complex* index into `twiddle`.
|
||||
fft_butterflies = 0,
|
||||
// QMF synthesis basis application, see kSynthesis* below.
|
||||
qmf_synthesis_basis,
|
||||
// Hybrid analysis low join, see kHybrid* below.
|
||||
hybrid_low_join,
|
||||
// Hybrid-domain path rendering, see kPathBands below.
|
||||
render_hybrid_path,
|
||||
// Scalar-times-vector accumulate over interleaved complexes.
|
||||
complex_axpy,
|
||||
// QMF analysis polyphase accumulate, see kQmfAnalysisBands below.
|
||||
qmf_analysis_taps,
|
||||
// Element-wise complex product of two whole runs.
|
||||
complex_multiply,
|
||||
count,
|
||||
};
|
||||
|
||||
// Both filterbanks apply the same map: for every output row and every one of the
|
||||
// 64 bands, four ranks each accumulate 128 taps.
|
||||
inline constexpr std::size_t kSynthesisBands = 64;
|
||||
inline constexpr std::size_t kSynthesisRanks = 4;
|
||||
inline constexpr std::size_t kSynthesisTaps = 128;
|
||||
|
||||
// ... and both derive the same 16 low hybrid bands from 78 terms: 3 parents x 2
|
||||
// components x 13 lags.
|
||||
inline constexpr std::size_t kHybridTerms = 78;
|
||||
inline constexpr std::size_t kHybridOutputs = 32;
|
||||
// Rows the callers stage at a time: enough to keep the kernel busy, small enough
|
||||
// that the staged values stay in the first-level cache.
|
||||
inline constexpr std::size_t kHybridJoinBlock = 32;
|
||||
|
||||
// Bands in one ear of a rendered path (the SOFA renderer's 77-band hybrid domain).
|
||||
inline constexpr std::size_t kPathBands = 77;
|
||||
|
||||
// The cascade packs whole groups into a vector, so it needs at least one full
|
||||
// packing: a size of 8 covers a 512-bit stage. Smaller transforms are rare enough
|
||||
// (and short enough) that the caller's portable loop is the right answer there.
|
||||
inline constexpr std::size_t kMinVectorFftSize = 8;
|
||||
|
||||
// Bands in one QMF analysis row (the polyphase accumulator of both filterbanks).
|
||||
inline constexpr std::size_t kQmfAnalysisBands = 64;
|
||||
|
||||
const char* isa_name(Isa isa) noexcept;
|
||||
|
||||
// Whether this CPU *and* the OS state it has to save can execute the ISA. The
|
||||
// answer never changes, so it is computed once.
|
||||
bool isa_supported(Isa isa) noexcept;
|
||||
|
||||
// The ISA the dispatcher settled on, after applying a JOC_SIMD override. This
|
||||
// is the widest ISA considered, not necessarily the one that serves every
|
||||
// kernel: active_isa() answers that per kernel.
|
||||
Isa selected_isa() noexcept;
|
||||
|
||||
// Which ISA actually implements `kernel` after the per-kernel fallback.
|
||||
Isa active_isa(Kernel kernel) noexcept;
|
||||
|
||||
// The dispatched kernels; a slot is never null.
|
||||
struct Kernels {
|
||||
void (*fft_butterflies)(double* data, std::size_t size, const double* twiddle,
|
||||
const std::size_t* stage_begin) noexcept = nullptr;
|
||||
|
||||
// values: [rows][kSynthesisTaps] real/imaginary of the 64 bands of one row.
|
||||
// basis: the same weights the caller holds, reordered rank-minor, so that
|
||||
// (band, tap) addresses its kSynthesisRanks weights contiguously:
|
||||
// basis[(band * kSynthesisTaps + tap) * kSynthesisRanks + rank].
|
||||
// out: [rows][kSynthesisBands][kSynthesisRanks], rank-minor as well.
|
||||
//
|
||||
// Each of the four ranks is an independent dot product over the row, so the
|
||||
// four of them are what shares a vector; every lane keeps the tap order and
|
||||
// the two roundings of the caller's `sum += values[tap] * weight`.
|
||||
void (*qmf_synthesis_basis)(const double* values, const double* basis, double* out,
|
||||
std::size_t rows) noexcept = nullptr;
|
||||
|
||||
// Hybrid analysis low join. Every output row accumulates kHybridTerms values
|
||||
// into kHybridOutputs outputs (16 bands x a real/imaginary pair, component
|
||||
// minor):
|
||||
// out[row][output] = sum over term of values[row][term] * kernel[term][output]
|
||||
// values is [rows][kHybridTerms] in the caller's own term order and kernel is
|
||||
// the caller's taps regrouped to that same order, so that one term's 32
|
||||
// weights are contiguous. Terms whose value is exactly zero are skipped, as
|
||||
// both callers do: adding a zero product to a lane can only leave it alone
|
||||
// (no lane's running sum can be a negative zero, since it starts at +0 and
|
||||
// sums without ever producing one).
|
||||
void (*hybrid_low_join)(const double* values, const double* kernel, double* out,
|
||||
std::size_t rows) noexcept = nullptr;
|
||||
|
||||
// One rendered path. For each ear e and band b, with the ear's history sample
|
||||
// h and the path's transfer t:
|
||||
// out[e][b].re += h.re * t[e][b].re * scale - h.im * t[e][b].im * scale
|
||||
// out[e][b].im += h.re * t[e][b].im * scale + h.im * t[e][b].re * scale
|
||||
// `history0` and `history1` are the two ears' [kPathBands] interleaved complex
|
||||
// rows (they come from different places in the history ring); `transfer` and
|
||||
// `out` are [2][kPathBands] interleaved complexes. The scale multiplies each
|
||||
// product separately, exactly as the caller writes it, so the bands -- which
|
||||
// are independent accumulations into independent outputs -- are what the lanes
|
||||
// carry.
|
||||
void (*render_hybrid_path)(double* out, const double* transfer, const double* history0,
|
||||
const double* history1, double scale) noexcept = nullptr;
|
||||
|
||||
// `count` interleaved complexes, accumulated in place:
|
||||
// out[i] += field[i] * scale
|
||||
// Every element is an independent accumulation of one product, so the lanes
|
||||
// carry neighbouring elements and each one keeps the caller's multiply-then-add.
|
||||
void (*complex_axpy)(double* out, const double* field, double scale,
|
||||
std::size_t count) noexcept = nullptr;
|
||||
|
||||
// QMF analysis polyphase accumulate: `rows` rows of kQmfAnalysisBands bands,
|
||||
// out[row][band] += source[row][band] * coefficients[band]
|
||||
// Neighbouring bands are neighbouring outputs, so they are what fills a vector;
|
||||
// each band accumulates its own product once, in the caller's order.
|
||||
void (*qmf_analysis_taps)(double* out, const double* source, const double* coefficients,
|
||||
std::size_t rows) noexcept = nullptr;
|
||||
|
||||
// `count` interleaved complexes, multiplied element by element:
|
||||
// out[i] = left[i] * right[i]
|
||||
// with the caller's `re * re - im * im` and `re * im + im * re`, two roundings
|
||||
// per component. Neighbouring complexes are independent products.
|
||||
void (*complex_multiply)(double* out, const double* left, const double* right,
|
||||
std::size_t count) noexcept = nullptr;
|
||||
};
|
||||
|
||||
const Kernels& kernels() noexcept;
|
||||
|
||||
// Convenience wrappers.
|
||||
inline void fft_butterflies(double* data, std::size_t size, const double* twiddle,
|
||||
const std::size_t* stage_begin) noexcept {
|
||||
kernels().fft_butterflies(data, size, twiddle, stage_begin);
|
||||
}
|
||||
|
||||
inline void qmf_synthesis_basis(const double* values, const double* basis, double* out,
|
||||
std::size_t rows) noexcept {
|
||||
kernels().qmf_synthesis_basis(values, basis, out, rows);
|
||||
}
|
||||
|
||||
inline void hybrid_low_join(const double* values, const double* kernel, double* out,
|
||||
std::size_t rows) noexcept {
|
||||
kernels().hybrid_low_join(values, kernel, out, rows);
|
||||
}
|
||||
|
||||
inline void render_hybrid_path(double* out, const double* transfer, const double* history0,
|
||||
const double* history1, double scale) noexcept {
|
||||
kernels().render_hybrid_path(out, transfer, history0, history1, scale);
|
||||
}
|
||||
|
||||
inline void complex_axpy(double* out, const double* field, double scale,
|
||||
std::size_t count) noexcept {
|
||||
kernels().complex_axpy(out, field, scale, count);
|
||||
}
|
||||
|
||||
inline void qmf_analysis_taps(double* out, const double* source, const double* coefficients,
|
||||
std::size_t rows) noexcept {
|
||||
kernels().qmf_analysis_taps(out, source, coefficients, rows);
|
||||
}
|
||||
|
||||
inline void complex_multiply(double* out, const double* left, const double* right,
|
||||
std::size_t count) noexcept {
|
||||
kernels().complex_multiply(out, left, right, count);
|
||||
}
|
||||
|
||||
} // namespace joc::simd
|
||||
Reference in new issue
Block a user