From faf8a921c2431680c5d87e571f442b448fb5bc00 Mon Sep 17 00:00:00 2001 From: TheM14 Date: Sun, 1 Nov 2026 00:57:49 +0800 Subject: [PATCH] Render SOFA directly with full HRIR interpolation SOFA inputs now use delay-aligned local interpolation and partitioned convolution instead of compiling a compressed directional field. - preserve full FIR support, complex phase and explicit delay handling - add log-magnitude/phase and local magnitude-corrected interpolation - support Near/Mid/Far with a shared calibration anchor - retain .jochrtf as an explicit lightweight input and offline export - keep personalized headphone rendering unchanged - update the CLI, reports and bilingual documentation Checked: 21 automated tests, 8 bit-exact personalized regressions, and full-song D2 renders in Near/Mid/Far. --- README.en.md | 36 ++-- README.md | 26 +-- docs/binaural.en.md | 289 ++++----------------------------- docs/binaural.md | 259 +++++------------------------ main.py | 92 +++++------ src/binaural_reference.py | 137 ++++++++++++++++ src/binaural_renderer.py | 66 ++++---- src/compile_hrtf.py | 26 +++ src/oamd_bits.py | 40 +++-- src/partitioned_convolution.py | 217 +++++++++++++++++++++++++ src/sofa_hrir.py | 220 +++++++++++++++++++++++++ src/sofa_metadata.py | 53 ++++++ src/sofa_reference_backend.py | 270 ++++++++++++++++++++++++++++++ 13 files changed, 1140 insertions(+), 591 deletions(-) create mode 100644 src/binaural_reference.py create mode 100644 src/compile_hrtf.py create mode 100644 src/partitioned_convolution.py create mode 100644 src/sofa_hrir.py create mode 100644 src/sofa_metadata.py create mode 100644 src/sofa_reference_backend.py diff --git a/README.en.md b/README.en.md index 4611f35..dfb4a5b 100644 --- a/README.en.md +++ b/README.en.md @@ -22,7 +22,7 @@ This is research code, not a complete, standards-compliant, or production-grade or `9.1.6` (`4.0` is the quad layout, a front and a back pair; see [Speaker layouts and channel order](#speaker-layouts-and-channel-order)). - Run public SOFA binaural rendering directly from `pcm16 + ID11/OAMD`, without a temporary ADM BWF. -- Keep the binaural DSP in float64/complex128, including the selected filterbank's latency compensation (961 or 448 samples), cross-frame state, and the room tail. +- Keep binaural DSP in float64/complex128: full SOFA HRIR convolution and independent reference state; the personalized route retains its existing filterbank and latency rules. - Use a shared float32/PCM24 WAV writer and explicit PCM24 clipping policy for direct outputs. - Implement everything in pure Python on NumPy; - Read or write metadata sidecars and produce metadata, timing, and output reports. @@ -85,11 +85,12 @@ python main.py input.m4a --speaker-layout 5.1 --speaker-format int24 python main.py input.m4a --speaker-layout 7.1.2 --speaker-output output.7.1.2.wav ``` -Write binaural stereo directly (ordinary objects are Near/Mid/Far only; Mid is -the default). The HRTF input accepts three sources: +Write binaural stereo directly. SOFA and compact inputs select Near/Mid/Far in an external +reference environment, default Mid; personalized rendering retains its existing three modes. +The HRTF input accepts three sources: ```powershell -# 1) SOFA (defaults to HRTF/binaural.sofa, or an explicit path) +# 1) Full SOFA HRIR (defaults to HRTF/binaural.sofa, or an explicit path) python main.py input.m4a --binaural python main.py input.m4a --binaural --sofa-hrtf C:\HRTF\subject.sofa @@ -102,30 +103,33 @@ python main.py input.m4a --binaural --compiled-hrtf-cache C:\HRTF\subject.jochrt # Common options python main.py input.m4a --binaural --sofa-hrtf C:\HRTF\subject.sofa ` - --binaural-mode near -python main.py input.m4a --binaural --sofa-hrtf C:\HRTF\subject.sofa ` - --hrtf-cache-policy disk + --sofa-interpolation lf_mca_zero --reference-config C:\config\reference.json +python src\compile_hrtf.py C:\HRTF\subject.sofa C:\HRTF\subject.jochrtf +python main.py input.m4a --binaural --personalized-headphone --binaural-mode near +python main.py input.m4a --binaural --sofa-hrtf C:\HRTF\subject.sofa --binaural-mode far python main.py input.m4a --binaural --personalized-headphone ` --rosella-filterbank 2 python main.py input.m4a --binaural --binaural-output output.binaural.wav ``` -With none of the three specified, resolution tries, in order: -`HRTF/binaural.sofa`, the unique `.jochrtf` under `output/hrtf-cache`, then -`HRTF/binaural.personalized_headphone`; if none exist, an error asks for an -explicit path. +With no explicit input, use `HRTF/binaural.sofa` under the project directory. +If missing, try the original personalized default. Use `--sofa-hrtf PATH` for +other locations. Compact input is always explicit, with no automatic export or fallback. - `.sofa` is the portable source of truth; it can hold self-scanned or any generic HRTF data. - `.personalized_headphone` is a model produced by Dolby's official personalization scan; its JSON parsing is implemented by this project (`src/rosella_model.py`) and does not invoke any Dolby software. -- `.jochrtf` is a project-internal cache compiled from SOFA; it is disposable, - rebuildable, and written to `output/hrtf-cache` by default. +- `.jochrtf` is an explicitly exported compact field; its format and loader/writer + remain available. High-precision rendering reads SOFA directly. -HRTF data lives under `HRTF/` (git-ignored): the default SOFA -`HRTF/binaural.sofa` and the default model -`HRTF/binaural.personalized_headphone`. Because the cache contains transformed +The personalized default remains `HRTF/binaural.personalized_headphone`. +SOFA and compact routes also require an external reference configuration, read +from local `local/binaural_reference.json`, `--reference-config`, or +`BINAURAL_REFERENCE_CONFIG`. Environment data is not supplied with the source; +see the configuration contract in [Binaural rendering](docs/binaural.en.md). +Because the cache contains transformed HRTF data, its use and redistribution remain subject to the source dataset's terms. See [Binaural Rendering](docs/binaural.en.md) and [Third-party notices](THIRD_PARTY_NOTICES.md) for format boundaries, formulas, diff --git a/README.md b/README.md index 218cee1..0fbe731 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,7 @@ - 直接渲染 `2.0`、`3.1`、`4.0`、`5.1`、`7.1`、`5.1.2`、`5.1.4`、`7.1.2`、`7.1.4`、`9.1.4`、 `9.1.6`(`4.0` 是 Quad 布局,前后各一对;声道排列见[扬声器布局与声道顺序](#扬声器布局与声道顺序)); - 从 `pcm16 + ID11/OAMD` 直接运行公开 SOFA 双耳渲染,不生成临时 ADM BWF; -- 双耳 DSP 全程使用 float64/complex128,并保留所选滤波器组的延迟补偿(公开版 961、捕获版 448 samples)、跨帧状态和 room 尾声; +- 双耳 DSP 全程使用 float64/complex128;SOFA保留完整HRIR、跨帧卷积和独立参考环境状态,个性化路线保留原滤波器组与延迟规则; - 直接输出统一支持 float32 或 PCM24 WAV,并在 PCM24 削波前提供明确处理策略; - 全程使用 NumPy 的纯 Python 实现; - 读取或写入 metadata sidecar,并生成元数据、运行时间和输出摘要。 @@ -84,10 +84,10 @@ python main.py input.m4a --speaker-layout 5.1 --speaker-format int24 python main.py input.m4a --speaker-layout 7.1.2 --speaker-output output.7.1.2.wav ``` -直接输出双耳渲染 WAV(普通对象仅 Near/Mid/Far,默认 Mid)。HRTF 输入支持三种来源: +直接输出双耳渲染 WAV。SOFA与压缩输入使用外部参考环境的 Near/Mid/Far,默认 Mid;个性化路线保持原三档行为。HRTF 输入支持三种来源: ```powershell -# 1) SOFA(缺省取 HRTF/binaural.sofa,也可显式指定) +# 1) 完整SOFA HRIR(默认 HRTF/binaural.sofa,可显式指定) python main.py input.m4a --binaural python main.py input.m4a --binaural --sofa-hrtf C:\HRTF\subject.sofa @@ -100,25 +100,27 @@ python main.py input.m4a --binaural --compiled-hrtf-cache C:\HRTF\subject.jochrt # 常用选项 python main.py input.m4a --binaural --sofa-hrtf C:\HRTF\subject.sofa ` - --binaural-mode near -python main.py input.m4a --binaural --sofa-hrtf C:\HRTF\subject.sofa ` - --hrtf-cache-policy disk + --sofa-interpolation lf_mca_zero --reference-config C:\config\reference.json +python src\compile_hrtf.py C:\HRTF\subject.sofa C:\HRTF\subject.jochrtf +python main.py input.m4a --binaural --personalized-headphone --binaural-mode near +python main.py input.m4a --binaural --sofa-hrtf C:\HRTF\subject.sofa --binaural-mode far python main.py input.m4a --binaural --personalized-headphone ` --rosella-filterbank 2 python main.py input.m4a --binaural --binaural-output output.binaural.wav ``` -三者都不指定时的自动选择顺序:`HRTF/binaural.sofa` → `output/hrtf-cache` 下唯一的 -`.jochrtf` → `HRTF/binaural.personalized_headphone`;都没有则报错并提示显式指定。 +三者都不指定时先选择项目目录下的 `HRTF/binaural.sofa`; +缺失时尝试原个性化默认文件。其他位置通过 `--sofa-hrtf PATH` 显式指定。 +`.jochrtf` 只显式选择,不自动生成或回退。 - `.sofa` 是可移植的 source of truth;可以是自行扫描或任何来源的通用 HRTF 数据。 - `.personalized_headphone` 是杜比官方软件个性化扫描得到的模型,其 JSON 解析由 本项目自行实现(`src/rosella_model.py`),不调用杜比软件。 -- `.jochrtf` 是从 SOFA 编译出的项目内部 cache,可删除、可从 SOFA 重建,默认写在 - `output/hrtf-cache`。 +- `.jochrtf` 是从SOFA显式离线导出的轻量方向场,原格式及读写器保留;高精度路线直接读取SOFA。 -HRTF 数据统一放在 `HRTF/`(git 忽略):默认 SOFA `HRTF/binaural.sofa`、默认模型 -`HRTF/binaural.personalized_headphone`。cache 含有源 HRTF 的变换数据,使用与再分发 +个性化默认模型仍为 `HRTF/binaural.personalized_headphone`。SOFA与压缩路线还需要外部参考环境配置, +默认读取本地 `local/binaural_reference.json`,或用 `--reference-config` / `BINAURAL_REFERENCE_CONFIG` 指定。 +具体环境数据不随源码提供,配置格式见[双耳渲染](docs/binaural.md)。cache 含有源 HRTF 的变换数据,使用与再分发 仍受源数据许可约束;格式边界、计算公式、状态、时间轴及发布注意事项见 [双耳渲染](docs/binaural.md) 和 [第三方通知](THIRD_PARTY_NOTICES.md)。 diff --git a/docs/binaural.en.md b/docs/binaural.en.md index 28c9632..264a08d 100644 --- a/docs/binaural.en.md +++ b/docs/binaural.en.md @@ -2,283 +2,66 @@ [中文](binaural.md) · [Back to README](../README.en.md) -JustOneCacophony's binaural backend supports three HRTF sources: -`SimpleFreeFieldHRIR` SOFA, the Rosella `.personalized_headphone` model exported -by Dolby's official personalization scan (its JSON parsing is implemented by -this project and invokes no Dolby software), and the `.jochrtf` cache compiled -from SOFA. SOFA is compiled into an in-memory directional field when the model -is loaded. A `.jochrtf` file is only a disposable, reproducible JOC compiled -HRTF cache; it is neither an interchange format nor a prerequisite for using -SOFA. +## Routes -```text -SOFA FIR - -> CanonicalHrtf - -> 48 kHz / one radius shell / delay-phase policy - -> 64-QMF / 77-hybrid projection - -> fifth-order ACN/N3D real-SH field - -> per-object direct + early reflections - -> shared unitary-FDN late room - -> float64 stereo -``` +- SOFA: full complex local HRIR interpolation, adaptive-length filters and 128-sample partitioned convolution. +- `.jochrtf`: the existing compact 77-band/fifth-order field and delay representation. +- `.personalized_headphone`: the existing DSP, filterbank selection and Near/Mid/Far behavior. -## Inputs - -The CLI has three mutually exclusive HRTF input sources; with none given, a -default rule resolves the input: +SOFA and compact inputs use the same externally configured reference geometry, distance gains and environment state. SOFA never compiles or silently falls back to the compact format. The old compact loader/writer and compiler remain available. ```powershell -# 1) SOFA: defaults to HRTF/binaural.sofa, or an explicit path python main.py input.m4a --binaural -python main.py input.m4a --binaural --sofa-hrtf C:\HRTF\subject.sofa - -# 2) Rosella .personalized_headphone: defaults to HRTF/binaural.personalized_headphone -python main.py input.m4a --binaural --personalized-headphone -python main.py input.m4a --binaural --personalized-headphone C:\HRTF\subject.personalized_headphone - -# 3) .jochrtf: explicitly load a compiled cache -python main.py input.m4a --binaural ` - --compiled-hrtf-cache C:\HRTF\subject.jochrtf - -# Optional: create/reuse a transparent disk cache for SOFA -python main.py input.m4a --binaural --sofa-hrtf C:\HRTF\subject.sofa ` - --hrtf-cache-policy disk +python main.py input.m4a --binaural --sofa-hrtf subject.sofa --reference-config reference.json +python main.py input.m4a --binaural --sofa-hrtf subject.sofa --sofa-interpolation lf_mca_zero +python main.py input.m4a --binaural --sofa-hrtf subject.sofa --binaural-mode near +python main.py input.m4a --binaural --sofa-hrtf subject.sofa --binaural-mode far +python src\compile_hrtf.py subject.sofa subject.jochrtf +python main.py input.m4a --binaural --compiled-hrtf-cache subject.jochrtf +python main.py input.m4a --binaural --personalized-headphone --binaural-mode far --rosella-filterbank 2 ``` -The default order is `HRTF/binaural.sofa`, then the unique `.jochrtf` under -`output/hrtf-cache`, then `HRTF/binaural.personalized_headphone`; if none of -the three exist, an error asks for an explicit path. Multiple `.jochrtf` files -under `output/hrtf-cache` are also an error requiring an explicit choice. +The default SOFA is `HRTF/binaural.sofa` under the project directory, independently of the current working directory. Use `--sofa-hrtf PATH` for other locations. A single measured shell is selected automatically; multi-shell data requires `--hrtf-radius-m`. The personalized default path is unchanged. All three binaural routes accept `--binaural-mode near|mid|far`, defaulting to mid in the CLI; off remains an ADM-only DBMD hint. -The `.personalized_headphone` JSON parsing is implemented by this project -(`src/rosella_model.py`) and does not invoke any Dolby software. +## Reference configuration -`--hrtf-cache-policy` accepts `none`, `memory`, or `disk`. The default is -`memory`; neither `none` nor `memory` creates a file. `disk` writes to -`output/hrtf-cache` by default, or to `--hrtf-cache-dir`. `--hrtf-radius-m` -selects the nearest measurement-radius shell. +Resolution order: `--reference-config`, `BINAURAL_REFERENCE_CONFIG`, then local `local/binaural_reference.json`. Missing data, a hash mismatch, or an unsupported mapping is an error. Environment data and its specific identity are not supplied with the source. Example: -The Python API also uses explicit factories: - -```python -from sofa_binaural_backend import SofaBinauralBackend - -renderer = SofaBinauralBackend.from_sofa( - "subject.sofa", - source_count=16, - default_profile="mid", - cache_policy="memory", -) - -cached = SofaBinauralBackend.from_compiled_cache( - "subject.jochrtf", - source_count=16, - default_profile="mid", -) +```json +{"schema":"binaural-reference-v1","profile_id":"reference-v1", + "model_path":"C:/path/to/reference-data.json","model_sha256":"<64 hex digits>", + "profile_index":3,"room_impulse_slots":4096,"level_calibration_directions":128, + "compiled_receiver_positions_m":[[0,0.09,0],[0,-0.09,0]]} ``` -The factories never guess a format from an unknown suffix: SOFA and `.jochrtf` -always use distinct loaders. +The external data must be accepted by the existing parameter loader. Relative paths resolve against the configuration file. `profile_index` fixes the shared level-calibration anchor; `--binaural-mode` selects the rendering profile without changing that anchor. Compact files do not store receiver positions; they require explicit configured geometry. Full SOFA uses its own ear positions. A measured radius does not set the scene scale or room size. -## Binaural render mode +Reference mapping uses the selected profile's quantization, axes, bounds and centre regularization. Outside positions extend the listener-to-boundary ray by the coded finite factor and are never clamped back inside. Near uses direct gain 1 and environment send 0. Mid/Far use `a=1/sqrt(1+c*(h0*r)^2)` and `b=h0*r*a`. Send gain is not final reverberant pressure. All modes share the same interpolation, radial adaptation, convolution and timing. There is no additional 1/r attenuation or absolute flight-time delay. -`--binaural-mode off|near|mid|far` (default `mid`) is a **human-specified -rendering hint**, not original binaural metadata extracted or recovered from the -input E-AC-3 JOC bitstream: +## Full HRIR model -- Direct binaural rendering (`--binaural`): near/mid/far apply, default `mid`; - `off` is an error; -- ADM BWF: the low 3 binaural-render-mode bits of the last 15 JOC object entries - in DBMD segment 10 carry `off=0/near=1/far=2/mid=3`, leaving the first 10 bed - entries unchanged; the default is `mid`, and `off` explicitly disables the - binaural metadata hint. +The strict importer validates SimpleFreeFieldHRIR 0.4/1.0/1.1, FIR, units, listener orientation, ear order, zero emitter offset, finite data and free-field declarations. Resampling scales Data.Delay in samples. The radial adapter requires fixed symmetric receiver positions centred on the listener. -## Canonical SOFA contract +Map ADM right/front/up to SOFA front/left/up with `(y,-x,z)`. A 3D convex hull supplies local spherical triangles. Ray/planar barycentric weights are nonnegative and sum to one; exact measured nodes preserve the full complex response. Large triangles over coverage holes still have limited support. -The strict importer currently accepts: +Per-ear delay comes from phase regression at 200–1500 Hz. Interpolate aligned complex residuals and restore weighted delay, including Data.Delay once. The default `lf_logmag_phase` combines interpolated log magnitude with aligned complex phase. `lf_mca_zero` is an independent local MCA approximation with 41 continuous Gammatone power bands, PCHIP and a ±12 dB gain bound; it is not a bitwise reproduction of a third-party implementation. -- `Conventions=SOFA`; -- `SOFAConventions=SimpleFreeFieldHRIR`, version `0.4`, `1.0`, or `1.1`; -- `DataType=FIR` and `Data.IR[M,2,N]`; -- one positive finite `Data.SamplingRate` in hertz/Hz; -- spherical or Cartesian `SourcePosition`; -- singleton or per-measurement `ListenerPosition/View/Up`; -- two receivers whose listener-local lateral geometry uniquely identifies L/R; -- one zero-offset emitter; -- causal `Data.Delay[I,2]` or `[M,2]`; -- an explicitly free-field/anechoic `RoomType`. +Radial adaptation multiplies by the complex normalized rigid-sphere ratio `H_sphere(r)/H_sphere(r_measured)`, exactly one at the measured radius. Ear geometry determines sphere radius. Near the centre, the configured regularization and an equal-ear, zero-ITD, spherical-area-weighted energy response give a continuous blend. This is an approximate human-head radial model; angular interpolation scores do not validate arbitrary distances. -Receiver order comes from geometry, never from the receiver array index. SOFA -listener coordinates are $+X$ front, -$+Y$ left, -$+Z$ up; ADM coordinates are -$+X$ right, -$+Y$ front, -$+Z$ up: +Calibration computes one global scalar per input at the configured anchor from equal-area directions and total binaural energy with logarithmic frequency weighting at 100–16000 Hz. All three modes share that scalar; no per-mode/ear/direction/frequency normalization is applied. -$$\bigl(x_{\mathrm{SOFA}},\ y_{\mathrm{SOFA}},\ z_{\mathrm{SOFA}}\bigr) = \bigl(y_{\mathrm{ADM}},\ -x_{\mathrm{ADM}},\ z_{\mathrm{ADM}}\bigr)$$ +The FFT size is at least 2048, automatically increased to a power of two that fits the complete HRIR and Data.Delay, up to 16384. All resulting FIR taps are retained, with a declared common shift of half the FFT size. Rendering rebases one global recording origin, the rounded median of mean-ear fitted delays; the environment branch receives matching delay before common compensation. Original spectrum queries retain the dataset time reference. The shift is independent of target source distance. -`CanonicalHrtf` keeps `Data.IR` and `Data.Delay` separate. Only a time-domain -baseline calls `materialized_measurement()` to apply delay once; the runtime SH -path never materializes and then restores the delay. Non-48-kHz HRIRs are -normalized with float64 `scipy.signal.resample_poly`, and delay samples scale by -the same ratio. +Per-object convolution retains input history and independent old/new overlap during a 128-sample output crossfade. Environment computation batches 1536 samples without changing the 128-sample parameter timeline; extra computation padding is trimmed when draining. Source PCM, delays and accumulation use float64; spectra use complex128, at 48 kHz. -GeneralFIR, BRIR, TF, multiple emitters, ambiguous receivers, and non-free-field -data require convention-specific adapters. They cannot enter the core importer -through a reshape. +## Scheduling and limits -## Exactly-once delay and phase +Each `[1536,16]` frame contains LFE followed by 15 already reconstructed objects/bed slots. Metadata retains the existing 1473-sample content offset and 32-sample alignment; new parameters update every 128 samples. A SOFA-only opt-in preserves negative elevation, outside distance and anchor fields. Legacy parsing and the personalized timeline retain their default behavior. -The compiler recognizes three mutually exclusive representations: +Multi-position blocks, ISF, explicit infinity and screen anchors requiring calibration return structured unsupported errors. Finite outside updates participate in timeline interpolation. `finish()` drains state only once; output gain is applied once without an implicit limiter. -1. Nonzero `Data.Delay` is external to `Data.IR`; the FIR is not de-rotated and - runtime applies the delay once. -2. With `Data.Delay=0` and an ordinary positive-onset HRIR, each ear's main peak - supplies arrival time. Compilation separates it and runtime restores it once. - The current threshold is a peak index greater than two samples. -3. With `Data.Delay=0` and both FIRs at a shared sample-zero origin, no external - delay is invented. The authored complex phase stays in the fifth-order field. +## Compact files -No path may add a second ear delay or phase-group delay. +Version-1 `.jochrtf` files retain their pickle-free ZIP schema, band centres, complex128 `[36,2,77]` coefficients, float64 `[36,2]` delay coefficients, `[2,2]` bounds, metadata, hash validation and atomic writer. Their inherent directional compression error remains. See [filterbank data](../data/README.en.md) for the public filterbank sources. -## Public filterbank and directional field - -The runtime is fixed at: - -- 48 kHz; -- a 64-sample QMF hop; -- 64-QMF / 77 hybrid bands; -- 961 samples of analysis/synthesis latency; -- fifth order, 36 terms, ACN/N3D real spherical harmonics; -- float64 PCM, delay, SH, and room state; complex128 band transfers and spectra. - -Those are the SOFA / `.jochrtf` path and the public filterbank. The Rosella -`.personalized_headphone` path selects its bank: the public bank above is the -default (961 samples), and `--rosella-filterbank 2` switches to the bank captured -from the reference renderer (11 taps / 3 slots, 448 samples). Parameters, -buffers, latency and validation of both are in -[Rosella filterbank](rosella_filterbank.en.md). - -Real and imaginary unit gains for every hybrid band pass through the same -analysis/synthesis chain to form a 154-real-parameter impulse dictionary. The -compiler does not sample 77 FFT bins. Defaults are `1e-3` projection ridge and -`1e-5` SH ridge. Coincident directions are merged before a spherical-Voronoi -weighted ridge fit. - -The fixed resource is `data/rosella_kernels.npz`, which implements publicly -standardized filter banks, computable from the following formulas. - -The hybrid analysis kernels are defined in [3GPP TS 26.405 / ETSI TS 126 405](https://www.etsi.org/deliver/etsi_ts/126400_126499/126405/06.00.00_60/ts_126405v060000p.pdf), -Section 5.2.2 (Table 1 $Q=8$/ -$Q=4$ coefficients, delay 6): - -$$G_q^p[n] = g^p[n]\cdot\exp\Bigl(j\,\frac{2\pi}{Q^p}\bigl(q+\tfrac12\bigr)(n-6)\Bigr),\qquad n=0,\dots,12$$ - -The QMF analysis table is the MPEG-4 AAC/SBR 64 complex QMF bank of -ISO/IEC 14496-3/AMD1:2003, subclause 4.B.18.2, stored as the polyphase -reordering of the public 640-tap prototype $c_0,\dots,c_{639}$: - -$$A_{r,t} = \frac{(-1)^t}{128}\,c_{63-r+64t},\qquad r=0,\dots,63,\ t=0,\dots,9$$ - -The QMF synthesis table is the causal left inverse of the analysis polyphase -matrix $\mathbf{A}$, i.e. the solution of -$\mathbf{A}\,\mathbf{W}=\mathbf{P}$ -($\mathbf{P}$ is the 577-sample delay permutation; total latency -$961 = 577 + 6\times64$), stored as a rank-4 factorization: - -$$W_{b,l} = \sum_{r=1}^{4} t_{b,l,r}\,\mathbf{b}_{b,r}^{\top}$$ - -The hybrid synthesis table is the 77→64 recombination: identity for the high -bands, $Y_{3+b}=X_{16+b}$, and for the low bands( -$C_p$ is the -$8+4+4$ child partition): - -$$Y_p = \sum_{q\in C_p}\Bigl(\mathrm{Re}X_q + j\,s_q\,\mathrm{Im}X_q\Bigr),\qquad s_q\in\{\pm1\}$$ - -The loader verifies the archive and every array by SHA-256; the table version, -all array hashes, and the 77 reference band-center values are part of the cache -key. Public availability of a standard does not by itself grant permission to -practice related patent claims. See -[`data/README.en.md`](../data/README.en.md) and -[`THIRD_PARTY_NOTICES.md`](../THIRD_PARTY_NOTICES.md) for the sources and the -rights boundary. - -## `.jochrtf` - -A `.jochrtf` file is a pickle-free compressed NumPy archive with an exact member set: - -| key | dtype / shape | -|---|---| -| `metadata_json` | NumPy Unicode scalar containing JSON text (`dtype.kind == "U"`) | -| `band_center_frequencies_hz` | little-endian `float64[77]` | -| `coefficients` | little-endian `complex128[36,2,77]` | -| `delay_coefficients` | little-endian `float64[36,2]` | -| `delay_bounds` | little-endian `float64[2,2]` | - -Metadata uses the `JOC-HRTF-CACHE` magic and records the schema, compiler and -phase-policy versions, ACN/N3D convention, filterbank hashes, SOFA content -SHA-256, sample rate, radius, order, both ridge values, payload hash, and fit -report. Every setting that changes compilation participates in the cache key. -Metadata never persists an absolute local `source_path`; it may keep a display -name only. - -Before constructing a field, the loader uses `allow_pickle=False` and validates -ZIP members and expanded sizes, shapes, dtypes, byte order, contiguous layout, -finite values, delay bounds, band centers, payload hash, and cache key. The -writer uses a same-directory temporary file, `fsync`, a process-held OS file -lock, and atomic `os.replace`. Its hidden `.lock` sidecar may remain and does not -mean that a writer still owns the lock. Outdated, damaged, or mismatched -caches cannot hit. SOFA input rebuilds an invalid cache; an explicitly selected -cache reports the error. - -Deleting a disk cache must not change the field or render produced from the same -SOFA and compiler configuration. - -A `.jochrtf` file contains directional-field coefficients and delay data -transformed from the source HRIRs. Its reproducibility therefore does not make -it licence-free. Creating a cache does not enlarge the rights granted by the -source SOFA/HRTF dataset: use, copying, and redistribution remain subject to -that dataset's terms. If those terms are unclear, keep `.jochrtf` as a private -local cache and do not ship it with the program or another build artifact. -`source_sha256` is only a content-integrity identifier, not proof of provenance -or permission. - -## JOC objects and room behavior - -The production adapter retains the existing JOC schedule: - -- `[1536,16]` input per frame; -- channel 0 is special LFE and channels 1..15 are JOC objects; -- ID11/OAMD positions use a sample-timed timeline; -- source parameters update every 512 samples; -- every object owns independent direct/early history while one late FDN is shared; -- `finish()` drains early/late tails; output gain is explicit, with no implicit - limiter or programme loudness normalization. - -Near/Mid/Far, equal-power direct level, six first-order shoebox image sources, -late sends, the unitary FDN, the 120–180 Hz cosine-squared LFE low-pass, and room -calibration are JOC project-defined behavior, not constants published by SOFA or -Dolby. - -The filterbank, the SH direction-field evaluation, the per-object direct/early -histories and the shared FDN all run in Python/NumPy, with the binaural DSP kept -in float64/complex128 throughout. - -## Technical references and rights boundary - -- [SOFA SimpleFreeFieldHRIR convention](https://www.sofaconventions.org/mediawiki/index.php/SimpleFreeFieldHRIR) -- [3GPP TS 26.405 / ETSI TS 126 405 (64-QMF/77-hybrid definition)](https://www.etsi.org/deliver/etsi_ts/126400_126499/126405/06.00.00_60/ts_126405v060000p.pdf) -- [Dolby binaural render-mode workflow](https://professionalsupport.dolby.com/s/article/What-is-Binaural-Render-Mode-and-how-do-the-settings-affect-my-mix) -- [EP3090576A1](https://patents.google.com/patent/EP3090576A1/en), used only as - architectural background for direct/early/late, subbands, and FDNs; it does - not establish that any product uses a particular embodiment. - -Public availability of a specification, source file, or patent document does -not by itself authorize copying its contents, redistribution of derivatives, -or practice of patent claims. These technical references grant no patent -licence and make no non-infringement representation. Anyone preparing a release -or product integration must assess the applicable data and software licences, -patent permissions, and freedom to operate. See -[`THIRD_PARTY_NOTICES.md`](../THIRD_PARTY_NOTICES.md) for the public-standard -provenance and rights boundary. +Mathematical sources: [Duda and Martens 1998](https://www.ece.ucdavis.edu/cipic/wp-content/uploads/sites/12/2015/04/cipic_JASA_Nov_1998.pdf), [MCA paper](https://doi.org/10.1109/TASLP.2023.3313908), [ETSI TS 103 420](https://www.etsi.org/deliver/etsi_ts/103400_103499/103420/01.02.01_60/ts_103420v010201p.pdf). HRTF algorithms are independently implemented; external data retain their own licence terms. diff --git a/docs/binaural.md b/docs/binaural.md index 06c5d5f..4a9b969 100644 --- a/docs/binaural.md +++ b/docs/binaural.md @@ -2,247 +2,76 @@ [English](binaural.en.md) · [返回 README](../README.md) -JustOneCacophony 的双耳后端支持三种 HRTF 来源:`SimpleFreeFieldHRIR` SOFA、 -杜比官方软件个性化扫描导出的 Rosella `.personalized_headphone`(JSON 解析由本项目 -自行实现,不调用杜比软件),以及从 SOFA 编译出的 `.jochrtf` 缓存。SOFA 在模型加载 -时编译成内存方向场;`.jochrtf` 只是可删除、可重建的 JOC compiled HRTF cache, -不是交换格式,也不是使用 SOFA 的前置步骤。 +## 三条独立路线 -```text -SOFA FIR - -> CanonicalHrtf - -> 48 kHz / 单 radius shell / delay-phase policy - -> 64-QMF / 77-hybrid projection - -> 五阶 ACN/N3D 实球谐场 - -> 逐对象 direct + early reflections - -> shared unitary-FDN late room - -> stereo float64 -``` +| 输入 | 直接声 | 距离与环境 | +|---|---|---| +| SOFA | 完整复数HRIR局部插值、自适应长度FIR、128样本分区卷积 | 外部参考配置的Near/Mid/Far | +| `.jochrtf` | 保留原77带/五阶方向场及延迟表示 | 与SOFA相同的三档参考合同 | +| `.personalized_headphone` | 原DSP及滤波器组 | 原Near/Mid/Far及环境处理 | -## 输入接口 - -CLI 有三个互斥的 HRTF 输入来源;都不指定时按默认规则自动选择: +SOFA不会调用编译器、生成压缩缓存或自动回退。原压缩格式、校验器和离线编译能力保留。旧 `sofa_binaural_backend.py` 作为已有底层实现仍在,CLI工厂使用新的 `ReferenceSofaBackend`。 ```powershell -# 1) SOFA:缺省取 HRTF/binaural.sofa,也可显式指定 python main.py input.m4a --binaural -python main.py input.m4a --binaural --sofa-hrtf C:\HRTF\subject.sofa - -# 2) Rosella .personalized_headphone:缺省取 HRTF/binaural.personalized_headphone -python main.py input.m4a --binaural --personalized-headphone -python main.py input.m4a --binaural --personalized-headphone C:\HRTF\subject.personalized_headphone - -# 3) .jochrtf:显式读取预编译 cache -python main.py input.m4a --binaural ` - --compiled-hrtf-cache C:\HRTF\subject.jochrtf - -# 可选:SOFA 透明生成/复用磁盘 cache -python main.py input.m4a --binaural --sofa-hrtf C:\HRTF\subject.sofa ` - --hrtf-cache-policy disk +python main.py input.m4a --binaural --sofa-hrtf C:\HRTF\subject.sofa --reference-config C:\config\reference.json +python main.py input.m4a --binaural --sofa-hrtf C:\HRTF\subject.sofa --sofa-interpolation lf_mca_zero +python main.py input.m4a --binaural --sofa-hrtf C:\HRTF\subject.sofa --binaural-mode near +python main.py input.m4a --binaural --sofa-hrtf C:\HRTF\subject.sofa --binaural-mode far +python src\compile_hrtf.py C:\HRTF\subject.sofa C:\HRTF\subject.jochrtf +python main.py input.m4a --binaural --compiled-hrtf-cache C:\HRTF\subject.jochrtf +python main.py input.m4a --binaural --personalized-headphone --binaural-mode far --rosella-filterbank 2 ``` -默认选择顺序:`HRTF/binaural.sofa` → `output/hrtf-cache` 下唯一的 `.jochrtf` → -`HRTF/binaural.personalized_headphone`;三者都没有时报错并提示显式指定。 -`output/hrtf-cache` 下有多个 `.jochrtf` 时同样报错,要求显式选择。 +默认SOFA为项目目录下的 `HRTF/binaural.sofa`,路径不依赖当前工作目录;其他位置用 `--sofa-hrtf PATH` 显式指定。单球壳自动选择,多球壳须用 `--hrtf-radius-m` 指定最近的测量球壳。个性化默认路径不变。三条双耳路线都支持 `--binaural-mode near|mid|far`,CLI默认mid;off仅适用于ADM DBMD提示。 -`.personalized_headphone` 的 JSON 解析由本项目自行实现(`src/rosella_model.py`), -不调用任何杜比软件。 +## 外部参考配置 -`--hrtf-cache-policy` 可取 `none`、`memory`、`disk`。默认是 `memory`;`none` 和 -`memory` 都不会创建磁盘文件。`disk` 默认写入 `output/hrtf-cache`,也可用 -`--hrtf-cache-dir` 指定。`--hrtf-radius-m` 选择距离目标最近的 measurement shell。 +`--reference-config` > `BINAURAL_REFERENCE_CONFIG` > 本地 `local/binaural_reference.json`。缺失、SHA-256不符或不支持的映射明确报错。环境数据与具体身份说明不随源码提供。配置示例: -Python API 使用显式 factory: - -```python -from sofa_binaural_backend import SofaBinauralBackend - -renderer = SofaBinauralBackend.from_sofa( - "subject.sofa", - source_count=16, - default_profile="mid", - cache_policy="memory", -) - -cached = SofaBinauralBackend.from_compiled_cache( - "subject.jochrtf", - source_count=16, - default_profile="mid", -) +```json +{"schema":"binaural-reference-v1","profile_id":"reference-v1", + "model_path":"C:/path/to/reference-data.json","model_sha256":"<64 hex digits>", + "profile_index":3,"room_impulse_slots":4096,"level_calibration_directions":128, + "compiled_receiver_positions_m":[[0,0.09,0],[0,-0.09,0]]} ``` -文件工厂不会按“未知后缀”猜格式:SOFA 和 `.jochrtf` 始终走不同 loader。 +`model_path` 指向受现有参数加载器支持的外部数据,支持相对配置文件的路径;`profile_index` 固定公共电平校准锚点,选择渲染档位使用 `--binaural-mode`,不改校准锚点。压缩格式没有耳位字段,因此压缩输入必须显式配置耳位,且报告其来源。完整SOFA使用文件自身耳位。不同SOFA共用环境尺度,测量半径不定义场景距离或房间大小。 -## 双耳渲染模式 +参考映射执行所选档位的量化、轴缩放、边界与近中心正则规则。有限盒外位置沿边界射线延伸,保留ETSI距离因子,不能再次裁回坐标盒。Near共同直接声系数为1、房间发送为0;Mid/Far为: -`--binaural-mode off|near|mid|far`(默认 `mid`)是**人为指定的渲染提示**,不是 -从输入 E-AC-3 JOC 码流提取或还原的原始双耳元数据: +```text +r_eff = h0 * reference_distance_m +a = 1 / sqrt(1 + c * r_eff^2) +b = r_eff * a +``` -- 直接双耳渲染(`--binaural`):near/mid/far 生效,默认 `mid`;`off` 报错; -- ADM BWF:DBMD segment 10 中后 15 个 JOC 对象的 binaural render mode 写 - `off=0/near=1/far=2/mid=3`,前 10 个 bed 保持不变,默认 `mid`;`off` 用于显式 - 关闭双耳元数据提示。 +房间发送乘对象信号后进入独立环境状态。完整SOFA直接声与环境分支分别处理,再对齐实现延迟并求和;没有额外的1/r或绝对飞行时间。输出增益只应用一次,不隐式限幅。 -## Canonical SOFA 契约 +## 完整HRIR数学与时间基准 -当前 strict importer 接受: +严格导入器继续校验 `SimpleFreeFieldHRIR` 0.4/1.0/1.1、FIR、两个可由位置识别左右的耳、零Emitter偏移、自由场、单位和有限数值。重采样时Data.Delay的采样点数同比例变化。当前距离模型要求固定、对称并以听者中心为原点的耳位;不支持的几何明确拒绝。 -- `Conventions=SOFA`; -- `SOFAConventions=SimpleFreeFieldHRIR`,version `0.4`、`1.0` 或 `1.1`; -- `DataType=FIR`,`Data.IR[M,2,N]`; -- 单一正有限 `Data.SamplingRate`,单位为 hertz/Hz; -- spherical 或 Cartesian `SourcePosition`; -- 单值或 per-measurement 的 `ListenerPosition/View/Up`; -- 两个能由 listener-local lateral 坐标唯一识别左右的 receiver; -- 单一且零偏移的 emitter; -- causal `Data.Delay[I,2]` 或 `[M,2]`; -- 明确的 free-field/anechoic `RoomType`。 +SOFA为前/左/上,ADM为右/前/上,变换为 `(y,-x,z)`。单位方向三维凸包构建局部三角网格,查询使用射线与平面三角形的非负、和为1的重心权重;不做经纬平面插值或外推。精确节点返回完整复数响应。跨越大测量缺口的面仍是低支持区域,网格包围原点不能证明覆盖精度。 -receiver 左右顺序由几何决定,不能假定 `Data.IR` 的 receiver index。SOFA listener -坐标为 $+X$ front、 -$+Y$ left、 -$+Z$ up;ADM 坐标为 -$+X$ right、 -$+Y$ front、 -$+Z$ up,转换为: +FFT最少2048点,按完整HRIR与Data.Delay自动扩展为满足时间预算的2次幂,自动长度上限16384点;不裁短输入。每耳在200–1500 Hz作线性相位回归,对齐后插值完整剩余复数响应,再恢复加权延迟。Data.Delay恰好施加一次,不能从耳位再加一次ITD。默认 `lf_logmag_phase` 对数幅度与对齐复数插值相位结合;可选 `lf_mca_zero` 使用41个连续Gammatone功率近似带、PCHIP和±12 dB修正限幅。这是独立MCA近似,并非第三方项目逐位复现。 -$$\bigl(x_{\mathrm{SOFA}},\ y_{\mathrm{SOFA}},\ z_{\mathrm{SOFA}}\bigr) = \bigl(y_{\mathrm{ADM}},\ -x_{\mathrm{ADM}},\ z_{\mathrm{ADM}}\bigr)$$ +距离补充采用归一化刚性球传递函数 `H_sphere(r)/H_sphere(r_measurement)` 的复数比值。测量半径处严格为1;不叠加第二套耳间延迟。球半径由对称耳位推导。中心使用参考正则半径与零ITD的球面面积加权能量平均响应平滑混合。该模型是人头径向变化的近似,不能把角度插值实验当作任意距离准确性证明。 -`CanonicalHrtf` 将 `Data.IR` 与 `Data.Delay` 分开保存。只有时域 baseline 才调用 -`materialized_measurement()` 将 delay 应用一次;运行时 SH 路径不先 materialize。 -非 48 kHz HRIR 使用 float64 `scipy.signal.resample_poly` 规范化,delay samples 按 -相同比例缩放。 +直接声电平按固定等面积方向及100–16000 Hz对数频率权重,与校准锚点的参考直接声匹配总双耳能量;每份SOFA只得到一个全局系数,三档共用该系数,不逐档、逐方向、逐耳或逐频率归一化。三档共用相同的HRIR插值、距离补充、卷积和时间补偿流程,只切换参考参数与其直接声/发送规则。 -GeneralFIR、BRIR、TF、多 emitter、多义 receiver 或非 free-field 数据需要单独的 -convention adapter,不能只通过 reshape 进入核心 importer。 +保留选定FFT长度的全部FIR采样,共同移位FFT长度的一半以存储前振铃。渲染时使用全方向双耳低频延迟平均值的中位数、取整为单一录音时间原点,给环境分支补齐该原点与实现延迟后统一补偿;报告此偏移。查询得到的原始复数HRIR不改时间基准。这个全局时间参考变换不根据目标距离增加传播时间。 -## Delay/phase:exactly once +每对象独立卷积历史,更新时旧/新输出线性交叉淡化128样本,保持输入历史及独立overlap;环境发送平滑更新。环境计算按1536样本批处理,不改变128样本参数时间线,排空时裁掉额外计算填充。采样率固定48 kHz,内部float64/complex128。 -编译器只允许三种互斥语义: +## 对象接入与支持范围 -1. 非零 `Data.Delay` 是 `Data.IR` 外部 delay;FIR 不去旋,运行时应用一次。 -2. `Data.Delay=0` 且 HRIR 有普通正 onset:以每耳 main peak 分离 arrival,拟合后 - 在运行时恢复一次;当前阈值为 peak index 大于 2 samples。 -3. `Data.Delay=0` 且双耳 FIR 共享 sample-0 起点:不发明外部 delay,原 complex - phase 直接进入五阶场。 +输入为每帧 `[1536,16]` PCM:0是LFE,1–15是已经重建的对象/bed槽。直接使用ID11时间线,不重新解码,不先转扬声器布局。保留1473样本内容时间补偿和32样本元数据对齐;新后端每128样本更新参数。 -任何路径都不能再叠加第二套 ear delay 或 phase-group delay。 +SOFA专用解析开关保留负高度、边界外因子及锚定字段;旧解析器默认行为与个性化时间线不变。有限盒外更新进入时间插值;当前多位置块、ISF、显式无穷远和需要屏幕校准的锚定明确返回结构化不支持错误。`finish()`只排空一次,并保留配置的尾声预算。 -## 公开 filterbank 与方向场 +## 轻量格式 -运行时固定为: +`.jochrtf` v1仍为无pickle的NumPy ZIP,保存JSON元数据、77个频带中心、`complex128[36,2,77]`方向系数、`float64[36,2]`延迟系数和`float64[2,2]`边界。原格式、cache key、哈希校验、原子写入及源数据许可关系不变。它保留方向压缩的固有误差;与完整SOFA的距离语义一致不代表频响精度相同。滤波器组公开来源见 [data说明](../data/README.md)。 -- 48 kHz; -- 64-sample QMF hop; -- 64-QMF / 77-hybrid; -- analysis/synthesis latency 961 samples; -- 五阶、36 项、ACN/N3D real spherical harmonics; -- PCM、delay、SH、room state 为 float64;频带传递和频域状态为 complex128。 - -上面是 SOFA / `.jochrtf` 路径与公开滤波器组的固定参数。Rosella -`.personalized_headphone` 路径的滤波器组可选:默认仍是这一套公开滤波器组(961 -samples),`--rosella-filterbank 2` 换成从参考渲染器捕获的版本(11 taps / 3 slots、 -448 samples)。两套实现的参数、缓冲、延迟与验证见 -[Rosella 滤波器组](rosella_filterbank.md)。 - -每个 hybrid band 的 real/imaginary 单位增益都通过同一套 analysis/synthesis 链生成 -脉冲字典,共 154 个实参数;编译不是直接读取 77 个 FFT bin。默认 projection -ridge 为 `1e-3`,SH ridge 为 `1e-5`。同方向 measurement 先合并,再用球面 Voronoi -面积权重做 ridge fit。 - -固定表位于 `data/rosella_kernels.npz`,实现公开标准化的滤波器组,各表可由如下 -公式计算。 - -hybrid 分析核定义于 [3GPP TS 26.405 / ETSI TS 126 405](https://www.etsi.org/deliver/etsi_ts/126400_126499/126405/06.00.00_60/ts_126405v060000p.pdf) -第 5.2.2 节(Table 1 的 $Q=8$/ -$Q=4$ 系数,delay 6): - -$$G_q^p[n] = g^p[n]\cdot\exp\Bigl(j\,\frac{2\pi}{Q^p}\bigl(q+\tfrac12\bigr)(n-6)\Bigr),\qquad n=0,\dots,12$$ - -QMF analysis 表即 MPEG-4 AAC/SBR(ISO/IEC 14496-3/AMD1:2003 第 4.B.18.2 节) -的 64 complex QMF bank;打包的 $64\times10$ 表是公开 640-tap prototype -$c_0,\dots,c_{639}$ 的多相重排: - -$$A_{r,t} = \frac{(-1)^t}{128}\,c_{63-r+64t},\qquad r=0,\dots,63,\ t=0,\dots,9$$ - -QMF synthesis 表为上述 analysis 多相矩阵 $\mathbf{A}$ 的因果左逆,即求解 -$\mathbf{A}\,\mathbf{W}=\mathbf{P}$( -$\mathbf{P}$ 为 577-sample 延迟置换; -全链 $961 = 577 + 6\times64$),以 rank-4 分解形式存储: - -$$W_{b,l} = \sum_{r=1}^{4} t_{b,l,r}\,\mathbf{b}_{b,r}^{\top}$$ - -hybrid synthesis 表为 77→64 重组:高频带恒等 $Y_{3+b}=X_{16+b}$;低频带( -$C_p$ 为 -$8+4+4$ 子带划分): - -$$Y_p = \sum_{q\in C_p}\Bigl(\mathrm{Re}X_q + j\,s_q\,\mathrm{Im}X_q\Bigr),\qquad s_q\in\{\pm1\}$$ - -loader 校验 archive 和每个数组的 SHA-256;table version、所有数组 hash 与 -77 个 band-center 参考值都属于 cache key。标准可公开获取不等于获准实施相关 -专利;更多来源信息见 [`data/README.md`](../data/README.md) 与 -[`THIRD_PARTY_NOTICES.md`](../THIRD_PARTY_NOTICES.md)。 - -## `.jochrtf` - -`.jochrtf` 是无 pickle 的压缩 NumPy archive,固定包含: - -| key | dtype / shape | -|---|---| -| `metadata_json` | 含 JSON 文本的 NumPy Unicode scalar(`dtype.kind == "U"`) | -| `band_center_frequencies_hz` | little-endian `float64[77]` | -| `coefficients` | little-endian `complex128[36,2,77]` | -| `delay_coefficients` | little-endian `float64[36,2]` | -| `delay_bounds` | little-endian `float64[2,2]` | - -metadata magic 固定为 `JOC-HRTF-CACHE`,并记录 schema/compiler/phase-policy、 -ACN/N3D、filterbank table hashes、SOFA content SHA-256、采样率、radius、order、 -两个 ridge、payload hash 和 fit report。cache key 覆盖所有会改变编译结果的字段。 -metadata 不保存本机绝对 `source_path`,仅可保存 source display name。 - -loader 使用 `allow_pickle=False`,并在构造对象前检查 ZIP 成员集、解压大小、shape、 -dtype、端序、连续布局、有限值、delay bounds、band centers、payload hash 和 cache -key。writer 使用同目录临时文件、`fsync`、进程持有的 OS 文件锁和原子 -`os.replace`;对应的隐藏 `.lock` sidecar 可保留,但不代表仍有 writer 持锁。 -旧版本、损坏或配置不匹配的 cache 不能命中;从 SOFA 启动时会重建,显式 cache -入口则直接报错。 - -删除磁盘 cache 后,从同一 SOFA 和同一编译配置得到的场与渲染结果不得改变。 - -`.jochrtf` 包含由源 HRIR 变换得到的方向场系数与 delay 数据,因此“可以重建”不表示 -它不受数据许可约束。生成 cache 不会扩大源 SOFA/HRTF 数据集授予的权利;cache 的 -使用、复制和再分发仍须遵守源数据集条款。不能确认条款时,应把 `.jochrtf` 作为本地 -私有 cache,不随程序或构建产物发布。`source_sha256` 只用于内容一致性校验,不是许可 -或来源证明。 - -## JOC 对象与房间 - -生产适配器继续使用现有 JOC 调度: - -- 每帧输入 `[1536,16]`; -- channel 0 是 special LFE,channel 1..15 是 JOC objects; -- ID11/OAMD position 使用 sample-timed timeline; -- 每 512 samples 更新方向/profile; -- 每个对象拥有独立 direct/early history,late FDN 全局共享; -- `finish()` 排空 early/late tail;输出增益显式应用,不隐含 limiter 或节目响度归一化。 - -Near/Mid/Far、equal-power direct level、六面 shoebox 一阶 image source、late send、 -unitary FDN、LFE 120–180 Hz cosine-squared 低通及 room calibration 都是 JOC -项目定义行为,不是 SOFA 或 Dolby 公布常数。 - -filterbank、SH 方向场求值、逐对象 early/direct 历史与共享 FDN 全部在 Python/NumPy -一侧执行,双耳 DSP 全程 float64/complex128。 - -## 技术引用与权利边界 - -- [SOFA SimpleFreeFieldHRIR convention](https://www.sofaconventions.org/mediawiki/index.php/SimpleFreeFieldHRIR) -- [3GPP TS 26.405 / ETSI TS 126 405(64-QMF/77-hybrid 定义)](https://www.etsi.org/deliver/etsi_ts/126400_126499/126405/06.00.00_60/ts_126405v060000p.pdf) -- [Dolby binaural render mode workflow](https://professionalsupport.dolby.com/s/article/What-is-Binaural-Render-Mode-and-how-do-the-settings-affect-my-mix) -- [EP3090576A1](https://patents.google.com/patent/EP3090576A1/en),仅作 direct/early/late、 - subband 与 FDN 架构背景,不证明某个产品使用特定实施例。 - -规范、源码或专利文献可公开获取,不等于获准复制其内容、再分发派生产物或实施其中的 -专利权利要求。本项目的技术引用本身不授予专利许可,也不作不侵权保证;准备发布或集成 -到产品的一方应自行审查适用的数据许可、软件许可、专利许可及 freedom-to-operate。 -公开标准来源与权利边界见 -[`THIRD_PARTY_NOTICES.md`](../THIRD_PARTY_NOTICES.md)。 +数学依据:[Duda与Martens 1998](https://www.ece.ucdavis.edu/cipic/wp-content/uploads/sites/12/2015/04/cipic_JASA_Nov_1998.pdf),[MCA原论文](https://doi.org/10.1109/TASLP.2023.3313908),[ETSI TS 103 420](https://www.etsi.org/deliver/etsi_ts/103400_103499/103420/01.02.01_60/ts_103420v010201p.pdf)。算法独立实现,不复制第三方HRTF源码;数据许可仍随外部数据适用。 diff --git a/main.py b/main.py index d69a6c3..59a25bb 100644 --- a/main.py +++ b/main.py @@ -72,27 +72,10 @@ def resolve_output(source, requested=None, speaker_layout=None, *, binaural=Fals return target.expanduser().resolve() -def _find_default_compiled_hrtf_cache(): - """在默认 cache 目录寻找唯一的 .jochrtf;无文件返回 None,多个则报错。""" - directory = DEFAULT_HRTF_CACHE_DIR - if not directory.is_dir(): - return None - candidates = sorted(directory.glob("*.jochrtf")) - if not candidates: - return None - if len(candidates) > 1: - listing = ", ".join(path.name for path in candidates[:8]) - raise ValueError( - f"{directory} 下有多个 .jochrtf 缓存({listing}…),无法自动选择;" - "请用 --compiled-hrtf-cache PATH 或 --sofa-hrtf PATH 显式指定") - return candidates[0] - - def resolve_binaural_hrtf_input(args, *, required): """解析 binaural 的 HRTF 输入。 - 无显式输入时按顺序回退:默认 HRTF/binaural.sofa → 默认 cache 目录下唯一的 - .jochrtf → 默认 HRTF/binaural.personalized_headphone → 报错。 + 默认选择完整SOFA;若缺失则尝试原个性化默认文件。压缩输入只能显式选择。 只校验路径,不做编译。 """ sofa = args.sofa_hrtf @@ -104,10 +87,10 @@ def resolve_binaural_hrtf_input(args, *, required): if compiled is not None and cache_policy is not None: raise ValueError("显式 .jochrtf 输入不能再指定 --hrtf-cache-policy") - if compiled is not None and radius != 1.0: + if compiled is not None and radius is not None: raise ValueError("显式 .jochrtf 输入不能再选择 SOFA radius shell") if private is not None and (cache_policy is not None or cache_dir is not None - or radius != 1.0): + or radius is not None): raise ValueError( "Rosella 模型输入不能使用 " "--hrtf-cache-policy/--hrtf-cache-dir/--hrtf-radius-m") @@ -116,12 +99,11 @@ def resolve_binaural_hrtf_input(args, *, required): if DEFAULT_SOFA_HRTF.is_file(): sofa = DEFAULT_SOFA_HRTF else: - compiled = _find_default_compiled_hrtf_cache() - if compiled is None and DEFAULT_PERSONALIZED_HEADPHONE.is_file(): + if DEFAULT_PERSONALIZED_HEADPHONE.is_file(): private = DEFAULT_PERSONALIZED_HEADPHONE if sofa is None and compiled is None and private is None: - if cache_policy is not None or cache_dir is not None or radius != 1.0: + if cache_policy is not None or cache_dir is not None or radius is not None: raise ValueError("HRTF cache/radius 选项需要 --sofa-hrtf") if required: raise ValueError( @@ -133,18 +115,16 @@ def resolve_binaural_hrtf_input(args, *, required): return None if sofa is None and (cache_policy is not None or cache_dir is not None - or radius != 1.0): + or radius is not None): raise ValueError("HRTF cache/radius 选项需要 --sofa-hrtf") - effective_policy = "memory" if cache_policy is None else cache_policy - if cache_dir is not None and (sofa is None or effective_policy != "disk"): - raise ValueError("--hrtf-cache-dir 仅与 SOFA 的 disk cache policy 一起使用") + if cache_policy == "disk" or cache_dir is not None: + raise ValueError("SOFA直接渲染完整HRIR;请用 src/compile_hrtf.py 显式导出 .jochrtf") if sofa is not None: return { "kind": "sofa", "path": resolve_sofa_hrtf(sofa), - "cache_policy": effective_policy, - "cache_dir": (DEFAULT_HRTF_CACHE_DIR if cache_dir is None else - cache_dir.expanduser().resolve()), + "cache_policy": None, + "cache_dir": None, } if compiled is not None: return { @@ -455,13 +435,13 @@ def render(index, bed_path, frame_count, raw_path, gain, progress_every, if speaker_renderer is not None: stage = time.perf_counter() speaker_pcm = speaker_renderer.render_frame( - pcm16.T, subs.get(11), speaker_metadata_offset) + pcm16.T, subs.get(11) if subs is not None else None, speaker_metadata_offset) speaker_render_seconds += time.perf_counter() - stage stage = time.perf_counter() speaker_sink.write_frame(speaker_pcm) speaker_write_seconds += time.perf_counter() - stage if binaural_renderer is not None: - payload = subs.get(11) + payload = subs.get(11) if subs is not None else None outer_offset = ( index.subpayload_sample_offset(row, 11) if payload is not None and hasattr(index, "subpayload_sample_offset") @@ -548,16 +528,13 @@ def build_parser(): parser.add_argument("--speaker-metadata-offset", type=int, default=1473, help="扬声器渲染 metadata 相对帧偏移,默认 1473 samples") parser.add_argument("--binaural-mode", choices=("off", "near", "mid", "far"), - default="mid", - help="双耳渲染模式,默认 mid(人为指定的渲染提示,非码流 " - "原始元数据);直接双耳渲染与 ADM BWF 的 DBMD 提示共用。" - "off 仅用于 ADM BWF:关闭 DBMD 双耳提示(编码 0)") + default=None, + help="双耳渲染模式或ADM DBMD提示,默认mid;支持near/mid/far。" + "off仅用于ADM BWF关闭DBMD双耳提示") hrtf_input = parser.add_mutually_exclusive_group() hrtf_input.add_argument( "--sofa-hrtf", type=Path, - help="SimpleFreeFieldHRIR SOFA;缺省时依次尝试 HRTF/binaural.sofa、" - "output/hrtf-cache 下唯一的 .jochrtf、" - "HRTF/binaural.personalized_headphone,均无则报错") + help=f"完整HRIR SOFA;默认 {DEFAULT_SOFA_HRTF};缺失时尝试原个性化默认文件") hrtf_input.add_argument( "--compiled-hrtf-cache", type=Path, help="高级入口:显式读取 JOC .jochrtf compiled cache") @@ -572,13 +549,17 @@ def build_parser(): "2=11 taps/3 slots;仅作用于个性化模型") parser.add_argument( "--hrtf-cache-policy", choices=("none", "memory", "disk"), default=None, - help="SOFA 编译缓存;默认 memory,disk 写入可删除的 .jochrtf") + help=argparse.SUPPRESS) parser.add_argument( "--hrtf-cache-dir", type=Path, - help="disk cache 目录;默认 output/hrtf-cache") + help=argparse.SUPPRESS) parser.add_argument( - "--hrtf-radius-m", type=float, default=1.0, - help="选择最近的 SOFA measurement-radius shell,默认 1.0 m") + "--hrtf-radius-m", type=float, default=None, + help="选择最近的SOFA测量球壳;单球壳默认自动选择,多球壳必须指定") + parser.add_argument("--reference-config", type=Path, + help="SOFA/.jochrtf的外部参考环境配置") + parser.add_argument("--sofa-interpolation", choices=("lf_logmag_phase", "lf_mca_zero"), + default=None, help="完整HRIR局部插值,默认 lf_logmag_phase") parser.add_argument("--binaural-tail-seconds", type=float, default=5.0, help="双耳 room/filterbank flush 上限,默认 5 秒") parser.add_argument("--binaural-tail-threshold", type=float, default=1.0e-8, @@ -625,6 +606,7 @@ def main(argv=None): if hasattr(stream, "reconfigure"): stream.reconfigure(encoding="utf-8", errors="backslashreplace") args = build_parser().parse_args(argv) + args.binaural_mode = args.binaural_mode or "mid" source = args.input.expanduser().resolve() if not source.is_file(): raise FileNotFoundError(source) @@ -634,7 +616,7 @@ def main(argv=None): if binaural_mode and binaural_render_mode == "off": raise ValueError( "--binaural-mode off 仅用于 ADM BWF 输出(关闭 DBMD 双耳提示);" - "直接双耳渲染请使用 near/mid/far") + "直接双耳渲染可使用near/mid/far") if args.speaker_output is not None and not speaker_mode: raise ValueError("--speaker-output 必须与 --speaker-layout 一起使用") if args.binaural_output is not None and not binaural_mode: @@ -653,7 +635,9 @@ def main(argv=None): args.personalized_headphone is not None, args.hrtf_cache_policy is not None, args.hrtf_cache_dir is not None, - args.hrtf_radius_m != 1.0, + args.hrtf_radius_m is not None, + args.reference_config is not None, + args.sofa_interpolation is not None, )) if hrtf_options_used and not binaural_mode: raise ValueError("SOFA/HRTF 选项仅与 --binaural 一起使用") @@ -669,7 +653,7 @@ def main(argv=None): raise ValueError("binaural-tail-threshold 必须是非负有限值") if args.binaural_chunk_frames <= 0: raise ValueError("binaural-chunk-frames 必须大于 0") - if not math.isfinite(args.hrtf_radius_m) or args.hrtf_radius_m <= 0.0: + if args.hrtf_radius_m is not None and (not math.isfinite(args.hrtf_radius_m) or args.hrtf_radius_m <= 0.0): raise ValueError("hrtf-radius-m 必须是正有限值") requested_output = (args.speaker_output if args.speaker_output is not None else args.binaural_output if args.binaural_output is not None @@ -694,6 +678,12 @@ def main(argv=None): raise ValueError("eac3-target-level 必须在 -31..0 之间") binaural_hrtf_input = resolve_binaural_hrtf_input( args, required=binaural_mode and not args.metadata_only) + if binaural_hrtf_input is not None: + kind = binaural_hrtf_input["kind"] + if kind == "rosella" and (args.reference_config is not None or args.sofa_interpolation is not None): + raise ValueError("参考环境/HRIR插值选项仅用于SOFA路径") + if kind == "compiled_cache" and args.sofa_interpolation is not None: + raise ValueError("--sofa-interpolation不适用于已编译输入") if (args.rosella_filterbank is not None and binaural_hrtf_input is not None and binaural_hrtf_input["kind"] != "rosella"): raise ValueError( @@ -824,6 +814,7 @@ def main(argv=None): "output_gain": gain_float64, "chunk_frames": args.binaural_chunk_frames, "bed_channels": getattr(index, "bed_channels", ()), + "reference_config": args.reference_config, } if hrtf_source["kind"] == "sofa": binaural_decoder = timed_call( @@ -833,6 +824,7 @@ def main(argv=None): cache_policy=hrtf_source["cache_policy"], cache_dir=hrtf_source["cache_dir"], shell_radius_m=args.hrtf_radius_m, + interpolation=args.sofa_interpolation or "lf_logmag_phase", **common_options) elif hrtf_source["kind"] == "rosella": binaural_decoder = timed_call( @@ -914,10 +906,10 @@ def main(argv=None): "source_sha256": ( binaural_backend_info["field"]["source_sha256"]), "cache_policy": binaural_backend_info["cache_policy"], - "cache_key": ( - binaural_backend_info["field"]["cache_key"]), - "format_version": ( - binaural_backend_info["field"]["format_version"]), + "cache_key": binaural_backend_info["field"].get("cache_key"), + "format_version": binaural_backend_info["field"].get("format_version"), + "representation": binaural_backend_info.get("direct_representation"), + "reference": binaural_backend_info.get("reference"), } finally: spool.close() diff --git a/src/binaural_reference.py b/src/binaural_reference.py new file mode 100644 index 0000000..9a8a6b1 --- /dev/null +++ b/src/binaural_reference.py @@ -0,0 +1,137 @@ +"""Configured reference environment, geometry and scalar level calibration.""" +from __future__ import annotations + +from dataclasses import dataclass +import hashlib +import json +import math +import os +from pathlib import Path +import numpy as np + +from rosella_bank import RosellaBank +from rosella_direct import _profile_geometry, direct_and_room_send +from rosella_model import load_personalized_headphone +from rosella_room import RosellaRoomFir + + +DEFAULT_REFERENCE_CONFIG = Path(__file__).resolve().parents[1]/"local"/"binaural_reference.json" +REFERENCE_MODES = {"near": 1, "mid": 3, "far": 2} + + +@dataclass(frozen=True) +class ReferencePosition: + direction: np.ndarray + distance_m: float + ear_distance_m: float + centre_weight: float + direct_gain: float + room_send: float + + +class ReferenceEnvironment: + def __init__(self, config=None, *, mode=None): + self.config_path = Path(config or os.environ.get("BINAURAL_REFERENCE_CONFIG") or + DEFAULT_REFERENCE_CONFIG).expanduser().resolve() + with self.config_path.open(encoding="utf-8") as stream: + spec = json.load(stream) + self.configuration_sha256 = hashlib.sha256( + json.dumps(spec, sort_keys=True, separators=(",", ":")).encode("utf-8")).hexdigest() + if spec.get("schema") != "binaural-reference-v1": + raise ValueError("unsupported reference configuration") + path = Path(spec["model_path"]).expanduser() + if not path.is_absolute(): + path = self.config_path.parent/path + self.model_sha256 = hashlib.sha256(path.read_bytes()).hexdigest() + if self.model_sha256.lower() != str(spec["model_sha256"]).lower(): + raise ValueError("reference data hash mismatch") + self.model = load_personalized_headphone(path) + self.calibration_profile_index = int(spec["profile_index"]) + self.profile_id = str(spec["profile_id"]) + if self.calibration_profile_index not in (2, 3) or self.model.sample_rate != 48000: + raise ValueError("unsupported reference profile") + self.mode = ({v: k for k, v in REFERENCE_MODES.items()}[self.calibration_profile_index] + if mode is None else str(mode).lower()) + if self.mode not in REFERENCE_MODES: + raise ValueError("reference mode must be near, mid, or far") + self.profile_index = REFERENCE_MODES[self.mode] + if int(self.model.header_integer_fields[0]) != 0: + raise ValueError("reference-v1 requires a Cartesian mapping") + self.profile = self.model.profiles[self.profile_index] + self.compiled_receivers = spec.get("compiled_receiver_positions_m") + self.room_slots = int(spec.get("room_impulse_slots", 4096)) + self.calibration_directions = int(spec.get("level_calibration_directions", 128)) + if not 128 <= self.room_slots <= 16384 or not 16 <= self.calibration_directions <= 2048: + raise ValueError("reference configuration exceeds supported bounds") + self.h0, self.coupling = map(float, self.model.header_float_scalars[:2]) + if self.h0 <= 0 or self.coupling <= 0: + raise ValueError("invalid reference energy coupling") + self.minimum_distance_m = float(self.profile.minimum_normalized_radius)*float(self.profile.distance_scale_m) + self.reference_distance_m = self.map_position((0., 1., 0.)).distance_m + + def map_position(self, position, *, profile_index=None): + position = np.asarray(position, dtype=np.float64) + if position.shape != (3,) or not np.isfinite(position).all(): + raise ValueError("reference position must contain three finite values") + extension = max(1., float(np.max(np.abs(position)))) + if extension > 50.1+1e-9: + raise ValueError("position exceeds the finite metadata domain") + index = self.profile_index if profile_index is None else int(profile_index) + profile, direction, radius, _, _ = _profile_geometry(self.model, position/extension, index) + distance = float(profile.distance_scale_m)*radius*extension + minimum = float(profile.minimum_normalized_radius)*float(profile.distance_scale_m) + regularized = max(distance, minimum) + alpha = distance/regularized + effective = self.h0*distance + if index == REFERENCE_MODES["near"]: + a, send = 1., 0. + else: + a = 1/math.sqrt(1+self.coupling*effective*effective) + send = effective*a + return ReferencePosition(np.asarray(direction), distance, regularized, alpha, a, send) + + def create_room(self): + return RosellaRoomFir(self.model, impulse_slots=self.room_slots) + + @staticmethod + def create_bank(source_channels): + return RosellaBank(1, source_channels=source_channels, output_channels=2) + + def calibrate(self, field, spectrum_at): + from public_filterbank import hybrid_gain_synthesis_dictionary + from rosella_direct import _FIELD_CACHE + # Discard stale cache entries when object ids are reused. + _FIELD_CACHE.pop(id(self.model), None) + count = self.calibration_directions + i = np.arange(count, dtype=np.float64) + z = 1-2*(i+.5)/count + phi = i*np.pi*(3-math.sqrt(5)) + unit = np.column_stack((np.sqrt(1-z*z)*np.cos(phi), np.sqrt(1-z*z)*np.sin(phi), z)) + dictionary = hybrid_gain_synthesis_dictionary(field.fft_size) + band = (field.frequencies >= 100) & (field.frequencies <= 16000) + weights = 1/field.frequencies[band] + reference_energy = 0. + new_energy = 0. + for direction in unit: + adm = (-direction[1], direction[0], direction[2]) + state = self.map_position(adm, profile_index=self.calibration_profile_index) + parameters = direct_and_room_send(self.model, adm, self.calibration_profile_index) + vector = np.empty((2, 154), dtype=np.float64) + vector[:, 0::2], vector[:, 1::2] = parameters.gains.real, parameters.gains.imag + reference = np.fft.rfft(vector @ dictionary.T, field.fft_size) + target = spectrum_at(state)*state.direct_gain + reference_energy += float(np.sum(np.abs(reference[:, band])**2*weights)) + new_energy += float(np.sum(np.abs(target[:, band])**2*weights)) + if min(reference_energy, new_energy) <= 1e-24 or not math.isfinite(new_energy): + raise ValueError("reference level calibration has no finite signal energy") + return math.sqrt(reference_energy/new_energy) + + def info(self): + return dict(profile_id=self.profile_id, model_sha256=self.model_sha256, + configuration_sha256=self.configuration_sha256, + mode=self.mode, selected_profile_index=self.profile_index, + calibration_profile_index=self.calibration_profile_index, + distance_semantics="reference-reproduction-metres", reference_distance_m=self.reference_distance_m, + regularized_distance_m=self.minimum_distance_m, + filterbank=1, room_impulse_slots=self.room_slots, + calibration_directions=self.calibration_directions) diff --git a/src/binaural_renderer.py b/src/binaural_renderer.py index 5ef7f49..008777a 100644 --- a/src/binaural_renderer.py +++ b/src/binaural_renderer.py @@ -1,4 +1,4 @@ -"""JOC frame adapter for the public SOFA binaural backend.""" +"""JOC frame adapter for independent SOFA and compact HRTF rendering.""" from __future__ import annotations import math @@ -8,7 +8,6 @@ import numpy as np from binaural_metadata import OamdPositionTimeline from public_filterbank import ANALYSIS_SYNTHESIS_LATENCY_SAMPLES -from sofa_binaural_backend import SofaBinauralBackend from sofa_hrtf_field import ( DEFAULT_HRTF_CACHE_DIR, DEFAULT_PROJECTION_RIDGE, @@ -50,14 +49,13 @@ def resolve_compiled_hrtf_cache(path: str | Path) -> Path: class SofaBinauralRenderer: """Render interleaved LFE plus fifteen JOC objects to stereo. - The adapter owns frame buffering and sample-timed OAMD updates. The - backend owns the 64-QMF/77-hybrid state, the 961-sample latency policy, - per-object direct/early state, and the shared late room. + The adapter owns frame buffering and sample-timed OAMD updates; the + selected backend owns direct filtering, reference state and latency. """ def __init__( self, backend, *, - mode: str = "mid", + mode: str | None = None, object_delay_samples: int = 1473, tail_seconds: float = 5.0, chunk_frames: int = 64, @@ -72,6 +70,7 @@ class SofaBinauralRenderer: f"missing: {', '.join(missing)}") if backend.source_count != SOURCE_CHANNELS: raise ValueError(f"JOC binaural backend must have {SOURCE_CHANNELS} sources") + mode = backend.default_profile if mode is None else mode if backend.default_profile != str(mode).lower(): raise ValueError("backend default profile does not match renderer mode") if int(object_delay_samples) < 0: @@ -90,7 +89,9 @@ class SofaBinauralRenderer: self.dsp_backend = getattr(backend, "dsp_backend", "python-sofa") from bed_layout import bed_object_positions - self.timeline = OamdPositionTimeline( + from sofa_metadata import SofaPositionTimeline + timeline_type = SofaPositionTimeline if getattr(backend, "fixed_reference", False) else OamdPositionTimeline + self.timeline = timeline_type( 15, bed_positions=bed_object_positions(bed_channels or ())) self._input_buffer = np.empty( @@ -105,28 +106,32 @@ class SofaBinauralRenderer: @classmethod def from_sofa( cls, sofa: str | Path, *, - mode: str = "mid", - cache_policy: str = "memory", - cache_dir: str | Path | None = DEFAULT_HRTF_CACHE_DIR, - shell_radius_m: float = 1.0, + mode: str | None = None, + cache_policy: str | None = None, + cache_dir: str | Path | None = None, + shell_radius_m: float | None = None, projection_ridge: float = DEFAULT_PROJECTION_RIDGE, sh_ridge: float = DEFAULT_SH_RIDGE, object_delay_samples: int = 1473, tail_seconds: float = 5.0, output_gain: float = 1.0, chunk_frames: int = 64, - bed_channels=None) -> "SofaBinauralRenderer": + bed_channels=None, reference_config=None, + interpolation: str = "lf_logmag_phase") -> "SofaBinauralRenderer": + from sofa_reference_backend import ReferenceSofaBackend + if cache_policy not in (None, "none", "memory") or cache_dir is not None: + raise ValueError("SOFA renders full HRIRs; use compile_hrtf.py to export a .jochrtf") + if projection_ridge != DEFAULT_PROJECTION_RIDGE or sh_ridge != DEFAULT_SH_RIDGE: + raise ValueError("full HRIR rendering does not use projection/SH ridge parameters") source = resolve_sofa_hrtf(sofa) - backend = SofaBinauralBackend.from_sofa( + backend = ReferenceSofaBackend( source, source_count=SOURCE_CHANNELS, - default_profile=mode, output_gain=output_gain, - cache_policy=cache_policy, - cache_dir=cache_dir, + reference_config=reference_config, + mode=mode, shell_radius_m=shell_radius_m, - projection_ridge=projection_ridge, - sh_ridge=sh_ridge) + method=interpolation) return cls( backend, mode=mode, @@ -138,18 +143,18 @@ class SofaBinauralRenderer: @classmethod def from_compiled_cache( cls, cache: str | Path, *, - mode: str = "mid", + mode: str | None = None, object_delay_samples: int = 1473, tail_seconds: float = 5.0, output_gain: float = 1.0, chunk_frames: int = 64, - bed_channels=None) -> "SofaBinauralRenderer": + bed_channels=None, reference_config=None) -> "SofaBinauralRenderer": + from sofa_reference_backend import ReferenceSofaBackend source = resolve_compiled_hrtf_cache(cache) - backend = SofaBinauralBackend.from_compiled_cache( + backend = ReferenceSofaBackend( source, source_count=SOURCE_CHANNELS, - default_profile=mode, - output_gain=output_gain) + compiled=True, reference_config=reference_config, output_gain=output_gain, mode=mode) return cls( backend, mode=mode, @@ -222,15 +227,16 @@ class SofaBinauralRenderer: values = np.asarray(source, dtype=np.float64) if values.ndim != 2 or values.shape[1] != SOURCE_CHANNELS: raise ValueError(f"expected [samples,{SOURCE_CHANNELS}], got {values.shape}") - if len(values) % BINAURAL_BLOCK_SAMPLES: - raise ValueError("binaural input must be divisible by 512 samples") + block_samples = int(getattr(self.backend, "block_samples", BINAURAL_BLOCK_SAMPLES)) + if len(values) % block_samples: + raise ValueError("binaural input must contain complete processing blocks") outputs = [] block_base = self.processed_input_samples - for start in range(0, len(values), BINAURAL_BLOCK_SAMPLES): + for start in range(0, len(values), block_samples): sample = block_base + start self._set_block_parameters(sample) outputs.append(self.backend.process( - values[start:start + BINAURAL_BLOCK_SAMPLES])) + values[start:start + block_samples])) self.processed_input_samples += len(values) nonempty = [value for value in outputs if len(value)] if not nonempty: @@ -240,7 +246,7 @@ class SofaBinauralRenderer: return output def finish(self) -> np.ndarray: - """Process pending source samples and drain early/late room state once.""" + """Process pending source samples and drain filter/reference state once.""" if self.finished: return np.empty((0, OUTPUT_CHANNELS), dtype=np.float64) outputs: list[np.ndarray] = [] @@ -264,10 +270,10 @@ class SofaBinauralRenderer: def backend_info(self) -> dict: info = self.backend.info() info.update({ - "adapter": "JOC 1536-frame / 512-sample metadata", + "adapter": f"JOC 1536-frame / {getattr(self.backend, 'block_samples', BINAURAL_BLOCK_SAMPLES)}-sample metadata", "dsp_backend": self.dsp_backend, "mode": self.mode, - "latency_compensated_samples": BINAURAL_LATENCY_SAMPLES, + "latency_compensated_samples": getattr(self.backend, "latency_samples", BINAURAL_LATENCY_SAMPLES), "object_delay_samples": self.object_delay_samples, "tail_seconds": self.tail_seconds, "metadata_payloads": self.timeline.payload_count, diff --git a/src/compile_hrtf.py b/src/compile_hrtf.py new file mode 100644 index 0000000..857d238 --- /dev/null +++ b/src/compile_hrtf.py @@ -0,0 +1,26 @@ +"""Explicit offline export of a SOFA source to the compact cache format.""" +import argparse +from pathlib import Path +from sofa_canonical import load_simple_free_field_hrir +from sofa_hrtf_field import compile_sofa_hrtf + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("sofa", type=Path) + parser.add_argument("output", type=Path) + parser.add_argument("--radius-m", type=float) + args = parser.parse_args(argv) + if args.output.suffix.lower() != ".jochrtf": + parser.error("output must use .jochrtf") + canonical = load_simple_free_field_hrir(args.sofa, target_sample_rate_hz=48000) + if args.radius_m is None and len(canonical.shells_m) != 1: + parser.error("multi-radius input requires --radius-m") + radius = float(canonical.shells_m[0]) if args.radius_m is None else args.radius_m + field = compile_sofa_hrtf(canonical, shell_radius_m=radius, cache_policy="none") + field.save(args.output) + print(args.output.resolve()) + + +if __name__ == "__main__": + main() diff --git a/src/oamd_bits.py b/src/oamd_bits.py index e1189f9..5d8991f 100644 --- a/src/oamd_bits.py +++ b/src/oamd_bits.py @@ -215,7 +215,7 @@ def _parse_program_assignment(reader, raw_payload): return program -def _parse_object_info_block(reader, object_index, in_bed_or_isf, raw_payload): +def _parse_object_info_block(reader, object_index, in_bed_or_isf, raw_payload, retain_spatial=False): """5.5.9 ``object_info_block(0)``:一个对象的属性更新。""" info = { "object": object_index, @@ -263,9 +263,16 @@ def _parse_object_info_block(reader, object_index, in_bed_or_isf, raw_payload): z_sign = reader.read(1) z = reader.read(4) info["position"] = (x, y, z if z_sign else -z) - if reader.read(1): # b_object_distance_specified - if not reader.read(1): # b_object_at_infinity - reader.read(4) # distance_factor_idx + outside = bool(reader.read(1)) + infinite = False + distance_index = None + if outside: + infinite = bool(reader.read(1)) + if not infinite: + distance_index = reader.read(4) + if retain_spatial: + info.update(distance_specified=outside, at_infinity=infinite, + distance_factor_idx=distance_index) if zone_present: info["zone_constraints_idx"] = reader.read(3) info["enable_elevation"] = bool(reader.read(1)) @@ -277,9 +284,12 @@ def _parse_object_info_block(reader, object_index, in_bed_or_isf, raw_payload): elif size_idx == 2: reader.read(15) if screen_present: - if reader.read(1): # b_object_use_screen_ref - reader.read(3) # screen_factor_bits - reader.read(2) # depth_factor_idx + screen_ref = bool(reader.read(1)) + screen_bits = reader.read(3) if screen_ref else None + depth_index = reader.read(2) if screen_ref else None + if retain_spatial: + info.update(use_screen_ref=screen_ref, screen_factor_bits=screen_bits, + depth_factor_idx=depth_index) info["snap"] = bool(reader.read(1)) if reader.read(1): # b_additional_table_data_exists info["additional_table_bytes"] = reader.read(4) + 1 @@ -287,7 +297,7 @@ def _parse_object_info_block(reader, object_index, in_bed_or_isf, raw_payload): return info -def _parse_object_element(reader, element, raw_payload): +def _parse_object_element(reader, element, raw_payload, retain_spatial=False): """5.5.5 ``object_element()``:时间信息 + 每个对象一条 object_info_block。""" object_count = element["object_count"] try: @@ -327,7 +337,7 @@ def _parse_object_element(reader, element, raw_payload): reader.read(5) objects = [ _parse_object_info_block( - reader, index, index < element["bed_isf_objects"], raw_payload) + reader, index, index < element["bed_isf_objects"], raw_payload, retain_spatial) for index in range(object_count) ] except UnsupportedVariantError: @@ -348,7 +358,7 @@ def _parse_object_element(reader, element, raw_payload): } -def _parse_elements(reader, alternate_present, object_count, program, raw_payload): +def _parse_elements(reader, alternate_present, object_count, program, raw_payload, retain_spatial=False): """5.5.4 ``oa_element_md()`` 目录;对象 element 按结构解析,其余按声明长度跳过。""" try: element_count = reader.read(4) @@ -425,7 +435,7 @@ def _parse_elements(reader, alternate_present, object_count, program, raw_payloa "payload": bytes_descriptor(raw_payload), }) object_element = _parse_object_element( - reader, element, raw_payload) + reader, element, raw_payload, retain_spatial) element["parsed_end_bit"] = reader.position element["object_element"] = object_element # 个别编码器声明的 oa_element_size 比实际内容短;以结构解析结果为准。 @@ -510,7 +520,7 @@ def program_bed_channels(bits_one): return program_bed_summary(bits_one)["channels"] -def frame_update(bits_one): +def frame_update(bits_one, *, retain_spatial=False): """单帧 OAMD → 位置字段增量及其 sample offset/ramp duration。""" bits, raw_payload = _payload_bits(bits_one) try: @@ -532,7 +542,7 @@ def frame_update(bits_one): program = _parse_program_assignment(reader, raw_payload) alternate_present = bool(reader.read(1)) object_element, elements = _parse_elements( - reader, alternate_present, object_count, program, raw_payload) + reader, alternate_present, object_count, program, raw_payload, retain_spatial) except UnsupportedVariantError: raise except ValueError as exc: @@ -584,8 +594,8 @@ def frame_update(bits_one): x, y, z = position out[(slot, "q1")] = q_of(x, N_Q12) if 0 <= x <= N_Q12 else None out[(slot, "q2")] = q_of(y, N_Q12) if 0 <= y <= N_Q12 else None - # 5.6.1.1.10/5.6.1.1.11:pos3D_Z 带符号;当前 16 槽模型只表达非负高度。 - out[(slot, "q3")] = q_of(z, N_Q3) if 0 <= z <= N_Q3 else ( + # Preserve legacy nonnegative height unless signed spatial output is requested. + out[(slot, "q3")] = q_of(z, N_Q3) if (0 <= z <= N_Q3 or retain_spatial and -N_Q3 <= z < 0) else ( 0 if z < 0 else None) size_mismatch = [ diff --git a/src/partitioned_convolution.py b/src/partitioned_convolution.py new file mode 100644 index 0000000..bca4604 --- /dev/null +++ b/src/partitioned_convolution.py @@ -0,0 +1,217 @@ +"""Uniform partitioned convolution with float64/complex128 state.""" +from __future__ import annotations + +import math +import numpy as np + + +class UniformPartitionedConvolver: + """Overlap-add FIR convolution with shared input history. + + Filter updates crossfade outputs with independent old/new overlap tails. + """ + + def __init__(self, filters, *, block_size: int = 128, + max_filter_taps: int | None = None): + values = np.asarray(filters, dtype=np.float64) + if values.ndim == 1: + values = values[None, :] + if values.ndim != 2 or values.shape[0] <= 0 or values.shape[1] <= 0: + raise ValueError("filters must have shape [outputs,taps]") + if not np.isfinite(values).all(): + raise ValueError("filters must be finite") + self.block_size = int(block_size) + if self.block_size <= 0: + raise ValueError("block_size must be positive") + self.fft_size = 2 * self.block_size + self.frequency_bins = self.block_size + 1 + self.outputs = int(values.shape[0]) + self.max_filter_taps = int(max_filter_taps or values.shape[1]) + if self.max_filter_taps < values.shape[1]: + raise ValueError("max_filter_taps is shorter than the initial filter") + self.partition_count = int(math.ceil(self.max_filter_taps / self.block_size)) + self.input_history = np.zeros( + (self.partition_count, self.frequency_bins), dtype=np.complex128) + self.write_index = -1 + self.processed_blocks = 0 + self.current_spectra = self._partition(values) + self.current_overlap = np.zeros( + (self.outputs, self.block_size), dtype=np.float64) + self.target_spectra: np.ndarray | None = None + self.target_overlap: np.ndarray | None = None + self.fade_samples = 0 + self.fade_position = 0 + + @property + def transition_active(self) -> bool: + return self.target_spectra is not None + + def _partition(self, filters) -> np.ndarray: + values = np.asarray(filters, dtype=np.float64) + if values.ndim == 1: + values = values[None, :] + if values.shape[0] != self.outputs or values.shape[1] > self.max_filter_taps: + raise ValueError( + f"filter update must have shape [{self.outputs}, <= {self.max_filter_taps}]") + if not np.isfinite(values).all(): + raise ValueError("filter update contains non-finite values") + padded = np.zeros( + (self.outputs, self.partition_count, self.fft_size), dtype=np.float64) + for partition in range(self.partition_count): + start = partition * self.block_size + stop = min(start + self.block_size, values.shape[1]) + if stop > start: + padded[:, partition, :stop - start] = values[:, start:stop] + return np.asarray(np.fft.rfft(padded, axis=-1), dtype=np.complex128) + + def _ordered_input_history(self) -> np.ndarray: + if self.write_index < 0: + return np.zeros_like(self.input_history) + indices = (self.write_index - np.arange(self.partition_count)) % self.partition_count + return self.input_history[indices] + + def _convolution_block(self, spectra: np.ndarray, + overlap: np.ndarray) -> tuple[np.ndarray, np.ndarray]: + ordered = self._ordered_input_history() + frequency = np.einsum( + "pf,opf->of", ordered, spectra, dtype=np.complex128, optimize=False) + time = np.asarray(np.fft.irfft(frequency, self.fft_size, axis=-1), + dtype=np.float64) + output = time[:, :self.block_size] + overlap + return output, time[:, self.block_size:].copy() + + def _overlap_for_previous_block(self, spectra: np.ndarray) -> np.ndarray: + if self.processed_blocks == 0: + return np.zeros((self.outputs, self.block_size), dtype=np.float64) + ordered = self._ordered_input_history() + frequency = np.einsum( + "pf,opf->of", ordered, spectra, dtype=np.complex128, optimize=False) + time = np.asarray(np.fft.irfft(frequency, self.fft_size, axis=-1), + dtype=np.float64) + return time[:, self.block_size:].copy() + + def set_filter(self, filters, *, fade_blocks: int = 0) -> None: + if self.transition_active: + raise RuntimeError("finish the current filter crossfade before another update") + target = self._partition(filters) + fade_blocks = int(fade_blocks) + if fade_blocks < 0: + raise ValueError("fade_blocks must be non-negative") + if fade_blocks == 0: + self.current_spectra = target + self.current_overlap = self._overlap_for_previous_block(target) + return + self.target_spectra = target + self.target_overlap = self._overlap_for_previous_block(target) + self.fade_samples = fade_blocks * self.block_size + self.fade_position = 0 + + def process_block(self, samples) -> np.ndarray: + values = np.asarray(samples, dtype=np.float64) + if values.shape != (self.block_size,) or not np.isfinite(values).all(): + raise ValueError(f"samples must have finite shape ({self.block_size},)") + padded = np.zeros(self.fft_size, dtype=np.float64) + padded[:self.block_size] = values + spectrum = np.asarray(np.fft.rfft(padded), dtype=np.complex128) + self.write_index = (self.write_index + 1) % self.partition_count + self.input_history[self.write_index] = spectrum + + current, next_current_overlap = self._convolution_block( + self.current_spectra, self.current_overlap) + if not self.transition_active: + self.current_overlap = next_current_overlap + self.processed_blocks += 1 + return current.T.copy() + + target, next_target_overlap = self._convolution_block( + self.target_spectra, self.target_overlap) + sample_indices = self.fade_position + np.arange( + 1, self.block_size + 1, dtype=np.float64) + amount = np.clip(sample_indices / float(self.fade_samples), 0.0, 1.0) + output = current * (1.0 - amount[None, :]) + target * amount[None, :] + self.fade_position += self.block_size + self.current_overlap = next_current_overlap + self.target_overlap = next_target_overlap + if self.fade_position >= self.fade_samples: + self.current_spectra = self.target_spectra + self.current_overlap = self.target_overlap + self.target_spectra = None + self.target_overlap = None + self.fade_samples = 0 + self.fade_position = 0 + self.processed_blocks += 1 + return output.T.copy() + + def reset(self) -> None: + self.input_history.fill(0.0) + self.current_overlap.fill(0.0) + if self.target_overlap is not None: + self.target_overlap.fill(0.0) + self.write_index = -1 + self.processed_blocks = 0 + self.target_spectra = None + self.target_overlap = None + self.fade_samples = 0 + self.fade_position = 0 + + +class MultiSourcePartitionedConvolver: + """Sum independent per-source convolution outputs.""" + + def __init__(self, source_count: int, *, block_size: int, + max_filter_taps: int, output_channels: int = 2): + self.source_count = int(source_count) + self.block_size = int(block_size) + self.output_channels = int(output_channels) + if min(self.source_count, self.block_size, self.output_channels, + int(max_filter_taps)) <= 0: + raise ValueError("source_count, block_size, outputs and taps must be positive") + zero = np.zeros((self.output_channels, int(max_filter_taps)), dtype=np.float64) + self.convolvers = [ + UniformPartitionedConvolver( + zero, block_size=self.block_size, max_filter_taps=max_filter_taps) + for _ in range(self.source_count) + ] + + def set_filter(self, source: int, filters, *, fade_blocks: int = 0) -> None: + source = int(source) + if not 0 <= source < self.source_count: + raise IndexError(source) + self.convolvers[source].set_filter(filters, fade_blocks=fade_blocks) + + def process_block(self, sources) -> np.ndarray: + values = np.asarray(sources, dtype=np.float64) + if values.shape != (self.block_size, self.source_count): + raise ValueError( + f"sources must have shape ({self.block_size},{self.source_count})") + if not np.isfinite(values).all(): + raise ValueError("sources contain non-finite values") + output = np.zeros((self.block_size, self.output_channels), dtype=np.float64) + for source in range(self.source_count): + output += self.convolvers[source].process_block(values[:, source]) + return output + + def process(self, sources) -> np.ndarray: + values = np.asarray(sources, dtype=np.float64) + if values.ndim != 2 or values.shape[1] != self.source_count: + raise ValueError(f"sources must have shape [samples,{self.source_count}]") + if len(values) % self.block_size: + raise ValueError("partitioned input length must be divisible by block_size") + blocks = [ + self.process_block(values[start:start + self.block_size]) + for start in range(0, len(values), self.block_size) + ] + return (np.concatenate(blocks, axis=0) if blocks + else np.zeros((0, self.output_channels), dtype=np.float64)) + + def flush(self, blocks: int | None = None) -> np.ndarray: + count = (max(conv.partition_count for conv in self.convolvers) + 2 + if blocks is None else int(blocks)) + if count < 0: + raise ValueError("flush block count must be non-negative") + zero = np.zeros((self.block_size, self.source_count), dtype=np.float64) + return self.process(np.tile(zero, (count, 1))) + + def reset(self) -> None: + for convolver in self.convolvers: + convolver.reset() diff --git a/src/sofa_hrir.py b/src/sofa_hrir.py new file mode 100644 index 0000000..a847ffa --- /dev/null +++ b/src/sofa_hrir.py @@ -0,0 +1,220 @@ +"""Local spherical interpolation of full complex HRIR spectra.""" +from __future__ import annotations + +from collections import OrderedDict +import math +import numpy as np +from scipy.interpolate import PchipInterpolator +from scipy.spatial import ConvexHull, SphericalVoronoi, cKDTree + +from sofa_canonical import CanonicalHrtf, load_simple_free_field_hrir + + +class FullHrirField: + def __init__(self, source, *, method="lf_logmag_phase", shell_radius_m=None, + fft_size=None): + if method not in ("lf_logmag_phase", "lf_mca_zero"): + raise ValueError("unsupported full HRIR interpolation method") + self.canonical = (source if isinstance(source, CanonicalHrtf) else + load_simple_free_field_hrir(source, target_sample_rate_hz=48000)) + c = self.canonical + if c.sample_rate_hz != 48000: + raise ValueError("full HRIR renderer requires 48000 Hz") + shells = c.shells_m + if shell_radius_m is not None and (not math.isfinite(shell_radius_m) or shell_radius_m <= 0): + raise ValueError("shell radius must be finite and positive") + if shell_radius_m is None and len(shells) != 1: + raise ValueError("multi-radius SOFA requires an explicit --hrtf-radius-m") + selected = c.shell_indices(shells[0] if shell_radius_m is None else shell_radius_m) + if np.any(np.sum(c.hrir[selected]**2, axis=-1) <= 0): + raise ValueError("each selected HRIR ear must contain signal energy") + self.measurement_radius_m = float(np.median(c.measurement_radius_m[selected])) + support = c.taps + int(math.ceil(float(np.max(c.delay_samples[selected])))) + 1 + if fft_size is None: + fft_size = max(2048, 1 << (2*support-1).bit_length()) + if fft_size > 16384: + raise ValueError("HRIR and Data.Delay exceed the automatic FFT budget") + self.method, self.fft_size = method, int(fft_size) + if self.fft_size < 64 or self.fft_size % 2 or c.taps + np.max(c.delay_samples[selected]) >= self.fft_size//2: + raise ValueError("HRIR and Data.Delay exceed the selected FFT time budget") + receiver = c.receiver_position_cartesian_m[selected] + if not np.allclose(receiver, receiver[0], atol=1e-9, rtol=0): + raise ValueError("moving receiver geometry is unsupported") + self.receivers = receiver[0].copy() + if np.linalg.norm(self.receivers.mean(axis=0)) > 1e-6: + raise ValueError("radial model requires ears centred on the listener") + self.head_radius_m = float(np.linalg.norm(self.receivers[0])) + if not (0.04 < self.head_radius_m < .15) or not np.isclose( + np.linalg.norm(self.receivers[1]), self.head_radius_m, atol=1e-6): + raise ValueError("unsupported receiver geometry for the radial model") + self.ear_directions = self.receivers/self.head_radius_m + unit = c.unit_directions[selected] + _, first, inverse = np.unique(np.round(unit, 10), axis=0, return_index=True, return_inverse=True) + for group in np.flatnonzero(np.bincount(inverse) > 1): + members = selected[inverse == group] + if not (np.allclose(c.hrir[members], c.hrir[members[0]], atol=1e-10, rtol=1e-8) + and np.allclose(c.delay_samples[members], c.delay_samples[members[0]], atol=1e-10, rtol=0)): + raise ValueError("coincident SOFA measurements disagree") + selected = selected[np.sort(first)] + self.directions = c.unit_directions[selected].copy() + self.tree = cKDTree(self.directions) + hull = ConvexHull(self.directions) + if np.any(hull.equations[:, 3] >= -1e-9): + raise ValueError("SOFA must enclose the listener; extrapolation is unsupported") + self.faces = hull.simplices + vertices = self.directions[self.faces] + dots = np.einsum("fij,fkj->fik", vertices, vertices) + self.max_triangle_edge_degrees = math.degrees(math.acos(float(np.clip(dots.min(), -1, 1)))) + matrix = self.directions[self.faces].transpose(0, 2, 1) + if np.max(np.linalg.cond(matrix)) > 1e8: + raise ValueError("ill-conditioned spherical interpolation mesh") + self.inverse = np.linalg.inv(matrix) + self.frequencies = np.fft.rfftfreq(self.fft_size, 1/c.sample_rate_hz) + self.omega = 2*np.pi*self.frequencies/c.sample_rate_hz + raw = np.fft.rfft(c.hrir[selected], self.fft_size) + fit = (self.frequencies >= 200) & (self.frequencies <= 1500) + x = self.omega[fit]-self.omega[fit].mean() + phase = np.unwrap(np.angle(raw), axis=-1)[..., fit] + self.delays = -np.sum(x*phase, axis=-1)/np.sum(x*x) + c.delay_samples[selected] + residual_phase = phase+(self.delays-c.delay_samples[selected])[..., None]*self.omega[fit] + residual_phase -= residual_phase.mean(axis=-1, keepdims=True) + self.phase_fit_rms_rad = np.sqrt(np.mean(residual_phase**2, axis=-1)) + self.recording_origin_samples = int(round(float(np.median(self.delays.mean(axis=1))))) + self.common_latency_samples = self.fft_size//2 + physical = raw*np.exp(-1j*self.omega*c.delay_samples[selected, :, None]) + self.floor = max(float(np.max(np.abs(physical)))*1e-6, 1e-15) + areas = SphericalVoronoi(self.directions).calculate_areas() + areas /= areas.sum() + energy = np.einsum("m,mef->f", areas, np.abs(physical)**2)/2 + self.centre_spectrum = np.sqrt(energy)*np.exp(-1j*self.omega*self.recording_origin_samples) + self.aligned = physical*np.exp(1j*self.omega*self.delays[..., None]) + self.log_magnitude = None + self.auditory = None + if method == "lf_logmag_phase": + self.log_magnitude = np.log(np.maximum(np.abs(physical), self.floor)) + else: + erb = lambda f: 21.4*np.log10(1+.00437*f) + self.centres = (10**(np.linspace(erb(50), erb(20000), 41)/21.4)-1)/.00437 + bw = 1.019*24.7*(1+.00437*self.centres) + self.bank = (1+((self.frequencies[None, :]-self.centres[:, None])/bw[:, None])**2)**-4 + self.bank[:, 1:-1] *= 2 + self.bank /= self.bank.sum(axis=1, keepdims=True) + self.auditory = self._auditory_db(physical) + self._cache = OrderedDict() + + def _auditory_db(self, spectrum): + return 10*np.log10(np.maximum(np.abs(spectrum)**2 @ self.bank.T, 1e-24)) + + def weights(self, direction): + q = np.asarray(direction, dtype=np.float64) + if q.shape != (3,) or not np.isfinite(q).all() or np.linalg.norm(q) <= 1e-12: + raise ValueError("query direction must be a finite nonzero vector") + q = q/np.linalg.norm(q) + distance, nearest = self.tree.query(q) + if distance < 1e-10: + return np.array([nearest]), np.array([1.]), 0. + g = self.inverse @ q + valid = np.flatnonzero(np.min(g, axis=1) >= -1e-10) + if not len(valid): + raise ValueError("no containing spherical triangle") + face = int(valid[0]) + weights = np.maximum(g[face], 0) + weights /= weights.sum() + ids = self.faces[face] + vertices = self.directions[ids] + edge = math.degrees(math.acos(float(np.clip(np.min(vertices @ vertices.T), -1, 1)))) + return ids, weights, edge + + def spectrum(self, direction): + q = np.asarray(direction, dtype=np.float64) + if q.shape != (3,) or not np.isfinite(q).all() or np.linalg.norm(q) <= 1e-12: + raise ValueError("query direction must be a finite nonzero vector") + q = q/np.linalg.norm(q) + key = tuple(q) + if key in self._cache: + self._cache.move_to_end(key) + return self._cache[key].copy() + ids, weights, _ = self.weights(q) + delay = weights @ self.delays[ids] + spectrum = np.einsum("i,ief->ef", weights, self.aligned[ids]) + spectrum *= np.exp(-1j*self.omega*delay[:, None]) + if len(ids) > 1: + if self.method == "lf_logmag_phase": + magnitude = np.exp(np.einsum("i,ief->ef", weights, self.log_magnitude[ids])) + spectrum = magnitude*spectrum/np.maximum(np.abs(spectrum), self.floor) + else: + target = np.einsum("i,ieb->eb", weights, self.auditory[ids]) + difference = target-self._auditory_db(spectrum) + correction = PchipInterpolator(self.centres, difference, axis=-1)( + np.clip(self.frequencies, self.centres[0], self.centres[-1])) + spectrum *= 10**(np.clip(correction, -12, 12)/20) + self._cache[key] = spectrum.copy() + if len(self._cache) > 128: + self._cache.popitem(last=False) + return spectrum + + def info(self): + arrays = [self.aligned, self.log_magnitude, self.auditory, self.delays, + self.inverse, self.directions, self.faces] + return dict(source_sha256=self.canonical.source_sha256, + sample_rate_hz=self.canonical.sample_rate_hz, + convention=self.canonical.convention, + measurement_radius_m=self.measurement_radius_m, + method=self.method, fft_size=self.fft_size, + retained_filter_taps=self.fft_size, + recording_origin_samples=self.recording_origin_samples, + interpolation_storage_bytes=sum(x.nbytes for x in arrays if x is not None), + max_triangle_edge_degrees=self.max_triangle_edge_degrees, + phase_fit_rms_rad_p95=float(np.percentile(self.phase_fit_rms_rad, 95)), + ear_model="complex-rigid-sphere-distance-ratio", + radial_model_validation="analytic-identity-and-series-checks; approximate-human-head") + + +def sphere_transfer(frequencies, radius, head_radius, cosines, *, sound_speed=343.3, + tolerance=1e-10): + """Point-source pressure / free-field centre pressure, FFT sign convention. + + Evaluate the rigid-sphere series using adjacent Hankel ratios, avoiding + overflow of high-order Neumann functions at low frequency. + """ + f = np.asarray(frequencies, dtype=np.float64) + r, a = float(radius), float(head_radius) + mu = np.asarray(cosines, dtype=np.float64) + if f.ndim != 1 or not np.isfinite(f).all() or np.any(f < 0) or not np.isfinite(mu).all(): + raise ValueError("invalid sphere frequency or incidence") + if not (math.isfinite(r) and math.isfinite(a) and r > a > 0): + raise ValueError("sphere source must be outside the head") + x = 2*np.pi*f*a/sound_speed + z = x*(r/a) + order = max(int(math.ceil(float(x.max(initial=0))))+32, + int(math.ceil(math.log(tolerance)/math.log(a/r)))+16) + if order > 4096: + raise ValueError("source too close to the sphere surface") + mu = np.clip(mu, -1., 1.) + p_before, p = np.ones_like(mu), mu.copy() + result = np.ones(mu.shape+(len(f),), dtype=np.complex128)/(1+1j*x) + ratio_a = x/(1+1j*x) + ratio_r = z/(1+1j*z) + product = np.ones_like(x, dtype=np.complex128) + for n in range(1, order+1): + quotient = np.divide(ratio_a, ratio_r, out=np.full_like(ratio_a, a/r), where=z != 0) + product *= quotient + result += (2*n+1)*p[..., None]*product/(n+1-x*ratio_a) + p_before, p = p, ((2*n+1)*mu*p-n*p_before)/(n+1) + ratio_a = x/((2*n+1)-x*ratio_a) + ratio_r = z/((2*n+1)-z*ratio_r) + result *= np.exp(1j*x) + if not np.isfinite(result).all(): + raise ValueError("sphere series failed to converge") + return result + + +def radial_ratio(field, direction, radius): + if abs(float(radius)-field.measurement_radius_m) < 1e-10: + return np.ones((2, len(field.frequencies)), dtype=np.complex128) + mu = field.ear_directions @ np.asarray(direction, dtype=np.float64) + source = sphere_transfer(field.frequencies, field.measurement_radius_m, field.head_radius_m, mu) + target = sphere_transfer(field.frequencies, radius, field.head_radius_m, mu) + if np.min(np.abs(source)) < 1e-10: + raise ValueError("singular radial reference response") + return target/source diff --git a/src/sofa_metadata.py b/src/sofa_metadata.py new file mode 100644 index 0000000..e6c4ea1 --- /dev/null +++ b/src/sofa_metadata.py @@ -0,0 +1,53 @@ +"""SOFA spatial metadata adaptation on a sample-timed trajectory.""" +from __future__ import annotations + +import numpy as np +from binaural_metadata import OamdPositionTimeline +from oamd_bits import frame_update +from variant_error import UnsupportedVariantError + + +DISTANCE_FACTORS = (1.1, 1.3, 1.6, 2., 2.5, 3.2, 4., 5., 6.3, 7.9, 10., 12.6, + 15.8, 20., 25.1, 50.1) + + +class SofaPositionTimeline(OamdPositionTimeline): + def __init__(self, object_count=15, bed_positions=None): + super().__init__(object_count, bed_positions) + self.distance_factors = np.ones(16, dtype=np.float64) + + def _targets(self): + targets = super()._targets() + for index, target in enumerate(targets, 1): + if index in self.bed_positions or self.distance_factors[index] == 1: + continue + value = np.asarray(target, dtype=np.float64) + boundary = float(np.max(np.abs(value))) + if boundary <= 1e-12: + raise UnsupportedVariantError("sofa", "undefined_outside_direction", + "Outside object has no ray from the listener") + targets[index-1] = tuple(value/boundary*self.distance_factors[index]) + return targets + + def submit_update(self, update, **timing): + if update.get("program", {}).get("num_isf_objects", 0): + raise UnsupportedVariantError("sofa", "isf_not_supported", "ISF spatial mapping is unsupported") + for info in update.get("objects", ()): + slot = int(info["object"]) + if info.get("at_infinity"): + raise UnsupportedVariantError("sofa", "infinite_distance", + "Infinite-distance rendering is unsupported") + if info.get("use_screen_ref"): + raise UnsupportedVariantError("sofa", "screen_mapping_required", + "Screen-anchored objects require a calibrated mapping") + if "distance_specified" in info: + index = info.get("distance_factor_idx") + if info["distance_specified"] and (index is None or not 0 <= index < 16): + raise ValueError("invalid outside distance index") + self.distance_factors[slot] = DISTANCE_FACTORS[index] if info["distance_specified"] else 1. + return super().submit_update(update, **timing) + + def submit_payload(self, payload, **timing): + update = frame_update(payload, retain_spatial=True) + self.submit_update(update, **timing) + return update diff --git a/src/sofa_reference_backend.py b/src/sofa_reference_backend.py new file mode 100644 index 0000000..b8aee1f --- /dev/null +++ b/src/sofa_reference_backend.py @@ -0,0 +1,270 @@ +"""Direct HRIR rendering with a configured reference environment.""" +from __future__ import annotations + +from collections import OrderedDict +import math +from pathlib import Path +from types import SimpleNamespace +import numpy as np +from scipy import signal + +from binaural_reference import ReferenceEnvironment +from partitioned_convolution import MultiSourcePartitionedConvolver +from sofa_hrir import FullHrirField, radial_ratio +from sofa_hrtf_field import SofaHrtfField +from sofa_binaural_backend import HybridPath, HybridObjectPathRenderer, _StereoDelay + + +class _CompiledField: + def __init__(self, path, receivers): + if receivers is None: + raise ValueError("compiled input requires configured receiver positions") + self.field = SofaHrtfField.load(path) + self.fft_size = 2048 + self.frequencies = np.fft.rfftfreq(self.fft_size, 1/48000) + self.measurement_radius_m = self.field.measurement_radius_m + receivers = np.asarray(receivers, dtype=np.float64) + if receivers.shape != (2, 3) or not np.isfinite(receivers).all(): + raise ValueError("invalid compiled receiver positions") + self.head_radius_m = float(np.linalg.norm(receivers[0])) + if not .04 < self.head_radius_m < .15 or not np.allclose(receivers[0], -receivers[1], atol=1e-6): + raise ValueError("unsupported compiled receiver geometry") + self.ear_directions = receivers/self.head_radius_m + self.band_geometry = SimpleNamespace(frequencies=self.field.band_center_frequencies_hz, + measurement_radius_m=self.measurement_radius_m, head_radius_m=self.head_radius_m, + ear_directions=self.ear_directions) + i = np.arange(128) + z = 1-2*(i+.5)/128 + phi = i*np.pi*(3-math.sqrt(5)) + directions = np.column_stack((np.sqrt(1-z*z)*np.cos(phi), np.sqrt(1-z*z)*np.sin(phi), z)) + evaluations = [self.field.evaluate_sofa(q) for q in directions] + self.recording_origin_samples = int(round(float(np.median([e.delay_samples.mean() for e in evaluations])))) + self.centre_band = np.sqrt(np.mean([np.abs(e.transfer_gains)**2 for e in evaluations], axis=(0, 1))) + from public_filterbank import hybrid_gain_synthesis_dictionary + self.dictionary = hybrid_gain_synthesis_dictionary(self.fft_size) + + def bands(self, state): + e = self.field.evaluate_sofa(state.direction) + ratio = radial_ratio(self.band_geometry, state.direction, state.ear_distance_m) + return e, ratio + + def spectrum_at(self, state): + e, ratio = self.bands(state) + f = self.field.band_center_frequencies_hz + centre = self.centre_band*np.exp(-2j*np.pi*f*self.recording_origin_samples/48000) + gains = state.centre_weight*e.transfer_gains*ratio+(1-state.centre_weight)*centre + values = np.empty((2, 154)) + values[:, 0::2], values[:, 1::2] = gains.real, gains.imag + return np.fft.rfft(values @ self.dictionary.T, self.fft_size) + + def info(self): + return dict(source_sha256=self.field.source_sha256, method="compiled-band-field", + measurement_radius_m=self.measurement_radius_m, + receiver_geometry_source="reference-configuration", + cache_key=self.field.cache_key, format_version=self.field.format_version, + recording_origin_samples=self.recording_origin_samples, + radial_model_validation="analytic-identity-and-series-checks; approximate-human-head") + + +class ReferenceSofaBackend: + fixed_reference = True + default_profile = "mid" + dsp_backend = "python-full-hrir" + block_samples = 128 + environment_batch_samples = 1536 + + def __init__(self, source, *, reference_config=None, compiled=False, source_count=16, + output_gain=1., method="lf_logmag_phase", shell_radius_m=None, mode=None): + if source_count <= 0 or not math.isfinite(output_gain): + raise ValueError("invalid source count or output gain") + self.source_count, self.output_gain = int(source_count), float(output_gain) + self.environment = ReferenceEnvironment(reference_config, mode=mode) + self.default_profile = self.environment.mode + self.compiled = bool(compiled) + self.field = (_CompiledField(source, self.environment.compiled_receivers) if compiled else + FullHrirField(source, method=method, shell_radius_m=shell_radius_m)) + self.hrtf_input_kind = "compiled_cache" if compiled else "sofa" + self.hrtf_input_path = str(Path(source).expanduser().resolve()) + self.cache_policy = None + if self.environment.minimum_distance_m <= self.field.head_radius_m: + raise ValueError("reference near-centre regularization lies inside the head") + self._spectra = OrderedDict() + self.level_calibration = self.environment.calibrate(self.field, self.spectrum_at) + self.bank = self.environment.create_bank(self.source_count) + self.room = self.environment.create_room() + self._last = [None]*self.source_count + self.distance_states = [None]*self.source_count + self._sends = np.zeros(self.source_count, dtype=np.float64) + self._send_targets = self._sends.copy() + self.processed_input_samples = self.output_samples = self.parameter_updates = 0 + self.finished = False + self._pending_room = [] + self._pending_direct = [] + self.origin = self.field.recording_origin_samples + if self.origin < 0: + raise ValueError("negative recording origin requires an explicit time-base adapter") + self.implementation_latency = (self.bank.latency_samples if compiled else self.field.common_latency_samples) + self.latency_samples = max(self.implementation_latency, self.bank.latency_samples)+max(self.origin, 0) + self._latency_remaining = self.latency_samples + self.room_delay = _StereoDelay(self.latency_samples-self.bank.latency_samples) + if compiled: + self.dsp_backend = "python-compiled-hrtf" + maximum = int(math.ceil(float(np.max(self.field.field.delay_bounds))/64)) + self.paths = HybridObjectPathRenderer(self.source_count, history_slots=maximum+8, transition_slots=2) + self.direct_delay = _StereoDelay(max(0, self.latency_samples-self.bank.latency_samples-self.origin)) + self.lfe_transfer = self._compiled_lfe() + self.direct_bank = self.environment.create_bank(self.source_count) + else: + self.convolver = MultiSourcePartitionedConvolver(self.source_count, block_size=self.block_samples, + max_filter_taps=self.field.fft_size) + self.direct_delay = _StereoDelay(max(0, self.latency_samples-self.implementation_latency-max(self.origin, 0))) + lowpass = signal.firwin(129, 150., fs=48000) + h = np.zeros((2, self.field.fft_size), dtype=np.float64) + start = self.latency_samples-64 + if start < 0 or start+len(lowpass) > self.field.fft_size: + raise ValueError("LFE latency exceeds filter budget") + h[:, start:start+len(lowpass)] = lowpass/math.sqrt(2) + self.lfe_filter = h + + def spectrum_at(self, state): + if self.compiled: + return self.field.spectrum_at(state) + key = (tuple(state.direction), state.ear_distance_m, state.centre_weight) + if key in self._spectra: + self._spectra.move_to_end(key) + return self._spectra[key].copy() + value = self.field.spectrum(state.direction)*radial_ratio(self.field, state.direction, state.ear_distance_m) + value = value*state.centre_weight+(1-state.centre_weight)*self.field.centre_spectrum + self._spectra[key] = value.copy() + if len(self._spectra) > 128: + self._spectra.popitem(last=False) + return value + + def _compiled_lfe(self): + from rosella_direct import special_lfe_direct + return special_lfe_direct().gains + + def set_source(self, source, position_adm, *, profile=None, gain=1., enabled=True, + special_lfe=False, fade=True): + if self.finished: + raise RuntimeError("renderer is finished") + source = int(source) + if not 0 <= source < self.source_count or not math.isfinite(gain): + raise ValueError("invalid source or gain") + if profile not in (None, self.default_profile): + raise ValueError("source profile must match the selected reference mode") + position = np.asarray(position_adm, dtype=np.float64) + if position.shape != (3,) or not np.isfinite(position).all(): + raise ValueError("source position must be finite") + gain = float(gain) if enabled else 0. + key = (tuple(position), gain, bool(special_lfe)) + if key == self._last[source]: + return + self._last[source] = key + first = self.processed_input_samples == 0 + if special_lfe: + self._send_targets[source] = 0. + if self.compiled: + f = self.field.field.band_center_frequencies_hz + transfer = self.lfe_transfer*gain*np.exp(-2j*np.pi*(self.origin % 64)*f/48000) + path = HybridPath("lfe", np.full(2, self.origin//64, dtype=np.int64), transfer) + self.paths.set_paths(source, (path,), fade_slots=0 if first else 2) + else: + self.convolver.set_filter(source, self.lfe_filter*gain, fade_blocks=0 if first else 1) + else: + state = self.environment.map_position(position) + self.distance_states[source] = state + self._send_targets[source] = state.room_send*gain + if self.compiled: + evaluation, ratio = self.field.bands(state) + slots = np.floor(evaluation.delay_samples/64).astype(np.int64) + residual = evaluation.delay_samples-slots*64 + f = self.field.field.band_center_frequencies_hz + transfer = evaluation.aligned_gains*np.exp(-2j*np.pi*residual[:, None]*f/48000)*ratio + transfer *= state.centre_weight*state.direct_gain*self.level_calibration*gain + paths = [HybridPath("direct", slots, transfer)] + if state.centre_weight < 1: + delay = max(0, self.origin) + centre = self.field.centre_band*np.exp(-2j*np.pi*(delay % 64)*f/48000) + centre = np.tile(centre, (2, 1))*(1-state.centre_weight)*state.direct_gain*self.level_calibration*gain + paths.append(HybridPath("centre", np.full(2, delay//64, dtype=np.int64), centre)) + self.paths.set_paths(source, paths, fade_slots=0 if first or not fade else 2) + else: + spectrum = self.spectrum_at(state)*state.direct_gain*self.level_calibration*gain + hrir = np.roll(np.fft.irfft(spectrum, self.field.fft_size), self.field.common_latency_samples, axis=-1) + self.convolver.set_filter(source, hrir, fade_blocks=0 if first or not fade else 1) + if first: + self._sends[source] = self._send_targets[source] + self.parameter_updates += 1 + + def process(self, source): + if self.finished: + raise RuntimeError("renderer is finished") + values = np.asarray(source, dtype=np.float64) + if values.ndim != 2 or values.shape[1] != self.source_count or len(values) % self.block_samples or not np.isfinite(values).all(): + raise ValueError("input must contain finite complete 128-sample source blocks") + outputs = [] + for start in range(0, len(values), self.block_samples): + block = values[start:start+self.block_samples] + hops = block.reshape(2, 64, self.source_count).transpose(0, 2, 1) + hybrid = self.bank.hybrid_analysis.process_chunk(self.bank.qmf_analysis.process_chunk(hops)) + amount = np.array([.5, 1.])[:, None] + sends = self._sends+(self._send_targets-self._sends)*amount + room_input = np.einsum("tsf,ts->tf", hybrid, sends) + if self.compiled: + self._pending_direct.append(self.paths.process(hybrid)) + else: + self._pending_direct.append(self.convolver.process(block)) + self._pending_room.append(room_input) + self._sends[:] = self._send_targets + self.processed_input_samples += self.block_samples + if len(self._pending_room)*self.block_samples < self.environment_batch_samples: + continue + room = self.room.process_chunk(np.concatenate(self._pending_room)) + time = self.bank.qmf_synthesis.process_chunk(self.bank.hybrid_synthesis.process_chunk(room)) + wet = time.transpose(0, 2, 1).reshape(-1, 2) + direct = np.concatenate(self._pending_direct) + if self.compiled: + direct = self.direct_bank.qmf_synthesis.process_chunk( + self.direct_bank.hybrid_synthesis.process_chunk(direct)) + direct = direct.transpose(0, 2, 1).reshape(-1, 2) + output = self.direct_delay.process(direct)+self.room_delay.process(wet) + self._pending_direct.clear() + self._pending_room.clear() + skip = min(len(output), self._latency_remaining) + self._latency_remaining -= skip + outputs.append(output[skip:]*self.output_gain) + output = np.concatenate(outputs) if outputs else np.empty((0, 2)) + self.output_samples += len(output) + return output + + def finish_output_capacity(self, tail_seconds): + tail = float(tail_seconds) + if not math.isfinite(tail) or tail < 0: + raise ValueError("tail must be finite and nonnegative") + return int(math.ceil((tail*48000+self.latency_samples+self.field.fft_size)/128)*128) + + def finish(self, *, tail_seconds=5.): + if self.finished: + return np.empty((0, 2)) + count = self.finish_output_capacity(tail_seconds) + pending = len(self._pending_room)*self.block_samples + padded = int(math.ceil((count+pending)/self.environment_batch_samples)*self.environment_batch_samples)-pending + expected = max(0, self.processed_input_samples+count-self.latency_samples-self.output_samples) + output = self.process(np.zeros((padded, self.source_count))) + self.output_samples -= max(0, len(output)-expected) + output = output[:expected] + self.finished = True + return output + + def info(self): + distances = [s.distance_m for s in self.distance_states if s is not None] + return dict(hrtf_input_kind=self.hrtf_input_kind, hrtf_input_path=self.hrtf_input_path, + cache_policy=None, field=self.field.info(), reference=self.environment.info(), + level_calibration=self.level_calibration, + latency_compensated_samples=self.latency_samples, + processing_batch_samples=self.environment_batch_samples, + recording_origin_compensated_samples=self.origin, + current_reference_distances_m=distances, + parameter_updates=self.parameter_updates, direct_representation=("77-band" if self.compiled else "full-HRIR"), + radial_model="approximate-rigid-sphere-ratio", precision="float64/complex128")