Update the vendored kernel to the bounded objects16 render-ahead

.gitignore

@@ -27,8 +27,9 @@
 *.personalized_headphone
 rosella_kernels.npz
 
-# The component writes its own log next to the DLL
+# The component writes its own log next to the DLL, and rolls it to a .1 sibling
 joc_decoder.log
+joc_decoder.log.1
 
 # Measurement record: local working notes, kept on disk and never published.
 VERIFICATION.md

README.en.md

@@ -94,7 +94,11 @@ and test-bed scripts. Settings also read `JOC_*` environment overrides for one r
 only); the list and what each one does is in `src\settings.cpp`.
 
 To diagnose a problem, read `joc_decoder.log` beside the DLL: the component writes its own
-version, the core version and the log path there at start-up.
+version, the core version and the log path there at start-up. The log is capped by **bytes**: a
+file that reaches 16 MiB is rolled to `joc_decoder.log.1` beside it, replacing the previous roll
+rather than accumulating, so the two together never exceed 32 MiB and the newest lines are always
+in `joc_decoder.log`. The version banner is written again after a roll. A Media Library scan logs
+every container in the library, which is where the volume comes from.
 
 ## Known limitations
 

README.md

@@ -82,7 +82,9 @@ pwsh -File tools/package.ps1            # 两个架构，打包到 dist\*.fb2k-c
 另有 `JOC_*` 环境变量覆盖（仅用于开发运行），清单与含义在 `src\settings.cpp`。
 
 排查问题看 DLL 旁边的 `joc_decoder.log`；组件启动时会把自己的版本、核心版本、日志路径写在
-里面。
+里面。日志按**字节**封顶：单个文件写满 16 MiB 就滚到同目录的 `joc_decoder.log.1`（覆盖上一次
+滚动，不累积），所以两份加起来不超过 32 MiB，最新的内容始终在 `joc_decoder.log`。滚动之后版本
+信息会重写一遍。媒体库扫描会给库里每个容器写日志，体量主要来自这里。
 
 ## 已知限制
 

THIRD_PARTY_NOTICES.md

@@ -1,31 +1,9 @@
 # Third-party notices / 第三方通知
 
-本仓库包含 JustOneCacophony 原生库的逐字节副本（下称"复用文件"），以及遵循公开标准实现的
+本文件记录 64-QMF / 77-hybrid 滤波器组表（`kernel/src/hrtf/public_filterbank.h`、
-滤波器组表。本文件记录这些来源、公开标准依据与权利边界。
+`kernel/src/joc_core/qmf_tables.h`）与 JOC Huffman 表
-
+（`kernel/src/joc_bitstream/joc_huffman_tables.h`）的公开标准来源，以及 HRTF 数据与
-## 复用文件
+专利的边界说明。
-
-以下文件是本项目（JustOneCacophony，MIT）原生 C++ 库的逐字节副本，不修改、不追加注释：
-
-| 本仓库路径 | 上游路径 | SHA-256（前 16 位） |
-|---|---|---|
-| `src/joc_core/eac3joc_core.cpp` | `native/src/eac3joc_core.cpp` | `1978eea64a2616fa` |
-| `src/joc_core/qmf_tables.h` | `native/src/qmf_tables.h` | `c205ea187e956e87` |
-| `src/speaker/speaker_renderer.cpp` | `native/src/speaker_renderer.cpp` | `96f36f40daf86eec` |
-| `src/speaker/speaker_layouts.h` | `native/src/speaker_layouts.h` | `51e24c11be09787f` |
-| `src/binaural/binaural_renderer.cpp` | `native/src/binaural_renderer.cpp` | `d579803f0f5a6699` |
-| `src/joc_bitstream/joc_huffman_tables.h` | `native/src/joc_huffman_tables.h` | `698498b3778d88db` |
-| `include/eac3joc_core.h` | `native/include/eac3joc_core.h` | `7392f48dfd840656` |
-
-上游修订：`6bc2c2885666bb151bb66af93472199af9a99b81`。
-
-## 派生文件
-
-`src/binaural/sofa_binaural_renderer.cpp` 是上游 `native/src/sofa_binaural_renderer.cpp` 的派生
-实现：同一 `ejoc_sofa_binaural_*` C ABI，增加表提升、结果记忆化、输入校验与 SIMD 派发。它
-**不属于**逐字节副本，也不受"不得修改"约束，但来源固定为上述上游修订，对应上游源文件的
-SHA-256 为 `81b485e4c71907672ab308ddc382284acf29161cee93d7906785a4cce58941c7`，输出与原
-实现逐位相同（见该文件头部的验证记录）。
 
 ## 公开标准来源
 

foo_input_joc.vcxproj

@@ -114,6 +114,7 @@
     <ClInclude Include="src\prefs.h" />
     <ClInclude Include="src\resource.h" />
     <ClInclude Include="src\settings.h" />
+    <ClInclude Include="src\win_path.h" />
   </ItemGroup>
 
   <ItemGroup>

kernel/include/joc_stream.h

@@ -39,12 +39,12 @@ typedef struct joc_stream joc_stream;
 
 typedef enum joc_stream_input {
     JOC_STREAM_IN_EAC3 = 0,          /* bare E-AC-3 syncframes (the metadata stream) */
-    JOC_STREAM_IN_PCM_OBJECTS16 = 1, /* 16-channel objects16, decoded by the host   */
+    JOC_STREAM_IN_PCM_OBJECTS16 = 1, /* objects16 frames, planar [16][1536] each    */
     JOC_STREAM_IN_CORE_PCM = 3       /* the 5.1 core PCM of the pushed E-AC-3 frames */
 } joc_stream_input;
 
 typedef enum joc_stream_output {
-    JOC_STREAM_OUT_PCM_OBJECTS16 = 0, /* planar [16][samples] float32        */
+    JOC_STREAM_OUT_PCM_OBJECTS16 = 0, /* objects16 frames, planar [16][1536] each    */
     JOC_STREAM_OUT_SPEAKER = 1,       /* interleaved [samples][channels] f32         */
     JOC_STREAM_OUT_BINAURAL = 2       /* interleaved [samples][2] f32                */
 } joc_stream_output;

kernel/joc_kernel.vcxproj

@@ -113,18 +113,17 @@
   </ItemDefinitionGroup>
 
   <!--
-    SIMD units, mirroring CMakeLists.txt (the JOC_SIMD_* block) exactly.
+    SIMD units, mirroring CMakeLists.txt (the JOC_SIMD_* block).
 
     The ISA in the file name, one translation unit per ISA, each compiled with
     its own /arch because MSVC has no function-level ISA attribute; dispatch.cpp
-    (a baseline unit) picks one at run time from CPUID/XGETBV.  The kernel
+    (a baseline unit) picks one at run time from CPUID/XGETBV.
-    enables SIMD only for x86_64 and aarch64, so:
 
-      * Win32 (x86) gets NO SIMD unit and NO SIMD define: the scalar reference
-        and the probe/dispatch baseline are all it builds, exactly as
-        CMAKE_SIZEOF_VOID_P EQUAL 8 gates them out upstream;
       * x64 gets the AVX2 and AVX-512 units, JOC_SIMD_HAVE_SSE2 / _AVX2 /
         _AVX512, and a per-file /arch for those two files only;
+      * Win32 (x86) gets the AVX2 unit and JOC_SIMD_HAVE_AVX2: AVX2 is not an
+        x86-64-only ISA.  AVX-512 stays x64-only, because 32-bit mode addresses
+        ZMM0-7 only.
       * src\simd\kernels_intrin_neon.cpp is excluded everywhere here: the CMake
         build lists it only for aarch64 (CMakeLists.txt lines 177-180), which no
         configuration of this project targets.
@@ -134,6 +133,11 @@
       <PreprocessorDefinitions>JOC_SIMD_HAVE_SSE2=1;JOC_SIMD_HAVE_AVX2=1;JOC_SIMD_HAVE_AVX512=1;%(PreprocessorDefinitions)</PreprocessorDefinitions>
     </ClCompile>
   </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Platform)'=='Win32'">
+    <ClCompile>
+      <PreprocessorDefinitions>JOC_SIMD_HAVE_AVX2=1;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+    </ClCompile>
+  </ItemDefinitionGroup>
 
   <ItemGroup>
     <!-- Verbatim copies of the upstream native library (JOC_REUSED_SOURCES). -->
@@ -203,6 +207,12 @@
     </ClCompile>
   </ItemGroup>
 
+  <ItemGroup Condition="'$(Platform)'=='Win32'">
+    <ClCompile Include="src\simd\kernels_intrin_avx2.cpp">
+      <AdditionalOptions>/arch:AVX2 %(AdditionalOptions)</AdditionalOptions>
+    </ClCompile>
+  </ItemGroup>
+
   <ItemGroup>
     <ClInclude Include="include\eac3joc_core.h" />
     <ClInclude Include="include\joc_core.h" />

kernel/src/simd/cpu_probe.cpp

@@ -7,10 +7,10 @@
 
 #include "simd/cpu_probe.h"
 
-#if defined(_M_X64)
+#if defined(_M_X64) || defined(_M_IX86)
 #include <immintrin.h>
 #include <intrin.h>
-#elif defined(__x86_64__)
+#elif defined(__x86_64__) || defined(__i386__)
 #include <cpuid.h>
 #endif
 
@@ -21,10 +21,11 @@
 namespace joc::simd {
 namespace {
 
-// ------------------------------------------------------------------- x86-64 --
+// ---------------------------------------------------------------------- x86 --
-#if defined(_M_X64) || defined(__x86_64__)
+// AVX2 is not an x86-64-only ISA, so both pointer sizes are probed.
+#if defined(_M_X64) || defined(_M_IX86) || defined(__x86_64__) || defined(__i386__)
 
-#if defined(_M_X64)
+#if defined(_M_X64) || defined(_M_IX86)
 
 // CPUID tells us what the silicon can do; XCR0 tells us whether the OS saves the
 // state the wider registers need.  Both have to agree, or the first AVX
@@ -53,7 +54,7 @@ CpuFeatures probe_x86() noexcept {
     return features;
 }
 
-#else  // GCC/Clang on x86-64
+#else  // GCC/Clang on x86
 
 // The compiler runtime performs the same CPUID + XGETBV probe (libgcc's cpuinfo
 // checks XCR0 before it reports AVX), which keeps this file free of inline

kernel/src/stream/stream.cpp

@@ -38,7 +38,10 @@ void Stream::reset_state() {
     reader_ = eac3::FrameReader();
     metadata_.clear();
     bed_pending_.clear();
+    bed_read_offset_ = 0;
     objects16_.clear();
+    objects_pending_.clear();
+    objects_read_offset_ = 0;
     output_.clear();
     read_offset_ = 0;
     info_ = Info();
@@ -118,7 +121,6 @@ Status Stream::create(const Config& config) {
             }
             render_options.object_delay_samples = config_.object_delay_samples;
             render_options.tail_seconds = config_.tail_seconds;
-            render_options.output_gain = 1.0;
             if (status.ok()) {
                 status = rosella_.open(model, render_options);
             }
@@ -222,7 +224,7 @@ Status Stream::push_eac3(const std::uint8_t* data, std::size_t size, std::size_t
         }
         metadata_.push_back(entry);
     }
-    return process_ready_frames();
+    return process_ready_frames(false);
 }
 
 Status Stream::push_bed(const float* interleaved6, std::size_t samples, std::size_t* consumed) {
@@ -236,7 +238,7 @@ Status Stream::push_bed(const float* interleaved6, std::size_t samples, std::siz
         bed_pending_.insert(bed_pending_.end(), interleaved6,
                             interleaved6 + samples * kBedChannels);
     }
-    return process_ready_frames();
+    return process_ready_frames(false);
 }
 
 Status Stream::push_objects16(const float* planar16, std::size_t samples, std::size_t* consumed) {
@@ -246,49 +248,82 @@ Status Stream::push_objects16(const float* planar16, std::size_t samples, std::s
     if (planar16 == nullptr || samples == 0u) {
         return Status::success();
     }
-    // Rendered immediately: the host has already done the JOC rebuild.
+    // Queued as whole frames, each planar [16][kFrameSamples]: the shape the
+    // objects16 output writes, so a host can feed a batch back unchanged.
+    constexpr std::size_t kFrameValues =
+        static_cast<std::size_t>(JOC_OUTPUT_CHANNELS) * kFrameSamples;
     for (std::size_t offset = 0; offset < samples; offset += kFrameSamples) {
         const std::size_t count = std::min(kFrameSamples, samples - offset);
-        std::vector<float> frame(static_cast<std::size_t>(JOC_OUTPUT_CHANNELS) * kFrameSamples,
+        const std::size_t base = objects_pending_.size();
-                                 0.0f);
+        const float* frame = planar16 + (offset / kFrameSamples) * kFrameValues;
+        objects_pending_.resize(base + kFrameValues, 0.0f);
         for (std::size_t channel = 0; channel < JOC_OUTPUT_CHANNELS; ++channel) {
-            std::memcpy(frame.data() + channel * kFrameSamples,
+            std::memcpy(objects_pending_.data() + base + channel * kFrameSamples,
-                        planar16 + channel * samples + offset, count * sizeof(float));
+                        frame + channel * kFrameSamples, count * sizeof(float));
         }
-        ++info_.frames_in;
+    }
-        info_.samples_in += count;
+    return process_objects16_frames(false);
-        const Status rendered = render_objects16(frame);
+}
+
+Status Stream::process_objects16_frames(bool drain_all) {
+    constexpr std::size_t kFrameValues =
+        static_cast<std::size_t>(JOC_OUTPUT_CHANNELS) * kFrameSamples;
+    while (objects_pending_.size() - objects_read_offset_ >= kFrameValues) {
+        // Leave the rest queued, in order, for a later push or for flush().
+        if (!drain_all && buffered_samples() >= kMaxRenderAheadSamples) {
+            break;
+        }
+        const auto first = objects_pending_.begin() +
+                           static_cast<std::ptrdiff_t>(objects_read_offset_);
+        objects_frame_.assign(first, first + static_cast<std::ptrdiff_t>(kFrameValues));
+        objects_read_offset_ += kFrameValues;
+        const Status rendered = render_objects16(objects_frame_);
         if (!rendered.ok()) {
             return rendered;
         }
-        if (count != kFrameSamples) {
+        ++info_.frames_in;
-            break;  // a partial frame is dropped; the host should push whole frames
+        info_.samples_in += kFrameSamples;
     }
+    if (objects_read_offset_ == objects_pending_.size()) {
+        objects_pending_.clear();
+        objects_read_offset_ = 0;
+    } else if (objects_read_offset_ >= (1u << 20)) {
+        // Erasing from the front moves the remainder, so it is only worth doing
+        // once the consumed prefix is large enough to pay for the move.
+        objects_pending_.erase(
+            objects_pending_.begin(),
+            objects_pending_.begin() + static_cast<std::ptrdiff_t>(objects_read_offset_));
+        objects_read_offset_ = 0;
     }
     return Status::success();
 }
 
-Status Stream::process_ready_frames() {
+Status Stream::process_ready_frames(bool drain_all) {
-    while (bed_pending_.size() / kBedChannels >= kFrameSamples && !metadata_.empty()) {
+    while (bed_pending_.size() - bed_read_offset_ >= kFrameSamples * kBedChannels &&
+           !metadata_.empty()) {
+        // Leave the rest queued, in order, for a later push or for flush().
+        if (!drain_all && buffered_samples() >= kMaxRenderAheadSamples) {
+            break;
+        }
         const FrameMetadata entry = metadata_.front();
         metadata_.pop_front();
 
-        std::vector<float> bed5(static_cast<std::size_t>(JOC_CORE_CHANNELS) * kFrameSamples, 0.0f);
+        const float* bed = bed_pending_.data() + bed_read_offset_;
-        std::vector<float> lfe(kFrameSamples, 0.0f);
+        bed5_.resize(static_cast<std::size_t>(JOC_CORE_CHANNELS) * kFrameSamples);
+        lfe_.resize(kFrameSamples);
         for (std::size_t sample = 0; sample < kFrameSamples; ++sample) {
             for (std::size_t channel = 0; channel < JOC_CORE_CHANNELS; ++channel) {
-                bed5[channel * kFrameSamples + sample] =
+                bed5_[channel * kFrameSamples + sample] =
-                    bed_pending_[sample * kBedChannels + kCoreChannels[channel]];
+                    bed[sample * kBedChannels + kCoreChannels[channel]];
             }
-            lfe[sample] = bed_pending_[sample * kBedChannels + kLfeChannel];
+            lfe_[sample] = bed[sample * kBedChannels + kLfeChannel];
         }
-        bed_pending_.erase(bed_pending_.begin(),
+        bed_read_offset_ += kFrameSamples * kBedChannels;
-                           bed_pending_.begin() + static_cast<std::ptrdiff_t>(kFrameSamples *
+        compact_bed_pending();
-                                                                             kBedChannels));
 
         std::string error;
-        const Status rebuilt = joc::rebuild_objects16(rebuilder_, entry.params, bed5.data(),
+        const Status rebuilt = joc::rebuild_objects16(rebuilder_, entry.params, bed5_.data(),
-                                                      lfe.data(), gain_, &objects16_, &error);
+                                                      lfe_.data(), gain_, &objects16_, &error);
         if (!rebuilt.ok()) {
             return Status::fail(rebuilt.code(), stage::kDsp, error);
         }
@@ -303,6 +338,22 @@ Status Stream::process_ready_frames() {
     return Status::success();
 }
 
+void Stream::compact_bed_pending() {
+    if (bed_read_offset_ == 0) {
+        return;
+    }
+    if (bed_read_offset_ == bed_pending_.size()) {
+        bed_pending_.clear();
+        bed_read_offset_ = 0;
+    } else if (bed_read_offset_ >= (1u << 20)) {
+        // Erasing from the front moves the remainder, so it is only worth doing
+        // once the consumed prefix is large enough to pay for the move.
+        bed_pending_.erase(bed_pending_.begin(),
+                           bed_pending_.begin() + static_cast<std::ptrdiff_t>(bed_read_offset_));
+        bed_read_offset_ = 0;
+    }
+}
+
 Status Stream::render_objects16(const std::vector<float>& objects16) {
     if (config_.output == JOC_STREAM_OUT_PCM_OBJECTS16) {
         output_.insert(output_.end(), objects16.begin(), objects16.end());
@@ -320,6 +371,7 @@ Status Stream::render_objects16(const std::vector<float>& objects16) {
         if (!stepped.ok()) {
             return Status::fail(stepped.code(), stage::kRender, error);
         }
+        output_.reserve(output_.size() + speaker_.output.size());
         for (const double value : speaker_.output) {
             output_.push_back(static_cast<float>(value));
         }
@@ -340,13 +392,13 @@ Status Stream::render_objects16(const std::vector<float>& objects16) {
     if (!submitted.ok()) {
         return submitted;
     }
-    std::vector<double> produced;
+    binaural_.take_output(&produced_);
-    binaural_.take_output(&produced);
+    output_.reserve(output_.size() + produced_.size());
-    for (const double value : produced) {
+    for (const double value : produced_) {
         output_.push_back(static_cast<float>(value));
     }
     info_.frames_out++;
-    info_.samples_out += produced.size() / 2u;
+    info_.samples_out += produced_.size() / 2u;
     return Status::success();
 }
 
@@ -367,16 +419,15 @@ Status Stream::render_rosella_objects16(const std::vector<float>& objects16) {
     if (!submitted.ok()) {
         return submitted;
     }
-    std::vector<double> produced;
+    rosella_.take_output(&produced_);
-    rosella_.take_output(&produced);
+    if (!produced_.empty()) {
-    if (!produced.empty()) {
+        rosella_pending_.insert(rosella_pending_.end(), produced_.begin(), produced_.end());
-        rosella_pending_.insert(rosella_pending_.end(), produced.begin(), produced.end());
     }
     release_rosella_output(kFrameSamples);
     info_.frames_out++;
     // Counted as the runtime produces it, which is also how the SOFA path counts:
     // the totals are identical, only the frame they appear on differs.
-    info_.samples_out += produced.size() / 2u;
+    info_.samples_out += produced_.size() / 2u;
     return Status::success();
 }
 
@@ -389,6 +440,7 @@ void Stream::release_rosella_output(std::size_t limit) {
         return;
     }
     const std::size_t values = count * 2u;
+    output_.reserve(output_.size() + values);
     for (std::size_t index = 0; index < values; ++index) {
         output_.push_back(static_cast<float>(rosella_pending_[rosella_read_offset_ + index]));
     }
@@ -430,6 +482,16 @@ Status Stream::pull(float* destination, std::size_t capacity_samples, std::size_
 }
 
 Status Stream::flush() {
+    // Input has ended, so drain what the cap held back: nothing else will
+    // trigger rendering.
+    const Status remaining = process_ready_frames(true);
+    if (!remaining.ok()) {
+        return remaining;
+    }
+    const Status objects = process_objects16_frames(true);
+    if (!objects.ok()) {
+        return objects;
+    }
     if (binaural_ready_) {
         std::vector<double> tail;
         const Status drained =
@@ -437,6 +499,7 @@ Status Stream::flush() {
         if (!drained.ok()) {
             return drained;
         }
+        output_.reserve(output_.size() + tail.size());
         for (const double value : tail) {
             output_.push_back(static_cast<float>(value));
         }
@@ -453,6 +516,7 @@ Status Stream::flush() {
         // tail only sounds after it.  The program samples were already counted by
         // render_rosella_objects16, so only the tail is added here.
         release_rosella_output(rosella_pending_samples());
+        output_.reserve(output_.size() + tail.size());
         for (const double value : tail) {
             output_.push_back(static_cast<float>(value));
         }

kernel/src/stream/stream.h

@@ -85,8 +85,13 @@ public:
                    : 0u;
     }
 
+    // Cap on rendered samples that have not been pulled.  A push renders what it
+    // makes ready, so a caller that feeds faster than it pulls renders ahead.
+    static constexpr std::size_t kMaxRenderAheadSamples = 16384;
+
 private:
-    Status process_ready_frames();
+    // `drain_all` ignores kMaxRenderAheadSamples and renders every ready frame.
+    Status process_ready_frames(bool drain_all);
     Status render_objects16(const std::vector<float>& objects16);
     Status render_rosella_objects16(const std::vector<float>& objects16);
     // Moves at most `limit` rendered stereo samples per channel out of the FIFO
@@ -96,6 +101,10 @@ private:
         return rosella_ready_ ? (rosella_pending_.size() - rosella_read_offset_) / 2u : 0u;
     }
     void reset_state();
+    // Drops the bed samples that have already been rendered, keeping the rest.
+    void compact_bed_pending();
+    // Renders the queued objects16 frames, bounded by kMaxRenderAheadSamples.
+    Status process_objects16_frames(bool drain_all);
 
     Config config_;
     Info info_;
@@ -103,7 +112,12 @@ private:
     std::deque<FrameMetadata> metadata_;
     FrameMetadata pending_metadata_;
     std::vector<float> bed_pending_;
-    std::vector<std::uint8_t> frame_copy_;
+    std::size_t bed_read_offset_ = 0;
+    std::vector<float> bed5_;
+    std::vector<float> lfe_;
+    std::vector<float> objects_pending_;
+    std::size_t objects_read_offset_ = 0;
+    std::vector<float> objects_frame_;
     std::vector<float> objects16_;
     std::vector<float> output_;
     std::size_t read_offset_ = 0;
@@ -115,6 +129,7 @@ private:
     hrtf::RosellaRuntime rosella_;
     std::vector<double> rosella_pending_;
     std::size_t rosella_read_offset_ = 0;
+    std::vector<double> produced_;
     bool speaker_enabled_ = false;
     bool binaural_enabled_ = false;
     bool binaural_ready_ = false;

kernel/src/task/task.cpp

@@ -1209,7 +1209,6 @@ Status run(const joc_task_config& raw, const joc_event_sink* sink, joc_task_resu
             }
             render_options.object_delay_samples = cfg.object_delay_samples;
             render_options.tail_seconds = cfg.tail_seconds;
-            render_options.output_gain = std::pow(10.0, cfg.gain_db / 20.0);
             if (status.ok()) {
                 status = rosella.open(model, render_options);
             }

src/container_scan.cpp

@@ -8,18 +8,19 @@
 #include <vector>
 
 #include "log.h"
+#include "win_path.h"
 
 namespace joc_container {
 namespace {
 
-std::wstring utf8_to_wide(const std::string& text) {
+// A library scan probes every container in the library, and most of them are
-    if (text.empty()) return {};
+// declined; they also mostly share a directory, so the path would spend the log's
-    const int needed = MultiByteToWideChar(CP_UTF8, 0, text.c_str(),
+// byte budget repeating the same prefix once per file.  The file name is what
-                                           static_cast<int>(text.size()), nullptr, 0);
+// identifies the line.  Files this component claims are logged with their full
-    std::wstring out(static_cast<std::size_t>(needed), L'\0');
+// path by the caller.
-    MultiByteToWideChar(CP_UTF8, 0, text.c_str(), static_cast<int>(text.size()), out.data(),
+std::string file_name_of(const std::string& path) {
-                        needed);
+    const std::string::size_type slash = path.find_last_of("\\/");
-    return out;
+    return (slash == std::string::npos) ? path : path.substr(slash + 1);
 }
 
 // A bounded, read-only view of the file.  Every accessor returns false instead of
@@ -27,7 +28,7 @@ std::wstring utf8_to_wide(const std::string& text) {
 class Window {
 public:
     bool open(const std::string& path) {
-        const std::wstring wide = utf8_to_wide(path);
+        const std::wstring wide = joc_path::to_wide_extended(path);
         handle_ = CreateFileW(wide.c_str(), GENERIC_READ, FILE_SHARE_READ | FILE_SHARE_WRITE,
                               nullptr, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, nullptr);
         if (handle_ == INVALID_HANDLE_VALUE) {
@@ -200,15 +201,24 @@ Result scan_mp4(const Window& file, std::size_t max_bytes) {
     Result result;
     result.kind = Kind::kMp4;
 
-    // moov is usually at the start for streamed files and at the end otherwise.
+    // moov is usually at the start for streamed files and at the end otherwise.  The
+    // walk reads box headers where they lie and steps over mdat in one go, so the
+    // whole file is walked from the top: a fixed prefix window cannot reach a moov
+    // that is larger than the window, which is what an MP4 with its cover art stored
+    // as a video track produces (a 4.9 MB moov against a 4 MiB window).  The tail
+    // range stays as a fallback for a file whose leading boxes do not parse.
     struct Range {
         std::uint64_t begin;
         std::uint64_t end;
     };
-    std::vector<Range> ranges{{0, (std::min<std::uint64_t>)(file.size(), max_bytes)}};
+    std::vector<Range> ranges{{0, file.size()}};
     if (file.size() > max_bytes) {
         ranges.push_back({file.size() - max_bytes, file.size()});
     }
+    // An oversized moov is a file this probe cannot describe, and walking it would be
+    // unbounded work; it is reported instead of entered.
+    constexpr std::uint64_t kMaxMoovBytes = 64ull * 1024ull * 1024ull;
+    bool oversized_moov = false;
 
     unsigned audio_seen = 0;
     bool found_any_audio = false;
@@ -218,6 +228,10 @@ Result scan_mp4(const Window& file, std::size_t max_bytes) {
                    [&](const std::string& type, std::uint64_t payload, std::uint64_t box_end,
                        std::size_t) {
                        if (type != "moov") return true;
+                       if (box_end - payload > kMaxMoovBytes) {
+                           oversized_moov = true;
+                           return true;
+                       }
                        walk_boxes(file, payload, box_end, 1,
                                   [&](const std::string& inner, std::uint64_t inner_payload,
                                       std::uint64_t inner_end, std::size_t) {
@@ -273,6 +287,8 @@ Result scan_mp4(const Window& file, std::size_t max_bytes) {
         result.detail = "mp4: first audio track is " +
                         (first_audio_format.empty() ? std::string("unknown")
                                                     : first_audio_format);
+    } else if (oversized_moov) {
+        result.detail = "mp4: moov is larger than the scan limit";
     } else {
         result.detail = "mp4: no audio track found in the scanned window";
     }
@@ -519,7 +535,7 @@ Result scan(const std::string& path, std::size_t max_bytes) {
     } else {
         result = scan_mp4(file, max_bytes);
     }
-    joc_log::line("container: %s -> %s (eac3=%d audio#%u codec=%s)", path.c_str(),
+    joc_log::line("container: %s -> %s (eac3=%d audio#%u codec=%s)", file_name_of(path).c_str(),
                   result.detail.c_str(), result.eac3 ? 1 : 0, result.audio_index,
                   result.codec.empty() ? "-" : result.codec.c_str());
     return result;

src/input_joc.cpp

@@ -17,6 +17,8 @@
 #include <SDK/input_impl.h>
 #include <SDK/tag_processor.h>
 
+#include <atomic>
+#include <cstdarg>
 #include <cstring>
 #include <string>
 
@@ -66,6 +68,31 @@ std::string file_name_of(const std::string& path) {
     return slash == std::string::npos ? path : path.substr(slash + 1);
 }
 
+// A library scan opens every container in the library through open(), and all but
+// a few of them are declined because their audio is not E-AC-3 JOC.  The per-file
+// lines are what makes a scan diagnosable -- "the file that should have been
+// claimed was not, and here is why" -- but a large library is hundreds of
+// thousands of them, so a running summary is logged as well.  It is short enough
+// to survive the log rolling over, which is what a scan of that size makes it do.
+constexpr unsigned kDeclineSummaryEvery = 1000;
+std::atomic<unsigned> g_declined{0};
+std::atomic<unsigned> g_claimed{0};
+
+// One place for "this file is not ours".  The message stays the caller's, so a
+// decline that failed rather than decided still reads as one.
+void decline(const char* fmt, ...) {
+    va_list args;
+    va_start(args, fmt);
+    joc_log::line_v(fmt, args);
+    va_end(args);
+
+    const unsigned count = g_declined.fetch_add(1, std::memory_order_relaxed) + 1;
+    if ((count % kDeclineSummaryEvery) == 0u) {
+        joc_log::line("scan: %u file(s) declined, %u claimed so far", count,
+                      g_claimed.load(std::memory_order_relaxed));
+    }
+}
+
 // ---------------------------------------------------------------------------
 // Tags of a file this component has taken over.
 //
@@ -200,8 +227,7 @@ public:
         try {
             m_native_path = filesystem::g_get_native_path(m_path.c_str());
         } catch (const pfc::exception& error) {
-            joc_log::line("decoder: not a native file path (%s): %s", m_path.c_str(),
+            decline("decoder: not a native file path (%s): %s", m_path.c_str(), error.what());
-                          error.what());
             throw exception_io_unsupported_format();
         }
 
@@ -249,10 +275,11 @@ public:
 
         if (scan.joc != joc_eac3::JocState::kYes) {
             // Hand the file to the next entry in the priority table.
-            joc_log::line("decoder: yielding to the built-in decoder (%s)", scan.detail);
+            decline("decoder: yielding to the built-in decoder (%s)", scan.detail);
             throw exception_io_unsupported_format();
         }
         joc_log::line("decoder: claiming this file as E-AC-3 JOC (bare stream)");
+        g_claimed.fetch_add(1, std::memory_order_relaxed);
         m_input_kind = joc_decode::InputKind::kBare;
     }
 
@@ -263,7 +290,7 @@ public:
     void open_container(const char* extension) {
         m_container = joc_container::scan(m_native_path.get_ptr());
         if (!m_container.eac3) {
-            joc_log::line("decoder: yielding to the built-in decoder (%s)",
+            decline("decoder: yielding to the built-in decoder (%s)",
                     m_container.detail.c_str());
             throw exception_io_unsupported_format();
         }
@@ -273,12 +300,11 @@ public:
         std::string detail;
         if (!joc_decode::probe_container_joc(settings.ffmpeg_path, m_native_path.get_ptr(),
                                             m_container.audio_index, &joc, &detail)) {
-            joc_log::line("decoder: cannot examine the E-AC-3 track (%s); yielding",
+            decline("decoder: cannot examine the E-AC-3 track (%s); yielding", detail.c_str());
-                          detail.c_str());
             throw exception_io_unsupported_format();
         }
         if (!joc) {
-            joc_log::line("decoder: yielding to the built-in decoder (E-AC-3 track %u carries "
+            decline("decoder: yielding to the built-in decoder (E-AC-3 track %u carries "
                     "no JOC: %s)",
                     m_container.audio_index, detail.c_str());
             throw exception_io_unsupported_format();
@@ -287,6 +313,7 @@ public:
         joc_log::line("decoder: claiming this file as E-AC-3 JOC (%s, audio track %u, .%s)",
                       joc_container::kind_name(m_container.kind), m_container.audio_index,
                       extension);
+        g_claimed.fetch_add(1, std::memory_order_relaxed);
         m_input_kind = joc_decode::InputKind::kContainer;
         m_audio_index = m_container.audio_index;
     }

src/joc_decode.cpp

@@ -14,11 +14,17 @@
 
 #include "eac3_scan.h"
 #include "log.h"
+#include "win_path.h"
 
 namespace joc_decode {
 namespace {
 
 constexpr std::size_t kEac3Chunk = 96u * 1024u;      // bytes read per push
+// The core renders during the push, so a read queues about as much as it is
+// about to consume rather than everything it just read.
+constexpr std::uint64_t kEac3FramesPerRead = 3u;
+// Rendered audio the caller has not taken yet.
+constexpr std::size_t kMaxRenderedAheadSamples = 4096u;
 constexpr std::size_t kBedFramesChunk = 8192u;       // staging capacity, in frames
 constexpr std::size_t kBedChannels = 6;              // ffmpeg -ac 6
 // A read on an anonymous pipe only completes once the whole request is available,
@@ -30,6 +36,32 @@ constexpr std::size_t kBedReadBytes = 48u * 1024u;
 constexpr std::size_t kBedPipeBytes = 1u << 20;
 constexpr std::size_t kFrameSamples = JOC_FRAME_SAMPLES;
 
+// The decoded bed has to stay on the E-AC-3 frame grid that the JOC matrix and the
+// OAMD are indexed by.  An mp4/mov edit list trims the decoded audio instead, which
+// takes the bed off that grid by however much the list removes -- a Dolby Atmos
+// download loses 2432 samples (1.58 frames) -- and every frame's matrix would then be
+// applied to audio tens of milliseconds away from it, so a subset of the objects comes
+// out attenuated.  The demuxer's own option drops the trim; the option exists only on
+// the mov/mp4 demuxer, so it is passed for that family alone.
+bool has_mov_timeline(const std::string& path) {
+    static const char* const kExtensions[] = {".mp4", ".m4a", ".m4b", ".m4v",
+                                              ".mov", ".3gp", ".3g2", ".mj2"};
+    std::string lowered = path;
+    for (char& character : lowered) {
+        if (character >= 'A' && character <= 'Z') {
+            character = static_cast<char>(character - 'A' + 'a');
+        }
+    }
+    for (const char* extension : kExtensions) {
+        const std::size_t length = std::strlen(extension);
+        if (lowered.size() >= length &&
+            lowered.compare(lowered.size() - length, length, extension) == 0) {
+            return true;
+        }
+    }
+    return false;
+}
+
 // ---------------------------------------------------------------------------
 // Kernel entry points.
 //
@@ -52,18 +84,8 @@ struct CoreApi {
     const char*(JOC_CALL* error_name)(joc_error) = joc_error_name;
 };
 
-std::wstring utf8_to_wide(const std::string& text) {
-    if (text.empty()) return {};
-    const int needed = MultiByteToWideChar(CP_UTF8, 0, text.c_str(),
-                                           static_cast<int>(text.size()), nullptr, 0);
-    std::wstring out(static_cast<std::size_t>(needed), L'\0');
-    MultiByteToWideChar(CP_UTF8, 0, text.c_str(), static_cast<int>(text.size()), out.data(),
-                        needed);
-    return out;
-}
-
 bool file_exists(const std::string& path) {
-    const std::wstring wide = utf8_to_wide(path);
+    const std::wstring wide = joc_path::to_wide_extended(path);
     if (wide.empty()) return false;
     const DWORD attributes = GetFileAttributesW(wide.c_str());
     return attributes != INVALID_FILE_ATTRIBUTES &&
@@ -121,11 +143,11 @@ public:
                                         FILE_SHARE_READ | FILE_SHARE_WRITE, &attributes,
                                         CREATE_ALWAYS, 0, nullptr);
 
-        std::wstring command = L"\"" + utf8_to_wide(ffmpeg_path) + L"\"";
+        std::wstring command = L"\"" + joc_path::to_wide_extended(ffmpeg_path) + L"\"";
         command += L" -hide_banner -loglevel error -nostdin -y ";
         command += input_arguments;  // input options must precede -i
         command += L" -i \"";
-        command += utf8_to_wide(input_path);
+        command += joc_path::to_wide_extended(input_path);
         command += L"\" ";
         command += output_arguments;
 
@@ -197,7 +219,7 @@ class InputFile {
 public:
     ~InputFile() { close(); }
     bool open(const std::string& path) {
-        handle_ = CreateFileW(utf8_to_wide(path).c_str(), GENERIC_READ,
+        handle_ = CreateFileW(joc_path::to_wide_extended(path).c_str(), GENERIC_READ,
                               FILE_SHARE_READ | FILE_SHARE_WRITE, nullptr, OPEN_EXISTING,
                               FILE_ATTRIBUTE_NORMAL | FILE_FLAG_SEQUENTIAL_SCAN, nullptr);
         return handle_ != INVALID_HANDLE_VALUE;
@@ -422,8 +444,8 @@ bool probe_container_joc(const std::string& ffmpeg_path, const std::string& path
     std::uint64_t modified = 0;
     {
         WIN32_FILE_ATTRIBUTE_DATA data{};
-        if (GetFileAttributesExW(utf8_to_wide(path).c_str(), GetFileExInfoStandard, &data) !=
+        if (GetFileAttributesExW(joc_path::to_wide_extended(path).c_str(), GetFileExInfoStandard,
-            FALSE) {
+                                 &data) != FALSE) {
             size = (static_cast<std::uint64_t>(data.nFileSizeHigh) << 32) | data.nFileSizeLow;
             modified = (static_cast<std::uint64_t>(data.ftLastWriteTime.dwHighDateTime) << 32) |
                        data.ftLastWriteTime.dwLowDateTime;
@@ -685,6 +707,7 @@ bool Engine::Impl::start_bed(std::uint64_t source_sample, std::string* error) {
     // at the start of the file, which is a low-level, noise-like difference from a
     // play-through rather than a misalignment: the position stays exact.
     std::wstring input_arguments = L"-drc_scale 0 -target_level 0";
+    if (has_mov_timeline(input_path)) input_arguments += L" -ignore_editlist 1";
     const std::wstring offset = seek_time(source_sample);
     if (!offset.empty()) input_arguments += L" -ss " + offset;
     std::wstring bed_arguments = L"-map 0:a:";
@@ -956,7 +979,19 @@ std::size_t Engine::read(float* destination, std::size_t frames, std::string* er
             impl.eac3_eof = true;
         }
 
-        if (!impl.eac3_eof && impl.frames_queued <= impl.bed_frames_pushed + 2u &&
+        // Queue more input only while less than one read's worth is waiting.
+        std::size_t rendered_ahead = 0;
+        {
+            joc_stream_status_info pending{};
+            pending.struct_size = sizeof(pending);
+            pending.struct_version = 1;
+            if (impl.api.status(impl.stream, &pending) == JOC_OK) {
+                rendered_ahead = static_cast<std::size_t>(pending.buffered_samples);
+            }
+        }
+
+        if (!impl.eac3_eof && rendered_ahead < kMaxRenderedAheadSamples &&
+            impl.frames_queued <= impl.bed_frames_pushed + 2u &&
             (impl.settings.input_frame_limit == 0 ||
              impl.frames_queued < impl.settings.input_frame_limit)) {
             // The tail of a chunk is usually the head of the next syncframe.  It
@@ -1005,17 +1040,20 @@ std::size_t Engine::read(float* destination, std::size_t frames, std::string* er
                     offset += bytes;
                     ++complete;
                 }
-                // With an input limit the chunk is cut at a frame boundary: the
+                // Both cuts land on a frame boundary because the renderer's output
-                // renderer's output depends on how many frames it was given, so a
+                // depends on how many frames it was given: the input limit has to
-                // limit that overshoots to the end of the read buffer would not
+                // stop where it says, and a read queues only its own budget.
-                // reproduce a run that stopped earlier.
+                std::uint64_t allowed = complete;
-                std::size_t push_bytes = offset;
-                std::uint64_t pushed_frames = complete;
                 if (impl.settings.input_frame_limit != 0) {
                     const std::uint64_t room =
                         impl.settings.input_frame_limit - impl.frames_queued;
-                    if (complete > room) {
+                    if (allowed > room) allowed = room;
-                        pushed_frames = room;
+                }
+                if (allowed > kEac3FramesPerRead) allowed = kEac3FramesPerRead;
+                std::size_t push_bytes = offset;
+                std::uint64_t pushed_frames = complete;
+                if (allowed < complete) {
+                    pushed_frames = allowed;
                     std::size_t walk = 0;
                     for (std::uint64_t index = 0; index < pushed_frames; ++index) {
                         const std::size_t bytes = joc_eac3::frame_bytes_at(
@@ -1025,7 +1063,6 @@ std::size_t Engine::read(float* destination, std::size_t frames, std::string* er
                     }
                     push_bytes = walk;
                 }
-                }
                 joc_stream_buffer input{};
                 input.struct_size = sizeof(input);
                 input.struct_version = 1;
@@ -1046,7 +1083,9 @@ std::size_t Engine::read(float* destination, std::size_t frames, std::string* er
                     std::memmove(impl.eac3_buffer.data(), impl.eac3_buffer.data() + push_bytes,
                                  impl.eac3_carry);
                 }
-                if (got == 0) impl.eac3_eof = true;
+                // The pipe can end with complete syncframes still buffered; they
+                // are queued next, so end of input is only real once none remain.
+                if (got == 0 && pushed_frames >= complete) impl.eac3_eof = true;
             }
         }
 

src/log.cpp

@@ -8,6 +8,8 @@
 #include <mutex>
 #include <string>
 
+#include "win_path.h"
+
 namespace joc_log {
 namespace {
 
@@ -16,12 +18,20 @@ FILE* g_file = nullptr;
 bool g_opened = false;
 std::string g_path;          // UTF-8, for reporting
 std::wstring g_wide_path;    // for _wfopen
-unsigned long long g_lines = 0;
+std::wstring g_wide_rolled;  // g_wide_path with the .1 suffix
-bool g_capped = false;
+unsigned long long g_bytes = 0;
 
-// A Media Library scan calls into us for every file; the cap keeps a long
+// A Media Library scan calls into us for every file it walks, so the cap has to
-// session from filling the disk, and says so once when it is reached.
+// be on bytes: the lines a scan produces run from 40 to 2048 bytes each, and a
-constexpr unsigned long long kMaxLines = 500000;
+// cap on their number leaves the file they build unpredictable by a factor of
+// fifty.  Sixteen mebibytes per file, one previous file kept, so the component
+// never holds more than 32 MiB of log.
+constexpr unsigned long long kMaxBytes = 16ull << 20;
+
+// The lines recorded between header_begin() and header_end(): the version
+// banner, a handful of lines written once per process.
+bool g_recording = false;
+std::string g_header;
 
 std::wstring module_directory() {
     HMODULE module = nullptr;
@@ -39,32 +49,26 @@ std::wstring module_directory() {
     return dir;
 }
 
-std::string to_utf8(const std::wstring& text) {
-    if (text.empty()) return {};
-    const int needed = WideCharToMultiByte(CP_UTF8, 0, text.c_str(),
-                                           static_cast<int>(text.size()), nullptr, 0,
-                                           nullptr, nullptr);
-    std::string out(static_cast<std::string::size_type>(needed), '\0');
-    WideCharToMultiByte(CP_UTF8, 0, text.c_str(), static_cast<int>(text.size()),
-                        out.data(), needed, nullptr, nullptr);
-    return out;
-}
-
 void open_locked() {
     g_opened = true;
 
     wchar_t from_env[4096] = {};
     const DWORD env_length = GetEnvironmentVariableW(L"JOC_LOG", from_env,
                                                      static_cast<DWORD>(4096));
+    std::wstring plain;
     if (env_length > 0 && env_length < 4096) {
-        g_wide_path.assign(from_env, env_length);
+        plain.assign(from_env, env_length);
     } else {
         const std::wstring dir = module_directory();
         if (dir.empty()) return;
-        g_wide_path = dir + L"\\joc_decoder.log";
+        plain = dir + L"\\joc_decoder.log";
     }
 
-    g_path = to_utf8(g_wide_path);
+    // g_path is the one the component reports and a user reads; the handles work on
+    // the extended form, because a portable install can sit deeper than MAX_PATH.
+    g_path = joc_path::to_utf8(plain);
+    g_wide_path = joc_path::to_wide_extended(plain);
+    g_wide_rolled = g_wide_path + L".1";
     g_file = _wfopen(g_wide_path.c_str(), L"wb");
     if (g_file == nullptr) {
         g_path.clear();
@@ -74,6 +78,52 @@ void open_locked() {
     std::setvbuf(g_file, nullptr, _IONBF, 0);
 }
 
+// Composes one line with its timestamp, writes it, and accounts for it.  Returns
+// the bytes written, 0 when there was nowhere to write them.
+unsigned long long write_locked(const char* text) {
+    if (g_file == nullptr) return 0;
+
+    SYSTEMTIME now;
+    GetLocalTime(&now);
+    char composed[2100];
+    const int length =
+        std::snprintf(composed, sizeof(composed), "%02u:%02u:%02u.%03u [t%05lu] %s\n", now.wHour,
+                      now.wMinute, now.wSecond, now.wMilliseconds,
+                      static_cast<unsigned long>(GetCurrentThreadId()), text);
+    if (length <= 0) return 0;
+    // snprintf reports the length it would have needed, which for a truncated
+    // line is past the end of the buffer.
+    const std::size_t size = (static_cast<std::size_t>(length) < sizeof(composed))
+                                 ? static_cast<std::size_t>(length)
+                                 : sizeof(composed) - 1;
+    std::fwrite(composed, 1, size, g_file);
+    g_bytes += size;
+    if (g_recording) g_header.append(composed, size);
+    return size;
+}
+
+// Moves the full file aside and starts an empty one.  Called with the mutex held,
+// once the budget is spent.  The banner is written again afterwards, so a log
+// that has rolled still opens with the versions it was produced by.
+void roll_locked() {
+    std::fclose(g_file);
+    g_file = nullptr;
+    // One previous window, replaced rather than chained: a chain of rolls would
+    // be the same unbounded log under another name.  A move that fails -- the .1
+    // file held open by a reader -- leaves the truncation below to bound the file
+    // anyway, at the cost of the window that would have been kept.
+    MoveFileExW(g_wide_path.c_str(), g_wide_rolled.c_str(), MOVEFILE_REPLACE_EXISTING);
+    g_file = _wfopen(g_wide_path.c_str(), L"wb");
+    if (g_file == nullptr) return;
+    std::setvbuf(g_file, nullptr, _IONBF, 0);
+    g_bytes = 0;
+    write_locked("[log rolled: the previous window is in the .1 file beside this one]");
+    if (!g_header.empty()) {
+        std::fwrite(g_header.c_str(), 1, g_header.size(), g_file);
+        g_bytes += g_header.size();
+    }
+}
+
 }  // namespace
 
 void open() {
@@ -86,30 +136,34 @@ const char* path() {
     return g_path.c_str();
 }
 
-void line(const char* fmt, ...) {
+unsigned long long budget_bytes() { return kMaxBytes; }
-    va_list args;
+
-    va_start(args, fmt);
+void line_v(const char* fmt, va_list args) {
     char text[2048];
     std::vsnprintf(text, sizeof(text), fmt, args);
-    va_end(args);
 
     std::lock_guard<std::mutex> guard(g_mutex);
     if (!g_opened) open_locked();
     if (g_file == nullptr) return;
-    if (g_lines >= kMaxLines) {
+    if (write_locked(text) != 0 && g_bytes >= kMaxBytes) roll_locked();
-        if (!g_capped) {
-            g_capped = true;
-            std::fputs("[log capped]\n", g_file);
 }
-        return;
-    }
-    ++g_lines;
 
-    SYSTEMTIME now;
+void line(const char* fmt, ...) {
-    GetLocalTime(&now);
+    va_list args;
-    std::fprintf(g_file, "%02u:%02u:%02u.%03u [t%05lu] %s\n", now.wHour, now.wMinute,
+    va_start(args, fmt);
-                 now.wSecond, now.wMilliseconds,
+    line_v(fmt, args);
-                 static_cast<unsigned long>(GetCurrentThreadId()), text);
+    va_end(args);
+}
+
+void header_begin() {
+    std::lock_guard<std::mutex> guard(g_mutex);
+    g_header.clear();
+    g_recording = true;
+}
+
+void header_end() {
+    std::lock_guard<std::mutex> guard(g_mutex);
+    g_recording = false;
 }
 
 }  // namespace joc_log

src/log.h

@@ -4,10 +4,22 @@
 // harness and so that it is safe to call from any thread the core queries us on.
 //
 // Path: %JOC_LOG% if set, otherwise <directory of this DLL>\joc_decoder.log.
-// The file is truncated once per process, line-buffered, and every line is
+// The file is truncated once per process, unbuffered, and every line is written
-// flushed so the log can be read while foobar2000 is still running.
+// straight out so the log can be read while foobar2000 is still running.
+//
+// Size: what is bounded is bytes, not lines.  A line is anywhere between 40 and
+// 2048 bytes, so a cap on the number of lines leaves the size of the file
+// unpredictable by a factor of fifty.  A file that reaches budget_bytes() is
+// moved to <path>.1 -- replacing any previous roll, so there is never a chain of
+// them -- and a fresh one is started.  Two files of the budget is therefore the
+// most the component ever holds, and the newest lines are always the ones in
+// <path>.  Lines written between header_begin() and header_end() are written
+// again at the top of every window, so a rolled log still says which build
+// produced it.
 #pragma once
 
+#include <cstdarg>
+
 namespace joc_log {
 
 // Opens (truncating) the log.  Called lazily by line(); call it explicitly to
@@ -17,7 +29,18 @@ void open();
 // Absolute path of the log file, or "" when it could not be opened.
 const char* path();
 
+// Bytes one file holds before it is rolled to <path>.1.
+unsigned long long budget_bytes();
+
 // printf-style, thread safe, appends one line.
 void line(const char* fmt, ...);
 
+// The va_list form, for a wrapper that forwards its own arguments.
+void line_v(const char* fmt, va_list args);
+
+// Remembers the lines written in between and writes them again after every roll.
+// Meant for the version banner: a handful of lines, written once.
+void header_begin();
+void header_end();
+
 }  // namespace joc_log

src/main.cpp

@@ -12,7 +12,7 @@
 
 // Kept in one place: the string reported to foobar2000 and written to the log
 // must not drift apart.
-#define JOC_VERSION "0.2.1"
+#define JOC_VERSION "0.3.0"
 
 DECLARE_COMPONENT_VERSION("JOC decoder (E-AC-3 JOC)", JOC_VERSION,
                           "Plays E-AC-3 JOC (Dolby Atmos) files: the JOC objects are "
@@ -30,6 +30,9 @@ class joc_initquit : public initquit {
 public:
     void on_init() override {
         joc_log::open();
+        // Recorded, so that a log which has rolled still opens with the versions
+        // it was produced by.
+        joc_log::header_begin();
         joc_log::line("=== foo_input_joc %s ===", JOC_VERSION);
         joc_log::line("foobar2000 core : %s", core_version_info::g_get_version_string());
         joc_log::line("component file  : %s", core_api::get_my_file_name());
@@ -37,6 +40,8 @@ public:
         joc_log::line("portable mode   : %s",
                       core_api::is_portable_mode_enabled() ? "yes" : "no");
         joc_log::line("log file        : %s", joc_log::path());
+        joc_log::line("log limit       : %llu byte(s) per file, then rolled to the .1 suffix",
+                      joc_log::budget_bytes());
         joc_log::line("compiled as     : %s",
 #if defined(_M_IX86)
                       "x86 (32-bit)"
@@ -46,6 +51,7 @@ public:
                       "other"
 #endif
         );
+        joc_log::header_end();
     }
 
     void on_quit() override { joc_log::line("=== foo_input_joc shutdown ==="); }

src/win_path.h

@@ -0,0 +1,90 @@
+// Paths handed to the Win32 file APIs.
+//
+// Windows refuses a path of MAX_PATH characters or more to any call that does not
+// carry the \\?\ prefix.  The prefix is a property of the path, not of the process:
+// a component cannot inherit it from the host's manifest, so whoever calls the API
+// has to add it.  A file that plays from the desktop therefore stops being found
+// once its folder tree is deep enough, which is what this header exists to prevent.
+//
+// Everything in this component that hands a path to Win32 -- or puts one on an
+// ffmpeg command line, which is the same call made by the child process -- goes
+// through to_wide_extended().  What is deliberately not routed through it is the
+// path the component keeps for itself: the one it logs, and the one it uses as the
+// container-probe cache key.  Those stay in their plain form, so a log line reads
+// like a path a user can paste into Explorer.
+#pragma once
+
+#include <windows.h>
+
+#include <string>
+
+namespace joc_path {
+
+// UTF-8 -> UTF-16, for the W APIs.  Every path crosses into the OS through here:
+// the narrow CRT would convert it through the ANSI code page, which is what breaks
+// names like "Les Fêtes d'Hébé".
+inline std::wstring to_wide(const std::string& utf8) {
+    if (utf8.empty()) return {};
+    const int needed = MultiByteToWideChar(CP_UTF8, 0, utf8.c_str(),
+                                           static_cast<int>(utf8.size()), nullptr, 0);
+    if (needed <= 0) return {};
+    std::wstring out(static_cast<std::size_t>(needed), L'\0');
+    MultiByteToWideChar(CP_UTF8, 0, utf8.c_str(), static_cast<int>(utf8.size()), out.data(),
+                        needed);
+    return out;
+}
+
+// The same path in the form the Win32 file APIs need.  Three kinds of path are left
+// exactly as they came in, and each of them would be broken by a prefix:
+//
+//   * one that already carries \\?\, which makes this idempotent;
+//   * one that is not fully qualified -- a relative path, or the ffmpeg setting
+//     when it is a bare executable name that PATH resolves.  \\?\ is undefined for
+//     those;
+//   * one with a "." or ".." segment, because \\?\ switches off the normalization
+//     that would resolve it.
+//
+// The separators are settled first for the same reason: once the prefix is on,
+// nothing turns a forward slash into a backslash any more, and both of the checks
+// above have to see the path the way the file system will.
+inline std::wstring to_wide_extended(const std::wstring& path) {
+    if (path.empty()) return path;
+    if (path.compare(0, 4, L"\\\\?\\") == 0) return path;
+
+    std::wstring plain = path;
+    for (wchar_t& character : plain) {
+        if (character == L'/') character = L'\\';
+    }
+
+    const bool unc = plain.size() >= 2 && plain[0] == L'\\' && plain[1] == L'\\';
+    const bool drive = plain.size() >= 3 && plain[1] == L':' && plain[2] == L'\\';
+    if (!unc && !drive) return path;
+    if (plain.find(L"\\.\\") != std::wstring::npos ||
+        plain.find(L"\\..\\") != std::wstring::npos) {
+        return path;
+    }
+
+    // A UNC path drops its two leading separators; everything else is copied whole.
+    std::wstring out = unc ? L"\\\\?\\UNC\\" : L"\\\\?\\";
+    out.append(plain, unc ? 2 : 0, std::wstring::npos);
+    return out;
+}
+
+inline std::wstring to_wide_extended(const std::string& utf8) {
+    return to_wide_extended(to_wide(utf8));
+}
+
+// UTF-16 -> UTF-8, the direction the component reports paths in.
+inline std::string to_utf8(const std::wstring& wide) {
+    if (wide.empty()) return {};
+    const int needed = WideCharToMultiByte(CP_UTF8, 0, wide.c_str(),
+                                           static_cast<int>(wide.size()), nullptr, 0, nullptr,
+                                           nullptr);
+    if (needed <= 0) return {};
+    std::string out(static_cast<std::string::size_type>(needed), '\0');
+    WideCharToMultiByte(CP_UTF8, 0, wide.c_str(), static_cast<int>(wide.size()), out.data(),
+                        needed, nullptr, nullptr);
+    return out;
+}
+
+}  // namespace joc_path

tests/log_rotation_test.cpp

@@ -0,0 +1,111 @@
+// Drives the diagnostic log past its byte budget, without foobar2000.
+//
+//   log_rotation_test
+//
+// The component's log is bounded by rolling: a file that reaches the budget is
+// moved to a .1 sibling and a fresh one is started.  What pushes it there is a
+// Media Library scan, which a test cannot arrange, so the budget is spent here
+// with lines of a known size instead.  What is checked is what the bound
+// promises: neither file passes the budget, there is no chain of rolls, the
+// newest lines are the ones in the live file, and the session banner is written
+// again after a roll -- which is the only reason a rolled log still says which
+// build produced it.
+
+#include <windows.h>
+
+#include <cstdio>
+#include <string>
+
+#include "../src/log.h"
+
+namespace {
+
+std::wstring temp_directory() {
+    wchar_t buffer[MAX_PATH + 1] = {};
+    const DWORD length = GetTempPathW(MAX_PATH, buffer);
+    return std::wstring(buffer, length) + L"joc_log_rotation_test";
+}
+
+bool read_file(const std::wstring& path, std::string* out) {
+    std::FILE* file = _wfopen(path.c_str(), L"rb");
+    if (file == nullptr) return false;
+    char buffer[64 * 1024];
+    std::size_t got = 0;
+    while ((got = std::fread(buffer, 1, sizeof(buffer), file)) != 0) out->append(buffer, got);
+    std::fclose(file);
+    return true;
+}
+
+unsigned long long file_size(const std::wstring& path) {
+    WIN32_FILE_ATTRIBUTE_DATA data{};
+    if (GetFileAttributesExW(path.c_str(), GetFileExInfoStandard, &data) == FALSE) return 0;
+    return (static_cast<unsigned long long>(data.nFileSizeHigh) << 32) | data.nFileSizeLow;
+}
+
+bool contains(const std::string& haystack, const std::string& needle) {
+    return haystack.find(needle) != std::string::npos;
+}
+
+int failures = 0;
+
+void check(bool ok, const char* what) {
+    std::printf("%-4s %s\n", ok ? "ok" : "FAIL", what);
+    if (!ok) ++failures;
+}
+
+}  // namespace
+
+int main() {
+    const std::wstring dir = temp_directory();
+    CreateDirectoryW(dir.c_str(), nullptr);
+    const std::wstring live = dir + L"\\rotation.log";
+    const std::wstring rolled = live + L".1";
+    const std::wstring chained = live + L".2";
+    DeleteFileW(live.c_str());
+    DeleteFileW(rolled.c_str());
+    DeleteFileW(chained.c_str());
+    SetEnvironmentVariableW(L"JOC_LOG", live.c_str());
+
+    const unsigned long long budget = joc_log::budget_bytes();
+    std::printf("budget %llu byte(s) per file\n", budget);
+
+    joc_log::open();
+    joc_log::header_begin();
+    joc_log::line("=== header marker ===");
+    joc_log::header_end();
+
+    // Enough lines to spend the budget twice over, so the roll has to happen and
+    // the file it produces has to be replaced rather than added to.
+    const std::string filler(160, 'x');
+    const unsigned lines = static_cast<unsigned>(budget / 190ull * 3ull);
+    for (unsigned i = 1; i <= lines; ++i) joc_log::line("filler %u %s", i, filler.c_str());
+
+    const unsigned long long live_size = file_size(live);
+    const unsigned long long rolled_size = file_size(rolled);
+    std::printf("wrote %u line(s): live %llu byte(s), rolled %llu byte(s)\n", lines, live_size,
+                rolled_size);
+
+    check(live_size > 0 && live_size <= budget + 4096, "live file is inside the budget");
+    check(rolled_size > 0 && rolled_size <= budget + 4096, "rolled file is inside the budget");
+    check(live_size + rolled_size <= 2 * budget + 8192, "the two together are inside twice it");
+    check(file_size(chained) == 0, "no second roll is kept");
+
+    std::string live_text;
+    std::string rolled_text;
+    check(read_file(live, &live_text), "live file reads back");
+    check(read_file(rolled, &rolled_text), "rolled file reads back");
+    check(contains(live_text, "[log rolled"), "the live file says it rolled");
+    check(contains(live_text, "=== header marker ==="), "the banner was written again");
+
+    char newest[64] = {};
+    std::snprintf(newest, sizeof(newest), "filler %u ", lines);
+    check(contains(live_text, newest), "the newest line is in the live file");
+    check(!contains(rolled_text, newest), "and not in the rolled one");
+
+    if (failures == 0) {
+        std::printf("log rotation holds\n");
+        return 0;
+    }
+    std::printf("%d check(s) failed\n", failures);
+    return 1;
+}

tests/long_path_test.cpp

@@ -0,0 +1,172 @@
+// Checks that the component works with a path past MAX_PATH.
+//
+//   long_path_test
+//
+// Windows refuses a path of MAX_PATH characters or more to any Win32 call that does
+// not carry the \\?\ prefix.  The prefix is a property of the path rather than of the
+// process, so it has to be added by whoever calls the API -- which is why a file that
+// plays from the desktop stops being found once its folder tree is deep enough, the
+// shape an Atmos download under a long album title has.
+//
+// The path is built here rather than taken from the command line, so the test needs
+// no fixture and cannot be defeated by the ANSI argv a console tool is handed.
+
+#include <windows.h>
+
+#include <cstdio>
+#include <string>
+
+#include "../src/container_scan.h"
+#include "../src/joc_decode.h"
+#include "../src/win_path.h"
+
+namespace {
+
+int failures = 0;
+
+void check(bool ok, const char* what) {
+    std::printf("%-4s %s\n", ok ? "ok" : "FAIL", what);
+    if (!ok) ++failures;
+}
+
+// %TEMP%\joc_long_path
+std::string temp_root() {
+    wchar_t buffer[MAX_PATH + 1] = {};
+    const DWORD length = GetTempPathW(MAX_PATH, buffer);
+    if (length == 0) return {};
+    return joc_path::to_utf8(std::wstring(buffer, length) + L"joc_long_path");
+}
+
+// One 45-character directory name.  Five of them put the file comfortably past the
+// limit whatever %TEMP% is.
+std::string level_name(int index) {
+    return "level-" + std::string(39, static_cast<char>('a' + index));
+}
+
+bool make_directory(const std::string& path) {
+    if (CreateDirectoryW(joc_path::to_wide_extended(path).c_str(), nullptr)) return true;
+    return GetLastError() == ERROR_ALREADY_EXISTS;
+}
+
+// Builds the deep tree one level at a time, so every parent already exists by the
+// time its child is asked for.
+std::string build_tree() {
+    std::string path = temp_root();
+    if (path.empty() || !make_directory(path)) return {};
+    for (int i = 0; i < 5; ++i) {
+        path += "\\" + level_name(i);
+        if (!make_directory(path)) return {};
+    }
+    return path;
+}
+
+// A file that is a container but holds no audio track: enough for the probe to walk
+// its boxes and report something of its own, rather than "cannot open the file".
+bool write_container(const std::string& path) {
+    unsigned char bytes[32] = {};
+    bytes[3] = sizeof(bytes);  // box size, big-endian
+    const char type[] = "ftypisom";
+    for (int i = 0; i < 8; ++i) bytes[4 + i] = static_cast<unsigned char>(type[i]);
+    std::FILE* file = _wfopen(joc_path::to_wide_extended(path).c_str(), L"wb");
+    if (file == nullptr) return false;
+    const std::size_t written = std::fwrite(bytes, 1, sizeof(bytes), file);
+    std::fclose(file);
+    return written == sizeof(bytes);
+}
+
+void remove_tree(const std::string& root, const std::string& leaf) {
+    // Files before the directories that hold them, deepest directory first, and
+    // never above root.  Worth doing at all because a tree past MAX_PATH is one
+    // Explorer cannot delete: leaving it behind would be worse than the disk it
+    // occupies.
+    DeleteFileW(joc_path::to_wide_extended(leaf + "\\probe.m4a").c_str());
+    DeleteFileW(joc_path::to_wide_extended(root + "\\short.m4a").c_str());
+    std::string current = leaf;
+    while (current.size() > root.size()) {
+        RemoveDirectoryW(joc_path::to_wide_extended(current).c_str());
+        const std::string::size_type slash = current.find_last_of('\\');
+        if (slash == std::string::npos) break;
+        current.resize(slash);
+    }
+    RemoveDirectoryW(joc_path::to_wide_extended(root).c_str());
+}
+
+// The rules that keep the prefix from being applied where it would break a path.
+void check_helper_rules() {
+    check(joc_path::to_wide_extended(std::string("ffmpeg")) == L"ffmpeg",
+          "a bare executable name is left for PATH to resolve");
+    check(joc_path::to_wide_extended(std::string("sub\\file.m4a")) == L"sub\\file.m4a",
+          "a relative path is left alone");
+    check(joc_path::to_wide_extended(std::string("C:\\a\\b.m4a")) == L"\\\\?\\C:\\a\\b.m4a",
+          "an absolute path gains the prefix");
+    check(joc_path::to_wide_extended(std::string("\\\\?\\C:\\a\\b.m4a")) ==
+              L"\\\\?\\C:\\a\\b.m4a",
+          "an already-prefixed path is not prefixed twice");
+    check(joc_path::to_wide_extended(std::string("\\\\server\\share\\b.m4a")) ==
+              L"\\\\?\\UNC\\server\\share\\b.m4a",
+          "a UNC path takes the UNC form");
+    check(joc_path::to_wide_extended(std::string("//server/share/b.m4a")) ==
+              L"\\\\?\\UNC\\server\\share\\b.m4a",
+          "and the same one in forward slashes, without doubling a separator");
+    check(joc_path::to_wide_extended(std::string("C:/a/b.m4a")) == L"\\\\?\\C:\\a\\b.m4a",
+          "forward slashes become backslashes, which the prefix stops normalizing");
+    check(joc_path::to_wide_extended(std::string("C:\\a\\..\\b.m4a")) == L"C:\\a\\..\\b.m4a",
+          "a .. segment is left alone, since the prefix would stop resolving it");
+    check(joc_path::to_wide_extended(std::string("C:/a/../b.m4a")) == L"C:/a/../b.m4a",
+          "and one in forward slashes is recognized as well");
+}
+
+}  // namespace
+
+int main() {
+    check_helper_rules();
+
+    const std::string tree = build_tree();
+    if (tree.empty()) {
+        std::printf("FAIL cannot build the tree under %s\n", temp_root().c_str());
+        return 1;
+    }
+    const std::string long_file = tree + "\\probe.m4a";
+    const std::string short_file = temp_root() + "\\short.m4a";
+    std::printf("long path  : %zu chars\n", long_file.size());
+    std::printf("short path : %zu chars\n", short_file.size());
+
+    check(long_file.size() > MAX_PATH, "the path is past MAX_PATH, so the test is on it");
+    check(write_container(long_file), "the file was written at the long path");
+    check(write_container(short_file), "the control file was written at the short path");
+
+    // The premise: without the prefix the very same path is refused.  If this ever
+    // starts passing, the machine lifts MAX_PATH for ordinary processes and the rest
+    // of the test no longer proves anything.
+    const HANDLE raw = CreateFileW(joc_path::to_wide(long_file).c_str(), GENERIC_READ,
+                                   FILE_SHARE_READ, nullptr, OPEN_EXISTING,
+                                   FILE_ATTRIBUTE_NORMAL, nullptr);
+    check(raw == INVALID_HANDLE_VALUE, "the raw path is still refused without the prefix");
+    if (raw != INVALID_HANDLE_VALUE) CloseHandle(raw);
+
+    const joc_container::Result long_result = joc_container::scan(long_file);
+    std::printf("container probe: kind=%s detail=%s\n",
+                joc_container::kind_name(long_result.kind), long_result.detail.c_str());
+    check(long_result.kind == joc_container::Kind::kMp4,
+          "the container probe reads the long path");
+    check(long_result.detail != "cannot open the file",
+          "and did not fall back to the unreadable-file answer");
+
+    const joc_decode::FileProbe probe = joc_decode::probe_file(long_file);
+    std::printf("decode reader  : readable=%d detail=%s\n", probe.readable ? 1 : 0,
+                probe.detail.c_str());
+    check(probe.readable, "the decode-side reader opens the long path too");
+
+    const joc_container::Result short_result = joc_container::scan(short_file);
+    check(short_result.kind == joc_container::Kind::kMp4 && short_result.detail != "cannot open the file",
+          "a short path in the same tree still probes, prefix and all");
+
+    remove_tree(temp_root(), tree);
+
+    if (failures == 0) {
+        std::printf("long paths are handled\n");
+        return 0;
+    }
+    std::printf("%d check(s) failed\n", failures);
+    return 1;
+}

tools/build_tests.ps1

@@ -40,6 +40,8 @@ $lines = @(
     "cd /d `"$projectRoot`"",
     # log.cpp comes along because the engine writes its diagnostics through it.
     "cl $common /Fe:`"$outDir\container_scan_test.exe`" /Fo:`"$outDir\\`" tests\container_scan_test.cpp src\container_scan.cpp src\log.cpp",
+    "cl $common /Fe:`"$outDir\log_rotation_test.exe`" /Fo:`"$outDir\\`" tests\log_rotation_test.cpp src\log.cpp",
+    "cl $common /Fe:`"$outDir\long_path_test.exe`" /Fo:`"$outDir\\`" tests\long_path_test.cpp src\container_scan.cpp src\joc_decode.cpp src\eac3_scan.cpp src\log.cpp `"$coreLib`" shell32.lib",
     "cl $common /Fe:`"$outDir\scan_selftest.exe`" /Fo:`"$outDir\\`" tests\scan_selftest.cpp src\eac3_scan.cpp",
     "cl $common /Fe:`"$outDir\prefs_layout_check.exe`" /Fo:`"$outDir\\`" tests\prefs_layout_check.cpp user32.lib gdi32.lib",
     "cl $common /Fe:`"$outDir\scan_crosscheck.exe`" /Fo:`"$outDir\\`" tests\scan_crosscheck.cpp src\eac3_scan.cpp `"$coreLib`" shell32.lib",

tools/run.ps1

@@ -39,7 +39,10 @@ if (Test-Path -LiteralPath $running) {
 }
 
 $arguments = @()
-if ($Play) { $arguments += $Play }
+# Start-Process joins the array with spaces and quotes nothing itself, so a path with
+# a space in it would reach foobar2000 as several arguments and nothing would play --
+# silently, which is the worst way for a test bed to fail.
+if ($Play) { $arguments += ($Play | ForEach-Object { '"' + $_ + '"' }) }
 $process = Start-Process -FilePath $exe -ArgumentList $arguments -PassThru
 Write-Host "started pid=$($process.Id) $($Play -join ', ')"
 Start-Sleep -Seconds $WaitSeconds