Fix files past MAX_PATH not being claimed or decoded

foo_input_joc.vcxproj

@@ -114,6 +114,7 @@
     <ClInclude Include="src\prefs.h" />
     <ClInclude Include="src\resource.h" />
     <ClInclude Include="src\settings.h" />
+    <ClInclude Include="src\win_path.h" />
   </ItemGroup>
 
   <ItemGroup>

src/container_scan.cpp

@@ -8,20 +8,11 @@
 #include <vector>
 
 #include "log.h"
+#include "win_path.h"
 
 namespace joc_container {
 namespace {
 
-std::wstring utf8_to_wide(const std::string& text) {
-    if (text.empty()) return {};
-    const int needed = MultiByteToWideChar(CP_UTF8, 0, text.c_str(),
-                                           static_cast<int>(text.size()), nullptr, 0);
-    std::wstring out(static_cast<std::size_t>(needed), L'\0');
-    MultiByteToWideChar(CP_UTF8, 0, text.c_str(), static_cast<int>(text.size()), out.data(),
-                        needed);
-    return out;
-}
-
 // A library scan probes every container in the library, and most of them are
 // declined; they also mostly share a directory, so the path would spend the log's
 // byte budget repeating the same prefix once per file.  The file name is what
@@ -37,7 +28,7 @@ std::string file_name_of(const std::string& path) {
 class Window {
 public:
     bool open(const std::string& path) {
-        const std::wstring wide = utf8_to_wide(path);
+        const std::wstring wide = joc_path::to_wide_extended(path);
         handle_ = CreateFileW(wide.c_str(), GENERIC_READ, FILE_SHARE_READ | FILE_SHARE_WRITE,
                               nullptr, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, nullptr);
         if (handle_ == INVALID_HANDLE_VALUE) {

src/joc_decode.cpp

@@ -14,6 +14,7 @@
 
 #include "eac3_scan.h"
 #include "log.h"
+#include "win_path.h"
 
 namespace joc_decode {
 namespace {
@@ -78,18 +79,8 @@ struct CoreApi {
     const char*(JOC_CALL* error_name)(joc_error) = joc_error_name;
 };
 
-std::wstring utf8_to_wide(const std::string& text) {
-    if (text.empty()) return {};
-    const int needed = MultiByteToWideChar(CP_UTF8, 0, text.c_str(),
-                                           static_cast<int>(text.size()), nullptr, 0);
-    std::wstring out(static_cast<std::size_t>(needed), L'\0');
-    MultiByteToWideChar(CP_UTF8, 0, text.c_str(), static_cast<int>(text.size()), out.data(),
-                        needed);
-    return out;
-}
-
 bool file_exists(const std::string& path) {
-    const std::wstring wide = utf8_to_wide(path);
+    const std::wstring wide = joc_path::to_wide_extended(path);
     if (wide.empty()) return false;
     const DWORD attributes = GetFileAttributesW(wide.c_str());
     return attributes != INVALID_FILE_ATTRIBUTES &&
@@ -147,11 +138,11 @@ public:
                                         FILE_SHARE_READ | FILE_SHARE_WRITE, &attributes,
                                         CREATE_ALWAYS, 0, nullptr);
 
-        std::wstring command = L"\"" + utf8_to_wide(ffmpeg_path) + L"\"";
+        std::wstring command = L"\"" + joc_path::to_wide_extended(ffmpeg_path) + L"\"";
         command += L" -hide_banner -loglevel error -nostdin -y ";
         command += input_arguments;  // input options must precede -i
         command += L" -i \"";
-        command += utf8_to_wide(input_path);
+        command += joc_path::to_wide_extended(input_path);
         command += L"\" ";
         command += output_arguments;
 
@@ -223,7 +214,7 @@ class InputFile {
 public:
     ~InputFile() { close(); }
     bool open(const std::string& path) {
-        handle_ = CreateFileW(utf8_to_wide(path).c_str(), GENERIC_READ,
+        handle_ = CreateFileW(joc_path::to_wide_extended(path).c_str(), GENERIC_READ,
                               FILE_SHARE_READ | FILE_SHARE_WRITE, nullptr, OPEN_EXISTING,
                               FILE_ATTRIBUTE_NORMAL | FILE_FLAG_SEQUENTIAL_SCAN, nullptr);
         return handle_ != INVALID_HANDLE_VALUE;
@@ -448,8 +439,8 @@ bool probe_container_joc(const std::string& ffmpeg_path, const std::string& path
     std::uint64_t modified = 0;
     {
         WIN32_FILE_ATTRIBUTE_DATA data{};
-        if (GetFileAttributesExW(utf8_to_wide(path).c_str(), GetFileExInfoStandard, &data) !=
-            FALSE) {
+        if (GetFileAttributesExW(joc_path::to_wide_extended(path).c_str(), GetFileExInfoStandard,
+                                 &data) != FALSE) {
             size = (static_cast<std::uint64_t>(data.nFileSizeHigh) << 32) | data.nFileSizeLow;
             modified = (static_cast<std::uint64_t>(data.ftLastWriteTime.dwHighDateTime) << 32) |
                        data.ftLastWriteTime.dwLowDateTime;

src/log.cpp

@@ -8,6 +8,8 @@
 #include <mutex>
 #include <string>
 
+#include "win_path.h"
+
 namespace joc_log {
 namespace {
 
@@ -47,33 +49,26 @@ std::wstring module_directory() {
     return dir;
 }
 
-std::string to_utf8(const std::wstring& text) {
-    if (text.empty()) return {};
-    const int needed = WideCharToMultiByte(CP_UTF8, 0, text.c_str(),
-                                           static_cast<int>(text.size()), nullptr, 0,
-                                           nullptr, nullptr);
-    std::string out(static_cast<std::string::size_type>(needed), '\0');
-    WideCharToMultiByte(CP_UTF8, 0, text.c_str(), static_cast<int>(text.size()),
-                        out.data(), needed, nullptr, nullptr);
-    return out;
-}
-
 void open_locked() {
     g_opened = true;
 
     wchar_t from_env[4096] = {};
     const DWORD env_length = GetEnvironmentVariableW(L"JOC_LOG", from_env,
                                                      static_cast<DWORD>(4096));
+    std::wstring plain;
     if (env_length > 0 && env_length < 4096) {
-        g_wide_path.assign(from_env, env_length);
+        plain.assign(from_env, env_length);
     } else {
         const std::wstring dir = module_directory();
         if (dir.empty()) return;
-        g_wide_path = dir + L"\\joc_decoder.log";
+        plain = dir + L"\\joc_decoder.log";
     }
 
+    // g_path is the one the component reports and a user reads; the handles work on
+    // the extended form, because a portable install can sit deeper than MAX_PATH.
+    g_path = joc_path::to_utf8(plain);
+    g_wide_path = joc_path::to_wide_extended(plain);
     g_wide_rolled = g_wide_path + L".1";
-    g_path = to_utf8(g_wide_path);
     g_file = _wfopen(g_wide_path.c_str(), L"wb");
     if (g_file == nullptr) {
         g_path.clear();

src/win_path.h

@@ -0,0 +1,90 @@
+// Paths handed to the Win32 file APIs.
+//
+// Windows refuses a path of MAX_PATH characters or more to any call that does not
+// carry the \\?\ prefix.  The prefix is a property of the path, not of the process:
+// a component cannot inherit it from the host's manifest, so whoever calls the API
+// has to add it.  A file that plays from the desktop therefore stops being found
+// once its folder tree is deep enough, which is what this header exists to prevent.
+//
+// Everything in this component that hands a path to Win32 -- or puts one on an
+// ffmpeg command line, which is the same call made by the child process -- goes
+// through to_wide_extended().  What is deliberately not routed through it is the
+// path the component keeps for itself: the one it logs, and the one it uses as the
+// container-probe cache key.  Those stay in their plain form, so a log line reads
+// like a path a user can paste into Explorer.
+#pragma once
+
+#include <windows.h>
+
+#include <string>
+
+namespace joc_path {
+
+// UTF-8 -> UTF-16, for the W APIs.  Every path crosses into the OS through here:
+// the narrow CRT would convert it through the ANSI code page, which is what breaks
+// names like "Les Fêtes d'Hébé".
+inline std::wstring to_wide(const std::string& utf8) {
+    if (utf8.empty()) return {};
+    const int needed = MultiByteToWideChar(CP_UTF8, 0, utf8.c_str(),
+                                           static_cast<int>(utf8.size()), nullptr, 0);
+    if (needed <= 0) return {};
+    std::wstring out(static_cast<std::size_t>(needed), L'\0');
+    MultiByteToWideChar(CP_UTF8, 0, utf8.c_str(), static_cast<int>(utf8.size()), out.data(),
+                        needed);
+    return out;
+}
+
+// The same path in the form the Win32 file APIs need.  Three kinds of path are left
+// exactly as they came in, and each of them would be broken by a prefix:
+//
+//   * one that already carries \\?\, which makes this idempotent;
+//   * one that is not fully qualified -- a relative path, or the ffmpeg setting
+//     when it is a bare executable name that PATH resolves.  \\?\ is undefined for
+//     those;
+//   * one with a "." or ".." segment, because \\?\ switches off the normalization
+//     that would resolve it.
+//
+// The separators are settled first for the same reason: once the prefix is on,
+// nothing turns a forward slash into a backslash any more, and both of the checks
+// above have to see the path the way the file system will.
+inline std::wstring to_wide_extended(const std::wstring& path) {
+    if (path.empty()) return path;
+    if (path.compare(0, 4, L"\\\\?\\") == 0) return path;
+
+    std::wstring plain = path;
+    for (wchar_t& character : plain) {
+        if (character == L'/') character = L'\\';
+    }
+
+    const bool unc = plain.size() >= 2 && plain[0] == L'\\' && plain[1] == L'\\';
+    const bool drive = plain.size() >= 3 && plain[1] == L':' && plain[2] == L'\\';
+    if (!unc && !drive) return path;
+    if (plain.find(L"\\.\\") != std::wstring::npos ||
+        plain.find(L"\\..\\") != std::wstring::npos) {
+        return path;
+    }
+
+    // A UNC path drops its two leading separators; everything else is copied whole.
+    std::wstring out = unc ? L"\\\\?\\UNC\\" : L"\\\\?\\";
+    out.append(plain, unc ? 2 : 0, std::wstring::npos);
+    return out;
+}
+
+inline std::wstring to_wide_extended(const std::string& utf8) {
+    return to_wide_extended(to_wide(utf8));
+}
+
+// UTF-16 -> UTF-8, the direction the component reports paths in.
+inline std::string to_utf8(const std::wstring& wide) {
+    if (wide.empty()) return {};
+    const int needed = WideCharToMultiByte(CP_UTF8, 0, wide.c_str(),
+                                           static_cast<int>(wide.size()), nullptr, 0, nullptr,
+                                           nullptr);
+    if (needed <= 0) return {};
+    std::string out(static_cast<std::string::size_type>(needed), '\0');
+    WideCharToMultiByte(CP_UTF8, 0, wide.c_str(), static_cast<int>(wide.size()), out.data(),
+                        needed, nullptr, nullptr);
+    return out;
+}
+
+}  // namespace joc_path

tests/long_path_test.cpp

@@ -0,0 +1,172 @@
+// Checks that the component works with a path past MAX_PATH.
+//
+//   long_path_test
+//
+// Windows refuses a path of MAX_PATH characters or more to any Win32 call that does
+// not carry the \\?\ prefix.  The prefix is a property of the path rather than of the
+// process, so it has to be added by whoever calls the API -- which is why a file that
+// plays from the desktop stops being found once its folder tree is deep enough, the
+// shape an Atmos download under a long album title has.
+//
+// The path is built here rather than taken from the command line, so the test needs
+// no fixture and cannot be defeated by the ANSI argv a console tool is handed.
+
+#include <windows.h>
+
+#include <cstdio>
+#include <string>
+
+#include "../src/container_scan.h"
+#include "../src/joc_decode.h"
+#include "../src/win_path.h"
+
+namespace {
+
+int failures = 0;
+
+void check(bool ok, const char* what) {
+    std::printf("%-4s %s\n", ok ? "ok" : "FAIL", what);
+    if (!ok) ++failures;
+}
+
+// %TEMP%\joc_long_path
+std::string temp_root() {
+    wchar_t buffer[MAX_PATH + 1] = {};
+    const DWORD length = GetTempPathW(MAX_PATH, buffer);
+    if (length == 0) return {};
+    return joc_path::to_utf8(std::wstring(buffer, length) + L"joc_long_path");
+}
+
+// One 45-character directory name.  Five of them put the file comfortably past the
+// limit whatever %TEMP% is.
+std::string level_name(int index) {
+    return "level-" + std::string(39, static_cast<char>('a' + index));
+}
+
+bool make_directory(const std::string& path) {
+    if (CreateDirectoryW(joc_path::to_wide_extended(path).c_str(), nullptr)) return true;
+    return GetLastError() == ERROR_ALREADY_EXISTS;
+}
+
+// Builds the deep tree one level at a time, so every parent already exists by the
+// time its child is asked for.
+std::string build_tree() {
+    std::string path = temp_root();
+    if (path.empty() || !make_directory(path)) return {};
+    for (int i = 0; i < 5; ++i) {
+        path += "\\" + level_name(i);
+        if (!make_directory(path)) return {};
+    }
+    return path;
+}
+
+// A file that is a container but holds no audio track: enough for the probe to walk
+// its boxes and report something of its own, rather than "cannot open the file".
+bool write_container(const std::string& path) {
+    unsigned char bytes[32] = {};
+    bytes[3] = sizeof(bytes);  // box size, big-endian
+    const char type[] = "ftypisom";
+    for (int i = 0; i < 8; ++i) bytes[4 + i] = static_cast<unsigned char>(type[i]);
+    std::FILE* file = _wfopen(joc_path::to_wide_extended(path).c_str(), L"wb");
+    if (file == nullptr) return false;
+    const std::size_t written = std::fwrite(bytes, 1, sizeof(bytes), file);
+    std::fclose(file);
+    return written == sizeof(bytes);
+}
+
+void remove_tree(const std::string& root, const std::string& leaf) {
+    // Files before the directories that hold them, deepest directory first, and
+    // never above root.  Worth doing at all because a tree past MAX_PATH is one
+    // Explorer cannot delete: leaving it behind would be worse than the disk it
+    // occupies.
+    DeleteFileW(joc_path::to_wide_extended(leaf + "\\probe.m4a").c_str());
+    DeleteFileW(joc_path::to_wide_extended(root + "\\short.m4a").c_str());
+    std::string current = leaf;
+    while (current.size() > root.size()) {
+        RemoveDirectoryW(joc_path::to_wide_extended(current).c_str());
+        const std::string::size_type slash = current.find_last_of('\\');
+        if (slash == std::string::npos) break;
+        current.resize(slash);
+    }
+    RemoveDirectoryW(joc_path::to_wide_extended(root).c_str());
+}
+
+// The rules that keep the prefix from being applied where it would break a path.
+void check_helper_rules() {
+    check(joc_path::to_wide_extended(std::string("ffmpeg")) == L"ffmpeg",
+          "a bare executable name is left for PATH to resolve");
+    check(joc_path::to_wide_extended(std::string("sub\\file.m4a")) == L"sub\\file.m4a",
+          "a relative path is left alone");
+    check(joc_path::to_wide_extended(std::string("C:\\a\\b.m4a")) == L"\\\\?\\C:\\a\\b.m4a",
+          "an absolute path gains the prefix");
+    check(joc_path::to_wide_extended(std::string("\\\\?\\C:\\a\\b.m4a")) ==
+              L"\\\\?\\C:\\a\\b.m4a",
+          "an already-prefixed path is not prefixed twice");
+    check(joc_path::to_wide_extended(std::string("\\\\server\\share\\b.m4a")) ==
+              L"\\\\?\\UNC\\server\\share\\b.m4a",
+          "a UNC path takes the UNC form");
+    check(joc_path::to_wide_extended(std::string("//server/share/b.m4a")) ==
+              L"\\\\?\\UNC\\server\\share\\b.m4a",
+          "and the same one in forward slashes, without doubling a separator");
+    check(joc_path::to_wide_extended(std::string("C:/a/b.m4a")) == L"\\\\?\\C:\\a\\b.m4a",
+          "forward slashes become backslashes, which the prefix stops normalizing");
+    check(joc_path::to_wide_extended(std::string("C:\\a\\..\\b.m4a")) == L"C:\\a\\..\\b.m4a",
+          "a .. segment is left alone, since the prefix would stop resolving it");
+    check(joc_path::to_wide_extended(std::string("C:/a/../b.m4a")) == L"C:/a/../b.m4a",
+          "and one in forward slashes is recognized as well");
+}
+
+}  // namespace
+
+int main() {
+    check_helper_rules();
+
+    const std::string tree = build_tree();
+    if (tree.empty()) {
+        std::printf("FAIL cannot build the tree under %s\n", temp_root().c_str());
+        return 1;
+    }
+    const std::string long_file = tree + "\\probe.m4a";
+    const std::string short_file = temp_root() + "\\short.m4a";
+    std::printf("long path  : %zu chars\n", long_file.size());
+    std::printf("short path : %zu chars\n", short_file.size());
+
+    check(long_file.size() > MAX_PATH, "the path is past MAX_PATH, so the test is on it");
+    check(write_container(long_file), "the file was written at the long path");
+    check(write_container(short_file), "the control file was written at the short path");
+
+    // The premise: without the prefix the very same path is refused.  If this ever
+    // starts passing, the machine lifts MAX_PATH for ordinary processes and the rest
+    // of the test no longer proves anything.
+    const HANDLE raw = CreateFileW(joc_path::to_wide(long_file).c_str(), GENERIC_READ,
+                                   FILE_SHARE_READ, nullptr, OPEN_EXISTING,
+                                   FILE_ATTRIBUTE_NORMAL, nullptr);
+    check(raw == INVALID_HANDLE_VALUE, "the raw path is still refused without the prefix");
+    if (raw != INVALID_HANDLE_VALUE) CloseHandle(raw);
+
+    const joc_container::Result long_result = joc_container::scan(long_file);
+    std::printf("container probe: kind=%s detail=%s\n",
+                joc_container::kind_name(long_result.kind), long_result.detail.c_str());
+    check(long_result.kind == joc_container::Kind::kMp4,
+          "the container probe reads the long path");
+    check(long_result.detail != "cannot open the file",
+          "and did not fall back to the unreadable-file answer");
+
+    const joc_decode::FileProbe probe = joc_decode::probe_file(long_file);
+    std::printf("decode reader  : readable=%d detail=%s\n", probe.readable ? 1 : 0,
+                probe.detail.c_str());
+    check(probe.readable, "the decode-side reader opens the long path too");
+
+    const joc_container::Result short_result = joc_container::scan(short_file);
+    check(short_result.kind == joc_container::Kind::kMp4 && short_result.detail != "cannot open the file",
+          "a short path in the same tree still probes, prefix and all");
+
+    remove_tree(temp_root(), tree);
+
+    if (failures == 0) {
+        std::printf("long paths are handled\n");
+        return 0;
+    }
+    std::printf("%d check(s) failed\n", failures);
+    return 1;
+}

tools/build_tests.ps1

@@ -41,6 +41,7 @@ $lines = @(
     # log.cpp comes along because the engine writes its diagnostics through it.
     "cl $common /Fe:`"$outDir\container_scan_test.exe`" /Fo:`"$outDir\\`" tests\container_scan_test.cpp src\container_scan.cpp src\log.cpp",
     "cl $common /Fe:`"$outDir\log_rotation_test.exe`" /Fo:`"$outDir\\`" tests\log_rotation_test.cpp src\log.cpp",
+    "cl $common /Fe:`"$outDir\long_path_test.exe`" /Fo:`"$outDir\\`" tests\long_path_test.cpp src\container_scan.cpp src\joc_decode.cpp src\eac3_scan.cpp src\log.cpp `"$coreLib`" shell32.lib",
     "cl $common /Fe:`"$outDir\scan_selftest.exe`" /Fo:`"$outDir\\`" tests\scan_selftest.cpp src\eac3_scan.cpp",
     "cl $common /Fe:`"$outDir\prefs_layout_check.exe`" /Fo:`"$outDir\\`" tests\prefs_layout_check.cpp user32.lib gdi32.lib",
     "cl $common /Fe:`"$outDir\scan_crosscheck.exe`" /Fo:`"$outDir\\`" tests\scan_crosscheck.cpp src\eac3_scan.cpp `"$coreLib`" shell32.lib",

tools/run.ps1

@@ -39,7 +39,10 @@ if (Test-Path -LiteralPath $running) {
 }
 
 $arguments = @()
-if ($Play) { $arguments += $Play }
+# Start-Process joins the array with spaces and quotes nothing itself, so a path with
+# a space in it would reach foobar2000 as several arguments and nothing would play --
+# silently, which is the worst way for a test bed to fail.
+if ($Play) { $arguments += ($Play | ForEach-Object { '"' + $_ + '"' }) }
 $process = Start-Process -FilePath $exe -ArgumentList $arguments -PassThru
 Write-Host "started pid=$($process.Id) $($Play -join ', ')"
 Start-Sleep -Seconds $WaitSeconds