From ea30786711cd5bad4b79b83f87252e0a0ed27123 Mon Sep 17 00:00:00 2001 From: TheM14 Date: Thu, 1 Oct 2026 14:07:34 +0800 Subject: [PATCH] Fix files past MAX_PATH not being claimed or decoded --- foo_input_joc.vcxproj | 1 + src/container_scan.cpp | 13 +-- src/joc_decode.cpp | 23 ++---- src/log.cpp | 23 ++---- src/win_path.h | 90 ++++++++++++++++++++ tests/long_path_test.cpp | 172 +++++++++++++++++++++++++++++++++++++++ tools/build_tests.ps1 | 1 + tools/run.ps1 | 5 +- 8 files changed, 286 insertions(+), 42 deletions(-) create mode 100644 src/win_path.h create mode 100644 tests/long_path_test.cpp diff --git a/foo_input_joc.vcxproj b/foo_input_joc.vcxproj index 740169b..44b9956 100644 --- a/foo_input_joc.vcxproj +++ b/foo_input_joc.vcxproj @@ -114,6 +114,7 @@ + diff --git a/src/container_scan.cpp b/src/container_scan.cpp index ef5503f..43936bc 100644 --- a/src/container_scan.cpp +++ b/src/container_scan.cpp @@ -8,20 +8,11 @@ #include #include "log.h" +#include "win_path.h" namespace joc_container { namespace { -std::wstring utf8_to_wide(const std::string& text) { - if (text.empty()) return {}; - const int needed = MultiByteToWideChar(CP_UTF8, 0, text.c_str(), - static_cast(text.size()), nullptr, 0); - std::wstring out(static_cast(needed), L'\0'); - MultiByteToWideChar(CP_UTF8, 0, text.c_str(), static_cast(text.size()), out.data(), - needed); - return out; -} - // A library scan probes every container in the library, and most of them are // declined; they also mostly share a directory, so the path would spend the log's // byte budget repeating the same prefix once per file. The file name is what @@ -37,7 +28,7 @@ std::string file_name_of(const std::string& path) { class Window { public: bool open(const std::string& path) { - const std::wstring wide = utf8_to_wide(path); + const std::wstring wide = joc_path::to_wide_extended(path); handle_ = CreateFileW(wide.c_str(), GENERIC_READ, FILE_SHARE_READ | FILE_SHARE_WRITE, nullptr, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, nullptr); if (handle_ == INVALID_HANDLE_VALUE) { diff --git a/src/joc_decode.cpp b/src/joc_decode.cpp index 59352f7..eb64342 100644 --- a/src/joc_decode.cpp +++ b/src/joc_decode.cpp @@ -14,6 +14,7 @@ #include "eac3_scan.h" #include "log.h" +#include "win_path.h" namespace joc_decode { namespace { @@ -78,18 +79,8 @@ struct CoreApi { const char*(JOC_CALL* error_name)(joc_error) = joc_error_name; }; -std::wstring utf8_to_wide(const std::string& text) { - if (text.empty()) return {}; - const int needed = MultiByteToWideChar(CP_UTF8, 0, text.c_str(), - static_cast(text.size()), nullptr, 0); - std::wstring out(static_cast(needed), L'\0'); - MultiByteToWideChar(CP_UTF8, 0, text.c_str(), static_cast(text.size()), out.data(), - needed); - return out; -} - bool file_exists(const std::string& path) { - const std::wstring wide = utf8_to_wide(path); + const std::wstring wide = joc_path::to_wide_extended(path); if (wide.empty()) return false; const DWORD attributes = GetFileAttributesW(wide.c_str()); return attributes != INVALID_FILE_ATTRIBUTES && @@ -147,11 +138,11 @@ public: FILE_SHARE_READ | FILE_SHARE_WRITE, &attributes, CREATE_ALWAYS, 0, nullptr); - std::wstring command = L"\"" + utf8_to_wide(ffmpeg_path) + L"\""; + std::wstring command = L"\"" + joc_path::to_wide_extended(ffmpeg_path) + L"\""; command += L" -hide_banner -loglevel error -nostdin -y "; command += input_arguments; // input options must precede -i command += L" -i \""; - command += utf8_to_wide(input_path); + command += joc_path::to_wide_extended(input_path); command += L"\" "; command += output_arguments; @@ -223,7 +214,7 @@ class InputFile { public: ~InputFile() { close(); } bool open(const std::string& path) { - handle_ = CreateFileW(utf8_to_wide(path).c_str(), GENERIC_READ, + handle_ = CreateFileW(joc_path::to_wide_extended(path).c_str(), GENERIC_READ, FILE_SHARE_READ | FILE_SHARE_WRITE, nullptr, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL | FILE_FLAG_SEQUENTIAL_SCAN, nullptr); return handle_ != INVALID_HANDLE_VALUE; @@ -448,8 +439,8 @@ bool probe_container_joc(const std::string& ffmpeg_path, const std::string& path std::uint64_t modified = 0; { WIN32_FILE_ATTRIBUTE_DATA data{}; - if (GetFileAttributesExW(utf8_to_wide(path).c_str(), GetFileExInfoStandard, &data) != - FALSE) { + if (GetFileAttributesExW(joc_path::to_wide_extended(path).c_str(), GetFileExInfoStandard, + &data) != FALSE) { size = (static_cast(data.nFileSizeHigh) << 32) | data.nFileSizeLow; modified = (static_cast(data.ftLastWriteTime.dwHighDateTime) << 32) | data.ftLastWriteTime.dwLowDateTime; diff --git a/src/log.cpp b/src/log.cpp index c0cfe0d..918e5d2 100644 --- a/src/log.cpp +++ b/src/log.cpp @@ -8,6 +8,8 @@ #include #include +#include "win_path.h" + namespace joc_log { namespace { @@ -47,33 +49,26 @@ std::wstring module_directory() { return dir; } -std::string to_utf8(const std::wstring& text) { - if (text.empty()) return {}; - const int needed = WideCharToMultiByte(CP_UTF8, 0, text.c_str(), - static_cast(text.size()), nullptr, 0, - nullptr, nullptr); - std::string out(static_cast(needed), '\0'); - WideCharToMultiByte(CP_UTF8, 0, text.c_str(), static_cast(text.size()), - out.data(), needed, nullptr, nullptr); - return out; -} - void open_locked() { g_opened = true; wchar_t from_env[4096] = {}; const DWORD env_length = GetEnvironmentVariableW(L"JOC_LOG", from_env, static_cast(4096)); + std::wstring plain; if (env_length > 0 && env_length < 4096) { - g_wide_path.assign(from_env, env_length); + plain.assign(from_env, env_length); } else { const std::wstring dir = module_directory(); if (dir.empty()) return; - g_wide_path = dir + L"\\joc_decoder.log"; + plain = dir + L"\\joc_decoder.log"; } + // g_path is the one the component reports and a user reads; the handles work on + // the extended form, because a portable install can sit deeper than MAX_PATH. + g_path = joc_path::to_utf8(plain); + g_wide_path = joc_path::to_wide_extended(plain); g_wide_rolled = g_wide_path + L".1"; - g_path = to_utf8(g_wide_path); g_file = _wfopen(g_wide_path.c_str(), L"wb"); if (g_file == nullptr) { g_path.clear(); diff --git a/src/win_path.h b/src/win_path.h new file mode 100644 index 0000000..306ddaf --- /dev/null +++ b/src/win_path.h @@ -0,0 +1,90 @@ +// Paths handed to the Win32 file APIs. +// +// Windows refuses a path of MAX_PATH characters or more to any call that does not +// carry the \\?\ prefix. The prefix is a property of the path, not of the process: +// a component cannot inherit it from the host's manifest, so whoever calls the API +// has to add it. A file that plays from the desktop therefore stops being found +// once its folder tree is deep enough, which is what this header exists to prevent. +// +// Everything in this component that hands a path to Win32 -- or puts one on an +// ffmpeg command line, which is the same call made by the child process -- goes +// through to_wide_extended(). What is deliberately not routed through it is the +// path the component keeps for itself: the one it logs, and the one it uses as the +// container-probe cache key. Those stay in their plain form, so a log line reads +// like a path a user can paste into Explorer. +#pragma once + +#include + +#include + +namespace joc_path { + +// UTF-8 -> UTF-16, for the W APIs. Every path crosses into the OS through here: +// the narrow CRT would convert it through the ANSI code page, which is what breaks +// names like "Les Fêtes d'Hébé". +inline std::wstring to_wide(const std::string& utf8) { + if (utf8.empty()) return {}; + const int needed = MultiByteToWideChar(CP_UTF8, 0, utf8.c_str(), + static_cast(utf8.size()), nullptr, 0); + if (needed <= 0) return {}; + std::wstring out(static_cast(needed), L'\0'); + MultiByteToWideChar(CP_UTF8, 0, utf8.c_str(), static_cast(utf8.size()), out.data(), + needed); + return out; +} + +// The same path in the form the Win32 file APIs need. Three kinds of path are left +// exactly as they came in, and each of them would be broken by a prefix: +// +// * one that already carries \\?\, which makes this idempotent; +// * one that is not fully qualified -- a relative path, or the ffmpeg setting +// when it is a bare executable name that PATH resolves. \\?\ is undefined for +// those; +// * one with a "." or ".." segment, because \\?\ switches off the normalization +// that would resolve it. +// +// The separators are settled first for the same reason: once the prefix is on, +// nothing turns a forward slash into a backslash any more, and both of the checks +// above have to see the path the way the file system will. +inline std::wstring to_wide_extended(const std::wstring& path) { + if (path.empty()) return path; + if (path.compare(0, 4, L"\\\\?\\") == 0) return path; + + std::wstring plain = path; + for (wchar_t& character : plain) { + if (character == L'/') character = L'\\'; + } + + const bool unc = plain.size() >= 2 && plain[0] == L'\\' && plain[1] == L'\\'; + const bool drive = plain.size() >= 3 && plain[1] == L':' && plain[2] == L'\\'; + if (!unc && !drive) return path; + if (plain.find(L"\\.\\") != std::wstring::npos || + plain.find(L"\\..\\") != std::wstring::npos) { + return path; + } + + // A UNC path drops its two leading separators; everything else is copied whole. + std::wstring out = unc ? L"\\\\?\\UNC\\" : L"\\\\?\\"; + out.append(plain, unc ? 2 : 0, std::wstring::npos); + return out; +} + +inline std::wstring to_wide_extended(const std::string& utf8) { + return to_wide_extended(to_wide(utf8)); +} + +// UTF-16 -> UTF-8, the direction the component reports paths in. +inline std::string to_utf8(const std::wstring& wide) { + if (wide.empty()) return {}; + const int needed = WideCharToMultiByte(CP_UTF8, 0, wide.c_str(), + static_cast(wide.size()), nullptr, 0, nullptr, + nullptr); + if (needed <= 0) return {}; + std::string out(static_cast(needed), '\0'); + WideCharToMultiByte(CP_UTF8, 0, wide.c_str(), static_cast(wide.size()), out.data(), + needed, nullptr, nullptr); + return out; +} + +} // namespace joc_path diff --git a/tests/long_path_test.cpp b/tests/long_path_test.cpp new file mode 100644 index 0000000..e332b7e --- /dev/null +++ b/tests/long_path_test.cpp @@ -0,0 +1,172 @@ +// Checks that the component works with a path past MAX_PATH. +// +// long_path_test +// +// Windows refuses a path of MAX_PATH characters or more to any Win32 call that does +// not carry the \\?\ prefix. The prefix is a property of the path rather than of the +// process, so it has to be added by whoever calls the API -- which is why a file that +// plays from the desktop stops being found once its folder tree is deep enough, the +// shape an Atmos download under a long album title has. +// +// The path is built here rather than taken from the command line, so the test needs +// no fixture and cannot be defeated by the ANSI argv a console tool is handed. + +#include + +#include +#include + +#include "../src/container_scan.h" +#include "../src/joc_decode.h" +#include "../src/win_path.h" + +namespace { + +int failures = 0; + +void check(bool ok, const char* what) { + std::printf("%-4s %s\n", ok ? "ok" : "FAIL", what); + if (!ok) ++failures; +} + +// %TEMP%\joc_long_path +std::string temp_root() { + wchar_t buffer[MAX_PATH + 1] = {}; + const DWORD length = GetTempPathW(MAX_PATH, buffer); + if (length == 0) return {}; + return joc_path::to_utf8(std::wstring(buffer, length) + L"joc_long_path"); +} + +// One 45-character directory name. Five of them put the file comfortably past the +// limit whatever %TEMP% is. +std::string level_name(int index) { + return "level-" + std::string(39, static_cast('a' + index)); +} + +bool make_directory(const std::string& path) { + if (CreateDirectoryW(joc_path::to_wide_extended(path).c_str(), nullptr)) return true; + return GetLastError() == ERROR_ALREADY_EXISTS; +} + +// Builds the deep tree one level at a time, so every parent already exists by the +// time its child is asked for. +std::string build_tree() { + std::string path = temp_root(); + if (path.empty() || !make_directory(path)) return {}; + for (int i = 0; i < 5; ++i) { + path += "\\" + level_name(i); + if (!make_directory(path)) return {}; + } + return path; +} + +// A file that is a container but holds no audio track: enough for the probe to walk +// its boxes and report something of its own, rather than "cannot open the file". +bool write_container(const std::string& path) { + unsigned char bytes[32] = {}; + bytes[3] = sizeof(bytes); // box size, big-endian + const char type[] = "ftypisom"; + for (int i = 0; i < 8; ++i) bytes[4 + i] = static_cast(type[i]); + std::FILE* file = _wfopen(joc_path::to_wide_extended(path).c_str(), L"wb"); + if (file == nullptr) return false; + const std::size_t written = std::fwrite(bytes, 1, sizeof(bytes), file); + std::fclose(file); + return written == sizeof(bytes); +} + +void remove_tree(const std::string& root, const std::string& leaf) { + // Files before the directories that hold them, deepest directory first, and + // never above root. Worth doing at all because a tree past MAX_PATH is one + // Explorer cannot delete: leaving it behind would be worse than the disk it + // occupies. + DeleteFileW(joc_path::to_wide_extended(leaf + "\\probe.m4a").c_str()); + DeleteFileW(joc_path::to_wide_extended(root + "\\short.m4a").c_str()); + std::string current = leaf; + while (current.size() > root.size()) { + RemoveDirectoryW(joc_path::to_wide_extended(current).c_str()); + const std::string::size_type slash = current.find_last_of('\\'); + if (slash == std::string::npos) break; + current.resize(slash); + } + RemoveDirectoryW(joc_path::to_wide_extended(root).c_str()); +} + +// The rules that keep the prefix from being applied where it would break a path. +void check_helper_rules() { + check(joc_path::to_wide_extended(std::string("ffmpeg")) == L"ffmpeg", + "a bare executable name is left for PATH to resolve"); + check(joc_path::to_wide_extended(std::string("sub\\file.m4a")) == L"sub\\file.m4a", + "a relative path is left alone"); + check(joc_path::to_wide_extended(std::string("C:\\a\\b.m4a")) == L"\\\\?\\C:\\a\\b.m4a", + "an absolute path gains the prefix"); + check(joc_path::to_wide_extended(std::string("\\\\?\\C:\\a\\b.m4a")) == + L"\\\\?\\C:\\a\\b.m4a", + "an already-prefixed path is not prefixed twice"); + check(joc_path::to_wide_extended(std::string("\\\\server\\share\\b.m4a")) == + L"\\\\?\\UNC\\server\\share\\b.m4a", + "a UNC path takes the UNC form"); + check(joc_path::to_wide_extended(std::string("//server/share/b.m4a")) == + L"\\\\?\\UNC\\server\\share\\b.m4a", + "and the same one in forward slashes, without doubling a separator"); + check(joc_path::to_wide_extended(std::string("C:/a/b.m4a")) == L"\\\\?\\C:\\a\\b.m4a", + "forward slashes become backslashes, which the prefix stops normalizing"); + check(joc_path::to_wide_extended(std::string("C:\\a\\..\\b.m4a")) == L"C:\\a\\..\\b.m4a", + "a .. segment is left alone, since the prefix would stop resolving it"); + check(joc_path::to_wide_extended(std::string("C:/a/../b.m4a")) == L"C:/a/../b.m4a", + "and one in forward slashes is recognized as well"); +} + +} // namespace + +int main() { + check_helper_rules(); + + const std::string tree = build_tree(); + if (tree.empty()) { + std::printf("FAIL cannot build the tree under %s\n", temp_root().c_str()); + return 1; + } + const std::string long_file = tree + "\\probe.m4a"; + const std::string short_file = temp_root() + "\\short.m4a"; + std::printf("long path : %zu chars\n", long_file.size()); + std::printf("short path : %zu chars\n", short_file.size()); + + check(long_file.size() > MAX_PATH, "the path is past MAX_PATH, so the test is on it"); + check(write_container(long_file), "the file was written at the long path"); + check(write_container(short_file), "the control file was written at the short path"); + + // The premise: without the prefix the very same path is refused. If this ever + // starts passing, the machine lifts MAX_PATH for ordinary processes and the rest + // of the test no longer proves anything. + const HANDLE raw = CreateFileW(joc_path::to_wide(long_file).c_str(), GENERIC_READ, + FILE_SHARE_READ, nullptr, OPEN_EXISTING, + FILE_ATTRIBUTE_NORMAL, nullptr); + check(raw == INVALID_HANDLE_VALUE, "the raw path is still refused without the prefix"); + if (raw != INVALID_HANDLE_VALUE) CloseHandle(raw); + + const joc_container::Result long_result = joc_container::scan(long_file); + std::printf("container probe: kind=%s detail=%s\n", + joc_container::kind_name(long_result.kind), long_result.detail.c_str()); + check(long_result.kind == joc_container::Kind::kMp4, + "the container probe reads the long path"); + check(long_result.detail != "cannot open the file", + "and did not fall back to the unreadable-file answer"); + + const joc_decode::FileProbe probe = joc_decode::probe_file(long_file); + std::printf("decode reader : readable=%d detail=%s\n", probe.readable ? 1 : 0, + probe.detail.c_str()); + check(probe.readable, "the decode-side reader opens the long path too"); + + const joc_container::Result short_result = joc_container::scan(short_file); + check(short_result.kind == joc_container::Kind::kMp4 && short_result.detail != "cannot open the file", + "a short path in the same tree still probes, prefix and all"); + + remove_tree(temp_root(), tree); + + if (failures == 0) { + std::printf("long paths are handled\n"); + return 0; + } + std::printf("%d check(s) failed\n", failures); + return 1; +} diff --git a/tools/build_tests.ps1 b/tools/build_tests.ps1 index 202ec52..17f538b 100644 --- a/tools/build_tests.ps1 +++ b/tools/build_tests.ps1 @@ -41,6 +41,7 @@ $lines = @( # log.cpp comes along because the engine writes its diagnostics through it. "cl $common /Fe:`"$outDir\container_scan_test.exe`" /Fo:`"$outDir\\`" tests\container_scan_test.cpp src\container_scan.cpp src\log.cpp", "cl $common /Fe:`"$outDir\log_rotation_test.exe`" /Fo:`"$outDir\\`" tests\log_rotation_test.cpp src\log.cpp", + "cl $common /Fe:`"$outDir\long_path_test.exe`" /Fo:`"$outDir\\`" tests\long_path_test.cpp src\container_scan.cpp src\joc_decode.cpp src\eac3_scan.cpp src\log.cpp `"$coreLib`" shell32.lib", "cl $common /Fe:`"$outDir\scan_selftest.exe`" /Fo:`"$outDir\\`" tests\scan_selftest.cpp src\eac3_scan.cpp", "cl $common /Fe:`"$outDir\prefs_layout_check.exe`" /Fo:`"$outDir\\`" tests\prefs_layout_check.cpp user32.lib gdi32.lib", "cl $common /Fe:`"$outDir\scan_crosscheck.exe`" /Fo:`"$outDir\\`" tests\scan_crosscheck.cpp src\eac3_scan.cpp `"$coreLib`" shell32.lib", diff --git a/tools/run.ps1 b/tools/run.ps1 index 77bcdc4..9b8b39d 100644 --- a/tools/run.ps1 +++ b/tools/run.ps1 @@ -39,7 +39,10 @@ if (Test-Path -LiteralPath $running) { } $arguments = @() -if ($Play) { $arguments += $Play } +# Start-Process joins the array with spaces and quotes nothing itself, so a path with +# a space in it would reach foobar2000 as several arguments and nothing would play -- +# silently, which is the worst way for a test bed to fail. +if ($Play) { $arguments += ($Play | ForEach-Object { '"' + $_ + '"' }) } $process = Start-Process -FilePath $exe -ArgumentList $arguments -PassThru Write-Host "started pid=$($process.Id) $($Play -join ', ')" Start-Sleep -Seconds $WaitSeconds