TheM14/foo_input_joc
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 4 Wiki Activity

Init foo_input_joc
build / windows (push) Has been cancelled
Details

Browse Source
This commit is contained in:
TheM14
2026-09-25 17:07:04 +08:00
commit ad62a914e6
127 changed files with 38545 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
kernel/src/io/adm_writer.cpp
+241
View File
@@ -0,0 +1,241 @@
#include "io/adm_writer.h"
#include "foundation/fs_utf8.h"
#include <cstring>
#include <filesystem>
#include "io/wav_writer.h" // pack_int24 (shared int24 quantisation)
namespace joc::io {
namespace {
constexpr long kDs64BodyOffset = 20;
constexpr long kDataSizeOffset = 76;
void put_u16(std::string* out, std::uint16_t value) {
char buffer[2];
std::memcpy(buffer, &value, 2);
out->append(buffer, 2);
}
void put_u32(std::string* out, std::uint32_t value) {
char buffer[4];
std::memcpy(buffer, &value, 4);
out->append(buffer, 4);
}
void put_u64(std::string* out, std::uint64_t value) {
char buffer[8];
std::memcpy(buffer, &value, 8);
out->append(buffer, 8);
}
} // namespace
AdmBwfWriter::~AdmBwfWriter() { abort(); }
Status AdmBwfWriter::open(const std::string& path, std::size_t block_samples) {
if (block_samples < JOC_FRAME_SAMPLES) {
return Status::fail(JOC_ERR_INVALID_ARGUMENT, stage::kOutput,
"ADM block size must hold at least one E-AC-3 frame");
}
path_ = path;
block_samples_ = block_samples;
used_ = 0;
frames_ = 0;
finalized_ = false;
buffer_.assign(block_samples * kChannels, 0.0f);
file_ = fs_utf8::fopen(path, "wb+");
if (file_ == nullptr) {
std::error_code ignored;
const std::filesystem::path parent = std::filesystem::path(path).parent_path();
if (!parent.empty()) {
std::filesystem::create_directories(parent, ignored);
}
file_ = fs_utf8::fopen(path, "wb+");
}
if (file_ == nullptr) {
return Status::fail(JOC_ERR_OUTPUT_OPEN, stage::kOutput, "cannot open " + path);
}
std::string header;
header.append("RF64", 4);
put_u32(&header, 0xFFFFFFFFu);
header.append("WAVE", 4);
if (std::fwrite(header.data(), 1, header.size(), file_) != header.size()) {
abort();
return Status::fail(JOC_ERR_OUTPUT_WRITE, stage::kOutput, "cannot write " + path);
}
Status status = write_chunk("ds64", std::string(28, '\0'));
if (!status.ok()) {
abort();
return status;
}
std::string fmt;
put_u16(&fmt, 1);
put_u16(&fmt, static_cast<std::uint16_t>(kChannels));
put_u32(&fmt, kRate);
put_u32(&fmt, kRate * kChannels * 3u);
put_u16(&fmt, static_cast<std::uint16_t>(kChannels * 3u));
put_u16(&fmt, 24);
status = write_chunk("fmt ", fmt);
if (!status.ok()) {
abort();
return status;
}
status = write_chunk("data", std::string());
if (!status.ok()) {
abort();
return status;
}
return Status::success();
}
Status AdmBwfWriter::write_chunk(const char id[4], const std::string& body) {
std::string header;
header.append(id, 4);
put_u32(&header, static_cast<std::uint32_t>(body.size()));
if (std::fwrite(header.data(), 1, header.size(), file_) != header.size()) {
return Status::fail(JOC_ERR_OUTPUT_WRITE, stage::kOutput, "chunk header write failed");
}
if (!body.empty() &&
std::fwrite(body.data(), 1, body.size(), file_) != body.size()) {
return Status::fail(JOC_ERR_OUTPUT_WRITE, stage::kOutput, "chunk body write failed");
}
if ((body.size() & 1u) != 0u) {
const char pad = '\0';
if (std::fwrite(&pad, 1, 1, file_) != 1) {
return Status::fail(JOC_ERR_OUTPUT_WRITE, stage::kOutput, "chunk padding write failed");
}
}
return Status::success();
}
Status AdmBwfWriter::flush() {
if (used_ == 0) {
return Status::success();
}
if (file_ == nullptr) {
return Status::fail(JOC_ERR_STATE, stage::kOutput, "ADM writer is not open");
}
packed_.clear();
pack_int24(buffer_.data(), used_, kChannels, &packed_);
if (std::fwrite(packed_.data(), 1, packed_.size(), file_) != packed_.size()) {
return Status::fail(JOC_ERR_OUTPUT_WRITE, stage::kOutput, "audio write failed for " + path_);
}
used_ = 0;
return Status::success();
}
Status AdmBwfWriter::write_objects16(const float* planar16) {
if (file_ == nullptr) {
return Status::fail(JOC_ERR_STATE, stage::kOutput, "ADM writer is not open");
}
if (planar16 == nullptr) {
return Status::fail(JOC_ERR_INVALID_ARGUMENT, stage::kOutput, "null frame");
}
std::size_t source = 0;
while (source < JOC_FRAME_SAMPLES) {
const std::size_t available = block_samples_ - used_;
const std::size_t count =
std::min(available, static_cast<std::size_t>(JOC_FRAME_SAMPLES) - source);
float* target = buffer_.data() + used_ * kChannels;
std::memset(target, 0, count * kChannels * sizeof(float));
for (std::size_t sample = 0; sample < count; ++sample) {
float* row = target + sample * kChannels;
row[3] = planar16[0u * JOC_FRAME_SAMPLES + source + sample];
for (std::size_t object = 0; object < 15u; ++object) {
row[10u + object] =
planar16[(object + 1u) * JOC_FRAME_SAMPLES + source + sample];
}
}
used_ += count;
source += count;
if (used_ == block_samples_) {
const Status status = flush();
if (!status.ok()) {
return status;
}
}
}
frames_ += JOC_FRAME_SAMPLES;
return Status::success();
}
Status AdmBwfWriter::finalize(const std::string& axml, const std::string& chna,
const std::string& dbmd) {
if (file_ == nullptr) {
return Status::fail(JOC_ERR_STATE, stage::kOutput, "ADM writer is not open");
}
if (finalized_) {
return Status::fail(JOC_ERR_STATE, stage::kOutput, "ADM writer already finalized");
}
Status status = flush();
if (!status.ok()) {
return status;
}
status = write_chunk("axml", axml);
if (!status.ok()) {
return status;
}
status = write_chunk("chna", chna);
if (!status.ok()) {
return status;
}
status = write_chunk("dbmd", dbmd);
if (!status.ok()) {
return status;
}
if (std::fseek(file_, 0, SEEK_END) != 0) {
return Status::fail(JOC_ERR_OUTPUT_WRITE, stage::kOutput, "seek failed for " + path_);
}
const long long total = std::ftell(file_);
if (total < 0) {
return Status::fail(JOC_ERR_OUTPUT_WRITE, stage::kOutput, "tell failed for " + path_);
}
const std::uint64_t data_len = frames_ * kChannels * 3u;
const std::uint32_t data_field =
data_len <= 0xFFFFFFFFull ? static_cast<std::uint32_t>(data_len) : 0xFFFFFFFFu;
if (std::fseek(file_, kDataSizeOffset, SEEK_SET) != 0) {
return Status::fail(JOC_ERR_OUTPUT_WRITE, stage::kOutput, "seek failed for " + path_);
}
char buffer[4];
std::memcpy(buffer, &data_field, 4);
if (std::fwrite(buffer, 1, 4, file_) != 4) {
return Status::fail(JOC_ERR_OUTPUT_WRITE, stage::kOutput, "data size patch failed");
}
std::string ds64;
put_u64(&ds64, static_cast<std::uint64_t>(total) - 8u);
put_u64(&ds64, data_len);
put_u64(&ds64, frames_);
put_u32(&ds64, 0);
if (std::fseek(file_, kDs64BodyOffset, SEEK_SET) != 0) {
return Status::fail(JOC_ERR_OUTPUT_WRITE, stage::kOutput, "seek failed for " + path_);
}
if (std::fwrite(ds64.data(), 1, ds64.size(), file_) != ds64.size()) {
return Status::fail(JOC_ERR_OUTPUT_WRITE, stage::kOutput, "ds64 patch failed");
}
finalized_ = true;
if (std::fclose(file_) != 0) {
file_ = nullptr;
return Status::fail(JOC_ERR_OUTPUT_WRITE, stage::kOutput, "close failed for " + path_);
}
file_ = nullptr;
return Status::success();
}
void AdmBwfWriter::abort() {
if (file_ != nullptr) {
std::fclose(file_);
file_ = nullptr;
}
if (!finalized_ && !path_.empty()) {
fs_utf8::remove(path_);
}
}
} // namespace joc::io
kernel/src/io/adm_writer.h
+52
View File
@@ -0,0 +1,52 @@
#pragma once
#include <cstdint>
#include <cstdio>
#include <string>
#include <vector>
#include "foundation/status.h"
#include "joc_core.h"
namespace joc::io {
class AdmBwfWriter {
public:
static constexpr std::uint32_t kChannels = 25;
static constexpr std::uint32_t kRate = 48000;
static constexpr std::size_t kDefaultBlockSamples = 131072;
AdmBwfWriter() = default;
~AdmBwfWriter();
AdmBwfWriter(const AdmBwfWriter&) = delete;
AdmBwfWriter& operator=(const AdmBwfWriter&) = delete;
Status open(const std::string& path, std::size_t block_samples = kDefaultBlockSamples);
Status write_objects16(const float* planar16);
Status finalize(const std::string& axml, const std::string& chna, const std::string& dbmd);
// Closes and removes a file that was never finalized (plan 28.3: abort must
void abort();
std::uint64_t frames() const { return frames_; }
bool open_ok() const { return file_ != nullptr; }
private:
Status write_chunk(const char id[4], const std::string& body);
Status flush();
std::FILE* file_ = nullptr;
std::string path_;
std::size_t block_samples_ = kDefaultBlockSamples;
std::size_t used_ = 0;
std::uint64_t frames_ = 0;
std::vector<float> buffer_;
std::string packed_;
bool finalized_ = false;
};
} // namespace joc::io
kernel/src/io/hdf5.cpp
+1430
View File
File diff suppressed because it is too large Load Diff
kernel/src/io/hdf5.h
+87
View File
@@ -0,0 +1,87 @@
#pragma once
#include <cstdint>
#include <memory>
#include <string>
#include <vector>
#include "foundation/status.h"
// Read-only subset of the HDF5 file format, sized for the SOFA files this
// project consumes: superblock v0, version 2 object headers, fractal-heap link
// and attribute storage, compact and contiguous datasets. The file is opened
// lazily: only the requested dataset's bytes are read into memory, everything
// else (superblock, object headers, heap blocks) is fetched on demand and the
// metadata that was parsed is cached by file address.
//
// Paths are HDF5 link paths ("Data.IR" is a single link name here, "Group/Set"
// walks two links); the empty path names the root group. Byte order is
// normalized on read, so callers never see the file's own endianness.
namespace joc::io {
enum class Hdf5Type {
Unknown,
Int8,
Int16,
Int32,
Int64,
UInt8,
UInt16,
UInt32,
UInt64,
Float32,
Float64,
String,
};
struct Hdf5TypeInfo {
Hdf5Type type = Hdf5Type::Unknown;
std::uint32_t size = 0; // bytes per element as stored in the file
bool big_endian = false;
bool is_signed = false;
};
struct Hdf5DatasetInfo {
std::vector<std::uint64_t> shape;
Hdf5TypeInfo type;
std::uint64_t element_count() const;
};
struct Hdf5Attribute {
Hdf5TypeInfo type;
std::vector<std::uint64_t> shape;
std::vector<std::uint8_t> raw; // C order, host byte order
std::string text; // decoded for fixed-length string attributes
};
class Hdf5File {
public:
Hdf5File();
~Hdf5File();
Hdf5File(Hdf5File&&) noexcept;
Hdf5File& operator=(Hdf5File&&) noexcept;
Hdf5File(const Hdf5File&) = delete;
Hdf5File& operator=(const Hdf5File&) = delete;
Status open(const std::string& path);
bool is_open() const;
// Names of the links of a group ("" is the root group).
Status links(const std::string& group_path, std::vector<std::string>* names) const;
bool has_dataset(const std::string& path) const;
Status dataset_info(const std::string& path, Hdf5DatasetInfo* out) const;
Status read_dataset_raw(const std::string& path, std::vector<std::uint8_t>* out) const;
Status read_dataset_double(const std::string& path, std::vector<double>* out) const;
Status attribute_names(const std::string& object_path, std::vector<std::string>* names) const;
Status attribute(const std::string& object_path, const std::string& name, Hdf5Attribute* out) const;
Status attribute_text(const std::string& object_path, const std::string& name, std::string* out) const;
private:
struct Impl;
std::unique_ptr<Impl> impl_;
};
} // namespace joc::io
kernel/src/io/inflate.cpp
+276
View File
@@ -0,0 +1,276 @@
#include "io/inflate.h"
#include <cstring>
namespace joc::io {
namespace {
class LsbBitReader {
public:
LsbBitReader(const std::uint8_t* data, std::size_t size) : data_(data), size_(size) {}
bool ok() const { return ok_; }
std::size_t byte_position() const { return position_ >> 3; }
std::uint32_t bits(unsigned count) {
std::uint32_t value = 0;
for (unsigned i = 0; i < count; ++i) {
if ((position_ >> 3) >= size_) {
ok_ = false;
return value;
}
const std::uint32_t bit = (data_[position_ >> 3] >> (position_ & 7u)) & 1u;
value |= bit << i;
++position_;
}
return value;
}
void align_to_byte() { position_ = (position_ + 7u) & ~static_cast<std::size_t>(7u); }
void skip_bytes(std::size_t count) { position_ += count * 8u; }
private:
const std::uint8_t* data_;
std::size_t size_;
std::size_t position_ = 0;
bool ok_ = true;
};
struct Huffman {
std::uint16_t counts[16] = {};
std::uint16_t symbols[288] = {};
int max_length = 0;
bool build(const std::uint8_t* lengths, int count) {
for (int i = 0; i < 16; ++i) {
counts[i] = 0;
}
for (int i = 0; i < count; ++i) {
counts[lengths[i]]++;
}
counts[0] = 0;
std::uint16_t offsets[16] = {};
std::uint16_t total = 0;
for (int length = 1; length < 16; ++length) {
offsets[length] = total;
total = static_cast<std::uint16_t>(total + counts[length]);
}
if (total == 0) {
return false;
}
for (int symbol = 0; symbol < count; ++symbol) {
const std::uint8_t length = lengths[symbol];
if (length != 0) {
symbols[offsets[length]++] = static_cast<std::uint16_t>(symbol);
}
}
max_length = 15;
while (max_length > 0 && counts[max_length] == 0) {
--max_length;
}
return max_length != 0;
}
int decode(LsbBitReader* reader) const {
int code = 0;
int first = 0;
int index = 0;
for (int length = 1; length <= max_length; ++length) {
code |= static_cast<int>(reader->bits(1));
if (!reader->ok()) {
return -1;
}
const int count = counts[length];
if (code - first < count) {
return symbols[index + (code - first)];
}
index += count;
first = (first + count) << 1;
code <<= 1;
}
return -1;
}
};
constexpr std::uint16_t kLengthBase[29] = {3, 4, 5, 6, 7, 8, 9, 10, 11, 13, 15, 17, 19,
23, 27, 31, 35, 43, 51, 59, 67, 83, 99, 115, 131, 163,
195, 227, 258};
constexpr std::uint8_t kLengthExtra[29] = {0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2,
2, 3, 3, 3, 3, 4, 4, 4, 4, 5, 5, 5, 5, 0};
constexpr std::uint16_t kDistanceBase[30] = {1, 2, 3, 4, 5, 7, 9, 13,
17, 25, 33, 49, 65, 97, 129, 193,
257, 385, 513, 769, 1025, 1537, 2049, 3073,
4097, 6145, 8193, 12289, 16385, 24577};
constexpr std::uint8_t kDistanceExtra[30] = {0, 0, 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6,
6, 7, 7, 8, 8, 9, 9, 10, 10, 11, 11, 12, 12, 13, 13};
constexpr std::uint8_t kCodeLengthOrder[19] = {16, 17, 18, 0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2,
14, 1, 15};
bool inflate_block_data(LsbBitReader* reader, const Huffman& literal, const Huffman& distance,
std::vector<std::uint8_t>* out) {
for (;;) {
const int symbol = literal.decode(reader);
if (symbol < 0) {
return false;
}
if (symbol < 256) {
out->push_back(static_cast<std::uint8_t>(symbol));
continue;
}
if (symbol == 256) {
return true;
}
const int length_index = symbol - 257;
if (length_index >= 29) {
return false;
}
const std::uint32_t length =
kLengthBase[length_index] + reader->bits(kLengthExtra[length_index]);
const int distance_symbol = distance.decode(reader);
if (distance_symbol < 0 || distance_symbol >= 30) {
return false;
}
const std::uint32_t distance_value =
kDistanceBase[distance_symbol] + reader->bits(kDistanceExtra[distance_symbol]);
if (!reader->ok() || distance_value == 0 || distance_value > out->size()) {
return false;
}
const std::size_t start = out->size() - distance_value;
for (std::uint32_t i = 0; i < length; ++i) {
out->push_back((*out)[start + i]);
}
}
}
bool inflate_fixed(LsbBitReader* reader, std::vector<std::uint8_t>* out) {
std::uint8_t lengths[288];
for (int i = 0; i < 144; ++i) { lengths[i] = 8; }
for (int i = 144; i < 256; ++i) { lengths[i] = 9; }
for (int i = 256; i < 280; ++i) { lengths[i] = 7; }
for (int i = 280; i < 288; ++i) { lengths[i] = 8; }
Huffman literal;
if (!literal.build(lengths, 288)) {
return false;
}
std::uint8_t distance_lengths[30];
for (int i = 0; i < 30; ++i) { distance_lengths[i] = 5; }
Huffman distance;
if (!distance.build(distance_lengths, 30)) {
return false;
}
return inflate_block_data(reader, literal, distance, out);
}
bool inflate_dynamic(LsbBitReader* reader, std::vector<std::uint8_t>* out) {
const int literal_count = static_cast<int>(reader->bits(5)) + 257;
const int distance_count = static_cast<int>(reader->bits(5)) + 1;
const int code_length_count = static_cast<int>(reader->bits(4)) + 4;
if (!reader->ok() || literal_count > 286 || distance_count > 30) {
return false;
}
std::uint8_t code_lengths[19] = {};
for (int i = 0; i < code_length_count; ++i) {
code_lengths[kCodeLengthOrder[i]] = static_cast<std::uint8_t>(reader->bits(3));
}
if (!reader->ok()) {
return false;
}
Huffman code_length_tree;
if (!code_length_tree.build(code_lengths, 19)) {
return false;
}
std::uint8_t lengths[288 + 30] = {};
const int total = literal_count + distance_count;
int index = 0;
while (index < total) {
const int symbol = code_length_tree.decode(reader);
if (symbol < 0) {
return false;
}
if (symbol < 16) {
lengths[index++] = static_cast<std::uint8_t>(symbol);
continue;
}
int repeat = 0;
std::uint8_t value = 0;
if (symbol == 16) {
if (index == 0) {
return false;
}
value = lengths[index - 1];
repeat = 3 + static_cast<int>(reader->bits(2));
} else if (symbol == 17) {
repeat = 3 + static_cast<int>(reader->bits(3));
} else {
repeat = 11 + static_cast<int>(reader->bits(7));
}
if (!reader->ok() || index + repeat > total) {
return false;
}
for (int i = 0; i < repeat; ++i) {
lengths[index++] = value;
}
}
Huffman literal;
if (!literal.build(lengths, literal_count)) {
return false;
}
Huffman distance;
if (!distance.build(lengths + literal_count, distance_count)) {
return false;
}
return inflate_block_data(reader, literal, distance, out);
}
} // namespace
bool inflate_raw(const std::uint8_t* data, std::size_t size, std::vector<std::uint8_t>* out) {
if (data == nullptr || out == nullptr) {
return false;
}
out->clear();
LsbBitReader reader(data, size);
for (;;) {
const std::uint32_t final_block = reader.bits(1);
const std::uint32_t type = reader.bits(2);
if (!reader.ok()) {
return false;
}
if (type == 0) {
reader.align_to_byte();
const std::size_t position = reader.byte_position();
if (position + 4 > size) {
return false;
}
const std::uint16_t length = static_cast<std::uint16_t>(data[position] | (data[position + 1] << 8));
const std::uint16_t complement =
static_cast<std::uint16_t>(data[position + 2] | (data[position + 3] << 8));
if (static_cast<std::uint16_t>(length ^ 0xFFFFu) != complement) {
return false;
}
if (position + 4 + length > size) {
return false;
}
out->insert(out->end(), data + position + 4, data + position + 4 + length);
reader.skip_bytes(4u + length);
} else if (type == 1) {
if (!inflate_fixed(&reader, out)) {
return false;
}
} else if (type == 2) {
if (!inflate_dynamic(&reader, out)) {
return false;
}
} else {
return false;
}
if (final_block != 0u) {
break;
}
}
return true;
}
} // namespace joc::io
kernel/src/io/inflate.h
+12
View File
@@ -0,0 +1,12 @@
#pragma once
#include <cstddef>
#include <cstdint>
#include <vector>
namespace joc::io {
bool inflate_raw(const std::uint8_t* data, std::size_t size, std::vector<std::uint8_t>* out);
} // namespace joc::io
kernel/src/io/npy.cpp
+420
View File
@@ -0,0 +1,420 @@
#include "io/npy.h"
#include <cstring>
namespace joc::io {
namespace {
std::uint16_t read_u16(const std::uint8_t* p) { return static_cast<std::uint16_t>(p[0] | (p[1] << 8)); }
std::uint32_t read_u32(const std::uint8_t* p) {
return static_cast<std::uint32_t>(p[0]) | (static_cast<std::uint32_t>(p[1]) << 8) |
(static_cast<std::uint32_t>(p[2]) << 16) | (static_cast<std::uint32_t>(p[3]) << 24);
}
NpyType classify(const std::string& descr) {
if (descr == "<f8" || descr == "=f8" || descr == "|f8") { return NpyType::Float64; }
if (descr == "<f4" || descr == "=f4") { return NpyType::Float32; }
if (descr == "<i8" || descr == "=i8") { return NpyType::Int64; }
if (descr == "<i4" || descr == "=i4") { return NpyType::Int32; }
if (descr == "<i2" || descr == "=i2") { return NpyType::Int16; }
if (descr == "|u1" || descr == "<u1") { return NpyType::UInt8; }
if (descr == "<c16" || descr == "=c16") { return NpyType::Complex128; }
if (descr.size() > 2 && descr[0] == '<' && descr[1] == 'U') {
return NpyType::Unicode;
}
if (descr.size() > 2 && descr[0] == '=' && descr[1] == 'U') {
return NpyType::Unicode;
}
return NpyType::Unknown;
}
std::size_t unicode_length(const std::string& descr) {
std::size_t index = 0;
while (index < descr.size() && (descr[index] == '<' || descr[index] == '=')) {
++index;
}
if (index >= descr.size() || descr[index] != 'U') {
return 0;
}
++index;
std::size_t value = 0;
bool any = false;
while (index < descr.size() && descr[index] >= '0' && descr[index] <= '9') {
value = value * 10 + static_cast<std::size_t>(descr[index] - '0');
++index;
any = true;
}
return any ? value : 0;
}
bool is_big_endian(const std::string& descr) { return !descr.empty() && descr[0] == '>'; }
bool header_value(const std::string& header, const std::string& key, std::string* out) {
const std::string needle = "'" + key + "'";
const std::size_t position = header.find(needle);
if (position == std::string::npos) {
return false;
}
const std::size_t colon = header.find(':', position + needle.size());
if (colon == std::string::npos) {
return false;
}
std::size_t start = colon + 1;
while (start < header.size() && (header[start] == ' ' || header[start] == '\t')) {
++start;
}
*out = header.substr(start);
return true;
}
} // namespace
std::size_t NpyArray::element_count() const {
std::size_t count = 1;
for (const std::int64_t dimension : shape) {
count *= static_cast<std::size_t>(dimension < 0 ? 0 : dimension);
}
return count;
}
std::size_t NpyArray::element_size() const {
switch (type) {
case NpyType::Float64: return 8;
case NpyType::Float32: return 4;
case NpyType::Int64: return 8;
case NpyType::Int32: return 4;
case NpyType::Int16: return 2;
case NpyType::UInt8: return 1;
case NpyType::Complex128: return 16;
case NpyType::Unicode: return item_bytes;
default: return 0;
}
}
bool parse_npy(const std::uint8_t* data, std::size_t size, NpyArray* out, std::string* error) {
if (data == nullptr || out == nullptr) {
return false;
}
const std::uint8_t magic[6] = {0x93u, 'N', 'U', 'M', 'P', 'Y'};
if (size < 10u || std::memcmp(data, magic, 6) != 0) {
if (error != nullptr) { *error = "not a .npy image"; }
return false;
}
const std::uint8_t major = data[6];
std::size_t header_length = 0;
std::size_t header_offset = 0;
if (major == 1u) {
header_length = read_u16(data + 8);
header_offset = 10;
} else if (major == 2u || major == 3u) {
if (size < 12u) {
if (error != nullptr) { *error = "truncated .npy v2 header"; }
return false;
}
header_length = read_u32(data + 8);
header_offset = 12;
} else {
if (error != nullptr) { *error = "unsupported .npy version " + std::to_string(major); }
return false;
}
if (header_offset + header_length > size) {
if (error != nullptr) { *error = "truncated .npy header"; }
return false;
}
const std::string header(reinterpret_cast<const char*>(data + header_offset), header_length);
out->descr.clear();
std::string value;
if (!header_value(header, "descr", &value)) {
if (error != nullptr) { *error = ".npy header without descr"; }
return false;
}
const std::size_t first_quote = value.find('\'');
const std::size_t second_quote =
first_quote == std::string::npos ? std::string::npos : value.find('\'', first_quote + 1);
if (first_quote == std::string::npos || second_quote == std::string::npos) {
if (error != nullptr) { *error = ".npy descr is not a quoted string"; }
return false;
}
out->descr = value.substr(first_quote + 1, second_quote - first_quote - 1);
out->type = classify(out->descr);
if (out->type == NpyType::Unknown) {
if (error != nullptr) { *error = "unsupported .npy dtype " + out->descr; }
return false;
}
out->item_bytes = 0;
if (out->type == NpyType::Unicode) {
const std::size_t length = unicode_length(out->descr);
if (length == 0) {
if (error != nullptr) { *error = "malformed unicode .npy dtype " + out->descr; }
return false;
}
out->item_bytes = length * 4u;
}
out->fortran_order = header.find("'fortran_order': True") != std::string::npos;
if (!header_value(header, "shape", &value)) {
if (error != nullptr) { *error = ".npy header without shape"; }
return false;
}
out->shape.clear();
for (std::size_t i = 0; i < value.size(); ++i) {
if (value[i] >= '0' && value[i] <= '9') {
long long dimension = 0;
while (i < value.size() && value[i] >= '0' && value[i] <= '9') {
dimension = dimension * 10 + (value[i] - '0');
++i;
}
out->shape.push_back(dimension);
} else if (value[i] == ')') {
break;
}
}
const std::size_t expected = out->element_count() * out->element_size();
if (header_offset + header_length + expected > size) {
if (error != nullptr) {
*error = ".npy payload truncated (need " + std::to_string(expected) + " bytes)";
}
return false;
}
out->data = data + header_offset + header_length;
out->data_bytes = expected;
return true;
}
bool npy_shape_is(const NpyArray& array, const std::vector<std::int64_t>& expected) {
return array.shape == expected;
}
namespace {
template <typename T>
void load_le(const std::uint8_t* source, std::size_t count, bool swap, std::vector<T>* out) {
out->resize(count);
std::memcpy(out->data(), source, count * sizeof(T));
if (swap) {
std::uint8_t* bytes = reinterpret_cast<std::uint8_t*>(out->data());
for (std::size_t i = 0; i < count; ++i) {
for (std::size_t b = 0; b < sizeof(T) / 2; ++b) {
const std::uint8_t temporary = bytes[i * sizeof(T) + b];
bytes[i * sizeof(T) + b] = bytes[i * sizeof(T) + sizeof(T) - 1 - b];
bytes[i * sizeof(T) + sizeof(T) - 1 - b] = temporary;
}
}
}
}
} // namespace
bool npy_to_double(const NpyArray& array, std::vector<double>* out, std::string* error) {
const bool swap = is_big_endian(array.descr);
const std::size_t count = array.element_count();
switch (array.type) {
case NpyType::Float64:
load_le(array.data, count, swap, out);
return true;
case NpyType::Complex128:
load_le(array.data, count * 2u, swap, out);
return true;
case NpyType::Float32: {
std::vector<float> values;
load_le(array.data, count, swap, &values);
out->resize(count);
for (std::size_t i = 0; i < count; ++i) {
(*out)[i] = static_cast<double>(values[i]);
}
return true;
}
case NpyType::Int64: {
std::vector<std::int64_t> values;
load_le(array.data, count, swap, &values);
out->resize(count);
for (std::size_t i = 0; i < count; ++i) {
(*out)[i] = static_cast<double>(values[i]);
}
return true;
}
case NpyType::Int32: {
std::vector<std::int32_t> values;
load_le(array.data, count, swap, &values);
out->resize(count);
for (std::size_t i = 0; i < count; ++i) {
(*out)[i] = static_cast<double>(values[i]);
}
return true;
}
case NpyType::Int16: {
std::vector<std::int16_t> values;
load_le(array.data, count, swap, &values);
out->resize(count);
for (std::size_t i = 0; i < count; ++i) {
(*out)[i] = static_cast<double>(values[i]);
}
return true;
}
case NpyType::UInt8: {
out->resize(count);
for (std::size_t i = 0; i < count; ++i) {
(*out)[i] = static_cast<double>(array.data[i]);
}
return true;
}
default:
if (error != nullptr) { *error = "cannot convert " + array.descr + " to double"; }
return false;
}
}
bool npy_to_int16(const NpyArray& array, std::vector<std::int16_t>* out, std::string* error) {
const bool swap = is_big_endian(array.descr);
const std::size_t count = array.element_count();
switch (array.type) {
case NpyType::Int16:
load_le(array.data, count, swap, out);
return true;
case NpyType::Int32: {
std::vector<std::int32_t> values;
load_le(array.data, count, swap, &values);
out->resize(count);
for (std::size_t i = 0; i < count; ++i) {
(*out)[i] = static_cast<std::int16_t>(values[i]);
}
return true;
}
case NpyType::Int64: {
std::vector<std::int64_t> values;
load_le(array.data, count, swap, &values);
out->resize(count);
for (std::size_t i = 0; i < count; ++i) {
(*out)[i] = static_cast<std::int16_t>(values[i]);
}
return true;
}
default:
if (error != nullptr) { *error = "cannot convert " + array.descr + " to int16"; }
return false;
}
}
bool npy_to_int32(const NpyArray& array, std::vector<std::int32_t>* out, std::string* error) {
const bool swap = is_big_endian(array.descr);
const std::size_t count = array.element_count();
switch (array.type) {
case NpyType::Int32:
load_le(array.data, count, swap, out);
return true;
case NpyType::Int64: {
std::vector<std::int64_t> values;
load_le(array.data, count, swap, &values);
out->resize(count);
for (std::size_t i = 0; i < count; ++i) {
(*out)[i] = static_cast<std::int32_t>(values[i]);
}
return true;
}
case NpyType::Int16: {
std::vector<std::int16_t> values;
load_le(array.data, count, swap, &values);
out->resize(count);
for (std::size_t i = 0; i < count; ++i) {
(*out)[i] = static_cast<std::int32_t>(values[i]);
}
return true;
}
default:
if (error != nullptr) { *error = "cannot convert " + array.descr + " to int32"; }
return false;
}
}
bool npy_to_uint8(const NpyArray& array, std::vector<std::uint8_t>* out, std::string* error) {
if (array.type != NpyType::UInt8) {
if (error != nullptr) { *error = "cannot convert " + array.descr + " to uint8"; }
return false;
}
out->assign(array.data, array.data + array.element_count());
return true;
}
bool npy_unicode_to_utf8(const NpyArray& array, std::string* out, std::string* error) {
if (array.type != NpyType::Unicode) {
if (error != nullptr) { *error = "not a unicode .npy member: " + array.descr; }
return false;
}
if (array.shape.size() != 0) {
if (error != nullptr) { *error = "unicode .npy member must be a scalar"; }
return false;
}
out->clear();
const std::size_t count = array.item_bytes / 4u;
for (std::size_t i = 0; i < count; ++i) {
const std::uint8_t* p = array.data + i * 4u;
const std::uint32_t code = static_cast<std::uint32_t>(p[0]) | (static_cast<std::uint32_t>(p[1]) << 8) |
(static_cast<std::uint32_t>(p[2]) << 16) |
(static_cast<std::uint32_t>(p[3]) << 24);
if (code == 0u) {
break;
}
if (code < 0x80u) {
out->push_back(static_cast<char>(code));
} else if (code < 0x800u) {
out->push_back(static_cast<char>(0xC0u | (code >> 6)));
out->push_back(static_cast<char>(0x80u | (code & 0x3Fu)));
} else if (code < 0x10000u) {
out->push_back(static_cast<char>(0xE0u | (code >> 12)));
out->push_back(static_cast<char>(0x80u | ((code >> 6) & 0x3Fu)));
out->push_back(static_cast<char>(0x80u | (code & 0x3Fu)));
} else {
out->push_back(static_cast<char>(0xF0u | (code >> 18)));
out->push_back(static_cast<char>(0x80u | ((code >> 12) & 0x3Fu)));
out->push_back(static_cast<char>(0x80u | ((code >> 6) & 0x3Fu)));
out->push_back(static_cast<char>(0x80u | (code & 0x3Fu)));
}
}
return true;
}
bool npy_to_c_order(const NpyArray& array, std::vector<std::uint8_t>* out, std::string* error) {
const std::size_t element = array.element_size();
if (element == 0) {
if (error != nullptr) { *error = "unsupported element size for " + array.descr; }
return false;
}
if (!array.fortran_order) {
out->assign(array.data, array.data + array.data_bytes);
return true;
}
const std::size_t dimensions = array.shape.size();
if (dimensions == 0) {
out->assign(array.data, array.data + element);
return true;
}
// Source (Fortran) strides in elements; destination is C order.
std::vector<std::size_t> source_stride(dimensions, 1);
std::size_t running = 1;
for (std::size_t d = 0; d < dimensions; ++d) {
source_stride[d] = running;
running *= static_cast<std::size_t>(array.shape[d]);
}
out->assign(array.data_bytes, 0);
std::vector<std::size_t> index(dimensions, 0);
const std::size_t total = array.element_count();
for (std::size_t linear = 0; linear < total; ++linear) {
std::size_t remainder = linear;
for (std::size_t d = dimensions; d-- > 0;) {
index[d] = remainder % static_cast<std::size_t>(array.shape[d]);
remainder /= static_cast<std::size_t>(array.shape[d]);
}
std::size_t source = 0;
for (std::size_t d = 0; d < dimensions; ++d) {
source += index[d] * source_stride[d];
}
std::memcpy(out->data() + linear * element, array.data + source * element, element);
}
return true;
}
} // namespace joc::io
kernel/src/io/npy.h
+42
View File
@@ -0,0 +1,42 @@
#pragma once
#include <cstdint>
#include <string>
#include <vector>
namespace joc::io {
enum class NpyType { Unknown, Float64, Float32, Int64, Int32, Int16, UInt8, Complex128, Unicode };
struct NpyArray {
std::string descr;
NpyType type = NpyType::Unknown;
bool fortran_order = false;
std::vector<std::int64_t> shape;
const std::uint8_t* data = nullptr;
std::size_t data_bytes = 0;
std::size_t item_bytes = 0; // bytes per element as stored
std::size_t element_count() const;
std::size_t element_size() const; // bytes per element in the file
};
// Parses the header of one `.npy` image. `data` must outlive the NpyArray.
bool parse_npy(const std::uint8_t* data, std::size_t size, NpyArray* out, std::string* error);
bool npy_to_double(const NpyArray& array, std::vector<double>* out, std::string* error);
bool npy_to_int16(const NpyArray& array, std::vector<std::int16_t>* out, std::string* error);
bool npy_to_int32(const NpyArray& array, std::vector<std::int32_t>* out, std::string* error);
bool npy_to_uint8(const NpyArray& array, std::vector<std::uint8_t>* out, std::string* error);
bool npy_unicode_to_utf8(const NpyArray& array, std::string* out, std::string* error);
// Materializes the array in C order as raw element bytes. Fortran-order members
// hybrid synthesis table as [count][4] row-major while the shipped table stores it
// Fortran-order, so passing the file bytes straight through would transpose it.
bool npy_to_c_order(const NpyArray& array, std::vector<std::uint8_t>* out, std::string* error);
bool npy_shape_is(const NpyArray& array, const std::vector<std::int64_t>& expected);
} // namespace joc::io
kernel/src/io/npy_writer.cpp
+263
View File
@@ -0,0 +1,263 @@
#include "io/npy_writer.h"
#include <array>
#include <charconv>
#include <cmath>
#include <cstdio>
#include <cstring>
#include <string>
#include "foundation/fs_utf8.h"
#include "io/zip_reader.h"
namespace joc::io {
namespace {
constexpr std::size_t kNpyHeaderAlignment = 64;
void append_u16(std::vector<std::uint8_t>* out, std::uint16_t value) {
out->push_back(static_cast<std::uint8_t>(value & 0xFFu));
out->push_back(static_cast<std::uint8_t>((value >> 8) & 0xFFu));
}
void append_u32(std::vector<std::uint8_t>* out, std::uint32_t value) {
for (int index = 0; index < 4; ++index) {
out->push_back(static_cast<std::uint8_t>((value >> (8 * index)) & 0xFFu));
}
}
void append_bytes(std::vector<std::uint8_t>* out, const void* data, std::size_t size) {
const std::uint8_t* bytes = static_cast<const std::uint8_t*>(data);
out->insert(out->end(), bytes, bytes + size);
}
std::string shape_literal(const std::vector<std::uint64_t>& shape) {
if (shape.empty()) {
return "()";
}
std::string text = "(";
for (std::size_t index = 0; index < shape.size(); ++index) {
if (index != 0u) {
text += ", ";
}
text += std::to_string(shape[index]);
}
if (shape.size() == 1u) {
text += ",";
}
text += ")";
return text;
}
} // namespace
std::vector<std::uint8_t> npy_image(const std::string& descr,
const std::vector<std::uint64_t>& shape,
const std::vector<std::uint8_t>& data) {
std::string header = "{'descr': '" + descr + "', 'fortran_order': False, 'shape': " +
shape_literal(shape) + ", }";
// NumPy pads the header so that the payload starts on a 64-byte boundary.
const std::size_t preamble = 10u; // magic, version, two byte header length
std::size_t total = preamble + header.size() + 1u;
const std::size_t padding = (kNpyHeaderAlignment - (total % kNpyHeaderAlignment)) %
kNpyHeaderAlignment;
header.append(padding, ' ');
header.push_back('\n');
std::vector<std::uint8_t> out;
out.reserve(preamble + header.size() + data.size());
static const std::uint8_t kMagic[6] = {0x93u, 'N', 'U', 'M', 'P', 'Y'};
append_bytes(&out, kMagic, sizeof(kMagic));
out.push_back(1u); // major
out.push_back(0u); // minor
append_u16(&out, static_cast<std::uint16_t>(header.size()));
append_bytes(&out, header.data(), header.size());
append_bytes(&out, data.data(), data.size());
return out;
}
std::vector<std::uint8_t> zip_bytes(const std::vector<NpyMember>& members) {
std::vector<std::uint8_t> out;
struct Entry {
std::string name;
std::uint32_t crc = 0;
std::uint32_t size = 0;
std::uint32_t offset = 0;
};
std::vector<Entry> entries;
entries.reserve(members.size());
for (const NpyMember& member : members) {
const std::string name = member.name + ".npy";
const std::vector<std::uint8_t> payload = npy_image(member.descr, member.shape, member.data);
Entry entry;
entry.name = name;
entry.crc = crc32_of(payload.data(), payload.size());
entry.size = static_cast<std::uint32_t>(payload.size());
entry.offset = static_cast<std::uint32_t>(out.size());
entries.push_back(entry);
append_u32(&out, 0x04034B50u); // local file header
append_u16(&out, 20u); // version needed
append_u16(&out, 0u); // flags
append_u16(&out, 0u); // method: stored
append_u16(&out, 0u); // time
append_u16(&out, 0x2821u); // date: 2000-01-01, fixed for reproducibility
append_u32(&out, entry.crc);
append_u32(&out, entry.size);
append_u32(&out, entry.size);
append_u16(&out, static_cast<std::uint16_t>(name.size()));
append_u16(&out, 0u); // extra length
append_bytes(&out, name.data(), name.size());
append_bytes(&out, payload.data(), payload.size());
}
const std::uint32_t directory_offset = static_cast<std::uint32_t>(out.size());
for (const Entry& entry : entries) {
append_u32(&out, 0x02014B50u); // central directory header
append_u16(&out, 20u); // version made by
append_u16(&out, 20u); // version needed
append_u16(&out, 0u); // flags
append_u16(&out, 0u); // method: stored
append_u16(&out, 0u); // time
append_u16(&out, 0x2821u); // date
append_u32(&out, entry.crc);
append_u32(&out, entry.size);
append_u32(&out, entry.size);
append_u16(&out, static_cast<std::uint16_t>(entry.name.size()));
append_u16(&out, 0u); // extra
append_u16(&out, 0u); // comment
append_u16(&out, 0u); // disk
append_u16(&out, 0u); // internal attributes
append_u32(&out, 0u); // external attributes
append_u32(&out, entry.offset);
append_bytes(&out, entry.name.data(), entry.name.size());
}
const std::uint32_t directory_size = static_cast<std::uint32_t>(out.size()) - directory_offset;
append_u32(&out, 0x06054B50u); // end of central directory
append_u16(&out, 0u);
append_u16(&out, 0u);
append_u16(&out, static_cast<std::uint16_t>(entries.size()));
append_u16(&out, static_cast<std::uint16_t>(entries.size()));
append_u32(&out, directory_size);
append_u32(&out, directory_offset);
append_u16(&out, 0u);
return out;
}
bool write_zip(const std::string& path, const std::vector<NpyMember>& members,
std::string* error) {
const std::vector<std::uint8_t> bytes = zip_bytes(members);
std::FILE* stream = fs_utf8::fopen(path, "wb");
if (stream == nullptr) {
if (error != nullptr) {
*error = "cannot open " + path + " for writing";
}
return false;
}
const std::size_t written = std::fwrite(bytes.data(), 1, bytes.size(), stream);
const bool flushed = std::fclose(stream) == 0;
if (written != bytes.size() || !flushed) {
if (error != nullptr) {
*error = "short write to " + path;
}
return false;
}
return true;
}
std::vector<std::uint8_t> utf8_to_utf32le(const std::string& text) {
std::vector<std::uint8_t> out;
out.reserve(text.size() * 4u);
std::size_t index = 0;
while (index < text.size()) {
const std::uint8_t lead = static_cast<std::uint8_t>(text[index]);
std::uint32_t code = 0;
std::size_t extra = 0;
if (lead < 0x80u) {
code = lead;
} else if ((lead & 0xE0u) == 0xC0u) {
code = lead & 0x1Fu;
extra = 1;
} else if ((lead & 0xF0u) == 0xE0u) {
code = lead & 0x0Fu;
extra = 2;
} else if ((lead & 0xF8u) == 0xF0u) {
code = lead & 0x07u;
extra = 3;
} else {
code = 0xFFFDu; // invalid lead byte: substitute rather than fail
extra = 0;
}
++index;
for (std::size_t count = 0; count < extra && index < text.size(); ++count) {
code = (code << 6) | (static_cast<std::uint8_t>(text[index]) & 0x3Fu);
++index;
}
for (int byte = 0; byte < 4; ++byte) {
out.push_back(static_cast<std::uint8_t>((code >> (8 * byte)) & 0xFFu));
}
}
return out;
}
std::string python_float_repr(double value) {
if (std::isnan(value)) {
return "NaN";
}
if (std::isinf(value)) {
return value > 0.0 ? "Infinity" : "-Infinity";
}
// to_chars gives the shortest round-trip digits; Python's repr uses the same
// digits but its own notation, so the digits are re-laid-out here.
std::array<char, 64> buffer{};
const std::to_chars_result converted =
std::to_chars(buffer.data(), buffer.data() + buffer.size(), value);
std::string text(buffer.data(), converted.ptr);
const bool negative = !text.empty() && text[0] == '-';
const std::string body = negative ? text.substr(1) : text;
const std::size_t exponent_at = body.find_first_of("eE");
std::string digits = body;
int exponent = 0;
if (exponent_at != std::string::npos) {
digits = body.substr(0, exponent_at);
exponent = std::atoi(body.c_str() + exponent_at + 1);
}
const std::size_t point = digits.find('.');
std::string mantissa = digits;
if (point != std::string::npos) {
mantissa = digits.substr(0, point) + digits.substr(point + 1);
exponent += static_cast<int>(point) - 1;
} else {
exponent += static_cast<int>(digits.size()) - 1;
}
while (mantissa.size() > 1u && mantissa.back() == '0') {
mantissa.pop_back();
}
// Python switches to exponent notation below 1e-4 and at 1e16 and above.
std::string result;
if (exponent < -4 || exponent >= 16) {
result = mantissa.substr(0, 1);
if (mantissa.size() > 1u) {
result += "." + mantissa.substr(1);
}
char tail[16];
std::snprintf(tail, sizeof(tail), "e%+03d", exponent);
result += tail;
} else if (exponent >= 0) {
if (static_cast<std::size_t>(exponent) + 1u >= mantissa.size()) {
result = mantissa + std::string(static_cast<std::size_t>(exponent) + 1u - mantissa.size(), '0');
result += ".0";
} else {
result = mantissa.substr(0, static_cast<std::size_t>(exponent) + 1u) + "." +
mantissa.substr(static_cast<std::size_t>(exponent) + 1u);
}
} else {
result = "0." + std::string(static_cast<std::size_t>(-exponent - 1), '0') + mantissa;
}
return negative ? "-" + result : result;
}
} // namespace joc::io
kernel/src/io/npy_writer.h
+39
View File
@@ -0,0 +1,39 @@
#pragma once
#include <cstdint>
#include <string>
#include <vector>
namespace joc::io {
// NPY 1.0 images and a minimal ZIP container, used to write the compiled HRTF
// cache in exactly the layout the reader (and NumPy) expects. Only what the
// cache needs is implemented: little-endian C-order arrays and stored members.
struct NpyMember {
std::string name; // archive member name, without the .npy suffix
std::string descr; // NumPy dtype string, e.g. "<f8", "<c16", "<U123"
std::vector<std::uint64_t> shape;
std::vector<std::uint8_t> data; // C order payload in the dtype's byte order
};
// Serializes one array as an NPY 1.0 image (magic, header, 64-byte aligned).
std::vector<std::uint8_t> npy_image(const std::string& descr,
const std::vector<std::uint64_t>& shape,
const std::vector<std::uint8_t>& data);
// Writes a ZIP archive with stored (uncompressed) members. The upstream reader
// accepts stored members, and compression would need a deflate encoder.
bool write_zip(const std::string& path, const std::vector<NpyMember>& members,
std::string* error);
// Serializes the archive in memory (same layout as write_zip).
std::vector<std::uint8_t> zip_bytes(const std::vector<NpyMember>& members);
// UTF-8 text as the payload of a NumPy Unicode scalar string ('<U<n>').
std::vector<std::uint8_t> utf8_to_utf32le(const std::string& text);
// Python's repr() for a double: shortest round-trip digits with Python's
// exponent rules, which is what json.dumps emits for the cache metadata.
std::string python_float_repr(double value);
} // namespace joc::io
kernel/src/io/process.cpp
+186
View File
@@ -0,0 +1,186 @@
#include "io/process.h"
#include "foundation/fs_utf8.h"
#include <cstdio>
#include <filesystem>
#include <fstream>
#include <random>
#if defined(_WIN32)
#define WIN32_LEAN_AND_MEAN
#define NOMINMAX
#include <windows.h>
#else
#include <sys/wait.h>
#endif
namespace joc::io {
namespace {
std::string quote_argument(const std::string& argument) {
if (!argument.empty() && argument.find_first_of(" \t\"") == std::string::npos) {
return argument;
}
std::string quoted = "\"";
unsigned backslashes = 0;
for (const char c : argument) {
if (c == '\\') {
++backslashes;
continue;
}
if (c == '"') {
quoted.append(backslashes * 2 + 1, '\\');
quoted.push_back('"');
backslashes = 0;
continue;
}
quoted.append(backslashes, '\\');
backslashes = 0;
quoted.push_back(c);
}
quoted.append(backslashes * 2, '\\');
quoted.push_back('"');
return quoted;
}
std::string tail_of(const std::string& text, std::size_t limit) {
if (text.size() <= limit) {
return text;
}
return text.substr(text.size() - limit);
}
} // namespace
Status run_process(const std::vector<std::string>& argv, ProcessResult* out) {
if (out == nullptr || argv.empty()) {
return Status::fail(JOC_ERR_INVALID_ARGUMENT, stage::kOutput, "empty command");
}
out->output.clear();
out->exit_code = 0;
const std::filesystem::path log_path =
std::filesystem::temp_directory_path() /
("joc_process_" + std::to_string(std::random_device{}()) + ".log");
auto read_log = [&]() {
#if defined(_WIN32)
return; // the Windows branch reads the handle it opened
#else
std::ifstream log = fs_utf8::open_input(fs_utf8::from_path(log_path));
if (log) {
std::string text((std::istreambuf_iterator<char>(log)),
std::istreambuf_iterator<char>());
out->output = tail_of(text, 4096);
}
#endif
};
#if defined(_WIN32)
std::string command;
for (std::size_t i = 0; i < argv.size(); ++i) {
if (i != 0) {
command.push_back(' ');
}
command += quote_argument(argv[i]);
}
auto widen = [](const std::string& text) {
if (text.empty()) {
return std::wstring();
}
const int size = MultiByteToWideChar(CP_UTF8, 0, text.c_str(),
static_cast<int>(text.size()), nullptr, 0);
std::wstring wide(static_cast<std::size_t>(size), L'\0');
MultiByteToWideChar(CP_UTF8, 0, text.c_str(), static_cast<int>(text.size()), wide.data(),
size);
return wide;
};
const std::wstring wide_command = widen(command);
const std::wstring wide_log = widen(fs_utf8::from_path(log_path));
SECURITY_ATTRIBUTES attributes{};
attributes.nLength = sizeof(attributes);
attributes.bInheritHandle = TRUE;
// DELETE access plus FILE_FLAG_DELETE_ON_CLOSE means the log disappears when
// the last handle goes away - including when this process is killed, which
// would otherwise leave joc_process_*.log litter in the temp directory.
HANDLE log_handle = CreateFileW(
wide_log.c_str(), GENERIC_READ | GENERIC_WRITE | DELETE,
FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, &attributes, CREATE_ALWAYS,
FILE_ATTRIBUTE_NORMAL | FILE_FLAG_DELETE_ON_CLOSE, nullptr);
if (log_handle == INVALID_HANDLE_VALUE) {
return Status::fail(JOC_ERR_IO, stage::kOutput, "cannot create the process log file");
}
// A delete-on-close file cannot be reopened by name (it is delete-pending), so
// the child's output is read back through the handle it wrote to.
auto read_log_handle = [&]() {
LARGE_INTEGER start{};
start.QuadPart = 0;
if (!SetFilePointerEx(log_handle, start, nullptr, FILE_BEGIN)) {
return;
}
std::string text;
char buffer[1024];
DWORD count = 0;
while (ReadFile(log_handle, buffer, sizeof(buffer), &count, nullptr) && count > 0) {
text.append(buffer, count);
}
out->output = tail_of(text, 4096);
};
STARTUPINFOW startup{};
startup.cb = sizeof(startup);
startup.dwFlags = STARTF_USESTDHANDLES;
startup.hStdOutput = log_handle;
startup.hStdError = log_handle;
startup.hStdInput = GetStdHandle(STD_INPUT_HANDLE);
PROCESS_INFORMATION process{};
std::vector<wchar_t> mutable_command(wide_command.begin(), wide_command.end());
mutable_command.push_back(L'\0');
const BOOL started = CreateProcessW(nullptr, mutable_command.data(), nullptr, nullptr, TRUE,
CREATE_NO_WINDOW, nullptr, nullptr, &startup, &process);
if (!started) {
CloseHandle(log_handle); // delete-on-close removes the file
return Status::fail(JOC_ERR_LIBRARY_MISSING, stage::kOutput,
"cannot start " + argv[0] + " (is it on PATH?)");
}
WaitForSingleObject(process.hProcess, INFINITE);
DWORD exit_code = 0;
GetExitCodeProcess(process.hProcess, &exit_code);
CloseHandle(process.hThread);
CloseHandle(process.hProcess);
// Read the log before the delete-on-close handle goes away.
read_log_handle();
CloseHandle(log_handle);
out->exit_code = static_cast<std::uint32_t>(exit_code);
#else
std::string command;
for (std::size_t i = 0; i < argv.size(); ++i) {
if (i != 0) {
command.push_back(' ');
}
command += quote_argument(argv[i]);
}
command += " > " + quote_argument(fs_utf8::from_path(log_path)) + " 2>&1";
const int status = std::system(command.c_str());
// system() reports a wait status, not the child's exit code.
out->exit_code = status == -1 ? 127u
: WIFEXITED(status) ? static_cast<std::uint32_t>(WEXITSTATUS(status))
: 128u;
read_log();
std::error_code ignored;
std::filesystem::remove(log_path, ignored);
#endif
if (out->exit_code != 0) {
return Status::fail(JOC_ERR_INPUT_FORMAT, stage::kOutput,
argv[0] + " failed with exit code " + std::to_string(out->exit_code) +
(out->output.empty() ? "" : ": " + tail_of(out->output, 400)));
}
return Status::success();
}
} // namespace joc::io
kernel/src/io/process.h
+19
View File
@@ -0,0 +1,19 @@
#pragma once
#include <cstdint>
#include <string>
#include <vector>
#include "foundation/status.h"
namespace joc::io {
struct ProcessResult {
std::uint32_t exit_code = 0;
std::string output;
};
Status run_process(const std::vector<std::string>& argv, ProcessResult* out);
} // namespace joc::io
kernel/src/io/wav_writer.cpp
+242
View File
@@ -0,0 +1,242 @@
#include "io/wav_writer.h"
#include "foundation/fs_utf8.h"
#include <cmath>
#include <cstring>
#include <filesystem>
#include <limits>
#include <vector>
namespace joc::io {
namespace {
constexpr std::uint16_t kWaveFormatPcm = 0x0001;
constexpr std::uint16_t kWaveFormatIeeeFloat = 0x0003;
constexpr std::uint16_t kWaveFormatExtensible = 0xFFFE;
constexpr std::uint8_t kPcmGuid[16] = {0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x00,
0x80, 0x00, 0x00, 0xAA, 0x00, 0x38, 0x9B, 0x71};
constexpr std::uint8_t kFloatGuid[16] = {0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x00,
0x80, 0x00, 0x00, 0xAA, 0x00, 0x38, 0x9B, 0x71};
void put_u16(std::string* out, std::uint16_t value) {
char buffer[2];
std::memcpy(buffer, &value, 2);
out->append(buffer, 2);
}
void put_u32(std::string* out, std::uint32_t value) {
char buffer[4];
std::memcpy(buffer, &value, 4);
out->append(buffer, 4);
}
void put_u64(std::string* out, std::uint64_t value) {
char buffer[8];
std::memcpy(buffer, &value, 8);
out->append(buffer, 8);
}
// Port of speaker_wav._fmt_chunk.
std::string fmt_chunk(std::uint32_t channels, std::uint32_t rate, SampleFormat format, WavInfo* info) {
std::uint16_t simple_tag = 0;
const std::uint8_t* guid = nullptr;
if (format == SampleFormat::Float32) {
info->bits_per_sample = 32;
info->bytes_per_sample = 4;
simple_tag = kWaveFormatIeeeFloat;
guid = kFloatGuid;
} else {
info->bits_per_sample = 24;
info->bytes_per_sample = 3;
simple_tag = kWaveFormatPcm;
guid = kPcmGuid;
}
const std::uint32_t block_align = channels * info->bytes_per_sample;
const std::uint32_t byte_rate = rate * block_align;
info->block_align = block_align;
std::string body;
if (channels <= 2) {
put_u16(&body, simple_tag);
put_u16(&body, static_cast<std::uint16_t>(channels));
put_u32(&body, rate);
put_u32(&body, byte_rate);
put_u16(&body, static_cast<std::uint16_t>(block_align));
put_u16(&body, static_cast<std::uint16_t>(info->bits_per_sample));
} else {
put_u16(&body, kWaveFormatExtensible);
put_u16(&body, static_cast<std::uint16_t>(channels));
put_u32(&body, rate);
put_u32(&body, byte_rate);
put_u16(&body, static_cast<std::uint16_t>(block_align));
put_u16(&body, static_cast<std::uint16_t>(info->bits_per_sample));
put_u16(&body, 22);
put_u16(&body, static_cast<std::uint16_t>(info->bits_per_sample));
put_u32(&body, 0);
body.append(reinterpret_cast<const char*>(guid), 16);
}
return body;
}
// int32 conversion identical to NumPy's float32 -> int32 cast after clipping.
std::int32_t to_int32(const float value) {
if (!std::isfinite(value)) {
return std::numeric_limits<std::int32_t>::min();
}
return static_cast<std::int32_t>(value);
}
} // namespace
void pack_int24(const float* interleaved, std::size_t frames, std::size_t channels,
std::string* out) {
const std::size_t count = frames * channels;
out->resize(count * 3);
char* target = out->data();
for (std::size_t i = 0; i < count; ++i) {
float value = interleaved[i];
if (value > 1.0f) {
value = 1.0f;
} else if (value < -1.0f) {
value = -1.0f;
}
const std::int32_t scaled = to_int32(value * 8388607.0f);
const std::uint32_t bits = static_cast<std::uint32_t>(scaled);
target[i * 3 + 0] = static_cast<char>(bits & 0xFFu);
target[i * 3 + 1] = static_cast<char>((bits >> 8) & 0xFFu);
target[i * 3 + 2] = static_cast<char>((bits >> 16) & 0xFFu);
}
}
WavWriter::~WavWriter() {
if (file_ != nullptr) {
std::fclose(file_);
file_ = nullptr;
}
}
Status WavWriter::open(const std::string& path, std::uint32_t channels, std::uint32_t rate,
SampleFormat format, std::uint64_t total_frames) {
if (channels == 0 || rate == 0) {
return Status::fail(JOC_ERR_INVALID_ARGUMENT, stage::kOutput,
"WAV writer needs a positive channel count and rate");
}
path_ = path;
channels_ = channels;
total_frames_ = total_frames;
frames_written_ = 0;
finalized_ = false;
info_ = WavInfo{};
info_.format = format;
const std::string fmt = fmt_chunk(channels, rate, format, &info_);
const std::uint64_t data_size = total_frames * info_.block_align;
info_.data_bytes = data_size;
const std::uint64_t riff_file_size = 12u + 8u + fmt.size() + 8u + data_size;
const bool rf64 = (riff_file_size - 8u) > 0xFFFFFFFFull;
info_.rf64 = rf64;
std::string header;
if (rf64) {
const std::uint64_t file_size = 12u + 36u + 8u + fmt.size() + 8u + data_size;
header.append("RF64", 4);
put_u32(&header, 0xFFFFFFFFu);
header.append("WAVE", 4);
header.append("ds64", 4);
put_u32(&header, 28);
put_u64(&header, file_size - 8u);
put_u64(&header, data_size);
put_u64(&header, total_frames);
put_u32(&header, 0);
} else {
header.append("RIFF", 4);
put_u32(&header, static_cast<std::uint32_t>(riff_file_size - 8u));
header.append("WAVE", 4);
}
header.append("fmt ", 4);
put_u32(&header, static_cast<std::uint32_t>(fmt.size()));
header.append(fmt);
header.append("data", 4);
put_u32(&header, rf64 ? 0xFFFFFFFFu : static_cast<std::uint32_t>(data_size));
file_ = fs_utf8::fopen(path, "wb");
if (file_ == nullptr) {
// The reference creates the parent directory itself.
std::error_code ignored;
const std::filesystem::path parent = std::filesystem::path(path).parent_path();
if (!parent.empty()) {
std::filesystem::create_directories(parent, ignored);
}
file_ = fs_utf8::fopen(path, "wb");
}
if (file_ == nullptr) {
return Status::fail(JOC_ERR_OUTPUT_OPEN, stage::kOutput, "cannot open " + path);
}
if (std::fwrite(header.data(), 1, header.size(), file_) != header.size()) {
std::fclose(file_);
file_ = nullptr;
return Status::fail(JOC_ERR_OUTPUT_WRITE, stage::kOutput, "cannot write header to " + path);
}
return Status::success();
}
Status WavWriter::write(const double* interleaved, std::size_t frames) {
if (file_ == nullptr) {
return Status::fail(JOC_ERR_STATE, stage::kOutput, "WAV writer is not open");
}
if (frames == 0) {
return Status::success();
}
if (frames_written_ + frames > total_frames_) {
return Status::fail(JOC_ERR_OUTPUT_WRITE, stage::kOutput,
"WAV writer received more frames than the header declared (declared " +
std::to_string(total_frames_) + ", written " +
std::to_string(frames_written_) + ", requested " +
std::to_string(frames) + ")");
}
const std::size_t count = frames * channels_;
if (info_.format == SampleFormat::Float32) {
std::vector<float> converted(count);
for (std::size_t i = 0; i < count; ++i) {
converted[i] = static_cast<float>(interleaved[i]);
}
if (std::fwrite(converted.data(), sizeof(float), count, file_) != count) {
return Status::fail(JOC_ERR_OUTPUT_WRITE, stage::kOutput, "write failed for " + path_);
}
} else {
std::vector<float> converted(count);
for (std::size_t i = 0; i < count; ++i) {
converted[i] = static_cast<float>(interleaved[i]);
}
std::string packed;
pack_int24(converted.data(), frames, channels_, &packed);
if (std::fwrite(packed.data(), 1, packed.size(), file_) != packed.size()) {
return Status::fail(JOC_ERR_OUTPUT_WRITE, stage::kOutput, "write failed for " + path_);
}
}
frames_written_ += frames;
return Status::success();
}
Status WavWriter::finalize() {
if (file_ == nullptr) {
return Status::fail(JOC_ERR_STATE, stage::kOutput, "WAV writer is not open");
}
if (frames_written_ != total_frames_) {
std::fclose(file_);
file_ = nullptr;
return Status::fail(JOC_ERR_OUTPUT_WRITE, stage::kOutput,
"WAV writer wrote " + std::to_string(frames_written_) + " of " +
std::to_string(total_frames_) + " frames");
}
const int result = std::fclose(file_);
file_ = nullptr;
finalized_ = true;
if (result != 0) {
return Status::fail(JOC_ERR_OUTPUT_WRITE, stage::kOutput, "close failed for " + path_);
}
return Status::success();
}
} // namespace joc::io
kernel/src/io/wav_writer.h
+61
View File
@@ -0,0 +1,61 @@
// Port of src/speaker_wav.py.
#pragma once
#include <cstdint>
#include <cstdio>
#include <string>
#include "foundation/status.h"
#include "joc_core.h"
namespace joc::io {
enum class SampleFormat { Float32, Int24 };
struct WavInfo {
SampleFormat format = SampleFormat::Float32;
std::uint32_t bits_per_sample = 32;
std::uint32_t bytes_per_sample = 4;
std::uint32_t block_align = 0;
std::uint64_t data_bytes = 0;
bool rf64 = false;
};
// int24 packing shared by the WAV and ADM writers:
// trunc(clip(v, -1, 1) * 8388607.0f) with the low three bytes written LE.
// NaN follows NumPy's float->int cast (INT32_MIN) so that the C++ conversion is
// never undefined; the reference passes it through unguarded (plan TD-3.11).
void pack_int24(const float* interleaved, std::size_t frames, std::size_t channels,
std::string* out);
class WavWriter {
public:
WavWriter() = default;
~WavWriter();
WavWriter(const WavWriter&) = delete;
WavWriter& operator=(const WavWriter&) = delete;
// `total_frames` must be known up front: the header depends on it.
Status open(const std::string& path, std::uint32_t channels, std::uint32_t rate,
SampleFormat format, std::uint64_t total_frames);
Status write(const double* interleaved, std::size_t frames);
Status finalize();
const WavInfo& info() const { return info_; }
std::uint64_t frames_written() const { return frames_written_; }
private:
std::FILE* file_ = nullptr;
std::string path_;
WavInfo info_;
std::uint32_t channels_ = 0;
std::uint64_t total_frames_ = 0;
std::uint64_t frames_written_ = 0;
bool finalized_ = false;
};
} // namespace joc::io
kernel/src/io/zip_reader.cpp
+215
View File
@@ -0,0 +1,215 @@
#include "io/zip_reader.h"
#include "foundation/fs_utf8.h"
#include <cstdio>
#include <cstring>
#include "io/inflate.h"
namespace joc::io {
namespace {
constexpr std::uint32_t kLocalHeaderSignature = 0x04034b50u;
constexpr std::uint32_t kCentralHeaderSignature = 0x02014b50u;
constexpr std::uint32_t kEndOfCentralDirectory = 0x06054b50u;
std::uint16_t read_u16(const std::uint8_t* p) {
return static_cast<std::uint16_t>(p[0] | (p[1] << 8));
}
std::uint32_t read_u32(const std::uint8_t* p) {
return static_cast<std::uint32_t>(p[0]) | (static_cast<std::uint32_t>(p[1]) << 8) |
(static_cast<std::uint32_t>(p[2]) << 16) | (static_cast<std::uint32_t>(p[3]) << 24);
}
} // namespace
std::uint32_t crc32_of(const std::uint8_t* data, std::size_t size) {
static std::uint32_t table[256];
static bool ready = false;
if (!ready) {
for (std::uint32_t i = 0; i < 256; ++i) {
std::uint32_t value = i;
for (int bit = 0; bit < 8; ++bit) {
value = (value & 1u) ? (0xEDB88320u ^ (value >> 1)) : (value >> 1);
}
table[i] = value;
}
ready = true;
}
std::uint32_t crc = 0xFFFFFFFFu;
for (std::size_t i = 0; i < size; ++i) {
crc = table[(crc ^ data[i]) & 0xFFu] ^ (crc >> 8);
}
return crc ^ 0xFFFFFFFFu;
}
bool ZipArchive::open(const std::string& path, std::string* error) {
entries_.clear();
data_.clear();
std::FILE* file = fs_utf8::fopen(path, "rb");
if (file == nullptr) {
if (error != nullptr) {
*error = "cannot open " + path;
}
return false;
}
std::fseek(file, 0, SEEK_END);
const long long size = std::ftell(file);
std::fseek(file, 0, SEEK_SET);
if (size <= 0) {
std::fclose(file);
if (error != nullptr) {
*error = "empty file " + path;
}
return false;
}
data_.resize(static_cast<std::size_t>(size));
const std::size_t got = std::fread(data_.data(), 1, data_.size(), file);
std::fclose(file);
if (got != data_.size()) {
if (error != nullptr) {
*error = "short read on " + path;
}
return false;
}
std::size_t eocd = std::string::npos;
const std::size_t scan_start = data_.size() > 65557u ? data_.size() - 65557u : 0u;
for (std::size_t i = data_.size(); i-- > scan_start;) {
if (i + 4u <= data_.size() && read_u32(&data_[i]) == kEndOfCentralDirectory) {
eocd = i;
break;
}
if (i == 0) {
break;
}
}
if (eocd == std::string::npos || eocd + 22u > data_.size()) {
if (error != nullptr) {
*error = "not a zip archive (no end-of-central-directory)";
}
return false;
}
const std::uint16_t entry_count = read_u16(&data_[eocd + 10]);
const std::uint32_t directory_offset = read_u32(&data_[eocd + 16]);
if (directory_offset >= data_.size()) {
if (error != nullptr) {
*error = "central directory offset out of range";
}
return false;
}
std::size_t cursor = directory_offset;
for (std::uint16_t index = 0; index < entry_count; ++index) {
if (cursor + 46u > data_.size() || read_u32(&data_[cursor]) != kCentralHeaderSignature) {
if (error != nullptr) {
*error = "malformed central directory entry " + std::to_string(index);
}
return false;
}
ZipEntry entry;
entry.method = read_u16(&data_[cursor + 10]);
entry.crc32 = read_u32(&data_[cursor + 16]);
entry.compressed_size = read_u32(&data_[cursor + 20]);
entry.uncompressed_size = read_u32(&data_[cursor + 24]);
const std::uint16_t name_length = read_u16(&data_[cursor + 28]);
const std::uint16_t extra_length = read_u16(&data_[cursor + 30]);
const std::uint16_t comment_length = read_u16(&data_[cursor + 32]);
entry.local_header_offset = read_u32(&data_[cursor + 42]);
if (entry.compressed_size == 0xFFFFFFFFu || entry.uncompressed_size == 0xFFFFFFFFu ||
entry.local_header_offset == 0xFFFFFFFFu) {
if (error != nullptr) {
*error = "zip64 archives are not supported";
}
return false;
}
if (cursor + 46u + name_length > data_.size()) {
if (error != nullptr) {
*error = "member name out of range";
}
return false;
}
entry.name.assign(reinterpret_cast<const char*>(&data_[cursor + 46]), name_length);
entries_.push_back(std::move(entry));
cursor += 46u + name_length + extra_length + comment_length;
}
return true;
}
const ZipEntry* ZipArchive::find(const std::string& name) const {
for (const ZipEntry& entry : entries_) {
if (entry.name == name) {
return &entry;
}
}
return nullptr;
}
bool ZipArchive::extract(const ZipEntry& entry, std::vector<std::uint8_t>* out,
std::string* error) const {
if (out == nullptr) {
return false;
}
const std::size_t offset = entry.local_header_offset;
if (offset + 30u > data_.size() || read_u32(&data_[offset]) != kLocalHeaderSignature) {
if (error != nullptr) {
*error = "bad local header for " + entry.name;
}
return false;
}
const std::uint16_t name_length = read_u16(&data_[offset + 26]);
const std::uint16_t extra_length = read_u16(&data_[offset + 28]);
const std::size_t start = offset + 30u + name_length + extra_length;
if (start + entry.compressed_size > data_.size()) {
if (error != nullptr) {
*error = "member data out of range for " + entry.name;
}
return false;
}
if (entry.method == 0u) {
out->assign(data_.begin() + static_cast<std::ptrdiff_t>(start),
data_.begin() + static_cast<std::ptrdiff_t>(start + entry.compressed_size));
} else if (entry.method == 8u) {
if (!inflate_raw(&data_[start], entry.compressed_size, out)) {
if (error != nullptr) {
*error = "deflate error in " + entry.name;
}
return false;
}
} else {
if (error != nullptr) {
*error = "unsupported compression method " + std::to_string(entry.method) + " for " +
entry.name;
}
return false;
}
if (entry.uncompressed_size != 0u && out->size() != entry.uncompressed_size) {
if (error != nullptr) {
*error = "size mismatch for " + entry.name + " (" + std::to_string(out->size()) +
" vs " + std::to_string(entry.uncompressed_size) + ")";
}
return false;
}
if (entry.crc32 != 0u && crc32_of(out->data(), out->size()) != entry.crc32) {
if (error != nullptr) {
*error = "CRC mismatch for " + entry.name;
}
return false;
}
return true;
}
bool ZipArchive::read_member(const std::string& name, std::vector<std::uint8_t>* out,
std::string* error) const {
const ZipEntry* entry = find(name);
if (entry == nullptr) {
if (error != nullptr) {
*error = "member not found: " + name;
}
return false;
}
return extract(*entry, out, error);
}
} // namespace joc::io
kernel/src/io/zip_reader.h
+38
View File
@@ -0,0 +1,38 @@
#pragma once
#include <cstdint>
#include <string>
#include <vector>
namespace joc::io {
struct ZipEntry {
std::string name;
std::uint16_t method = 0;
std::uint32_t crc32 = 0;
std::uint32_t compressed_size = 0;
std::uint32_t uncompressed_size = 0;
std::uint32_t local_header_offset = 0;
};
class ZipArchive {
public:
bool open(const std::string& path, std::string* error);
const std::vector<ZipEntry>& entries() const { return entries_; }
const ZipEntry* find(const std::string& name) const;
bool extract(const ZipEntry& entry, std::vector<std::uint8_t>* out, std::string* error) const;
bool read_member(const std::string& name, std::vector<std::uint8_t>* out, std::string* error) const;
private:
std::vector<std::uint8_t> data_;
std::vector<ZipEntry> entries_;
};
std::uint32_t crc32_of(const std::uint8_t* data, std::size_t size);
} // namespace joc::io