Files
JustOneCacophony/src/emdf.py
T

488 lines
19 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""从常见 E-AC-3 同步帧直接提取连续 EMDF 容器。
扫描器检查八种全局位对齐,定位 ``0x5838`` 同步字,验证容器长度并解析各
payload config,因此不要求 EMDF 在原始 E-AC-3 文件中按字节对齐。
本模块有意只覆盖“完整 EMDF 容器在一个同步帧中连续出现”的常见情形。不解析
E-AC-3 mantissa,也不重组被音频数据隔开的多个 skip-field 碎片;遇到这种输入会
明确报错,让上层决定是否使用兼容桥。
"""
from dataclasses import dataclass
import csv
import hashlib
from pathlib import Path
import numpy as np
from variant_error import UnsupportedVariantError
SYNCWORD = 0x5838
REQUIRED_JOC_IDS = frozenset((11, 14))
class EmdfError(ValueError):
"""EMDF 或其 E-AC-3 传输结构不符合本实现支持的范围。"""
class BitReader:
"""MSB-first 位读取器;位置以源数据的绝对 bit offset 表示。"""
def __init__(self, data, position=0, limit=None):
self.data = memoryview(data)
self.position = int(position)
self.limit = len(self.data) * 8 if limit is None else int(limit)
def read(self, count):
count = int(count)
if count < 0 or self.position + count > self.limit:
raise EmdfError(f"位流越界 @bit{self.position}, need={count}, limit={self.limit}")
value = 0
while count:
byte_pos = self.position >> 3
removed_left = self.position & 7
take = min(count, 8 - removed_left)
shift = 8 - removed_left - take
value = (value << take) | ((self.data[byte_pos] >> shift) & ((1 << take) - 1))
self.position += take
count -= take
return value
def skip(self, count):
self.read(count)
def read_bytes(self, count):
return bytes(self.read(8) for _ in range(count))
def variable_bits(reader, width, max_groups=8):
"""读取 EMDF ``variable_bits(width)`` 变长整数。"""
value = 0
for _ in range(max_groups):
value += reader.read(width)
more = reader.read(1)
if not more:
return value
value = (value + 1) << width
raise EmdfError(f"variable_bits({width}) 延伸组过多")
@dataclass(frozen=True)
class EmdfContainer:
start_bit: int
raw: bytes
payloads: dict
sample_offsets: dict
# payload_id -> group_id(未声明 groupid 的载荷为 None)。规格 §8.2/Table 56:
# 同一帧的 OAMD 与 JOC 载荷属于同一 group,凭此把一对载荷关联起来。
group_ids: dict = None
def _parse_at(data, start_bit):
"""在已知 syncword 的 bit offset 解析一个 EMDF 容器。"""
reader = BitReader(data, start_bit)
if reader.read(16) != SYNCWORD:
raise EmdfError(f"EMDF syncword 不匹配 @bit{start_bit}")
length = reader.read(16)
body_start = reader.position
body_end = body_start + length * 8
if body_end > reader.limit:
raise EmdfError(f"EMDF 容器越界 @bit{start_bit}: length={length}")
reader.limit = body_end
version = reader.read(2)
if version == 3:
version += variable_bits(reader, 2)
key_id = reader.read(3)
if key_id == 7:
key_id += variable_bits(reader, 3)
# TS 103 420 JOC 使用 version=0/key_id=0;严格限制也能排除音频中的伪 marker。
if version != 0 or key_id != 0:
raise EmdfError(f"不支持的 EMDF version/key_id: {version}/{key_id}")
payloads = {}
sample_offsets = {}
group_ids = {}
terminated = False
while reader.position + 5 <= body_end:
payload_id = reader.read(5)
if payload_id == 0:
terminated = True
break
if payload_id == 0x1F:
payload_id += variable_bits(reader, 5)
if payload_id in payloads:
raise EmdfError(f"同一 EMDF 容器重复 payload id {payload_id}")
has_sample_offset = bool(reader.read(1))
sample_offset = (reader.read(12) >> 1) if has_sample_offset else 0
if reader.read(1):
variable_bits(reader, 11) # duration
if reader.read(1):
group_ids[payload_id] = variable_bits(reader, 2) # group id
else:
group_ids[payload_id] = None
if reader.read(1):
reader.skip(8) # codec data
if not reader.read(1): # discard_unknown_payload
frame_aligned = False
if not has_sample_offset:
frame_aligned = bool(reader.read(1))
if frame_aligned:
reader.skip(2)
if has_sample_offset or frame_aligned:
reader.skip(7)
payload_size = variable_bits(reader, 8)
if reader.position + payload_size * 8 > body_end:
raise EmdfError(
f"payload id {payload_id} 越界: size={payload_size}, @bit{reader.position}")
payloads[payload_id] = reader.read_bytes(payload_size)
sample_offsets[payload_id] = sample_offset
if not terminated:
raise EmdfError("EMDF 容器缺少 payload id 0 终止符")
total_bytes = 4 + length
raw_reader = BitReader(data, start_bit, start_bit + total_bytes * 8)
raw = raw_reader.read_bytes(total_bytes)
return EmdfContainer(start_bit, raw, payloads, sample_offsets, group_ids)
def _marker_offsets(data):
"""以 NumPy 批量检查八种位移,返回可能的 0x5838 bit offsets。"""
source = np.frombuffer(data, dtype=np.uint8)
if source.size < 4:
return []
offsets = []
for shift in range(8):
if shift == 0:
aligned = source
else:
aligned = np.bitwise_or(
np.left_shift(source[:-1].astype(np.uint16), shift) & 0xFF,
np.right_shift(source[1:].astype(np.uint16), 8 - shift),
).astype(np.uint8)
hits = np.flatnonzero((aligned[:-1] == 0x58) & (aligned[1:] == 0x38))
offsets.extend(int(hit) * 8 + shift for hit in hits)
return sorted(offsets)
def find_joc_emdf(frame, usable=None, diagnostics=None, frame_number=None):
"""返回同步帧中唯一、顶层连续且包含 ID11/ID14 的 JOC EMDF 容器。
EMDF payload 是不透明字节串,其中可能自然出现另一个 ``0x5838``。若从这个
内嵌 marker 开始的后续随机位恰好也能通过容器语法探测,它仍不是一个独立的
transport 容器。因此,候选的起点一旦落在较早 JOC 容器的声明范围内,就只把
它记作内嵌伪候选,不参与“多个容器”的判定。
一个同步帧里可能出现多个顶层容器(同一帧可能同时带两套 ID11/ID14)。
``usable`` 给出“这个容器能否作为本帧元数据”的分值(0 = 不是,越大越可信):
只有唯一最高分时用它,其余记入 ``diagnostics``;没有候选或最高分并列仍按变体
报错,不猜。
"""
matches = []
offsets = _marker_offsets(frame)
parsed_candidates = []
parse_errors = []
for start_bit in offsets:
try:
container = _parse_at(frame, start_bit)
except EmdfError as exc:
if len(parse_errors) < 8:
parse_errors.append({"start_bit": start_bit, "error": str(exc)})
continue
parsed_candidates.append({
"start_bit": start_bit,
"payload_ids": list(container.payloads),
"payload_lengths": {str(k): len(v) for k, v in container.payloads.items()},
})
if REQUIRED_JOC_IDS.issubset(container.payloads):
matches.append(container)
if not matches:
raise UnsupportedVariantError(
"emdf_transport", "no_contiguous_joc_container",
"同步帧中未找到可连续解析且同时包含 ID11/ID14 的 EMDF 容器",
details={
"syncframe_bytes": len(frame),
"marker_bit_offsets": offsets,
"parsed_candidates": parsed_candidates,
"candidate_parse_errors": parse_errors,
"repair_hint": "检查 EMDF 是否跨多个 audio-block skip field 分片,或 payload config 是否变化",
})
top_level_matches = []
nested_matches = []
for container in sorted(matches, key=lambda item: item.start_bit):
parent = next((candidate for candidate in top_level_matches
if candidate.start_bit < container.start_bit <
candidate.start_bit + len(candidate.raw) * 8), None)
if parent is None:
top_level_matches.append(container)
else:
nested_matches.append({
"start_bit": container.start_bit,
"end_bit": container.start_bit + len(container.raw) * 8,
"parent_start_bit": parent.start_bit,
"parent_end_bit": parent.start_bit + len(parent.raw) * 8,
})
if len(top_level_matches) != 1:
starts = [item.start_bit for item in top_level_matches]
if usable is not None:
scored = [(usable(item), item) for item in top_level_matches]
best = max(score for score, _item in scored)
winners = [item for score, item in scored if score == best]
if best > 0 and len(winners) == 1:
if diagnostics is not None:
diagnostics.append({
"stage": "emdf",
"variant": "extra_container_ignored",
"frame": frame_number,
"message": "同步帧内有多个含 ID11/ID14 的 EMDF 容器,只有一个能作为本帧元数据",
"details": {"joc_container_start_bits": starts,
"used_start_bit": winners[0].start_bit,
"pair_score": best},
})
return winners[0]
raise UnsupportedVariantError(
"emdf_transport", "multiple_joc_containers",
"同步帧中存在多个可用 JOC EMDF,当前无法自动选择",
details={
"syncframe_bytes": len(frame),
"joc_container_start_bits": starts,
"nested_joc_candidates": nested_matches,
})
return top_level_matches[0]
def parse_container(data):
"""解析从 syncword 开始、已经重新按字节对齐保存的 EMDF 容器。"""
container = _parse_at(data, 0)
if len(container.raw) != len(data):
raise EmdfError(f"EMDF 文件尾有额外数据: parsed={len(container.raw)}, file={len(data)}")
return container
# AC-3 同步帧长度表(16-bit word),索引为 fscod(0=48 kHz、1=44.1 kHz、2=32 kHz)
# 再取 frmsizecod;与 A/52 的比特率表等价。
_AC3_FRAME_WORDS = (
(64, 64, 80, 80, 96, 96, 112, 112, 128, 128, 160, 160, 192, 192, 224, 224,
256, 256, 320, 320, 384, 384, 448, 448, 512, 512, 640, 640, 768, 768,
896, 896, 1024, 1024, 1152, 1152, 1280, 1280),
(69, 70, 87, 88, 104, 105, 121, 122, 139, 140, 174, 175, 208, 209, 243, 244,
278, 278, 348, 348, 417, 418, 487, 487, 557, 557, 696, 696, 835, 835,
975, 975, 1114, 1114, 1253, 1253, 1393, 1393),
(96, 96, 120, 120, 144, 144, 168, 168, 192, 192, 240, 240, 288, 288, 336, 336,
384, 384, 480, 480, 576, 576, 672, 672, 768, 768, 960, 960, 1152, 1152,
1344, 1344, 1536, 1536, 1728, 1728, 1920, 1920),
)
# E-AC-3 chanmap(TS 102 366 Annex E Table E2.5):字段的最高位对应 bit 0。
CHANMAP_LOCATIONS = (
"L", "C", "R", "Ls", "Rs", "Lc/Rc", "Lrs/Rrs", "Cs", "Ts", "Lsd/Rsd",
"Lw/Rw", "Vhl/Vhr", "Vhc", "Lts/Rts", "LFE2", "LFE",
)
# 单个 bit 展开为两个相邻声道的成对位置;展开顺序为先左后右。
CHANMAP_PAIR_BITS = frozenset((5, 6, 9, 10, 11, 13))
# A/52 acmod 对应的全频段声道位置(不含 LFE,按解码顺序)。
ACMOD_LOCATIONS = {
0: ("C",), # 1+1 双单声道
1: ("C",),
2: ("L", "R"),
3: ("L", "C", "R"),
4: ("L", "R", "Cs"),
5: ("L", "C", "R", "Cs"),
6: ("L", "R", "Ls", "Rs"),
7: ("L", "C", "R", "Ls", "Rs"),
}
def expand_chanmap(chanmap):
"""按 Table E2.5 把 16 bit chanmap 展开为有序声道位置列表。"""
locations = []
for bit in range(16):
if not (int(chanmap) >> (15 - bit)) & 1:
continue
locations.extend(CHANMAP_LOCATIONS[bit].split("/"))
return locations
@dataclass(frozen=True)
class SyncFrame:
"""一个同步帧及其传输层(L2)字段。"""
offset: int
size: int
codec: str # "ac3" 或 "eac3"
bsid: int
strmtyp: int # AC-3 没有子流类型,固定记为 0
substreamid: int
acmod: int
lfeon: int
chanmap: object # 依赖子流的 16 bit chanmap;未声明为 None
@property
def key(self):
"""同一音频帧内唯一的子流标识。"""
return (self.codec, self.strmtyp, self.substreamid)
def _read_bits(data, start, count):
value = 0
for index in range(start, start + count):
value = (value << 1) | ((data[index >> 3] >> (7 - (index & 7))) & 1)
return value
def sync_frame_at(data, offset):
"""解析 offset 处同步帧的传输层头。
区分 AC-3 与 E-AC-3 的依据是 bsid(bit 40 起 5 bit):bsid <= 8 为 AC-3,
其帧长由 fscod/frmsizecod 的比特率表给出;bsid >= 16 为 E-AC-3,帧长由
frmsiz 给出。依赖子流(strmtyp=1)在这里顺带解析出 chanmap。
"""
if offset + 6 > len(data) or data[offset] != 0x0B or data[offset + 1] != 0x77:
return None
base = offset * 8
bsid = _read_bits(data, base + 40, 5)
if bsid <= 8:
fscod = _read_bits(data, base + 32, 2)
frmsizecod = _read_bits(data, base + 34, 6)
if fscod > 2 or frmsizecod >= 38:
return None
size = _AC3_FRAME_WORDS[fscod][frmsizecod] * 2
acmod = _read_bits(data, base + 48, 3)
bit = 51
if acmod & 0x1 and acmod != 0x1:
bit += 2 # cmixlev
if acmod & 0x4:
bit += 2 # surmixlev
if acmod == 0x2:
bit += 2 # dsurmod
lfeon = _read_bits(data, base + bit, 1)
return SyncFrame(offset, size, "ac3", bsid, 0, 0, acmod, lfeon, None)
strmtyp = _read_bits(data, base + 16, 2)
substreamid = _read_bits(data, base + 18, 3)
frmsiz = _read_bits(data, base + 21, 11)
# fscod==3 时的 fscod2 同样是 2 bit,因此 acmod 恒在 bit 36。
acmod = _read_bits(data, base + 36, 3)
lfeon = _read_bits(data, base + 39, 1)
bit = 45 # bsid 之后
bit += 5 # dialnorm
compre = _read_bits(data, base + bit, 1)
bit += 1
if compre:
bit += 8 # compr
if acmod == 0:
bit += 5 # dialnorm2
compr2 = _read_bits(data, base + bit, 1)
bit += 1
if compr2:
bit += 8
chanmap = None
if strmtyp == 1:
if _read_bits(data, base + bit, 1):
chanmap = _read_bits(data, base + bit + 1, 16)
return SyncFrame(offset, (frmsiz + 1) * 2, "eac3", bsid, strmtyp, substreamid,
acmod, lfeon, chanmap)
def iter_syncframes(data):
"""按同步字长度遍历输入中的 AC-3 与 E-AC-3 同步帧。"""
pos = 0
while pos < len(data):
frame = sync_frame_at(data, pos)
if frame is None:
raise UnsupportedVariantError(
"eac3_transport", "syncframe_header",
"同步帧头无效或出现了未处理的子流排列",
details={
"byte_offset": pos,
"remaining_bytes": len(data) - pos,
"next_16_bytes_hex": data[pos:pos + 16].hex(),
})
if pos + frame.size > len(data):
raise UnsupportedVariantError(
"eac3_transport", "truncated_syncframe",
"末帧长度超过输入剩余数据",
details={
"byte_offset": pos,
"codec": frame.codec,
"declared_frame_bytes": frame.size,
"remaining_bytes": len(data) - pos,
})
yield frame
pos += frame.size
def audio_frame_groups(frames):
"""把同步帧序列按音频帧分组。
一个音频帧内每个 (编码, strmtyp, substreamid) 组合只出现一次,因此再次遇到
已出现过的组合即表示下一个音频帧开始。独立子流缺席(只有依赖子流)或核心是
AC-3 时同样成立。
"""
group = []
keys = set()
for frame in frames:
if frame.key in keys:
yield group
group = []
keys = set()
keys.add(frame.key)
group.append(frame)
if group:
yield group
def iter_eac3_frames(data):
"""兼容入口:只产出同步帧字节。"""
for frame in iter_syncframes(data):
yield data[frame.offset:frame.offset + frame.size]
def extract_index(eac3_path, output_dir, max_frames=None):
"""将裸 E-AC-3 的连续 EMDF 保存为 ``frames.csv + emdf/``。"""
output_dir = Path(output_dir)
emdf_dir = output_dir / "emdf"
emdf_dir.mkdir(parents=True, exist_ok=True)
frames = iter_eac3_frames(Path(eac3_path).read_bytes())
rows = []
for frame_number, frame in enumerate(frames):
if max_frames is not None and frame_number >= max_frames:
break
try:
container = find_joc_emdf(frame)
except UnsupportedVariantError as exc:
exc.add_context(frame=frame_number, details={"syncframe_bytes": len(frame)})
raise
except EmdfError as exc:
raise UnsupportedVariantError(
"emdf_transport", "container_syntax",
"EMDF 容器语法无法解析",
frame=frame_number,
details={"syncframe_bytes": len(frame), "parser_error": str(exc)}) from exc
digest = hashlib.sha256(container.raw).hexdigest()
target = emdf_dir / f"{digest}.bin"
if not target.is_file():
target.write_bytes(container.raw)
rows.append({
"frame": frame_number,
"emdf_hash": digest,
"emdf_size": len(container.raw),
"emdf_start_bit": container.start_bit,
"payload_ids": ";".join(str(x) for x in container.payloads),
"error": "",
})
if (frame_number + 1) % 1000 == 0:
print(f"[metadata] {frame_number + 1} frames", flush=True)
if not rows:
raise EmdfError("E-AC-3 输入中没有可处理的同步帧")
with (output_dir / "frames.csv").open("w", encoding="utf-8", newline="") as fp:
fields = ("frame", "emdf_hash", "emdf_size", "emdf_start_bit", "payload_ids", "error")
writer = csv.DictWriter(fp, fieldnames=fields)
writer.writeheader()
writer.writerows(rows)
return output_dir