From 99d553753e3414a129b9e37f347c3dca125eb3cb Mon Sep 17 00:00:00 2001 From: Flechazo <2558755403@qq.com> Date: Wed, 30 Sep 2026 19:32:46 +0800 Subject: [PATCH] fix(all): synchronize complete local Addressables catalog --- tools/python/localize_bundles.py | 251 ++++++++++++-------- tools/python/tests/test_localize_bundles.py | 108 +++++++++ 2 files changed, 266 insertions(+), 93 deletions(-) create mode 100644 tools/python/tests/test_localize_bundles.py diff --git a/tools/python/localize_bundles.py b/tools/python/localize_bundles.py index 58e78b1..fdf5b46 100644 --- a/tools/python/localize_bundles.py +++ b/tools/python/localize_bundles.py @@ -1,118 +1,183 @@ #!/usr/bin/env python3 -"""把远程 bundle 变成本地直读：硬链接进 StreamingAssets/aa + 改写 catalog。 +"""Install one complete downloaded ServerData release as the built-in local catalog. -## 为什么要用 {RuntimePath} 而不是绝对路径 +The source directory must contain the exact ``catalog_alpha.json`` and every +bundle referenced by it. Referenced bundles are hard-linked into the game's +``StreamingAssets/aa`` directory, then a localized copy of the *same* catalog +atomically replaces ``catalog.json``. No bundle data is duplicated when source +and game are on the same filesystem. -试过把 catalog 里的条目改成 `E:/bd2/dl/.../xxx.bundle` 这种绝对路径，结果带子目录的 -bundle 报： +Usage: - RemoteProviderException : Invalid path in AssetBundleProvider - -原因是 Addressables 的 `AssetBundleResource.GetLoadInfo()`： - - if (!(location?.Data is AssetBundleRequestOptions)) { loadType = None; } // ← 落到这 - else if (ShouldPathUseWebRequest(path)) loadType = Web; - else if (UseUnityWebRequestForLocalBundles) loadType = Web; - else loadType = Local; - - ... - default: // LoadType.None - Complete(null, false, new RemoteProviderException( - $"Invalid path in AssetBundleProvider: '{path}'.")); - -绝对路径这种 location 上挂不到 `AssetBundleRequestOptions`，于是 LoadType.None。 - -**客户端自带那 10 个本地 bundle 用的是 `{RuntimePath}\\<包名>`** —— -那是 Addressables 原生的本地形态，一定挂得上 options。照抄它。 - -## 硬链接 - -`RuntimePath` 解析成 `StreamingAssets/aa`，所以 bundle 得摆进去。用 NTFS 硬链接 -**零额外空间**（同一份数据两个路径）。包名里带子目录的（798 个）要建同样的子目录。 - -## 用法 - - python localize_bundles.py + python localize_bundles.py """ + import hashlib import io import json import os -import re import shutil import sys +import tempfile + BS = chr(92) +REMOTE_PREFIX = "{BDNetwork.CdnInfo.Info}" + BS LOCAL_PREFIX = "{UnityEngine.AddressableAssets.Addressables.RuntimePath}" + BS +BACKUP_NAME = "catalog.json.bak-before-local-catalog-sync" -def strip_prefix(s): - """`{BDNetwork.CdnInfo.Info}\\平台\\{分辨率}\\{版本}\\<包名>` -> `<包名>`""" - parts = s.replace(BS, "/").split("/") - return "/".join(parts[4:]) +def relative_bundle(internal_id): + """Return the path below the version root for one remote internal ID.""" + if not internal_id.startswith(REMOTE_PREFIX): + return None + parts = internal_id.replace(BS, "/").split("/") + if len(parts) < 5: + raise ValueError("remote internal ID has no bundle path: %r" % internal_id) + relative = "/".join(parts[4:]) + if not relative or "{" in relative or not relative.endswith(".bundle"): + raise ValueError("remote internal ID has an invalid bundle path: %r" % internal_id) + if relative.startswith("/") or any(part in ("", ".", "..") for part in relative.split("/")): + raise ValueError("remote internal ID escapes the release directory: %r" % internal_id) + return relative + + +def inside(root, relative): + root = os.path.realpath(root) + candidate = os.path.realpath(os.path.join(root, *relative.split("/"))) + if os.path.commonpath((root, candidate)) != root: + raise ValueError("bundle path escapes its root: %s" % relative) + return candidate + + +def digest(path): + value = hashlib.sha256() + with open(path, "rb") as stream: + while True: + block = stream.read(1024 * 1024) + if not block: + return value.digest() + value.update(block) + + +def same_content(left, right): + try: + if os.path.samefile(left, right): + return True + except OSError: + pass + return os.path.getsize(left) == os.path.getsize(right) and digest(left) == digest(right) + + +def load_catalog(path): + with io.open(path, encoding="utf-8") as stream: + value = json.load(stream) + internal_ids = value.get("m_InternalIds") + if not isinstance(internal_ids, list) or not internal_ids: + raise ValueError("catalog has no non-empty m_InternalIds array") + if not all(isinstance(item, str) for item in internal_ids): + raise ValueError("catalog m_InternalIds contains a non-string value") + return value + + +def prepare_catalog(source_directory, aa_directory): + source_directory = os.path.realpath(source_directory) + aa_directory = os.path.realpath(aa_directory) + source_catalog = os.path.join(source_directory, "catalog_alpha.json") + target_catalog = os.path.join(aa_directory, "catalog.json") + if not os.path.isfile(source_catalog): + raise FileNotFoundError("source release is missing catalog_alpha.json: " + source_catalog) + if not os.path.isdir(aa_directory): + raise NotADirectoryError("game Addressables directory does not exist: " + aa_directory) + if not os.path.isfile(target_catalog): + raise FileNotFoundError("game Addressables catalog does not exist: " + target_catalog) + + catalog = load_catalog(source_catalog) + referenced = {} + changed = 0 + for index, internal_id in enumerate(catalog["m_InternalIds"]): + relative = relative_bundle(internal_id) + if relative is None: + continue + source = inside(source_directory, relative) + if not os.path.isfile(source): + raise FileNotFoundError("catalog references a missing source bundle: " + source) + referenced[relative] = source + catalog["m_InternalIds"][index] = LOCAL_PREFIX + relative.replace("/", BS) + changed += 1 + if changed == 0: + raise ValueError("source catalog contains no BDNetwork CDN bundle entries") + + linked = existing = 0 + for relative, source in sorted(referenced.items()): + destination = inside(aa_directory, relative) + os.makedirs(os.path.dirname(destination), exist_ok=True) + if os.path.exists(destination): + if not os.path.isfile(destination) or not same_content(source, destination): + raise FileExistsError("game bundle conflicts with downloaded release: " + destination) + existing += 1 + continue + try: + os.link(source, destination) + except OSError as error: + raise OSError( + "could not hard-link bundle without duplicating data; keep the download and game on the same filesystem: " + + destination + ) from error + linked += 1 + + for internal_id in catalog["m_InternalIds"]: + if not internal_id.startswith(LOCAL_PREFIX): + continue + relative = internal_id[len(LOCAL_PREFIX):].replace(BS, "/") + if not os.path.isfile(inside(aa_directory, relative)): + raise FileNotFoundError("localized catalog references a missing game bundle: " + relative) + + raw = json.dumps(catalog, ensure_ascii=False, separators=(",", ":")).encode("utf-8") + backup = os.path.join(aa_directory, BACKUP_NAME) + if not os.path.exists(backup): + shutil.copy2(target_catalog, backup) + elif not os.path.isfile(backup): + raise FileExistsError("catalog backup path is not a regular file: " + backup) + + descriptor, temporary = tempfile.mkstemp(prefix=".catalog-local-", suffix=".json", dir=aa_directory) + try: + with os.fdopen(descriptor, "wb") as stream: + stream.write(raw) + stream.flush() + os.fsync(stream.fileno()) + os.replace(temporary, target_catalog) + except BaseException: + try: + os.unlink(temporary) + except FileNotFoundError: + pass + raise + return { + "changed_ids": changed, + "unique_bundles": len(referenced), + "linked": linked, + "existing": existing, + "catalog_bytes": len(raw), + "catalog_sha256": hashlib.sha256(raw).hexdigest(), + "backup": backup, + } def main(): if len(sys.argv) != 3: print(__doc__) + return 2 + try: + result = prepare_catalog(sys.argv[1], sys.argv[2]) + except Exception as error: + print("local catalog sync failed: %s" % error, file=sys.stderr) return 1 - src, aa = sys.argv[1], sys.argv[2] - cat = os.path.join(aa, "catalog.json") - if not os.path.isfile(cat): - print("找不到 %s" % cat) - return 1 - if not os.path.isdir(src): - print("找不到 bundle 目录 %s" % src) - return 1 - - # ---- 1. 备份 ---- - bak = cat + ".bak-before-localize" - if not os.path.exists(bak): - shutil.copy2(cat, bak) - print("备份 -> %s" % os.path.basename(bak)) - else: - print("备份已存在，跳过") - - # ---- 2. 硬链接 ---- - linked = skipped = copied = 0 - for dp, _, fn in os.walk(src): - for f in fn: - if not f.endswith(".bundle"): - continue - s = os.path.join(dp, f) - d = os.path.join(aa, os.path.relpath(s, src)) - os.makedirs(os.path.dirname(d), exist_ok=True) - if os.path.exists(d): - skipped += 1 - continue - try: - os.link(s, d) - linked += 1 - except OSError: - shutil.copy2(s, d) - copied += 1 - print("硬链接 新建%d 已存在%d 退回复制%d" % (linked, skipped, copied)) - - # ---- 3. 改写 internal id ---- - o = json.load(io.open(cat, encoding="utf-8")) - ids = o["m_InternalIds"] - changed = 0 - for i, s in enumerate(ids): - if not s.startswith("{BDNetwork.CdnInfo.Info}"): - continue - name = strip_prefix(s) - if not name or "{" in name: - print("解析异常，中止：%r -> %r" % (s, name)) - return 1 - ids[i] = LOCAL_PREFIX + name - changed += 1 - print("改写 %d 条" % changed) - - # 长度不变，下标引用依然有效 - raw = json.dumps(o, ensure_ascii=False, separators=(",", ":")).encode("utf-8") - io.open(cat, "wb").write(raw) - print("写回 %s（%d 字节）" % (os.path.basename(cat), len(raw))) - print(" md5 = %s" % hashlib.md5(raw).hexdigest()) + print("localized current catalog: %d IDs, %d unique bundles" % ( + result["changed_ids"], result["unique_bundles"])) + print("hard links: %d new, %d already exact" % (result["linked"], result["existing"])) + print("catalog: %d bytes, sha256=%s" % (result["catalog_bytes"], result["catalog_sha256"])) + print("backup: %s" % result["backup"]) return 0 diff --git a/tools/python/tests/test_localize_bundles.py b/tools/python/tests/test_localize_bundles.py new file mode 100644 index 0000000..a36efca --- /dev/null +++ b/tools/python/tests/test_localize_bundles.py @@ -0,0 +1,108 @@ +import json +import os +import pathlib +import sys +import tempfile +import unittest + +TOOLS = pathlib.Path(__file__).resolve().parents[1] +sys.path.insert(0, str(TOOLS)) + +import localize_bundles + + +class LocalizeBundlesTest(unittest.TestCase): + def test_uses_source_catalog_and_hard_links_every_reference(self): + with tempfile.TemporaryDirectory() as root: + source = pathlib.Path(root, "release") + aa = pathlib.Path(root, "aa") + source.mkdir() + aa.mkdir() + bundle = source / "nested" / "current.bundle" + bundle.parent.mkdir() + bundle.write_bytes(b"current bundle") + builtin = aa / "builtin.bundle" + builtin.write_bytes(b"builtin") + old_catalog = {"m_InternalIds": [localize_bundles.LOCAL_PREFIX + "obsolete.bundle"]} + (aa / "catalog.json").write_text(json.dumps(old_catalog), encoding="utf-8") + current_catalog = { + "m_BuildResultHash": "current", + "m_InternalIds": [ + localize_bundles.REMOTE_PREFIX + + "StandaloneWindows64\\HD\\20260921135230\\nested/current.bundle", + localize_bundles.LOCAL_PREFIX + "builtin.bundle", + ], + } + (source / "catalog_alpha.json").write_text(json.dumps(current_catalog), encoding="utf-8") + + result = localize_bundles.prepare_catalog(str(source), str(aa)) + + localized = json.loads((aa / "catalog.json").read_text(encoding="utf-8")) + self.assertEqual("current", localized["m_BuildResultHash"]) + self.assertEqual( + localize_bundles.LOCAL_PREFIX + "nested\\current.bundle", + localized["m_InternalIds"][0], + ) + self.assertTrue(os.path.samefile(bundle, aa / "nested" / "current.bundle")) + self.assertEqual(1, result["linked"]) + self.assertTrue((aa / localize_bundles.BACKUP_NAME).is_file()) + + repeated = localize_bundles.prepare_catalog(str(source), str(aa)) + self.assertEqual(0, repeated["linked"]) + self.assertEqual(1, repeated["existing"]) + self.assertEqual(result["catalog_sha256"], repeated["catalog_sha256"]) + + def test_missing_referenced_bundle_does_not_replace_catalog(self): + with tempfile.TemporaryDirectory() as root: + source = pathlib.Path(root, "release") + aa = pathlib.Path(root, "aa") + source.mkdir() + aa.mkdir() + original = b'{"m_InternalIds":["original"]}' + (aa / "catalog.json").write_bytes(original) + catalog = { + "m_InternalIds": [ + localize_bundles.REMOTE_PREFIX + + "StandaloneWindows64\\HD\\20260921135230\\missing.bundle" + ] + } + (source / "catalog_alpha.json").write_text(json.dumps(catalog), encoding="utf-8") + + with self.assertRaises(FileNotFoundError): + localize_bundles.prepare_catalog(str(source), str(aa)) + + self.assertEqual(original, (aa / "catalog.json").read_bytes()) + self.assertFalse((aa / localize_bundles.BACKUP_NAME).exists()) + + def test_conflicting_existing_bundle_does_not_replace_catalog(self): + with tempfile.TemporaryDirectory() as root: + source = pathlib.Path(root, "release") + aa = pathlib.Path(root, "aa") + source.mkdir() + aa.mkdir() + (source / "current.bundle").write_bytes(b"source") + (aa / "current.bundle").write_bytes(b"conflict") + original = b'{"m_InternalIds":["original"]}' + (aa / "catalog.json").write_bytes(original) + (source / "catalog_alpha.json").write_text(json.dumps({ + "m_InternalIds": [ + localize_bundles.REMOTE_PREFIX + + "StandaloneWindows64\\HD\\20260921135230\\current.bundle" + ] + }), encoding="utf-8") + + with self.assertRaises(FileExistsError): + localize_bundles.prepare_catalog(str(source), str(aa)) + + self.assertEqual(original, (aa / "catalog.json").read_bytes()) + + def test_traversal_reference_is_rejected(self): + with self.assertRaises(ValueError): + localize_bundles.relative_bundle( + localize_bundles.REMOTE_PREFIX + + "StandaloneWindows64\\HD\\20260921135230\\..\\escape.bundle" + ) + + +if __name__ == "__main__": + unittest.main()