fix(all): synchronize complete local Addressables catalog

tools/python/localize_bundles.py

@@ -1,118 +1,183 @@
 #!/usr/bin/env python3
-"""把远程 bundle 变成本地直读：硬链接进 StreamingAssets/aa + 改写 catalog。
+"""Install one complete downloaded ServerData release as the built-in local catalog.
 
-## 为什么要用 {RuntimePath} 而不是绝对路径
+The source directory must contain the exact ``catalog_alpha.json`` and every
+bundle referenced by it. Referenced bundles are hard-linked into the game's
+``StreamingAssets/aa`` directory, then a localized copy of the *same* catalog
+atomically replaces ``catalog.json``. No bundle data is duplicated when source
+and game are on the same filesystem.
 
-试过把 catalog 里的条目改成 `E:/bd2/dl/.../xxx.bundle` 这种绝对路径，结果带子目录的
+Usage:
-bundle 报：
 
-    RemoteProviderException : Invalid path in AssetBundleProvider
+    python localize_bundles.py <ServerData version directory> <game aa directory>
-
-原因是 Addressables 的 `AssetBundleResource.GetLoadInfo()`：
-
-    if (!(location?.Data is AssetBundleRequestOptions)) { loadType = None; }   // ← 落到这
-    else if (ShouldPathUseWebRequest(path))      loadType = Web;
-    else if (UseUnityWebRequestForLocalBundles)  loadType = Web;
-    else                                         loadType = Local;
-
-    ...
-    default:  // LoadType.None
-        Complete(null, false, new RemoteProviderException(
-            $"Invalid path in AssetBundleProvider: '{path}'."));
-
-绝对路径这种 location 上挂不到 `AssetBundleRequestOptions`，于是 LoadType.None。
-
-**客户端自带那 10 个本地 bundle 用的是 `{RuntimePath}\\<包名>`** ——
-那是 Addressables 原生的本地形态，一定挂得上 options。照抄它。
-
-## 硬链接
-
-`RuntimePath` 解析成 `StreamingAssets/aa`，所以 bundle 得摆进去。用 NTFS 硬链接
-**零额外空间**（同一份数据两个路径）。包名里带子目录的（798 个）要建同样的子目录。
-
-## 用法
-
-    python localize_bundles.py <bundle 所在目录> <aa 目录>
 """
+
 import hashlib
 import io
 import json
 import os
-import re
 import shutil
 import sys
+import tempfile
+
 
 BS = chr(92)
+REMOTE_PREFIX = "{BDNetwork.CdnInfo.Info}" + BS
 LOCAL_PREFIX = "{UnityEngine.AddressableAssets.Addressables.RuntimePath}" + BS
+BACKUP_NAME = "catalog.json.bak-before-local-catalog-sync"
 
 
-def strip_prefix(s):
+def relative_bundle(internal_id):
-    """`{BDNetwork.CdnInfo.Info}\\平台\\{分辨率}\\{版本}\\<包名>` -> `<包名>`"""
+    """Return the path below the version root for one remote internal ID."""
-    parts = s.replace(BS, "/").split("/")
+    if not internal_id.startswith(REMOTE_PREFIX):
-    return "/".join(parts[4:])
+        return None
+    parts = internal_id.replace(BS, "/").split("/")
+    if len(parts) < 5:
+        raise ValueError("remote internal ID has no bundle path: %r" % internal_id)
+    relative = "/".join(parts[4:])
+    if not relative or "{" in relative or not relative.endswith(".bundle"):
+        raise ValueError("remote internal ID has an invalid bundle path: %r" % internal_id)
+    if relative.startswith("/") or any(part in ("", ".", "..") for part in relative.split("/")):
+        raise ValueError("remote internal ID escapes the release directory: %r" % internal_id)
+    return relative
+
+
+def inside(root, relative):
+    root = os.path.realpath(root)
+    candidate = os.path.realpath(os.path.join(root, *relative.split("/")))
+    if os.path.commonpath((root, candidate)) != root:
+        raise ValueError("bundle path escapes its root: %s" % relative)
+    return candidate
+
+
+def digest(path):
+    value = hashlib.sha256()
+    with open(path, "rb") as stream:
+        while True:
+            block = stream.read(1024 * 1024)
+            if not block:
+                return value.digest()
+            value.update(block)
+
+
+def same_content(left, right):
+    try:
+        if os.path.samefile(left, right):
+            return True
+    except OSError:
+        pass
+    return os.path.getsize(left) == os.path.getsize(right) and digest(left) == digest(right)
+
+
+def load_catalog(path):
+    with io.open(path, encoding="utf-8") as stream:
+        value = json.load(stream)
+    internal_ids = value.get("m_InternalIds")
+    if not isinstance(internal_ids, list) or not internal_ids:
+        raise ValueError("catalog has no non-empty m_InternalIds array")
+    if not all(isinstance(item, str) for item in internal_ids):
+        raise ValueError("catalog m_InternalIds contains a non-string value")
+    return value
+
+
+def prepare_catalog(source_directory, aa_directory):
+    source_directory = os.path.realpath(source_directory)
+    aa_directory = os.path.realpath(aa_directory)
+    source_catalog = os.path.join(source_directory, "catalog_alpha.json")
+    target_catalog = os.path.join(aa_directory, "catalog.json")
+    if not os.path.isfile(source_catalog):
+        raise FileNotFoundError("source release is missing catalog_alpha.json: " + source_catalog)
+    if not os.path.isdir(aa_directory):
+        raise NotADirectoryError("game Addressables directory does not exist: " + aa_directory)
+    if not os.path.isfile(target_catalog):
+        raise FileNotFoundError("game Addressables catalog does not exist: " + target_catalog)
+
+    catalog = load_catalog(source_catalog)
+    referenced = {}
+    changed = 0
+    for index, internal_id in enumerate(catalog["m_InternalIds"]):
+        relative = relative_bundle(internal_id)
+        if relative is None:
+            continue
+        source = inside(source_directory, relative)
+        if not os.path.isfile(source):
+            raise FileNotFoundError("catalog references a missing source bundle: " + source)
+        referenced[relative] = source
+        catalog["m_InternalIds"][index] = LOCAL_PREFIX + relative.replace("/", BS)
+        changed += 1
+    if changed == 0:
+        raise ValueError("source catalog contains no BDNetwork CDN bundle entries")
+
+    linked = existing = 0
+    for relative, source in sorted(referenced.items()):
+        destination = inside(aa_directory, relative)
+        os.makedirs(os.path.dirname(destination), exist_ok=True)
+        if os.path.exists(destination):
+            if not os.path.isfile(destination) or not same_content(source, destination):
+                raise FileExistsError("game bundle conflicts with downloaded release: " + destination)
+            existing += 1
+            continue
+        try:
+            os.link(source, destination)
+        except OSError as error:
+            raise OSError(
+                "could not hard-link bundle without duplicating data; keep the download and game on the same filesystem: "
+                + destination
+            ) from error
+        linked += 1
+
+    for internal_id in catalog["m_InternalIds"]:
+        if not internal_id.startswith(LOCAL_PREFIX):
+            continue
+        relative = internal_id[len(LOCAL_PREFIX):].replace(BS, "/")
+        if not os.path.isfile(inside(aa_directory, relative)):
+            raise FileNotFoundError("localized catalog references a missing game bundle: " + relative)
+
+    raw = json.dumps(catalog, ensure_ascii=False, separators=(",", ":")).encode("utf-8")
+    backup = os.path.join(aa_directory, BACKUP_NAME)
+    if not os.path.exists(backup):
+        shutil.copy2(target_catalog, backup)
+    elif not os.path.isfile(backup):
+        raise FileExistsError("catalog backup path is not a regular file: " + backup)
+
+    descriptor, temporary = tempfile.mkstemp(prefix=".catalog-local-", suffix=".json", dir=aa_directory)
+    try:
+        with os.fdopen(descriptor, "wb") as stream:
+            stream.write(raw)
+            stream.flush()
+            os.fsync(stream.fileno())
+        os.replace(temporary, target_catalog)
+    except BaseException:
+        try:
+            os.unlink(temporary)
+        except FileNotFoundError:
+            pass
+        raise
+    return {
+        "changed_ids": changed,
+        "unique_bundles": len(referenced),
+        "linked": linked,
+        "existing": existing,
+        "catalog_bytes": len(raw),
+        "catalog_sha256": hashlib.sha256(raw).hexdigest(),
+        "backup": backup,
+    }
 
 
 def main():
     if len(sys.argv) != 3:
         print(__doc__)
-        return 1
+        return 2
-    src, aa = sys.argv[1], sys.argv[2]
-    cat = os.path.join(aa, "catalog.json")
-    if not os.path.isfile(cat):
-        print("找不到 %s" % cat)
-        return 1
-    if not os.path.isdir(src):
-        print("找不到 bundle 目录 %s" % src)
-        return 1
-
-    # ---- 1. 备份 ----
-    bak = cat + ".bak-before-localize"
-    if not os.path.exists(bak):
-        shutil.copy2(cat, bak)
-        print("备份 -> %s" % os.path.basename(bak))
-    else:
-        print("备份已存在，跳过")
-
-    # ---- 2. 硬链接 ----
-    linked = skipped = copied = 0
-    for dp, _, fn in os.walk(src):
-        for f in fn:
-            if not f.endswith(".bundle"):
-                continue
-            s = os.path.join(dp, f)
-            d = os.path.join(aa, os.path.relpath(s, src))
-            os.makedirs(os.path.dirname(d), exist_ok=True)
-            if os.path.exists(d):
-                skipped += 1
-                continue
     try:
-                os.link(s, d)
+        result = prepare_catalog(sys.argv[1], sys.argv[2])
-                linked += 1
+    except Exception as error:
-            except OSError:
+        print("local catalog sync failed: %s" % error, file=sys.stderr)
-                shutil.copy2(s, d)
-                copied += 1
-    print("硬链接 新建%d 已存在%d 退回复制%d" % (linked, skipped, copied))
-
-    # ---- 3. 改写 internal id ----
-    o = json.load(io.open(cat, encoding="utf-8"))
-    ids = o["m_InternalIds"]
-    changed = 0
-    for i, s in enumerate(ids):
-        if not s.startswith("{BDNetwork.CdnInfo.Info}"):
-            continue
-        name = strip_prefix(s)
-        if not name or "{" in name:
-            print("解析异常，中止：%r -> %r" % (s, name))
         return 1
-        ids[i] = LOCAL_PREFIX + name
+    print("localized current catalog: %d IDs, %d unique bundles" % (
-        changed += 1
+        result["changed_ids"], result["unique_bundles"]))
-    print("改写 %d 条" % changed)
+    print("hard links: %d new, %d already exact" % (result["linked"], result["existing"]))
-
+    print("catalog: %d bytes, sha256=%s" % (result["catalog_bytes"], result["catalog_sha256"]))
-    # 长度不变，下标引用依然有效
+    print("backup: %s" % result["backup"])
-    raw = json.dumps(o, ensure_ascii=False, separators=(",", ":")).encode("utf-8")
-    io.open(cat, "wb").write(raw)
-    print("写回 %s（%d 字节）" % (os.path.basename(cat), len(raw)))
-    print("  md5 = %s" % hashlib.md5(raw).hexdigest())
     return 0
 
 

tools/python/tests/test_localize_bundles.py

@@ -0,0 +1,108 @@
+import json
+import os
+import pathlib
+import sys
+import tempfile
+import unittest
+
+TOOLS = pathlib.Path(__file__).resolve().parents[1]
+sys.path.insert(0, str(TOOLS))
+
+import localize_bundles
+
+
+class LocalizeBundlesTest(unittest.TestCase):
+    def test_uses_source_catalog_and_hard_links_every_reference(self):
+        with tempfile.TemporaryDirectory() as root:
+            source = pathlib.Path(root, "release")
+            aa = pathlib.Path(root, "aa")
+            source.mkdir()
+            aa.mkdir()
+            bundle = source / "nested" / "current.bundle"
+            bundle.parent.mkdir()
+            bundle.write_bytes(b"current bundle")
+            builtin = aa / "builtin.bundle"
+            builtin.write_bytes(b"builtin")
+            old_catalog = {"m_InternalIds": [localize_bundles.LOCAL_PREFIX + "obsolete.bundle"]}
+            (aa / "catalog.json").write_text(json.dumps(old_catalog), encoding="utf-8")
+            current_catalog = {
+                "m_BuildResultHash": "current",
+                "m_InternalIds": [
+                    localize_bundles.REMOTE_PREFIX
+                    + "StandaloneWindows64\\HD\\20260921135230\\nested/current.bundle",
+                    localize_bundles.LOCAL_PREFIX + "builtin.bundle",
+                ],
+            }
+            (source / "catalog_alpha.json").write_text(json.dumps(current_catalog), encoding="utf-8")
+
+            result = localize_bundles.prepare_catalog(str(source), str(aa))
+
+            localized = json.loads((aa / "catalog.json").read_text(encoding="utf-8"))
+            self.assertEqual("current", localized["m_BuildResultHash"])
+            self.assertEqual(
+                localize_bundles.LOCAL_PREFIX + "nested\\current.bundle",
+                localized["m_InternalIds"][0],
+            )
+            self.assertTrue(os.path.samefile(bundle, aa / "nested" / "current.bundle"))
+            self.assertEqual(1, result["linked"])
+            self.assertTrue((aa / localize_bundles.BACKUP_NAME).is_file())
+
+            repeated = localize_bundles.prepare_catalog(str(source), str(aa))
+            self.assertEqual(0, repeated["linked"])
+            self.assertEqual(1, repeated["existing"])
+            self.assertEqual(result["catalog_sha256"], repeated["catalog_sha256"])
+
+    def test_missing_referenced_bundle_does_not_replace_catalog(self):
+        with tempfile.TemporaryDirectory() as root:
+            source = pathlib.Path(root, "release")
+            aa = pathlib.Path(root, "aa")
+            source.mkdir()
+            aa.mkdir()
+            original = b'{"m_InternalIds":["original"]}'
+            (aa / "catalog.json").write_bytes(original)
+            catalog = {
+                "m_InternalIds": [
+                    localize_bundles.REMOTE_PREFIX
+                    + "StandaloneWindows64\\HD\\20260921135230\\missing.bundle"
+                ]
+            }
+            (source / "catalog_alpha.json").write_text(json.dumps(catalog), encoding="utf-8")
+
+            with self.assertRaises(FileNotFoundError):
+                localize_bundles.prepare_catalog(str(source), str(aa))
+
+            self.assertEqual(original, (aa / "catalog.json").read_bytes())
+            self.assertFalse((aa / localize_bundles.BACKUP_NAME).exists())
+
+    def test_conflicting_existing_bundle_does_not_replace_catalog(self):
+        with tempfile.TemporaryDirectory() as root:
+            source = pathlib.Path(root, "release")
+            aa = pathlib.Path(root, "aa")
+            source.mkdir()
+            aa.mkdir()
+            (source / "current.bundle").write_bytes(b"source")
+            (aa / "current.bundle").write_bytes(b"conflict")
+            original = b'{"m_InternalIds":["original"]}'
+            (aa / "catalog.json").write_bytes(original)
+            (source / "catalog_alpha.json").write_text(json.dumps({
+                "m_InternalIds": [
+                    localize_bundles.REMOTE_PREFIX
+                    + "StandaloneWindows64\\HD\\20260921135230\\current.bundle"
+                ]
+            }), encoding="utf-8")
+
+            with self.assertRaises(FileExistsError):
+                localize_bundles.prepare_catalog(str(source), str(aa))
+
+            self.assertEqual(original, (aa / "catalog.json").read_bytes())
+
+    def test_traversal_reference_is_rejected(self):
+        with self.assertRaises(ValueError):
+            localize_bundles.relative_bundle(
+                localize_bundles.REMOTE_PREFIX
+                + "StandaloneWindows64\\HD\\20260921135230\\..\\escape.bundle"
+            )
+
+
+if __name__ == "__main__":
+    unittest.main()