diff --git a/Cargo.lock b/Cargo.lock index 02577a3..46b3d59 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -109,6 +109,12 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "base64ct" +version = "1.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2af50177e190e07a26ab74f8b1efbfe2ef87da2116221318cb1c2e82baf7de06" + [[package]] name = "bcdec_rs" version = "0.1.2" @@ -303,6 +309,12 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "const-oid" +version = "0.9.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" + [[package]] name = "convert_case" version = "0.9.0" @@ -392,20 +404,6 @@ dependencies = [ "memchr", ] -[[package]] -name = "dashmap" -version = "6.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5041cc499144891f3790297212f32a74fb938e5136a14943f338ef9e0ae276cf" -dependencies = [ - "cfg-if", - "crossbeam-utils", - "hashbrown 0.14.5", - "lock_api", - "once_cell", - "parking_lot_core", -] - [[package]] name = "ddsfile" version = "0.5.2" @@ -418,6 +416,17 @@ dependencies = [ "num-traits", ] +[[package]] +name = "der" +version = "0.7.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7c1832837b905bbfb5101e07cc24c8deddf52f93225eee6ead5f4d63d53ddcb" +dependencies = [ + "const-oid", + "pem-rfc7468", + "zeroize", +] + [[package]] name = "deranged" version = "0.5.5" @@ -434,6 +443,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ "block-buffer", + "const-oid", "crypto-common", ] @@ -647,12 +657,6 @@ dependencies = [ "zerocopy", ] -[[package]] -name = "hashbrown" -version = "0.14.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" - [[package]] name = "hashbrown" version = "0.16.1" @@ -806,7 +810,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7714e70437a7dc3ac8eb7e6f8df75fd8eb422675fc7678aff7364301092b1017" dependencies = [ "equivalent", - "hashbrown 0.16.1", + "hashbrown", "serde", "serde_core", ] @@ -880,12 +884,14 @@ name = "lazy_static" version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" +dependencies = [ + "spin", +] [[package]] name = "league-toolkit" -version = "0.2.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "731977c88708f1645c6b6c661a4529d025ddd186f45205146388c3e8df30e78e" +version = "0.2.24" +source = "git+https://github.com/LeagueToolkit/league-toolkit?rev=22d1990421e980107cb8a436a3206a7a4b47b418#22d1990421e980107cb8a436a3206a7a4b47b418" dependencies = [ "ltk_anim", "ltk_file", @@ -893,6 +899,7 @@ dependencies = [ "ltk_mesh", "ltk_meta", "ltk_primitives", + "ltk_rst", "ltk_texture", "ltk_wad", ] @@ -903,6 +910,12 @@ version = "0.2.180" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bcc35a38544a891a5f7c865aca548a982ccb3b8650a5b06d0fd33a10283c56fc" +[[package]] +name = "libm" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" + [[package]] name = "libredox" version = "0.1.12" @@ -936,9 +949,8 @@ checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" [[package]] name = "ltk_anim" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d308e5232aec9875ed2dcfecd931bb32c6dceac068eb6afe18283ceccf352bb" +version = "0.3.6" +source = "git+https://github.com/LeagueToolkit/league-toolkit?rev=22d1990421e980107cb8a436a3206a7a4b47b418#22d1990421e980107cb8a436a3206a7a4b47b418" dependencies = [ "bitflags", "bytemuck", @@ -952,9 +964,8 @@ dependencies = [ [[package]] name = "ltk_file" -version = "0.2.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1df39d71a40db8007946675db305495da04602ffae3bdcd6e43b7dec6dfcd07d" +version = "0.2.10" +source = "git+https://github.com/LeagueToolkit/league-toolkit?rev=22d1990421e980107cb8a436a3206a7a4b47b418#22d1990421e980107cb8a436a3206a7a4b47b418" dependencies = [ "serde", "strum", @@ -962,9 +973,13 @@ dependencies = [ [[package]] name = "ltk_hash" -version = "0.2.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a0081deb694867c5ca11658f3df01425ae4181bdf5c6fbdc5fb1818fa0b52e5" +version = "0.4.0" +source = "git+https://github.com/LeagueToolkit/league-toolkit?rev=22d1990421e980107cb8a436a3206a7a4b47b418#22d1990421e980107cb8a436a3206a7a4b47b418" +dependencies = [ + "byteorder", + "serde", + "xxhash-rust", +] [[package]] name = "ltk_hashdb" @@ -980,9 +995,8 @@ dependencies = [ [[package]] name = "ltk_io_ext" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "751c5641617ea4fe4bb1bf92ac82d88db597f35d5ff4ea40b735bcf674c15107" +version = "0.4.4" +source = "git+https://github.com/LeagueToolkit/league-toolkit?rev=22d1990421e980107cb8a436a3206a7a4b47b418#22d1990421e980107cb8a436a3206a7a4b47b418" dependencies = [ "byteorder", "glam", @@ -992,9 +1006,8 @@ dependencies = [ [[package]] name = "ltk_mesh" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8cb8f4cd64cf9b9fd0cf367d06e244de0c4135f7b964b8cb57a1272e5b79aa5" +version = "0.4.4" +source = "git+https://github.com/LeagueToolkit/league-toolkit?rev=22d1990421e980107cb8a436a3206a7a4b47b418#22d1990421e980107cb8a436a3206a7a4b47b418" dependencies = [ "bitflags", "byteorder", @@ -1008,15 +1021,15 @@ dependencies = [ [[package]] name = "ltk_meta" -version = "0.3.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "198e9f23797a8899b1273a1c49c6d7bf194055a32064b5dd20724f5469f9ffb1" +version = "0.6.1" +source = "git+https://github.com/LeagueToolkit/league-toolkit?rev=22d1990421e980107cb8a436a3206a7a4b47b418#22d1990421e980107cb8a436a3206a7a4b47b418" dependencies = [ "byteorder", "enum_dispatch", "glam", "indexmap", "log", + "ltk_hash", "ltk_io_ext", "ltk_primitives", "miette", @@ -1043,9 +1056,8 @@ dependencies = [ [[package]] name = "ltk_primitives" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f5379b9ad4c75f5a12d43ae941ab9cb8b14b49f4e15c3a04cb213962b26b6fcb" +version = "0.3.5" +source = "git+https://github.com/LeagueToolkit/league-toolkit?rev=22d1990421e980107cb8a436a3206a7a4b47b418#22d1990421e980107cb8a436a3206a7a4b47b418" dependencies = [ "byteorder", "glam", @@ -1053,38 +1065,54 @@ dependencies = [ "thiserror 1.0.69", ] +[[package]] +name = "ltk_rst" +version = "0.2.1" +source = "git+https://github.com/LeagueToolkit/league-toolkit?rev=22d1990421e980107cb8a436a3206a7a4b47b418#22d1990421e980107cb8a436a3206a7a4b47b418" +dependencies = [ + "byteorder", + "num_enum", + "serde", + "thiserror 1.0.69", + "xxhash-rust", +] + [[package]] name = "ltk_texture" -version = "0.4.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a01e9a5f09e5c4ce36b9d852d32b2339fae47972a3066d03c305ace46f85f2b" +version = "0.6.0" +source = "git+https://github.com/LeagueToolkit/league-toolkit?rev=22d1990421e980107cb8a436a3206a7a4b47b418#22d1990421e980107cb8a436a3206a7a4b47b418" dependencies = [ "bitflags", + "bytemuck", "byteorder", "ddsfile", + "half", "image", "image_dds", "ltk_io_ext", "num_enum", + "texpresso", "texture2ddecoder", "thiserror 1.0.69", ] [[package]] name = "ltk_wad" -version = "0.2.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0b540870b2dbf58257ec0bcb2cb64f26195e4eaef6e5955bad82f97067234a4" +version = "0.3.1" +source = "git+https://github.com/LeagueToolkit/league-toolkit?rev=22d1990421e980107cb8a436a3206a7a4b47b418#22d1990421e980107cb8a436a3206a7a4b47b418" dependencies = [ "byteorder", "camino", "flate2", "itertools 0.14.0", "ltk_file", + "ltk_hash", "ltk_io_ext", "memchr", "num_enum", + "rsa", "serde", + "sha2", "thiserror 1.0.69", "xxhash-rust", "zstd", @@ -1156,12 +1184,47 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "num-bigint-dig" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e661dda6640fad38e827a6d4a310ff4763082116fe217f279885c97f511bb0b7" +dependencies = [ + "lazy_static", + "libm", + "num-integer", + "num-iter", + "num-traits", + "rand", + "smallvec", + "zeroize", +] + [[package]] name = "num-conv" version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf97ec579c3c42f953ef76dbf8d55ac91fb219dde70e49aa4a6b7d74e9919050" +[[package]] +name = "num-integer" +version = "0.1.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ce2d95d4b3734dc35aa2f45e1aa22cd416814592a4f9d9205e11affd5b8e10b" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-iter" +version = "0.1.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c92800bd69a1eac91786bcfe9da64a897eb72911b8dc3095decbd07429e8048b" +dependencies = [ + "num-integer", + "num-traits", +] + [[package]] name = "num-traits" version = "0.2.19" @@ -1169,6 +1232,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" dependencies = [ "autocfg", + "libm", ] [[package]] @@ -1264,6 +1328,15 @@ version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" +[[package]] +name = "pem-rfc7468" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88b39c9bfcfc231068454382784bb460aae594343fb030d46e9f50a645418412" +dependencies = [ + "base64ct", +] + [[package]] name = "percent-encoding" version = "2.3.2" @@ -1276,6 +1349,27 @@ version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3b3cff922bd51709b605d9ead9aa71031d81447142d828eb4a6eba76fe619f9b" +[[package]] +name = "pkcs1" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8ffb9f10fa047879315e6625af03c164b16962a5368d724ed16323b68ace47f" +dependencies = [ + "der", + "pkcs8", + "spki", +] + +[[package]] +name = "pkcs8" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7" +dependencies = [ + "der", + "spki", +] + [[package]] name = "pkg-config" version = "0.3.32" @@ -1303,6 +1397,15 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + [[package]] name = "proc-macro-crate" version = "3.4.0" @@ -1345,6 +1448,35 @@ version = "5.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" +[[package]] +name = "rand" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22f6172bdec972074665ed81ed53b71da00bfc44b65a753cfde883ec4c702a1a" +dependencies = [ + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom 0.2.17", +] + [[package]] name = "rayon" version = "1.11.0" @@ -1416,6 +1548,27 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "rsa" +version = "0.9.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8573f03f5883dcaebdfcf4725caa1ecb9c15b2ef50c43a07b816e06799bb12d" +dependencies = [ + "const-oid", + "digest", + "num-bigint-dig", + "num-integer", + "num-traits", + "pkcs1", + "pkcs8", + "rand_core", + "sha2", + "signature", + "spki", + "subtle", + "zeroize", +] + [[package]] name = "rustc-demangle" version = "0.1.27" @@ -1571,6 +1724,16 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" +[[package]] +name = "signature" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77549399552de45a898a580c1b41d445bf730df867cc44e6c0233bbc4b8329de" +dependencies = [ + "digest", + "rand_core", +] + [[package]] name = "simd-adler32" version = "0.3.8" @@ -1583,6 +1746,22 @@ version = "1.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" +[[package]] +name = "spin" +version = "0.9.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3763264f6b73151db08c50ff20d7d8a0b8796e021cdea7ceedad07b80155fa0e" + +[[package]] +name = "spki" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d91ed6c858b01f942cd56b37a94b3e0a1798290327d1236e4d9cf4eaca44d29d" +dependencies = [ + "base64ct", + "der", +] + [[package]] name = "stable_deref_trait" version = "1.2.1" @@ -1655,6 +1834,16 @@ dependencies = [ "syn 2.0.114", ] +[[package]] +name = "texpresso" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8550677e2259d675a7841cb1403db35f330cc9e58674c8c5caa12dd12c51dc71" +dependencies = [ + "libm", + "rayon", +] + [[package]] name = "texture2ddecoder" version = "0.1.2" @@ -2053,7 +2242,6 @@ dependencies = [ "colored", "convert_case", "csv", - "dashmap", "directories-next", "dirs-next", "eyre", @@ -2064,7 +2252,6 @@ dependencies = [ "ltk_hashdb", "ltk_mimir_cache", "parking_lot", - "rayon", "serde", "serde_json", "time", @@ -2076,7 +2263,6 @@ dependencies = [ "walkdir", "winreg", "winresource", - "xxhash-rust", ] [[package]] diff --git a/README.md b/README.md index fb81b12..baf7969 100644 --- a/README.md +++ b/README.md @@ -305,8 +305,8 @@ hashtable format (a ` ` text file, as in CommunityDragon's `hash [Mimir](https://github.com/LeagueToolkit/Mimir) `.lhdb` hash table. Alias: `rip`. "Resolvable" means every chunk whose hash we can attribute to a real path: names already known -to the shared [Mimir](https://github.com/LeagueToolkit/Mimir) cache **plus** names recovered by scanning the WAD's `.bin` files (dependency -links and string properties, the same recovery used during extraction). Chunks that would only +to the shared [Mimir](https://github.com/LeagueToolkit/Mimir) cache **plus** names recovered by scanning the WAD's `.bin` files (every +path string a bin holds, the same recovery used during extraction). Chunks that would only render as their 16-character hex fallback are skipped, so the output is a clean, meaningful list. The `.lhdb` output is written in the Game-table configuration (64-bit XXH64 keys, case-insensitive), @@ -321,8 +321,7 @@ Common flags: - `-H, --hashtable ` (also `-d`): optional supplemental hashtable file to resolve additional names - `-x, --pattern `: only include chunks whose resolved path matches this regex - `-v, --filter-invert`: invert the `-x` filter (exclude matching paths instead of including them) -- `--no-bin-paths`: disable scanning `.bin` files to recover names (enabled by default) -- `--full-bin-scan`: scan every chunk (not just known `.bin` files), recovering the most names at the cost of a full decompression pass +- `--no-bin-paths`: disable scanning `.bin` files to recover names (enabled by default). The scan reads every bin of the archive, named or not Basic examples: @@ -340,8 +339,8 @@ wadtools paths -i Aatrox.wad.client -F lhdb # Combine several WADs into one hashtable wadtools paths -i Aatrox.wad.client -i Ahri.wad.client -o champions.txt -# Only rip character asset paths, using the deepest bin scan -wadtools paths -i Aatrox.wad.client --full-bin-scan -x "^assets/characters/" +# Only rip character asset paths +wadtools paths -i Aatrox.wad.client -x "^assets/characters/" ``` ### Diff diff --git a/crates/wadtools/CHANGELOG.md b/crates/wadtools/CHANGELOG.md index 6aa0fe6..fbaeefe 100644 --- a/crates/wadtools/CHANGELOG.md +++ b/crates/wadtools/CHANGELOG.md @@ -7,6 +7,20 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Changed + +- Extraction and bin-name recovery run through `ltk_wad`'s `WadExtractor` and `NameRecovery`, + which took over wadtools' own pipeline. The naming rules, the skip-or-overwrite policy, the + stats and the parallel reader and writer pipeline are the same, now shared with LTK Manager. +- The bin scan tells a bin by its magic from the first compressed block alone and reads its + strings with no parse of the property tree, so it recovers the names the old `--full-bin-scan` + found in a fraction of the time. + +### Removed + +- `--full-bin-scan` on `extract` and `paths`. The scan reads every bin of the archive now, so the + flag had nothing left to switch on. + ## [0.5.7](https://github.com/LeagueToolkit/wadtools/compare/v0.5.6...v0.5.7) - 2026-07-13 ### Added diff --git a/crates/wadtools/Cargo.toml b/crates/wadtools/Cargo.toml index 3921c7e..f97260e 100644 --- a/crates/wadtools/Cargo.toml +++ b/crates/wadtools/Cargo.toml @@ -8,7 +8,9 @@ clap = { version = "4.5.20", features = ["derive"] } eyre = "0.6.12" toml = "0.8.19" fancy-regex = "0.16.2" -league-toolkit = { version = "0.2.17", features = ["serde"] } +# Pinned to a commit of the `feat/wad-extractor` branch (LeagueToolkit/league-toolkit#183) +# until `ltk_wad` 0.4 ships on crates.io. The extractor and the bin-name recovery live there now. +league-toolkit = { git = "https://github.com/LeagueToolkit/league-toolkit", rev = "22d1990421e980107cb8a436a3206a7a4b47b418", features = ["serde"] } color-eyre = "0.6" tracing = "0.1.41" parking_lot = "0.12.3" @@ -26,9 +28,6 @@ directories-next = "2.0" dirs-next = "2.0" camino = "1.1" convert_case = "0.9.0" -rayon = "1.10" -dashmap = "6" -xxhash-rust = { version = "0.8", features = ["xxh64"] } # `download-hashes` streams release assets through our own progress-bar `Fetch` # (mimir's bundled `UreqFetch` buffers the whole body, so it can't drive a bar). ureq = "2.12" diff --git a/crates/wadtools/src/bin_scan.rs b/crates/wadtools/src/bin_scan.rs deleted file mode 100644 index 5d3180c..0000000 --- a/crates/wadtools/src/bin_scan.rs +++ /dev/null @@ -1,444 +0,0 @@ -//! Pre-extraction scan of `.bin` property files to recover chunk names. -//! -//! WAD chunks are keyed only by `xxh64(lowercase_path)`. Chunks whose hash is not in -//! the loaded community hashtable would otherwise extract as anonymous 16-hex names. -//! League `.bin` files reference other assets by path - both as explicit dependency -//! links (`BinTree::dependencies`) and as string properties (textures, meshes, vfx, -//! ...). Hashing those discovered paths and matching them against the WAD's own chunk -//! hashes lets us fill in real names/folders before extraction. -//! -//! Matching is exact: a discovered string only contributes a name when its hash equals -//! an actual unresolved chunk hash, so non-path strings simply never match. - -use std::{ - collections::{HashMap, HashSet, VecDeque}, - io::{Cursor, Read, Seek}, - sync::Arc, -}; - -use camino::Utf8Path; -use league_toolkit::{ - file::LeagueFileKind, - meta::{BinTree, PropertyValueEnum}, - wad::{decompress_raw, Wad, WadChunk}, -}; -use xxhash_rust::xxh64::xxh64; - -use crate::utils::WadHashtable; - -/// Computes the WAD path hash for a string, matching how chunk hashes are produced -/// (`xxh64` of the lowercased path, seed 0). -pub fn hash_wad_path(path: &str) -> u64 { - xxh64(path.to_lowercase().as_bytes(), 0) -} - -/// Parses a decompressed `.bin` file and appends every path-like string it contains -/// (dependency links plus all string property values) to `out`. -/// -/// Parse failures are non-fatal: a malformed bin is logged at debug level and skipped. -pub fn collect_bin_paths(bin_bytes: &[u8], out: &mut Vec) { - let tree = match BinTree::from_reader(&mut Cursor::new(bin_bytes)) { - Ok(tree) => tree, - Err(error) => { - tracing::debug!("failed to parse bin during scan: {error}"); - return; - } - }; - - for dependency in &tree.dependencies { - out.push(dependency.clone()); - } - - for (_path_hash, object) in &tree.objects { - for property in object.properties.values() { - walk_value(&property.value, out); - } - } -} - -/// Recursively collects string values from a property value, descending into every -/// container/struct/map variant. -fn walk_value(value: &PropertyValueEnum, out: &mut Vec) { - match value { - PropertyValueEnum::String(string) => { - if !string.0.is_empty() { - out.push(string.0.clone()); - } - } - PropertyValueEnum::Container(container) => { - for item in &container.items { - walk_value(item, out); - } - } - PropertyValueEnum::UnorderedContainer(container) => { - for item in &container.0.items { - walk_value(item, out); - } - } - PropertyValueEnum::Struct(value) => { - for property in value.properties.values() { - walk_value(&property.value, out); - } - } - PropertyValueEnum::Embedded(value) => { - for property in value.0.properties.values() { - walk_value(&property.value, out); - } - } - PropertyValueEnum::Optional(optional) => { - if let Some(inner) = &optional.value { - walk_value(inner, out); - } - } - PropertyValueEnum::Map(map) => { - for (key, value) in &map.entries { - walk_value(&key.0, out); - walk_value(value, out); - } - } - // Hash / WadChunkLink / ObjectLink hold hashes that are not reversible to a - // string path, and the remaining variants are scalars. Nothing to collect. - _ => {} - } -} - -/// Returns true if `path` has a `.bin` extension (case-insensitive). -fn is_bin_path(path: &str) -> bool { - Utf8Path::new(path) - .extension() - .is_some_and(|extension| extension.eq_ignore_ascii_case("bin")) -} - -/// Loads and decompresses a chunk, returning `None` (with a debug log) on failure. -fn load_and_decompress(wad: &mut Wad, chunk: &WadChunk) -> Option> { - let raw = match wad.load_chunk_raw(chunk) { - Ok(raw) => raw, - Err(error) => { - tracing::debug!("failed to read chunk {:016x}: {error}", chunk.path_hash); - return None; - } - }; - match decompress_raw(&raw, chunk.compression_type, chunk.uncompressed_size) { - Ok(data) => Some(data), - Err(error) => { - tracing::debug!( - "failed to decompress chunk {:016x}: {error}", - chunk.path_hash - ); - None - } - } -} - -/// Parses one decompressed bin and folds its discovered names into `discovered`. A name -/// is kept only when its hash matches a real chunk in `chunk_hashes` and is not already -/// known - the overlay is purely additive gap-fill. Returns the chunk hashes of any -/// newly discovered paths that are themselves `.bin` files, so the caller can follow -/// links transitively. -fn harvest_bin( - data: &[u8], - chunk_hashes: &HashSet, - hashtable: &WadHashtable, - discovered: &mut HashMap>, - scratch: &mut Vec, -) -> Vec { - scratch.clear(); - collect_bin_paths(data, scratch); - - let mut new_bins = Vec::new(); - for string in scratch.drain(..) { - let hash = hash_wad_path(&string); - if !chunk_hashes.contains(&hash) - || discovered.contains_key(&hash) - || hashtable.contains(hash) - { - continue; - } - let is_bin = is_bin_path(&string); - discovered.insert(hash, Arc::from(string)); - if is_bin { - new_bins.push(hash); - } - } - new_bins -} - -/// Scans the `.bin` files of a WAD and returns an overlay of `path_hash -> path` for -/// paths discovered inside them (dependency links plus string properties). A discovered -/// path is kept only when its hash matches an actual chunk in `chunk_hashes` and the -/// community `hashtable` does not already resolve it - the overlay is purely additive -/// gap-fill that never overrides official names. -/// -/// Two modes: -/// - default (`full == false`): start from chunks the `hashtable` already names `*.bin`, -/// then follow discovered bin links transitively until no new bins are found. Only bins -/// we have a name for are decompressed, so the cost is a small slice of the WAD. -/// - full (`full == true`): decompress every chunk and magic-detect bins, so even bins -/// that are anonymous *and* unreferenced get harvested. Recovers the most names, at the -/// cost of a full extra decompression pass over the WAD. -pub fn scan_wad_bin_paths( - wad: &mut Wad, - hashtable: &WadHashtable, - chunk_hashes: &HashSet, - full: bool, -) -> HashMap> { - let mut discovered: HashMap> = HashMap::new(); - let mut scratch: Vec = Vec::new(); - let chunks: Vec = wad.chunks().iter().copied().collect(); - - if full { - for chunk in &chunks { - let Some(data) = load_and_decompress(wad, chunk) else { - continue; - }; - if !matches!( - LeagueFileKind::identify_from_bytes(&data), - LeagueFileKind::PropertyBin | LeagueFileKind::PropertyBinOverride - ) { - continue; - } - harvest_bin( - &data, - chunk_hashes, - hashtable, - &mut discovered, - &mut scratch, - ); - } - return discovered; - } - - // Seed the queue with chunks already named `*.bin`, then follow discovered bin links. - let chunk_by_hash: HashMap = chunks - .iter() - .map(|chunk| (chunk.path_hash, *chunk)) - .collect(); - let mut queue: VecDeque = chunks - .iter() - .filter(|chunk| is_bin_path(&hashtable.resolve_path(chunk.path_hash))) - .map(|chunk| chunk.path_hash) - .collect(); - let mut parsed: HashSet = HashSet::new(); - - while let Some(hash) = queue.pop_front() { - if !parsed.insert(hash) { - continue; - } - let Some(&chunk) = chunk_by_hash.get(&hash) else { - continue; - }; - let Some(data) = load_and_decompress(wad, &chunk) else { - continue; - }; - let new_bins = harvest_bin( - &data, - chunk_hashes, - hashtable, - &mut discovered, - &mut scratch, - ); - queue.extend(new_bins); - } - - discovered -} - -#[cfg(test)] -mod tests { - use super::*; - use league_toolkit::meta::value::{ContainerValue, StringValue}; - use league_toolkit::meta::{BinPropertyKind, BinTree, BinTreeObject}; - - #[test] - fn hash_wad_path_is_lowercased_xxh64() { - assert_eq!(hash_wad_path("ASSETS/X.dds"), xxh64(b"assets/x.dds", 0)); - assert_eq!(hash_wad_path("assets/x.dds"), xxh64(b"assets/x.dds", 0)); - } - - #[test] - fn collect_bin_paths_gathers_dependencies_and_nested_strings() { - // A nested container holding a string, plus a top-level string property and a - // dependency link. - let container = ContainerValue { - item_kind: BinPropertyKind::String, - items: vec![StringValue("ASSETS/Characters/Foo/Foo.dds".into()).into()], - }; - let object = BinTreeObject::builder(0x1, 0x2) - .property(0xAA, StringValue("ASSETS/Top.bin".into())) - .property(0xBB, container) - .build(); - let tree = BinTree::builder() - .dependency("DATA/base.bin") - .object(object) - .build(); - - let mut buffer = Cursor::new(Vec::new()); - tree.to_writer(&mut buffer).unwrap(); - let bytes = buffer.into_inner(); - - let mut out = Vec::new(); - collect_bin_paths(&bytes, &mut out); - - assert!(out.contains(&"DATA/base.bin".to_string())); - assert!(out.contains(&"ASSETS/Top.bin".to_string())); - assert!(out.contains(&"ASSETS/Characters/Foo/Foo.dds".to_string())); - } - - #[test] - fn collect_bin_paths_ignores_garbage() { - let mut out = Vec::new(); - collect_bin_paths(&[0, 1, 2, 3, 4, 5, 6, 7], &mut out); - assert!(out.is_empty()); - } - - #[test] - fn scan_wad_recovers_referenced_asset_name() { - use league_toolkit::wad::{Wad, WadBuilder, WadChunkBuilder}; - use std::io::Write; - - // A bin (with a known name) references an asset whose hash is otherwise unknown. - let asset_path = "assets/characters/foo/recovered.dds"; - let tree = BinTree::builder().dependency(asset_path).build(); - let mut bin_buffer = Cursor::new(Vec::new()); - tree.to_writer(&mut bin_buffer).unwrap(); - let bin_bytes = bin_buffer.into_inner(); - - let bin_hash = hash_wad_path("data/test.bin"); - let asset_hash = hash_wad_path(asset_path); - - // Build an in-memory WAD with the bin chunk and the referenced (unnamed) asset. - let mut wad_buffer = Cursor::new(Vec::new()); - WadBuilder::default() - .with_chunk(WadChunkBuilder::default().with_path("data/test.bin")) - .with_chunk(WadChunkBuilder::default().with_path(asset_path)) - .build_to_writer(&mut wad_buffer, |path_hash, cursor| { - if path_hash == bin_hash { - cursor.write_all(&bin_bytes)?; - } else { - cursor.write_all(&[0xAB; 32])?; - } - Ok(()) - }) - .unwrap(); - wad_buffer.set_position(0); - let mut wad = Wad::mount(wad_buffer).unwrap(); - - // Hashtable resolves only the bin path - the asset is anonymous. - let mut hashtable = WadHashtable::new().unwrap(); - hashtable.insert(bin_hash, "data/test.bin"); - - let chunk_hashes: HashSet = [bin_hash, asset_hash].into_iter().collect(); - let discovered = scan_wad_bin_paths(&mut wad, &hashtable, &chunk_hashes, false); - - // The asset's real path is recovered from the bin's dependency link... - assert_eq!( - discovered.get(&asset_hash).map(|path| path.as_ref()), - Some(asset_path) - ); - // ...and the already-known bin hash is not duplicated into the overlay. - assert!(!discovered.contains_key(&bin_hash)); - } - - /// Helper: serialize a `BinTree` to bytes. - fn serialize_bin(tree: &BinTree) -> Vec { - let mut buffer = Cursor::new(Vec::new()); - tree.to_writer(&mut buffer).unwrap(); - buffer.into_inner() - } - - #[test] - fn default_scan_follows_links_into_discovered_bins() { - use league_toolkit::wad::{Wad, WadBuilder, WadChunkBuilder}; - use std::io::Write; - - // Chain: root.bin (known) -> child.bin (anonymous) -> asset.dds (anonymous). - // The asset is only reachable by parsing child.bin, which is itself only found - // by following root.bin's link - so this only works with transitive following. - let child_path = "data/child.bin"; - let asset_path = "assets/deep/asset.dds"; - let root_bytes = serialize_bin(&BinTree::builder().dependency(child_path).build()); - let child_bytes = serialize_bin(&BinTree::builder().dependency(asset_path).build()); - - let root_hash = hash_wad_path("data/root.bin"); - let child_hash = hash_wad_path(child_path); - let asset_hash = hash_wad_path(asset_path); - - let mut wad_buffer = Cursor::new(Vec::new()); - WadBuilder::default() - .with_chunk(WadChunkBuilder::default().with_path("data/root.bin")) - .with_chunk(WadChunkBuilder::default().with_path(child_path)) - .with_chunk(WadChunkBuilder::default().with_path(asset_path)) - .build_to_writer(&mut wad_buffer, |path_hash, cursor| { - if path_hash == root_hash { - cursor.write_all(&root_bytes)?; - } else if path_hash == child_hash { - cursor.write_all(&child_bytes)?; - } else { - cursor.write_all(&[0xCD; 16])?; - } - Ok(()) - }) - .unwrap(); - wad_buffer.set_position(0); - let mut wad = Wad::mount(wad_buffer).unwrap(); - - // Only the root bin is known up front. - let mut hashtable = WadHashtable::new().unwrap(); - hashtable.insert(root_hash, "data/root.bin"); - - let chunk_hashes: HashSet = [root_hash, child_hash, asset_hash].into_iter().collect(); - let discovered = scan_wad_bin_paths(&mut wad, &hashtable, &chunk_hashes, false); - - assert_eq!( - discovered.get(&child_hash).map(|path| path.as_ref()), - Some(child_path) - ); - assert_eq!( - discovered.get(&asset_hash).map(|path| path.as_ref()), - Some(asset_path) - ); - } - - #[test] - fn full_scan_finds_unreferenced_anonymous_bins() { - use league_toolkit::wad::{Wad, WadBuilder, WadChunkBuilder}; - use std::io::Write; - - // An anonymous bin with no hashtable name that is not linked from anything. - let asset_path = "assets/orphan/asset.dds"; - let bin_bytes = serialize_bin(&BinTree::builder().dependency(asset_path).build()); - let bin_hash = hash_wad_path("data/orphan.bin"); - let asset_hash = hash_wad_path(asset_path); - - let mut wad_buffer = Cursor::new(Vec::new()); - WadBuilder::default() - .with_chunk(WadChunkBuilder::default().with_path("data/orphan.bin")) - .with_chunk(WadChunkBuilder::default().with_path(asset_path)) - .build_to_writer(&mut wad_buffer, |path_hash, cursor| { - if path_hash == bin_hash { - cursor.write_all(&bin_bytes)?; - } else { - cursor.write_all(&[0xEE; 16])?; - } - Ok(()) - }) - .unwrap(); - wad_buffer.set_position(0); - let mut wad = Wad::mount(wad_buffer).unwrap(); - - // Empty hashtable: nothing is known, so the bin is anonymous and unreferenced. - let hashtable = WadHashtable::new().unwrap(); - let chunk_hashes: HashSet = [bin_hash, asset_hash].into_iter().collect(); - - // The default (fast) scan cannot find an unnamed, unreferenced bin. - let fast = scan_wad_bin_paths(&mut wad, &hashtable, &chunk_hashes, false); - assert!(fast.is_empty()); - - // The full scan magic-detects the bin and recovers the asset name. - let full = scan_wad_bin_paths(&mut wad, &hashtable, &chunk_hashes, true); - assert_eq!( - full.get(&asset_hash).map(|path| path.as_ref()), - Some(asset_path) - ); - } -} diff --git a/crates/wadtools/src/commands/diff.rs b/crates/wadtools/src/commands/diff.rs index 347b9a0..eb9978f 100644 --- a/crates/wadtools/src/commands/diff.rs +++ b/crates/wadtools/src/commands/diff.rs @@ -5,12 +5,12 @@ use std::{ use colored::Colorize; use fancy_regex::Regex; -use league_toolkit::wad::{Wad, WadChunk}; +use league_toolkit::wad::{Wad, WadChunk, WadHash}; use serde::Serialize; use std::borrow::Cow; use crate::{ - extractor::{should_skip_hash, should_skip_pattern}, + filters::{should_skip_hash, should_skip_pattern}, utils::{format_chunk_path_hash, format_size, WadHashtable}, }; @@ -46,7 +46,7 @@ pub struct DiffArgs { pub target: String, pub output: Option, pub pattern: Option, - pub hash: Option>, + pub hash: Option>, pub filter_invert: bool, } @@ -85,7 +85,7 @@ pub fn diff(args: DiffArgs, hashtable: &WadHashtable) -> eyre::Result<()> { } /// Returns the primary path hash for a diff entry (used for filtering). -fn diff_primary_path_hash(diff: &ChunkDiff) -> u64 { +fn diff_primary_path_hash(diff: &ChunkDiff) -> WadHash { match diff { ChunkDiff::New(chunk) => chunk.path_hash, ChunkDiff::Removed(chunk) => chunk.path_hash, @@ -109,7 +109,7 @@ fn should_skip_diff( diff: &ChunkDiff, hashtable: &WadHashtable, filter_pattern: Option<&Regex>, - hash_filter: Option<&[u64]>, + hash_filter: Option<&[WadHash]>, filter_invert: bool, ) -> bool { let path_hash = diff_primary_path_hash(diff); @@ -127,7 +127,7 @@ fn print_diffs( diffs: &[ChunkDiff], hashtable: &WadHashtable, filter_pattern: Option<&Regex>, - hash_filter: Option<&[u64]>, + hash_filter: Option<&[WadHash]>, filter_invert: bool, ) { // Sort by resolved path @@ -284,7 +284,7 @@ fn write_diffs_to_csv( hashtable: &WadHashtable, output_path: &str, filter_pattern: Option<&Regex>, - hash_filter: Option<&[u64]>, + hash_filter: Option<&[WadHash]>, filter_invert: bool, ) -> eyre::Result<()> { tracing::info!("Writing diffs to CSV file: {}", output_path.bright_cyan()); @@ -315,7 +315,7 @@ fn create_csv_records( diffs: &[ChunkDiff], hashtable: &WadHashtable, filter_pattern: Option<&Regex>, - hash_filter: Option<&[u64]>, + hash_filter: Option<&[WadHash]>, filter_invert: bool, ) -> Vec { let mut records = Vec::::new(); diff --git a/crates/wadtools/src/commands/extract.rs b/crates/wadtools/src/commands/extract.rs index 6e41613..4b95c59 100644 --- a/crates/wadtools/src/commands/extract.rs +++ b/crates/wadtools/src/commands/extract.rs @@ -1,138 +1,177 @@ use camino::{Utf8Path, Utf8PathBuf}; use color_eyre::owo_colors::OwoColorize; -use std::collections::{HashMap, HashSet}; +use convert_case::{Case, Casing}; +use fancy_regex::Regex; use std::fs::File; -use std::sync::Arc; -use league_toolkit::{file::LeagueFileKind, wad::Wad}; +use league_toolkit::{ + file::LeagueFileKind, + wad::{ExistingFilePolicy, ExtractReport, Wad, WadExtractor, WadHash}, +}; +use tracing_indicatif::span_ext::IndicatifSpanExt; +use tracing_indicatif::style::ProgressStyle; use crate::{ - bin_scan::scan_wad_bin_paths, - extractor::Extractor, - utils::{create_filter_pattern, format_size, WadHashtable}, + filters::{should_skip_hash, should_skip_pattern, should_skip_type}, + utils::{create_filter_pattern, format_size, truncate_middle, WadHashtable}, }; -use convert_case::{Case, Casing}; + +const MAX_LOG_PATH_LEN: usize = 120; pub struct ExtractArgs { pub input: String, pub output: Option, pub filter_type: Option>, pub pattern: Option, - pub hash: Option>, + pub hash: Option>, pub filter_invert: bool, pub overwrite: bool, pub show_stats: bool, pub resolve_bin_paths: bool, - pub full_bin_scan: bool, } -/// Scan the WAD's `.bin` files for path strings so otherwise-anonymous chunks can be -/// extracted under their real names. -fn discover_bin_paths( - wad: &mut Wad, - hashtable: &WadHashtable, - full_bin_scan: bool, -) -> HashMap> { - if full_bin_scan { - tracing::info!("scanning all chunks for bin paths (full mode)"); - } +pub fn extract(mut args: ExtractArgs, hashtable: &WadHashtable) -> eyre::Result<()> { + let source = File::open(&args.input)?; + let mut wad = Wad::mount(source)?; - let chunk_hashes: HashSet = wad.chunks().iter().map(|c| c.path_hash).collect(); - let discovered = scan_wad_bin_paths(wad, hashtable, &chunk_hashes, full_bin_scan); - if !discovered.is_empty() { + let filter_pattern = create_filter_pattern(args.pattern.take())?; + let output_dir = resolve_output_dir(&args.input, args.output.as_deref()); + + let report = run( + &mut wad, + hashtable, + &args, + filter_pattern.as_ref(), + &output_dir, + )?; + + if !report.recovered.is_empty() { tracing::info!( "recovered {} chunk name(s) from bin files", - discovered.len() + report.recovered.len() ); } - discovered + if args.show_stats { + print_stats(&args.input, &report); + } else if report.skipped_existing > 0 { + tracing::info!( + "extracted {} chunks, skipped {} existing :)", + report.extracted, + report.skipped_existing + ); + } else { + tracing::info!("extracted {} chunks :)", report.extracted); + } + + Ok(()) } -pub fn extract(args: ExtractArgs, hashtable: &WadHashtable) -> eyre::Result<()> { - let source = File::open(&args.input)?; +/// Runs the extraction under its progress span, so the bar is gone before the +/// summary prints. +fn run( + wad: &mut Wad, + hashtable: &WadHashtable, + args: &ExtractArgs, + filter_pattern: Option<&Regex>, + output_dir: &Utf8Path, +) -> eyre::Result { + let filter_invert = args.filter_invert; - let mut wad = Wad::mount(source)?; + // The hash filter is the cheapest check, so it picks the chunks before anything is read. + let selected: Vec = wad + .chunks() + .iter() + .map(|chunk| chunk.path_hash) + .filter(|&path_hash| !should_skip_hash(path_hash, args.hash.as_deref(), filter_invert)) + .collect(); - let discovered = match args.resolve_bin_paths { - true => discover_bin_paths(&mut wad, hashtable, args.full_bin_scan), - false => HashMap::new(), - }; - let recovered = discovered.len(); + let span = tracing::info_span!("extract", total = selected.len()); + let _entered = span.enter(); + span.pb_set_style( + &ProgressStyle::with_template("{wide_bar:40.cyan/blue} {pos}/{len} \n {spinner} {msg}") + .unwrap(), + ); + span.pb_set_length(selected.len() as u64); + span.pb_set_message("Extracting chunks"); - let mut extractor = Extractor::new(&mut wad, hashtable); - extractor.set_discovered(discovered); + let existing = if args.overwrite { + ExistingFilePolicy::Overwrite + } else { + ExistingFilePolicy::Skip + }; + let mut extractor = WadExtractor::new(hashtable) + .with_existing_file_policy(existing) + .on_progress(|progress| { + span.pb_set_message(&truncate_middle(progress.path(), MAX_LOG_PATH_LEN)); + span.pb_set_position(progress.done() as u64); + }); + if let Some(pattern) = filter_pattern { + extractor = extractor + .with_filter(move |path| !should_skip_pattern(path, Some(pattern), filter_invert)); + } + if let Some(kinds) = args.filter_type.as_deref() { + // The crate's type filter only keeps, so `-v` hands it the complement. + let kept = LeagueFileKind::iter() + .filter(|&kind| !should_skip_type(kind, Some(kinds), filter_invert)); + extractor = extractor.with_type_filter(kept); + } + if args.resolve_bin_paths { + extractor = extractor.with_name_recovery(); + } - let filter_pattern = create_filter_pattern(args.pattern)?; + Ok(extractor.extract_chunks(wad, selected, output_dir)?) +} - extractor.set_filter_pattern(filter_pattern); - extractor.set_hash_filter(args.hash); - extractor.set_filter_invert(args.filter_invert); - let output_dir: Utf8PathBuf = match &args.output { - Some(path) => Utf8PathBuf::from(path.as_str()), +/// The output directory, or a sibling directory named after the input file +/// without its extension. +fn resolve_output_dir(input: &str, output: Option<&str>) -> Utf8PathBuf { + match output { + Some(path) => Utf8PathBuf::from(path), None => { - // Construct sibling dir named after input file (without extension) - let input_path = Utf8Path::new(&args.input); + let input_path = Utf8Path::new(input); let parent = input_path.parent().unwrap_or(Utf8Path::new(".")); let stem = input_path.file_stem().unwrap_or("extracted"); parent.join(stem) } - }; - let stats = - extractor.extract_chunks(&output_dir, args.filter_type.as_deref(), args.overwrite)?; + } +} - if args.show_stats { - println!(); +fn print_stats(input: &str, report: &ExtractReport) { + println!(); + println!("{}: {}", "WAD".bright_cyan().bold(), input.bright_white()); + println!( + "{}: {} chunks ({})", + "Extracted".bright_cyan().bold(), + report.extracted.to_string().bright_green(), + format_size(report.bytes_written).bright_white() + ); + println!( + "{}: {} existing", + "Skipped".bright_cyan().bold(), + report.skipped_existing.to_string().bright_yellow() + ); + if !report.recovered.is_empty() { println!( - "{}: {}", - "WAD".bright_cyan().bold(), - args.input.bright_white() + "{}: {} names from bins", + "Recovered".bright_cyan().bold(), + report.recovered.len().to_string().bright_green() ); - println!( - "{}: {} chunks ({})", - "Extracted".bright_cyan().bold(), - stats.extracted_count.to_string().bright_green(), - format_size(stats.bytes_written).bright_white() - ); - println!( - "{}: {} existing", - "Skipped".bright_cyan().bold(), - stats.skipped_existing.to_string().bright_yellow() - ); - if recovered > 0 { + } + if !report.by_kind.is_empty() { + println!(); + println!("{}:", "By type".bright_cyan().bold()); + let mut type_entries: Vec<_> = report.by_kind.iter().collect(); + type_entries.sort_by(|a, b| b.1.cmp(a.1)); + for (kind, count) in type_entries { + let name = format!("{:?}", kind).to_case(Case::Snake); println!( - "{}: {} names from bins", - "Recovered".bright_cyan().bold(), - recovered.to_string().bright_green() + " {:24} {}", + name.bright_magenta(), + count.to_string().bright_white() ); } - if !stats.by_type.is_empty() { - println!(); - println!("{}:", "By type".bright_cyan().bold()); - let mut type_entries: Vec<_> = stats.by_type.iter().collect(); - type_entries.sort_by(|a, b| b.1.cmp(a.1)); - for (kind, count) in type_entries { - let name = format!("{:?}", kind).to_case(Case::Snake); - println!( - " {:24} {}", - name.bright_magenta(), - count.to_string().bright_white() - ); - } - } - } else { - if stats.skipped_existing > 0 { - tracing::info!( - "extracted {} chunks, skipped {} existing :)", - stats.extracted_count, - stats.skipped_existing - ); - } else { - tracing::info!("extracted {} chunks :)", stats.extracted_count); - } } - - Ok(()) } pub fn print_supported_filters() { diff --git a/crates/wadtools/src/commands/list.rs b/crates/wadtools/src/commands/list.rs index 82209b8..4359168 100644 --- a/crates/wadtools/src/commands/list.rs +++ b/crates/wadtools/src/commands/list.rs @@ -1,11 +1,14 @@ use camino::Utf8Path; use color_eyre::owo_colors::OwoColorize; -use league_toolkit::{file::LeagueFileKind, wad::Wad}; +use league_toolkit::{ + file::LeagueFileKind, + wad::{Wad, WadHash}, +}; use serde::Serialize; use std::fs::File; use crate::{ - extractor::{should_skip_hash, should_skip_pattern, should_skip_type}, + filters::{should_skip_hash, should_skip_pattern, should_skip_type}, utils::{create_filter_pattern, format_chunk_path_hash, format_size, WadHashtable}, }; @@ -26,7 +29,7 @@ pub struct ListArgs { pub input: String, pub filter_type: Option>, pub pattern: Option, - pub hash: Option>, + pub hash: Option>, pub filter_invert: bool, pub format: ListOutputFormat, pub show_stats: bool, @@ -66,7 +69,7 @@ pub fn list(args: ListArgs, hashtable: &WadHashtable) -> eyre::Result<()> { let mut total_uncompressed: u64 = 0; let chunks = wad.chunks().as_slice(); - let path_hashes: Vec = chunks.iter().map(|chunk| chunk.path_hash).collect(); + let path_hashes: Vec = chunks.iter().map(|chunk| chunk.path_hash.0).collect(); let resolved_paths = hashtable.resolve_batch(&path_hashes); for (chunk, path_str) in chunks.iter().zip(&resolved_paths) { diff --git a/crates/wadtools/src/commands/paths.rs b/crates/wadtools/src/commands/paths.rs index 22205e0..b65f7fa 100644 --- a/crates/wadtools/src/commands/paths.rs +++ b/crates/wadtools/src/commands/paths.rs @@ -4,9 +4,9 @@ //! //! "Resolvable" means every chunk whose hash we can attribute to a real path - //! names already known to the shared mimir cache *plus* names recovered by -//! scanning the WAD's `.bin` files (dependency links and string properties, see -//! [`crate::bin_scan`]). Chunks that would only render as their 16-hex fallback are -//! skipped, so the output is a clean, human-meaningful path list. +//! scanning the WAD's `.bin` files (see [`NameRecovery`]). Chunks that would only +//! render as their 16-hex fallback are skipped, so the output is a clean, +//! human-meaningful path list. //! //! The `.lhdb` output is written in the Game-table configuration (64-bit XXH64 keys, //! case-insensitive), so it is a drop-in supplemental table for any LeagueToolkit @@ -15,17 +15,15 @@ use camino::Utf8Path; use color_eyre::owo_colors::OwoColorize; use eyre::Context; -use league_toolkit::wad::Wad; -use std::collections::{BTreeMap, HashMap, HashSet}; +use league_toolkit::wad::{NameRecovery, PathResolver, RecoveredNames, Wad, WadHash}; +use std::collections::BTreeMap; use std::fs::File; use std::io::{BufWriter, Write}; -use std::sync::Arc; use ltk_hashdb::{Casing, Compression, HashDbWriter, HashKind, KeyWidth}; use crate::{ - bin_scan::scan_wad_bin_paths, - extractor::should_skip_pattern, + filters::should_skip_pattern, utils::{create_filter_pattern, format_chunk_path_hash, WadHashtable}, }; @@ -58,8 +56,6 @@ pub struct PathsArgs { pub filter_invert: bool, /// Scan `.bin` files to recover otherwise-anonymous chunk names (default: true). pub resolve_bin_paths: bool, - /// Decompress every chunk to magic-detect bins, recovering the most names. - pub full_bin_scan: bool, pub show_stats: bool, } @@ -73,7 +69,7 @@ pub fn rip_paths(args: PathsArgs, hashtable: &WadHashtable) -> eyre::Result<()> // Deduplicated across every input WAD; keyed by chunk hash so a path shared by // multiple WADs is written once. - let mut collected: BTreeMap> = BTreeMap::new(); + let mut collected: BTreeMap = BTreeMap::new(); let mut recovered_total = 0usize; for input in &args.inputs { @@ -81,23 +77,20 @@ pub fn rip_paths(args: PathsArgs, hashtable: &WadHashtable) -> eyre::Result<()> let mut wad = Wad::mount(source).wrap_err_with(|| format!("failed to mount WAD '{input}'"))?; - let chunk_hashes: HashSet = wad.chunks().iter().map(|c| c.path_hash).collect(); - - let discovered = if args.resolve_bin_paths { - scan_wad_bin_paths(&mut wad, hashtable, &chunk_hashes, args.full_bin_scan) + let recovered = if args.resolve_bin_paths { + NameRecovery::new() + .run(&mut wad, hashtable) + .wrap_err_with(|| format!("failed to scan the bins of WAD '{input}'"))? } else { - HashMap::new() + RecoveredNames::default() }; - recovered_total += discovered.len(); - - for &hash in &chunk_hashes { - // Prefer bin-recovered names, then fall back to the shared cache. A chunk - // that neither source resolves would only be the 16-hex fallback - skip it. - let path: Arc = if let Some(path) = discovered.get(&hash) { - path.clone() - } else if hashtable.contains(hash) { - Arc::from(hashtable.resolve_path(hash).as_ref()) - } else { + recovered_total += recovered.len(); + + // Bin-recovered names first, then the shared cache. A chunk that neither + // source resolves would only be the 16-hex fallback - skip it. + let resolver = recovered.over(hashtable); + for chunk in wad.chunks() { + let Some(path) = resolver.resolve(chunk.path_hash) else { continue; }; @@ -105,7 +98,9 @@ pub fn rip_paths(args: PathsArgs, hashtable: &WadHashtable) -> eyre::Result<()> continue; } - collected.entry(hash).or_insert(path); + collected + .entry(chunk.path_hash) + .or_insert_with(|| path.into_owned()); } } @@ -179,8 +174,8 @@ fn resolve_output_path( /// Writes the CDragon hashtable format (` ` text), sorted by path /// for readable, deterministic diffs. -fn write_txt(output: &str, collected: &BTreeMap>) -> eyre::Result<()> { - let mut entries: Vec<(&u64, &Arc)> = collected.iter().collect(); +fn write_txt(output: &str, collected: &BTreeMap) -> eyre::Result<()> { + let mut entries: Vec<(&WadHash, &String)> = collected.iter().collect(); entries.sort_by(|a, b| a.1.cmp(b.1)); let file = File::create(output) @@ -197,13 +192,13 @@ fn write_txt(output: &str, collected: &BTreeMap>) -> eyre::Result< /// cleanly on top of the shared cache (XXH64 keys over the lowercased path). fn write_lhdb( output: &str, - collected: &BTreeMap>, + collected: &BTreeMap, ) -> eyre::Result { let mut writer = HashDbWriter::new(KeyWidth::U64, Compression::default()) .hash_kind(HashKind::Xxh64) .casing(Casing::Insensitive); for (&hash, path) in collected { - writer.insert(hash, path); + writer.insert(hash.0, path); } let file = File::create(output) @@ -217,7 +212,7 @@ fn write_lhdb( fn print_stats( output: &str, format: PathsFormat, - collected: &BTreeMap>, + collected: &BTreeMap, recovered_total: usize, build_stats: Option, ) { @@ -254,8 +249,8 @@ fn print_stats( #[cfg(test)] mod tests { use super::*; - use crate::bin_scan::hash_wad_path; - use league_toolkit::meta::BinTree; + use league_toolkit::hash::Hash as _; + use league_toolkit::meta::{property::NoMeta, Bin}; use league_toolkit::wad::{WadBuilder, WadChunkBuilder}; use ltk_hashdb::HashDb; use std::io::{Cursor, Write}; @@ -309,9 +304,9 @@ mod tests { struct Fixture { wad_path: std::path::PathBuf, hashtable: WadHashtable, - bin_hash: u64, - asset_hash: u64, - unknown_hash: u64, + bin_hash: WadHash, + asset_hash: WadHash, + unknown_hash: WadHash, } fn build_fixture(tag: &str) -> Fixture { @@ -321,15 +316,15 @@ mod tests { // The known bin links to the anonymous asset, so a scan recovers its name. let bin_bytes = { - let tree = BinTree::builder().dependency(asset_path).build(); + let tree = Bin::::builder().dependency(asset_path).build(); let mut buffer = Cursor::new(Vec::new()); tree.to_writer(&mut buffer).unwrap(); buffer.into_inner() }; - let bin_hash = hash_wad_path(bin_path); - let asset_hash = hash_wad_path(asset_path); - let unknown_hash = hash_wad_path(unknown_path); + let bin_hash = WadHash::hash_str(bin_path); + let asset_hash = WadHash::hash_str(asset_path); + let unknown_hash = WadHash::hash_str(unknown_path); let wad_path = std::env::temp_dir().join(format!( "wadtools_paths_{tag}_{}.wad.client", @@ -373,7 +368,6 @@ mod tests { pattern: None, filter_invert: false, resolve_bin_paths: true, - full_bin_scan: false, show_stats: false, } } @@ -431,17 +425,17 @@ mod tests { // by the same XXH64 chunk hashes the WAD keys on. let db = HashDb::open(&out).unwrap(); assert_eq!( - db.get(fx.bin_hash).as_deref(), + db.get(fx.bin_hash.0).as_deref(), Some("data/test.bin"), "bin path not resolvable from the .lhdb" ); assert_eq!( - db.get(fx.asset_hash).as_deref(), + db.get(fx.asset_hash.0).as_deref(), Some("assets/characters/foo/recovered.dds"), "recovered asset not resolvable from the .lhdb" ); assert!( - !db.contains(fx.unknown_hash), + !db.contains(fx.unknown_hash.0), "hex-only orphan should not be present in the .lhdb" ); diff --git a/crates/wadtools/src/extractor.rs b/crates/wadtools/src/extractor.rs deleted file mode 100644 index 3b35c5c..0000000 --- a/crates/wadtools/src/extractor.rs +++ /dev/null @@ -1,483 +0,0 @@ -use crate::utils::{is_hex_chunk_path, truncate_middle, WadHashtable}; -use camino::{Utf8Path, Utf8PathBuf}; -use color_eyre::eyre::{self, Ok}; -use dashmap::DashMap; -use eyre::Context; -use fancy_regex::Regex; -use league_toolkit::{ - file::LeagueFileKind, - wad::{decompress_raw, Wad, WadChunk}, -}; -use std::{ - borrow::Cow, - collections::HashMap, - fs::{self, File, OpenOptions}, - io::{self, Write}, - sync::{ - atomic::{AtomicU64, AtomicUsize, Ordering}, - mpsc, Arc, - }, -}; -use tracing_indicatif::span_ext::IndicatifSpanExt; -use tracing_indicatif::style::ProgressStyle; - -const MAX_LOG_PATH_LEN: usize = 120; - -pub struct ExtractStats { - pub extracted_count: usize, - pub skipped_existing: usize, - pub bytes_written: u64, - pub by_type: HashMap, -} - -enum ChunkResult { - Extracted(LeagueFileKind, u64), - SkippedFilter, - SkippedExisting, -} - -pub struct Extractor<'a> { - wad: &'a mut Wad, - hashtable: &'a WadHashtable, - /// Names recovered from `.bin` files for this WAD, consulted before the shared - /// hashtable. Gap-fill only - never contains a hash the hashtable already resolves. - discovered: HashMap>, - filter_pattern: Option, - hash_filter: Option>, - filter_invert: bool, -} - -impl<'a> Extractor<'a> { - pub fn new(wad: &'a mut Wad, hashtable: &'a WadHashtable) -> Self { - Self { - wad, - hashtable, - discovered: HashMap::new(), - filter_pattern: None, - hash_filter: None, - filter_invert: false, - } - } - - pub fn set_discovered(&mut self, discovered: HashMap>) { - self.discovered = discovered; - } - - /// Resolves a chunk's path, preferring names recovered from bins over the shared - /// hashtable (which falls back to a hex string when the hash is unknown). - fn resolve_path(&self, path_hash: u64) -> Cow<'_, str> { - match self.discovered.get(&path_hash) { - Some(path) => Cow::Borrowed(path), - None => self.hashtable.resolve_path(path_hash), - } - } - - pub fn set_filter_pattern(&mut self, filter_pattern: Option) { - self.filter_pattern = filter_pattern; - } - - pub fn set_hash_filter(&mut self, hash_filter: Option>) { - self.hash_filter = hash_filter; - } - - pub fn set_filter_invert(&mut self, filter_invert: bool) { - self.filter_invert = filter_invert; - } - - pub fn extract_chunks( - &mut self, - extract_directory: impl AsRef, - filter_type: Option<&[LeagueFileKind]>, - overwrite: bool, - ) -> eyre::Result { - let extract_directory = extract_directory.as_ref().to_path_buf(); - - let chunks: Vec = self.wad.chunks().iter().copied().collect(); - let total = chunks.len() as u64; - - let span = tracing::info_span!("extract", total = total); - let _entered = span.enter(); - span.pb_set_style( - &ProgressStyle::with_template("{wide_bar:40.cyan/blue} {pos}/{len} \n {spinner} {msg}") - .unwrap(), - ); - span.pb_set_length(total); - span.pb_set_message("Extracting chunks"); - - // Bounded channel: caps in-flight raw chunks to limit memory usage. - // The sequential reader blocks when the channel is full (workers are busy). - let buffer_size = rayon::current_num_threads().max(1); - let (tx, rx) = mpsc::sync_channel::<(WadChunk, String, Box<[u8]>)>(buffer_size); - - let counter = AtomicUsize::new(0); - let extracted_counter = AtomicUsize::new(0); - let skipped_existing_counter = AtomicUsize::new(0); - let bytes_written_counter = AtomicU64::new(0); - let by_type: DashMap = DashMap::new(); - let filter_invert = self.filter_invert; - let extract_dir = &extract_directory; - let err_holder: std::sync::Mutex> = std::sync::Mutex::new(None); - - std::thread::scope(|s| -> eyre::Result<()> { - // Worker thread: receive chunks and decompress+write in parallel via rayon::scope - let worker = s.spawn(|| { - rayon::scope(|rayon_scope| { - for (chunk, path_str, raw) in rx { - let counter = &counter; - let extracted_counter = &extracted_counter; - let skipped_existing_counter = &skipped_existing_counter; - let bytes_written_counter = &bytes_written_counter; - let by_type = &by_type; - let err_holder = &err_holder; - let progress_span = &span; - - rayon_scope.spawn(move |_| { - let result = process_chunk( - &chunk, - &path_str, - &raw, - extract_dir, - filter_type, - filter_invert, - overwrite, - ); - - match result { - std::result::Result::Ok(ChunkResult::Extracted(kind, size)) => { - extracted_counter.fetch_add(1, Ordering::Relaxed); - bytes_written_counter.fetch_add(size, Ordering::Relaxed); - *by_type.entry(kind).or_insert(0) += 1; - } - std::result::Result::Ok(ChunkResult::SkippedExisting) => { - skipped_existing_counter.fetch_add(1, Ordering::Relaxed); - } - std::result::Result::Ok(ChunkResult::SkippedFilter) => {} - Err(e) => { - let mut guard = err_holder.lock().unwrap(); - if guard.is_none() { - *guard = Some(e); - } - } - } - - let done = counter.fetch_add(1, Ordering::Relaxed) + 1; - progress_span.pb_set_position(done as u64); - }); - } - }); - // rayon::scope blocks until all spawned tasks complete - }); - - // Sequential reader: read raw bytes and send through bounded channel - for chunk in chunks.iter() { - if err_holder.lock().unwrap().is_some() { - break; - } - - // Hash filter is the cheapest check - do it before resolving path - if should_skip_hash( - chunk.path_hash, - self.hash_filter.as_deref(), - self.filter_invert, - ) { - let done = counter.fetch_add(1, Ordering::Relaxed) + 1; - span.pb_set_position(done as u64); - continue; - } - - let chunk_path_str = self.resolve_path(chunk.path_hash).into_owned(); - - span.pb_set_message(&truncate_middle(chunk_path_str.as_ref(), MAX_LOG_PATH_LEN)); - - if should_skip_pattern( - chunk_path_str.as_ref(), - self.filter_pattern.as_ref(), - self.filter_invert, - ) { - let done = counter.fetch_add(1, Ordering::Relaxed) + 1; - span.pb_set_position(done as u64); - continue; - } - - let raw = self.wad.load_chunk_raw(chunk).wrap_err(format!( - "failed to read raw chunk (chunk_path: {})", - chunk_path_str.as_str() - ))?; - - // Blocks if the channel is full, bounding memory - let _ = tx.send((*chunk, chunk_path_str, raw)); - } - - drop(tx); - worker.join().unwrap(); - - Ok(()) - })?; - - if let Some(err) = err_holder.into_inner().unwrap() { - return Err(err); - } - - let by_type_map: HashMap = by_type.into_iter().collect(); - - Ok(ExtractStats { - extracted_count: extracted_counter.load(Ordering::Relaxed), - skipped_existing: skipped_existing_counter.load(Ordering::Relaxed), - bytes_written: bytes_written_counter.load(Ordering::Relaxed), - by_type: by_type_map, - }) - } -} - -fn process_chunk( - chunk: &WadChunk, - path_str: &str, - raw: &[u8], - extract_dir: &Utf8Path, - filter_type: Option<&[LeagueFileKind]>, - filter_invert: bool, - overwrite: bool, -) -> eyre::Result { - let chunk_data = decompress_raw(raw, chunk.compression_type, chunk.uncompressed_size) - .wrap_err(format!( - "failed to decompress chunk (chunk_path: {})", - path_str - ))?; - - let chunk_kind = LeagueFileKind::identify_from_bytes(&chunk_data); - if should_skip_type(chunk_kind, filter_type, filter_invert) { - return Ok(ChunkResult::SkippedFilter); - } - - let chunk_path = Utf8Path::new(path_str); - let final_path = resolve_final_chunk_path(extract_dir, chunk_path, &chunk_data, chunk_kind); - let full_path = extract_dir.join(&final_path); - - if let Some(parent) = full_path.parent() { - fs::create_dir_all(parent.as_std_path())?; - } - - let size = chunk_data.len() as u64; - match write_chunk_file(full_path.as_std_path(), &chunk_data, overwrite) { - std::result::Result::Ok(ChunkWriteResult::Written) => { - Ok(ChunkResult::Extracted(chunk_kind, size)) - } - std::result::Result::Ok(ChunkWriteResult::SkippedExisting) => { - Ok(ChunkResult::SkippedExisting) - } - Err(error) if error.kind() == io::ErrorKind::InvalidFilename => write_long_filename_chunk( - chunk, - final_path, - extract_dir, - &chunk_data, - chunk_kind, - overwrite, - ), - Err(error) => Err(error).wrap_err(format!( - "failed to write chunk (chunk_path: {})", - truncate_middle(full_path.as_str(), MAX_LOG_PATH_LEN) - )), - } -} - -enum ChunkWriteResult { - Written, - SkippedExisting, -} - -/// Writes chunk data to a file. When `overwrite` is false, uses `create_new(true)` for an -/// atomic existence check, returning `SkippedExisting` on `AlreadyExists`. This avoids the -/// TOCTOU race of a separate exists() check followed by write(). -fn write_chunk_file( - path: &std::path::Path, - data: &[u8], - overwrite: bool, -) -> io::Result { - if overwrite { - fs::write(path, data)?; - return std::result::Result::Ok(ChunkWriteResult::Written); - } - - match OpenOptions::new().write(true).create_new(true).open(path) { - std::result::Result::Ok(mut file) => { - file.write_all(data)?; - std::result::Result::Ok(ChunkWriteResult::Written) - } - Err(e) if e.kind() == io::ErrorKind::AlreadyExists => { - tracing::debug!("skipping existing file: {}", path.display()); - std::result::Result::Ok(ChunkWriteResult::SkippedExisting) - } - Err(e) => Err(e), - } -} - -fn resolve_final_chunk_path( - extract_directory: impl AsRef, - chunk_path: impl AsRef, - chunk_data: &[u8], - chunk_kind: LeagueFileKind, -) -> Utf8PathBuf { - let mut final_path = chunk_path.as_ref().to_path_buf(); - - // Hashed filenames should keep the 16-hex base, but we can append a real extension - if is_hex_chunk_path(final_path.as_path()) { - if let Some(ext) = chunk_kind.extension() { - final_path.set_extension(ext); - } - return final_path; - } - - // - If the original path has no extension, affix .ltk (and real extension if known) - // - OR if the destination path collides with an existing directory, affix .ltk - let has_extension = final_path.extension().is_some(); - let collides_with_dir = extract_directory.as_ref().join(&final_path).is_dir(); - if !has_extension || collides_with_dir { - let original_stem = chunk_path.as_ref().file_stem().unwrap_or(""); - let new_name = build_ltk_name(original_stem, chunk_data); - final_path.set_file_name(&new_name); - } - - final_path -} - -fn build_ltk_name(file_stem: &str, chunk_data: &[u8]) -> String { - let kind = LeagueFileKind::identify_from_bytes(chunk_data); - match kind.extension() { - Some(ext) => format!("{}.ltk.{}", file_stem, ext), - None => format!("{}.ltk", file_stem), - } -} - -/// Returns true if the chunk should be skipped based on the pattern filter. -pub(crate) fn should_skip_pattern( - path: &str, - filter_pattern: Option<&Regex>, - filter_invert: bool, -) -> bool { - if let Some(regex) = filter_pattern { - let matched = regex.is_match(path).unwrap_or(false); - return matched == filter_invert; - } - false -} - -/// Returns true if the chunk should be skipped based on the hash filter. -pub(crate) fn should_skip_hash( - path_hash: u64, - hash_filter: Option<&[u64]>, - filter_invert: bool, -) -> bool { - hash_filter.is_some_and(|hashes| hashes.contains(&path_hash) == filter_invert) -} - -/// Returns true if the chunk should be skipped based on the type filter. -pub(crate) fn should_skip_type( - chunk_kind: LeagueFileKind, - filter_type: Option<&[LeagueFileKind]>, - filter_invert: bool, -) -> bool { - filter_type.is_some_and(|filter| filter.contains(&chunk_kind) == filter_invert) -} - -fn write_long_filename_chunk( - chunk: &WadChunk, - chunk_path: impl AsRef, - extract_directory: impl AsRef, - chunk_data: &[u8], - chunk_kind: LeagueFileKind, - overwrite: bool, -) -> eyre::Result { - let mut hashed_path = Utf8PathBuf::from(format!("{:016x}", chunk.path_hash)); - if let Some(ext) = chunk_kind.extension() { - hashed_path.set_extension(ext); - } - - let full_path = extract_directory.as_ref().join(&hashed_path); - - let disp = chunk_path.as_ref().as_str().to_string(); - let truncated = truncate_middle(&disp, MAX_LOG_PATH_LEN); - tracing::warn!( - "Long filename detected (chunk_path: {}, hashed_path: {})", - truncated, - &hashed_path - ); - - let size = chunk_data.len() as u64; - match write_chunk_file(full_path.as_std_path(), chunk_data, overwrite)? { - ChunkWriteResult::Written => Ok(ChunkResult::Extracted(chunk_kind, size)), - ChunkWriteResult::SkippedExisting => Ok(ChunkResult::SkippedExisting), - } -} - -#[cfg(test)] -mod tests { - use super::*; - - fn regex(pattern: &str) -> Regex { - Regex::new(pattern).unwrap() - } - - // --- should_skip_pattern tests --- - - #[test] - fn no_pattern_never_skips() { - assert!(!should_skip_pattern("anything.dds", None, false)); - assert!(!should_skip_pattern("anything.dds", None, true)); - } - - #[test] - fn pattern_includes_matching() { - let re = regex(r"(?i)\.dds$"); - assert!(!should_skip_pattern("textures/foo.dds", Some(&re), false)); - } - - #[test] - fn pattern_excludes_non_matching() { - let re = regex(r"(?i)\.dds$"); - assert!(should_skip_pattern("sounds/bar.wav", Some(&re), false)); - } - - #[test] - fn pattern_inverted_excludes_matching() { - let re = regex(r"(?i)\.dds$"); - assert!(should_skip_pattern("textures/foo.dds", Some(&re), true)); - } - - #[test] - fn pattern_inverted_includes_non_matching() { - let re = regex(r"(?i)\.dds$"); - assert!(!should_skip_pattern("sounds/bar.wav", Some(&re), true)); - } - - // --- should_skip_type tests --- - - #[test] - fn no_type_filter_never_skips() { - assert!(!should_skip_type(LeagueFileKind::Png, None, false)); - assert!(!should_skip_type(LeagueFileKind::Png, None, true)); - } - - #[test] - fn type_filter_includes_matching() { - let types = [LeagueFileKind::Png]; - assert!(!should_skip_type(LeagueFileKind::Png, Some(&types), false)); - } - - #[test] - fn type_filter_excludes_non_matching() { - let types = [LeagueFileKind::Png]; - assert!(should_skip_type(LeagueFileKind::Jpeg, Some(&types), false)); - } - - #[test] - fn type_filter_inverted_excludes_matching() { - let types = [LeagueFileKind::Png]; - assert!(should_skip_type(LeagueFileKind::Png, Some(&types), true)); - } - - #[test] - fn type_filter_inverted_includes_non_matching() { - let types = [LeagueFileKind::Png]; - assert!(!should_skip_type(LeagueFileKind::Jpeg, Some(&types), true)); - } -} diff --git a/crates/wadtools/src/filters.rs b/crates/wadtools/src/filters.rs new file mode 100644 index 0000000..b3a24fc --- /dev/null +++ b/crates/wadtools/src/filters.rs @@ -0,0 +1,131 @@ +//! The `-f`, `-x` and `--hash` filters and their `-v` inversion, as one rule each +//! for every command. + +use fancy_regex::Regex; +use league_toolkit::{file::LeagueFileKind, wad::WadHash}; + +/// Returns true if the chunk should be skipped based on the pattern filter. +pub(crate) fn should_skip_pattern( + path: &str, + filter_pattern: Option<&Regex>, + filter_invert: bool, +) -> bool { + if let Some(regex) = filter_pattern { + let matched = regex.is_match(path).unwrap_or(false); + return matched == filter_invert; + } + false +} + +/// Returns true if the chunk should be skipped based on the hash filter. +pub(crate) fn should_skip_hash( + path_hash: WadHash, + hash_filter: Option<&[WadHash]>, + filter_invert: bool, +) -> bool { + hash_filter.is_some_and(|hashes| hashes.contains(&path_hash) == filter_invert) +} + +/// Returns true if the chunk should be skipped based on the type filter. +pub(crate) fn should_skip_type( + chunk_kind: LeagueFileKind, + filter_type: Option<&[LeagueFileKind]>, + filter_invert: bool, +) -> bool { + filter_type.is_some_and(|filter| filter.contains(&chunk_kind) == filter_invert) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn regex(pattern: &str) -> Regex { + Regex::new(pattern).unwrap() + } + + // --- should_skip_pattern tests --- + + #[test] + fn no_pattern_never_skips() { + assert!(!should_skip_pattern("anything.dds", None, false)); + assert!(!should_skip_pattern("anything.dds", None, true)); + } + + #[test] + fn pattern_includes_matching() { + let re = regex(r"(?i)\.dds$"); + assert!(!should_skip_pattern("textures/foo.dds", Some(&re), false)); + } + + #[test] + fn pattern_excludes_non_matching() { + let re = regex(r"(?i)\.dds$"); + assert!(should_skip_pattern("sounds/bar.wav", Some(&re), false)); + } + + #[test] + fn pattern_inverted_excludes_matching() { + let re = regex(r"(?i)\.dds$"); + assert!(should_skip_pattern("textures/foo.dds", Some(&re), true)); + } + + #[test] + fn pattern_inverted_includes_non_matching() { + let re = regex(r"(?i)\.dds$"); + assert!(!should_skip_pattern("sounds/bar.wav", Some(&re), true)); + } + + // --- should_skip_hash tests --- + + #[test] + fn no_hash_filter_never_skips() { + assert!(!should_skip_hash(WadHash(1), None, false)); + assert!(!should_skip_hash(WadHash(1), None, true)); + } + + #[test] + fn hash_filter_keeps_listed_and_drops_the_rest() { + let hashes = [WadHash(1), WadHash(2)]; + assert!(!should_skip_hash(WadHash(1), Some(&hashes), false)); + assert!(should_skip_hash(WadHash(3), Some(&hashes), false)); + } + + #[test] + fn hash_filter_inverted_drops_listed_and_keeps_the_rest() { + let hashes = [WadHash(1), WadHash(2)]; + assert!(should_skip_hash(WadHash(1), Some(&hashes), true)); + assert!(!should_skip_hash(WadHash(3), Some(&hashes), true)); + } + + // --- should_skip_type tests --- + + #[test] + fn no_type_filter_never_skips() { + assert!(!should_skip_type(LeagueFileKind::Png, None, false)); + assert!(!should_skip_type(LeagueFileKind::Png, None, true)); + } + + #[test] + fn type_filter_includes_matching() { + let types = [LeagueFileKind::Png]; + assert!(!should_skip_type(LeagueFileKind::Png, Some(&types), false)); + } + + #[test] + fn type_filter_excludes_non_matching() { + let types = [LeagueFileKind::Png]; + assert!(should_skip_type(LeagueFileKind::Jpeg, Some(&types), false)); + } + + #[test] + fn type_filter_inverted_excludes_matching() { + let types = [LeagueFileKind::Png]; + assert!(should_skip_type(LeagueFileKind::Png, Some(&types), true)); + } + + #[test] + fn type_filter_inverted_includes_non_matching() { + let types = [LeagueFileKind::Png]; + assert!(!should_skip_type(LeagueFileKind::Jpeg, Some(&types), true)); + } +} diff --git a/crates/wadtools/src/main.rs b/crates/wadtools/src/main.rs index 5019d05..2adb9f8 100644 --- a/crates/wadtools/src/main.rs +++ b/crates/wadtools/src/main.rs @@ -2,7 +2,7 @@ use camino::Utf8Path; use clap::builder::{styling::AnsiColor, Styles}; use clap::error::ErrorKind; use clap::{Parser, Subcommand, ValueEnum}; -use league_toolkit::file::LeagueFileKind; +use league_toolkit::{file::LeagueFileKind, wad::WadHash}; use ltk_mimir_cache::HashStore; use serde::de::value::Error; use serde::de::IntoDeserializer; @@ -18,9 +18,8 @@ use tracing_subscriber::{filter, fmt}; use utils::config::{default_config_path, load_or_create_config, resolve_and_persist_progress}; use utils::{cleanup_legacy_hashtables, expand_wad_inputs, resolve_inputs, WadHashtable}; -mod bin_scan; mod commands; -mod extractor; +mod filters; mod launch; mod shell; mod utils; @@ -149,11 +148,6 @@ pub enum Commands { /// Disable scanning .bin files to recover chunk names before extracting (enabled by default) #[arg(long)] no_bin_paths: bool, - - /// Scan every chunk (not just known .bin files) for paths, recovering the most - /// names at the cost of an extra full decompression pass. Ignored with --no-bin-paths - #[arg(long)] - full_bin_scan: bool, }, /// Compare two wad files /// @@ -288,11 +282,6 @@ pub enum Commands { #[arg(long)] no_bin_paths: bool, - /// Scan every chunk (not just known .bin files) for paths, recovering the most - /// names at the cost of an extra full decompression pass. Ignored with --no-bin-paths - #[arg(long)] - full_bin_scan: bool, - /// Show summary statistics after ripping: true/false (default: true) #[arg(short = 's', long, value_name = "true|false", default_missing_value = "true", num_args = 0..=1, default_value_t = true)] stats: bool, @@ -371,7 +360,6 @@ fn run(args: Args) -> eyre::Result<()> { overwrite, stats, no_bin_paths, - full_bin_scan, } => { if list_filters { print_supported_filters(); @@ -398,7 +386,6 @@ fn run(args: Args) -> eyre::Result<()> { overwrite, show_stats: stats, resolve_bin_paths: !no_bin_paths, - full_bin_scan, }, &ht, )?; @@ -476,7 +463,6 @@ fn run(args: Args) -> eyre::Result<()> { pattern, filter_invert, no_bin_paths, - full_bin_scan, stats, } => { let resolved = expand_wad_inputs(resolve_inputs(&input)); @@ -493,7 +479,6 @@ fn run(args: Args) -> eyre::Result<()> { pattern, filter_invert, resolve_bin_paths: !no_bin_paths, - full_bin_scan, show_stats: stats, }, &ht, @@ -605,7 +590,7 @@ fn parse_filter_type(s: &str) -> Result { } } -fn parse_hashes(raw: Option>) -> eyre::Result>> { +fn parse_hashes(raw: Option>) -> eyre::Result>> { let Some(strings) = raw else { return Ok(None); }; @@ -617,7 +602,7 @@ fn parse_hashes(raw: Option>) -> eyre::Result>> { s ) })?; - hashes.push(h); + hashes.push(WadHash(h)); } Ok(Some(hashes)) } diff --git a/crates/wadtools/src/utils/hashtable.rs b/crates/wadtools/src/utils/hashtable.rs index 7a1ecf6..f489265 100644 --- a/crates/wadtools/src/utils/hashtable.rs +++ b/crates/wadtools/src/utils/hashtable.rs @@ -1,4 +1,5 @@ use color_eyre::eyre::{self, eyre, Result}; +use league_toolkit::wad::{PathResolver, WadHash}; use ltk_hashdb::LayeredHashDb; use ltk_mimir_cache::{HashStore, Table}; use std::{ @@ -14,12 +15,13 @@ use super::format_chunk_path_hash; /// /// A thin wrapper over mimir's [`LayeredHashDb`]: the Game and Lcu `.lhdb` base /// tables (opened lazily via `mmap` from the shared cache) sit under an in-memory -/// overlay for supplemental names - a user-provided `-H` text file or names -/// recovered at runtime. Lookups check the overlay first, then each base table. +/// overlay for supplemental names from a user-provided `-H` text file. Lookups +/// check the overlay first, then each base table. /// /// The only League-domain policy layered on top of mimir is the **hex fallback**: /// mimir returns `Option`, and a total miss is rendered as the 16-hex form of the -/// hash here (mimir never invents hex strings). +/// hash here (mimir never invents hex strings). As a [`PathResolver`] the table +/// answers `None` instead, and `ltk_wad` applies the same fallback itself. #[derive(Default)] pub struct WadHashtable(LayeredHashDb); @@ -55,30 +57,28 @@ impl WadHashtable { /// Resolves a chunk path hash to a readable path, falling back to the 16-hex /// representation when unknown. - pub fn resolve_path(&self, path_hash: u64) -> Cow<'_, str> { + pub fn resolve_path(&self, path_hash: WadHash) -> Cow<'_, str> { self.0 - .get(path_hash) + .get(path_hash.0) .unwrap_or_else(|| Cow::Owned(format_chunk_path_hash(path_hash))) } /// Resolves many chunk path hashes at once, in input order. - #[allow(dead_code)] + /// + /// Takes the raw hashes, as mimir's bulk lookup does. pub fn resolve_batch<'a>(&'a self, path_hashes: &'a [u64]) -> Vec> { self.0 .get_batch(path_hashes) - .map(|(hash, path)| path.unwrap_or_else(|| Cow::Owned(format_chunk_path_hash(hash)))) + .map(|(hash, path)| { + path.unwrap_or_else(|| Cow::Owned(format_chunk_path_hash(WadHash(hash)))) + }) .collect() } - /// Returns true if any base table or the overlay knows this hash. - pub fn contains(&self, path_hash: u64) -> bool { - self.0.contains(path_hash) - } - /// Inserts a supplemental `hash -> path` mapping into the overlay. #[allow(dead_code)] - pub fn insert(&mut self, path_hash: u64, path: impl Into>) { - self.0.insert(path_hash, path); + pub fn insert(&mut self, path_hash: WadHash, path: impl Into>) { + self.0.insert(path_hash.0, path); } /// Loads supplemental ` ` lines from a text file into the @@ -101,3 +101,13 @@ impl WadHashtable { Ok(()) } } + +impl PathResolver for WadHashtable { + fn resolve(&self, path_hash: WadHash) -> Option> { + self.0.get(path_hash.0) + } + + fn is_known(&self, path_hash: WadHash) -> bool { + self.0.contains(path_hash.0) + } +} diff --git a/crates/wadtools/src/utils/mod.rs b/crates/wadtools/src/utils/mod.rs index 7a0d0de..7cd729c 100644 --- a/crates/wadtools/src/utils/mod.rs +++ b/crates/wadtools/src/utils/mod.rs @@ -3,6 +3,7 @@ mod hashtable; use camino::{Utf8Path, Utf8PathBuf}; use fancy_regex::Regex; +use league_toolkit::wad::WadHash; pub use hashtable::*; @@ -72,13 +73,8 @@ pub fn create_filter_pattern(pattern: Option) -> eyre::Result String { - format!("{:016x}", path_hash) -} - -pub fn is_hex_chunk_path(path: &Utf8Path) -> bool { - let file_name = path.file_name().unwrap_or(""); - file_name.len() == 16 && file_name.chars().all(|c| c.is_ascii_hexdigit()) +pub fn format_chunk_path_hash(path_hash: WadHash) -> String { + format!("{path_hash:016x}") } /// Truncates a string in the middle