From e3216855c3797da9121931e0840200ed3e560f1d Mon Sep 17 00:00:00 2001 From: Byte Northbridge Date: Fri, 8 Aug 2025 14:15:09 -0500 Subject: [PATCH 1/2] fix: criterion benchmark paths --- ck3-tiger/benches/criterion.rs | 13 ++++++++++++- vic3-tiger/benches/criterion.rs | 13 ++++++++++++- 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/ck3-tiger/benches/criterion.rs b/ck3-tiger/benches/criterion.rs index c3934cc58..229a59338 100644 --- a/ck3-tiger/benches/criterion.rs +++ b/ck3-tiger/benches/criterion.rs @@ -22,12 +22,23 @@ struct Config { sample_size: Option, } +fn workspace_path(s: &str) -> PathBuf { + let p = PathBuf::from(s); + if p.is_relative() { + PathBuf::from("..").join(p) + } + else { + p + } +} + fn bench_multiple(c: &mut Criterion) { let content = fs::read_to_string(CONFIG_PATH).unwrap(); let config: Config = toml::from_str(&content).unwrap(); - let mut modfile_paths = config.modfile_paths.iter().map(PathBuf::from).collect::>(); + let mut modfile_paths = config.modfile_paths.iter().map(|p| workspace_path(p)).collect::>(); if let Some(modfile_dir) = config.modfile_dir { + let modfile_dir = workspace_path(&modfile_dir); let iter = fs::read_dir(modfile_dir).unwrap().filter_map(|entry| entry.ok()).filter_map(|entry| { entry.file_name().to_string_lossy().ends_with(".mod").then(|| entry.path()) diff --git a/vic3-tiger/benches/criterion.rs b/vic3-tiger/benches/criterion.rs index 3093e8869..0a972560f 100644 --- a/vic3-tiger/benches/criterion.rs +++ b/vic3-tiger/benches/criterion.rs @@ -23,13 +23,24 @@ struct Config { sample_size: Option, } +fn workspace_path(s: &str) -> PathBuf { + let p = PathBuf::from(s); + if p.is_relative() { + PathBuf::from("..").join(p) + } + else { + p + } +} + fn bench_multiple(c: &mut Criterion) { Game::set(Game::Vic3).unwrap(); let content = fs::read_to_string(CONFIG_PATH).unwrap(); let config: Config = toml::from_str(&content).unwrap(); - let mut mod_paths = config.mod_paths.iter().map(PathBuf::from).collect::>(); + let mut mod_paths = config.mod_paths.iter().map(|p| workspace_path(p)).collect::>(); if let Some(mod_dir) = config.mod_dir { + let mod_dir = workspace_path(&mod_dir); let iter = fs::read_dir(mod_dir).unwrap().filter_map(|entry| entry.ok()).filter_map(|entry| { entry.path().join(".metadata/metadata.json").is_file().then(|| entry.path()) From d66848fb74799ff4f0fa68eff538d3d0f17ba8cf Mon Sep 17 00:00:00 2001 From: Byte Northbridge Date: Fri, 8 Aug 2025 13:57:20 -0500 Subject: [PATCH 2/2] refactor: Give ownership of file content to FileEntry Clean up Fileset construction and fix bug Don't bother normalising paths fix tests --- ck3-tiger/benches/criterion.rs | 35 +- ck3-tiger/src/bin/scan-mod-ck3-tiger.rs | 16 +- src/block/comparator.rs | 1 + src/ck3/data/characters.rs | 7 +- src/ck3/data/doctrines.rs | 7 +- src/ck3/data/gameconcepts.rs | 7 +- src/ck3/data/interaction_cats.rs | 7 +- src/ck3/data/maa.rs | 7 +- src/ck3/data/prov_history.rs | 7 +- src/ck3/data/prov_terrain.rs | 11 +- src/ck3/data/provinces.rs | 13 +- src/ck3/data/religions.rs | 2 +- src/ck3/data/title_history.rs | 7 +- src/ck3/data/titles.rs | 6 +- src/ck3/data/traits.rs | 7 +- src/ck3/data/wars.rs | 7 +- src/data/assets.rs | 13 +- src/data/coa.rs | 7 +- src/data/data_binding.rs | 7 +- src/data/defines.rs | 7 +- src/data/events.rs | 8 +- src/data/gui.rs | 6 +- src/data/localization.rs | 20 +- src/data/music.rs | 7 +- src/data/on_actions.rs | 7 +- src/data/script_values.rs | 8 +- src/data/scripted_effects.rs | 7 +- src/data/scripted_lists.rs | 8 +- src/data/scripted_modifiers.rs | 7 +- src/data/scripted_triggers.rs | 7 +- src/dds.rs | 20 +- src/everything.rs | 68 ++-- src/files/filecontent.rs | 143 +++++++ src/files/filedb.rs | 22 + src/{ => files}/fileset.rs | 509 +++++++----------------- src/files/fileset_builder.rs | 429 ++++++++++++++++++++ src/files/mod.rs | 11 + src/hoi4/data/events.rs | 8 +- src/hoi4/data/gfx.rs | 7 +- src/hoi4/data/music.rs | 7 +- src/hoi4/data/provinces.rs | 14 +- src/imperator/data/decisions.rs | 7 +- src/imperator/data/provinces.rs | 13 +- src/launcher_settings.rs | 9 +- src/lib.rs | 4 +- src/mod_metadata.rs | 11 +- src/modfile.rs | 9 +- src/parse/csv.rs | 2 +- src/parse/json.rs | 27 +- src/parse/localization.rs | 18 +- src/parse/pdxfile.rs | 43 +- src/pathtable.rs | 11 +- src/pdxfile.rs | 89 +---- src/report/error_loc.rs | 16 +- src/report/errors.rs | 55 +-- src/report/filter.rs | 2 +- src/report/writer.rs | 2 +- src/rivers.rs | 43 +- src/token.rs | 46 +-- src/vic3/data/history.rs | 7 +- src/vic3/data/provinces.rs | 12 +- tests/files/mod1/descriptor.mod | 3 + tests/files/mod2/descriptor.mod | 3 + tests/files/mod3/descriptor.mod | 3 + tests/test.rs | 7 +- tiger-bin-shared/src/auto.rs | 54 +-- tiger-bin-shared/src/tiger.rs | 63 +-- vic3-tiger/benches/criterion.rs | 15 +- 68 files changed, 1167 insertions(+), 911 deletions(-) create mode 100644 src/files/filecontent.rs create mode 100644 src/files/filedb.rs rename src/{ => files}/fileset.rs (51%) create mode 100644 src/files/fileset_builder.rs create mode 100644 src/files/mod.rs create mode 100644 tests/files/mod1/descriptor.mod create mode 100644 tests/files/mod2/descriptor.mod create mode 100644 tests/files/mod3/descriptor.mod diff --git a/ck3-tiger/benches/criterion.rs b/ck3-tiger/benches/criterion.rs index 229a59338..b33fdb1ef 100644 --- a/ck3-tiger/benches/criterion.rs +++ b/ck3-tiger/benches/criterion.rs @@ -4,7 +4,7 @@ use std::{ fs, path::{Path, PathBuf}, }; -use tiger_lib::{Everything, ModFile}; +use tiger_lib::{Everything, Fileset}; static CONFIG_PATH: &str = "../benches/ck3.toml"; @@ -26,8 +26,7 @@ fn workspace_path(s: &str) -> PathBuf { let p = PathBuf::from(s); if p.is_relative() { PathBuf::from("..").join(p) - } - else { + } else { p } } @@ -35,7 +34,8 @@ fn workspace_path(s: &str) -> PathBuf { fn bench_multiple(c: &mut Criterion) { let content = fs::read_to_string(CONFIG_PATH).unwrap(); let config: Config = toml::from_str(&content).unwrap(); - let mut modfile_paths = config.modfile_paths.iter().map(|p| workspace_path(p)).collect::>(); + let mut modfile_paths = + config.modfile_paths.iter().map(|p| workspace_path(p)).collect::>(); if let Some(modfile_dir) = config.modfile_dir { let modfile_dir = workspace_path(&modfile_dir); @@ -46,18 +46,16 @@ fn bench_multiple(c: &mut Criterion) { modfile_paths.extend(iter); } + let vanilla_dir = PathBuf::from(config.vanilla_dir); + let mut group = c.benchmark_group("benchmark"); group.sample_size(config.sample_size.unwrap_or(10)); for (index, modfile_path) in modfile_paths.iter().enumerate() { - let modfile = ModFile::read(modfile_path).unwrap(); group.bench_with_input( - BenchmarkId::new( - "mods", - format!("{}. {}", index + 1, modfile.display_name().unwrap_or_default()), - ), - &modfile, - |b, modfile_ref| { - b.iter(|| bench_mod(&config.vanilla_dir, modfile_ref)); + BenchmarkId::new("mods", format!("{}. {}", index + 1, modfile_path.display())), + modfile_path, + |b, modfile_path| { + b.iter(|| bench_mod(&vanilla_dir, modfile_path.clone())); }, ); } @@ -65,16 +63,9 @@ fn bench_multiple(c: &mut Criterion) { group.finish(); } -fn bench_mod(vanilla_dir: &str, modfile: &ModFile) { - let mut everything = Everything::new( - None, - Some(Path::new(vanilla_dir)), - None, - None, - &modfile.modpath(), - modfile.replace_paths(), - ) - .unwrap(); +fn bench_mod(vanilla_dir: &Path, modfile_path: PathBuf) { + let fileset = Fileset::builder(Some(vanilla_dir)).with_modfile(modfile_path).unwrap(); + let mut everything = Everything::new(fileset, None, None, None).unwrap(); everything.load_all(); everything.validate_all(); everything.check_rivers(); diff --git a/ck3-tiger/src/bin/scan-mod-ck3-tiger.rs b/ck3-tiger/src/bin/scan-mod-ck3-tiger.rs index 1dc19d087..824e2cf59 100644 --- a/ck3-tiger/src/bin/scan-mod-ck3-tiger.rs +++ b/ck3-tiger/src/bin/scan-mod-ck3-tiger.rs @@ -5,12 +5,12 @@ use std::fs::write; use std::mem::forget; use std::path::PathBuf; -use anyhow::{bail, Result}; +use anyhow::Result; use clap::Parser; use serde_json::{json, to_string_pretty, Value}; use strum::IntoEnumIterator; -use tiger_lib::{Everything, FileKind, Game, Item, ModFile}; +use tiger_lib::{Everything, FileKind, Fileset, Game, Item}; #[derive(Parser)] struct Cli { @@ -34,16 +34,8 @@ fn main() -> Result<()> { if args.modpath.is_dir() { args.modpath.push("descriptor.mod"); } - let modfile = ModFile::read(&args.modpath)?; - let modpath = modfile.modpath(); - if !modpath.exists() { - eprintln!("Looking for mod in {}", modpath.display()); - bail!("Cannot find mod directory. Please make sure the .mod file is correct."); - } - eprintln!("Using mod directory: {}", modpath.display()); - - let mut everything = - Everything::new(None, None, None, None, &modpath, modfile.replace_paths())?; + let fileset = Fileset::builder(None).with_modfile(args.modpath)?; + let mut everything = Everything::new(fileset, None, None, None)?; everything.load_all(); diff --git a/src/block/comparator.rs b/src/block/comparator.rs index 380c6460a..b211e4b17 100644 --- a/src/block/comparator.rs +++ b/src/block/comparator.rs @@ -32,6 +32,7 @@ pub enum Eq { Question, } +#[derive(Debug, Copy, Clone)] pub struct UnknownComparatorError; impl FromStr for Comparator { diff --git a/src/ck3/data/characters.rs b/src/ck3/data/characters.rs index 8c80ae785..4c14c0da0 100644 --- a/src/ck3/data/characters.rs +++ b/src/ck3/data/characters.rs @@ -4,6 +4,7 @@ use std::fmt::{Display, Formatter}; use std::path::PathBuf; use std::str::FromStr; use std::sync::atomic::Ordering; +use std::sync::Arc; use atomic_enum::atomic_enum; @@ -14,7 +15,7 @@ use crate::context::ScopeContext; use crate::date::Date; use crate::effect::{validate_effect, validate_effect_field}; use crate::everything::Everything; -use crate::fileset::{FileEntry, FileHandler}; +use crate::files::{FileEntry, FileHandler}; use crate::helpers::{TigerHashMap, TigerHashSet}; use crate::item::Item; use crate::lowercase::Lowercase; @@ -255,7 +256,7 @@ impl FileHandler for Characters { PathBuf::from("history/characters") } - fn load_file(&self, entry: &FileEntry, parser: &ParserMemory) -> Option { + fn load_file(&self, entry: &Arc, parser: &ParserMemory) -> Option { if !entry.filename().to_string_lossy().ends_with(".txt") { return None; } @@ -263,7 +264,7 @@ impl FileHandler for Characters { PdxFile::read(entry, parser) } - fn handle_file(&mut self, _entry: &FileEntry, mut block: Block) { + fn handle_file(&mut self, _entry: &Arc, mut block: Block) { for (key, block) in block.drain_definitions_warn() { self.load_item(key, block); } diff --git a/src/ck3/data/doctrines.rs b/src/ck3/data/doctrines.rs index 4bc5b4678..8f24e6b9e 100644 --- a/src/ck3/data/doctrines.rs +++ b/src/ck3/data/doctrines.rs @@ -1,4 +1,5 @@ use std::path::PathBuf; +use std::sync::Arc; use crate::block::Block; use crate::ck3::modif::ModifKinds; @@ -6,7 +7,7 @@ use crate::ck3::validate::validate_traits; use crate::context::ScopeContext; use crate::desc::validate_desc; use crate::everything::Everything; -use crate::fileset::{FileEntry, FileHandler}; +use crate::files::{FileEntry, FileHandler}; use crate::helpers::{dup_error, TigerHashMap, TigerHashSet}; use crate::item::Item; use crate::modif::validate_modifs; @@ -105,7 +106,7 @@ impl FileHandler for Doctrines { PathBuf::from("common/religion/doctrines") } - fn load_file(&self, entry: &FileEntry, parser: &ParserMemory) -> Option { + fn load_file(&self, entry: &Arc, parser: &ParserMemory) -> Option { if !entry.filename().to_string_lossy().ends_with(".txt") { return None; } @@ -113,7 +114,7 @@ impl FileHandler for Doctrines { PdxFile::read(entry, parser) } - fn handle_file(&mut self, _entry: &FileEntry, mut block: Block) { + fn handle_file(&mut self, _entry: &Arc, mut block: Block) { for (key, block) in block.drain_definitions_warn() { self.load_item(key, block); } diff --git a/src/ck3/data/gameconcepts.rs b/src/ck3/data/gameconcepts.rs index 279174b0e..b6a96ce59 100644 --- a/src/ck3/data/gameconcepts.rs +++ b/src/ck3/data/gameconcepts.rs @@ -1,8 +1,9 @@ use std::path::PathBuf; +use std::sync::Arc; use crate::block::Block; use crate::everything::Everything; -use crate::fileset::{FileEntry, FileHandler}; +use crate::files::{FileEntry, FileHandler}; use crate::helpers::{dup_error, TigerHashMap}; use crate::item::Item; use crate::parse::ParserMemory; @@ -47,7 +48,7 @@ impl FileHandler for GameConcepts { PathBuf::from("common/game_concepts") } - fn load_file(&self, entry: &FileEntry, parser: &ParserMemory) -> Option { + fn load_file(&self, entry: &Arc, parser: &ParserMemory) -> Option { if !entry.filename().to_string_lossy().ends_with(".txt") { return None; } @@ -55,7 +56,7 @@ impl FileHandler for GameConcepts { PdxFile::read(entry, parser) } - fn handle_file(&mut self, _entry: &FileEntry, mut block: Block) { + fn handle_file(&mut self, _entry: &Arc, mut block: Block) { for (key, block) in block.drain_definitions_warn() { self.load_item(key, block); } diff --git a/src/ck3/data/interaction_cats.rs b/src/ck3/data/interaction_cats.rs index bb67f053f..3a3f71593 100644 --- a/src/ck3/data/interaction_cats.rs +++ b/src/ck3/data/interaction_cats.rs @@ -1,8 +1,9 @@ use std::path::PathBuf; +use std::sync::Arc; use crate::block::Block; use crate::everything::Everything; -use crate::fileset::{FileEntry, FileHandler}; +use crate::files::{FileEntry, FileHandler}; use crate::helpers::{dup_error, TigerHashMap}; use crate::item::Item; use crate::parse::ParserMemory; @@ -54,7 +55,7 @@ impl FileHandler for CharacterInteractionCategories { PathBuf::from("common/character_interaction_categories") } - fn load_file(&self, entry: &FileEntry, parser: &ParserMemory) -> Option { + fn load_file(&self, entry: &Arc, parser: &ParserMemory) -> Option { if !entry.filename().to_string_lossy().ends_with(".txt") { return None; } @@ -62,7 +63,7 @@ impl FileHandler for CharacterInteractionCategories { PdxFile::read(entry, parser) } - fn handle_file(&mut self, _entry: &FileEntry, mut block: Block) { + fn handle_file(&mut self, _entry: &Arc, mut block: Block) { for (key, block) in block.drain_definitions_warn() { self.load_item(key, block); } diff --git a/src/ck3/data/maa.rs b/src/ck3/data/maa.rs index bbfb3eb92..89500f085 100644 --- a/src/ck3/data/maa.rs +++ b/src/ck3/data/maa.rs @@ -1,10 +1,11 @@ use std::path::PathBuf; +use std::sync::Arc; use crate::block::Block; use crate::ck3::validate::{validate_cost, validate_maa_stats}; use crate::context::ScopeContext; use crate::everything::Everything; -use crate::fileset::{FileEntry, FileHandler}; +use crate::files::{FileEntry, FileHandler}; use crate::helpers::{dup_error, TigerHashMap, TigerHashSet}; use crate::item::Item; use crate::lowercase::Lowercase; @@ -73,7 +74,7 @@ impl FileHandler for MenAtArmsTypes { PathBuf::from("common/men_at_arms_types") } - fn load_file(&self, entry: &FileEntry, parser: &ParserMemory) -> Option { + fn load_file(&self, entry: &Arc, parser: &ParserMemory) -> Option { if !entry.filename().to_string_lossy().ends_with(".txt") { return None; } @@ -81,7 +82,7 @@ impl FileHandler for MenAtArmsTypes { PdxFile::read(entry, parser) } - fn handle_file(&mut self, _entry: &FileEntry, mut block: Block) { + fn handle_file(&mut self, _entry: &Arc, mut block: Block) { for (key, block) in block.drain_definitions_warn() { self.load_item(key, block); } diff --git a/src/ck3/data/prov_history.rs b/src/ck3/data/prov_history.rs index 93096866e..93263bd4f 100644 --- a/src/ck3/data/prov_history.rs +++ b/src/ck3/data/prov_history.rs @@ -1,11 +1,12 @@ use std::path::PathBuf; +use std::sync::Arc; use crate::block::{Block, BV}; use crate::ck3::data::provinces::ProvId; use crate::ck3::data::titles::Titles; use crate::date::Date; use crate::everything::Everything; -use crate::fileset::{FileEntry, FileHandler}; +use crate::files::{FileEntry, FileHandler}; use crate::helpers::{dup_error, TigerHashMap}; use crate::item::Item; use crate::parse::ParserMemory; @@ -83,7 +84,7 @@ impl FileHandler for ProvinceHistories { PathBuf::from("history/provinces") } - fn load_file(&self, entry: &FileEntry, parser: &ParserMemory) -> Option { + fn load_file(&self, entry: &Arc, parser: &ParserMemory) -> Option { if !entry.filename().to_string_lossy().ends_with(".txt") { return None; } @@ -91,7 +92,7 @@ impl FileHandler for ProvinceHistories { PdxFile::read_detect_encoding(entry, parser) } - fn handle_file(&mut self, _entry: &FileEntry, mut block: Block) { + fn handle_file(&mut self, _entry: &Arc, mut block: Block) { for (key, block) in block.drain_definitions_warn() { if let Ok(id) = key.as_str().parse() { self.load_item(id, key, block); diff --git a/src/ck3/data/prov_terrain.rs b/src/ck3/data/prov_terrain.rs index c808eb0ff..dda273a7d 100644 --- a/src/ck3/data/prov_terrain.rs +++ b/src/ck3/data/prov_terrain.rs @@ -1,8 +1,9 @@ use std::path::PathBuf; +use std::sync::Arc; use crate::block::Block; use crate::everything::Everything; -use crate::fileset::{FileEntry, FileHandler}; +use crate::files::{FileEntry, FileHandler}; use crate::helpers::{dup_error, TigerHashMap}; use crate::item::Item; use crate::parse::ParserMemory; @@ -63,7 +64,7 @@ impl FileHandler for ProvinceTerrains { PathBuf::from("common/province_terrain") } - fn load_file(&self, entry: &FileEntry, parser: &ParserMemory) -> Option { + fn load_file(&self, entry: &Arc, parser: &ParserMemory) -> Option { if !entry.filename().to_string_lossy().ends_with("province_terrain.txt") { // Omit _province_properties.txt return None; @@ -72,7 +73,7 @@ impl FileHandler for ProvinceTerrains { PdxFile::read_detect_encoding(entry, parser) } - fn handle_file(&mut self, _entry: &FileEntry, mut block: Block) { + fn handle_file(&mut self, _entry: &Arc, mut block: Block) { self.file_loc = Some(block.loc); for (key, value) in block.drain_assignments_warn() { if let Ok(id) = key.as_str().parse() { @@ -140,7 +141,7 @@ impl FileHandler for ProvinceProperties { PathBuf::from("common/province_terrain") } - fn load_file(&self, entry: &FileEntry, parser: &ParserMemory) -> Option { + fn load_file(&self, entry: &Arc, parser: &ParserMemory) -> Option { if !entry.filename().to_string_lossy().ends_with("province_properties.txt") { // Omit _province_terrain.txt return None; @@ -148,7 +149,7 @@ impl FileHandler for ProvinceProperties { PdxFile::read_detect_encoding(entry, parser) } - fn handle_file(&mut self, _entry: &FileEntry, mut block: Block) { + fn handle_file(&mut self, _entry: &Arc, mut block: Block) { for (key, block) in block.drain_definitions_warn() { if let Ok(id) = key.as_str().parse() { self.load_item(id, key, block); diff --git a/src/ck3/data/provinces.rs b/src/ck3/data/provinces.rs index 1bb3ce8f2..2a149394f 100644 --- a/src/ck3/data/provinces.rs +++ b/src/ck3/data/provinces.rs @@ -1,5 +1,6 @@ use std::path::PathBuf; use std::str::FromStr; +use std::sync::Arc; use bitvec::bitbox; use bitvec::boxed::BitBox; @@ -9,7 +10,7 @@ use itertools::Itertools; use crate::block::Block; use crate::db::{Db, DbKind}; use crate::everything::Everything; -use crate::fileset::{FileEntry, FileHandler}; +use crate::files::{FileEntry, FileHandler}; use crate::game::GameFlags; use crate::helpers::{TigerHashMap, TigerHashSet}; use crate::item::{Item, ItemLoader, LoadAsFile, Recursive}; @@ -73,7 +74,7 @@ pub struct Ck3Provinces { provinces: TigerHashMap, /// Kept and used for error reporting. - definition_csv: Option, + definition_csv: Option, adjacencies: Vec, @@ -234,7 +235,7 @@ impl FileHandler for Ck3Provinces { PathBuf::from("map_data") } - fn load_file(&self, entry: &FileEntry, parser: &ParserMemory) -> Option { + fn load_file(&self, entry: &Arc, parser: &ParserMemory) -> Option { if entry.path().components().count() == 2 { match &*entry.filename().to_string_lossy() { "adjacencies.csv" => { @@ -293,7 +294,7 @@ impl FileHandler for Ck3Provinces { None } - fn handle_file(&mut self, entry: &FileEntry, content: FileContent) { + fn handle_file(&mut self, entry: &Arc, content: FileContent) { match content { FileContent::Adjacencies(content) => { let mut seen_terminator = false; @@ -314,7 +315,7 @@ impl FileHandler for Ck3Provinces { } } FileContent::Definitions(content) => { - self.definition_csv = Some(entry.clone()); + self.definition_csv = Some(Loc::from(entry)); for csv in parse_csv(entry, 0, &content) { self.parse_definition(&csv); } @@ -341,7 +342,7 @@ impl FileHandler for Ck3Provinces { eprintln!("map_data/definition.csv is missing?!?"); return; } - let definition_csv = self.definition_csv.as_ref().unwrap(); + let definition_csv = self.definition_csv.unwrap(); let mut seen_colors = TigerHashMap::default(); #[allow(clippy::cast_possible_truncation)] diff --git a/src/ck3/data/religions.rs b/src/ck3/data/religions.rs index a3b1e47a2..6e815288e 100644 --- a/src/ck3/data/religions.rs +++ b/src/ck3/data/religions.rs @@ -3,7 +3,7 @@ use crate::ck3::validate::validate_traits; use crate::context::ScopeContext; use crate::db::{Db, DbKind}; use crate::everything::Everything; -use crate::fileset::FileKind; +use crate::files::FileKind; use crate::game::GameFlags; use crate::helpers::TigerHashMap; use crate::item::{Item, ItemLoader}; diff --git a/src/ck3/data/title_history.rs b/src/ck3/data/title_history.rs index 75ecbef43..c63afe9e5 100644 --- a/src/ck3/data/title_history.rs +++ b/src/ck3/data/title_history.rs @@ -1,10 +1,11 @@ use std::path::PathBuf; +use std::sync::Arc; use crate::block::Block; use crate::ck3::data::titles::Tier; use crate::date::Date; use crate::everything::Everything; -use crate::fileset::{FileEntry, FileHandler}; +use crate::files::{FileEntry, FileHandler}; use crate::helpers::TigerHashMap; use crate::item::Item; use crate::parse::ParserMemory; @@ -73,7 +74,7 @@ impl FileHandler for TitleHistories { PathBuf::from("history/titles") } - fn load_file(&self, entry: &FileEntry, parser: &ParserMemory) -> Option { + fn load_file(&self, entry: &Arc, parser: &ParserMemory) -> Option { if !entry.filename().to_string_lossy().ends_with(".txt") { return None; } @@ -81,7 +82,7 @@ impl FileHandler for TitleHistories { PdxFile::read_detect_encoding(entry, parser) } - fn handle_file(&mut self, _entry: &FileEntry, mut block: Block) { + fn handle_file(&mut self, _entry: &Arc, mut block: Block) { for (key, block) in block.drain_definitions_warn() { if Tier::try_from(&key).is_ok() { self.load_item(key, block); diff --git a/src/ck3/data/titles.rs b/src/ck3/data/titles.rs index ca6297a78..4ea2312a3 100644 --- a/src/ck3/data/titles.rs +++ b/src/ck3/data/titles.rs @@ -6,7 +6,7 @@ use crate::block::Block; use crate::ck3::data::provinces::ProvId; use crate::context::ScopeContext; use crate::everything::Everything; -use crate::fileset::{FileEntry, FileHandler}; +use crate::files::{FileEntry, FileHandler}; use crate::helpers::{dup_error, TigerHashMap}; use crate::item::Item; use crate::parse::ParserMemory; @@ -158,7 +158,7 @@ impl FileHandler for Titles { PathBuf::from("common/landed_titles") } - fn load_file(&self, entry: &FileEntry, parser: &ParserMemory) -> Option { + fn load_file(&self, entry: &Arc, parser: &ParserMemory) -> Option { if !entry.filename().to_string_lossy().ends_with(".txt") { return None; } @@ -166,7 +166,7 @@ impl FileHandler for Titles { PdxFile::read(entry, parser) } - fn handle_file(&mut self, _entry: &FileEntry, mut block: Block) { + fn handle_file(&mut self, _entry: &Arc, mut block: Block) { for (key, block) in block.drain_definitions_warn() { if Tier::try_from(&key).is_ok() { self.load_item(key, &block, None, false); diff --git a/src/ck3/data/traits.rs b/src/ck3/data/traits.rs index b6661f91a..d365a321a 100644 --- a/src/ck3/data/traits.rs +++ b/src/ck3/data/traits.rs @@ -1,11 +1,12 @@ use std::path::PathBuf; +use std::sync::Arc; use crate::block::{Block, BV}; use crate::ck3::modif::ModifKinds; use crate::context::ScopeContext; use crate::desc::{validate_desc, validate_desc_map}; use crate::everything::Everything; -use crate::fileset::{FileEntry, FileHandler}; +use crate::files::{FileEntry, FileHandler}; use crate::helpers::{dup_error, TigerHashMap, TigerHashSet}; use crate::item::Item; use crate::lowercase::Lowercase; @@ -110,7 +111,7 @@ impl FileHandler for Traits { PathBuf::from("common/traits") } - fn load_file(&self, entry: &FileEntry, parser: &ParserMemory) -> Option { + fn load_file(&self, entry: &Arc, parser: &ParserMemory) -> Option { if !entry.filename().to_string_lossy().ends_with(".txt") { return None; } @@ -118,7 +119,7 @@ impl FileHandler for Traits { PdxFile::read(entry, parser) } - fn handle_file(&mut self, _entry: &FileEntry, mut block: Block) { + fn handle_file(&mut self, _entry: &Arc, mut block: Block) { for (key, block) in block.drain_definitions_warn() { self.load_item(key, block); } diff --git a/src/ck3/data/wars.rs b/src/ck3/data/wars.rs index b8a0c58a8..412ae3814 100644 --- a/src/ck3/data/wars.rs +++ b/src/ck3/data/wars.rs @@ -1,8 +1,9 @@ use std::path::PathBuf; +use std::sync::Arc; use crate::block::Block; use crate::everything::Everything; -use crate::fileset::{FileEntry, FileHandler}; +use crate::files::{FileEntry, FileHandler}; use crate::item::Item; use crate::parse::ParserMemory; use crate::pdxfile::PdxFile; @@ -37,7 +38,7 @@ impl FileHandler for Wars { PathBuf::from("history/wars") } - fn load_file(&self, entry: &FileEntry, parser: &ParserMemory) -> Option { + fn load_file(&self, entry: &Arc, parser: &ParserMemory) -> Option { if !entry.filename().to_string_lossy().ends_with(".txt") { return None; } @@ -45,7 +46,7 @@ impl FileHandler for Wars { PdxFile::read_optional_bom(entry, parser) } - fn handle_file(&mut self, _entry: &FileEntry, mut block: Block) { + fn handle_file(&mut self, _entry: &Arc, mut block: Block) { for (key, block) in block.drain_definitions_warn() { self.load_item(key, block); } diff --git a/src/data/assets.rs b/src/data/assets.rs index 970b2df53..f84ee6408 100644 --- a/src/data/assets.rs +++ b/src/data/assets.rs @@ -1,8 +1,9 @@ use std::path::PathBuf; +use std::sync::Arc; use crate::block::{Block, BV}; use crate::everything::Everything; -use crate::fileset::{FileEntry, FileHandler}; +use crate::files::{FileEntry, FileHandler}; use crate::game::Game; use crate::helpers::{dup_error, TigerHashMap, TigerHashSet}; use crate::item::Item; @@ -24,7 +25,7 @@ pub struct Assets { attributes: TigerHashSet, blend_shapes: TigerHashSet, musics: TigerHashSet, - textures: TigerHashMap, + textures: TigerHashMap, Token)>, } impl Assets { @@ -109,7 +110,7 @@ impl Assets { self.textures.values().map(|(_, token)| token) } - pub fn get_texture(&self, key: &str) -> Option<&FileEntry> { + pub fn get_texture(&self, key: &str) -> Option<&Arc> { self.textures.get(key).map(|(entry, _)| entry) } @@ -126,7 +127,7 @@ impl FileHandler> for Assets { } /// TODO: should probably simplify this `FileHandler` by keeping the textures in a separate `FileHandler`. - fn load_file(&self, entry: &FileEntry, parser: &ParserMemory) -> Option> { + fn load_file(&self, entry: &Arc, parser: &ParserMemory) -> Option> { let name = entry.filename().to_string_lossy(); if name.ends_with(".dds") { @@ -138,7 +139,7 @@ impl FileHandler> for Assets { } } - fn handle_file(&mut self, entry: &FileEntry, loaded: Option) { + fn handle_file(&mut self, entry: &Arc, loaded: Option) { let name = entry.filename().to_string_lossy(); if name.ends_with(".dds") { if let Some((other, _)) = self.textures.get(&*name) { @@ -151,7 +152,7 @@ impl FileHandler> for Assets { } } let entry_token = Token::new(&entry.filename().to_string_lossy(), entry.into()); - self.textures.insert(name.to_string(), (entry.clone(), entry_token)); + self.textures.insert(name.to_string(), (Arc::clone(entry), entry_token)); return; } diff --git a/src/data/coa.rs b/src/data/coa.rs index a6920110b..8d66bd394 100644 --- a/src/data/coa.rs +++ b/src/data/coa.rs @@ -1,10 +1,11 @@ use std::path::PathBuf; +use std::sync::Arc; use crate::block::{Block, BV}; use crate::context::ScopeContext; use crate::db::{Db, DbKind}; use crate::everything::Everything; -use crate::fileset::{FileEntry, FileHandler}; +use crate::files::{FileEntry, FileHandler}; use crate::game::{Game, GameFlags}; use crate::helpers::{dup_error, exact_dup_advice, TigerHashMap}; use crate::item::{Item, ItemLoader, LoadAsFile, Recursive}; @@ -105,7 +106,7 @@ impl FileHandler for Coas { PathBuf::from("common/coat_of_arms/coat_of_arms/") } - fn load_file(&self, entry: &FileEntry, parser: &ParserMemory) -> Option { + fn load_file(&self, entry: &Arc, parser: &ParserMemory) -> Option { if !entry.filename().to_string_lossy().ends_with(".txt") { return None; } @@ -113,7 +114,7 @@ impl FileHandler for Coas { PdxFile::read_optional_bom(entry, parser) } - fn handle_file(&mut self, _entry: &FileEntry, block: Block) { + fn handle_file(&mut self, _entry: &Arc, block: Block) { for (key, bv) in block.iter_assignments_and_definitions_warn() { self.load_item(key, bv); } diff --git a/src/data/data_binding.rs b/src/data/data_binding.rs index 1a5804d8d..7250fe19f 100644 --- a/src/data/data_binding.rs +++ b/src/data/data_binding.rs @@ -1,11 +1,12 @@ use std::mem::take; use std::path::PathBuf; +use std::sync::Arc; use crate::block::Block; use crate::data::localization::LocaValue; use crate::datatype::{Code, CodeArg, CodeChain}; use crate::everything::Everything; -use crate::fileset::{FileEntry, FileHandler}; +use crate::files::{FileEntry, FileHandler}; use crate::helpers::{dup_error, TigerHashMap}; use crate::parse::localization::ValueParser; use crate::parse::ParserMemory; @@ -56,7 +57,7 @@ impl FileHandler for DataBindings { PathBuf::from("data_binding") } - fn load_file(&self, entry: &FileEntry, parser: &ParserMemory) -> Option { + fn load_file(&self, entry: &Arc, parser: &ParserMemory) -> Option { if !entry.filename().to_string_lossy().ends_with(".txt") { return None; } @@ -64,7 +65,7 @@ impl FileHandler for DataBindings { PdxFile::read(entry, parser) } - fn handle_file(&mut self, _entry: &FileEntry, mut block: Block) { + fn handle_file(&mut self, _entry: &Arc, mut block: Block) { for (key, block) in block.drain_definitions_warn() { if key.is("macro") { self.load_macro(block); diff --git a/src/data/defines.rs b/src/data/defines.rs index 7118f8591..709293edd 100644 --- a/src/data/defines.rs +++ b/src/data/defines.rs @@ -1,8 +1,9 @@ use std::path::PathBuf; +use std::sync::Arc; use crate::block::{Block, BV}; use crate::everything::Everything; -use crate::fileset::{FileEntry, FileHandler}; +use crate::files::{FileEntry, FileHandler}; use crate::game::Game; use crate::helpers::{dup_error, TigerHashMap}; #[cfg(feature = "ck3")] @@ -56,7 +57,7 @@ impl FileHandler for Defines { PathBuf::from("common/defines") } - fn load_file(&self, entry: &FileEntry, parser: &ParserMemory) -> Option { + fn load_file(&self, entry: &Arc, parser: &ParserMemory) -> Option { if !entry.filename().to_string_lossy().ends_with(".txt") { return None; } @@ -64,7 +65,7 @@ impl FileHandler for Defines { PdxFile::read(entry, parser) } - fn handle_file(&mut self, _entry: &FileEntry, mut block: Block) { + fn handle_file(&mut self, _entry: &Arc, mut block: Block) { // TODO HOI4: Hoi4 has a toplevel group for (group, block) in block.drain_definitions_warn() { for (name, bv) in block.iter_assignments_and_definitions_warn() { diff --git a/src/data/events.rs b/src/data/events.rs index 4a41454d9..9ebce8beb 100644 --- a/src/data/events.rs +++ b/src/data/events.rs @@ -1,6 +1,6 @@ use std::path::PathBuf; use std::str::FromStr; -use std::sync::Mutex; +use std::sync::{Arc, Mutex}; use rayon::prelude::*; @@ -9,7 +9,7 @@ use crate::context::{Reason, ScopeContext, Signature}; use crate::data::scripted_effects::Effect; use crate::data::scripted_triggers::Trigger; use crate::everything::Everything; -use crate::fileset::{FileEntry, FileHandler}; +use crate::files::{FileEntry, FileHandler}; use crate::game::Game; use crate::helpers::{dup_error, TigerHashMap, TigerHashSet}; use crate::item::Item; @@ -161,7 +161,7 @@ impl FileHandler for Events { PathBuf::from("events") } - fn load_file(&self, entry: &FileEntry, parser: &ParserMemory) -> Option { + fn load_file(&self, entry: &Arc, parser: &ParserMemory) -> Option { if !entry.filename().to_string_lossy().ends_with(".txt") { return None; } @@ -169,7 +169,7 @@ impl FileHandler for Events { PdxFile::read(entry, parser) } - fn handle_file(&mut self, _entry: &FileEntry, mut block: Block) { + fn handle_file(&mut self, _entry: &Arc, mut block: Block) { #[derive(Copy, Clone)] enum Expecting { Event, diff --git a/src/data/gui.rs b/src/data/gui.rs index 02d4c4a2b..e3d19179b 100644 --- a/src/data/gui.rs +++ b/src/data/gui.rs @@ -6,7 +6,7 @@ use std::sync::{Arc, RwLock}; use crate::block::{Block, BlockItem, Field, BV}; use crate::everything::Everything; -use crate::fileset::{FileEntry, FileHandler}; +use crate::files::{FileEntry, FileHandler}; use crate::game::Game; use crate::gui::{BuiltinWidget, GuiBlock, GuiBlockFrom}; use crate::helpers::{dup_error, TigerHashMap, TigerHashSet}; @@ -282,7 +282,7 @@ impl FileHandler for Gui { } } - fn load_file(&self, entry: &FileEntry, parser: &ParserMemory) -> Option { + fn load_file(&self, entry: &Arc, parser: &ParserMemory) -> Option { if !entry.filename().to_string_lossy().ends_with(".gui") { return None; } @@ -290,7 +290,7 @@ impl FileHandler for Gui { PdxFile::read_optional_bom(entry, parser) } - fn handle_file(&mut self, entry: &FileEntry, mut block: Block) { + fn handle_file(&mut self, entry: &Arc, mut block: Block) { #[derive(Clone, Debug)] enum Expecting { Widget, diff --git a/src/data/localization.rs b/src/data/localization.rs index 1e5f8c18e..ae198b4d2 100644 --- a/src/data/localization.rs +++ b/src/data/localization.rs @@ -4,13 +4,12 @@ use std::borrow::Borrow; use std::cmp::Ordering; use std::collections::hash_map::Entry; use std::ffi::OsStr; -use std::fs::read_to_string; #[cfg(any(feature = "ck3", feature = "vic3"))] use std::io::Cursor; use std::path::{Path, PathBuf}; use std::sync::atomic::AtomicBool; use std::sync::atomic::Ordering::Relaxed; -use std::sync::LazyLock; +use std::sync::{Arc, LazyLock}; use bitvec::order::Lsb0; use bitvec::{bitarr, BitArr}; @@ -26,7 +25,7 @@ use crate::ck3::tables::localization::{BUILTIN_MACROS_CK3, COMPLEX_TOOLTIPS_CK3} use crate::context::ScopeContext; use crate::datatype::{validate_datatypes, CodeChain, Datatype}; use crate::everything::Everything; -use crate::fileset::{FileEntry, FileHandler, FileKind}; +use crate::files::{FileEntry, FileHandler, FileKind}; use crate::game::Game; use crate::helpers::{dup_error, stringify_list, TigerHashMap}; #[cfg(feature = "hoi4")] @@ -739,7 +738,7 @@ impl FileHandler<(Language, Vec)> for Localization { fn load_file( &self, - entry: &FileEntry, + entry: &Arc, _parser: &ParserMemory, ) -> Option<(Language, Vec)> { if !entry.filename().to_string_lossy().ends_with(".yml") { @@ -769,16 +768,7 @@ impl FileHandler<(Language, Vec)> for Localization { warn(ErrorKey::Filename).msg(msg).info(info).loc(entry).push(); } } - match read_to_string(entry.fullpath()) { - Ok(content) => { - return Some((filelang, parse_loca(entry, content, filelang).collect())); - } - Err(e) => { - let msg = "could not read file"; - let info = &format!("{e:#}"); - err(ErrorKey::ReadError).msg(msg).info(info).loc(entry).push(); - } - } + return Some((filelang, parse_loca(entry, filelang)?.collect())); } else if entry.kind() >= FileKind::Vanilla { // Check for `FileKind::Vanilla` because Jomini and Clausewitz support more languages let msg = "could not determine language from filename"; @@ -791,7 +781,7 @@ impl FileHandler<(Language, Vec)> for Localization { None } - fn handle_file(&mut self, entry: &FileEntry, loaded: (Language, Vec)) { + fn handle_file(&mut self, entry: &Arc, loaded: (Language, Vec)) { let (filelang, mut vec) = loaded; let hash = &mut self.locas[filelang.to_idx()]; diff --git a/src/data/music.rs b/src/data/music.rs index b67c9a289..c24694288 100644 --- a/src/data/music.rs +++ b/src/data/music.rs @@ -1,10 +1,11 @@ use std::path::PathBuf; +use std::sync::Arc; use crate::block::Block; use crate::context::ScopeContext; use crate::db::{Db, DbKind}; use crate::everything::Everything; -use crate::fileset::{FileEntry, FileHandler}; +use crate::files::{FileEntry, FileHandler}; use crate::game::{Game, GameFlags}; use crate::helpers::{dup_error, TigerHashMap}; use crate::item::{Item, ItemLoader}; @@ -85,7 +86,7 @@ impl FileHandler for Musics { PathBuf::from("music") } - fn load_file(&self, entry: &FileEntry, parser: &ParserMemory) -> Option { + fn load_file(&self, entry: &Arc, parser: &ParserMemory) -> Option { if entry.path().parent().unwrap().ends_with("music_player_categories") { return None; } @@ -96,7 +97,7 @@ impl FileHandler for Musics { PdxFile::read(entry, parser) } - fn handle_file(&mut self, _entry: &FileEntry, mut block: Block) { + fn handle_file(&mut self, _entry: &Arc, mut block: Block) { for (key, block) in block.drain_definitions_warn() { self.load_item(key, block); } diff --git a/src/data/on_actions.rs b/src/data/on_actions.rs index 25935c655..74ec3b507 100644 --- a/src/data/on_actions.rs +++ b/src/data/on_actions.rs @@ -1,10 +1,11 @@ use std::path::PathBuf; +use std::sync::Arc; use crate::block::Block; use crate::context::ScopeContext; use crate::effect::validate_effect; use crate::everything::Everything; -use crate::fileset::{FileEntry, FileHandler}; +use crate::files::{FileEntry, FileHandler}; use crate::game::Game; use crate::helpers::TigerHashMap; use crate::item::Item; @@ -83,7 +84,7 @@ impl FileHandler for OnActions { } } - fn load_file(&self, entry: &FileEntry, parser: &ParserMemory) -> Option { + fn load_file(&self, entry: &Arc, parser: &ParserMemory) -> Option { if !entry.filename().to_string_lossy().ends_with(".txt") { return None; } @@ -95,7 +96,7 @@ impl FileHandler for OnActions { PdxFile::read(entry, parser) } - fn handle_file(&mut self, _entry: &FileEntry, mut block: Block) { + fn handle_file(&mut self, _entry: &Arc, mut block: Block) { for (key, block) in block.drain_definitions_warn() { if Game::is_hoi4() { #[cfg(feature = "hoi4")] diff --git a/src/data/script_values.rs b/src/data/script_values.rs index 0972809db..5758b822d 100644 --- a/src/data/script_values.rs +++ b/src/data/script_values.rs @@ -1,10 +1,10 @@ use std::path::PathBuf; -use std::sync::RwLock; +use std::sync::{Arc, RwLock}; use crate::block::{Block, BV}; use crate::context::ScopeContext; use crate::everything::Everything; -use crate::fileset::{FileEntry, FileHandler}; +use crate::files::{FileEntry, FileHandler}; use crate::helpers::{dup_error, exact_dup_error, TigerHashMap, BANNED_NAMES}; use crate::parse::ParserMemory; use crate::pdxfile::PdxFile; @@ -102,7 +102,7 @@ impl FileHandler for ScriptValues { PathBuf::from("common/script_values") } - fn load_file(&self, entry: &FileEntry, parser: &ParserMemory) -> Option { + fn load_file(&self, entry: &Arc, parser: &ParserMemory) -> Option { if !entry.filename().to_string_lossy().ends_with(".txt") { return None; } @@ -110,7 +110,7 @@ impl FileHandler for ScriptValues { PdxFile::read(entry, parser) } - fn handle_file(&mut self, _entry: &FileEntry, block: Block) { + fn handle_file(&mut self, _entry: &Arc, block: Block) { for (key, bv) in block.iter_assignments_and_definitions_warn() { self.load_item(key, bv); } diff --git a/src/data/scripted_effects.rs b/src/data/scripted_effects.rs index 2cedd1ebd..fc3d66087 100644 --- a/src/data/scripted_effects.rs +++ b/src/data/scripted_effects.rs @@ -1,11 +1,12 @@ use std::fmt::Debug; use std::path::PathBuf; +use std::sync::Arc; use crate::block::Block; use crate::context::ScopeContext; use crate::effect::validate_effect; use crate::everything::Everything; -use crate::fileset::{FileEntry, FileHandler}; +use crate::files::{FileEntry, FileHandler}; #[cfg(feature = "hoi4")] use crate::game::Game; use crate::helpers::{dup_error, exact_dup_error, TigerHashMap, BANNED_NAMES}; @@ -98,7 +99,7 @@ impl FileHandler for Effects { PathBuf::from("common/scripted_effects") } - fn load_file(&self, entry: &FileEntry, parser: &ParserMemory) -> Option { + fn load_file(&self, entry: &Arc, parser: &ParserMemory) -> Option { if !entry.filename().to_string_lossy().ends_with(".txt") { return None; } @@ -110,7 +111,7 @@ impl FileHandler for Effects { PdxFile::read(entry, parser) } - fn handle_file(&mut self, _entry: &FileEntry, mut block: Block) { + fn handle_file(&mut self, _entry: &Arc, mut block: Block) { for (key, block) in block.drain_definitions_warn() { self.load_item(key, block); } diff --git a/src/data/scripted_lists.rs b/src/data/scripted_lists.rs index 068186bae..e586edd4c 100644 --- a/src/data/scripted_lists.rs +++ b/src/data/scripted_lists.rs @@ -1,10 +1,10 @@ use std::path::PathBuf; -use std::sync::RwLock; +use std::sync::{Arc, RwLock}; use crate::block::Block; use crate::context::ScopeContext; use crate::everything::Everything; -use crate::fileset::{FileEntry, FileHandler}; +use crate::files::{FileEntry, FileHandler}; use crate::helpers::{dup_error, TigerHashMap}; use crate::parse::ParserMemory; use crate::pdxfile::PdxFile; @@ -67,7 +67,7 @@ impl FileHandler for ScriptedLists { PathBuf::from("common/scripted_lists") } - fn load_file(&self, entry: &FileEntry, parser: &ParserMemory) -> Option { + fn load_file(&self, entry: &Arc, parser: &ParserMemory) -> Option { if !entry.filename().to_string_lossy().ends_with(".txt") { return None; } @@ -75,7 +75,7 @@ impl FileHandler for ScriptedLists { PdxFile::read(entry, parser) } - fn handle_file(&mut self, _entry: &FileEntry, mut block: Block) { + fn handle_file(&mut self, _entry: &Arc, mut block: Block) { for (key, block) in block.drain_definitions_warn() { self.load_item(key, block); } diff --git a/src/data/scripted_modifiers.rs b/src/data/scripted_modifiers.rs index 0e432d907..4bfffb92d 100644 --- a/src/data/scripted_modifiers.rs +++ b/src/data/scripted_modifiers.rs @@ -1,9 +1,10 @@ use std::path::PathBuf; +use std::sync::Arc; use crate::block::Block; use crate::context::ScopeContext; use crate::everything::Everything; -use crate::fileset::{FileEntry, FileHandler}; +use crate::files::{FileEntry, FileHandler}; use crate::helpers::{dup_error, TigerHashMap, BANNED_NAMES}; use crate::macros::{MacroCache, MACRO_MAP}; use crate::parse::ParserMemory; @@ -69,7 +70,7 @@ impl FileHandler for ScriptedModifiers { PathBuf::from("common/scripted_modifiers") } - fn load_file(&self, entry: &FileEntry, parser: &ParserMemory) -> Option { + fn load_file(&self, entry: &Arc, parser: &ParserMemory) -> Option { if !entry.filename().to_string_lossy().ends_with(".txt") { return None; } @@ -77,7 +78,7 @@ impl FileHandler for ScriptedModifiers { PdxFile::read(entry, parser) } - fn handle_file(&mut self, _entry: &FileEntry, mut block: Block) { + fn handle_file(&mut self, _entry: &Arc, mut block: Block) { for (key, block) in block.drain_definitions_warn() { self.load_item(key, block); } diff --git a/src/data/scripted_triggers.rs b/src/data/scripted_triggers.rs index e2fdf8905..b2bf6218e 100644 --- a/src/data/scripted_triggers.rs +++ b/src/data/scripted_triggers.rs @@ -1,9 +1,10 @@ use std::path::PathBuf; +use std::sync::Arc; use crate::block::Block; use crate::context::ScopeContext; use crate::everything::Everything; -use crate::fileset::{FileEntry, FileHandler}; +use crate::files::{FileEntry, FileHandler}; #[cfg(feature = "hoi4")] use crate::game::Game; use crate::helpers::{dup_error, exact_dup_error, TigerHashMap, BANNED_NAMES}; @@ -104,7 +105,7 @@ impl FileHandler for Triggers { PathBuf::from("common/scripted_triggers") } - fn load_file(&self, entry: &FileEntry, parser: &ParserMemory) -> Option { + fn load_file(&self, entry: &Arc, parser: &ParserMemory) -> Option { if !entry.filename().to_string_lossy().ends_with(".txt") { return None; } @@ -116,7 +117,7 @@ impl FileHandler for Triggers { PdxFile::read(entry, parser) } - fn handle_file(&mut self, _entry: &FileEntry, mut block: Block) { + fn handle_file(&mut self, _entry: &Arc, mut block: Block) { for (key, block) in block.drain_definitions_warn() { self.load_item(key, block); } diff --git a/src/dds.rs b/src/dds.rs index 487b1b768..e71dd1da5 100644 --- a/src/dds.rs +++ b/src/dds.rs @@ -3,13 +3,15 @@ use std::fs::{metadata, File}; use std::io::{Read, Result}; use std::path::PathBuf; +use std::sync::Arc; -use crate::fileset::{FileEntry, FileHandler}; +use crate::files::{FileEntry, FileHandler}; use crate::helpers::TigerHashMap; use crate::parse::ParserMemory; use crate::report::{err, tips, warn, ErrorKey}; #[cfg(feature = "ck3")] use crate::token::Token; +use crate::Loc; const DDS_HEADER_SIZE: usize = 124; const DDS_HEIGHT_OFFSET: usize = 12; @@ -29,7 +31,7 @@ pub struct DdsFiles { } impl DdsFiles { - fn load_dds(entry: &FileEntry) -> Result> { + fn load_dds(entry: &Arc) -> Result> { if metadata(entry.fullpath())?.len() == 0 { warn(ErrorKey::ImageFormat).msg("empty file").loc(entry).push(); return Ok(None); @@ -48,7 +50,7 @@ impl DdsFiles { err(ErrorKey::ImageFormat).msg("not a DDS file").loc(entry).push(); return Ok(None); } - Ok(Some(DdsInfo::new(entry.clone(), &buffer))) + Ok(Some(DdsInfo::new(Loc::from(entry), &buffer))) } fn handle_dds(&mut self, entry: &FileEntry, info: DdsInfo) { @@ -84,7 +86,7 @@ impl FileHandler for DdsFiles { PathBuf::from("gfx") } - fn load_file(&self, entry: &FileEntry, _parser: &ParserMemory) -> Option { + fn load_file(&self, entry: &Arc, _parser: &ParserMemory) -> Option { if !entry.filename().to_string_lossy().ends_with(".dds") { return None; } @@ -102,23 +104,23 @@ impl FileHandler for DdsFiles { } } - fn handle_file(&mut self, entry: &FileEntry, info: DdsInfo) { + fn handle_file(&mut self, entry: &Arc, info: DdsInfo) { self.handle_dds(entry, info); } } #[derive(Clone, Debug)] pub struct DdsInfo { - entry: FileEntry, + loc: Loc, compressed: bool, width: u32, height: u32, } impl DdsInfo { - pub fn new(entry: FileEntry, header: &[u8]) -> Self { + pub fn new(loc: Loc, header: &[u8]) -> Self { Self { - entry, + loc, compressed: (from_le32(header, DDS_PIXELFORMAT_FLAGS_OFFSET) & 0x04) != 0, width: from_le32(header, DDS_WIDTH_OFFSET), height: from_le32(header, DDS_HEIGHT_OFFSET), @@ -132,7 +134,7 @@ impl DdsInfo { "DDS file is {}x{}, which can cause scaling problems and graphical artifacts", self.width, self.height ); - err(ErrorKey::ImageSize).msg(msg).info(info).loc(&self.entry).push(); + err(ErrorKey::ImageSize).msg(msg).info(info).loc(self.loc).push(); } } } diff --git a/src/everything.rs b/src/everything.rs index 2b9bf46ac..9978dd7fd 100644 --- a/src/everything.rs +++ b/src/everything.rs @@ -55,7 +55,7 @@ use crate::data::{ }; use crate::db::{Db, DbKind}; use crate::dds::DdsFiles; -use crate::fileset::{FileEntry, FileKind, Fileset}; +use crate::files::{Fileset, FilesetBuilderWithMod}; use crate::game::Game; #[cfg(any(feature = "ck3", feature = "vic3"))] use crate::helpers::TigerHashSet; @@ -80,7 +80,7 @@ use crate::pdxfile::PdxFile; use crate::report::err; use crate::report::{report, set_output_style, ErrorKey, OutputStyle, Severity}; use crate::rivers::Rivers; -use crate::token::{Loc, Token}; +use crate::token::Token; use crate::variables::Variables; #[cfg(feature = "vic3")] use crate::vic3::data::{ @@ -112,9 +112,6 @@ pub enum FilesError { /// * During validation, `Everything` is immutable and cross-checking between item types can be done safely. #[derive(Debug)] pub struct Everything { - /// Config from file - config: Block, - /// The global parser state, carrying information between files. /// Currently only used by the pdxfile parser, to handle the `reader_export` directory, /// which is specially processed before all other files. @@ -234,15 +231,11 @@ impl Everything { /// /// `replace_paths` is from the similarly named field in the `.mod` file. pub fn new( + fileset: FilesetBuilderWithMod, config_filepath: Option<&Path>, - vanilla_dir: Option<&Path>, workshop_dir: Option<&Path>, paradox_dir: Option<&Path>, - mod_root: &Path, - replace_paths: Vec, ) -> Result { - let mut fileset = Fileset::new(vanilla_dir, mod_root.to_path_buf(), replace_paths); - let config_file_name = match Game::game() { #[cfg(feature = "ck3")] Game::Ck3 => "ck3-tiger.conf", @@ -256,26 +249,17 @@ impl Everything { let config_file = match config_filepath { Some(path) => path.to_path_buf(), - None => mod_root.join(config_file_name), - }; - - let config = if config_file.is_file() { - Self::read_config(config_file_name, &config_file) - .ok_or(FilesError::ConfigUnreadable { path: config_file })? - } else { - Block::new(Loc::for_file(config_file.clone(), FileKind::Mod, config_file.clone())) + None => fileset.the_mod.root().join(config_file_name), }; - fileset.config(config.clone(), workshop_dir, paradox_dir)?; - + let mut fileset = fileset.config(config_file, workshop_dir, paradox_dir)?; fileset.scan_all()?; - fileset.finalize(); + let fileset = fileset.finalize(); Ok(Everything { parser: ParserMemory::default(), fileset, dds: DdsFiles::default(), - config, #[cfg(any(feature = "ck3", feature = "vic3"))] warned_defines: RwLock::new(TigerHashSet::default()), database: Db::default(), @@ -346,14 +330,9 @@ impl Everything { }) } - fn read_config(name: &str, path: &Path) -> Option { - let entry = FileEntry::new(PathBuf::from(name), FileKind::Mod, path.to_path_buf()); - PdxFile::read_optional_bom(&entry, &ParserMemory::default()) - } - pub fn load_config_filtering_rules(&self) { - check_for_legacy_ignore(&self.config); - load_filter(&self.config); + check_for_legacy_ignore(&self.fileset.config); + load_filter(&self.fileset.config); } /// Load the `OutputStyle` settings from the config. @@ -361,9 +340,9 @@ impl Everything { /// by supplying the --no-color flag. fn load_output_styles(&self, default_color: bool) -> OutputStyle { // Treat a missing output_style block and an empty output_style block exactly the same. - let block = match self.config.get_field_block("output_style") { + let block = match self.fileset.config.get_field_block("output_style") { Some(block) => Cow::Borrowed(block), - None => Cow::Owned(Block::new(self.config.loc)), + None => Cow::Owned(Block::new(self.fileset.config.loc)), }; if !block.get_field_bool("enable").unwrap_or(default_color) { return OutputStyle::no_color(); @@ -1366,8 +1345,10 @@ mod benchmark { fn load_provinces_ck3(bencher: Bencher, (vanilla_dir, modpath): (&str, &PathBuf)) { bencher .with_inputs(|| { - Everything::new(None, Some(Path::new(vanilla_dir)), None, None, modpath, vec![]) - .unwrap() + let fileset = Fileset::builder(Some(Path::new(vanilla_dir))) + .with_modfile(modpath.clone()) + .unwrap(); + Everything::new(fileset, None, None, None).unwrap() }) .bench_local_refs(|everything| { everything.fileset.handle(&mut everything.provinces_ck3, &everything.parser); @@ -1379,8 +1360,10 @@ mod benchmark { fn load_provinces_vic3(bencher: Bencher, (vanilla_dir, modpath): (&str, &PathBuf)) { bencher .with_inputs(|| { - Everything::new(None, Some(Path::new(vanilla_dir)), None, None, modpath, vec![]) - .unwrap() + let fileset = Fileset::builder(Some(Path::new(vanilla_dir))) + .with_metadata(modpath.clone()) + .unwrap(); + Everything::new(fileset, None, None, None).unwrap() }) .bench_local_refs(|everything| { everything.fileset.handle(&mut everything.provinces_vic3, &everything.parser); @@ -1391,8 +1374,19 @@ mod benchmark { fn load_localization(bencher: Bencher, (vanilla_dir, modpath): (&str, &PathBuf)) { bencher .with_inputs(|| { - Everything::new(None, Some(Path::new(vanilla_dir)), None, None, modpath, vec![]) - .unwrap() + let fileset = Fileset::builder(Some(Path::new(vanilla_dir))); + let fileset = match Game::game() { + #[cfg(feature = "ck3")] + Game::Ck3 => fileset.with_modfile(modpath.clone()), + #[cfg(feature = "vic3")] + Game::Vic3 => fileset.with_metadata(modpath.clone()), + #[cfg(feature = "imperator")] + Game::Imperator => fileset.with_modfile(modpath.clone()), + #[cfg(feature = "hoi4")] + Game::Hoi4 => fileset.with_modfile(modpath.clone()), + } + .unwrap(); + Everything::new(fileset, None, None, None).unwrap() }) .bench_local_refs(|everything| { everything.fileset.handle(&mut everything.localization, &everything.parser); diff --git a/src/files/filecontent.rs b/src/files/filecontent.rs new file mode 100644 index 000000000..4f88206dc --- /dev/null +++ b/src/files/filecontent.rs @@ -0,0 +1,143 @@ +use std::{ + borrow::Cow, + fs::{read, read_to_string}, + slice::from_raw_parts, + sync::OnceLock, +}; + +use encoding_rs::WINDOWS_1252; + +use crate::{ + files::FileEntry, + report::{err, ErrorKey}, +}; + +pub const BOM_UTF8_BYTES: [u8; 3] = *b"\xef\xbb\xbf"; +pub const BOM_CHAR: char = '\u{feff}'; + +#[derive(Debug)] +pub struct FileContent { + /// Full content of the file. It must never be moved or modified. + // full: Cow<'static, str>, + full: &'static str, + /// The file content split into lines. Cached to avoid doing that work again. + // Stored as raw pointers to make the self reference work + lines: OnceLock>, +} + +impl FileContent { + pub fn new(mut s: String) -> Self { + s.shrink_to_fit(); + //Self { full: Cow::from(s), lines: OnceLock::new() } + Self { full: String::leak(s), lines: OnceLock::new() } + } + + pub fn new_static(s: &'static str) -> Self { + //Self { full: Cow::from(s), lines: OnceLock::new() } + Self { full: s, lines: OnceLock::new() } + } + + pub fn read(entry: &FileEntry) -> Option { + let contents = match read_to_string(entry.fullpath()) { + Ok(contents) => contents, + Err(e) => { + err(ErrorKey::ReadError) + .msg("could not read file") + .info(format!("{e:#}")) + .abbreviated(entry) + .push(); + return None; + } + }; + Some(FileContent::new(contents)) + } + + pub fn read_detect_encoding(entry: &FileEntry) -> Option { + let bytes = match read(entry.fullpath()) { + Ok(bytes) => bytes, + Err(e) => { + err(ErrorKey::ReadError) + .msg("could not read file") + .info(format!("{e:#}")) + .abbreviated(entry) + .push(); + return None; + } + }; + if bytes.starts_with(&BOM_UTF8_BYTES) { + let contents = match String::from_utf8(bytes) { + Ok(utf8) => utf8, + Err(e) => { + err(ErrorKey::ReadError) + .msg("could not decode UTF-8 file") + .info(format!("{e:#}")) + .abbreviated(entry) + .push(); + return None; + } + }; + Some(FileContent::new(contents)) + } else { + let contents = + match WINDOWS_1252.decode_without_bom_handling_and_without_replacement(&bytes) { + Some(Cow::Owned(utf8)) => utf8, + Some(Cow::Borrowed(_)) => unsafe { + // If the result is borrowed, we can transform the original byte vec + // and avoid the data copy we'd otherwise get doirg Cow::to_owned() + // Safety: the byte vec is already confirmed to be valid uft8. + String::from_utf8_unchecked(bytes) + }, + None => { + err(ErrorKey::Encoding) + .msg("could not decode WINDOWS-1252 file") + .abbreviated(entry) + .push(); + return None; + } + }; + Some(FileContent::new(contents)) + } + } + + pub fn full(&self) -> &'static str { + self.full + } + + pub fn has_bom(&self) -> bool { + self.full.starts_with(BOM_CHAR) + } + + pub fn nobom(&self) -> &'static str { + self.full.strip_prefix(BOM_CHAR).unwrap_or(self.full) + } + + fn lines_get_or_init(&self) -> &[(*const u8, usize)] { + self.lines + .get_or_init(|| self.nobom().lines().map(|r| (r.as_ptr(), r.len())).collect()) + .as_ref() + } + + pub fn lines(&self) -> impl Iterator { + self.lines_get_or_init().iter().map(|(p, len)| unsafe { + // Safety: p and len are constructed from the owned or static string in `full`. + // It is never modified or moved, and guaranteed to be valid utf8. + std::str::from_utf8_unchecked(from_raw_parts(*p, *len)) + }) + } + + pub fn line(&self, line_num: usize) -> Option<&str> { + if line_num == 0 { + None + } else { + self.lines_get_or_init().get(line_num - 1).map(|(p, len)| unsafe { + // Safety: p and len are constructed from the owned or static string in `full`. + // It is never modified or moved, and guaranteed to be valid utf8. + std::str::from_utf8_unchecked(from_raw_parts(*p, *len)) + }) + } + } +} + +// Safety: The raw pointers are never exposed, so threads can't do anything funny with them +unsafe impl Send for FileContent {} +unsafe impl Sync for FileContent {} diff --git a/src/files/filedb.rs b/src/files/filedb.rs new file mode 100644 index 000000000..659254f95 --- /dev/null +++ b/src/files/filedb.rs @@ -0,0 +1,22 @@ +use std::{path::PathBuf, sync::Arc}; + +use crate::{files::FileEntry, FileKind, TigerHashMap}; + +#[derive(Debug, Default)] +pub struct FileDb { + /// The base game and mod files in arbitrary order. + pub(super) files: TigerHashMap>, +} + +impl FileDb { + /// Gets a reference to a file entry. + /// If the fileset has not been finalized, a new entry will be created and returned if needed. + pub(crate) fn get_or_create_entry( + &mut self, + path: PathBuf, + kind: FileKind, + fullpath: PathBuf, + ) -> &Arc { + self.files.entry(fullpath.clone()).or_insert_with(|| FileEntry::new(path, kind, fullpath)) + } +} diff --git a/src/fileset.rs b/src/files/fileset.rs similarity index 51% rename from src/fileset.rs rename to src/files/fileset.rs index f9cf41f91..40a5d773b 100644 --- a/src/fileset.rs +++ b/src/files/fileset.rs @@ -1,33 +1,29 @@ //! Track all the files (vanilla and mods) that are relevant to the current validation. -use std::borrow::ToOwned; use std::cmp::Ordering; use std::ffi::OsStr; use std::fmt::{Display, Formatter}; +use std::hash::{Hash, Hasher}; use std::path::{Path, PathBuf}; use std::string::ToString; -use std::sync::RwLock; +use std::sync::{Arc, OnceLock, RwLock}; -use anyhow::{bail, Result}; +use anyhow::Result; use rayon::prelude::*; -use walkdir::WalkDir; +use super::filecontent::FileContent; use crate::block::Block; -use crate::everything::{Everything, FilesError}; +use crate::everything::Everything; +use crate::files::fileset_builder::FilesetBuilder; +use crate::files::FileDb; use crate::game::Game; use crate::helpers::TigerHashSet; use crate::item::Item; -#[cfg(feature = "vic3")] -use crate::mod_metadata::ModMetadata; -#[cfg(any(feature = "ck3", feature = "imperator", feature = "hoi4"))] -use crate::modfile::ModFile; use crate::parse::ParserMemory; use crate::pathtable::{PathTable, PathTableIndex}; -use crate::report::{ - add_loaded_dlc_root, add_loaded_mod_root, err, fatal, report, ErrorKey, Severity, -}; +use crate::report::{err, fatal, report, store_source_file, ErrorKey, Severity}; use crate::token::Token; -use crate::util::fix_slashes_for_target_platform; +use crate::TigerHashMap; /// Note that ordering of these enum values matters. /// Files later in the order will override files of the same name before them, @@ -59,7 +55,7 @@ impl FileKind { } } -#[derive(Clone, Debug, PartialEq, Eq)] +#[derive(Debug)] pub struct FileEntry { /// Pathname components below the mod directory or the vanilla game dir /// Must not be empty. @@ -69,15 +65,49 @@ pub struct FileEntry { /// Index into the `PathTable`. Used to initialize `Loc`, which doesn't carry a copy of the pathbuf. /// A `FileEntry` might not have this index, because `FileEntry` needs to be usable before the (ordered) /// path table is created. - idx: Option, + idx: OnceLock, /// The full filesystem path of this entry. Not used for ordering or equality. fullpath: PathBuf, + /// The files contents. Not used for ordering or equality. + contents: OnceLock>, } impl FileEntry { - pub fn new(path: PathBuf, kind: FileKind, fullpath: PathBuf) -> Self { - assert!(path.file_name().is_some()); - Self { path, kind, idx: None, fullpath } + // We make sure every file entry is wrapped in an Arc so that everyone has access + // to late initialized fields + pub(super) fn new(path: PathBuf, kind: FileKind, fullpath: PathBuf) -> Arc { + debug_assert!(path.file_name().is_some()); + let entry = Arc::new(Self { + path, + kind, + idx: OnceLock::new(), + fullpath, + contents: OnceLock::new(), + }); + store_source_file(Arc::clone(&entry)); + entry + } + + pub fn new_internal(description: PathBuf, contents: &'static str) -> Self { + Self { + path: description.clone(), + kind: FileKind::Internal, + idx: OnceLock::from(PathTableIndex::UNTRACKED), + fullpath: description, + contents: OnceLock::from(Some(FileContent::new_static(contents))), + } + } + + /// Creates a new file entry which will not have errors logged. + pub fn new_untracked(path: PathBuf, kind: FileKind, fullpath: PathBuf) -> Self { + debug_assert!(path.file_name().is_some()); + Self { + path, + kind, + idx: OnceLock::from(PathTableIndex::UNTRACKED), + fullpath, + contents: OnceLock::new(), + } } pub fn kind(&self) -> FileKind { @@ -99,13 +129,28 @@ impl FileEntry { self.path.file_name().unwrap() } - fn store_in_pathtable(&mut self) { - assert!(self.idx.is_none()); - self.idx = Some(PathTable::store(self.path.clone(), self.fullpath.clone())); + /// Prefer `path_idx` when possible. + /// Using this funtion before the file set is finalized will + /// result in an unsorted path table + pub(crate) fn get_or_init_path_idx(&self) -> PathTableIndex { + *self.idx.get_or_init(|| PathTable::store(self.path.clone(), self.fullpath.clone())) } pub fn path_idx(&self) -> Option { - self.idx + self.idx.get().copied() + } + + /// Tests if this entry exists as a file on disk + pub fn is_file(&self) -> bool { + self.fullpath().is_file() + } + + pub fn contents(&self) -> Option<&FileContent> { + self.contents.get_or_init(|| FileContent::read(self)).as_ref() + } + + pub fn contents_detect_encoding(&self) -> Option<&FileContent> { + self.contents.get_or_init(|| FileContent::read_detect_encoding(self)).as_ref() } } @@ -115,6 +160,22 @@ impl Display for FileEntry { } } +impl PartialEq for FileEntry { + fn eq(&self, other: &Self) -> bool { + (self.idx.get().is_some() == other.idx.get().is_some() && self.idx == other.idx) + || (self.path == other.path && self.kind == other.kind) + } +} + +impl Eq for FileEntry {} + +impl Hash for FileEntry { + fn hash(&self, state: &mut H) { + self.path.hash(state); + self.kind.hash(state); + } +} + impl PartialOrd for FileEntry { fn partial_cmp(&self, other: &Self) -> Option { Some(self.cmp(other)) @@ -124,8 +185,9 @@ impl PartialOrd for FileEntry { impl Ord for FileEntry { fn cmp(&self, other: &Self) -> Ordering { // Compare idx if available (for speed), otherwise compare the paths. - let path_ord = if self.idx.is_some() && other.idx.is_some() { - self.idx.unwrap().cmp(&other.idx.unwrap()) + let path_ord = if let (Some(self_idx), Some(other_idx)) = (self.idx.get(), other.idx.get()) + { + self_idx.cmp(other_idx) } else { self.path.cmp(&other.path) }; @@ -153,17 +215,32 @@ pub trait FileHandler: Sync + Send { /// If a `T` is returned, it will be passed to `handle_file` later. /// Since `load_file` is executed multi-threaded while `handle_file` /// is single-threaded, try to do the heavy work in this function. - fn load_file(&self, entry: &FileEntry, parser: &ParserMemory) -> Option; + fn load_file(&self, entry: &Arc, parser: &ParserMemory) -> Option; /// This is called for each matching file in turn, in lexical order. /// That's the order in which the CK3 game engine loads them too. - fn handle_file(&mut self, entry: &FileEntry, loaded: T); + fn handle_file(&mut self, entry: &Arc, loaded: T); /// This is called after all files have been handled. /// The `FileHandler` can generate indexes, perform full-data checks, etc. fn finalize(&mut self) {} } +#[derive(Debug)] +#[allow(clippy::struct_field_names)] +pub(super) struct FilePaths { + /// The CK3 game directory. + pub(super) vanilla_root: Option, + + /// Extra CK3 directory loaded before vanilla. + #[cfg(feature = "jomini")] + pub(super) clausewitz_root: Option, + + /// Extra CK3 directory loaded before vanilla. + #[cfg(feature = "jomini")] + pub(super) jomini_root: Option, +} + #[derive(Clone, Debug)] pub struct LoadedMod { /// The `FileKind` to use for file entries from this mod. @@ -181,11 +258,16 @@ pub struct LoadedMod { } impl LoadedMod { - fn new_main_mod(root: PathBuf, replace_paths: Vec) -> Self { + pub(super) fn new_main_mod(root: PathBuf, replace_paths: Vec) -> Self { Self { kind: FileKind::Mod, label: "MOD".to_string(), root, replace_paths } } - fn new(kind: FileKind, label: String, root: PathBuf, replace_paths: Vec) -> Self { + pub(super) fn new( + kind: FileKind, + label: String, + root: PathBuf, + replace_paths: Vec, + ) -> Self { Self { kind, label, root, replace_paths } } @@ -204,333 +286,101 @@ impl LoadedMod { #[derive(Debug)] pub struct Fileset { - /// The CK3 game directory. - vanilla_root: Option, - - /// Extra CK3 directory loaded before vanilla. - #[cfg(feature = "jomini")] - clausewitz_root: Option, - - /// Extra CK3 directory loaded before vanilla. - #[cfg(feature = "jomini")] - jomini_root: Option, + #[allow(dead_code)] + pub(super) paths: FilePaths, /// The mod being analyzed. - the_mod: LoadedMod, + #[allow(dead_code)] + pub(super) the_mod: LoadedMod, /// Other mods to be loaded before `mod`, in order. pub loaded_mods: Vec, /// DLC directories to be loaded after vanilla, in order. - loaded_dlcs: Vec, + #[allow(dead_code)] + pub(super) loaded_dlcs: Vec, /// The ck3-tiger config. - config: Option, + pub(crate) config: Block, - /// The CK3 and mod files in arbitrary order (will be empty after `finalize`). - files: Vec, + /// Collection of all known files. + #[allow(dead_code)] + pub(super) db: FileDb, - /// The CK3 and mod files in the order the game would load them. - ordered_files: Vec, + /// The bose game and mod files in the order the game would load them. + pub(super) ordered_files: Vec>, /// Filename Tokens for the files in `ordered_files`. /// Used for [`Fileset::iter_keys()`]. - filename_tokens: Vec, + pub(super) filename_tokens: Vec, /// All filenames from `ordered_files`, for quick lookup. - filenames: TigerHashSet, + pub(super) filenames: TigerHashMap>, /// All directories that have been looked up, for quick lookup. - directories: RwLock>, + pub(super) directories: RwLock>, /// Filenames that have been looked up during validation. Used to filter the --unused output. - used: RwLock>, + pub(super) used: RwLock>, } impl Fileset { - pub fn new(vanilla_dir: Option<&Path>, mod_root: PathBuf, replace_paths: Vec) -> Self { - let vanilla_root = if Game::is_jomini() { - vanilla_dir.map(|dir| dir.join("game")) - } else { - vanilla_dir.map(ToOwned::to_owned) - }; - #[cfg(feature = "jomini")] - let clausewitz_root = vanilla_dir.map(|dir| dir.join("clausewitz")); - #[cfg(feature = "jomini")] - let jomini_root = vanilla_dir.map(|dir| dir.join("jomini")); - - Fileset { - vanilla_root, - #[cfg(feature = "jomini")] - clausewitz_root, - #[cfg(feature = "jomini")] - jomini_root, - the_mod: LoadedMod::new_main_mod(mod_root, replace_paths), - loaded_mods: Vec::new(), - loaded_dlcs: Vec::new(), - config: None, - files: Vec::new(), - ordered_files: Vec::new(), - filename_tokens: Vec::new(), - filenames: TigerHashSet::default(), - directories: RwLock::new(TigerHashSet::default()), - used: RwLock::new(TigerHashSet::default()), - } - } - - pub fn config( - &mut self, - config: Block, - #[allow(unused_variables)] workshop_dir: Option<&Path>, - #[allow(unused_variables)] paradox_dir: Option<&Path>, - ) -> Result<()> { - let config_path = config.loc.fullpath(); - for block in config.get_field_blocks("load_mod") { - let mod_idx; - if let Ok(idx) = u8::try_from(self.loaded_mods.len()) { - mod_idx = idx; - } else { - bail!("too many loaded mods, cannot process more"); - } - - let default_label = || format!("MOD{mod_idx}"); - let label = - block.get_field_value("label").map_or_else(default_label, ToString::to_string); - - if Game::is_ck3() || Game::is_imperator() || Game::is_hoi4() { - #[cfg(any(feature = "ck3", feature = "imperator", feature = "hoi4"))] - if let Some(path) = get_modfile(&label, config_path, block, paradox_dir) { - let modfile = ModFile::read(&path)?; - eprintln!( - "Loading secondary mod {label} from: {}{}", - modfile.modpath().display(), - modfile - .display_name() - .map_or_else(String::new, |name| format!(" \"{name}\"")), - ); - let kind = FileKind::LoadedMod(mod_idx); - let loaded_mod = LoadedMod::new( - kind, - label.clone(), - modfile.modpath().clone(), - modfile.replace_paths(), - ); - add_loaded_mod_root(label); - self.loaded_mods.push(loaded_mod); - } else { - bail!("could not load secondary mod from config; missing valid `modfile` or `workshop_id` field"); - } - } else if Game::is_vic3() { - #[cfg(feature = "vic3")] - if let Some(pathdir) = get_mod(&label, config_path, block, workshop_dir) { - match ModMetadata::read(&pathdir) { - Ok(metadata) => { - eprintln!( - "Loading secondary mod {label} from: {}{}", - pathdir.display(), - metadata - .display_name() - .map_or_else(String::new, |name| format!(" \"{name}\"")), - ); - let kind = FileKind::LoadedMod(mod_idx); - let loaded_mod = LoadedMod::new( - kind, - label.clone(), - pathdir, - metadata.replace_paths(), - ); - add_loaded_mod_root(label); - self.loaded_mods.push(loaded_mod); - } - Err(e) => { - eprintln!( - "could not load secondary mod {label} from: {}", - pathdir.display() - ); - eprintln!(" because: {e}"); - } - } - } else { - bail!("could not load secondary mod from config; missing valid `mod` or `workshop_id` field"); - } - } - } - self.config = Some(config); - Ok(()) + pub fn builder(vanilla_dir: Option<&Path>) -> FilesetBuilder { + FilesetBuilder::new(vanilla_dir) } - fn should_replace(&self, path: &Path, kind: FileKind) -> bool { - if kind == FileKind::Mod { - return false; - } - if kind < FileKind::Mod && self.the_mod.should_replace(path) { - return true; - } - for loaded_mod in &self.loaded_mods { - if kind < loaded_mod.kind && loaded_mod.should_replace(path) { - return true; - } - } - false - } - - fn scan(&mut self, path: &Path, kind: FileKind) -> Result<(), walkdir::Error> { - for entry in WalkDir::new(path) { - let entry = entry?; - if entry.depth() == 0 || !entry.file_type().is_file() { - continue; - } - // unwrap is safe here because WalkDir gives us paths with this prefix. - let inner_path = entry.path().strip_prefix(path).unwrap(); - if inner_path.starts_with(".git") { - continue; - } - let inner_dir = inner_path.parent().unwrap_or_else(|| Path::new("")); - if self.should_replace(inner_dir, kind) { - continue; - } - self.files.push(FileEntry::new( - inner_path.to_path_buf(), - kind, - entry.path().to_path_buf(), - )); - } - Ok(()) - } - - pub fn scan_all(&mut self) -> Result<(), FilesError> { - #[cfg(feature = "jomini")] - if let Some(clausewitz_root) = self.clausewitz_root.clone() { - self.scan(&clausewitz_root.clone(), FileKind::Clausewitz).map_err(|e| { - FilesError::VanillaUnreadable { path: clausewitz_root.clone(), source: e } - })?; - } - #[cfg(feature = "jomini")] - if let Some(jomini_root) = &self.jomini_root.clone() { - self.scan(&jomini_root.clone(), FileKind::Jomini).map_err(|e| { - FilesError::VanillaUnreadable { path: jomini_root.clone(), source: e } - })?; - } - if let Some(vanilla_root) = &self.vanilla_root.clone() { - self.scan(&vanilla_root.clone(), FileKind::Vanilla).map_err(|e| { - FilesError::VanillaUnreadable { path: vanilla_root.clone(), source: e } - })?; - #[cfg(feature = "hoi4")] - if Game::is_hoi4() { - self.load_dlcs(&vanilla_root.join("integrated_dlc"))?; - } - self.load_dlcs(&vanilla_root.join("dlc"))?; - } - // loaded_mods is cloned here for the borrow checker - for loaded_mod in &self.loaded_mods.clone() { - self.scan(loaded_mod.root(), loaded_mod.kind()).map_err(|e| { - FilesError::ModUnreadable { path: loaded_mod.root().to_path_buf(), source: e } - })?; - } - #[allow(clippy::unnecessary_to_owned)] // borrow checker requires to_path_buf here - self.scan(&self.the_mod.root().to_path_buf(), FileKind::Mod).map_err(|e| { - FilesError::ModUnreadable { path: self.the_mod.root().to_path_buf(), source: e } - })?; - Ok(()) - } - - pub fn load_dlcs(&mut self, dlc_root: &Path) -> Result<(), FilesError> { - for entry in WalkDir::new(dlc_root).max_depth(1).sort_by_file_name().into_iter().flatten() { - if entry.depth() == 1 && entry.file_type().is_dir() { - let label = entry.file_name().to_string_lossy().to_string(); - let idx = - u8::try_from(self.loaded_dlcs.len()).expect("more than 256 DLCs installed"); - let dlc = LoadedMod::new( - FileKind::Dlc(idx), - label.clone(), - entry.path().to_path_buf(), - Vec::new(), - ); - self.scan(dlc.root(), dlc.kind()).map_err(|e| FilesError::VanillaUnreadable { - path: dlc.root().to_path_buf(), - source: e, - })?; - self.loaded_dlcs.push(dlc); - add_loaded_dlc_root(label); - } - } - Ok(()) - } - - pub fn finalize(&mut self) { - // This sorts by pathname but where pathnames are equal it places `Mod` entries after `Vanilla` entries - // and `LoadedMod` entries between them in order - self.files.sort(); - - // When there are identical paths, only keep the last entry of them. - for entry in self.files.drain(..) { - if let Some(prev) = self.ordered_files.last_mut() { - if entry.path == prev.path { - *prev = entry; - } else { - self.ordered_files.push(entry); - } - } else { - self.ordered_files.push(entry); - } - } - - for entry in &mut self.ordered_files { - let token = Token::new(&entry.filename().to_string_lossy(), (&*entry).into()); - self.filename_tokens.push(token); - entry.store_in_pathtable(); - self.filenames.insert(entry.path.clone()); - } - } - - pub fn get_files_under<'a>(&'a self, subpath: &'a Path) -> &'a [FileEntry] { + pub fn get_files_under<'a>(&'a self, subpath: &'a Path) -> &'a [Arc] { let start = self.ordered_files.partition_point(|entry| entry.path < subpath); let end = start + self.ordered_files[start..].partition_point(|entry| entry.path.starts_with(subpath)); &self.ordered_files[start..end] } - pub fn filter_map_under(&self, subpath: &Path, f: F) -> Vec + pub fn filter_map_under<'a, F, T>(&'a self, subpath: &'a Path, f: F) -> Vec where - F: Fn(&FileEntry) -> Option + Sync + Send, + F: Fn(&'a Arc) -> Option + Sync + Send, T: Send, { self.get_files_under(subpath).par_iter().filter_map(f).collect() } pub fn handle>(&self, handler: &mut H, parser: &ParserMemory) { - if let Some(config) = &self.config { - handler.config(config); - } + handler.config(&self.config); let subpath = handler.subpath(); let entries = self.filter_map_under(&subpath, |entry| { - handler.load_file(entry, parser).map(|loaded| (entry.clone(), loaded)) + handler.load_file(entry, parser).map(|loaded| (entry, loaded)) }); for (entry, loaded) in entries { - handler.handle_file(&entry, loaded); + handler.handle_file(entry, loaded); } handler.finalize(); } + /// # Panics pub fn mark_used(&self, file: &str) { let file = file.strip_prefix('/').unwrap_or(file); self.used.write().unwrap().insert(file.to_string()); } - pub fn exists(&self, key: &str) -> bool { + pub fn entry(&self, key: &str) -> Option<&Arc> { let key = key.strip_prefix('/').unwrap_or(key); let filepath = if Game::is_hoi4() && key.contains('\\') { PathBuf::from(key.replace('\\', "/")) } else { PathBuf::from(key) }; - self.filenames.contains(&filepath) + self.filenames.get(&filepath) + } + + pub fn exists(&self, key: &str) -> bool { + self.entry(key).is_some() } pub fn iter_keys(&self) -> impl Iterator { self.filename_tokens.iter() } + /// # Panics pub fn entry_exists(&self, key: &str) -> bool { // file exists if self.exists(key) { @@ -598,6 +448,7 @@ impl Fileset { } } + /// # Panics pub fn validate(&self, _data: &Everything) { let common_dirs = match Game::game() { #[cfg(feature = "ck3")] @@ -647,7 +498,7 @@ impl Fileset { if joined.starts_with(valid) { let msg = format!("file in unexpected directory {}", dirname.display()); let info = format!("did you mean common/{} ?", dirname.display()); - err(ErrorKey::Filename).msg(msg).info(info).loc(entry).push(); + err(ErrorKey::Filename).msg(msg).info(info).loc(&**entry).push(); warned.push(dirname); continue 'outer; } @@ -694,6 +545,8 @@ impl Fileset { } } + /// # Panics + /// Will panic if a lock on self.used can not be obtained pub fn check_unused_dds(&self, _data: &Everything) { let mut vec = Vec::new(); for entry in &self.ordered_files { @@ -713,73 +566,3 @@ impl Fileset { } } } - -#[cfg(any(feature = "ck3", feature = "imperator", feature = "hoi4"))] -fn get_modfile( - label: &String, - config_path: &Path, - block: &Block, - paradox_dir: Option<&Path>, -) -> Option { - let mut path: Option = None; - if let Some(modfile) = block.get_field_value("modfile") { - let modfile_path = fix_slashes_for_target_platform( - config_path - .parent() - .unwrap() // SAFETY: known to be for a file in a directory - .join(modfile.as_str()), - ); - if modfile_path.exists() { - path = Some(modfile_path); - } else { - eprintln!("Could not find mod {label} at: {}", modfile_path.display()); - } - } - if path.is_none() { - if let Some(workshop_id) = block.get_field_value("workshop_id") { - match paradox_dir { - Some(p) => { - path = Some(fix_slashes_for_target_platform( - p.join(format!("mod/ugc_{workshop_id}.mod")), - )); - } - None => eprintln!("workshop_id defined, but could not find paradox directory"), - } - } - } - path -} - -#[cfg(feature = "vic3")] -fn get_mod( - label: &String, - config_path: &Path, - block: &Block, - workshop_dir: Option<&Path>, -) -> Option { - let mut path: Option = None; - if let Some(modfile) = block.get_field_value("mod") { - let mod_path = fix_slashes_for_target_platform( - config_path - .parent() - .unwrap() // SAFETY: known to be for a file in a directory - .join(modfile.as_str()), - ); - if mod_path.exists() { - path = Some(mod_path); - } else { - eprintln!("Could not find mod {label} at: {}", mod_path.display()); - } - } - if path.is_none() { - if let Some(workshop_id) = block.get_field_value("workshop_id") { - match workshop_dir { - Some(w) => { - path = Some(fix_slashes_for_target_platform(w.join(workshop_id.as_str()))); - } - None => eprintln!("workshop_id defined, but could not find workshop"), - } - } - } - path -} diff --git a/src/files/fileset_builder.rs b/src/files/fileset_builder.rs new file mode 100644 index 000000000..a6755527e --- /dev/null +++ b/src/files/fileset_builder.rs @@ -0,0 +1,429 @@ +use std::path::Path; +use std::path::PathBuf; +use std::sync::Arc; +use std::sync::RwLock; + +use anyhow::bail; +use anyhow::Result; +use itertools::Itertools as _; +use walkdir::WalkDir; + +use super::filedb::FileDb; +use super::fileset::FilePaths; +use super::fileset::LoadedMod; +use super::FileKind; +use crate::add_loaded_mod_root; +use crate::block::Block; +use crate::everything::FilesError; +use crate::files::FileEntry; +use crate::parse::ParserMemory; +use crate::pdxfile::PdxFile; +use crate::report::add_loaded_dlc_root; +use crate::util::fix_slashes_for_target_platform; +use crate::Fileset; +use crate::Game; +use crate::Loc; +#[cfg(any(feature = "ck3", feature = "imperator", feature = "hoi4"))] +use crate::ModFile; +#[cfg(feature = "vic3")] +use crate::ModMetadata; +use crate::TigerHashMap; +use crate::TigerHashSet; +use crate::Token; + +#[derive(Debug)] +pub struct FilesetBuilder { + paths: FilePaths, + db: FileDb, +} + +impl FilesetBuilder { + pub(crate) fn new(vanilla_dir: Option<&Path>) -> Self { + Self { + paths: FilePaths { + vanilla_root: if Game::is_jomini() { + vanilla_dir.map(|dir| dir.join("game")) + } else { + vanilla_dir.map(ToOwned::to_owned) + }, + #[cfg(feature = "jomini")] + clausewitz_root: vanilla_dir.map(|dir| dir.join("clausewitz")), + #[cfg(feature = "jomini")] + jomini_root: vanilla_dir.map(|dir| dir.join("jomini")), + }, + db: FileDb::default(), + } + } + + #[cfg(any(feature = "ck3", feature = "imperator", feature = "hoi4"))] + pub fn with_modfile(mut self, mut modpath: PathBuf) -> Result { + if modpath.is_dir() { + modpath.push("descriptor.mod"); + } + let modfile_entry = self.db.get_or_create_entry(modpath.clone(), FileKind::Mod, modpath); + let modfile = ModFile::read(modfile_entry)?; + + let modpath = modfile.modpath(); + if modpath.exists() { + eprintln!("Using mod directory: {}", modpath.display()); + } else { + eprintln!("Looking for mod in {}", modpath.display()); + bail!("Cannot find mod directory. Please make sure the .mod file is correct."); + } + + let replace_paths = modfile.replace_paths(); + + Ok(FilesetBuilderWithMod { + paths: self.paths, + db: self.db, + the_mod: LoadedMod::new_main_mod(modfile.modpath(), replace_paths), + }) + } + + #[cfg(feature = "vic3")] + pub fn with_metadata(mut self, mod_root: PathBuf) -> Result { + let metadata = ModMetadata::read(&mut self.db, mod_root)?; + + eprintln!("Using mod directory: {}", metadata.modpath().display()); + + let replace_paths = metadata.replace_paths(); + + Ok(FilesetBuilderWithMod { + paths: self.paths, + db: self.db, + the_mod: LoadedMod::new_main_mod(metadata.modpath().to_path_buf(), replace_paths), + }) + } +} + +#[derive(Debug)] +pub struct FilesetBuilderWithMod { + paths: FilePaths, + db: FileDb, + pub(crate) the_mod: LoadedMod, +} + +impl FilesetBuilderWithMod { + pub fn config( + mut self, + config_file: PathBuf, + #[allow(unused_variables)] workshop_dir: Option<&Path>, + #[allow(unused_variables)] paradox_dir: Option<&Path>, + ) -> Result { + let config_entry = + self.db.get_or_create_entry(config_file.clone(), FileKind::Mod, config_file); + let config = if config_entry.is_file() { + PdxFile::read_optional_bom(config_entry, &ParserMemory::default()) + .ok_or(FilesError::ConfigUnreadable { path: config_entry.path().to_path_buf() })? + } else { + Block::new(Loc::from(config_entry)) + }; + + let mut loaded_mods = vec![]; + + let config_path = config.loc.fullpath(); + for block in config.get_field_blocks("load_mod") { + let mod_idx; + if let Ok(idx) = u8::try_from(loaded_mods.len()) { + mod_idx = idx; + } else { + bail!("too many loaded mods, cannot process more"); + } + + let default_label = || format!("MOD{mod_idx}"); + let label = + block.get_field_value("label").map_or_else(default_label, ToString::to_string); + + if Game::is_ck3() || Game::is_imperator() || Game::is_hoi4() { + #[cfg(any(feature = "ck3", feature = "imperator", feature = "hoi4"))] + if let Some(path) = get_modfile(&label, config_path, block, paradox_dir) { + let kind = FileKind::LoadedMod(mod_idx); + let modfile_entry = self.db.get_or_create_entry(path.clone(), kind, path); + let modfile = ModFile::read(modfile_entry)?; + eprintln!( + "Loading secondary mod {label} from: {}{}", + modfile.modpath().display(), + modfile + .display_name() + .map_or_else(String::new, |name| format!(" \"{name}\"")), + ); + let loaded_mod = LoadedMod::new( + kind, + label.clone(), + modfile.modpath().clone(), + modfile.replace_paths(), + ); + add_loaded_mod_root(label); + loaded_mods.push(loaded_mod); + } else { + bail!("could not load secondary mod from config; missing valid `modfile` or `workshop_id` field"); + } + } else if Game::is_vic3() { + #[cfg(feature = "vic3")] + if let Some(pathdir) = get_mod(&label, config_path, block, workshop_dir) { + match ModMetadata::read(&mut self.db, pathdir.clone()) { + Ok(metadata) => { + eprintln!( + "Loading secondary mod {label} from: {}{}", + pathdir.display(), + metadata + .display_name() + .map_or_else(String::new, |name| format!(" \"{name}\"")), + ); + let kind = FileKind::LoadedMod(mod_idx); + let loaded_mod = LoadedMod::new( + kind, + label.clone(), + pathdir, + metadata.replace_paths(), + ); + add_loaded_mod_root(label); + loaded_mods.push(loaded_mod); + } + Err(e) => { + eprintln!( + "could not load secondary mod {label} from: {}", + pathdir.display() + ); + eprintln!(" because: {e}"); + } + } + } else { + bail!("could not load secondary mod from config; missing valid `mod` or `workshop_id` field"); + } + } + } + Ok(FilesetBuilderWithConfig { + paths: self.paths, + db: self.db, + config, + the_mod: self.the_mod, + loaded_mods, + loaded_dlcs: vec![], + }) + } +} + +#[derive(Debug)] +pub struct FilesetBuilderWithConfig { + paths: FilePaths, + db: FileDb, + config: Block, + the_mod: LoadedMod, + loaded_mods: Vec, + loaded_dlcs: Vec, +} + +impl FilesetBuilderWithConfig { + fn should_replace(&self, path: &Path, kind: FileKind) -> bool { + if kind == FileKind::Mod { + return false; + } + if kind < FileKind::Mod && self.the_mod.should_replace(path) { + return true; + } + for loaded_mod in &self.loaded_mods { + if kind < loaded_mod.kind() && loaded_mod.should_replace(path) { + return true; + } + } + false + } + + fn scan(&mut self, path: &Path, kind: FileKind) -> Result<(), walkdir::Error> { + for entry in WalkDir::new(path) { + let entry = entry?; + if entry.depth() == 0 || !entry.file_type().is_file() { + continue; + } + // unwrap is safe here because WalkDir gives us paths with this prefix. + let inner_path = entry.path().strip_prefix(path).unwrap(); + if inner_path.starts_with(".git") { + continue; + } + let inner_dir = inner_path.parent().unwrap_or_else(|| Path::new("")); + if self.should_replace(inner_dir, kind) { + continue; + } + self.db.files.entry(entry.path().to_path_buf()).or_insert_with(|| { + FileEntry::new(inner_path.to_path_buf(), kind, entry.path().to_path_buf()) + }); + } + Ok(()) + } + + pub fn scan_all(&mut self) -> Result<(), FilesError> { + #[cfg(feature = "jomini")] + if let Some(clausewitz_root) = self.paths.clausewitz_root.clone() { + self.scan(&clausewitz_root.clone(), FileKind::Clausewitz).map_err(|e| { + FilesError::VanillaUnreadable { path: clausewitz_root.clone(), source: e } + })?; + } + #[cfg(feature = "jomini")] + if let Some(jomini_root) = &self.paths.jomini_root.clone() { + self.scan(&jomini_root.clone(), FileKind::Jomini).map_err(|e| { + FilesError::VanillaUnreadable { path: jomini_root.clone(), source: e } + })?; + } + if let Some(vanilla_root) = &self.paths.vanilla_root.clone() { + self.scan(&vanilla_root.clone(), FileKind::Vanilla).map_err(|e| { + FilesError::VanillaUnreadable { path: vanilla_root.clone(), source: e } + })?; + #[cfg(feature = "hoi4")] + if Game::is_hoi4() { + self.load_dlcs(&vanilla_root.join("integrated_dlc"))?; + } + self.load_dlcs(&vanilla_root.join("dlc"))?; + } + // loaded_mods is cloned here for the borrow checker + for loaded_mod in &self.loaded_mods.clone() { + self.scan(loaded_mod.root(), loaded_mod.kind()).map_err(|e| { + FilesError::ModUnreadable { path: loaded_mod.root().to_path_buf(), source: e } + })?; + } + #[allow(clippy::unnecessary_to_owned)] // borrow checker requires to_path_buf here + self.scan(&self.the_mod.root().to_path_buf(), FileKind::Mod).map_err(|e| { + FilesError::ModUnreadable { path: self.the_mod.root().to_path_buf(), source: e } + })?; + Ok(()) + } + + /// # Panics + /// Will panic if more than 256 DLCs are installed + pub fn load_dlcs(&mut self, dlc_root: &Path) -> Result<(), FilesError> { + for entry in WalkDir::new(dlc_root).max_depth(1).sort_by_file_name().into_iter().flatten() { + if entry.depth() == 1 && entry.file_type().is_dir() { + let label = entry.file_name().to_string_lossy().to_string(); + let idx = + u8::try_from(self.loaded_dlcs.len()).expect("more than 256 DLCs installed"); + let dlc = LoadedMod::new( + FileKind::Dlc(idx), + label.clone(), + entry.path().to_path_buf(), + Vec::new(), + ); + self.scan(dlc.root(), dlc.kind()).map_err(|e| FilesError::VanillaUnreadable { + path: dlc.root().to_path_buf(), + source: e, + })?; + self.loaded_dlcs.push(dlc); + add_loaded_dlc_root(label); + } + } + Ok(()) + } + + pub fn finalize(self) -> Fileset { + // This sorts by pathname but where pathnames are equal it places `Mod` entries after `Vanilla` entries + // and `LoadedMod` entries between them in order + let sorted = self.db.files.values().sorted(); + + let mut ordered_files: Vec> = vec![]; + + // When there are identical paths, only keep the last entry of them. + for entry in sorted { + if let Some(prev) = ordered_files.last_mut() { + if entry.path() == prev.path() { + *prev = Arc::clone(entry); + } else { + ordered_files.push(Arc::clone(entry)); + } + } else { + ordered_files.push(Arc::clone(entry)); + } + } + + let mut filename_tokens = vec![]; + let mut filenames = TigerHashMap::default(); + for entry in &ordered_files { + entry.get_or_init_path_idx(); + let token = Token::new(&entry.filename().to_string_lossy(), entry.into()); + filename_tokens.push(token); + filenames.insert(entry.path().to_path_buf(), Arc::clone(entry)); + } + + Fileset { + paths: self.paths, + the_mod: self.the_mod, + loaded_mods: self.loaded_mods, + loaded_dlcs: self.loaded_dlcs, + config: self.config, + db: self.db, + ordered_files, + filename_tokens, + filenames, + directories: RwLock::new(TigerHashSet::default()), + used: RwLock::new(TigerHashSet::default()), + } + } +} + +#[cfg(any(feature = "ck3", feature = "imperator", feature = "hoi4"))] +fn get_modfile( + label: &String, + config_path: &Path, + block: &Block, + paradox_dir: Option<&Path>, +) -> Option { + let mut path: Option = None; + if let Some(modfile) = block.get_field_value("modfile") { + let modfile_path = fix_slashes_for_target_platform( + config_path + .parent() + .unwrap() // SAFETY: known to be for a file in a directory + .join(modfile.as_str()), + ); + if modfile_path.exists() { + path = Some(modfile_path); + } else { + eprintln!("Could not find mod {label} at: {}", modfile_path.display()); + } + } + if path.is_none() { + if let Some(workshop_id) = block.get_field_value("workshop_id") { + match paradox_dir { + Some(p) => { + path = Some(fix_slashes_for_target_platform( + p.join(format!("mod/ugc_{workshop_id}.mod")), + )); + } + None => eprintln!("workshop_id defined, but could not find paradox directory"), + } + } + } + path +} + +#[cfg(feature = "vic3")] +fn get_mod( + label: &String, + config_path: &Path, + block: &Block, + workshop_dir: Option<&Path>, +) -> Option { + let mut path: Option = None; + if let Some(modfile) = block.get_field_value("mod") { + let mod_path = fix_slashes_for_target_platform( + config_path + .parent() + .unwrap() // SAFETY: known to be for a file in a directory + .join(modfile.as_str()), + ); + if mod_path.exists() { + path = Some(mod_path); + } else { + eprintln!("Could not find mod {label} at: {}", mod_path.display()); + } + } + if path.is_none() { + if let Some(workshop_id) = block.get_field_value("workshop_id") { + match workshop_dir { + Some(w) => { + path = Some(fix_slashes_for_target_platform(w.join(workshop_id.as_str()))); + } + None => eprintln!("workshop_id defined, but could not find workshop"), + } + } + } + path +} diff --git a/src/files/mod.rs b/src/files/mod.rs new file mode 100644 index 000000000..4787ab0df --- /dev/null +++ b/src/files/mod.rs @@ -0,0 +1,11 @@ +mod filecontent; +mod filedb; +mod fileset; +mod fileset_builder; + +pub(crate) use filedb::FileDb; +pub use fileset::FileEntry; +pub use fileset::FileHandler; +pub use fileset::FileKind; +pub use fileset::Fileset; +pub(crate) use fileset_builder::FilesetBuilderWithMod; diff --git a/src/hoi4/data/events.rs b/src/hoi4/data/events.rs index adb92d5a4..8bd708ab1 100644 --- a/src/hoi4/data/events.rs +++ b/src/hoi4/data/events.rs @@ -1,11 +1,11 @@ use std::path::PathBuf; use std::str::FromStr; -use std::sync::Mutex; +use std::sync::{Arc, Mutex}; use crate::block::{Block, BlockItem, Field}; use crate::context::{Reason, ScopeContext, Signature}; use crate::everything::Everything; -use crate::fileset::{FileEntry, FileHandler}; +use crate::files::{FileEntry, FileHandler}; use crate::helpers::{dup_error, TigerHashMap, TigerHashSet}; use crate::hoi4::events::{get_event_scope, validate_event}; use crate::item::Item; @@ -112,7 +112,7 @@ impl FileHandler for Hoi4Events { PathBuf::from("events") } - fn load_file(&self, entry: &FileEntry, parser: &ParserMemory) -> Option { + fn load_file(&self, entry: &Arc, parser: &ParserMemory) -> Option { if !entry.filename().to_string_lossy().ends_with(".txt") { return None; } @@ -120,7 +120,7 @@ impl FileHandler for Hoi4Events { PdxFile::read_optional_bom(entry, parser) } - fn handle_file(&mut self, _entry: &FileEntry, mut block: Block) { + fn handle_file(&mut self, _entry: &Arc, mut block: Block) { for item in block.drain() { if let BlockItem::Field(Field(key, _, bv)) = item { if key.is("add_namespace") { diff --git a/src/hoi4/data/gfx.rs b/src/hoi4/data/gfx.rs index e1fbe018d..4f47512d3 100644 --- a/src/hoi4/data/gfx.rs +++ b/src/hoi4/data/gfx.rs @@ -1,10 +1,11 @@ //! Process .gfx files, which contain sprite and mesh definitions. use std::path::PathBuf; +use std::sync::Arc; use crate::block::Block; use crate::everything::Everything; -use crate::fileset::{FileEntry, FileHandler}; +use crate::files::{FileEntry, FileHandler}; use crate::helpers::{dup_error, exact_dup_advice, TigerHashMap}; use crate::item::Item; use crate::parse::ParserMemory; @@ -79,7 +80,7 @@ impl FileHandler for Gfx { PathBuf::from("") } - fn load_file(&self, entry: &FileEntry, parser: &ParserMemory) -> Option { + fn load_file(&self, entry: &Arc, parser: &ParserMemory) -> Option { // Don't descend into the dlc directories directly. // Wait for them to be processed as Dlc FileKind. if entry.path().starts_with("dlc") || entry.path().starts_with("integrated_dlc") { @@ -95,7 +96,7 @@ impl FileHandler for Gfx { } } - fn handle_file(&mut self, _entry: &FileEntry, mut block: Block) { + fn handle_file(&mut self, _entry: &Arc, mut block: Block) { for (key, mut block) in block.drain_definitions_warn() { if key.lowercase_is("spritetypes") { for (key, block) in block.drain_definitions_warn() { diff --git a/src/hoi4/data/music.rs b/src/hoi4/data/music.rs index fa1c1478d..c1835b2fa 100644 --- a/src/hoi4/data/music.rs +++ b/src/hoi4/data/music.rs @@ -1,9 +1,10 @@ use std::path::PathBuf; +use std::sync::Arc; use crate::block::{Block, Field}; use crate::context::ScopeContext; use crate::everything::Everything; -use crate::fileset::{FileEntry, FileHandler}; +use crate::files::{FileEntry, FileHandler}; use crate::helpers::{dup_error, TigerHashMap}; use crate::item::Item; use crate::parse::ParserMemory; @@ -73,7 +74,7 @@ impl FileHandler for Hoi4Musics { PathBuf::from("music") } - fn load_file(&self, entry: &FileEntry, parser: &ParserMemory) -> Option { + fn load_file(&self, entry: &Arc, parser: &ParserMemory) -> Option { if !entry.filename().to_string_lossy().ends_with(".txt") { return None; } @@ -81,7 +82,7 @@ impl FileHandler for Hoi4Musics { PdxFile::read_no_bom(entry, parser) } - fn handle_file(&mut self, _entry: &FileEntry, mut block: Block) { + fn handle_file(&mut self, _entry: &Arc, mut block: Block) { let mut station = None; for item in block.drain() { diff --git a/src/hoi4/data/provinces.rs b/src/hoi4/data/provinces.rs index ab4faff7b..52ca2e07c 100644 --- a/src/hoi4/data/provinces.rs +++ b/src/hoi4/data/provinces.rs @@ -5,6 +5,7 @@ use std::io::{Read, Seek, SeekFrom}; use std::num::NonZero; use std::path::PathBuf; use std::str::FromStr; +use std::sync::Arc; use ahash::HashMapExt; use bitvec::bitbox; @@ -13,13 +14,14 @@ use image::{DynamicImage, Rgb, RgbImage}; use strum_macros::EnumString; use crate::everything::Everything; -use crate::fileset::{FileEntry, FileHandler}; +use crate::files::{FileEntry, FileHandler}; use crate::helpers::{TigerHashMap, TigerHashSet}; use crate::item::Item; use crate::parse::csv::{parse_csv, read_csv}; use crate::parse::ParserMemory; use crate::report::{err, report, untidy, warn, ErrorKey, Severity}; use crate::token::Token; +use crate::Loc; use super::terrain::Terrain; @@ -83,7 +85,7 @@ pub struct Hoi4Provinces { provinces: TigerHashSet, /// Kept and used for error reporting. - definition_csv: Option, + definition_csv: Option, adjacencies: Vec, } @@ -146,7 +148,7 @@ impl Hoi4Provinces { } fn validate_provinces(&self) { - let Some(definition_csv) = self.definition_csv.as_ref() else { + let Some(definition_csv) = self.definition_csv else { // Shouldn't happen, it should come from vanilla if not from the mod eprintln!("map/definition.csv is missing?!?"); return; @@ -272,7 +274,7 @@ impl FileHandler for Hoi4Provinces { PathBuf::from("map") } - fn load_file(&self, entry: &FileEntry, _parser: &ParserMemory) -> Option { + fn load_file(&self, entry: &Arc, _parser: &ParserMemory) -> Option { if entry.path().components().count() == 2 { match &*entry.filename().to_string_lossy() { "adjacencies.csv" => { @@ -360,7 +362,7 @@ impl FileHandler for Hoi4Provinces { None } - fn handle_file(&mut self, entry: &FileEntry, content: FileContent) { + fn handle_file(&mut self, entry: &Arc, content: FileContent) { match content { FileContent::Adjacencies(content) => { let mut seen_terminator = false; @@ -381,7 +383,7 @@ impl FileHandler for Hoi4Provinces { } } FileContent::Definitions(content) => { - self.definition_csv = Some(entry.clone()); + self.definition_csv = Some(Loc::from(entry)); // Assume first line has province ID 0. for csv in parse_csv(entry, 1, &content) { self.parse_definition(&csv); diff --git a/src/imperator/data/decisions.rs b/src/imperator/data/decisions.rs index 349ddf8f0..ca72d8aa1 100644 --- a/src/imperator/data/decisions.rs +++ b/src/imperator/data/decisions.rs @@ -1,9 +1,10 @@ use std::path::PathBuf; +use std::sync::Arc; use crate::block::Block; use crate::context::ScopeContext; use crate::everything::Everything; -use crate::fileset::{FileEntry, FileHandler}; +use crate::files::{FileEntry, FileHandler}; use crate::helpers::TigerHashMap; use crate::item::Item; use crate::parse::ParserMemory; @@ -54,7 +55,7 @@ impl FileHandler for Decisions { PathBuf::from("decisions/") } - fn load_file(&self, entry: &FileEntry, parser: &ParserMemory) -> Option { + fn load_file(&self, entry: &Arc, parser: &ParserMemory) -> Option { if !entry.filename().to_string_lossy().ends_with(".txt") { return None; } @@ -62,7 +63,7 @@ impl FileHandler for Decisions { PdxFile::read(entry, parser) } - fn handle_file(&mut self, _entry: &FileEntry, mut block: Block) { + fn handle_file(&mut self, _entry: &Arc, mut block: Block) { for (key, block) in block.drain_definitions_warn() { self.load_item(&key, &block); } diff --git a/src/imperator/data/provinces.rs b/src/imperator/data/provinces.rs index 35c0e0b23..c33471630 100644 --- a/src/imperator/data/provinces.rs +++ b/src/imperator/data/provinces.rs @@ -1,12 +1,13 @@ use std::path::PathBuf; use std::str::FromStr; +use std::sync::Arc; use image::{DynamicImage, Rgb}; use itertools::Itertools; use crate::block::Block; use crate::everything::Everything; -use crate::fileset::{FileEntry, FileHandler}; +use crate::files::{FileEntry, FileHandler}; use crate::helpers::{TigerHashMap, TigerHashSet}; use crate::item::Item; use crate::parse::csv::{parse_csv, read_csv}; @@ -28,7 +29,7 @@ pub struct ImperatorProvinces { provinces: TigerHashMap, /// Kept and used for error reporting. - definition_csv: Option, + definition_csv: Option, adjacencies: Vec, @@ -176,7 +177,7 @@ impl FileHandler for ImperatorProvinces { PathBuf::from("map_data") } - fn load_file(&self, entry: &FileEntry, parser: &ParserMemory) -> Option { + fn load_file(&self, entry: &Arc, parser: &ParserMemory) -> Option { if entry.path().components().count() == 2 { match &*entry.filename().to_string_lossy() { "adjacencies.csv" => { @@ -235,7 +236,7 @@ impl FileHandler for ImperatorProvinces { None } - fn handle_file(&mut self, entry: &FileEntry, content: FileContent) { + fn handle_file(&mut self, entry: &Arc, content: FileContent) { match content { FileContent::Adjacencies(content) => { let mut seen_terminator = false; @@ -256,7 +257,7 @@ impl FileHandler for ImperatorProvinces { } } FileContent::Definitions(content) => { - self.definition_csv = Some(entry.clone()); + self.definition_csv = Some(Loc::from(entry)); for csv in parse_csv(entry, 0, &content) { self.parse_definition(&csv); } @@ -278,7 +279,7 @@ impl FileHandler for ImperatorProvinces { eprintln!("map_data/definition.csv is missing?!?"); return; } - let definition_csv = self.definition_csv.as_ref().unwrap(); + let definition_csv = self.definition_csv.unwrap(); let mut seen_colors = TigerHashMap::default(); #[allow(clippy::cast_possible_truncation)] diff --git a/src/launcher_settings.rs b/src/launcher_settings.rs index c3f3298f4..7655d21bd 100644 --- a/src/launcher_settings.rs +++ b/src/launcher_settings.rs @@ -4,7 +4,7 @@ use std::path::Path; use anyhow::{bail, Context, Result}; -use crate::fileset::{FileEntry, FileKind}; +use crate::files::{FileEntry, FileKind}; use crate::game::Game; use crate::parse::json::parse_json_file; @@ -15,8 +15,11 @@ pub fn get_version_from_launcher(game_dir: &Path) -> Result { } else { game_dir.join("launcher/launcher-settings.json") }; - let launcher_entry = - FileEntry::new(launcher_pathname.clone(), FileKind::Vanilla, launcher_pathname.clone()); + let launcher_entry = FileEntry::new_untracked( + launcher_pathname.clone(), + FileKind::Vanilla, + launcher_pathname.clone(), + ); let block = parse_json_file(&launcher_entry).with_context(|| { format!("Could not parse launcher file {}", launcher_pathname.display()) })?; diff --git a/src/lib.rs b/src/lib.rs index 9d6269453..23651b016 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -19,7 +19,7 @@ compile_error!( pub use crate::config_load::validate_config_file; pub use crate::everything::Everything; -pub use crate::fileset::FileKind; +pub use crate::files::{FileKind, Fileset}; pub use crate::game::Game; pub use crate::helpers::{TigerHashMap, TigerHashSet}; pub use crate::item::Item; @@ -61,7 +61,7 @@ mod effect; #[cfg(feature = "jomini")] mod effect_validation; mod everything; -mod fileset; +pub mod files; mod game; mod gui; mod helpers; diff --git a/src/mod_metadata.rs b/src/mod_metadata.rs index 0b22c2f22..4618701dd 100644 --- a/src/mod_metadata.rs +++ b/src/mod_metadata.rs @@ -5,7 +5,8 @@ use std::path::{Path, PathBuf}; use anyhow::{Context, Result}; use crate::block::Block; -use crate::fileset::{FileEntry, FileKind}; +use crate::files::FileDb; +use crate::files::FileKind; use crate::parse::json::parse_json_file; use crate::token::Token; use crate::util::fix_slashes_for_target_platform; @@ -21,13 +22,13 @@ pub struct ModMetadata { impl ModMetadata { /// Read and parse the metadata file for the given mod dir - pub fn read(mod_dir: &Path) -> Result { + pub fn read(filedb: &mut FileDb, mod_dir: PathBuf) -> Result { let in_mod_path = PathBuf::from(".metadata/metadata.json"); let pathname = fix_slashes_for_target_platform(mod_dir.join(&in_mod_path)); - let entry = FileEntry::new(in_mod_path, FileKind::Mod, pathname.clone()); - let block = parse_json_file(&entry) + let entry = filedb.get_or_create_entry(in_mod_path, FileKind::Mod, pathname.clone()); + let block = parse_json_file(entry) .with_context(|| format!("could not read metadata file {}", pathname.display()))?; - Ok(Self { modpath: mod_dir.to_path_buf(), block }) + Ok(Self { modpath: mod_dir, block }) } /// Return the full path to the mod root. diff --git a/src/modfile.rs b/src/modfile.rs index 3e5daf46f..db162994d 100644 --- a/src/modfile.rs +++ b/src/modfile.rs @@ -7,7 +7,7 @@ use std::string::ToString; use anyhow::{Context, Result}; use crate::block::Block; -use crate::fileset::{FileEntry, FileKind}; +use crate::files::FileEntry; use crate::game::Game; use crate::parse::ParserMemory; use crate::pdxfile::PdxFile; @@ -72,10 +72,9 @@ fn validate_modfile(block: &Block) -> ModFile { impl ModFile { /// Take the path to a `.mod` file, validate it, and return its parsed structure. - pub fn read(pathname: &Path) -> Result { - let entry = FileEntry::new(pathname.to_path_buf(), FileKind::Mod, pathname.to_path_buf()); - let block = PdxFile::read_optional_bom(&entry, &ParserMemory::default()) - .with_context(|| format!("Could not read .mod file {}", pathname.display()))?; + pub fn read(entry: &FileEntry) -> Result { + let block = PdxFile::read_optional_bom(entry, &ParserMemory::default()) + .with_context(|| format!("Could not read .mod file {}", entry.path().display()))?; Ok(validate_modfile(&block)) } diff --git a/src/parse/csv.rs b/src/parse/csv.rs index 678a1fe48..1082dd374 100644 --- a/src/parse/csv.rs +++ b/src/parse/csv.rs @@ -6,7 +6,7 @@ use std::str::Chars; use anyhow::{bail, Result}; use encoding_rs::WINDOWS_1252; -use crate::fileset::FileEntry; +use crate::files::FileEntry; use crate::report::ErrorLoc; use crate::token::{Loc, Token}; diff --git a/src/parse/json.rs b/src/parse/json.rs index 22376063f..bf84419d1 100644 --- a/src/parse/json.rs +++ b/src/parse/json.rs @@ -3,12 +3,11 @@ //! `Block` is used, instead of a JSON-specific representation, for compatibility with the rest of the code. //! Unfortunately can't use serde-json because we need the locations for error reporting. -use std::fs::read_to_string; use std::mem::{swap, take}; use crate::block::Eq::Single; use crate::block::{Block, Comparator, BV}; -use crate::fileset::FileEntry; +use crate::files::FileEntry; use crate::report::{err, warn, ErrorKey}; use crate::token::{Loc, Token}; @@ -315,29 +314,9 @@ fn parse(blockloc: Loc, content: &str) -> Block { } #[allow(clippy::module_name_repetitions)] -pub fn parse_json(entry: &FileEntry, content: &str) -> Block { +pub fn parse_json_file(entry: &FileEntry) -> Option { let mut loc = Loc::from(entry); loc.line = 1; loc.column = 1; - parse(loc, content) -} - -#[allow(clippy::module_name_repetitions)] -pub fn parse_json_file(entry: &FileEntry) -> Option { - let contents = match read_to_string(entry.fullpath()) { - Ok(contents) => contents, - Err(e) => { - err(ErrorKey::ReadError) - .msg("could not read file") - .info(format!("{e:#}")) - .loc(entry) - .push(); - return None; - } - }; - if let Some(bomless) = contents.strip_prefix('\u{feff}') { - Some(parse_json(entry, bomless)) - } else { - Some(parse_json(entry, &contents)) - } + Some(parse(loc, entry.contents()?.nobom())) } diff --git a/src/parse/localization.rs b/src/parse/localization.rs index 069280bde..bdcf0ee4d 100644 --- a/src/parse/localization.rs +++ b/src/parse/localization.rs @@ -4,13 +4,13 @@ use std::str::Chars; use crate::data::localization::{Language, LocaEntry, LocaValue, MacroValue}; use crate::datatype::{Code, CodeArg, CodeChain}; -use crate::fileset::FileEntry; +use crate::files::FileEntry; use crate::game::Game; use crate::parse::cob::Cob; use crate::parse::ignore::{parse_comment, IgnoreFilter, IgnoreSize}; use crate::report::register_ignore_filter; use crate::report::{untidy, warn, ErrorKey}; -use crate::token::{leak, Loc, Token}; +use crate::token::{Loc, Token}; fn is_key_char(c: char) -> bool { c.is_alphanumeric() || c == '-' || c == '_' || c == '.' || c == '\'' @@ -37,7 +37,8 @@ struct LocaParser { } impl LocaParser { - fn new(entry: &FileEntry, content: &'static str, lang: Language) -> Self { + fn new(entry: &FileEntry, lang: Language) -> Option { + let content = entry.contents()?.full(); let mut chars = content.chars().peekable(); let mut offset = 0; @@ -67,7 +68,7 @@ impl LocaParser { // From here on we are reporting on file content loc.line = 1; - LocaParser { + Some(LocaParser { loc, offset, content, @@ -78,7 +79,7 @@ impl LocaParser { loca_end: 0, pending_line_ignores: Vec::new(), active_range_ignores: Vec::new(), - } + }) } fn next_char(&mut self) { @@ -920,8 +921,7 @@ impl Iterator for LocaReader { } } -pub fn parse_loca(entry: &FileEntry, content: String, lang: Language) -> LocaReader { - let content = leak(content); - let parser = LocaParser::new(entry, content, lang); - LocaReader { parser } +pub fn parse_loca(entry: &FileEntry, lang: Language) -> Option { + let parser = LocaParser::new(entry, lang)?; + Some(LocaReader { parser }) } diff --git a/src/parse/pdxfile.rs b/src/parse/pdxfile.rs index 734ebe30d..c2e2fd360 100644 --- a/src/parse/pdxfile.rs +++ b/src/parse/pdxfile.rs @@ -9,15 +9,15 @@ use std::sync::LazyLock; use lalrpop_util::{lalrpop_mod, ParseError}; use crate::block::{Block, Comparator, Eq}; -use crate::fileset::{FileEntry, FileKind}; +use crate::files::FileEntry; use crate::game::Game; use crate::parse::cob::Cob; use crate::parse::pdxfile::lexer::{LexError, Lexeme, Lexer}; use crate::parse::pdxfile::memory::CombinedMemory; pub use crate::parse::pdxfile::memory::PdxfileMemory; use crate::parse::ParserMemory; -use crate::report::{err, store_source_file, ErrorKey}; -use crate::token::{leak, Loc, Token}; +use crate::report::{err, ErrorKey}; +use crate::token::{Loc, Token}; mod lexer; pub mod memory; @@ -47,7 +47,8 @@ pub fn parse_pdx_macro(inputs: &[Token], global: &PdxfileMemory, local: &Pdxfile } /// Parse a whole file into a `Block`. -fn parse_pdx(entry: &FileEntry, content: &'static str, memory: &ParserMemory) -> Block { +fn parse_pdx(entry: &FileEntry, memory: &ParserMemory) -> Option { + let content = entry.contents()?.nobom(); let file_loc = Loc::from(entry); let mut loc = file_loc; loc.line = 1; @@ -57,42 +58,30 @@ fn parse_pdx(entry: &FileEntry, content: &'static str, memory: &ParserMemory) -> match FILE_PARSER.parse(&inputs, &mut combined, Lexer::new(&inputs)) { Ok(mut block) => { block.loc = file_loc; - block + Some(block) } Err(e) => { eprintln!("Internal error: parsing file {} failed.\n{e}", entry.path().display()); - Block::new(inputs[0].loc) + Some(Block::new(inputs[0].loc)) } } } /// Parse the content associated with the [`FileEntry`]. -pub fn parse_pdx_file( - entry: &FileEntry, - content: String, - offset: usize, - parser: &ParserMemory, -) -> Block { - let content = leak(content); - store_source_file(entry.fullpath().to_path_buf(), &content[offset..]); - parse_pdx(entry, &content[offset..], parser) +pub fn parse_pdx_file(entry: &FileEntry, parser: &ParserMemory) -> Option { + parse_pdx(entry, parser) } /// Parse the content associated with the [`FileEntry`], and update the global parser memory. #[cfg(feature = "ck3")] -pub fn parse_reader_export( - entry: &FileEntry, - content: String, - offset: usize, - global: &mut PdxfileMemory, -) { - let content = leak(content); - store_source_file(entry.fullpath().to_path_buf(), &content[offset..]); - let content = &content[offset..]; +pub fn parse_reader_export(entry: &FileEntry, global: &mut PdxfileMemory) { + let Some(contents) = entry.contents() else { + return; + }; let mut loc = Loc::from(entry); loc.line = 1; loc.column = 1; - let inputs = [Token::from_static_str(content, loc)]; + let inputs = [Token::from_static_str(contents.nobom(), loc)]; let mut combined = CombinedMemory::new(global); match FILE_PARSER.parse(&inputs, &mut combined, Lexer::new(&inputs)) { Ok(_) => { @@ -107,8 +96,8 @@ pub fn parse_reader_export( /// Parse a string into a [`Block`]. This function is meant for use by the validator itself, to /// allow it to load game description data from internal strings that are in pdx script format. pub fn parse_pdx_internal(input: &'static str, desc: &str) -> Block { - let entry = FileEntry::new(PathBuf::from(desc), FileKind::Internal, PathBuf::from(desc)); - parse_pdx(&entry, input, &ParserMemory::default()) + let entry = FileEntry::new_internal(PathBuf::from(desc), input); + parse_pdx(&entry, &ParserMemory::default()).unwrap() } #[derive(Debug, PartialEq, Eq, Clone, Copy)] diff --git a/src/pathtable.rs b/src/pathtable.rs index b8edc2779..bf2235926 100644 --- a/src/pathtable.rs +++ b/src/pathtable.rs @@ -2,7 +2,7 @@ //! //! Using this will make the often-cloned Loc faster to copy, since it will just contain an index into the global table. //! It also makes it faster to compare pathnames, because the table will be created in lexical order by the caller -//! ([`Fileset`](crate::fileset::Fileset)), with the exception of some stray files (such as the config file) +//! ([`Fileset`](crate::files::Fileset)), with the exception of some stray files (such as the config file) //! where the order doesn't matter. use std::hash::Hash; use std::path::{Path, PathBuf}; @@ -10,6 +10,9 @@ use std::sync::{LazyLock, RwLock}; #[derive(Debug, Default, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] pub struct PathTableIndex(u32); +impl PathTableIndex { + pub const UNTRACKED: PathTableIndex = PathTableIndex(u32::MAX); +} static PATHTABLE: LazyLock> = LazyLock::new(|| RwLock::new(PathTable::default())); @@ -47,12 +50,18 @@ impl PathTable { /// Return the local path based on its index. /// This can panic if the index is not one provided by `PathTable::store`. pub fn lookup_path(idx: PathTableIndex) -> &'static Path { + if idx == PathTableIndex::UNTRACKED { + return Path::new(""); + } PATHTABLE.read().unwrap().lookup_paths_inner(idx).0 } /// Return the full path based on its index. /// This can panic if the index is not one provided by `PathTable::store`. pub fn lookup_fullpath(idx: PathTableIndex) -> &'static Path { + if idx == PathTableIndex::UNTRACKED { + return Path::new(""); + } PATHTABLE.read().unwrap().lookup_paths_inner(idx).1 } diff --git a/src/pdxfile.rs b/src/pdxfile.rs index ab68282ae..10458c68c 100644 --- a/src/pdxfile.rs +++ b/src/pdxfile.rs @@ -2,24 +2,15 @@ //! //! The main entry point is [`PdxFile`]. -#[cfg(feature = "ck3")] -use std::fs::read; -use std::fs::read_to_string; - -#[cfg(feature = "ck3")] -use encoding_rs::{UTF_8, WINDOWS_1252}; - use crate::block::Block; -use crate::fileset::FileEntry; +use crate::files::FileEntry; use crate::parse::pdxfile::parse_pdx_file; #[cfg(feature = "ck3")] use crate::parse::pdxfile::{parse_reader_export, PdxfileMemory}; use crate::parse::ParserMemory; -use crate::report::{err, warn, ErrorKey}; - -const BOM_UTF8_BYTES: &[u8] = b"\xef\xbb\xbf"; -const BOM_UTF8_LEN: usize = BOM_UTF8_BYTES.len(); -const BOM_CHAR: char = '\u{feff}'; +#[cfg(feature = "hoi4")] +use crate::report::err; +use crate::report::{warn, ErrorKey}; #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] pub enum PdxEncoding { @@ -35,85 +26,35 @@ pub enum PdxEncoding { pub struct PdxFile {} impl PdxFile { - /// Internal function to read a file in UTF-8 encoding. - fn read_utf8(entry: &FileEntry) -> Option { - match read_to_string(entry.fullpath()) { - Ok(contents) => Some(contents), - Err(e) => { - let msg = "could not read file"; - let info = &format!("{e:#}"); - err(ErrorKey::ReadError).msg(msg).info(info).loc(entry).push(); - None - } - } - } - /// Parse a UTF-8 file that should start with a BOM (Byte Order Marker). pub fn read(entry: &FileEntry, parser: &ParserMemory) -> Option { - let contents = Self::read_utf8(entry)?; - if contents.starts_with(BOM_CHAR) { - Some(parse_pdx_file(entry, contents, BOM_UTF8_LEN, parser)) - } else { + if !entry.contents()?.has_bom() { let msg = "Expected UTF-8 BOM encoding"; warn(ErrorKey::Encoding).msg(msg).abbreviated(entry).push(); - Some(parse_pdx_file(entry, contents, 0, parser)) } + parse_pdx_file(entry, parser) } /// Parse a UTF-8 file that may must start with a BOM (Byte Order Marker). #[cfg(feature = "hoi4")] pub fn read_no_bom(entry: &FileEntry, parser: &ParserMemory) -> Option { - let contents = Self::read_utf8(entry)?; - if contents.starts_with(BOM_CHAR) { + if entry.contents()?.has_bom() { let msg = "Expected UTF-8 encoding without BOM"; err(ErrorKey::Encoding).msg(msg).abbreviated(entry).push(); - Some(parse_pdx_file(entry, contents, BOM_UTF8_LEN, parser)) - } else { - Some(parse_pdx_file(entry, contents, 0, parser)) } + parse_pdx_file(entry, parser) } /// Parse a UTF-8 file that may optionally start with a BOM (Byte Order Marker). pub fn read_optional_bom(entry: &FileEntry, parser: &ParserMemory) -> Option { - let contents = Self::read_utf8(entry)?; - if contents.starts_with(BOM_CHAR) { - Some(parse_pdx_file(entry, contents, BOM_UTF8_LEN, parser)) - } else { - Some(parse_pdx_file(entry, contents, 0, parser)) - } + parse_pdx_file(entry, parser) } /// Parse a file that may be in UTF-8 with BOM encoding, or Windows-1252 encoding. #[cfg(feature = "ck3")] pub fn read_detect_encoding(entry: &FileEntry, parser: &ParserMemory) -> Option { - let bytes = match read(entry.fullpath()) { - Ok(bytes) => bytes, - Err(e) => { - let msg = "could not read file"; - let info = format!("{e:#}"); - err(ErrorKey::ReadError).msg(msg).info(info).abbreviated(entry).push(); - return None; - } - }; - if bytes.starts_with(BOM_UTF8_BYTES) { - let (contents, errors) = UTF_8.decode_without_bom_handling(&bytes[BOM_UTF8_LEN..]); - if errors { - let msg = "could not decode UTF-8 file"; - err(ErrorKey::Encoding).msg(msg).abbreviated(entry).push(); - None - } else { - Some(parse_pdx_file(entry, contents.into_owned(), 0, parser)) - } - } else { - let (contents, errors) = WINDOWS_1252.decode_without_bom_handling(&bytes); - if errors { - let msg = "could not decode WINDOWS-1252 file"; - err(ErrorKey::Encoding).msg(msg).abbreviated(entry).push(); - None - } else { - Some(parse_pdx_file(entry, contents.into_owned(), 0, parser)) - } - } + entry.contents_detect_encoding()?; + parse_pdx_file(entry, parser) } pub fn read_encoded( @@ -134,14 +75,12 @@ impl PdxFile { #[cfg(feature = "ck3")] pub fn reader_export(entry: &FileEntry, memory: &mut PdxfileMemory) { - if let Some(contents) = Self::read_utf8(entry) { - if contents.starts_with(BOM_CHAR) { - parse_reader_export(entry, contents, BOM_UTF8_LEN, memory); - } else { + if let Some(contents) = entry.contents() { + if !contents.has_bom() { let msg = "Expected UTF-8 BOM encoding"; warn(ErrorKey::Encoding).msg(msg).abbreviated(entry).push(); - parse_reader_export(entry, contents, 0, memory); } + parse_reader_export(entry, memory); } } } diff --git a/src/report/error_loc.rs b/src/report/error_loc.rs index 63d2729b6..fab09825f 100644 --- a/src/report/error_loc.rs +++ b/src/report/error_loc.rs @@ -1,5 +1,7 @@ +use std::sync::Arc; + use crate::block::{Block, BlockItem, Field, BV}; -use crate::fileset::FileEntry; +use crate::files::FileEntry; use crate::token::{Loc, Token}; use crate::trigger::Part; use crate::validator::ValueValidator; @@ -134,6 +136,18 @@ impl ErrorLoc for &FileEntry { } } +impl ErrorLoc for Arc { + fn into_loc(self) -> Loc { + Loc::from(self) + } +} + +impl ErrorLoc for &Arc { + fn into_loc(self) -> Loc { + Loc::from(&**self) + } +} + impl ErrorLoc for Loc { fn into_loc(self) -> Loc { self diff --git a/src/report/errors.rs b/src/report/errors.rs index 3a34135de..0b2a0fecd 100644 --- a/src/report/errors.rs +++ b/src/report/errors.rs @@ -1,18 +1,15 @@ //! Collect error reports and then write them out. use std::borrow::Cow; -use std::cell::RefCell; use std::cmp::{min_by, Ordering}; -use std::fs::read; use std::io::Write; use std::iter::{empty, once}; use std::mem::take; use std::ops::{Bound, RangeBounds}; use std::path::PathBuf; -use std::sync::{LazyLock, Mutex, MutexGuard}; - -use encoding_rs::{UTF_8, WINDOWS_1252}; +use std::sync::{Arc, LazyLock, Mutex, MutexGuard}; +use crate::files::FileEntry; use crate::helpers::{TigerHashMap, TigerHashSet}; use crate::macros::MACRO_MAP; use crate::parse::ignore::IgnoreFilter; @@ -26,7 +23,7 @@ use crate::report::{ OutputStyle, PointedMessage, }; use crate::set; -use crate::token::{leak, Loc}; +use crate::token::Loc; /// Error types that should be logged once when consolidating reports static LOG_ONCE: LazyLock> = LazyLock::new(|| { @@ -220,8 +217,8 @@ impl Errors<'_> { result } - pub fn store_source_file(&mut self, fullpath: PathBuf, source: &'static str) { - self.cache.filecache.borrow_mut().insert(fullpath, source); + pub fn store_source_file(&mut self, entry: Arc) { + self.cache.filecache.insert(entry.fullpath().to_owned(), entry); } /// Get a mutable lock on the global ERRORS struct. @@ -248,44 +245,22 @@ impl Errors<'_> { pub(crate) struct Cache { /// Files that have been read in to get the lines where errors occurred. /// Cached here to avoid duplicate I/O and UTF-8 parsing. - filecache: RefCell>, - - /// Files that have been linesplit, cached to avoid doing that work again - linecache: RefCell>>, + filecache: TigerHashMap>, } impl Cache { /// Fetch the contents of a single line from a script file. - pub(crate) fn get_line(&self, loc: Loc) -> Option<&'static str> { - let mut filecache = self.filecache.borrow_mut(); - let mut linecache = self.linecache.borrow_mut(); - + pub(crate) fn get_line(&self, loc: Loc) -> Option<&str> { if loc.line == 0 { return None; } let fullpath = loc.fullpath(); - if let Some(lines) = linecache.get(fullpath) { - return lines.get(loc.line as usize - 1).copied(); - } - if let Some(contents) = filecache.get(fullpath) { - let lines: Vec<_> = contents.lines().collect(); - let line = lines.get(loc.line as usize - 1).copied(); - linecache.insert(fullpath.to_path_buf(), lines); - return line; - } - let bytes = read(fullpath).ok()?; - // Try decoding it as UTF-8. If that succeeds without errors, use it, otherwise fall back - // to WINDOWS_1252. The decode method will do BOM stripping. - let contents = match UTF_8.decode(&bytes) { - (contents, _, false) => contents, - (_, _, true) => WINDOWS_1252.decode(&bytes).0, - }; - let contents = leak(contents.into_owned()); - filecache.insert(fullpath.to_path_buf(), contents); - - let lines: Vec<_> = contents.lines().collect(); - let line = lines.get(loc.line as usize - 1).copied(); - linecache.insert(fullpath.to_path_buf(), lines); + let entry = self.filecache.get(fullpath); + debug_assert!(entry.is_some()); + let contents = entry?.contents(); + debug_assert!(contents.is_some()); + let line = contents?.line(loc.line as usize); + debug_assert!(line.is_some()); line } } @@ -365,8 +340,8 @@ pub fn take_reports() -> TigerHashMap) { + Errors::get_mut().store_source_file(entry); } pub fn register_ignore_filter(pathname: PathBuf, lines: R, filter: IgnoreFilter) diff --git a/src/report/filter.rs b/src/report/filter.rs index 03a8ed83a..ff014e46c 100644 --- a/src/report/filter.rs +++ b/src/report/filter.rs @@ -3,7 +3,7 @@ use std::path::Path; use crate::block::Comparator; -use crate::fileset::FileKind; +use crate::files::FileKind; use crate::report::{ err, Confidence, ErrorKey, ErrorLoc, LogReportMetadata, LogReportPointers, Severity, }; diff --git a/src/report/writer.rs b/src/report/writer.rs index 94af6a13d..ce3c01a66 100644 --- a/src/report/writer.rs +++ b/src/report/writer.rs @@ -3,7 +3,7 @@ use std::io::Write; use ansiterm::{ANSIString, ANSIStrings}; use unicode_width::UnicodeWidthChar; -use crate::fileset::FileKind; +use crate::files::FileKind; use crate::game::Game; use crate::report::errors::Errors; use crate::report::output_style::Styled; diff --git a/src/rivers.rs b/src/rivers.rs index 92a20aec4..f598a7634 100644 --- a/src/rivers.rs +++ b/src/rivers.rs @@ -5,6 +5,7 @@ use std::fs; use std::ops::{RangeInclusive, RangeToInclusive}; use std::path::PathBuf; +use std::sync::Arc; #[cfg(feature = "jomini")] use png::{ColorType, Decoder}; @@ -12,11 +13,11 @@ use png::{ColorType, Decoder}; use tinybmp::{Bpp, CompressionMethod, RawBmp}; use crate::everything::Everything; -use crate::fileset::{FileEntry, FileHandler}; +use crate::files::{FileEntry, FileHandler}; use crate::helpers::{TigerHashMap, TigerHashSet}; use crate::parse::ParserMemory; use crate::report::{err, warn, will_maybe_log, ErrorKey}; -use crate::Game; +use crate::{Game, Loc}; #[inline] fn river_image_path() -> &'static str { @@ -55,7 +56,7 @@ impl RiverPixels { #[derive(Clone, Debug, Default)] pub struct Rivers { /// for error reporting - entry: Option, + loc: Option, width: u32, height: u32, pixels: Vec, @@ -201,7 +202,7 @@ impl Rivers { fn validate_segments( &self, - entry: &FileEntry, + loc: Loc, river_segments: TigerHashMap<(u32, u32), (u32, u32)>, mut specials: TigerHashMap<(u32, u32), bool>, ) { @@ -221,16 +222,16 @@ impl Rivers { "({}, {}) river pixel connects two special pixels", start.0, start.1 ); - warn(ErrorKey::Rivers).msg(msg).loc(entry).push(); + warn(ErrorKey::Rivers).msg(msg).loc(loc).push(); } else if special_neighbors.is_empty() { let msg = format!("({}, {}) orphan river pixel", start.0, start.1); - warn(ErrorKey::Rivers).msg(msg).loc(entry).push(); + warn(ErrorKey::Rivers).msg(msg).loc(loc).push(); } else { let s = special_neighbors[0]; if specials[&s] { let msg = format!("({}, {}) pixel terminates multiple river segments", s.0, s.1); - warn(ErrorKey::Rivers).msg(msg).loc(entry).push(); + warn(ErrorKey::Rivers).msg(msg).loc(loc).push(); } else { specials.insert(s, true); } @@ -243,19 +244,19 @@ impl Rivers { "({}, {}) - ({}, {}) orphan river segment", start.0, start.1, end.0, end.1 ); - warn(ErrorKey::Rivers).msg(msg).loc(entry).push(); + warn(ErrorKey::Rivers).msg(msg).loc(loc).push(); } else if special_neighbors.len() > 1 { let msg = format!( "({}, {}) - ({}, {}) river segment has two terminators", start.0, start.1, end.0, end.1 ); - warn(ErrorKey::Rivers).msg(msg).loc(entry).push(); + warn(ErrorKey::Rivers).msg(msg).loc(loc).push(); } else { let s = special_neighbors[0]; if specials[&s] { let msg = format!("({}, {}) pixel terminates multiple river segments", s.0, s.1); - warn(ErrorKey::Rivers).msg(msg).loc(entry).push(); + warn(ErrorKey::Rivers).msg(msg).loc(loc).push(); } else { specials.insert(s, true); } @@ -267,14 +268,14 @@ impl Rivers { pub fn validate(&self, _data: &Everything) { // TODO: check image width and height against world defines - let Some(entry) = self.entry.as_ref() else { + let Some(loc) = self.loc else { // Shouldn't happen, it should come from vanilla if not from the mod eprintln!("{} is missing?!?", river_image_path()); return; }; // Early exit before expensive loop, if errors won't be logged anyway - if !will_maybe_log(entry, ErrorKey::Rivers) { + if !will_maybe_log(loc, ErrorKey::Rivers) { return; } @@ -304,7 +305,7 @@ impl Rivers { } else { let msg = format!("({x}, {y}) river source (green) not at source of a river"); - warn(ErrorKey::Rivers).msg(msg).loc(entry).push(); + warn(ErrorKey::Rivers).msg(msg).loc(loc).push(); bad_problem = true; } } @@ -316,7 +317,7 @@ impl Rivers { let msg = format!( "({x}, {y}) river tributary (red) not joining another river", ); - warn(ErrorKey::Rivers).msg(msg).loc(entry).push(); + warn(ErrorKey::Rivers).msg(msg).loc(loc).push(); bad_problem = true; } } @@ -328,7 +329,7 @@ impl Rivers { let msg = format!( "({x}, {y}) river split (yellow) not splitting off from a river", ); - warn(ErrorKey::Rivers).msg(msg).loc(entry).push(); + warn(ErrorKey::Rivers).msg(msg).loc(loc).push(); bad_problem = true; } } @@ -346,7 +347,7 @@ impl Rivers { // though. if third_end == (x, y) { let msg = format!("({x}, {y}) river forms a loop"); - warn(ErrorKey::Rivers).msg(msg).loc(entry).push(); + warn(ErrorKey::Rivers).msg(msg).loc(loc).push(); bad_problem = true; } else { river_segments.insert(other_end, third_end); @@ -371,7 +372,7 @@ impl Rivers { "({x}, {y}) river pixel has {} neighbors", river_neighbors.len() ); - warn(ErrorKey::Rivers).msg(msg).loc(entry).push(); + warn(ErrorKey::Rivers).msg(msg).loc(loc).push(); bad_problem = true; } } @@ -380,7 +381,7 @@ impl Rivers { } } if !bad_problem { - self.validate_segments(entry, river_segments, specials); + self.validate_segments(loc, river_segments, specials); } } } @@ -390,7 +391,7 @@ impl FileHandler> for Rivers { PathBuf::from(river_image_path()) } - fn load_file(&self, entry: &FileEntry, _parser: &ParserMemory) -> Option> { + fn load_file(&self, entry: &Arc, _parser: &ParserMemory) -> Option> { match fs::read(entry.fullpath()) { Err(e) => { err(ErrorKey::ReadError) @@ -403,8 +404,8 @@ impl FileHandler> for Rivers { } } - fn handle_file(&mut self, entry: &FileEntry, loaded: Vec) { - self.entry = Some(entry.clone()); + fn handle_file(&mut self, entry: &Arc, loaded: Vec) { + self.loc = Some(Loc::from(entry)); self.handle_image(&loaded, entry); } } diff --git a/src/token.rs b/src/token.rs index 55abbecde..6048bf355 100644 --- a/src/token.rs +++ b/src/token.rs @@ -7,13 +7,14 @@ use std::fmt::{Debug, Display, Error, Formatter}; use std::hash::Hash; use std::mem::ManuallyDrop; use std::ops::{Bound, Range, RangeBounds}; -use std::path::{Path, PathBuf}; +use std::path::Path; use std::slice::SliceIndex; +use std::sync::Arc; use bumpalo::Bump; use crate::date::Date; -use crate::fileset::{FileEntry, FileKind}; +use crate::files::{FileEntry, FileKind}; use crate::macros::MacroMapIndex; use crate::pathtable::{PathTable, PathTableIndex}; use crate::report::{err, untidy, ErrorKey}; @@ -31,12 +32,6 @@ pub struct Loc { } impl Loc { - #[must_use] - pub(crate) fn for_file(pathname: PathBuf, kind: FileKind, fullpath: PathBuf) -> Self { - let idx = PathTable::store(pathname, fullpath); - Loc { idx, kind, line: 0, column: 0, link_idx: None } - } - pub fn filename(self) -> Cow<'static, str> { PathTable::lookup_path(self.idx) .file_name() @@ -60,10 +55,12 @@ impl Loc { impl From<&FileEntry> for Loc { fn from(entry: &FileEntry) -> Self { - if let Some(idx) = entry.path_idx() { - Loc { idx, kind: entry.kind(), line: 0, column: 0, link_idx: None } - } else { - Self::for_file(entry.path().to_path_buf(), entry.kind(), entry.fullpath().to_path_buf()) + Loc { + idx: entry.get_or_init_path_idx(), + kind: entry.kind(), + line: 0, + column: 0, + link_idx: None, } } } @@ -80,6 +77,18 @@ impl From for Loc { } } +impl From> for Loc { + fn from(entry: Arc) -> Self { + (&*entry).into() + } +} + +impl From<&Arc> for Loc { + fn from(entry: &Arc) -> Self { + (&**entry).into() + } +} + impl Debug for Loc { /// Roll our own `Debug` implementation to handle the path field fn fmt(&self, f: &mut Formatter) -> Result<(), Error> { @@ -95,19 +104,6 @@ impl Debug for Loc { } } -/// Leak the string, including any excess capacity. -/// -/// It should only be used for large strings, rather than for small, individuals strings, -/// due to the memory overhead. Use [`bump`] instead, which uses a bump allocator to store -/// the strings. -pub(crate) fn leak(s: String) -> &'static str { - let s = ManuallyDrop::new(s); - unsafe { - let s_ptr: *const str = s.as_ref(); - &*s_ptr - } -} - thread_local!(static STR_BUMP: ManuallyDrop = ManuallyDrop::new(Bump::new())); /// Allocate the string on heap with a bump allocator. diff --git a/src/vic3/data/history.rs b/src/vic3/data/history.rs index d879aff70..db06e9595 100644 --- a/src/vic3/data/history.rs +++ b/src/vic3/data/history.rs @@ -1,10 +1,11 @@ use std::path::PathBuf; +use std::sync::Arc; use crate::block::Block; use crate::context::ScopeContext; use crate::effect::validate_effect; use crate::everything::Everything; -use crate::fileset::{FileEntry, FileHandler}; +use crate::files::{FileEntry, FileHandler}; use crate::helpers::TigerHashMap; use crate::parse::ParserMemory; use crate::pdxfile::PdxFile; @@ -69,7 +70,7 @@ impl FileHandler for History { PathBuf::from("common/history/") } - fn load_file(&self, entry: &FileEntry, parser: &ParserMemory) -> Option { + fn load_file(&self, entry: &Arc, parser: &ParserMemory) -> Option { if !entry.filename().to_string_lossy().ends_with(".txt") { return None; } @@ -77,7 +78,7 @@ impl FileHandler for History { PdxFile::read(entry, parser) } - fn handle_file(&mut self, _entry: &FileEntry, mut block: Block) { + fn handle_file(&mut self, _entry: &Arc, mut block: Block) { for (key, block) in block.drain_definitions_warn() { self.load_item(key, block); } diff --git a/src/vic3/data/provinces.rs b/src/vic3/data/provinces.rs index eed576f7d..96162c78e 100644 --- a/src/vic3/data/provinces.rs +++ b/src/vic3/data/provinces.rs @@ -1,15 +1,17 @@ use std::path::PathBuf; +use std::sync::Arc; use image::{DynamicImage, Rgb}; use itertools::Itertools; use crate::everything::Everything; -use crate::fileset::{FileEntry, FileHandler}; +use crate::files::{FileEntry, FileHandler}; use crate::helpers::TigerHashSet; use crate::item::Item; use crate::parse::ParserMemory; use crate::report::{err, report, ErrorKey, Severity}; use crate::token::Token; +use crate::Loc; #[derive(Clone, Debug, Default)] pub struct Vic3Provinces { @@ -17,7 +19,7 @@ pub struct Vic3Provinces { colors: TigerHashSet>, /// Kept and used for error reporting. - provinces_png: Option, + provinces_png: Option, } impl Vic3Provinces { @@ -59,7 +61,7 @@ impl FileHandler for Vic3Provinces { PathBuf::from("map_data/provinces.png") } - fn load_file(&self, entry: &FileEntry, _parser: &ParserMemory) -> Option { + fn load_file(&self, entry: &Arc, _parser: &ParserMemory) -> Option { if entry.path().components().count() == 2 { let img = match image::open(entry.fullpath()) { Ok(img) => img, @@ -84,8 +86,8 @@ impl FileHandler for Vic3Provinces { None } - fn handle_file(&mut self, entry: &FileEntry, img: DynamicImage) { - self.provinces_png = Some(entry.clone()); + fn handle_file(&mut self, entry: &Arc, img: DynamicImage) { + self.provinces_png = Some(Loc::from(entry)); if let DynamicImage::ImageRgb8(img) = img { for pixel in img.pixels().dedup() { self.colors.insert(*pixel); diff --git a/tests/files/mod1/descriptor.mod b/tests/files/mod1/descriptor.mod new file mode 100644 index 000000000..9f352643d --- /dev/null +++ b/tests/files/mod1/descriptor.mod @@ -0,0 +1,3 @@ +version="1.0" +name="Test mod" +supported_version="1.9.2.1" diff --git a/tests/files/mod2/descriptor.mod b/tests/files/mod2/descriptor.mod new file mode 100644 index 000000000..9f352643d --- /dev/null +++ b/tests/files/mod2/descriptor.mod @@ -0,0 +1,3 @@ +version="1.0" +name="Test mod" +supported_version="1.9.2.1" diff --git a/tests/files/mod3/descriptor.mod b/tests/files/mod3/descriptor.mod new file mode 100644 index 000000000..9f352643d --- /dev/null +++ b/tests/files/mod3/descriptor.mod @@ -0,0 +1,3 @@ +version="1.0" +name="Test mod" +supported_version="1.9.2.1" diff --git a/tests/test.rs b/tests/test.rs index 0b3ea0aae..79776c6b3 100644 --- a/tests/test.rs +++ b/tests/test.rs @@ -2,7 +2,8 @@ use std::path::PathBuf; use std::sync::{LazyLock, Mutex}; use tiger_lib::{ - take_reports, Everything, LogReportMetadata, LogReportPointers, TigerHashMap, TigerHashSet, + take_reports, Everything, Fileset, LogReportMetadata, LogReportPointers, TigerHashMap, + TigerHashSet, }; static TEST_MUTEX: LazyLock> = LazyLock::new(|| Mutex::new(())); @@ -15,8 +16,8 @@ fn check_mod_helper( let vanilla_dir = PathBuf::from("tests/files/ck3"); let mod_root = PathBuf::from(format!("tests/files/{}", modname)); - let mut everything = - Everything::new(None, Some(&vanilla_dir), None, None, &mod_root, Vec::new()).unwrap(); + let fileset = Fileset::builder(Some(&vanilla_dir)).with_modfile(mod_root).unwrap(); + let mut everything = Everything::new(fileset, None, None, None).unwrap(); everything.load_all(); everything.validate_all(); diff --git a/tiger-bin-shared/src/auto.rs b/tiger-bin-shared/src/auto.rs index c8b98acbf..36868d533 100644 --- a/tiger-bin-shared/src/auto.rs +++ b/tiger-bin-shared/src/auto.rs @@ -4,11 +4,7 @@ use std::path::{Path, PathBuf}; use anyhow::{bail, Context, Result}; use console::Term; -#[cfg(any(feature = "ck3", feature = "imperator", feature = "hoi4"))] -use tiger_lib::ModFile; -#[cfg(feature = "vic3")] -use tiger_lib::ModMetadata; -use tiger_lib::{emit_reports, Everything}; +use tiger_lib::{emit_reports, Everything, Fileset, Game}; use crate::gamedir::{ find_game_directory_steam, find_paradox_directory, find_workshop_directory_steam, @@ -55,7 +51,7 @@ pub fn run(game_consts: &GameConsts) -> Result<()> { &game, workshop.as_deref(), Some(&pdx), - &entries[0].path(), + entries[0].path(), &pdxlogs, )?; } else if entries.is_empty() { @@ -94,7 +90,7 @@ pub fn run(game_consts: &GameConsts) -> Result<()> { &game, workshop.as_deref(), Some(&pdx), - &entries[modnr].path(), + entries[modnr].path(), &pdxlogs, )?; return Ok(()); @@ -114,26 +110,24 @@ fn validate_mod( game: &Path, workshop: Option<&Path>, paradox: Option<&Path>, - modpath: &Path, + modpath: PathBuf, logdir: &Path, ) -> Result<()> { let mut everything; let mut modpath = modpath; - #[cfg(any(feature = "ck3", feature = "imperator", feature = "hoi4"))] - let modfile = ModFile::read(modpath)?; - #[cfg(any(feature = "ck3", feature = "imperator", feature = "hoi4"))] - let modpath_owned = modfile.modpath(); - #[cfg(any(feature = "ck3", feature = "imperator", feature = "hoi4"))] - { - modpath = &modpath_owned; - if !modpath.is_dir() { - eprintln!("Looking for mod in {}", modpath.display()); - bail!("Cannot find mod directory. Please make sure the .mod file is correct."); - } - } + let fileset = Fileset::builder(Some(game)); + let fileset = match Game::game() { + #[cfg(feature = "ck3")] + Game::Ck3 => fileset.with_modfile(modpath.clone()), + #[cfg(feature = "vic3")] + Game::Vic3 => fileset.with_metadata(modpath.clone()), + #[cfg(feature = "imperator")] + Game::Imperator => fileset.with_modfile(modpath.clone()), + #[cfg(feature = "hoi4")] + Game::Hoi4 => fileset.with_modfile(modpath.clone()), + }?; - eprintln!("Using mod directory: {}", modpath.display()); let output_filename = format!("{name_short}-tiger-{}.log", modpath.file_name().unwrap().to_string_lossy()); let output_file = &logdir.join(output_filename); @@ -141,23 +135,7 @@ fn validate_mod( eprintln!("Writing error reports to {} ...", output_file.display()); eprintln!("This will take a few seconds."); - #[cfg(any(feature = "ck3", feature = "imperator", feature = "hoi4"))] - { - everything = - Everything::new(None, Some(game), workshop, paradox, modpath, modfile.replace_paths())?; - } - #[cfg(feature = "vic3")] - { - let metadata = ModMetadata::read(modpath)?; - everything = Everything::new( - None, - Some(game), - workshop, - paradox, - modpath, - metadata.replace_paths(), - )?; - } + everything = Everything::new(fileset, None, workshop, paradox)?; // Unfortunately have to disable the colors by default because // on Windows there's no easy way to view a file that contains those escape sequences. diff --git a/tiger-bin-shared/src/tiger.rs b/tiger-bin-shared/src/tiger.rs index 14114093d..54a2476c5 100644 --- a/tiger-bin-shared/src/tiger.rs +++ b/tiger-bin-shared/src/tiger.rs @@ -3,13 +3,9 @@ use std::{mem::forget, path::PathBuf}; use anyhow::{bail, Result}; use clap::{error::ErrorKind, Args, Parser, Subcommand}; -#[cfg(any(feature = "ck3", feature = "imperator", feature = "hoi4"))] -use tiger_lib::ModFile; -#[cfg(feature = "vic3")] -use tiger_lib::ModMetadata; use tiger_lib::{ disable_ansi_colors, emit_reports, get_version_from_launcher, set_show_loaded_mods, - set_show_vanilla, suppress_from_json, validate_config_file, Everything, + set_show_vanilla, suppress_from_json, validate_config_file, Everything, Fileset, Game, }; use crate::gamedir::{ @@ -210,45 +206,24 @@ pub fn run( disable_ansi_colors(); } - let mut everything; - - #[cfg(any(feature = "ck3", feature = "imperator", feature = "hoi4"))] - { - if args.modpath.is_dir() { - args.modpath.push("descriptor.mod"); - } - - let modfile = ModFile::read(&args.modpath)?; - let modpath = modfile.modpath(); - if !modpath.exists() { - eprintln!("Looking for mod in {}", modpath.display()); - bail!("Cannot find mod directory. Please make sure the .mod file is correct."); - } - eprintln!("Using mod directory: {}", modpath.display()); - - everything = Everything::new( - args.config.as_deref(), - args.game.as_deref(), - args.workshop.as_deref(), - args.paradox.as_deref(), - &modpath, - modfile.replace_paths(), - )?; - } - #[cfg(feature = "vic3")] - { - let metadata = ModMetadata::read(&args.modpath)?; - eprintln!("Using mod directory: {}", metadata.modpath().display()); - - everything = Everything::new( - args.config.as_deref(), - args.game.as_deref(), - args.workshop.as_deref(), - args.paradox.as_deref(), - &args.modpath, - metadata.replace_paths(), - )?; - } + let fileset = Fileset::builder(args.game.as_deref()); + let fileset = match Game::game() { + #[cfg(feature = "ck3")] + Game::Ck3 => fileset.with_modfile(args.modpath), + #[cfg(feature = "vic3")] + Game::Vic3 => fileset.with_metadata(args.modpath), + #[cfg(feature = "imperator")] + Game::Imperator => fileset.with_modfile(args.modpath), + #[cfg(feature = "hoi4")] + Game::Hoi4 => fileset.with_modfile(args.modpath), + }?; + + let mut everything = Everything::new( + fileset, + args.config.as_deref(), + args.workshop.as_deref(), + args.paradox.as_deref(), + )?; // Print a blank line between the preamble and the first report: eprintln!(); diff --git a/vic3-tiger/benches/criterion.rs b/vic3-tiger/benches/criterion.rs index 0a972560f..d70657a47 100644 --- a/vic3-tiger/benches/criterion.rs +++ b/vic3-tiger/benches/criterion.rs @@ -5,7 +5,7 @@ use std::{ fs, path::{Path, PathBuf}, }; -use tiger_lib::{Everything, Game}; +use tiger_lib::{Everything, Fileset, Game}; static CONFIG_PATH: &str = "../benches/vic3.toml"; @@ -27,8 +27,7 @@ fn workspace_path(s: &str) -> PathBuf { let p = PathBuf::from(s); if p.is_relative() { PathBuf::from("..").join(p) - } - else { + } else { p } } @@ -48,6 +47,8 @@ fn bench_multiple(c: &mut Criterion) { mod_paths.extend(iter); } + let vanilla_dir = PathBuf::from(config.vanilla_dir); + let mut group = c.benchmark_group("benchmark"); group.sample_size(config.sample_size.unwrap_or(10)); for (index, mod_path) in mod_paths.iter().enumerate() { @@ -58,7 +59,7 @@ fn bench_multiple(c: &mut Criterion) { BenchmarkId::new("mods", format!("{}. {}", index + 1, mod_data["name"])), &mod_path, |b, modpath_ref| { - b.iter(|| bench_mod(&config.vanilla_dir, modpath_ref)); + b.iter(|| bench_mod(&vanilla_dir, modpath_ref)); }, ); } @@ -66,9 +67,9 @@ fn bench_multiple(c: &mut Criterion) { group.finish(); } -fn bench_mod(vanilla_dir: &str, modpath: &Path) { - let mut everything = - Everything::new(None, Some(Path::new(vanilla_dir)), None, None, modpath, vec![]).unwrap(); +fn bench_mod(vanilla_dir: &Path, modpath: &PathBuf) { + let fileset = Fileset::builder(Some(vanilla_dir)).with_metadata(modpath).unwrap(); + let mut everything = Everything::new(fileset, None, None, None).unwrap(); everything.load_all(); everything.validate_all(); everything.check_rivers();