diff --git a/data/README.md b/data/README.md index 2988eabb1..49962b57f 100644 --- a/data/README.md +++ b/data/README.md @@ -6,14 +6,16 @@ All examples are run from the repository root and write their outputs under `tmp ## Per-command configs -A per-command config is exactly the command's argument object. Every key has a matching `--flag`, and an explicit flag on the command line still overrides the file (flag > config > default). +A per-command config is exactly the command's argument object. Its keys map one-to-one to the CLI flags: `--alignment a.fasta` is `alignment: [a.fasta]`, `--clock-rate 0.003` is `clock_rate: 0.003`, `--output-all dir` is `output_all: dir`. An explicit flag on the command line still overrides the file (flag > config > default). | File | Command | Demonstrates | | ------------------------------------------------------------------------ | ----------- | ----------------------------------------------------------- | | [`ebola/20/ancestral-parsimony.yaml`](ebola/20/ancestral-parsimony.yaml) | `ancestral` | Fitch parsimony: fast, model-free ancestral states | | [`zika/20/ancestral.yaml`](zika/20/ancestral.yaml) | `ancestral` | Marginal reconstruction with an inferred GTR model | +| [`sc2/2844/ancestral.yaml`](sc2/2844/ancestral.yaml) | `ancestral` | Marginal reconstruction on 2844 SARS-CoV-2 genomes | | [`dengue/100/clock.yaml`](dengue/100/clock.yaml) | `clock` | Root-to-tip regression with a non-default id column | | [`flu/h3n2/20/optimize.yaml`](flu/h3n2/20/optimize.yaml) | `optimize` | Branch-length optimization and min-divergence rerooting | +| [`mpox/clade-ii/1000/optimize.yaml`](mpox/clade-ii/1000/optimize.yaml) | `optimize` | The mpox `augur refine` replacement (mutations, no indels) | | [`rsv/a/100/prune.yaml`](rsv/a/100/prune.yaml) | `prune` | Dropping short, empty, and shared-mutation branches | | [`zika/20/mugration.yaml`](zika/20/mugration.yaml) | `mugration` | Discrete ancestral geography over sampling country | | [`flu/h3n2/200/timetree.yaml`](flu/h3n2/200/timetree.yaml) | `timetree` | Dated tree with a fixed clock rate and confidence intervals | @@ -28,12 +30,14 @@ treetime ancestral --config data/zika/20/ancestral.yaml A pipeline file lists named steps, each an analysis command with its arguments. Steps run sequentially in one process. A step reads an earlier step's output with `{{ steps..outputs. }}`, so intermediate paths are never hand-written. `vars` names shared values once; a top-level `output_all` gives each step its own `//` output directory. -| File | Chain | Demonstrates | -| ------------------------------------------------------------------ | --------------------------------------- | -------------------------------------------------- | -| [`mpox/clade-ii/20/pipeline.yaml`](mpox/clade-ii/20/pipeline.yaml) | `optimize` -> `ancestral` | A Nextstrain-style mpox build in one file | -| [`zika/20/pipeline.yaml`](zika/20/pipeline.yaml) | `timetree` -> `mugration` | Phylodynamics and phylogeography end to end | -| [`ebola/362/pipeline.yaml`](ebola/362/pipeline.yaml) | `optimize` -> `ancestral` -> `timetree` | A full three-step build on a real epidemic dataset | -| [`flu/h3n2/200/pipeline.yaml`](flu/h3n2/200/pipeline.yaml) | `optimize` -> `timetree` | A shared, typed clock rate defined once in `vars` | +| File | Chain | Demonstrates | +| ---------------------------------------------------------------------- | --------------------------------------- | ---------------------------------------------------------------- | +| [`zika/20/pipeline.yaml`](zika/20/pipeline.yaml) | `timetree` -> `mugration` | Phylodynamics and phylogeography end to end | +| [`flu/h3n2/200/pipeline.yaml`](flu/h3n2/200/pipeline.yaml) | `optimize` -> `timetree` | A shared, typed clock rate defined once in `vars` | +| [`flu/h3n2/500/pipeline.yaml`](flu/h3n2/500/pipeline.yaml) | `optimize` -> `ancestral` -> `timetree` | A seasonal-influenza build with a fixed clock (500 sequences) | +| [`ebola/362/pipeline.yaml`](ebola/362/pipeline.yaml) | `optimize` -> `ancestral` -> `timetree` | A three-step build on the 2014 Ebola epidemic dataset | +| [`sc2/2844/pipeline.yaml`](sc2/2844/pipeline.yaml) | `optimize` -> `ancestral` | A date-free build on 2844 SARS-CoV-2 genomes | +| [`mpox/clade-ii/1000/pipeline.yaml`](mpox/clade-ii/1000/pipeline.yaml) | `optimize` -> `ancestral` -> `timetree` | The full Nextstrain mpox TreeTime chain (fixed clock, mutations) | Run a pipeline, preview its plan, or run only part of it: @@ -60,7 +64,7 @@ treetime pipeline --config data/zika/20/pipeline.yaml --steps mugration - Validation is static and typed. `--check` and the JSON-schema-backed argument objects catch a bad parameter or a dangling step reference before anything runs; a Snakemake rule only surfaces a bad TreeTime parameter when that rule executes. - Reproducible and portable. The config is the exact serialized argument shape, with one dependency, committed beside the data. -This is not a replacement for a full multi-sample workflow manager: incremental rebuilds, cluster scheduling, and non-TreeTime rules (subsampling, alignment, tree building, export) still belong to Snakemake and Augur. The pipeline replaces the TreeTime glue inside such a build. On the Nextstrain mpox `WIP/new_tt` phylogenetic workflow, for instance, `treetime optimize` replaces `augur refine` (branch-length optimization) and `treetime ancestral` folds in `augur translate`; the [`mpox/clade-ii/20/pipeline.yaml`](mpox/clade-ii/20/pipeline.yaml) example runs exactly that pair as one step chain. +This is not a replacement for a full multi-sample workflow manager: incremental rebuilds, cluster scheduling, and non-TreeTime rules (subsampling, alignment, tree building, export) still belong to Snakemake and Augur. The pipeline replaces the TreeTime glue inside such a build. On the Nextstrain mpox `WIP/new_tt` phylogenetic workflow, for instance, three Snakemake rules drive TreeTime: `treetime optimize` replaces `augur refine` (branch-length optimization), `treetime ancestral` folds in `augur translate`, and `treetime timetree` dates the tree. The [`mpox/clade-ii/1000/pipeline.yaml`](mpox/clade-ii/1000/pipeline.yaml) example runs that chain as three steps in one file. ## Editor support diff --git a/data/dengue/100/clock.yaml b/data/dengue/100/clock.yaml index 4edfe3b17..7d40fabd1 100644 --- a/data/dengue/100/clock.yaml +++ b/data/dengue/100/clock.yaml @@ -10,9 +10,7 @@ tree: "data/dengue/100/tree.nwk" metadata: "data/dengue/100/metadata.tsv" -metadata_id: - metadata_id_columns: - - "genbank_accession" +metadata_id_columns: + - "genbank_accession" -output: - output_all: "tmp/examples/dengue/100/clock" +output_all: "tmp/examples/dengue/100/clock" diff --git a/data/ebola/20/ancestral-parsimony.yaml b/data/ebola/20/ancestral-parsimony.yaml index 06bd150db..0f5211fcc 100644 --- a/data/ebola/20/ancestral-parsimony.yaml +++ b/data/ebola/20/ancestral-parsimony.yaml @@ -9,11 +9,9 @@ tree: "data/ebola/20/tree.nwk" alignment: - alignment: - - "data/ebola/20/aln.fasta.xz" + - "data/ebola/20/aln.fasta.xz" # Fitch parsimony: fast, model-free ancestral states and mutations. method_anc: "parsimony" -output: - output_all: "tmp/examples/ebola/20/ancestral-parsimony" +output_all: "tmp/examples/ebola/20/ancestral-parsimony" diff --git a/data/ebola/362/pipeline.yaml b/data/ebola/362/pipeline.yaml index a62913cdf..389bd3cf4 100644 --- a/data/ebola/362/pipeline.yaml +++ b/data/ebola/362/pipeline.yaml @@ -23,8 +23,7 @@ steps: optimize: tree: "{{ vars.data }}/tree.nwk" alignment: - alignment: - - "{{ vars.data }}/aln.fasta.xz" + - "{{ vars.data }}/aln.fasta.xz" reroot: "min-dev" # 2. Reconstruct ancestral sequences and mutations on the optimized tree. @@ -32,8 +31,7 @@ steps: ancestral: tree: "{{ steps.optimize.outputs.nwk }}" alignment: - alignment: - - "{{ vars.data }}/aln.fasta.xz" + - "{{ vars.data }}/aln.fasta.xz" method_anc: "marginal" # 3. Date the optimized tree, keeping the branch lengths from step 1. @@ -42,7 +40,6 @@ steps: tree: "{{ steps.optimize.outputs.nwk }}" metadata: "{{ vars.data }}/metadata.tsv" alignment: - alignment: - - "{{ vars.data }}/aln.fasta.xz" + - "{{ vars.data }}/aln.fasta.xz" coalescent_opt: true max_iter: 3 diff --git a/data/flu/h3n2/20/optimize.yaml b/data/flu/h3n2/20/optimize.yaml index 8ec9359c1..0e41fca03 100644 --- a/data/flu/h3n2/20/optimize.yaml +++ b/data/flu/h3n2/20/optimize.yaml @@ -9,11 +9,7 @@ tree: "data/flu/h3n2/20/tree.nwk" alignment: - alignment: - - "data/flu/h3n2/20/aln.fasta.xz" - -model_args: - model: "infer" + - "data/flu/h3n2/20/aln.fasta.xz" # Per-edge optimizer: Brent's method in sqrt(t) space (the default, matches v0). opt_method: "brent-sqrt" @@ -21,5 +17,4 @@ opt_method: "brent-sqrt" # Reroot by minimizing root-to-tip divergence variance (a date-free method). reroot: "min-dev" -output: - output_all: "tmp/examples/flu/h3n2/20/optimize" +output_all: "tmp/examples/flu/h3n2/20/optimize" diff --git a/data/flu/h3n2/200/pipeline.yaml b/data/flu/h3n2/200/pipeline.yaml index 69082c8a5..4b9e78e10 100644 --- a/data/flu/h3n2/200/pipeline.yaml +++ b/data/flu/h3n2/200/pipeline.yaml @@ -16,15 +16,12 @@ vars: output_all: "tmp/examples/flu/h3n2/200/pipeline" steps: - # Optimize branch lengths under an inferred GTR model. + # Optimize branch lengths under an inferred GTR model (the default). - name: "optimize" optimize: tree: "{{ vars.data }}/tree.nwk" alignment: - alignment: - - "{{ vars.data }}/aln.fasta.xz" - model_args: - model: "infer" + - "{{ vars.data }}/aln.fasta.xz" # Date the optimized tree with the fixed clock rate from `vars`. - name: "timetree" @@ -32,8 +29,7 @@ steps: tree: "{{ steps.optimize.outputs.nwk }}" metadata: "{{ vars.data }}/metadata.tsv" alignment: - alignment: - - "{{ vars.data }}/aln.fasta.xz" + - "{{ vars.data }}/aln.fasta.xz" clock_rate: "{{ vars.clock_rate }}" clock_std_dev: "{{ vars.clock_std_dev }}" confidence: true diff --git a/data/flu/h3n2/200/timetree.yaml b/data/flu/h3n2/200/timetree.yaml index 167a6f82e..57b5176e4 100644 --- a/data/flu/h3n2/200/timetree.yaml +++ b/data/flu/h3n2/200/timetree.yaml @@ -12,8 +12,7 @@ tree: "data/flu/h3n2/200/tree.nwk" metadata: "data/flu/h3n2/200/metadata.tsv" alignment: - alignment: - - "data/flu/h3n2/200/aln.fasta.xz" + - "data/flu/h3n2/200/aln.fasta.xz" clock_rate: 0.003 clock_std_dev: 0.0006 @@ -21,7 +20,6 @@ confidence: true coalescent_opt: true max_iter: 3 -output: - output_all: "tmp/examples/flu/h3n2/200/timetree" +output_all: "tmp/examples/flu/h3n2/200/timetree" # Confidence intervals are written to their own TSV when a path is given. output_confidence_tsv: "tmp/examples/flu/h3n2/200/timetree/confidence.tsv" diff --git a/data/flu/h3n2/500/pipeline.yaml b/data/flu/h3n2/500/pipeline.yaml new file mode 100644 index 000000000..896146893 --- /dev/null +++ b/data/flu/h3n2/500/pipeline.yaml @@ -0,0 +1,49 @@ +# Seasonal influenza H3N2 build: optimize, reconstruct, date (flu/h3n2/500). +# +# Run from the repository root: +# treetime pipeline --config data/flu/h3n2/500/pipeline.yaml +# +# Preview the plan, or resume from a later step after editing it: +# treetime pipeline --config data/flu/h3n2/500/pipeline.yaml --check +# treetime pipeline --config data/flu/h3n2/500/pipeline.yaml --steps timetree +# +# optimize -> ancestral -> timetree on 500 H3N2 hemagglutinin sequences. Seasonal +# influenza has a well-characterized clock, so the rate is fixed +# (`clock_rate` 0.003 substitutions/site/year) instead of estimated, defined once +# in `vars` as a typed number. The optimized tree flows to the later steps through +# `{{ steps.optimize.outputs.nwk }}`. + +vars: + data: "data/flu/h3n2/500" + clock_rate: 0.003 + clock_std_dev: 0.0006 + +output_all: "tmp/examples/flu/h3n2/500/pipeline" + +steps: + # Optimize branch lengths under an inferred GTR model. + - name: "optimize" + optimize: + tree: "{{ vars.data }}/tree.nwk" + alignment: + - "{{ vars.data }}/aln.fasta.xz" + + # Reconstruct ancestral sequences and mutations on the optimized tree. + - name: "ancestral" + ancestral: + tree: "{{ steps.optimize.outputs.nwk }}" + alignment: + - "{{ vars.data }}/aln.fasta.xz" + method_anc: "marginal" + + # Date the optimized tree with the fixed clock rate from `vars`. + - name: "timetree" + timetree: + tree: "{{ steps.optimize.outputs.nwk }}" + metadata: "{{ vars.data }}/metadata.tsv" + alignment: + - "{{ vars.data }}/aln.fasta.xz" + clock_rate: "{{ vars.clock_rate }}" + clock_std_dev: "{{ vars.clock_std_dev }}" + confidence: true + max_iter: 3 diff --git a/data/mpox/clade-ii/1000/optimize.yaml b/data/mpox/clade-ii/1000/optimize.yaml new file mode 100644 index 000000000..a272f0c3b --- /dev/null +++ b/data/mpox/clade-ii/1000/optimize.yaml @@ -0,0 +1,20 @@ +# Branch-length optimization for a Nextstrain mpox build (mpox/clade-ii/1000). +# +# Run from the repository root: +# treetime optimize --config data/mpox/clade-ii/1000/optimize.yaml +# +# This is the date-free step that replaces `augur refine` in the nextstrain/mpox +# `WIP/new_tt` workflow. It re-estimates every branch length and reroots on the +# minimum-divergence branch. mpox divergence is counted in absolute mutations +# (`divergence_units: mutations`), and indels are excluded from the branch-length +# likelihood (`no_indels`), matching the pipeline's `treetime optimize` rule. + +tree: "data/mpox/clade-ii/1000/tree.nwk" +alignment: + - "data/mpox/clade-ii/1000/aln.fasta.xz" + +reroot: "min-dev" +divergence_units: "mutations" +no_indels: true + +output_all: "tmp/examples/mpox/clade-ii/1000/optimize" diff --git a/data/mpox/clade-ii/1000/pipeline.yaml b/data/mpox/clade-ii/1000/pipeline.yaml new file mode 100644 index 000000000..4da8cfd2a --- /dev/null +++ b/data/mpox/clade-ii/1000/pipeline.yaml @@ -0,0 +1,65 @@ +# Nextstrain mpox build (clade IIb, hmpxv1), TreeTime part in one file (mpox/clade-ii/1000). +# +# Run from the repository root: +# treetime pipeline --config data/mpox/clade-ii/1000/pipeline.yaml +# +# Preview the whole plan, or resume after a failure, without recomputing earlier steps: +# treetime pipeline --config data/mpox/clade-ii/1000/pipeline.yaml --check +# treetime pipeline --config data/mpox/clade-ii/1000/pipeline.yaml --steps ancestral,timetree +# +# On the nextstrain/mpox `WIP/new_tt` phylogenetic workflow, three Snakemake +# rules drive TreeTime: `treetime optimize` (replacing `augur refine`), then +# `treetime ancestral` (folding in `augur translate`), then `treetime timetree`. +# This file runs that same chain in one process. +# +# mpox is a slow-evolving DNA virus: divergence is measured in absolute mutations +# (`divergence_units: mutations`), the clock rate is fixed rather than estimated +# (`clock_rate` 5.7e-5 substitutions/site/year, with `clock_std_dev` for date +# confidence), and indels are excluded from the branch-length likelihood +# (`no_indels`). The clock constants live in `vars` as typed numbers, defined +# once and reused. `{{ steps.optimize.outputs.nwk }}` carries the optimized tree +# to the later steps with no hand-written path. + +vars: + data: "data/mpox/clade-ii/1000" + clock_rate: 5.7e-5 + clock_std_dev: 2e-5 + +output_all: "tmp/examples/mpox/clade-ii/1000/pipeline" + +steps: + # 1. Optimize branch lengths and reroot on the minimum-divergence branch. + - name: "optimize" + optimize: + tree: "{{ vars.data }}/tree.nwk" + alignment: + - "{{ vars.data }}/aln.fasta.xz" + reroot: "min-dev" + divergence_units: "mutations" + no_indels: true + + # 2. Reconstruct ancestral sequences and mutations on the optimized tree. + - name: "ancestral" + ancestral: + tree: "{{ steps.optimize.outputs.nwk }}" + alignment: + - "{{ vars.data }}/aln.fasta.xz" + method_anc: "marginal" + + # 3. Date the optimized tree with the fixed clock, keeping its root and polytomies. + - name: "timetree" + timetree: + tree: "{{ steps.optimize.outputs.nwk }}" + metadata: "{{ vars.data }}/metadata.tsv" + metadata_id_columns: + - "accession" + alignment: + - "{{ vars.data }}/aln.fasta.xz" + clock_rate: "{{ vars.clock_rate }}" + clock_std_dev: "{{ vars.clock_std_dev }}" + keep_root: true + keep_polytomies: true + divergence_units: "mutations" + no_indels: true + confidence: true + max_iter: 3 diff --git a/data/mpox/clade-ii/20/pipeline.yaml b/data/mpox/clade-ii/20/pipeline.yaml deleted file mode 100644 index 3fdf81ac8..000000000 --- a/data/mpox/clade-ii/20/pipeline.yaml +++ /dev/null @@ -1,46 +0,0 @@ -# Nextstrain-style mpox build: branch-length optimization, then ancestral reconstruction. -# -# Run from the repository root: -# treetime pipeline --config data/mpox/clade-ii/20/pipeline.yaml -# -# Resolve the whole plan (inputs, outputs, chained paths) without running anything: -# treetime pipeline --config data/mpox/clade-ii/20/pipeline.yaml --check -# -# On the Nextstrain mpox `WIP/new_tt` phylogenetic workflow, `treetime optimize` -# replaces `augur refine` (branch-length optimization only) and `treetime -# ancestral` folds in `augur translate`. This one file runs both steps in one -# process, on one dataset, with no snakemake, augur, or shell glue. -# -# `vars` names shared values once. `{{ steps.optimize.outputs.nwk }}` resolves -# to the optimized tree the first step writes, so the second step never repeats -# a path. `output_all` gives each step its own `//` output directory. - -vars: - data: "data/mpox/clade-ii/20" - -output_all: "tmp/examples/mpox/clade-ii/20/pipeline" - -steps: - # Re-estimate branch lengths and reroot on the min-divergence branch. - - name: "optimize" - optimize: - tree: "{{ vars.data }}/tree.nwk" - alignment: - alignment: - - "{{ vars.data }}/aln.fasta.xz" - model_args: - model: "infer" - reroot: "min-dev" - # mpox is measured in absolute substitutions rather than per-site rates. - divergence_units: "mutations" - - # Reconstruct ancestral sequences and map mutations onto the optimized tree. - - name: "ancestral" - ancestral: - tree: "{{ steps.optimize.outputs.nwk }}" - alignment: - alignment: - - "{{ vars.data }}/aln.fasta.xz" - method_anc: "marginal" - model_args: - model: "infer" diff --git a/data/rsv/a/100/prune.yaml b/data/rsv/a/100/prune.yaml index 49c6071e8..dba102f3b 100644 --- a/data/rsv/a/100/prune.yaml +++ b/data/rsv/a/100/prune.yaml @@ -9,12 +9,10 @@ tree: "data/rsv/a/100/tree.nwk" alignment: - alignment: - - "data/rsv/a/100/aln.fasta.xz" + - "data/rsv/a/100/aln.fasta.xz" prune_short: 1.0e-6 prune_empty: true merge_shared_mutations: true -output: - output_all: "tmp/examples/rsv/a/100/prune" +output_all: "tmp/examples/rsv/a/100/prune" diff --git a/data/sc2/2844/ancestral.yaml b/data/sc2/2844/ancestral.yaml new file mode 100644 index 000000000..0732967c4 --- /dev/null +++ b/data/sc2/2844/ancestral.yaml @@ -0,0 +1,18 @@ +# Marginal ancestral reconstruction on 2844 SARS-CoV-2 genomes (sc2/2844). +# +# Run from the repository root: +# treetime ancestral --config data/sc2/2844/ancestral.yaml +# +# A large single-command run: marginal reconstruction treats every site +# independently, fits a GTR rate matrix from the data (the default +# `model: infer`), and maps nucleotide mutations onto the tree. The sparse +# representation (the default) stores only variable sites, which keeps a +# genome-length SARS-CoV-2 alignment tractable. + +tree: "data/sc2/2844/tree.nwk" +alignment: + - "data/sc2/2844/aln.fasta.xz" + +method_anc: "marginal" + +output_all: "tmp/examples/sc2/2844/ancestral" diff --git a/data/sc2/2844/pipeline.yaml b/data/sc2/2844/pipeline.yaml new file mode 100644 index 000000000..9e6d855f5 --- /dev/null +++ b/data/sc2/2844/pipeline.yaml @@ -0,0 +1,35 @@ +# SARS-CoV-2 branch-length optimization and ancestral reconstruction (sc2/2844). +# +# Run from the repository root: +# treetime pipeline --config data/sc2/2844/pipeline.yaml +# +# Preview the plan without running anything: +# treetime pipeline --config data/sc2/2844/pipeline.yaml --check +# +# A date-free build on 2844 SARS-CoV-2 genomes: `optimize` re-estimates branch +# lengths and reroots on the minimum-divergence branch, then `ancestral` maps +# nucleotide mutations onto that tree. Neither step needs sampling dates, so this +# runs on the tree and alignment alone. `{{ steps.optimize.outputs.nwk }}` passes +# the optimized tree to the second step. + +vars: + data: "data/sc2/2844" + +output_all: "tmp/examples/sc2/2844/pipeline" + +steps: + # Re-estimate branch lengths under an inferred GTR model and reroot. + - name: "optimize" + optimize: + tree: "{{ vars.data }}/tree.nwk" + alignment: + - "{{ vars.data }}/aln.fasta.xz" + reroot: "min-dev" + + # Reconstruct ancestral sequences and mutations on the optimized tree. + - name: "ancestral" + ancestral: + tree: "{{ steps.optimize.outputs.nwk }}" + alignment: + - "{{ vars.data }}/aln.fasta.xz" + method_anc: "marginal" diff --git a/data/zika/20/ancestral.yaml b/data/zika/20/ancestral.yaml index 4e339ae3f..19a303e39 100644 --- a/data/zika/20/ancestral.yaml +++ b/data/zika/20/ancestral.yaml @@ -4,17 +4,13 @@ # treetime ancestral --config data/zika/20/ancestral.yaml # # Marginal reconstruction treats every site independently and reports the -# maximum-likelihood ancestral state per node. `--model infer` (the default) -# fits a GTR rate matrix from the data and writes it alongside the tree. +# maximum-likelihood ancestral state per node. It fits a GTR rate matrix from the +# data (the default `model: infer`) and writes it alongside the tree. tree: "data/zika/20/tree.nwk" alignment: - alignment: - - "data/zika/20/aln.fasta.xz" + - "data/zika/20/aln.fasta.xz" method_anc: "marginal" -model_args: - model: "infer" -output: - output_all: "tmp/examples/zika/20/ancestral" +output_all: "tmp/examples/zika/20/ancestral" diff --git a/data/zika/20/mugration.yaml b/data/zika/20/mugration.yaml index 26cded39a..1bbdcea3f 100644 --- a/data/zika/20/mugration.yaml +++ b/data/zika/20/mugration.yaml @@ -15,6 +15,5 @@ attribute: "country" weights: "data/zika/country_weights.csv" pc: 1.0 -output: - output_all: "tmp/examples/zika/20/mugration" +output_all: "tmp/examples/zika/20/mugration" output_confidence_csv: "tmp/examples/zika/20/mugration/confidence.csv" diff --git a/data/zika/20/pipeline.yaml b/data/zika/20/pipeline.yaml index 4ecd33652..cdefc24db 100644 --- a/data/zika/20/pipeline.yaml +++ b/data/zika/20/pipeline.yaml @@ -25,8 +25,7 @@ steps: tree: "{{ vars.data }}/tree.nwk" metadata: "{{ vars.data }}/metadata.tsv" alignment: - alignment: - - "{{ vars.data }}/aln.fasta.xz" + - "{{ vars.data }}/aln.fasta.xz" # Reconstruct ancestral sampling country on the dated tree from step 1. - name: "mugration" diff --git a/packages/app-cli/src/cli/pipeline/inputs.rs b/packages/app-cli/src/cli/pipeline/inputs.rs index 1f1dc52e3..56bd36aa5 100644 --- a/packages/app-cli/src/cli/pipeline/inputs.rs +++ b/packages/app-cli/src/cli/pipeline/inputs.rs @@ -7,7 +7,7 @@ use serde_json::Value; /// disagree about what a step reads. pub const INPUT_FIELDS: [(&str, &[&str]); 5] = [ ("tree", &["tree"]), - ("alignment", &["alignment", "alignment"]), + ("alignment", &["alignment"]), ("metadata", &["metadata"]), ("weights", &["weights"]), ("vcf-reference", &["vcf_reference"]), diff --git a/packages/app-cli/src/cli/pipeline/resolve.rs b/packages/app-cli/src/cli/pipeline/resolve.rs index 265ed8021..32fcc064d 100644 --- a/packages/app-cli/src/cli/pipeline/resolve.rs +++ b/packages/app-cli/src/cli/pipeline/resolve.rs @@ -302,18 +302,15 @@ fn resolve_selection_path(step: &str, producer: &ResolvedStep, selection: &str) } } -/// Set `output.output_all` to `dir` unless the step already configured its own output directory. +/// Set the top-level `output_all` to `dir` unless the step already configured its own output +/// directory. `output_all` is a flattened field, so it sits at the top level of the step payload. fn set_output_all_if_absent(payload: &mut Value, dir: &Path) { let Value::Object(map) = payload else { return; }; - let output = map.entry("output").or_insert_with(|| Value::Object(Map::new())); - let Value::Object(output) = output else { - return; - }; - let already_set = matches!(output.get("output_all"), Some(value) if !value.is_null()); + let already_set = matches!(map.get("output_all"), Some(value) if !value.is_null()); if !already_set { - output.insert( + map.insert( "output_all".to_owned(), Value::String(dir.to_string_lossy().into_owned()), ); @@ -395,8 +392,8 @@ mod tests { fn test_resolve_chain_uses_explicit_step_output_all() { let config = json!({ "steps": [ - { "name": "tt", "timetree": { "tree": "in.nwk", "metadata": "m.tsv", "output": { "output_all": "out/tt" } } }, - { "name": "anc", "ancestral": { "tree": "{{ steps.tt.outputs.nwk }}", "output": { "output_all": "out/anc" } } } + { "name": "tt", "timetree": { "tree": "in.nwk", "metadata": "m.tsv", "output_all": "out/tt" } }, + { "name": "anc", "ancestral": { "tree": "{{ steps.tt.outputs.nwk }}", "output_all": "out/anc" } } ] }); let resolved = resolve(config).unwrap(); @@ -409,7 +406,7 @@ mod tests { let config = json!({ "output_all": "tmp/run", "steps": [ - { "name": "tt", "timetree": { "tree": "in.nwk", "metadata": "m.tsv", "output": { "output_nwk_style": ["plain", "beast"] } } }, + { "name": "tt", "timetree": { "tree": "in.nwk", "metadata": "m.tsv", "output_nwk_style": ["plain", "beast"] } }, { "name": "anc", "ancestral": { "tree": "{{ steps.tt.outputs.nwk }}" } } ] }); diff --git a/packages/app-cli/src/cli/pipeline/safety.rs b/packages/app-cli/src/cli/pipeline/safety.rs index 265385cc3..d46826244 100644 --- a/packages/app-cli/src/cli/pipeline/safety.rs +++ b/packages/app-cli/src/cli/pipeline/safety.rs @@ -141,8 +141,8 @@ mod tests { fn test_safety_output_collision_rejected() { let pipeline = resolve(json!({ "steps": [ - { "name": "a", "ancestral": { "tree": "in.nwk", "output": { "output_tree_nwk": "shared.nwk" } } }, - { "name": "b", "ancestral": { "tree": "in.nwk", "output": { "output_tree_nwk": "shared.nwk" } } } + { "name": "a", "ancestral": { "tree": "in.nwk", "output_tree_nwk": "shared.nwk" } }, + { "name": "b", "ancestral": { "tree": "in.nwk", "output_tree_nwk": "shared.nwk" } } ] })); let result = validate_plan(&pipeline, None); @@ -157,7 +157,7 @@ mod tests { fn test_safety_self_truncation_rejected() { let pipeline = resolve(json!({ "steps": [ - { "name": "a", "ancestral": { "tree": "same.nwk", "output": { "output_tree_nwk": "same.nwk" } } } + { "name": "a", "ancestral": { "tree": "same.nwk", "output_tree_nwk": "same.nwk" } } ] })); let result = validate_plan(&pipeline, None); diff --git a/packages/treetime/src/commands/ancestral/args.rs b/packages/treetime/src/commands/ancestral/args.rs index ebe8378fc..6e9e80f86 100644 --- a/packages/treetime/src/commands/ancestral/args.rs +++ b/packages/treetime/src/commands/ancestral/args.rs @@ -26,6 +26,7 @@ pub struct TreetimeAncestralArgsRaw { pub config_args: ConfigArgs, #[cfg_attr(feature = "clap", clap(flatten))] + #[serde(flatten)] pub alignment: AlignmentArgs, /// FASTA file of the sequence the VCF was mapped to (only for vcf input) @@ -41,9 +42,11 @@ pub struct TreetimeAncestralArgsRaw { pub tree: Option, #[cfg_attr(feature = "clap", clap(flatten))] + #[serde(flatten)] pub alphabet_args: AlphabetArgs, #[cfg_attr(feature = "clap", clap(flatten))] + #[serde(flatten)] pub model_args: ModelArgs, /// Method used for reconstructing ancestral sequences @@ -58,6 +61,7 @@ pub struct TreetimeAncestralArgsRaw { pub dense: Option, #[cfg_attr(feature = "clap", clap(flatten))] + #[serde(flatten)] pub gap_fill_args: GapFillArgs, /// Zero-based mutation indexing @@ -161,6 +165,7 @@ pub struct TreetimeAncestralArgsRaw { pub output_reconstructed_aa_fasta: Option, #[cfg_attr(feature = "clap", clap(flatten))] + #[serde(flatten)] pub output: OutputCoreArgs, /// Comma-separated list of outputs to produce with `--output-all`. @@ -175,6 +180,7 @@ pub struct TreetimeAncestralArgsRaw { pub output_selection: Vec, #[cfg_attr(feature = "clap", clap(flatten))] + #[serde(flatten)] pub topology_order: TopologyOrderArgs, /// Number of outer GTR refinement iterations. diff --git a/packages/treetime/src/commands/clock/args.rs b/packages/treetime/src/commands/clock/args.rs index 771c49920..aadec44ad 100644 --- a/packages/treetime/src/commands/clock/args.rs +++ b/packages/treetime/src/commands/clock/args.rs @@ -27,6 +27,7 @@ pub struct TreetimeClockArgsRaw { pub config_args: ConfigArgs, #[cfg_attr(feature = "clap", clap(flatten))] + #[serde(flatten)] pub alignment: AlignmentArgs, /// Name of file containing the tree in newick, nexus, or phylip format. @@ -47,9 +48,11 @@ pub struct TreetimeClockArgsRaw { pub metadata: Option, #[cfg_attr(feature = "clap", clap(flatten))] + #[serde(flatten)] pub metadata_id: MetadataIdArgs, #[cfg_attr(feature = "clap", clap(flatten))] + #[serde(flatten)] pub date_column: DateColumnArgs, /// Length of the sequence, used to calculate expected variation in branch length. Not required if alignment is provided. @@ -57,6 +60,7 @@ pub struct TreetimeClockArgsRaw { pub sequence_length: Option, #[cfg_attr(feature = "clap", clap(flatten))] + #[serde(flatten)] pub model_args: ModelArgs, /// If set to 'input', the provided branch length will be used without modification. Note that branch lengths optimized by treetime are only accurate at short evolutionary distances. @@ -73,6 +77,7 @@ pub struct TreetimeClockArgsRaw { pub clock_filter: f64, #[cfg_attr(feature = "clap", clap(flatten))] + #[serde(flatten)] pub reroot: RerootArgs, /// don't reroot the tree. Otherwise, reroot to minimize the residual of the regression of @@ -97,6 +102,7 @@ pub struct TreetimeClockArgsRaw { pub allow_negative_rate: bool, #[cfg_attr(feature = "clap", clap(flatten))] + #[serde(flatten)] pub output: OutputCoreArgs, /// Path to output clock model JSON. @@ -123,6 +129,7 @@ pub struct TreetimeClockArgsRaw { pub output_selection: Vec, #[cfg_attr(feature = "clap", clap(flatten))] + #[serde(flatten)] pub topology_order: TopologyOrderArgs, /// Random seed @@ -246,14 +253,17 @@ pub struct BranchSplitArgs { /// Grid search parameters #[cfg_attr(feature = "clap", clap(flatten))] + #[serde(flatten)] pub grid_params: GridSearchParams, /// Brent's method parameters #[cfg_attr(feature = "clap", clap(flatten))] + #[serde(flatten)] pub brent_params: BrentParams, /// Golden section search parameters #[cfg_attr(feature = "clap", clap(flatten))] + #[serde(flatten)] pub golden_params: GoldenSectionParams, } @@ -264,5 +274,6 @@ pub struct BranchSplitArgs { pub struct ClockRegressionArgs { /// Clock regression model parameters #[cfg_attr(feature = "clap", clap(flatten))] + #[serde(flatten)] pub clock_params: ClockParams, } diff --git a/packages/treetime/src/commands/mugration/args.rs b/packages/treetime/src/commands/mugration/args.rs index 7fd03a3cd..ffb6e11e8 100644 --- a/packages/treetime/src/commands/mugration/args.rs +++ b/packages/treetime/src/commands/mugration/args.rs @@ -41,6 +41,7 @@ pub struct TreetimeMugrationArgsRaw { pub weights: Option, #[cfg_attr(feature = "clap", clap(flatten))] + #[serde(flatten)] pub metadata_id: MetadataIdArgs, /// Path to output state-probability-profile CSV. @@ -118,6 +119,7 @@ pub struct TreetimeMugrationArgsRaw { pub seed: Option, #[cfg_attr(feature = "clap", clap(flatten))] + #[serde(flatten)] pub output: OutputCoreArgs, /// Comma-separated list of outputs to produce with `--output-all`. @@ -132,6 +134,7 @@ pub struct TreetimeMugrationArgsRaw { pub output_selection: Vec, #[cfg_attr(feature = "clap", clap(flatten))] + #[serde(flatten)] pub topology_order: TopologyOrderArgs, } diff --git a/packages/treetime/src/commands/optimize/args.rs b/packages/treetime/src/commands/optimize/args.rs index d48d4573b..d460ca750 100644 --- a/packages/treetime/src/commands/optimize/args.rs +++ b/packages/treetime/src/commands/optimize/args.rs @@ -45,6 +45,7 @@ pub struct TreetimeOptimizeArgsRaw { pub config_args: ConfigArgs, #[cfg_attr(feature = "clap", clap(flatten))] + #[serde(flatten)] pub alignment: AlignmentArgs, /// Name of file containing the tree in newick, nexus, or phylip format. @@ -55,9 +56,11 @@ pub struct TreetimeOptimizeArgsRaw { pub tree: Option, #[cfg_attr(feature = "clap", clap(flatten))] + #[serde(flatten)] pub alphabet_args: AlphabetArgs, #[cfg_attr(feature = "clap", clap(flatten))] + #[serde(flatten)] pub model_args: ModelArgs, /// Use dense representation of sequences on the tree @@ -69,6 +72,7 @@ pub struct TreetimeOptimizeArgsRaw { pub dense: Option, #[cfg_attr(feature = "clap", clap(flatten))] + #[serde(flatten)] pub output: OutputCoreArgs, /// Units for divergence values in augur node data JSON output. @@ -107,6 +111,7 @@ pub struct TreetimeOptimizeArgsRaw { pub output_selection: Vec, #[cfg_attr(feature = "clap", clap(flatten))] + #[serde(flatten)] pub topology_order: TopologyOrderArgs, /// Maximum number of iterations @@ -192,6 +197,7 @@ pub struct TreetimeOptimizeArgsRaw { pub keep_root: bool, #[cfg_attr(feature = "clap", clap(flatten))] + #[serde(flatten)] pub gap_fill_args: GapFillArgs, } diff --git a/packages/treetime/src/commands/prune/args.rs b/packages/treetime/src/commands/prune/args.rs index 4888afc76..823df02e3 100644 --- a/packages/treetime/src/commands/prune/args.rs +++ b/packages/treetime/src/commands/prune/args.rs @@ -21,6 +21,7 @@ pub struct TreetimePruneArgsRaw { pub config_args: ConfigArgs, #[cfg_attr(feature = "clap", clap(flatten))] + #[serde(flatten)] pub alignment: AlignmentArgs, /// Name of file containing the tree in newick, nexus, or phylip format. @@ -29,9 +30,11 @@ pub struct TreetimePruneArgsRaw { pub tree: Option, #[cfg_attr(feature = "clap", clap(flatten))] + #[serde(flatten)] pub alphabet_args: AlphabetArgs, #[cfg_attr(feature = "clap", clap(flatten))] + #[serde(flatten)] pub output: OutputCoreArgs, /// Path to output GTR model JSON. @@ -52,6 +55,7 @@ pub struct TreetimePruneArgsRaw { pub output_selection: Vec, #[cfg_attr(feature = "clap", clap(flatten))] + #[serde(flatten)] pub topology_order: TopologyOrderArgs, /// Threshold value for pruning of branches diff --git a/packages/treetime/src/commands/timetree/args.rs b/packages/treetime/src/commands/timetree/args.rs index 9d87fca35..f99ff9f5c 100644 --- a/packages/treetime/src/commands/timetree/args.rs +++ b/packages/treetime/src/commands/timetree/args.rs @@ -37,6 +37,7 @@ pub struct TreetimeTimetreeArgsRaw { pub config_args: ConfigArgs, #[cfg_attr(feature = "clap", clap(flatten))] + #[serde(flatten)] pub alignment: AlignmentArgs, /// Name of file containing the tree in newick, nexus, or phylip format. @@ -57,9 +58,11 @@ pub struct TreetimeTimetreeArgsRaw { pub metadata: Option, #[cfg_attr(feature = "clap", clap(flatten))] + #[serde(flatten)] pub metadata_id: MetadataIdArgs, #[cfg_attr(feature = "clap", clap(flatten))] + #[serde(flatten)] pub date_column_args: DateColumnArgs, /// Length of the sequence, used to calculate expected variation in branch length. Not required if alignment is provided. @@ -213,6 +216,7 @@ pub struct TreetimeTimetreeArgsRaw { pub n_iqd: Option, #[cfg_attr(feature = "clap", clap(flatten))] + #[serde(flatten)] pub reroot: RerootArgs, /// don't reroot the tree. Otherwise, reroot to minimize the residual of the regression of @@ -234,6 +238,7 @@ pub struct TreetimeTimetreeArgsRaw { pub covariation: bool, #[cfg_attr(feature = "clap", clap(flatten))] + #[serde(flatten)] pub model_args: ModelArgs, /// Method used for reconstructing ancestral sequences @@ -241,6 +246,7 @@ pub struct TreetimeTimetreeArgsRaw { pub method_anc: MethodAncestral, #[cfg_attr(feature = "clap", clap(flatten))] + #[serde(flatten)] pub alphabet_args: AlphabetArgs, /// Use dense representation for sequences (store full probability distributions) @@ -248,6 +254,7 @@ pub struct TreetimeTimetreeArgsRaw { pub dense: Option, #[cfg_attr(feature = "clap", clap(flatten))] + #[serde(flatten)] pub gap_fill_args: GapFillArgs, /// Zero-based mutation indexing @@ -357,6 +364,7 @@ pub struct TreetimeTimetreeArgsRaw { pub output_coalescent_json: Option, #[cfg_attr(feature = "clap", clap(flatten))] + #[serde(flatten)] pub output: OutputCoreArgs, /// Comma-separated list of outputs to produce with `--output-all`. @@ -371,6 +379,7 @@ pub struct TreetimeTimetreeArgsRaw { pub output_selection: Vec, #[cfg_attr(feature = "clap", clap(flatten))] + #[serde(flatten)] pub topology_order: TopologyOrderArgs, /// Random seed