From 756b2628ea8d27a8088fa5fc66ca1ce3adeb17b7 Mon Sep 17 00:00:00 2001 From: Jerome Kelleher Date: Mon, 26 Oct 2026 20:17:08 +0000 Subject: [PATCH] Join ancestral state annotations to sources strictly The [ancestral_state] annotation was joined to the sources by position, silently treating ambiguous duplicate rows as unknown and only checking REF alleles at duplicate positions. Sources with inconsistent REF alleles were dropped with a warning. - [ancestral_state] accepts include, exclude, regions and targets filters, applied as for sources. - More than one annotation variant at a site after filtering is an error. - Any difference in REF allele at a site, between the sources or between the sources and the annotation, is an error naming the sources and position. Sources may have different ALT alleles. - Ancestral states ".", "N" and "" are unknown, and sites with an unknown ancestral allele are skipped by augment_sites rather than polarised by parsimony. --- CHANGELOG.md | 7 ++ docs/config.md | 16 ++++ tests/test_ancestors.py | 17 +++- tests/test_config.py | 25 +++++ tests/test_pipeline.py | 31 ++++++ tests/test_vcz.py | 188 +++++++++++++++++++++++++++---------- tsinfer/config.py | 29 +++++- tsinfer/pipeline.py | 33 +++---- tsinfer/vcz.py | 202 +++++++++++++++++++++++----------------- 9 files changed, 391 insertions(+), 157 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bbbfb2ca..4535ac3c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,13 @@ In development - Require Python >= 3.12 and vcztools >= 0.2 +- The `[ancestral_state]` annotation is joined to the sources strictly: it + accepts `include`, `exclude`, `regions` and `targets` filters; more than one + annotation variant at a site is an error, as is any difference in REF allele + at a site between the sources, or between the sources and the annotation + (sources may have different ALT alleles); and sites with a missing ancestral state + (`.`, `N` or empty) are excluded from both inference and `augment_sites`. + ## [0.5.1] - 2026-03-05 - Require tskit >= 1.0 diff --git a/docs/config.md b/docs/config.md index 1fed7727..2a48b178 100644 --- a/docs/config.md +++ b/docs/config.md @@ -52,6 +52,22 @@ Specifies where to read the ancestral allele for each variant position. | `path` | string | (required) | Path to VCZ containing ancestral alleles | | `field` | string | — | Array name in the store (e.g. `"variant_AA"`). Required unless `is_reference` is set. | | `is_reference` | bool | `false` | Use the REF allele (`variant_allele[:, 0]`) as the ancestral state. Useful for simulations. `field` must not be set when this is `true`. | +| `contig` | string | — | Name of the contig in this store's header (see the top-level `contig` option) | +| `include` | string | — | bcftools-style filter expression selecting annotation variants (e.g. `'TYPE="snp"'`) | +| `exclude` | string | — | bcftools-style filter expression excluding annotation variants | +| `regions` | string | — | Genomic region, half-open (e.g. `"chr20:1000-50000"`) | +| `targets` | string | — | Exact target positions | + +The annotation is joined to the sources by position. After filtering, there +must be at most one annotation variant at each site, and its REF allele must +equal the site's REF allele in the sources; otherwise an error is raised. All +sources with a variant at a site must also have the same REF allele there, +although their ALT alleles may differ. If +the annotation contains, for example, both a SNP and an indel at the same +position, use `include = 'TYPE="snp"'` to keep only the SNP. An ancestral +state of `.`, `N` or the empty string means the ancestral allele is unknown, +as does one that is not among the site's alleles; such sites are excluded from +inference and from `augment_sites`. ## `[[ancestors]]` diff --git a/tests/test_ancestors.py b/tests/test_ancestors.py index 461ba57a..5a0343d3 100644 --- a/tests/test_ancestors.py +++ b/tests/test_ancestors.py @@ -1409,18 +1409,18 @@ def test_two_sources_combined_triallelic_raises(self): with pytest.raises(ValueError, match="position 100 "): ancestors.infer_ancestors(sources, _cfg(), _anc_state(store_a)) - def test_two_sources_same_alleles(self): + def _two_sources(self, alleles_a, alleles_b): store_a = _haploid_store( [[0, 1, 0, 1]], positions=[100], - alleles=[["A", "T"]], + alleles=[alleles_a], anc_states=["A"], sample_ids=np.array(["a0", "a1", "a2", "a3"]), ) store_b = _haploid_store( [[0, 1, 1, 0]], positions=[100], - alleles=[["T", "A"]], + alleles=[alleles_b], anc_states=["A"], sample_ids=np.array(["b0", "b1", "b2", "b3"]), ) @@ -1428,9 +1428,18 @@ def test_two_sources_same_alleles(self): config.Source(path=store_a, name="a"), config.Source(path=store_b, name="b"), ] - anc = ancestors.infer_ancestors(sources, _cfg(), _anc_state(store_a)) + return sources, store_a + + def test_two_sources_same_alleles(self): + sources, ann = self._two_sources(["A", "T"], ["A", "T"]) + anc = ancestors.infer_ancestors(sources, _cfg(), _anc_state(ann)) np.testing.assert_array_equal(anc["variant_position"][:], [100]) + def test_two_sources_swapped_ref_raises(self): + sources, ann = self._two_sources(["A", "T"], ["T", "A"]) + with pytest.raises(ValueError, match="source 'b' has REF 'T'"): + ancestors.infer_ancestors(sources, _cfg(), _anc_state(ann)) + def test_unknown_ancestral_allele_skipped(self, caplog): gt = _haploid_store( [[0, 1, 0, 1], [0, 1, 1, 0], [0, 1, 2, 0]], diff --git a/tests/test_config.py b/tests/test_config.py index 8d82fcec..7744ad03 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -294,6 +294,21 @@ def test_ancestral_state(self, tmp_path): assert cfg.ancestral_state is not None assert cfg.ancestral_state.field == "variant_ancestral_allele" + def test_ancestral_state_filters(self, tmp_path): + toml = _STANDARD_TOML.replace( + 'field = "variant_ancestral_allele"\n', + 'field = "variant_ancestral_allele"\n' + "include = 'TYPE=\"snp\"'\n" + 'exclude = "POS == 5"\n' + 'regions = "20:1-1000"\n' + 'targets = "^20:7"\n', + ) + cfg = config.Config.from_toml(_write_toml(tmp_path, toml)) + assert cfg.ancestral_state.include == 'TYPE="snp"' + assert cfg.ancestral_state.exclude == "POS == 5" + assert cfg.ancestral_state.regions == "20:1-1000" + assert cfg.ancestral_state.targets == "^20:7" + def test_ancestors(self, tmp_path): cfg = config.Config.from_toml(_write_toml(tmp_path, _STANDARD_TOML)) assert len(cfg.ancestors) == 1 @@ -805,6 +820,16 @@ def test_includes_ancestral_state(self): assert "ann.vcz" in text assert "anc_allele" in text + def test_ancestral_state_filters(self): + cfg = _minimal_config( + ancestral_state=config.AncestralState( + path="ann.vcz", field="anc_allele", include='TYPE="snp"' + ), + ) + text = cfg.format() + assert 'include = TYPE="snp"' in text + assert "exclude" not in text.split("[ancestral_state]")[1].split("[")[0] + def test_includes_post_process(self): cfg = _minimal_config(post_process=config.PostProcessConfig()) text = cfg.format() diff --git a/tests/test_pipeline.py b/tests/test_pipeline.py index 6d4dc2c0..b74aad55 100644 --- a/tests/test_pipeline.py +++ b/tests/test_pipeline.py @@ -1704,6 +1704,37 @@ def test_adds_new_sites(self): aug_positions = set(aug_ts.sites_position) assert 200.0 in aug_positions + @pytest.mark.parametrize("ancestral", [".", "N", "", "C"]) + def test_skips_unknown_ancestral(self, ancestral, caplog): + """Sites with a missing ancestral state, or one not among the site's + alleles, are not added.""" + main_store, sing_store, _ = self._make_hand_constructed_stores() + ts, _ = _run_pipeline_no_augment(main_store) + ann_store = helpers.make_sample_vcz( + genotypes=np.zeros((3, 1, 1), dtype=np.int8), + positions=np.array([100, 200, 300], dtype=np.int32), + alleles=np.array([["A", "T"], ["A", "G"], ["C", "G"]]), + ancestral_state=np.array(["A", ancestral, "C"]), + sequence_length=1000, + ) + cfg = config.Config( + sources={ + "test": config.Source(path=main_store, name="test"), + "singletons": config.Source(path=sing_store, name="singletons"), + }, + ancestors=[], + match=config.MatchConfig( + sources={}, output="output.trees", reference_ts="ref.trees" + ), + augment_sites=config.AugmentSitesConfig(sources=["singletons"]), + ancestral_state=_anc_state(ann_store), + ) + with caplog.at_level(logging.INFO, logger="tsinfer"): + aug_ts = pipeline.augment_sites(ts, cfg) + assert aug_ts.num_sites == ts.num_sites + assert "skipping 1 sites with unknown ancestral allele" in caplog.text + assert "no new sites to add" in caplog.text + def test_skips_existing_sites(self): """Sites already in the TS are not duplicated.""" main_store, sing_store, ann_store = self._make_hand_constructed_stores() diff --git a/tests/test_vcz.py b/tests/test_vcz.py index 38440074..080e4d27 100644 --- a/tests/test_vcz.py +++ b/tests/test_vcz.py @@ -2408,61 +2408,94 @@ def test_unknown_sample(self, store): class TestLookupAncestralAlleles: - def _lookup(self, ann, positions, refs, is_reference=False): + def _lookup(self, ann, positions, is_reference=False, **filters): if is_reference: ancestral_state = config.AncestralState( - path=ann, is_reference=True, contig="1" + path=ann, is_reference=True, contig="1", **filters ) else: ancestral_state = config.AncestralState( - path=ann, field="variant_ancestral_allele", contig="1" + path=ann, field="variant_ancestral_allele", contig="1", **filters ) return vcz.lookup_ancestral_alleles( - ancestral_state, - np.array(positions, dtype=np.int32), - np.array(refs, dtype=object), - ).tolist() + ancestral_state, np.array(positions, dtype=np.int32) + ) def test_single_rows(self): - ann = _store([100, 200], [["A", "T"], ["C", "G"]], ancestral=["T", "."]) - assert self._lookup(ann, [100, 200], ["A", "C"]) == ["T", "."] + ann = _store([100, 200], [["A", "T"], ["C", "G"]], ancestral=["T", "G"]) + result = self._lookup(ann, [100, 200]) + assert result.alleles.tolist() == ["T", "G"] + assert result.refs.tolist() == ["A", "C"] def test_no_annotation(self): ann = _store([100], [["A", "T"]], ancestral=["A"]) - assert self._lookup(ann, [50, 100, 150], ["A", "A", "A"]) == ["", "A", ""] + result = self._lookup(ann, [50, 100, 150]) + assert result.alleles.tolist() == ["", "A", ""] + assert result.refs.tolist() == ["", "A", ""] def test_no_positions(self): ann = _store([100], [["A", "T"]], ancestral=["A"]) - assert self._lookup(ann, [], []) == [] + result = self._lookup(ann, []) + assert result.alleles.tolist() == [] + assert result.refs.tolist() == [] + + def test_missing_values_unknown(self, caplog): + ann = _store( + [100, 200, 300, 400], [["A", "T"]] * 4, ancestral=[".", "N", "", "T"] + ) + with caplog.at_level(logging.INFO, logger="tsinfer"): + result = self._lookup(ann, [100, 200, 300, 400]) + assert result.alleles.tolist() == ["", "", "", "T"] + assert result.refs.tolist() == ["A", "A", "A", "A"] + assert "3 with a missing ancestral state" in caplog.text + + def test_duplicates_raise(self): + ann = _store([100, 100], [["C", "G"], ["C", "T"]], ancestral=["G", "T"]) + with pytest.raises(ValueError, match="more than one variant at 1 position"): + self._lookup(ann, [100]) - def test_duplicates_resolved_by_ref(self): - # As in the Ensembl chr20 annotation at 10031018 + def test_duplicates_at_unused_position_ignored(self): ann = _store( [100, 100, 200], - [["CAC", "TAG"], ["C", "G", "T"], ["A", "T"]], - ancestral=["TAC", "T", "A"], + [["C", "G"], ["C", "T"], ["A", "T"]], + ancestral=["G", "T", "A"], ) - assert self._lookup(ann, [100, 200], ["C", "A"]) == ["T", "A"] - assert self._lookup(ann, [100], ["CAC"]) == ["TAC"] + assert self._lookup(ann, [200]).alleles.tolist() == ["A"] - def test_duplicates_agreeing(self): - ann = _store([100, 100], [["C", "G"], ["C", "T"]], ancestral=["T", "T"]) - assert self._lookup(ann, [100], ["C"]) == ["T"] + def test_snp_filter_removes_duplicate(self): + # As in the Ensembl annotation at GRCh37 20:10002611 + ann = _store( + [100, 100, 200], + [["ATTA", "A"], ["A", "G"], ["C", "T"]], + ancestral=["ATTA", "A", "C"], + ) + result = self._lookup(ann, [100, 200], include='TYPE="snp"') + assert result.alleles.tolist() == ["A", "C"] + assert result.refs.tolist() == ["A", "C"] - def test_duplicates_ambiguous(self, caplog): - ann = _store([100, 100], [["C", "G"], ["C", "T"]], ancestral=["G", "T"]) - with caplog.at_level(logging.INFO, logger="tsinfer"): - assert self._lookup(ann, [100], ["C"]) == [""] - assert "1 with ambiguous annotations" in caplog.text + def test_exclude_filter(self): + ann = _store([100, 200], [["A", "T"]] * 2, ancestral=["T", "T"]) + result = self._lookup(ann, [100, 200], exclude="POS == 200") + assert result.alleles.tolist() == ["T", ""] - def test_duplicates_without_matching_ref(self): - ann = _store([100, 100], [["CA", "C"], ["CAC", "C"]], ancestral=["C", "C"]) - assert self._lookup(ann, [100], ["G"]) == [""] + def test_regions_filter(self): + ann = helpers.add_region_index( + _store([100, 200, 300], [["A", "T"]] * 3, ancestral=["T"] * 3) + ) + result = self._lookup(ann, [100, 200, 300], regions="1:150-250") + assert result.alleles.tolist() == ["", "T", ""] + + def test_targets_filter(self): + ann = helpers.add_region_index( + _store([100, 200, 300], [["A", "T"]] * 3, ancestral=["T"] * 3) + ) + result = self._lookup(ann, [100, 200, 300], targets="^1:200") + assert result.alleles.tolist() == ["T", "", "T"] def test_is_reference(self): ann = _store([100, 100, 200], [["CAC", "C"], ["C", "T"], ["G", "A"]]) - result = self._lookup(ann, [100, 200], ["C", "G"], is_reference=True) - assert result == ["C", "G"] + result = self._lookup(ann, [100, 200], is_reference=True, include='TYPE="snp"') + assert result.alleles.tolist() == ["C", "G"] def test_contig_of_multi_contig_store(self): singles = [ @@ -2470,20 +2503,25 @@ def test_contig_of_multi_contig_store(self): _store([10, 20], [["A", "T"]] * 2, ancestral=["A", "A"]), ] ann = helpers.combine_contigs(singles, ["1", "2"], 1) - assert self._lookup(ann, [10, 20], ["A", "A"]) == ["T", "T"] + assert self._lookup(ann, [10, 20]).alleles.tolist() == ["T", "T"] class TestFilteredMultiSourceView: """MultiSourceView presents only the variants that pass source filters.""" - def _view(self, stores, ann=None, **filters): + def _view(self, stores, ann=None, ann_filters=None, **filters): sources = [ config.Source(path=store, name=f"s{j}", **filters.get(f"s{j}", {})) for j, store in enumerate(stores) ] if ann is None: ann = stores[0] - return _view(sources, _anc_field(ann), "1") + if ann_filters is None: + ann_filters = {} + ancestral_state = config.AncestralState( + path=ann, field="variant_ancestral_allele", **ann_filters + ) + return _view(sources, ancestral_state, "1") def test_num_sites_reflects_filters(self): store = _store([100, 200, 300], [["A", "T"]] * 3) @@ -2498,7 +2536,7 @@ def test_snp_kept_at_indel_position_in_two_sources(self): _store(_SNP_INDEL_POSITIONS, _SNP_INDEL_ALLELES, sample_ids=["b0", "b1"]), ] snp = {"include": 'TYPE="snp"'} - view = self._view(stores, s0=snp, s1=snp) + view = self._view(stores, ann_filters=snp, s0=snp, s1=snp) np.testing.assert_array_equal(view.positions, [100, 200]) assert view.ancestral_alleles.tolist() == ["A", "C"] variants = list(view.iter_genotypes()) @@ -2507,33 +2545,84 @@ def test_snp_kept_at_indel_position_in_two_sources(self): def test_filtered_alleles_not_in_allele_map(self): store = _store(_SNP_INDEL_POSITIONS, _SNP_INDEL_ALLELES) - view = self._view([store], s0={"include": 'TYPE="snp"'}) + snp = {"include": 'TYPE="snp"'} + view = self._view([store], ann_filters=snp, s0=snp) assert "ATTAAC" not in view.alleles.flatten().tolist() - def test_ref_conflict_dropped(self, caplog): + def test_unfiltered_duplicate_annotation_raises(self): + store = _store(_SNP_INDEL_POSITIONS, _SNP_INDEL_ALLELES) + with pytest.raises(ValueError, match="more than one variant"): + self._view([store], s0={"include": 'TYPE="snp"'}) + + def test_snp_given_snp_annotation_not_deletion(self): + # GRCh37 20:10002611: sources have the SNP only; the Ensembl + # annotation has a deletion and the SNP at the same position + store = _store([100], [["A", "G"]]) + ann = _store([100, 100], [["ATTA", "A"], ["A", "G"]], ancestral=["ATTA", "A"]) + view = self._view([store], ann=ann, ann_filters={"include": 'TYPE="snp"'}) + assert view.ancestral_alleles.tolist() == ["A"] + + def test_ref_mismatch_with_annotation_raises(self): + store = _store([100, 200], [["A", "G"], ["C", "T"]], sample_ids=["a0", "a1"]) + ann = _store([100, 200], [["A", "G"], ["G", "T"]], ancestral=["A", "T"]) + with pytest.raises(ValueError, match="REF allele mismatch at 1 site") as info: + self._view([store], ann=ann) + message = str(info.value) + assert "position 200" in message + assert "source 's0'" in message + assert "'C'" in message + assert "'G'" in message + + def test_ref_mismatch_names_first_source_with_site(self): + store_a = _store([100], [["A", "G"]], sample_ids=["a0", "a1"]) + store_b = _store([100, 200], [["A", "G"], ["C", "T"]], sample_ids=["b0", "b1"]) + ann = _store([100, 200], [["A", "G"], ["T", "C"]], ancestral=["A", "T"]) + with pytest.raises(ValueError, match="source 's1' has REF 'C'"): + self._view([store_a, store_b], ann=ann) + + def test_snp_and_indel_in_different_sources_raises(self): snps = _store([100, 200], [["A", "G"], ["C", "T"]], sample_ids=["a0", "a1"]) indel = _store( [100, 200], [["ATTAAC", "A"], ["C", "T"]], sample_ids=["b0", "b1"] ) - with caplog.at_level(logging.WARNING, logger="tsinfer"): - view = self._view([snps, indel]) - np.testing.assert_array_equal(view.positions, [200]) - assert "Dropping 1 site(s)" in caplog.text - assert "[100]" in caplog.text + with pytest.raises(ValueError, match="REF allele mismatch at 1 site") as info: + self._view([snps, indel], ann=snps) + message = str(info.value) + assert "position 100" in message + assert "source 's0' has REF 'A'" in message + assert "source 's1' has REF 'ATTAAC'" in message - def test_ref_conflict_dropped_when_alleles_overlap(self): - # The indel's ALT equals the SNP's REF, but its REF is not a SNP allele + def test_ref_mismatch_when_alleles_overlap_raises(self): + # The indel's ALT equals the SNP's REF snps = _store([100], [["A", "G"]], sample_ids=["a0", "a1"]) indel = _store([100], [["AT", "A"]], sample_ids=["b0", "b1"]) - assert self._view([snps, indel]).num_sites == 0 + with pytest.raises(ValueError, match="source 's1' has REF 'AT'"): + self._view([snps, indel]) - def test_swapped_alleles_not_a_conflict(self): + def test_swapped_ref_and_alt_raises(self): store_a = _store([100], [["A", "T"]], sample_ids=["a0", "a1"]) store_b = _store([100], [["T", "A"]], ancestral=["A"], sample_ids=["b0", "b1"]) + with pytest.raises(ValueError, match="source 's1' has REF 'T'"): + self._view([store_a, store_b]) + + def test_different_alt_alleles(self): + store_a = _store([100], [["A", "T"]], sample_ids=["a0", "a1"]) + store_b = _store([100], [["A", "G"]], sample_ids=["b0", "b1"]) view = self._view([store_a, store_b]) np.testing.assert_array_equal(view.positions, [100]) variants = list(view.iter_genotypes()) - np.testing.assert_array_equal(variants[0].genotypes, [0, 1, 1, 0]) + assert variants[0].alleles == ("A", "T", "G") + np.testing.assert_array_equal(variants[0].genotypes, [0, 1, 0, 2]) + + def test_different_numbers_of_alt_alleles(self): + store_a = _store([100], [["A", "T"]], sample_ids=["a0", "a1"]) + store_b = _store( + [100], [["A", "G", "C"]], genotypes=[[2, 1]], sample_ids=["b0", "b1"] + ) + view = self._view([store_a, store_b]) + variants = list(view.iter_genotypes()) + assert variants[0].alleles == ("A", "T", "G", "C") + np.testing.assert_array_equal(variants[0].genotypes, [0, 1, 3, 2]) def test_sources_filtered_differently(self): store_a = _store([100, 200], [["A", "T"]] * 2, sample_ids=["a0", "a1"]) @@ -2550,7 +2639,8 @@ def test_ancestral_not_among_alleles_is_unknown(self, ancestral, caplog): with caplog.at_level(logging.INFO, logger="tsinfer"): view = self._view([store]) assert view.ancestral_alleles.tolist() == ["T", ""] - expected = 0 if ancestral == "" else 1 + # Missing values are unknown before the among-alleles check + expected = 1 if ancestral == "G" else 0 assert f"{expected} site(s) with an ancestral allele that is not" in caplog.text variants = list(view.iter_genotypes()) # Site 100 is polarised on T; at site 200 code 0 is unknown @@ -2568,12 +2658,12 @@ def test_ancestral_among_alleles_of_other_source(self): assert variants[0].alleles == ("G", "A", "T") np.testing.assert_array_equal(variants[0].genotypes, [1, 2, 1, 0]) - def test_ancestral_from_duplicate_annotation_rows(self): + def test_ancestral_from_filtered_annotation_rows(self): store = _store([100], [["C", "T"]]) ann = _store( [100, 100], [["CAC", "TAG"], ["C", "G", "T"]], ancestral=["TAC", "T"] ) - view = self._view([store], ann=ann) + view = self._view([store], ann=ann, ann_filters={"include": 'TYPE="snp"'}) assert view.ancestral_alleles.tolist() == ["T"] def test_external_annotation_missing_positions(self): diff --git a/tsinfer/config.py b/tsinfer/config.py index 74361711..8a18d076 100644 --- a/tsinfer/config.py +++ b/tsinfer/config.py @@ -129,12 +129,20 @@ class AncestralState: *contig* names the contig in this store's header; if None, it is resolved from the top-level contig or the store itself (see ``vcz.resolve_contig``). + + *include*, *exclude*, *regions* and *targets* select the annotation + variants to use, as for a ``Source``. After filtering, at most one variant + may remain at each position used for inference. """ path: str | pathlib.Path field: str | None = None is_reference: bool | None = None contig: str | None = None + include: str | None = None + exclude: str | None = None + regions: str | None = None + targets: str | None = None def __post_init__(self): if self.is_reference is None and self.field is None: @@ -289,8 +297,10 @@ def format(self) -> str: lines.append("[ancestral_state]") lines.append(f" path = {self.ancestral_state.path}") lines.append(f" field = {self.ancestral_state.field}") - if self.ancestral_state.contig is not None: - lines.append(f" contig = {self.ancestral_state.contig}") + for key in ["contig", "include", "exclude", "regions", "targets"]: + value = getattr(self.ancestral_state, key) + if value is not None: + lines.append(f" {key} = {value}") lines.append("") for anc in self.ancestors: @@ -417,7 +427,16 @@ def from_toml(cls, path: str | pathlib.Path) -> Config: "contig", } -_KNOWN_ANCESTRAL_STATE_KEYS = {"path", "field", "is_reference", "contig"} +_KNOWN_ANCESTRAL_STATE_KEYS = { + "path", + "field", + "is_reference", + "contig", + "include", + "exclude", + "regions", + "targets", +} _KNOWN_ANCESTORS_KEYS = { "name", @@ -503,6 +522,10 @@ def _parse_ancestral_state(raw: dict) -> AncestralState: field=entry.get("field"), is_reference=entry.get("is_reference"), contig=entry.get("contig"), + include=entry.get("include"), + exclude=entry.get("exclude"), + regions=entry.get("regions"), + targets=entry.get("targets"), ) diff --git a/tsinfer/pipeline.py b/tsinfer/pipeline.py index 7c2ceb27..bc98a9eb 100644 --- a/tsinfer/pipeline.py +++ b/tsinfer/pipeline.py @@ -812,7 +812,7 @@ def augment_sites( excluded (same behaviour as ``infer_ancestors``). The ancestral allele is read from the ``[ancestral_state]`` config section and passed to ``map_mutations`` so that the output always uses the specified ancestral - state. + state. Sites with an unknown ancestral allele are skipped. """ aug_cfg = cfg.augment_sites if aug_cfg is None or len(aug_cfg.sources) == 0: @@ -846,6 +846,17 @@ def augment_sites( positions = view.positions_in_intervals(intervals) positions = view.filter_positions(ts.sites_position, positions) + # Sites with an unknown ancestral allele cannot be polarised + unknown = view.ancestral_alleles == "" + unknown_positions = view.positions[unknown] + is_unknown = np.isin(positions, unknown_positions) + num_unknown = int(np.sum(is_unknown)) + if num_unknown > 0: + logger.info( + "augment_sites: skipping %d sites with unknown ancestral allele", + num_unknown, + ) + positions = positions[~is_unknown] n_sites = len(positions) if n_sites == 0: @@ -876,24 +887,10 @@ def augment_sites( continue # Canonical code 0 = ancestral allele - anc_allele = alleles[0] if alleles else "" tree.seek(var.position) - if anc_allele and anc_allele != "": - ancestral_state, mutations = tree.map_mutations( - genotypes, alleles, ancestral_state=anc_allele - ) - else: - # No ancestral allele known; let parsimony choose - real_alleles = [a for a in alleles if a != ""] - code_to_real = {} - for c, a in enumerate(alleles): - if a != "": - code_to_real[c] = real_alleles.index(a) - remapped = np.full_like(genotypes, tskit.MISSING_DATA) - for i in range(len(genotypes)): - if genotypes[i] >= 0 and genotypes[i] in code_to_real: - remapped[i] = code_to_real[genotypes[i]] - ancestral_state, mutations = tree.map_mutations(remapped, real_alleles) + ancestral_state, mutations = tree.map_mutations( + genotypes, alleles, ancestral_state=alleles[0] + ) site_id = tables.sites.add_row( position=var.position, ancestral_state=ancestral_state ) diff --git a/tsinfer/vcz.py b/tsinfer/vcz.py index e5518188..2391df43 100644 --- a/tsinfer/vcz.py +++ b/tsinfer/vcz.py @@ -577,7 +577,7 @@ class SourceVariants: alleles: np.ndarray # (num_variants, max_alleles) allele strings -def _source_has_filters(source: config.Source) -> bool: +def _source_has_filters(source: config.Source | config.AncestralState) -> bool: return ( source.include is not None or source.exclude is not None @@ -586,7 +586,9 @@ def _source_has_filters(source: config.Source) -> bool: ) -def _region_chunk_plan(reader: vcztools.VczReader, source: config.Source) -> list: +def _region_chunk_plan( + reader: vcztools.VczReader, source: config.Source | config.AncestralState +) -> list: """ Return the vcztools chunk plan selecting the variants that pass the source's regions and targets filters. Only the variant chunks that @@ -615,7 +617,9 @@ def _region_chunk_plan(reader: vcztools.VczReader, source: config.Source) -> lis def _filtered_rows( - store: zarr.Group, source: config.Source, contig_rows: slice + store: zarr.Group, + source: config.Source | config.AncestralState, + contig_rows: slice, ) -> np.ndarray: """ Return the ascending row indexes of the variants in a store that pass the @@ -640,6 +644,22 @@ def _filtered_rows( return np.concatenate(parts) +def _selected_rows( + store: zarr.Group, + source: config.Source | config.AncestralState, + contig_rows: slice, +) -> np.ndarray: + """ + Return the ascending row indexes of the variants in *contig_rows* that + pass the source's filters. + """ + if not _source_has_filters(source): + return np.arange(contig_rows.start, contig_rows.stop, dtype=np.int64) + rows = _filtered_rows(store, source, contig_rows) + in_contig = (rows >= contig_rows.start) & (rows < contig_rows.stop) + return rows[in_contig] + + def select_source_variants(store: zarr.Group, source: config.Source) -> SourceVariants: """ Return the variants of the source on its contig (``source.contig``, as @@ -651,12 +671,7 @@ def select_source_variants(store: zarr.Group, source: config.Source) -> SourceVa contig_id = source.contig contig_rows = contig_variant_rows(store, contig_id) num_on_contig = contig_rows.stop - contig_rows.start - if _source_has_filters(source): - rows = _filtered_rows(store, source, contig_rows) - in_contig = (rows >= contig_rows.start) & (rows < contig_rows.stop) - rows = rows[in_contig] - else: - rows = np.arange(contig_rows.start, contig_rows.stop, dtype=np.int64) + rows = _selected_rows(store, source, contig_rows) num_passing = len(rows) if num_passing > 0: @@ -687,59 +702,76 @@ def select_source_variants(store: zarr.Group, source: config.Source) -> SourceVa return SourceVariants(rows=rows, positions=positions, alleles=alleles) +# Ancestral state values meaning that the ancestral allele is unknown +MISSING_ANCESTRAL_STATES = {".", "N", ""} + + +@dataclasses.dataclass +class AncestralAnnotation: + """The result of ``lookup_ancestral_alleles``.""" + + alleles: np.ndarray # (num_positions,) ancestral allele, "" if unknown + refs: np.ndarray # (num_positions,) annotation REF, "" if no annotation + + def lookup_ancestral_alleles( - ancestral_state: config.AncestralState, - positions: np.ndarray, - ref_alleles: np.ndarray, -) -> np.ndarray: + ancestral_state: config.AncestralState, positions: np.ndarray +) -> AncestralAnnotation: """ - Return the ancestral allele recorded at each of the specified positions - on the ancestral state store's contig (``ancestral_state.contig``) - ("" if unknown). Where the annotation store has several variants at a - position (e.g. a SNP and an indel), only those whose REF allele equals - the corresponding entry of *ref_alleles* are used; if these do not - give a single ancestral allele, it is unknown. + Return the ancestral allele and REF allele of the annotation variant at + each of the specified positions on the ancestral state store's contig + (``ancestral_state.contig``), using only the variants that pass the + ancestral state's filters. Raises a ValueError if more than one variant + remains at any of the positions. Ancestral alleles in + ``MISSING_ANCESTRAL_STATES`` are returned as "" (unknown). """ ann_store = open_store(ancestral_state.path) - ann_rows = contig_variant_rows(ann_store, ancestral_state.contig) - ann_positions = np.asarray(ann_store["variant_position"][ann_rows], dtype=np.int32) + contig_rows = contig_variant_rows(ann_store, ancestral_state.contig) + rows = _selected_rows(ann_store, ancestral_state, contig_rows) + all_positions = np.asarray( + ann_store["variant_position"][contig_rows], dtype=np.int32 + ) + ann_positions = all_positions[rows - contig_rows.start] lo = np.searchsorted(ann_positions, positions, side="left") hi = np.searchsorted(ann_positions, positions, side="right") counts = hi - lo - # Read the annotation rows at the requested positions only. The rows for - # position j are needed_rows[offsets[j]: offsets[j] + counts[j]]. - offsets = np.cumsum(counts) - counts - within = np.arange(np.sum(counts)) - np.repeat(offsets, counts) - needed_rows = np.repeat(lo, counts) + within + ann_rows.start + duplicated = counts > 1 + num_duplicated = int(np.sum(duplicated)) + if num_duplicated > 0: + first = np.flatnonzero(duplicated)[0] + first_rows = rows[lo[first] : hi[first]] + first_alleles = ann_store["variant_allele"].oindex[first_rows, :2] + raise ValueError( + f"The ancestral state store {ancestral_state.path} has more than one " + f"variant at {num_duplicated} position(s) used for inference (e.g. " + f"positions {positions[duplicated][:5].tolist()}; alleles at " + f"{int(positions[first])}: {first_alleles.tolist()}). Use the " + "include, exclude, regions or targets options of [ancestral_state] " + "to select one variant per position, e.g. include = 'TYPE=\"snp\"'." + ) + + found = counts == 1 + needed_rows = rows[lo[found]] if ancestral_state.is_reference: values = np.asarray(ann_store["variant_allele"].oindex[needed_rows, 0]) else: values = np.asarray(ann_store[ancestral_state.field].oindex[needed_rows]) - - result = np.full(len(positions), "", dtype=object) - single = counts == 1 - result[single] = values[offsets[single]] - - multi = np.flatnonzero(counts > 1) - num_ambiguous = 0 - if len(multi) > 0: - refs = np.asarray(ann_store["variant_allele"].oindex[needed_rows, 0]) - for j in multi: - k = slice(offsets[j], offsets[j] + counts[j]) - ref_matches = refs[k] == ref_alleles[j] - candidates = set(values[k][ref_matches].tolist()) - if len(candidates) == 1: - result[j] = candidates.pop() - else: - num_ambiguous += 1 + ann_refs = np.asarray(ann_store["variant_allele"].oindex[needed_rows, 0]) + + alleles = np.full(len(positions), "", dtype=object) + alleles[found] = values + refs = np.full(len(positions), "", dtype=object) + refs[found] = ann_refs + missing = np.isin(alleles, list(MISSING_ANCESTRAL_STATES)) + alleles[missing] = "" logger.info( - "Ancestral alleles: %d positions with no annotation, %d with ambiguous " - "annotations at duplicate positions", - int(np.sum(counts == 0)), - num_ambiguous, + "Ancestral alleles: %d positions with no annotation, %d with a missing " + "ancestral state", + int(np.sum(~found)), + int(np.sum(missing & found)), ) - return result + return AncestralAnnotation(alleles=alleles, refs=refs) # --------------------------------------------------------------------------- @@ -2614,11 +2646,13 @@ class MultiSourceView: Unified view over one or more VCZ sample stores for variant iteration. The view's sites are the union of the variants of each source on its - contig that pass the source's filters (see ``select_source_variants``), - excluding sites where the sources have inconsistent REF alleles. Each - site's ancestral allele is looked up in the ``ancestral_state`` store, - and is unknown ("") if it is not among the site's alleles in the - sources. Allele codes are fixed at construction: code 0 is the + contig that pass the source's filters (see ``select_source_variants``). + Each site's ancestral allele is looked up in the ``ancestral_state`` + store (see ``lookup_ancestral_alleles``). A ValueError is raised if the + sources, or the sources and the annotation, have different REF alleles + at any site; sources may have different ALT alleles. The ancestral + allele is unknown ("") if it is missing or not among the site's alleles + in the sources. Allele codes are fixed at construction: code 0 is the ancestral allele and codes 1+ are the other alleles. Each source and the ancestral state must have their ``contig`` set to @@ -2668,45 +2702,47 @@ def __init__(self, sources, ancestral_state): site_idx = np.searchsorted(all_positions, variants.positions) source_row[site_idx, src_idx] = np.arange(len(variants.positions)) - # --- Drop sites where the sources describe different variants --- - # The first source with a site defines its REF and alleles. Another - # source conflicts if its REF is not among these alleles, or the - # first source's REF is not among its alleles (e.g. a SNP and an - # indel at the same position). + self._positions = all_positions + self._source_row = source_row + + # --- REF alleles must agree across the sources --- + # The first source with a site defines its REF allele first_src = np.argmax(source_row >= 0, axis=1) - first_alleles = np.full( - (len(all_positions), max(v.alleles.shape[1] for v in self._variants)), - "", - dtype=object, - ) + refs = np.full(len(all_positions), "", dtype=object) for src_idx, variants in enumerate(self._variants): is_first = first_src == src_idx rows = source_row[is_first, src_idx] - first_alleles[is_first, : variants.alleles.shape[1]] = variants.alleles[rows] - refs = first_alleles[:, 0] - conflict = np.zeros(len(all_positions), dtype=bool) + refs[is_first] = variants.alleles[rows, 0] for src_idx, variants in enumerate(self._variants): has_site = source_row[:, src_idx] >= 0 - src_alleles = variants.alleles[source_row[has_site, src_idx]] - src_ref = src_alleles[:, 0] - ref_known = np.any(first_alleles[has_site] == src_ref[:, None], axis=1) - ref_shared = np.any(src_alleles == refs[has_site, None], axis=1) - conflict[has_site] |= ~(ref_known & ref_shared) - num_conflicts = int(np.sum(conflict)) - if num_conflicts > 0: - logger.warning( - "Dropping %d site(s) where sources have inconsistent REF alleles " - "(e.g. positions %s)", - num_conflicts, - all_positions[conflict][:5].tolist(), + src_refs = variants.alleles[source_row[has_site, src_idx], 0] + mismatch = np.flatnonzero(src_refs != refs[has_site]) + if len(mismatch) > 0: + site = np.flatnonzero(has_site)[mismatch[0]] + first_name = sources[first_src[site]].name + raise ValueError( + f"REF allele mismatch at {len(mismatch)} site(s), e.g. " + f"position {int(all_positions[site])}: source {first_name!r} " + f"has REF {refs[site]!r} but source {sources[src_idx].name!r} " + f"has REF {src_refs[mismatch[0]]!r}" + ) + + # --- REF alleles must agree with the ancestral state annotation --- + annotation = lookup_ancestral_alleles(ancestral_state, all_positions) + annotated = annotation.refs != "" + mismatch = np.flatnonzero(annotated & (annotation.refs != refs)) + if len(mismatch) > 0: + site = mismatch[0] + source_name = sources[first_src[site]].name + raise ValueError( + f"REF allele mismatch at {len(mismatch)} site(s), e.g. position " + f"{int(all_positions[site])}: source {source_name!r} has REF " + f"{refs[site]!r} but the ancestral state store " + f"{ancestral_state.path} has REF {annotation.refs[site]!r}" ) - keep = ~conflict - self._positions = all_positions[keep] - self._source_row = source_row[keep] - refs = refs[keep] # --- Ancestral alleles: unknown unless among the site's alleles --- - ancestral = lookup_ancestral_alleles(ancestral_state, self._positions, refs) + ancestral = annotation.alleles among_alleles = np.zeros(len(self._positions), dtype=bool) for src_idx, variants in enumerate(self._variants): has_site = self._source_row[:, src_idx] >= 0