add pipeline steps to prepare release browser hts

Author: rileyhgrant

data-pipeline/src/data_pipeline/data_types/gene.py

@@ -226,6 +226,16 @@ def import_hgnc(path):
     return ds
 
 
+def prepare_gene_table_for_release(genes_path, keep_mane_version_global_annotation):
+    ds = hl.read_table(genes_path)
+    if keep_mane_version_global_annotation:
+        globals_dict = ds.index_globals()
+        ds = ds.select_globals(mane_select_version=globals_dict["annotations"]["mane_select_transcript"]["version"])
+    else:
+        ds = ds.select_globals()
+    return ds
+
+
 def prepare_genes(gencode_path, hgnc_path, reference_genome):
     genes = import_gencode(gencode_path, reference_genome)
 

data-pipeline/src/data_pipeline/data_types/variant/transcript_consequence/annotate_transcript_consequences.py

@@ -85,7 +85,7 @@ def annotate_transcript_consequences(variants_path, transcripts_path, mane_trans
 
     if mane_transcripts_path:
         mane_transcripts = hl.read_table(mane_transcripts_path)
-        mane_transcripts_version = hl.eval(mane_transcripts.globals.version)
+        mane_select_transcripts_version = hl.eval(mane_transcripts.globals.version)
 
         mane_transcripts = hl.dict([(row.gene_id, row.drop("gene_id")) for row in mane_transcripts.collect()])
 
@@ -138,7 +138,7 @@ def annotate_transcript_consequences(variants_path, transcripts_path, mane_trans
         )
 
         ds = ds.annotate(transcript_consequences=transcript_consequences).drop("vep")
-        ds = ds.annotate_globals(mane_transcripts_version=mane_transcripts_version)
+        ds = ds.annotate_globals(mane_select_version=mane_select_transcripts_version)
 
     else:
         transcript_consequences = hl.sorted(

data-pipeline/src/data_pipeline/datasets/gnomad_v4/gnomad_v4_constraint.py

@@ -43,3 +43,9 @@ def prepare_gnomad_v4_constraint(path):
     ds = ds.key_by("transcript_id")
 
     return ds
+
+
+def remove_gnomad_v4_constraint(genes_path):
+    ds = hl.read_table(genes_path)
+    ds = ds.drop("gnomad_constraint")
+    return ds

data-pipeline/src/data_pipeline/datasets/gnomad_v4/gnomad_v4_variants.py

@@ -391,6 +391,17 @@ def freq_joint(ds, subset=None, pop=None, sex=None, raw=False):
     return ds
 
 
+def prepare_table_for_release(variants_table_path):
+    ds = hl.read_table(variants_table_path)
+    ds = ds.annotate(
+        exomes=ds.exomes.drop("faf95", "faf99"),
+        genomes=ds.genomes.drop("faf95", "faf99"),
+        joint=ds.joint.drop("faf99_joint", "faf95_joint"),
+    )
+    ds = ds.select_globals(mane_select_version=ds.globals.mane_transcripts_version)
+    return ds
+
+
 def prepare_gnomad_v4_variants(exome_variants_path: str, genome_variants_path: str, variants_joint_frequency_path: str):
     exome_variants = prepare_gnomad_v4_variants_helper(exome_variants_path, "exome")
     genome_variants = prepare_gnomad_v4_variants_helper(genome_variants_path, "genome")

data-pipeline/src/data_pipeline/pipelines/genes.py

@@ -4,7 +4,7 @@
 
 from data_pipeline.helpers import annotate_table
 
-from data_pipeline.data_types.gene import prepare_genes
+from data_pipeline.data_types.gene import prepare_genes, prepare_gene_table_for_release
 from data_pipeline.data_types.canonical_transcript import get_canonical_transcripts
 from data_pipeline.data_types.mane_select_transcript import import_mane_select_transcripts
 from data_pipeline.data_types.transcript import (
@@ -27,7 +27,10 @@
 )
 from data_pipeline.data_types.gene import reject_par_y_genes
 
-from data_pipeline.datasets.gnomad_v4.gnomad_v4_constraint import prepare_gnomad_v4_constraint
+from data_pipeline.datasets.gnomad_v4.gnomad_v4_constraint import (
+    prepare_gnomad_v4_constraint,
+    remove_gnomad_v4_constraint,
+)
 
 pipeline = Pipeline()
 
@@ -319,6 +322,17 @@ def annotate_with_preferred_transcript(table_path):
     },
 )
 
+# naming scheme follows methods naming scheme for consistency
+pipeline.add_task(
+    "prepare_grch37_genes_table_for_public_release",
+    prepare_gene_table_for_release,
+    f"/{genes_subdir}/gnomad.browser.GRCh37.GENCODEv19.ht",
+    {
+        "genes_path": pipeline.get_task("annotate_grch37_genes_step_5"),
+        "keep_mane_version_global_annotation": False,
+    },
+)
+
 pipeline.add_task(
     "annotate_grch38_genes_step_1",
     annotate_table,
@@ -373,6 +387,27 @@ def annotate_with_constraint(genes_path, constraint_path):
     },
 )
 
+pipeline.add_task(
+    "remove_grch38_genes_constraint_for_release",
+    remove_gnomad_v4_constraint,
+    f"/{genes_subdir}/genes_grch38_annotate_5_removed_constraint",
+    {
+        "genes_path": pipeline.get_task("annotate_grch38_genes_step_5"),
+    },
+)
+
+
+# naming scheme follows methods naming scheme for consistency
+pipeline.add_task(
+    "prepare_grch38_genes_table_for_public_release",
+    prepare_gene_table_for_release,
+    f"/{genes_subdir}/gnomad.browser.GRCh38.GENCODEv39.ht",
+    {
+        "genes_path": pipeline.get_task("remove_grch38_genes_constraint_for_release"),
+        "keep_mane_version_global_annotation": True,
+    },
+)
+
 ###############################################
 # Extract transcripts
 ###############################################

data-pipeline/src/data_pipeline/pipelines/gnomad_v4_variants.py

@@ -8,6 +8,7 @@
 
 from data_pipeline.datasets.gnomad_v4.gnomad_v4_variants import (
     prepare_gnomad_v4_variants,
+    prepare_table_for_release,
 )
 
 
@@ -102,6 +103,17 @@
     },
 )
 
+# removes several duplicated values, as well as constraint to prepare for release to the general public
+#   naming scheme follows methods naming scheme for consistency
+pipeline.add_task(
+    name="prepare_table_for_release",
+    task_function=prepare_table_for_release,
+    output_path=f"{output_sub_dir}/gnomad.browser.v4.1.sites.ht",
+    inputs={
+        "variants_table_path": pipeline.get_task("annotate_vrs_ids"),
+    },
+)
+
 ###############################################
 # Outputs
 ###############################################