Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add VA/VRS GraphQL API #1426

Merged
merged 3 commits into from
Jun 24, 2024
Merged

Add VA/VRS GraphQL API #1426

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
747900b
Add VRS data to v4 variant data pipeline
phildarnowsky-broad May 23, 2024
83cb92c
Add VRS data to GraphQL API
phildarnowsky-broad Jun 17, 2024
ef8be18
Add some annotations to document new VA fields in API
phildarnowsky-broad Jun 24, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion data-pipeline/src/data_pipeline/data_types/variant/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
from .annotate_variants import annotate_variants, annotate_caids
from .annotate_variants import annotate_variants, annotate_caids, annotate_vrs_ids
from .transcript_consequence.annotate_transcript_consequences import annotate_transcript_consequences
from .variant_id import variant_id, variant_ids, compressed_variant_id

Expand All @@ -9,4 +9,5 @@
"variant_id",
"variant_ids",
"compressed_variant_id",
"annotate_vrs_ids",
]
28 changes: 28 additions & 0 deletions data-pipeline/src/data_pipeline/data_types/variant/annotate_variants.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,3 +23,31 @@ def annotate_caids(variants_path, caids_path=None):
ds = ds.annotate(caid=caids[ds.key].caid)

return ds


def annotate_vrs_ids(variants_path, exome_variants_path, genome_variants_path):
ds = hl.read_table(variants_path)
exomes = hl.read_table(exome_variants_path)
genomes = hl.read_table(genome_variants_path)
exome_vrs = exomes.select(vrs=exomes.info.vrs)
genome_vrs = genomes.select(vrs=genomes.info.vrs)
vrs = exome_vrs.union(genome_vrs)
vrs = vrs.group_by(vrs.locus, vrs.alleles).aggregate(vrs=hl.agg.collect(vrs.vrs)[0])
vrs = vrs.transmute(
vrs=hl.struct(
ref=hl.struct(
allele_id=vrs.vrs.VRS_Allele_IDs[0],
start=vrs.vrs.VRS_Starts[0],
end=vrs.vrs.VRS_Ends[0],
state=vrs.vrs.VRS_States[0],
),
alt=hl.struct(
allele_id=vrs.vrs.VRS_Allele_IDs[1],
start=vrs.vrs.VRS_Starts[1],
end=vrs.vrs.VRS_Ends[1],
state=vrs.vrs.VRS_States[1],
),
)
)
ds = ds.join(vrs)
return ds
2 changes: 1 addition & 1 deletion data-pipeline/src/data_pipeline/pipeline.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -102,7 +102,7 @@ def run(self, force=False):
elapsed = stop - start
logger.info("Finished %s in %dm%02ds", self._name, elapsed // 60, elapsed % 60)
else:
logger.info("Skipping %s", self._name)
logger.info(f"Skipping {self._name}")


@attr.define
Expand Down
3 changes: 2 additions & 1 deletion data-pipeline/src/data_pipeline/pipelines/export_to_elasticsearch.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -128,10 +128,11 @@ def add_liftover_document_id(ds):
"document_id",
"variant_id",
"rsids",
# "caid",
"caid",
"locus",
"transcript_consequences.gene_id",
"transcript_consequences.transcript_id",
"vrs.alt.allele_id",
Copy link
Contributor

@rileyhgrant rileyhgrant Jun 17, 2024 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Not directly related, but I realized that caid is still commented out a few lines above this.

Might be worth it to uncomment that before (as I think this is still before) we do the next full load of the variants, as the table has that information in it already.

],
"id_field": "document_id",
"num_shards": 48,
Expand Down
17 changes: 15 additions & 2 deletions data-pipeline/src/data_pipeline/pipelines/gnomad_v4_variants.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@
annotate_variants,
annotate_transcript_consequences,
annotate_caids,
annotate_vrs_ids,
)

RUN = True
Expand Down Expand Up @@ -90,11 +91,22 @@
},
)

pipeline.add_task(
name="annotate_vrs_ids",
task_function=annotate_vrs_ids,
output_path=f"{output_sub_dir}/gnomad_v4_variants_annotated_4.ht",
inputs={
"variants_path": pipeline.get_task("annotate_gnomad_v4_caids"),
"exome_variants_path": "gs://gcp-public-data--gnomad/release/4.1/ht/exomes/gnomad.exomes.v4.1.sites.ht",
"genome_variants_path": "gs://gcp-public-data--gnomad/release/4.1/ht/genomes/gnomad.genomes.v4.1.sites.ht",
},
)

###############################################
# Outputs
###############################################

pipeline.set_outputs({"variants": "annotate_gnomad_v4_caids"})
pipeline.set_outputs({"variants": "annotate_vrs_ids"})

###############################################
# Run
Expand All @@ -106,12 +118,13 @@

write_schemas(
[pipeline],
os.path.join("/home/msolomon", "schemas"),
os.path.expanduser("~/schemas"),
task_names=[
"prepare_gnomad_v4_variants",
"annotate_gnomad_v4_variants",
"annotate_gnomad_v4_transcript_consequences",
"annotate_gnomad_v4_caids",
"annotate_vrs_ids",
],
)
# copy locally using:
Expand Down
22 changes: 22 additions & 0 deletions dataset-metadata/metadata.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,7 @@ export type DatasetMetadata = {
structuralVariantDatasetId: DatasetId
copyNumberVariantDatasetId: DatasetId
hasJointFrequencyData: boolean
hasVRSData: boolean
}

const metadata: Record<DatasetId, DatasetMetadata> = {
Expand Down Expand Up @@ -124,6 +125,7 @@ const metadata: Record<DatasetId, DatasetMetadata> = {
copyNumberVariantDatasetId: 'gnomad_cnv_r4',
hasCopyNumberVariantCoverage: false,
hasJointFrequencyData: false,
hasVRSData: false,
},
gnomad_r2_1: {
isSubset: false,
Expand Down Expand Up @@ -173,6 +175,7 @@ const metadata: Record<DatasetId, DatasetMetadata> = {
copyNumberVariantDatasetId: 'gnomad_cnv_r4',
hasCopyNumberVariantCoverage: false,
hasJointFrequencyData: false,
hasVRSData: false,
},
gnomad_r2_1_controls: {
isSubset: true,
Expand Down Expand Up @@ -222,6 +225,7 @@ const metadata: Record<DatasetId, DatasetMetadata> = {
copyNumberVariantDatasetId: 'gnomad_cnv_r4',
hasCopyNumberVariantCoverage: false,
hasJointFrequencyData: false,
hasVRSData: false,
},
gnomad_r2_1_non_cancer: {
isSubset: true,
Expand Down Expand Up @@ -271,6 +275,7 @@ const metadata: Record<DatasetId, DatasetMetadata> = {
copyNumberVariantDatasetId: 'gnomad_cnv_r4',
hasCopyNumberVariantCoverage: false,
hasJointFrequencyData: false,
hasVRSData: false,
},
gnomad_r2_1_non_neuro: {
isSubset: true,
Expand Down Expand Up @@ -320,6 +325,7 @@ const metadata: Record<DatasetId, DatasetMetadata> = {
copyNumberVariantDatasetId: 'gnomad_cnv_r4',
hasCopyNumberVariantCoverage: false,
hasJointFrequencyData: false,
hasVRSData: false,
},
gnomad_r2_1_non_topmed: {
isSubset: true,
Expand Down Expand Up @@ -369,6 +375,7 @@ const metadata: Record<DatasetId, DatasetMetadata> = {
copyNumberVariantDatasetId: 'gnomad_cnv_r4',
hasCopyNumberVariantCoverage: false,
hasJointFrequencyData: false,
hasVRSData: false,
},
gnomad_r3: {
isSubset: false,
Expand Down Expand Up @@ -418,6 +425,7 @@ const metadata: Record<DatasetId, DatasetMetadata> = {
copyNumberVariantDatasetId: 'gnomad_cnv_r4',
hasCopyNumberVariantCoverage: false,
hasJointFrequencyData: false,
hasVRSData: false,
},
gnomad_r3_controls_and_biobanks: {
isSubset: true,
Expand Down Expand Up @@ -467,6 +475,7 @@ const metadata: Record<DatasetId, DatasetMetadata> = {
copyNumberVariantDatasetId: 'gnomad_cnv_r4',
hasCopyNumberVariantCoverage: false,
hasJointFrequencyData: false,
hasVRSData: false,
},
gnomad_r3_non_cancer: {
isSubset: true,
Expand Down Expand Up @@ -516,6 +525,7 @@ const metadata: Record<DatasetId, DatasetMetadata> = {
copyNumberVariantDatasetId: 'gnomad_cnv_r4',
hasCopyNumberVariantCoverage: false,
hasJointFrequencyData: false,
hasVRSData: false,
},
gnomad_r3_non_neuro: {
isSubset: true,
Expand Down Expand Up @@ -565,6 +575,7 @@ const metadata: Record<DatasetId, DatasetMetadata> = {
copyNumberVariantDatasetId: 'gnomad_cnv_r4',
hasCopyNumberVariantCoverage: false,
hasJointFrequencyData: false,
hasVRSData: false,
},
gnomad_r3_non_topmed: {
isSubset: true,
Expand Down Expand Up @@ -614,6 +625,7 @@ const metadata: Record<DatasetId, DatasetMetadata> = {
copyNumberVariantDatasetId: 'gnomad_cnv_r4',
hasCopyNumberVariantCoverage: false,
hasJointFrequencyData: false,
hasVRSData: false,
},
gnomad_r3_non_v2: {
isSubset: true,
Expand Down Expand Up @@ -663,6 +675,7 @@ const metadata: Record<DatasetId, DatasetMetadata> = {
copyNumberVariantDatasetId: 'gnomad_cnv_r4',
hasCopyNumberVariantCoverage: false,
hasJointFrequencyData: false,
hasVRSData: false,
},
gnomad_sv_r2_1: {
isSubset: false,
Expand Down Expand Up @@ -712,6 +725,7 @@ const metadata: Record<DatasetId, DatasetMetadata> = {
copyNumberVariantDatasetId: 'gnomad_cnv_r4',
hasCopyNumberVariantCoverage: false,
hasJointFrequencyData: false,
hasVRSData: false,
},
gnomad_sv_r2_1_controls: {
isSubset: true,
Expand Down Expand Up @@ -761,6 +775,7 @@ const metadata: Record<DatasetId, DatasetMetadata> = {
copyNumberVariantDatasetId: 'gnomad_cnv_r4',
hasCopyNumberVariantCoverage: false,
hasJointFrequencyData: false,
hasVRSData: false,
},
gnomad_sv_r2_1_non_neuro: {
isSubset: true,
Expand Down Expand Up @@ -810,6 +825,7 @@ const metadata: Record<DatasetId, DatasetMetadata> = {
copyNumberVariantDatasetId: 'gnomad_cnv_r4',
hasCopyNumberVariantCoverage: false,
hasJointFrequencyData: false,
hasVRSData: false,
},
gnomad_sv_r4: {
isSubset: false,
Expand Down Expand Up @@ -859,6 +875,7 @@ const metadata: Record<DatasetId, DatasetMetadata> = {
copyNumberVariantDatasetId: 'gnomad_cnv_r4',
hasCopyNumberVariantCoverage: false,
hasJointFrequencyData: true,
hasVRSData: false,
},
gnomad_cnv_r4: {
isSubset: false,
Expand Down Expand Up @@ -908,6 +925,7 @@ const metadata: Record<DatasetId, DatasetMetadata> = {
copyNumberVariantDatasetId: 'gnomad_cnv_r4',
hasCopyNumberVariantCoverage: true,
hasJointFrequencyData: true,
hasVRSData: false,
},
gnomad_r4: {
isSubset: false,
Expand Down Expand Up @@ -957,6 +975,7 @@ const metadata: Record<DatasetId, DatasetMetadata> = {
copyNumberVariantDatasetId: 'gnomad_cnv_r4',
hasCopyNumberVariantCoverage: false,
hasJointFrequencyData: true,
hasVRSData: true,
},
gnomad_r4_non_ukb: {
isSubset: true,
Expand Down Expand Up @@ -1006,6 +1025,7 @@ const metadata: Record<DatasetId, DatasetMetadata> = {
hasCopyNumberVariantCoverage: false,
hasRelatedVariants: true,
hasJointFrequencyData: false,
hasVRSData: true,
},
}

Expand Down Expand Up @@ -1156,3 +1176,5 @@ export const getTopLevelDataset = (datasetId: DatasetId) => {

return 'default'
}

export const hasVRSData = (datasetId: DatasetId) => getMetadata(datasetId, 'hasVRSData')
Loading
Loading