From 2c87873ec8d18c05cd4345f7985d458d9277eaaa Mon Sep 17 00:00:00 2001 From: Dave Lawrence Date: Tue, 6 Aug 2024 21:46:15 +0930 Subject: [PATCH] issue SACGF/variantgrid_private#3676 - do some migrations to solve problem --- ...remind_rematch_allele_info_bad_end_norm.py | 39 +++++++++++++++++++ manual/operations/manual_operations.py | 6 ++- .../0141_one_off_fix_variant_end2.py | 22 +++++++++++ ...f_delete_bad_end_bcftools_symbolic_norm.py | 32 +++++++++++++++ 4 files changed, 98 insertions(+), 1 deletion(-) create mode 100644 classification/migrations/0153_one_off_remind_rematch_allele_info_bad_end_norm.py create mode 100644 snpdb/migrations/0141_one_off_fix_variant_end2.py create mode 100644 upload/migrations/0022_one_off_delete_bad_end_bcftools_symbolic_norm.py diff --git a/classification/migrations/0153_one_off_remind_rematch_allele_info_bad_end_norm.py b/classification/migrations/0153_one_off_remind_rematch_allele_info_bad_end_norm.py new file mode 100644 index 000000000..9bf15c202 --- /dev/null +++ b/classification/migrations/0153_one_off_remind_rematch_allele_info_bad_end_norm.py @@ -0,0 +1,39 @@ +# Generated by Django 4.2.11 on 2024-08-06 11:08 + +from django.db import migrations +from django.db.models import Q + +from manual.operations.manual_operations import ManualOperation + + +def _get_iai_qs(apps): + Variant = apps.get_model("snpdb", "Variant") + ImportedAlleleInfo = apps.get_model("classification", "ImportedAlleleInfo") + bad_norm_qs = Variant.objects.filter(svlen__isnull=False, modifiedimportedvariant__isnull=False) + return ImportedAlleleInfo.objects.filter(Q(grch37__variant__in=bad_norm_qs) | Q(grch38__variant__in=bad_norm_qs)) + + +def _needs_hard_rematch(apps): + qs = _get_iai_qs(apps) + return qs.exists() + + +def _get_note(apps): + qs = _get_iai_qs(apps) + instructions = None + if imported_allele_ids := list(qs.values_list("pk", flat=True)): + iais = ", ".join((str(pk) for pk in imported_allele_ids)) + instructions = f"Go to /admin/classification/importedalleleinfo/ and select {iais}" + return instructions + + +class Migration(migrations.Migration): + dependencies = [ + ("classification", "0152_allele_origin_confirmed_ekey"), + ("snpdb", "0118_reminder_one_off_fix_variant_end"), + ] + + operations = [ + ManualOperation.operation_other(args="AFTER fix_variant_end - Rematch hard imported alleles to badly normalized symbolic variants. ", + note=_get_note, test=_needs_hard_rematch), + ] diff --git a/manual/operations/manual_operations.py b/manual/operations/manual_operations.py index 64f16b1a1..f7e928d0a 100644 --- a/manual/operations/manual_operations.py +++ b/manual/operations/manual_operations.py @@ -56,7 +56,11 @@ def run(self, apps, reverse=False): ManualMigrationTask.objects.filter(pk=self.task_id).delete() else: task, _ = ManualMigrationTask.objects.get_or_create(pk=self.task_id) - ManualMigrationRequsted.objects.create(task=task, note=self.note) + if callable(self.note): + note = self.note(apps) + else: + note = self.note + ManualMigrationRequsted.objects.create(task=task, note=note) @staticmethod def operation_manage(args: list[str], note: Optional[str] = None, test: Callable = None): diff --git a/snpdb/migrations/0141_one_off_fix_variant_end2.py b/snpdb/migrations/0141_one_off_fix_variant_end2.py new file mode 100644 index 000000000..4c1d4194e --- /dev/null +++ b/snpdb/migrations/0141_one_off_fix_variant_end2.py @@ -0,0 +1,22 @@ +# Generated by Django 4.2.11 on 2024-08-06 10:59 + +from django.db import migrations + +from manual.operations.manual_operations import ManualOperation + + +def _existing_deploy_has_variants(apps): + Variant = apps.get_model("snpdb", "Variant") + return Variant.objects.exists() + + +class Migration(migrations.Migration): + dependencies = [ + ("snpdb", "0140_case_insensitive_collation"), + ] + + operations = [ + ManualOperation(task_id=ManualOperation.task_id_manage(["one_off_fix_variant_end"]), + test=_existing_deploy_has_variants), + + ] diff --git a/upload/migrations/0022_one_off_delete_bad_end_bcftools_symbolic_norm.py b/upload/migrations/0022_one_off_delete_bad_end_bcftools_symbolic_norm.py new file mode 100644 index 000000000..cfda0ee30 --- /dev/null +++ b/upload/migrations/0022_one_off_delete_bad_end_bcftools_symbolic_norm.py @@ -0,0 +1,32 @@ +# Generated by Django 4.2.11 on 2024-08-06 12:04 +import json +import os + +from django.conf import settings +from django.db import migrations + +from library.utils import mk_path + + +def _delete_bad_modified_imported_variants(apps, schema_editor): + ModifiedImportedVariant = apps.get_model("upload", "ModifiedImportedVariant") + qs = ModifiedImportedVariant.objects.all().filter(variant__svlen__isnull=False) + if miv_values := list(qs.values()): + migrations_dir = os.path.join(settings.PRIVATE_DATA_ROOT, "migrations") + mk_path(migrations_dir) + bad_modifications_filename = os.path.join(migrations_dir, "bad_modifications_filename.json") + with open(bad_modifications_filename, "w") as f: + json.dump(miv_values, f) + print(f"Wrote {len(miv_values)} incorretly normalized symbolic variants to '{bad_modifications_filename}'") + qs.delete() + + +class Migration(migrations.Migration): + dependencies = [ + ("upload", "0021_remove_uploadedfile_md5_hash"), + ("classification", "0153_one_off_remind_rematch_allele_info_bad_end_norm") + ] + + operations = [ + migrations.RunPython(_delete_bad_modified_imported_variants) + ]