feat(FTA-192): export gene Nomenclature History fields (etymology + gene change events)

src/AGR_data_retrieval_curation_gene.py

@@ -119,6 +119,13 @@ def main():
         curation_tsv.write_notes_tsv(
             filename=tsv_filename.replace('.tsv', '_notes.tsv'), entities=entities,
         )
+        curation_tsv.write_gene_change_events_tsv(
+            filename=tsv_filename.replace('.tsv', '_gene_change_events.tsv'), entities=entities,
+        )
+        curation_tsv.write_skipped_identity_source_tsv(
+            filename=tsv_filename.replace('.tsv', '_skipped_identity_source.tsv'),
+            skipped=gene_handler.skipped_identity_source,
+        )
         log.info(f'Generated TSV: {tsv_filename}')
 
     log.info('Ended main function.\n')

src/agr_datatypes.py

@@ -203,6 +203,7 @@ def __init__(self):
         self.gene_secondary_id_dtos = []        # Annotation IDs and 2o FlyBase IDs.
         # self.reference_curies = []              # Not yet part of LinkML, so not exported - should be added to LinkML model?
         self.note_dtos = []                     # Will be NoteDTO objects.
+        self.gene_change_event_dtos = []        # Will be GeneChangeEventSlotAnnotationDTO objects.
         self.gcrp_cross_reference_dto = None    # Will be a single CrossReferenceDTO object for UniProt/GCRP xref, if any.
         self.required_fields.extend(['gene_symbol_dto'])
 
@@ -661,6 +662,53 @@ def __init__(self, rel_type: str, component_symbol: str, taxon_curie: str, taxon
         self.required_fields.extend([])
 
 
+class ChangeEventSlotAnnotationDTO(SlotAnnotationDTO):
+    """ChangeEventSlotAnnotationDTO class."""
+    def __init__(self, event_type_name: str, evidence_curies: list):
+        """Create a ChangeEventSlotAnnotationDTO for a FlyBase entity change event.
+
+        Args:
+            event_type_name (str): The name of the VocabularyTerm for the change event category.
+            evidence_curies (list): A list of FB:FBrf or PMID:### curies.
+
+        Notes:
+            event_status_name and current_version were made optional in agr_curation_schema
+            PR #326. FlyBase has no nomenclature event status or versioning data to import, so
+            both are left as None and dropped from the export (see FTA-193).
+
+        """
+        super().__init__(evidence_curies)
+        self.event_type_name = event_type_name
+        self.event_status_name = None
+        self.current_version = None
+        self.note_dtos = []
+        self.symbol_renamed_to = None
+        self.symbol_renamed_from = None
+        self.full_name_renamed_to = None
+        self.full_name_renamed_from = None
+        self.acquires_in_merge_curies = []
+        self.merged_into_curie = None
+        self.split_from_curie = None
+        self.split_into_curies = []
+        self.required_fields.extend(['event_type_name'])
+
+
+class GeneChangeEventSlotAnnotationDTO(ChangeEventSlotAnnotationDTO):
+    """GeneChangeEventSlotAnnotationDTO class."""
+    def __init__(self, gene_identifier: str, event_type_name: str, evidence_curies: list):
+        """Create a GeneChangeEventSlotAnnotationDTO for a FlyBase gene change event.
+
+        Args:
+            gene_identifier (str): The curie/primary_external_id/mod_internal_id of the gene.
+            event_type_name (str): The name of the VocabularyTerm for the change event category.
+            evidence_curies (list): A list of FB:FBrf or PMID:### curies.
+
+        """
+        super().__init__(event_type_name, evidence_curies)
+        self.gene_identifier = gene_identifier
+        self.required_fields.extend(['gene_identifier'])
+
+
 class NameSlotAnnotationDTO(SlotAnnotationDTO):
     """NameSlotAnnotationDTO class."""
     def __init__(self, name_type_name: str, format_text: str, display_text: str, evidence_curies: list):

src/curation_tsv.py

@@ -22,7 +22,13 @@
 PRIMARY_TSV_HEADER = (
     "# Primary FBid\tValid symbol\tValid full name\tsecondary FBid(s)\tsynonyms\tinternal\n"
 )
-NOTES_TSV_HEADER = "# Primary FBid\ttype\tcomment\n"
+NOTES_TSV_HEADER = "# Primary FBid\ttype\tcomment\tevidence\n"
+GENE_CHANGE_EVENTS_TSV_HEADER = (
+    "# Primary FBid\tevent_type\tsymbol_renamed_from\tsymbol_renamed_to\tnote\tevidence\n"
+)
+SKIPPED_IDENTITY_SOURCE_TSV_HEADER = (
+    "# Primary FBid\traw_value\ttoken_count\tinternal\tobsolete\n"
+)
 COMPONENTS_TSV_HEADER = "# Primary FBid\tsymbol\trelation\ttaxon\tevidence\n"
 # NB: existing tool_uses TSVs write rows as primary, tools, evidence; the
 # header preserves that historic column order verbatim.
@@ -93,7 +99,53 @@ def write_notes_tsv(*, filename, entities):
                 continue
             primary = entity_dict["primary_external_id"]
             for note in entity_dict.get("note_dtos", []):
-                outfile.write(f"{primary}\t{note['note_type_name']}\t{note['free_text']}\n")
+                evidence = EVIDENCE_DELIMITER.join(note.get('evidence_curies', []))
+                outfile.write(f"{primary}\t{note['note_type_name']}\t{note['free_text']}\t{evidence}\n")
+
+
+def write_gene_change_events_tsv(*, filename, entities):
+    """Write the gene change events TSV (`gene_change_event_dtos` slot).
+
+    One row per change event. Rename events (from 'identity_source') fill the
+    symbol columns; nomenclature comment events fill the note column with the
+    inner note's free_text. Evidence curies are pipe-joined.
+    """
+    skip = should_skip_obsolete()
+    with open(filename, 'w') as outfile:
+        outfile.write(GENE_CHANGE_EVENTS_TSV_HEADER)
+        for entity_dict in entities:
+            if skip and _is_excluded(entity_dict):
+                continue
+            primary = entity_dict["primary_external_id"]
+            for event in entity_dict.get("gene_change_event_dtos", []):
+                event_type = event.get("event_type_name", "")
+                renamed_from = event.get("symbol_renamed_from", "")
+                renamed_to = event.get("symbol_renamed_to", "")
+                inner_notes = event.get("note_dtos", [])
+                note = inner_notes[0]["free_text"] if inner_notes else ""
+                evidence = EVIDENCE_DELIMITER.join(event.get("evidence_curies", []))
+                outfile.write(
+                    f"{primary}\t{event_type}\t{renamed_from}\t{renamed_to}\t{note}\t{evidence}\n"
+                )
+
+
+def write_skipped_identity_source_tsv(*, filename, skipped):
+    """Write the diagnostic TSV of skipped multi-token 'identity_source' props.
+
+    These are values that did not split into exactly two symbols (gene merges
+    with multiple old IDs, or values with embedded provenance sentences) and so
+    were not exported as rename events. One row per skipped prop. Unlike the
+    other writers this is NOT filtered by should_skip_obsolete(): obsolete/internal
+    rows are included so curators can check whether bad-syntax values sit on
+    obsolete FBgns.
+    """
+    with open(filename, 'w') as outfile:
+        outfile.write(SKIPPED_IDENTITY_SOURCE_TSV_HEADER)
+        for item in skipped:
+            outfile.write(
+                f"{item['fb_id']}\t{item['raw_value']}\t{item['token_count']}\t"
+                f"{item['internal']}\t{item['obsolete']}\n"
+            )
 
 
 def write_components_tsv(*, filename, entities, datatype):

src/gene_handler.py

@@ -12,10 +12,25 @@
 import csv
 import re
 from logging import Logger
+from harvdev_utils.char_conversions import clean_free_text
 import agr_datatypes
 import fb_datatypes
 from feature_handler import FeatureHandler
 
+# event_type_name VocabularyTerm for both 'identity_source' (G28b) and 'gene_nomenclature_comment'
+# (G41) gene change events: Steven confirmed both are 'rename' - G41 is a note specifically about
+# renames (FTA-193). event_status_name and current_version were made optional in schema PR #326 and
+# are omitted (FlyBase has no event status or versioning data to import).
+GENE_CHANGE_EVENT_TYPE_RENAME = 'rename'
+# Steven confirmed gene symbols never contain whitespace, so splitting the 'identity_source' value on
+# whitespace is safe; None => split on any run of whitespace. Values not yielding exactly two symbols
+# are skipped and logged for Steven to review (FTA-193).
+IDENTITY_SOURCE_DELIMITER = None
+# Every 'identity_source' featureprop value carries this leading boilerplate before the two symbols
+# (e.g. 'Source for identity of: Bar B'). It must be stripped before splitting, else the symbols are
+# never recovered and the rename event is silently skipped (FTA-192).
+IDENTITY_SOURCE_PREFIX = 'Source for identity of: '
+
 
 class GeneHandler(FeatureHandler):
     """This object gets, synthesizes and filters gene data for export."""
@@ -26,6 +41,7 @@ def __init__(self, log: Logger, testing: bool):
         self.fb_export_type = fb_datatypes.FBGene
         self.agr_export_type = agr_datatypes.GeneDTO
         self.primary_export_set = 'gene_ingest_set'
+        self.skipped_identity_source = []    # Multi-token "identity_source" props skipped during change event mapping.
 
     test_set = {
         'FBgn0284084': 'wg',                  # Current annotated nuclear protein_coding gene.
@@ -56,6 +72,7 @@ def __init__(self, log: Logger, testing: bool):
 
     gene_prop_to_note_mapping = {
         'misc': ('comment', 'note_dtos'),
+        'etymology': ('gene_etymology', 'note_dtos'),
     }
 
     # Elaborate on get_general_data() for the GeneHandler.
@@ -239,7 +256,10 @@ def map_gene_gcrp_xrefs(self):
             for xref in gene.dbxrefs:
                 if xref.dbxref.db.name != 'UniProt/GCRP':
                     continue
-                prefix = self.fb_agr_db_dict[xref.dbxref.db.name]
+                # 'UniProt/GCRP' is intentionally excluded from fb_agr_db_dict (it must only
+                # populate gcrp_cross_reference_dto, not the general xrefs - see handler.py), so
+                # hardcode its AGR prefix here rather than looking it up.
+                prefix = 'UniProtKB'
                 page_area = self.agr_page_area_dict[prefix]
                 curie = f'{prefix}:{xref.dbxref.accession}'
                 display_name = curie
@@ -268,6 +288,74 @@ def map_fb_data_to_alliance(self):
         self.flag_internal_fb_entities('fb_data_entities')
         self.flag_unexportable_genes()
         self.map_entity_props_to_notes('gene_prop_to_note_mapping')
+        self.map_gene_change_events()
+        return
+
+    def map_gene_change_events(self):
+        """Map 'Nomenclature History' featureprops to GeneChangeEventSlotAnnotationDTOs.
+
+        Handles two FlyBase gene featureprops (FTA-193 / FTA-192):
+        - 'gene_nomenclature_comment' (G41): one change event per comment, the comment carried as
+          an inner 'gene_nomenclature_note' NoteDTO in the event's note_dtos.
+        - 'identity_source' (G28b): the value holds two symbols (new then old) mapped to
+          symbol_renamed_to and symbol_renamed_from.
+
+        Both reuse props_by_type (already loaded by get_entityprops, no new query) and derive
+        evidence from each prop's pubs via lookup_pub_curies(). Both use event_type_name 'rename'
+        (confirmed by Steven, FTA-193); event_status_name and current_version are omitted (made
+        optional in schema PR #326, no FlyBase data to import).
+
+        """
+        self.log.info('Map gene "Nomenclature History" props to Alliance gene change events.')
+        nomen_comment_counter = 0
+        identity_source_counter = 0
+        skipped_identity_source_counter = 0
+        for gene in self.fb_data_entities.values():
+            if gene.linkmldto is None:
+                continue
+            gene_id = gene.linkmldto.primary_external_id
+            # 'gene_nomenclature_comment' -> one change event per comment, comment held as inner note.
+            nomen_notes = self.convert_prop_to_note(gene, 'gene_nomenclature_comment', 'gene_nomenclature_note')
+            for note in nomen_notes:
+                change_event = agr_datatypes.GeneChangeEventSlotAnnotationDTO(
+                    gene_id,
+                    GENE_CHANGE_EVENT_TYPE_RENAME,
+                    note.get('evidence_curies', []),
+                )
+                change_event.note_dtos = [note]
+                gene.linkmldto.gene_change_event_dtos.append(change_event.dict_export())
+                nomen_comment_counter += 1
+            # 'identity_source' -> symbol_renamed_to (new, first) and symbol_renamed_from (old, second).
+            for fb_prop in gene.props_by_type.get('identity_source', []):
+                raw_value = fb_prop.chado_obj.value
+                if raw_value.startswith(IDENTITY_SOURCE_PREFIX):
+                    raw_value = raw_value[len(IDENTITY_SOURCE_PREFIX):]
+                symbols = [clean_free_text(part) for part in raw_value.split(IDENTITY_SOURCE_DELIMITER) if part.strip()]
+                if len(symbols) != 2:
+                    self.log.warning(f'Skipping "identity_source" prop for {gene_id} - expected 2 symbols, '
+                                     f'got {len(symbols)} from value: "{raw_value}"')
+                    self.skipped_identity_source.append({
+                        'fb_id': gene_id,
+                        'raw_value': raw_value,
+                        'token_count': len(symbols),
+                        'internal': gene.linkmldto.internal,
+                        'obsolete': gene.linkmldto.obsolete,
+                    })
+                    skipped_identity_source_counter += 1
+                    continue
+                pub_curies = self.lookup_pub_curies(fb_prop.pubs)
+                change_event = agr_datatypes.GeneChangeEventSlotAnnotationDTO(
+                    gene_id,
+                    GENE_CHANGE_EVENT_TYPE_RENAME,
+                    pub_curies,
+                )
+                change_event.symbol_renamed_to = symbols[0]
+                change_event.symbol_renamed_from = symbols[1]
+                gene.linkmldto.gene_change_event_dtos.append(change_event.dict_export())
+                identity_source_counter += 1
+        self.log.info(f'Mapped {nomen_comment_counter} "gene_nomenclature_comment" change events.')
+        self.log.info(f'Mapped {identity_source_counter} "identity_source" change events '
+                      f'({skipped_identity_source_counter} skipped due to unexpected value format).')
         return
 
     # Elaborate on query_chado_and_export() for the GeneHandler.