diff --git a/packages/scripts/import/data/example-v4-senses/example-v4-senses.csv b/packages/scripts/import/data/example-v4-senses/example-v4-senses.csv index cb4f919e4..155f6c2d9 100644 --- a/packages/scripts/import/data/example-v4-senses/example-v4-senses.csv +++ b/packages/scripts/import/data/example-v4-senses/example-v4-senses.csv @@ -1,4 +1,4 @@ -lexeme,variant,es_gloss,partOfSpeech ,vernacular_exampleSentence ,es_exampleSentence ,s2.es_gloss,s2.partOfSpeech,s2.default_vernacular_exampleSentence,s2.es_exampleSentence,s3.es_gloss,s3.partOfSpeech,s3.default_vernacular_exampleSentence,s3.es_exampleSentence,s4.es_gloss,s4.partOfSpeech,s4.default_vernacular_exampleSentence,s4.es_exampleSentence,notes +lexeme,variant,es_gloss,partOfSpeech,default_vernacular_exampleSentence,es_exampleSentence,s2.es_gloss,s2.partOfSpeech,s2.default_vernacular_exampleSentence,s2.es_exampleSentence,s3.es_gloss,s3.partOfSpeech,s3.default_vernacular_exampleSentence,s3.es_exampleSentence,s4.es_gloss,s4.partOfSpeech,s4.default_vernacular_exampleSentence,s4.es_exampleSentence,notes kꞌahkꞌal,kꞌajkꞌal,sol,n,Lokꞌix tal kꞌahkꞌal,Ya salió el sol,fiebre,n,Ay ta kꞌahkꞌal te chꞌin alale,El niño tiene fiebre,día,n,Cheb kꞌahkꞌal ya x-aꞌtejotik,Trabajaremos dos días,calor,n,Toyol kꞌahkꞌal ya kaꞌiy,Siento mucho calor,16/jul./2019. Bachajon kꞌaal,kꞌahkꞌal,sol,n,Jaꞌnix lek-a lokꞌix tel kꞌaal,"Que bueno, ya salió el sol",fiebre,n,Ay bayal skꞌaal te chꞌin x-Ixchele,Mi hijita Ixchel tiene mucha fiebre,día,n,"Bajtix kaal mamtik, yorailix ich lewa","Ya transcurrió el día mi estimado señor, es momento de tomar un descanso",calor,n,Toyol kꞌaal ya jkaꞌiy,Siento mucho calor,26/dic./2020 kꞌajkꞌal,kꞌahkꞌal,sol,n,,,día,n,,,calor,n,,,fiebre,n,,,14/dic./2019 diff --git a/packages/scripts/import/data/example-v4/example-v4.csv b/packages/scripts/import/data/example-v4/example-v4.csv index d3d72f636..50377fb62 100644 --- a/packages/scripts/import/data/example-v4/example-v4.csv +++ b/packages/scripts/import/data/example-v4/example-v4.csv @@ -1,10 +1,10 @@ -lexeme,phonetic,localOrthography,morphology,interlinearization,en_gloss,es_gloss,scientificName,partOfSpeech,,dialects,semanticDomain,,semanticDomain.2,,semanticDomain_custom,notes,soundFile,speakerName,speakerAge,speakerGender,speakerHometown,photoFile,vernacular_exampleSentence,en_exampleSentence,es_exampleSentence,source,pluralForm -(word/phrase),,,,,English,Spanish,,abbrev.,,,key,,key,,"(try not to use, separate multiples with a |)",,,,,,,,,English,Spanish,, -voiture,vwatyʁ,,,,car,auto,,"n,v",noun,Modern Parisian French,5.15,Transportation,5,Daily life,vehicle|cars,small automobile,helloworld.mp3,Celine Dorion,43,f,"Paris, France",mountain.jpg,Je conduis ma voiture,I drive my car,Conduzco mi auto,, -arbre,aʁbʁ,,,,tree,árbol,Acer rubrum,"n, adj",noun,"Modern Parisian French, Quebec French",1.4,"Plants, trees and other vegetation",1.2,"Earth, geology and landscape",,generic term for all kinds of trees,missing.mp3,Celine Dorion,43,f,"Paris, France",,L'arbre nous donne de l'ombre,The tree gives us shade,El árbol nos da sombra,, -tube,tyb,,,,tube,tubo,,n,noun,Modern Parisian French,5.9,Tools and weapons,,,plumbing,a cylindrical device for liquids,,,,,,missing.jpg,L'eau passe à travers les tubes,The water goes through the tubes,El agua pasa a través de los tubos,,tubes -voiture,vwɑtYʁ,,,,car,auto,,n,noun,Quebec French,5.15,Transportation,,,vehicle,small automobile,,,,,,,Je conduis ma voiture,I drive my car,Conduzco mi auto,testing sources, -neutre,nøʏ̯tʁ̥,,,,neutral,neutro,,adj,adjective,Quebec French,,,,,,,0005-neutre.mp3,David Boucher,31,m,"Montreal, Canada",,Ma chambre est peinte d'une couleur neutre.,My room is painted with a neutral color.,Mi habitación está pintada con un color neutro.,, -fêter,fɛɪ̯te,,,,to celebrate,celebrar,,v,verb,Quebec French,,,,,,to have a party,0006-feter.mp3,David Boucher,31,m,"Montreal, Canada",,On va vraiment fêter à soir,We will really party tonight,Vamos a celebrar esta noche,"test source|with multiples sources, test|https://example.com", -njakulaba,,,n-ja-ku-lab-a,1SG-Fut-2SG-see-Fin.V,I will see you,Voy a verte,,vp,verb phrase,Central Luganda,,,,,,,751-I-will-see-you.mp3,EB,50,m,,,,,,, -vale,,,,,bye,adiós,,,,,,,,,,,,,,,,,,,,, \ No newline at end of file +lexeme,phonetic,localOrthography,morphology,interlinearization,en_gloss,es_gloss,scientificName,partOfSpeech,,partOfSpeech.2,,dialects,semanticDomain,,semanticDomain.2,,notes,soundFile,speakerName,speakerAge,speakerGender,speakerHometown,photoFile,default_vernacular_exampleSentence,en_exampleSentence,es_exampleSentence,source,pluralForm +(word/phrase),,,,,English,Spanish,,abbrev.,,abbrev.,,,key,,key,,,,,,,,,,English,Spanish,, +voiture,vwatyʁ,,,,car,auto,,n,noun,v,verb,Modern Parisian French,5.15,Transportation,5,Daily life,small automobile,helloworld.mp3,Celine Dorion,43,f,"Paris, France",mountain.jpg,Je conduis ma voiture,I drive my car,Conduzco mi auto,, +arbre,aʁbʁ,,,,tree,árbol,Acer rubrum,n,noun,adj,adjective,Modern Parisian French | Quebec French,1.4,"Plants, trees and other vegetation",1.2,"Earth, geology and landscape",generic term for all kinds of trees,missing.mp3,Celine Dorion,43,f,"Paris, France",,L'arbre nous donne de l'ombre,The tree gives us shade,El árbol nos da sombra,, +tube,tyb,,,,tube,tubo,,n,noun,,,Modern Parisian French,5.9,Tools and weapons,,,a cylindrical device for liquids,,,,,,missing.jpg,L'eau passe à travers les tubes,The water goes through the tubes,El agua pasa a través de los tubos,,tubes +voiture,vwɑtYʁ,,,,car,auto,,n,noun,,,Quebec French,5.15,Transportation,,,small automobile,,,,,,,Je conduis ma voiture,I drive my car,Conduzco mi auto,testing sources, +neutre,nøʏ̯tʁ̥,,,,neutral,neutro,,adj,adjective,,,Quebec French,,,,,,0005-neutre.mp3,David Boucher,31,m,"Montreal, Canada",,Ma chambre est peinte d'une couleur neutre.,My room is painted with a neutral color.,Mi habitación está pintada con un color neutro.,, +fêter,fɛɪ̯te,,,,to celebrate,celebrar,,v,verb,,,Quebec French,,,,,to have a party,0006-feter.mp3,David Boucher,31,m,"Montreal, Canada",,On va vraiment fêter à soir,We will really party tonight,Vamos a celebrar esta noche,"test source|with multiples sources, test|https://example.com", +njakulaba,,,n-ja-ku-lab-a,1SG-Fut-2SG-see-Fin.V,I will see you,Voy a verte,,vp,verb phrase,,,Central Luganda,,,,,,751-I-will-see-you.mp3,EB,50,m,,,,,,, +vale,,,,,bye,adiós,,,,,,,,,,,,,,,,,,,,,, \ No newline at end of file diff --git a/packages/scripts/import/generate-sql-statements.ts b/packages/scripts/import/generate-sql-statements.ts index e5edf127a..4ffcdf5be 100644 --- a/packages/scripts/import/generate-sql-statements.ts +++ b/packages/scripts/import/generate-sql-statements.ts @@ -7,9 +7,11 @@ import { sql_file_string } from './to-sql-string' import { millisecond_incrementing_timestamp } from './incrementing-timestamp' export interface Upload_Operations { - upload_photo: (filepath: string, entry_id: string) => Promise<{ storage_path: string, serving_url: string }> - upload_audio: (filepath: string, entry_id: string) => Promise<{ storage_path: string }> - // upload_video: (filepath: string) => Promise<{ storage_path: string }> + upload_photo: (filepath: string, entry_id: string) => Promise< + { storage_path: string, serving_url: string, error: null } | { storage_path: null, serving_url: null, error: string } + > + upload_audio: (filepath: string, entry_id: string) => Promise<{ storage_path: string, error: null } | { storage_path: null, error: string }> + // upload_video: (filepath: string) => Promise<{ storage_path: string, error: null } | { storage_path: null, error: string }> } export async function generate_sql_statements({ @@ -38,14 +40,17 @@ export async function generate_sql_statements({ const entry_id = randomUUID() - const c_meta = { + const c_meta = () => ({ created_by: diego_ld_user_id, created_at: millisecond_incrementing_timestamp(), - } - const c_u_meta = { - ...c_meta, - updated_by: c_meta.created_by, - updated_at: c_meta.created_at, + }) + const c_u_meta = () => { + const meta = c_meta() + return { + ...meta, + updated_by: meta.created_by, + updated_at: meta.created_at, + } } const assemble_content_update = ({ data, ...rest }: ImportContentUpdate) => { const data_without_meta = { ...data } @@ -66,8 +71,8 @@ export async function generate_sql_statements({ id: randomUUID(), import_id, dictionary_id, - user_id: c_meta.created_by, - timestamp: c_meta.created_at, + user_id: c_meta().created_by, + timestamp: c_meta().created_at, data: data_without_meta, } return content_update @@ -76,7 +81,7 @@ export async function generate_sql_statements({ const entry: TablesInsert<'entries'> = { id: entry_id, dictionary_id, - ...c_u_meta, + ...c_u_meta(), lexeme: { default: row.lexeme, ...(row.localOrthography && { lo1: row.localOrthography }), @@ -104,7 +109,7 @@ export async function generate_sql_statements({ dialect_id = randomUUID() const dialect: TablesInsert<'dialects'> = { id: dialect_id, - ...c_u_meta, + ...c_u_meta(), dictionary_id, name: { default: dialect_to_assign }, } @@ -113,7 +118,7 @@ export async function generate_sql_statements({ } sql_statements += sql_file_string('entry_dialects', { - ...c_meta, + ...c_meta(), dialect_id, entry_id, }) @@ -128,7 +133,7 @@ export async function generate_sql_statements({ tag_id = randomUUID() const tag: TablesInsert<'tags'> = { id: tag_id, - ...c_u_meta, + ...c_u_meta(), dictionary_id, name: tag_to_assign, } @@ -137,7 +142,7 @@ export async function generate_sql_statements({ } sql_statements += sql_file_string('entry_tags', { - ...c_meta, + ...c_meta(), tag_id, entry_id, }) @@ -151,12 +156,13 @@ export async function generate_sql_statements({ const row_entries = (Object.entries(row) as [keyof Row, string][]) .sort(([keyA], [keyB]) => keyA.localeCompare(keyB)) + type Sense_Label = 's1' | 's2' | 's3' | 's4'// etc, for type-safety we just need a few here const first_sense_label = 's1' - const sense_labels = new Set([first_sense_label]) // always have at least one sense + const sense_labels = new Set([first_sense_label]) // always have at least one sense const sense_regex = /^(?s\d+)\./ for (const key of Object.keys(row)) { const match = key.match(sense_regex) - if (match) sense_labels.add(match.groups.sense_index) + if (match) sense_labels.add(match.groups.sense_index as Sense_Label) } for (const sense_label of sense_labels) { @@ -164,7 +170,7 @@ export async function generate_sql_statements({ const sense: TablesInsert<'senses'> = { entry_id, - ...c_u_meta, + ...c_u_meta(), id: sense_id, glosses: {}, } @@ -202,7 +208,7 @@ export async function generate_sql_statements({ } } - senses.push(sense) + if (sense_label === 's1' || Object.keys(sense.glosses).length > 0) senses.push(sense) //* It only adds an additional sense if it has any glosses, otherwise it won't be added const sense_sentence_number_suffix = new Set() @@ -225,7 +231,7 @@ export async function generate_sql_statements({ const sentence_id = randomUUID() const sentence: TablesInsert<'sentences'> = { dictionary_id, - ...c_u_meta, + ...c_u_meta(), id: sentence_id, text: {}, } @@ -263,7 +269,7 @@ export async function generate_sql_statements({ sentences.push(sentence) senses_in_sentences.push({ - ...c_meta, + ...c_meta(), sentence_id, sense_id, }) @@ -286,10 +292,15 @@ export async function generate_sql_statements({ if (!key.includes('soundFile')) continue if (!value) continue - const { storage_path } = await upload_audio(value, entry_id) + const { storage_path, error } = await upload_audio(value, entry_id) + if (error) { + console.log(error) + continue + } + const audio_id = randomUUID() const audio: TablesInsert<'audio'> = { - ...c_u_meta, + ...c_u_meta(), id: audio_id, dictionary_id, entry_id, @@ -305,7 +316,7 @@ export async function generate_sql_statements({ speaker_id = randomUUID() const speaker: TablesInsert<'speakers'> = { - ...c_u_meta, + ...c_u_meta(), id: speaker_id, dictionary_id, name: row.speakerName, @@ -319,18 +330,25 @@ export async function generate_sql_statements({ } sql_statements += sql_file_string('audio_speakers', { - ...c_meta, + ...c_meta(), audio_id, speaker_id, }) } } - if (row.photoFile) { - const { storage_path, serving_url } = await upload_photo(row.photoFile, entry_id) + for (const [key, value] of row_entries) { + if (!key.includes('photoFile')) continue + if (!value) continue + + const { storage_path, serving_url, error } = await upload_photo(value, entry_id) + if (error) { + console.log(error) + continue + } const photo_id = randomUUID() const photo: TablesInsert<'photos'> = { - ...c_u_meta, + ...c_u_meta(), id: photo_id, dictionary_id, storage_path, @@ -339,7 +357,7 @@ export async function generate_sql_statements({ sql_statements += sql_file_string('photos', photo) const sense_id = senses[0].id const sense_photo: TablesInsert<'sense_photos'> = { - ...c_meta, + ...c_meta(), photo_id, sense_id, } @@ -356,7 +374,3 @@ export async function generate_sql_statements({ console.error(err) } } - -function returnArrayFromCommaSeparatedItems(string: string): string[] { - return string?.split(',').map(item => item.trim()) || null -} diff --git a/packages/scripts/import/import-data.snap.json b/packages/scripts/import/import-data.snap.json index 203236adc..e0d630bcd 100644 --- a/packages/scripts/import/import-data.snap.json +++ b/packages/scripts/import/import-data.snap.json @@ -2,7 +2,7 @@ "entries": [ { "audios": null, - "created_at": "2024-03-08T00:44:04.6+00:00", + "created_at": "2024-03-08T00:44:04.666+00:00", "deleted": null, "dialect_ids": null, "dictionary_id": "example-v4-senses", @@ -16,18 +16,6 @@ }, }, "senses": [ - { - "glosses": { - "es": "calor", - }, - "id": "11111111-1111-1111-1111-111111100050", - "parts_of_speech": [ - "n", - ], - "sentence_ids": [ - "11111111-1111-1111-1111-111111100051", - ], - }, { "glosses": { "es": "sol", @@ -67,13 +55,25 @@ "11111111-1111-1111-1111-111111100049", ], }, + { + "glosses": { + "es": "calor", + }, + "id": "11111111-1111-1111-1111-111111100050", + "parts_of_speech": [ + "n", + ], + "sentence_ids": [ + "11111111-1111-1111-1111-111111100051", + ], + }, ], "tag_ids": null, - "updated_at": "2024-03-08T00:44:04.6+00:00", + "updated_at": "2024-03-08T00:44:04.68+00:00", }, { "audios": null, - "created_at": "2024-03-08T00:44:04.6+00:00", + "created_at": "2024-03-08T00:44:04.681+00:00", "deleted": null, "dialect_ids": null, "dictionary_id": "example-v4-senses", @@ -89,24 +89,24 @@ "senses": [ { "glosses": { - "es": "fiebre", + "es": "sol", }, - "id": "11111111-1111-1111-1111-111111100057", + "id": "11111111-1111-1111-1111-111111100054", "parts_of_speech": [ "n", ], + "variant": { + "default": "kꞌahkꞌal", + }, }, { "glosses": { - "es": "sol", + "es": "día", }, - "id": "11111111-1111-1111-1111-111111100054", + "id": "11111111-1111-1111-1111-111111100055", "parts_of_speech": [ "n", ], - "variant": { - "default": "kꞌahkꞌal", - }, }, { "glosses": { @@ -119,20 +119,20 @@ }, { "glosses": { - "es": "día", + "es": "fiebre", }, - "id": "11111111-1111-1111-1111-111111100055", + "id": "11111111-1111-1111-1111-111111100057", "parts_of_speech": [ "n", ], }, ], "tag_ids": null, - "updated_at": "2024-03-08T00:44:04.6+00:00", + "updated_at": "2024-03-08T00:44:04.687+00:00", }, { "audios": null, - "created_at": "2024-03-08T00:44:04.6+00:00", + "created_at": "2024-03-08T00:44:04.688+00:00", "deleted": null, "dialect_ids": null, "dictionary_id": "example-v4-senses", @@ -148,26 +148,29 @@ "senses": [ { "glosses": { - "es": "bravo", + "es": "fuego", }, - "id": "11111111-1111-1111-1111-111111100062", + "id": "11111111-1111-1111-1111-111111100060", "parts_of_speech": [ - "adj", + "n", ], "sentence_ids": [ - "11111111-1111-1111-1111-111111100063", + "11111111-1111-1111-1111-111111100061", ], + "variant": { + "default": "kꞌahkꞌ", + }, }, { "glosses": { - "es": "caliente", + "es": "bravo", }, - "id": "11111111-1111-1111-1111-111111100066", + "id": "11111111-1111-1111-1111-111111100062", "parts_of_speech": [ "adj", ], "sentence_ids": [ - "11111111-1111-1111-1111-111111100067", + "11111111-1111-1111-1111-111111100063", ], }, { @@ -184,26 +187,23 @@ }, { "glosses": { - "es": "fuego", + "es": "caliente", }, - "id": "11111111-1111-1111-1111-111111100060", + "id": "11111111-1111-1111-1111-111111100066", "parts_of_speech": [ - "n", + "adj", ], "sentence_ids": [ - "11111111-1111-1111-1111-111111100061", + "11111111-1111-1111-1111-111111100067", ], - "variant": { - "default": "kꞌahkꞌ", - }, }, ], "tag_ids": null, - "updated_at": "2024-03-08T00:44:04.6+00:00", + "updated_at": "2024-03-08T00:44:04.702+00:00", }, { "audios": null, - "created_at": "2024-03-08T00:44:04.6+00:00", + "created_at": "2024-03-08T00:44:04.703+00:00", "deleted": null, "dialect_ids": null, "dictionary_id": "example-v4-senses", @@ -267,11 +267,11 @@ }, ], "tag_ids": null, - "updated_at": "2024-03-08T00:44:04.6+00:00", + "updated_at": "2024-03-08T00:44:04.717+00:00", }, { "audios": null, - "created_at": "2024-03-08T00:44:04.6+00:00", + "created_at": "2024-03-08T00:44:04.718+00:00", "deleted": null, "dialect_ids": null, "dictionary_id": "example-v4-senses", @@ -285,15 +285,6 @@ }, }, "senses": [ - { - "glosses": { - "es": "abrir", - }, - "id": "11111111-1111-1111-1111-111111100082", - "sentence_ids": [ - "11111111-1111-1111-1111-111111100083", - ], - }, { "glosses": { "es": "abierto", @@ -310,36 +301,38 @@ }, }, { - "glosses": {}, - "id": "11111111-1111-1111-1111-111111100085", - }, - { - "glosses": {}, - "id": "11111111-1111-1111-1111-111111100084", + "glosses": { + "es": "abrir", + }, + "id": "11111111-1111-1111-1111-111111100082", + "sentence_ids": [ + "11111111-1111-1111-1111-111111100083", + ], }, ], "tag_ids": null, - "updated_at": "2024-03-08T00:44:04.6+00:00", + "updated_at": "2024-03-08T00:44:04.726+00:00", }, ], "sentences": [ { - "created_at": "2024-03-08T00:44:04.6+00:00", + "created_at": "2024-03-08T00:44:04.67+00:00", "created_by": "be43b1dd-6c64-494d-b5da-10d70c384433", "deleted": null, "dictionary_id": "example-v4-senses", "id": "11111111-1111-1111-1111-111111100045", - "text": {}, + "text": { + "default": "Jaꞌnix lek-a lokꞌix tel kꞌaal", + }, "text_id": null, "translation": { "es": "Que bueno, ya salió el sol", - "vernacular": "Jaꞌnix lek-a lokꞌix tel kꞌaal", }, - "updated_at": "2024-03-08T00:44:04.6+00:00", + "updated_at": "2024-03-08T00:44:04.67+00:00", "updated_by": "be43b1dd-6c64-494d-b5da-10d70c384433", }, { - "created_at": "2024-03-08T00:44:04.6+00:00", + "created_at": "2024-03-08T00:44:04.673+00:00", "created_by": "be43b1dd-6c64-494d-b5da-10d70c384433", "deleted": null, "dictionary_id": "example-v4-senses", @@ -351,11 +344,11 @@ "translation": { "es": "Mi hijita Ixchel tiene mucha fiebre", }, - "updated_at": "2024-03-08T00:44:04.6+00:00", + "updated_at": "2024-03-08T00:44:04.673+00:00", "updated_by": "be43b1dd-6c64-494d-b5da-10d70c384433", }, { - "created_at": "2024-03-08T00:44:04.6+00:00", + "created_at": "2024-03-08T00:44:04.676+00:00", "created_by": "be43b1dd-6c64-494d-b5da-10d70c384433", "deleted": null, "dictionary_id": "example-v4-senses", @@ -367,11 +360,11 @@ "translation": { "es": "Ya transcurrió el día mi estimado señor, es momento de tomar un descanso", }, - "updated_at": "2024-03-08T00:44:04.6+00:00", + "updated_at": "2024-03-08T00:44:04.676+00:00", "updated_by": "be43b1dd-6c64-494d-b5da-10d70c384433", }, { - "created_at": "2024-03-08T00:44:04.6+00:00", + "created_at": "2024-03-08T00:44:04.679+00:00", "created_by": "be43b1dd-6c64-494d-b5da-10d70c384433", "deleted": null, "dictionary_id": "example-v4-senses", @@ -383,26 +376,27 @@ "translation": { "es": "Siento mucho calor", }, - "updated_at": "2024-03-08T00:44:04.6+00:00", + "updated_at": "2024-03-08T00:44:04.679+00:00", "updated_by": "be43b1dd-6c64-494d-b5da-10d70c384433", }, { - "created_at": "2024-03-08T00:44:04.6+00:00", + "created_at": "2024-03-08T00:44:04.692+00:00", "created_by": "be43b1dd-6c64-494d-b5da-10d70c384433", "deleted": null, "dictionary_id": "example-v4-senses", "id": "11111111-1111-1111-1111-111111100061", - "text": {}, + "text": { + "default": "Tilix kuꞌun-i kꞌajkꞌi", + }, "text_id": null, "translation": { "es": "Ya hice el fuego", - "vernacular": "Tilix kuꞌun-i kꞌajkꞌi", }, - "updated_at": "2024-03-08T00:44:04.6+00:00", + "updated_at": "2024-03-08T00:44:04.692+00:00", "updated_by": "be43b1dd-6c64-494d-b5da-10d70c384433", }, { - "created_at": "2024-03-08T00:44:04.6+00:00", + "created_at": "2024-03-08T00:44:04.695+00:00", "created_by": "be43b1dd-6c64-494d-b5da-10d70c384433", "deleted": null, "dictionary_id": "example-v4-senses", @@ -414,11 +408,11 @@ "translation": { "es": "El mestizo es muy bravo", }, - "updated_at": "2024-03-08T00:44:04.6+00:00", + "updated_at": "2024-03-08T00:44:04.695+00:00", "updated_by": "be43b1dd-6c64-494d-b5da-10d70c384433", }, { - "created_at": "2024-03-08T00:44:04.6+00:00", + "created_at": "2024-03-08T00:44:04.698+00:00", "created_by": "be43b1dd-6c64-494d-b5da-10d70c384433", "deleted": null, "dictionary_id": "example-v4-senses", @@ -430,11 +424,11 @@ "translation": { "es": "El bebé tiene mucha fiebre", }, - "updated_at": "2024-03-08T00:44:04.6+00:00", + "updated_at": "2024-03-08T00:44:04.698+00:00", "updated_by": "be43b1dd-6c64-494d-b5da-10d70c384433", }, { - "created_at": "2024-03-08T00:44:04.6+00:00", + "created_at": "2024-03-08T00:44:04.701+00:00", "created_by": "be43b1dd-6c64-494d-b5da-10d70c384433", "deleted": null, "dictionary_id": "example-v4-senses", @@ -446,26 +440,27 @@ "translation": { "es": "Kꞌajkꞌ te kajpele, kꞌume xa awuchꞌ", }, - "updated_at": "2024-03-08T00:44:04.6+00:00", + "updated_at": "2024-03-08T00:44:04.701+00:00", "updated_by": "be43b1dd-6c64-494d-b5da-10d70c384433", }, { - "created_at": "2024-03-08T00:44:04.6+00:00", + "created_at": "2024-03-08T00:44:04.707+00:00", "created_by": "be43b1dd-6c64-494d-b5da-10d70c384433", "deleted": null, "dictionary_id": "example-v4-senses", "id": "11111111-1111-1111-1111-111111100071", - "text": {}, + "text": { + "default": "¿Beluk apas? - Yakalon ta skꞌoponel jun", + }, "text_id": null, "translation": { "es": "¿Qué haces? - Estoy leyendo un libro", - "vernacular": "¿Beluk apas? - Yakalon ta skꞌoponel jun", }, - "updated_at": "2024-03-08T00:44:04.6+00:00", + "updated_at": "2024-03-08T00:44:04.707+00:00", "updated_by": "be43b1dd-6c64-494d-b5da-10d70c384433", }, { - "created_at": "2024-03-08T00:44:04.6+00:00", + "created_at": "2024-03-08T00:44:04.71+00:00", "created_by": "be43b1dd-6c64-494d-b5da-10d70c384433", "deleted": null, "dictionary_id": "example-v4-senses", @@ -477,11 +472,11 @@ "translation": { "es": "Alcancé a rayar mi cuaderno", }, - "updated_at": "2024-03-08T00:44:04.6+00:00", + "updated_at": "2024-03-08T00:44:04.71+00:00", "updated_by": "be43b1dd-6c64-494d-b5da-10d70c384433", }, { - "created_at": "2024-03-08T00:44:04.6+00:00", + "created_at": "2024-03-08T00:44:04.713+00:00", "created_by": "be43b1dd-6c64-494d-b5da-10d70c384433", "deleted": null, "dictionary_id": "example-v4-senses", @@ -493,11 +488,11 @@ "translation": { "es": "No vayas a arrugar tu documento", }, - "updated_at": "2024-03-08T00:44:04.6+00:00", + "updated_at": "2024-03-08T00:44:04.713+00:00", "updated_by": "be43b1dd-6c64-494d-b5da-10d70c384433", }, { - "created_at": "2024-03-08T00:44:04.6+00:00", + "created_at": "2024-03-08T00:44:04.716+00:00", "created_by": "be43b1dd-6c64-494d-b5da-10d70c384433", "deleted": null, "dictionary_id": "example-v4-senses", @@ -509,26 +504,27 @@ "translation": { "es": "La schꞌiꞌ jun te Zoe", }, - "updated_at": "2024-03-08T00:44:04.6+00:00", + "updated_at": "2024-03-08T00:44:04.716+00:00", "updated_by": "be43b1dd-6c64-494d-b5da-10d70c384433", }, { - "created_at": "2024-03-08T00:44:04.6+00:00", + "created_at": "2024-03-08T00:44:04.722+00:00", "created_by": "be43b1dd-6c64-494d-b5da-10d70c384433", "deleted": null, "dictionary_id": "example-v4-senses", "id": "11111111-1111-1111-1111-111111100081", - "text": {}, + "text": { + "default": "Jeꞌel jilel stiꞌ jna", + }, "text_id": null, "translation": { "es": "La puerta de mi casa quedó abierta", - "vernacular": "Jeꞌel jilel stiꞌ jna", }, - "updated_at": "2024-03-08T00:44:04.6+00:00", + "updated_at": "2024-03-08T00:44:04.722+00:00", "updated_by": "be43b1dd-6c64-494d-b5da-10d70c384433", }, { - "created_at": "2024-03-08T00:44:04.6+00:00", + "created_at": "2024-03-08T00:44:04.725+00:00", "created_by": "be43b1dd-6c64-494d-b5da-10d70c384433", "deleted": null, "dictionary_id": "example-v4-senses", @@ -540,7 +536,7 @@ "translation": { "es": "Abre un poco la puerta, hace mucho calor", }, - "updated_at": "2024-03-08T00:44:04.6+00:00", + "updated_at": "2024-03-08T00:44:04.725+00:00", "updated_by": "be43b1dd-6c64-494d-b5da-10d70c384433", }, ], diff --git a/packages/scripts/import/import-data.test.ts b/packages/scripts/import/import-data.test.ts index 4be8ef2e5..dbbb6debf 100644 --- a/packages/scripts/import/import-data.test.ts +++ b/packages/scripts/import/import-data.test.ts @@ -23,8 +23,14 @@ vi.mock('node:crypto', () => { }) vi.mock('./incrementing-timestamp', () => { + const yesterday = new Date('2024-03-08T00:44:04.600392+00:00') + let milliseconds_to_add = 0 + return { - millisecond_incrementing_timestamp: () => new Date('2024-03-08T00:44:04.600392+00:00').toISOString(), + millisecond_incrementing_timestamp: () => { + milliseconds_to_add += 1 + return new Date(yesterday.getTime() + milliseconds_to_add).toISOString() + }, } }) @@ -76,7 +82,7 @@ describe(import_data, () => { "storage_path": "2.mp3", }, ], - "created_at": "2024-03-08T00:44:04.6+00:00", + "created_at": "2024-03-08T00:44:04.601+00:00", "deleted": null, "dialect_ids": null, "dictionary_id": "test_dictionary_id", @@ -98,7 +104,7 @@ describe(import_data, () => { }, ], "tag_ids": null, - "updated_at": "2024-03-08T00:44:04.6+00:00", + "updated_at": "2024-03-08T00:44:04.608+00:00", }, ] `) @@ -110,7 +116,7 @@ describe(import_data, () => { [ { "audios": null, - "created_at": "2024-03-08T00:44:04.6+00:00", + "created_at": "2024-03-08T00:44:04.609+00:00", "deleted": null, "dialect_ids": null, "dictionary_id": "test_dictionary_id", @@ -129,7 +135,7 @@ describe(import_data, () => { }, ], "tag_ids": null, - "updated_at": "2024-03-08T00:44:04.6+00:00", + "updated_at": "2024-03-08T00:44:04.612+00:00", }, ] `) @@ -156,7 +162,7 @@ describe(import_data, () => { "table": null, "tag_id": null, "text_id": null, - "timestamp": "2024-03-08T00:44:04.6+00:00", + "timestamp": "2024-03-08T00:44:04.611+00:00", "type": "insert_entry", "user_id": "be43b1dd-6c64-494d-b5da-10d70c384433", "video_id": null, @@ -185,14 +191,14 @@ describe(import_data, () => { expect(speakers[0]).toMatchInlineSnapshot(` { "birthplace": "Whoville", - "created_at": "2024-03-08T00:44:04.6+00:00", + "created_at": "2024-03-08T00:44:04.632+00:00", "decade": 12, "deleted": null, "dictionary_id": "test_dictionary_id", "gender": "m", "id": "11111111-1111-1111-1111-111111100021", "name": "speaker 1", - "updated_at": "2024-03-08T00:44:04.6+00:00", + "updated_at": "2024-03-08T00:44:04.632+00:00", } `) expect(entries[0].audios[0].speaker_ids[0]).toEqual(speakers[0].id) @@ -251,7 +257,7 @@ describe(import_data, () => { [ { "audios": null, - "created_at": "2024-03-08T00:44:04.6+00:00", + "created_at": "2024-03-08T00:44:04.646+00:00", "deleted": null, "dialect_ids": [ "11111111-1111-1111-1111-111111100032", @@ -301,16 +307,6 @@ describe(import_data, () => { "default": "variant", }, }, - { - "glosses": { - "fr": "auch", - }, - "id": "11111111-1111-1111-1111-111111100039", - "sentence_ids": [ - "11111111-1111-1111-1111-111111100040", - "11111111-1111-1111-1111-111111100041", - ], - }, { "glosses": { "en": "bye", @@ -321,12 +317,22 @@ describe(import_data, () => { "2.3", ], }, + { + "glosses": { + "fr": "auch", + }, + "id": "11111111-1111-1111-1111-111111100039", + "sentence_ids": [ + "11111111-1111-1111-1111-111111100040", + "11111111-1111-1111-1111-111111100041", + ], + }, ], "tag_ids": [ "11111111-1111-1111-1111-111111100034", "11111111-1111-1111-1111-111111100035", ], - "updated_at": "2024-03-08T00:44:04.6+00:00", + "updated_at": "2024-03-08T00:44:04.665+00:00", }, ] `) diff --git a/packages/scripts/import/import-media.ts b/packages/scripts/import/import-media.ts index bd5fccc6b..d09ae6174 100644 --- a/packages/scripts/import/import-media.ts +++ b/packages/scripts/import/import-media.ts @@ -19,30 +19,35 @@ export async function upload_audio_to_gcs({ entry_id: string dictionary_id: string live?: boolean -}): Promise { +}) { const audioDir = join(__dirname, `data/${dictionary_id}/audio`) const audioFilePath = join(audioDir, filepath) if (!fs.existsSync(audioFilePath)) { - console.log(`>> Missing audio file: ${filepath}`) - return null + return { + error: `!!! Missing audio file: ${filepath}`, + } } try { const [fileTypeSuffix] = filepath.match(/\.[0-9a-z]+$/i) - const uploadedAudioPath = `${dictionary_id}/audio/${entry_id}_${new Date().getTime()}${fileTypeSuffix}` + const storage_path = `${dictionary_id}/audio/${entry_id}_${new Date().getTime()}${fileTypeSuffix}` if (live) { await storage.bucket(fileBucket).upload(audioFilePath, { - destination: uploadedAudioPath, + destination: storage_path, metadata: { originalFileName: filepath, }, }) } - return uploadedAudioPath + return { + storage_path, + } } catch (err) { - throw new Error(`Not adding audio ${filepath} as the server had trouble uploading it. Double-check the file to see if there is a problem with it or perhaps there is code/server/network-connection problem. Error: ${err}`) + return { + error: `!!! Trouble uploading ${filepath}. Error: ${err}`, + } } } @@ -61,35 +66,39 @@ export async function upload_photo_to_gcs({ const imageFilePath = join(imageDir, filepath) if (!fs.existsSync(imageFilePath)) { - console.log(`>> Missing image file: ${filepath}`) - return null + return { + error: `!!! Missing image file: ${filepath}`, + } } - try { - const [fileTypeSuffix] = filepath.match(/\.[0-9a-z]+$/i) - const storage_path = `${dictionary_id}/images/${entry_id}_${new Date().getTime()}${fileTypeSuffix}` - if (!live) - return { storage_path, serving_url: 'no-serving_url-bc-dry-run' } + const [fileTypeSuffix] = filepath.match(/\.[0-9a-z]+$/i) + const storage_path = `${dictionary_id}/images/${entry_id}_${new Date().getTime()}${fileTypeSuffix}` + + if (!live) + return { storage_path, serving_url: 'no-serving_url-bc-dry-run' } + try { await storage.bucket(fileBucket).upload(imageFilePath, { destination: storage_path, metadata: { originalFileName: filepath, }, }) - - let serving_url - try { - serving_url = await getImageServingUrl(storage_path, environment) - } catch (err) { - throw new Error(`!!! Error getting image serving URL: ${err}`) + } catch (err) { + return { + error: `!!! Trouble uploading ${filepath}. Double-check the file to see if it is just a corrupted jpg (as some are) or if the file is good and perhaps there is code/server/network-connection problem. Error: ${err}`, } + } + try { + const serving_url = await getImageServingUrl(storage_path, environment) return { storage_path, serving_url, } } catch (err) { - throw new Error(`!!! Not adding image ${filepath} as the server had trouble digesting it. Double-check the file to see if it is just a corrupted jpg (as some are) or if the file is good and perhaps there is code/server/network-connection problem. Error: ${err}`) + return { + error: `!!! Error getting image serving URL for ${storage_path}: ${err}`, + } } } diff --git a/packages/scripts/import/import.ts b/packages/scripts/import/import.ts index 79a136010..4b8f020a2 100644 --- a/packages/scripts/import/import.ts +++ b/packages/scripts/import/import.ts @@ -28,7 +28,7 @@ async function import_from_spreadsheet({ dictionary_id, live }: { dictionary_id: const file = readFileSync(`./import/data/${dictionary_id}/${dictionary_id}.csv`, 'utf8') const rows = parseCSVFrom(file) - rows.shift() // remove header row + if (rows[0].lexeme.includes('word/phrase')) rows.shift() // remove header row await import_data({ dictionary_id, rows, import_id, live, upload_operations: { upload_photo, upload_audio } }) console.log( diff --git a/packages/scripts/package.json b/packages/scripts/package.json index 8e4ada7c3..c253b106b 100644 --- a/packages/scripts/package.json +++ b/packages/scripts/package.json @@ -14,9 +14,9 @@ }, "main": "index.ts", "scripts": { - "import-dictionary:dev:dry": "tsx import/import.ts --id tseltal", - "import-dictionary:dev:live": "tsx import/import.ts --id tseltal --live", - "import-dictionary:prod:live": "tsx import/import.ts --id tseltal -e prod --live", + "import-dictionary:dev:dry": "tsx import/import.ts --id example-v4", + "import-dictionary:dev:live": "tsx import/import.ts --id example-v4 --live", + "import-dictionary:prod:live": "tsx import/import.ts --id example-v4 -e prod --live", "update-locales": "tsx locales/update-locales.ts", "create-indexes": "tsx create-indexes/add-to-cloudflare.ts", "migrate-users": "tsx migrate-to-supabase/auth.ts", diff --git a/packages/types/exampe-sentence.interface.ts b/packages/types/example-sentence.interface.ts similarity index 100% rename from packages/types/exampe-sentence.interface.ts rename to packages/types/example-sentence.interface.ts diff --git a/packages/types/index.ts b/packages/types/index.ts index 0599c87a8..bfe23990e 100644 --- a/packages/types/index.ts +++ b/packages/types/index.ts @@ -4,7 +4,7 @@ export type { IAbout, IDictionary, IGrammar, Citation, Partner } from './diction export type { Coordinates, IPoint, IRegion } from './coordinates.interface' export type { IGlossLanguages, IGlossLanguage } from './gloss-language.interface' export type { MultiString } from './gloss.interface' -export type { IExampleSentence } from './exampe-sentence.interface' +export type { IExampleSentence } from './example-sentence.interface' export type { DictionaryPhoto, PartnerPhoto } from './photo.interface' export type { SemanticDomain } from './semantic-domain.interface' export type { IUser, GoogleAuthUserMetaData } from './user.interface'