Improve import functionality

packages/scripts/import/data/example-v4-senses/example-v4-senses.csv

@@ -1,4 +1,4 @@
-lexeme,variant,es_gloss,partOfSpeech ,vernacular_exampleSentence ,es_exampleSentence ,s2.es_gloss,s2.partOfSpeech,s2.default_vernacular_exampleSentence,s2.es_exampleSentence,s3.es_gloss,s3.partOfSpeech,s3.default_vernacular_exampleSentence,s3.es_exampleSentence,s4.es_gloss,s4.partOfSpeech,s4.default_vernacular_exampleSentence,s4.es_exampleSentence,notes
+lexeme,variant,es_gloss,partOfSpeech,default_vernacular_exampleSentence,es_exampleSentence,s2.es_gloss,s2.partOfSpeech,s2.default_vernacular_exampleSentence,s2.es_exampleSentence,s3.es_gloss,s3.partOfSpeech,s3.default_vernacular_exampleSentence,s3.es_exampleSentence,s4.es_gloss,s4.partOfSpeech,s4.default_vernacular_exampleSentence,s4.es_exampleSentence,notes
 kꞌahkꞌal,kꞌajkꞌal,sol,n,Lokꞌix tal kꞌahkꞌal,Ya salió el sol,fiebre,n,Ay ta kꞌahkꞌal te chꞌin alale,El niño tiene fiebre,día,n,Cheb kꞌahkꞌal ya x-aꞌtejotik,Trabajaremos dos días,calor,n,Toyol kꞌahkꞌal ya kaꞌiy,Siento mucho calor,16/jul./2019. Bachajon
 kꞌaal,kꞌahkꞌal,sol,n,Jaꞌnix lek-a lokꞌix tel kꞌaal,"Que bueno, ya salió el sol",fiebre,n,Ay bayal skꞌaal te chꞌin x-Ixchele,Mi hijita Ixchel tiene mucha fiebre,día,n,"Bajtix kaal mamtik, yorailix ich lewa","Ya transcurrió el día mi estimado señor, es momento de tomar un descanso",calor,n,Toyol kꞌaal ya jkaꞌiy,Siento mucho calor,26/dic./2020
 kꞌajkꞌal,kꞌahkꞌal,sol,n,,,día,n,,,calor,n,,,fiebre,n,,,14/dic./2019

packages/scripts/import/data/example-v4/example-v4.csv

@@ -1,10 +1,10 @@
-lexeme,phonetic,localOrthography,morphology,interlinearization,en_gloss,es_gloss,scientificName,partOfSpeech,,dialects,semanticDomain,,semanticDomain.2,,semanticDomain_custom,notes,soundFile,speakerName,speakerAge,speakerGender,speakerHometown,photoFile,vernacular_exampleSentence,en_exampleSentence,es_exampleSentence,source,pluralForm
-(word/phrase),,,,,English,Spanish,,abbrev.,,,key,,key,,"(try not to use, separate multiples with a |)",,,,,,,,,English,Spanish,,
-voiture,vwatyʁ,,,,car,auto,,"n,v",noun,Modern Parisian French,5.15,Transportation,5,Daily life,vehicle|cars,small automobile,helloworld.mp3,Celine Dorion,43,f,"Paris, France",mountain.jpg,Je conduis ma voiture,I drive my car,Conduzco mi auto,,
-arbre,aʁbʁ,,,,tree,árbol,Acer rubrum,"n, adj",noun,"Modern Parisian French, Quebec French",1.4,"Plants, trees and other vegetation",1.2,"Earth, geology and landscape",,generic term for all kinds of trees,missing.mp3,Celine Dorion,43,f,"Paris, France",,L'arbre nous donne de l'ombre,The tree gives us shade,El árbol nos da sombra,,
-tube,tyb,,,,tube,tubo,,n,noun,Modern Parisian French,5.9,Tools and weapons,,,plumbing,a cylindrical device for liquids,,,,,,missing.jpg,L'eau passe à travers les tubes,The water goes through the tubes,El agua pasa a través de los tubos,,tubes
-voiture,vwɑtYʁ,,,,car,auto,,n,noun,Quebec French,5.15,Transportation,,,vehicle,small automobile,,,,,,,Je conduis ma voiture,I drive my car,Conduzco mi auto,testing sources,
-neutre,nøʏ̯tʁ̥,,,,neutral,neutro,,adj,adjective,Quebec French,,,,,,,0005-neutre.mp3,David Boucher,31,m,"Montreal, Canada",,Ma chambre est peinte d'une couleur neutre.,My room is painted with a neutral color.,Mi habitación está pintada con un color neutro.,,
-fêter,fɛɪ̯te,,,,to celebrate,celebrar,,v,verb,Quebec French,,,,,,to have a party,0006-feter.mp3,David Boucher,31,m,"Montreal, Canada",,On va vraiment fêter à soir,We will really party tonight,Vamos a celebrar esta noche,"test source|with multiples sources, test|https://example.com",
-njakulaba,,,n-ja-ku-lab-a,1SG-Fut-2SG-see-Fin.V,I will see you,Voy a verte,,vp,verb phrase,Central Luganda,,,,,,,751-I-will-see-you.mp3,EB,50,m,,,,,,,
-vale,,,,,bye,adiós,,,,,,,,,,,,,,,,,,,,,
+lexeme,phonetic,localOrthography,morphology,interlinearization,en_gloss,es_gloss,scientificName,partOfSpeech,,partOfSpeech.2,,dialects,semanticDomain,,semanticDomain.2,,notes,soundFile,speakerName,speakerAge,speakerGender,speakerHometown,photoFile,default_vernacular_exampleSentence,en_exampleSentence,es_exampleSentence,source,pluralForm
+(word/phrase),,,,,English,Spanish,,abbrev.,,abbrev.,,,key,,key,,,,,,,,,,English,Spanish,,
+voiture,vwatyʁ,,,,car,auto,,n,noun,v,verb,Modern Parisian French,5.15,Transportation,5,Daily life,small automobile,helloworld.mp3,Celine Dorion,43,f,"Paris, France",mountain.jpg,Je conduis ma voiture,I drive my car,Conduzco mi auto,,
+arbre,aʁbʁ,,,,tree,árbol,Acer rubrum,n,noun,adj,adjective,Modern Parisian French | Quebec French,1.4,"Plants, trees and other vegetation",1.2,"Earth, geology and landscape",generic term for all kinds of trees,missing.mp3,Celine Dorion,43,f,"Paris, France",,L'arbre nous donne de l'ombre,The tree gives us shade,El árbol nos da sombra,,
+tube,tyb,,,,tube,tubo,,n,noun,,,Modern Parisian French,5.9,Tools and weapons,,,a cylindrical device for liquids,,,,,,missing.jpg,L'eau passe à travers les tubes,The water goes through the tubes,El agua pasa a través de los tubos,,tubes
+voiture,vwɑtYʁ,,,,car,auto,,n,noun,,,Quebec French,5.15,Transportation,,,small automobile,,,,,,,Je conduis ma voiture,I drive my car,Conduzco mi auto,testing sources,
+neutre,nøʏ̯tʁ̥,,,,neutral,neutro,,adj,adjective,,,Quebec French,,,,,,0005-neutre.mp3,David Boucher,31,m,"Montreal, Canada",,Ma chambre est peinte d'une couleur neutre.,My room is painted with a neutral color.,Mi habitación está pintada con un color neutro.,,
+fêter,fɛɪ̯te,,,,to celebrate,celebrar,,v,verb,,,Quebec French,,,,,to have a party,0006-feter.mp3,David Boucher,31,m,"Montreal, Canada",,On va vraiment fêter à soir,We will really party tonight,Vamos a celebrar esta noche,"test source|with multiples sources, test|https://example.com",
+njakulaba,,,n-ja-ku-lab-a,1SG-Fut-2SG-see-Fin.V,I will see you,Voy a verte,,vp,verb phrase,,,Central Luganda,,,,,,751-I-will-see-you.mp3,EB,50,m,,,,,,,
+vale,,,,,bye,adiós,,,,,,,,,,,,,,,,,,,,,,

packages/scripts/import/generate-sql-statements.ts

@@ -202,7 +202,7 @@ export async function generate_sql_statements({
         }
       }
 
-      senses.push(sense)
+      if (Object.keys(sense.glosses).length > 0 || sense_label[1] === '1') senses.push(sense) //* It only adds an additional sense if it has any glosses, otherwise it won't be added
 
       const sense_sentence_number_suffix = new Set<Number_Suffix>()
 
@@ -287,63 +287,69 @@ export async function generate_sql_statements({
       if (!value) continue
 
       const { storage_path } = await upload_audio(value, entry_id)
-      const audio_id = randomUUID()
-      const audio: TablesInsert<'audio'> = {
-        ...c_u_meta,
-        id: audio_id,
-        dictionary_id,
-        entry_id,
-        storage_path,
-      }
-      sql_statements += sql_file_string('audio', audio)
-
-      // TODO: the code above will properly import multiple audio files to the same entry but the code below will only import the metadata from the first audio file. Late on when adding multiple audio import ability, use the next line to get the number suffix from the key
-      // const number_suffix_with_period = key.replace('soundFile', '') as Number_Suffix
-      if (row.speakerName) {
-        let speaker_id = speakers.find(({ name }) => name === row.speakerName)?.id
-        if (!speaker_id) {
-          speaker_id = randomUUID()
-
-          const speaker: TablesInsert<'speakers'> = {
-            ...c_u_meta,
-            id: speaker_id,
-            dictionary_id,
-            name: row.speakerName,
-            birthplace: row.speakerHometown || '',
-            decade: Number.parseInt(row.speakerAge) || null,
-            gender: row.speakerGender as 'm' | 'f' | 'o' || null,
+      let audio_id: string
+      if (storage_path) {
+        audio_id = randomUUID()
+        const audio: TablesInsert<'audio'> = {
+          ...c_u_meta,
+          id: audio_id,
+          dictionary_id,
+          entry_id,
+          storage_path,
+        }
+        sql_statements += sql_file_string('audio', audio)
+
+        // TODO: the code above will properly import multiple audio files to the same entry but the code below will only import the metadata from the first audio file. Late on when adding multiple audio import ability, use the next line to get the number suffix from the key
+        // const number_suffix_with_period = key.replace('soundFile', '') as Number_Suffix
+        if (row.speakerName) {
+          let speaker_id = speakers.find(({ name }) => name === row.speakerName)?.id
+          if (!speaker_id) {
+            speaker_id = randomUUID()
+
+            const speaker: TablesInsert<'speakers'> = {
+              ...c_u_meta,
+              id: speaker_id,
+              dictionary_id,
+              name: row.speakerName,
+              birthplace: row.speakerHometown || '',
+              decade: Number.parseInt(row.speakerAge) || null,
+              gender: row.speakerGender as 'm' | 'f' | 'o' || null,
+            }
+
+            sql_statements += sql_file_string('speakers', speaker)
+            speakers.push({ id: speaker_id, name: row.speakerName })
           }
 
-          sql_statements += sql_file_string('speakers', speaker)
-          speakers.push({ id: speaker_id, name: row.speakerName })
+          sql_statements += sql_file_string('audio_speakers', {
+            ...c_meta,
+            audio_id,
+            speaker_id,
+          })
         }
-
-        sql_statements += sql_file_string('audio_speakers', {
-          ...c_meta,
-          audio_id,
-          speaker_id,
-        })
       }
     }
 
     if (row.photoFile) {
-      const { storage_path, serving_url } = await upload_photo(row.photoFile, entry_id)
-      const photo_id = randomUUID()
-      const photo: TablesInsert<'photos'> = {
-        ...c_u_meta,
-        id: photo_id,
-        dictionary_id,
-        storage_path,
-        serving_url,
-      }
-      sql_statements += sql_file_string('photos', photo)
-      const sense_id = senses[0].id
-      const sense_photo: TablesInsert<'sense_photos'> = {
-        ...c_meta,
-        photo_id,
-        sense_id,
+      const photo_media_result = await upload_photo(row.photoFile, entry_id)
+      if (photo_media_result) {
+        const { storage_path, serving_url } = photo_media_result
+        const photo_id = randomUUID()
+        const photo: TablesInsert<'photos'> = {
+          ...c_u_meta,
+          id: photo_id,
+          dictionary_id,
+          storage_path,
+          serving_url,
+        }
+        sql_statements += sql_file_string('photos', photo)
+        const sense_id = senses[0].id
+        const sense_photo: TablesInsert<'sense_photos'> = {
+          ...c_meta,
+          photo_id,
+          sense_id,
+        }
+        sql_statements += sql_file_string('sense_photos', sense_photo)
       }
-      sql_statements += sql_file_string('sense_photos', sense_photo)
     }
 
     // TablesInsert<'videos'>

packages/scripts/import/import.ts

@@ -28,7 +28,7 @@ async function import_from_spreadsheet({ dictionary_id, live }: { dictionary_id:
 
   const file = readFileSync(`./import/data/${dictionary_id}/${dictionary_id}.csv`, 'utf8')
   const rows = parseCSVFrom<Row>(file)
-  rows.shift() // remove header row
+  if (rows[0].lexeme.startsWith('(word/phrase)')) rows.shift() // remove header row
   await import_data({ dictionary_id, rows, import_id, live, upload_operations: { upload_photo, upload_audio } })
 
   console.log(

packages/scripts/package.json

@@ -14,9 +14,9 @@
   },
   "main": "index.ts",
   "scripts": {
-    "import-dictionary:dev:dry": "tsx import/import.ts --id tseltal",
-    "import-dictionary:dev:live": "tsx import/import.ts --id tseltal --live",
-    "import-dictionary:prod:live": "tsx import/import.ts --id tseltal -e prod --live",
+    "import-dictionary:dev:dry": "tsx import/import.ts --id example-v4",
+    "import-dictionary:dev:live": "tsx import/import.ts --id example-v4 --live",
+    "import-dictionary:prod:live": "tsx import/import.ts --id example-v4 -e prod --live",
     "update-locales": "tsx locales/update-locales.ts",
     "create-indexes": "tsx create-indexes/add-to-cloudflare.ts",
     "migrate-users": "tsx migrate-to-supabase/auth.ts",

packages/types/index.ts

@@ -4,7 +4,7 @@ export type { IAbout, IDictionary, IGrammar, Citation, Partner } from './diction
 export type { Coordinates, IPoint, IRegion } from './coordinates.interface'
 export type { IGlossLanguages, IGlossLanguage } from './gloss-language.interface'
 export type { MultiString } from './gloss.interface'
-export type { IExampleSentence } from './exampe-sentence.interface'
+export type { IExampleSentence } from './example-sentence.interface'
 export type { DictionaryPhoto, PartnerPhoto } from './photo.interface'
 export type { SemanticDomain } from './semantic-domain.interface'
 export type { IUser, GoogleAuthUserMetaData } from './user.interface'