Upload version and model registry on deploy (#43)

* Upload version and model registry on deploy * Update enit and move to prod * Add missing models * Update evaluation results [skip ci] * Update model registry [skip ci] * Delete obsolete folder * Update evaluation results [skip ci] Co-authored-by: CircleCI evaluation job <ci-models-evaluation@firefox-translations>
mozilla · Apr 29, 2022 · 47ff2cc · 47ff2cc
1 parent 1b630c7
commit 47ff2cc
Show file tree

Hide file tree

Showing 30 changed files with 70 additions and 67 deletions.
diff --git a/evaluation/dev/en-it/flores-dev.bergamot.it.bleu b/evaluation/dev/en-it/flores-dev.bergamot.it.bleu
diff --git a/evaluation/dev/en-it/flores-dev.google.it.bleu b/evaluation/dev/en-it/flores-dev.google.it.bleu
diff --git a/evaluation/dev/en-it/flores-dev.microsoft.it.bleu b/evaluation/dev/en-it/flores-dev.microsoft.it.bleu
diff --git a/evaluation/dev/en-it/flores-test.google.it.bleu b/evaluation/dev/en-it/flores-test.google.it.bleu
diff --git a/evaluation/dev/en-it/flores-test.microsoft.it.bleu b/evaluation/dev/en-it/flores-test.microsoft.it.bleu
diff --git a/evaluation/dev/en-it/wmt09.bergamot.it.bleu b/evaluation/dev/en-it/wmt09.bergamot.it.bleu
diff --git a/evaluation/dev/en-it/wmt09.microsoft.it.bleu b/evaluation/dev/en-it/wmt09.microsoft.it.bleu
diff --git a/evaluation/dev/img/avg.png b/evaluation/dev/img/avg.png
diff --git a/evaluation/dev/img/en-it.png b/evaluation/dev/img/en-it.png
diff --git a/evaluation/dev/results.md b/evaluation/dev/results.md
@@ -57,11 +57,11 @@ Both absolute and relative differences in BLEU scores between Bergamot and other
 
 ## avg
 
-| Translator/Dataset | fa-en | ru-en | en-ru | en-it | en-fa | is-en |
-| --- | --- | --- | --- | --- | --- | --- |
-| bergamot | 28.70 | 33.37 | 30.47 | 29.53 | 17.30 | 23.50 |
-| google | 36.05 (+7.35, +25.61%) | 36.53 (+3.15, +9.45%) | 33.72 (+3.25, +10.67%) | 29.47 (-0.07, -0.23%) | 27.70 (+10.40, +60.12%) | 34.95 (+11.45, +48.72%) |
-| microsoft | 36.15 (+7.45, +25.96%) | 36.87 (+3.50, +10.49%) | 33.68 (+3.21, +10.53%) | 32.20 (+2.67, +9.03%) | 20.50 (+3.20, +18.50%) | 34.90 (+11.40, +48.51%) |
+| Translator/Dataset | fa-en | ru-en | en-ru | en-fa | is-en |
+| --- | --- | --- | --- | --- | --- |
+| bergamot | 28.70 | 33.37 | 30.47 | 17.30 | 23.50 |
+| google | 36.05 (+7.35, +25.61%) | 36.53 (+3.15, +9.45%) | 33.72 (+3.25, +10.67%) | 27.70 (+10.40, +60.12%) | 34.95 (+11.45, +48.72%) |
+| microsoft | 36.15 (+7.45, +25.96%) | 36.87 (+3.50, +10.49%) | 33.68 (+3.21, +10.53%) | 20.50 (+3.20, +18.50%) | 34.90 (+11.40, +48.51%) |
 
 ![Results](img/avg.png)
 
@@ -95,16 +95,6 @@ Both absolute and relative differences in BLEU scores between Bergamot and other
 
 ![Results](img/en-ru.png)
 
-## en-it
-
-| Translator/Dataset | flores-dev | flores-test | wmt09 |
-| --- | --- | --- | --- |
-| bergamot | 28.40 | 29.20 | 31.00 |
-| google | 29.00 (+0.60, +2.11%) | 30.10 (+0.90, +3.08%) | 29.30 (-1.70, -5.48%) |
-| microsoft | 31.00 (+2.60, +9.15%) | 32.00 (+2.80, +9.59%) | 33.60 (+2.60, +8.39%) |
-
-![Results](img/en-it.png)
-
 ## en-fa
 
 | Translator/Dataset | flores-dev | flores-test |

diff --git a/...on/dev/en-it/flores-test.bergamot.it.bleu → ...on/prod/en-it/flores-dev.bergamot.it.bleu b/...on/dev/en-it/flores-test.bergamot.it.bleu → ...on/prod/en-it/flores-dev.bergamot.it.bleu
diff --git a/evaluation/prod/en-it/flores-dev.google.it.bleu b/evaluation/prod/en-it/flores-dev.google.it.bleu
@@ -0,0 +1 @@
+28.5
diff --git a/evaluation/prod/en-it/flores-dev.microsoft.it.bleu b/evaluation/prod/en-it/flores-dev.microsoft.it.bleu
@@ -0,0 +1 @@
+31.1
diff --git a/evaluation/dev/en-it/wmt09.google.it.bleu → ...n/prod/en-it/flores-test.bergamot.it.bleu b/evaluation/dev/en-it/wmt09.google.it.bleu → ...n/prod/en-it/flores-test.bergamot.it.bleu
diff --git a/evaluation/prod/en-it/flores-test.google.it.bleu b/evaluation/prod/en-it/flores-test.google.it.bleu
@@ -0,0 +1 @@
+29.6
diff --git a/evaluation/prod/en-it/flores-test.microsoft.it.bleu b/evaluation/prod/en-it/flores-test.microsoft.it.bleu
@@ -0,0 +1 @@
+32.1
diff --git a/evaluation/prod/en-it/wmt09.bergamot.it.bleu b/evaluation/prod/en-it/wmt09.bergamot.it.bleu
@@ -0,0 +1 @@
+30.8
diff --git a/evaluation/prod/en-it/wmt09.google.it.bleu b/evaluation/prod/en-it/wmt09.google.it.bleu
@@ -0,0 +1 @@
+28.8
diff --git a/evaluation/prod/en-it/wmt09.microsoft.it.bleu b/evaluation/prod/en-it/wmt09.microsoft.it.bleu
@@ -0,0 +1 @@
+33.7
diff --git a/evaluation/prod/img/avg.png b/evaluation/prod/img/avg.png
diff --git a/evaluation/prod/img/en-it.png b/evaluation/prod/img/en-it.png
diff --git a/evaluation/prod/results.md b/evaluation/prod/results.md
@@ -57,11 +57,11 @@ Both absolute and relative differences in BLEU scores between Bergamot and other
 
 ## avg
 
-| Translator/Dataset | es-en | nb-en | bg-en | pt-en | it-en | et-en | en-cs | cs-en | de-en | en-es | en-pt | en-et | en-bg | en-de |
-| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |
-| bergamot | 32.38 | 37.60 | 38.50 | 44.87 | 32.67 | 32.37 | 24.65 | 30.34 | 33.51 | 32.41 | 49.85 | 25.50 | 42.10 | 32.27 |
-| google | 33.64 (+1.27, +3.91%) | 42.05 (+4.45, +11.84%) | 41.30 (+2.80, +7.27%) | 46.60 (+1.73, +3.86%) | 34.50 (+1.83, +5.59%) | 35.80 (+3.43, +10.61%) | 26.73 (+2.09, +8.47%) | 32.40 (+2.06, +6.80%) | 35.98 (+2.48, +7.39%) | 34.74 (+2.32, +7.17%) | 53.75 (+3.90, +7.82%) | 28.60 (+3.10, +12.16%) | 44.60 (+2.50, +5.94%) | 33.05 (+0.77, +2.40%) |
-| microsoft | 32.93 (+0.56, +1.72%) | 42.90 (+5.30, +14.10%) | 41.20 (+2.70, +7.01%) | 46.47 (+1.60, +3.57%) | 34.55 (+1.88, +5.74%) | 36.17 (+3.80, +11.74%) | 27.75 (+3.11, +12.60%) | 33.53 (+3.19, +10.53%) | 38.21 (+4.70, +14.03%) | 33.76 (+1.35, +4.17%) | 50.15 (+0.30, +0.60%) | 28.47 (+2.97, +11.63%) | 38.55 (-3.55, -8.43%) | 33.54 (+1.27, +3.93%) |
+| Translator/Dataset | es-en | nb-en | bg-en | pt-en | it-en | et-en | en-cs | cs-en | en-it | de-en | en-es | en-pt | en-et | en-bg | en-de |
+| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |
+| bergamot | 32.38 | 37.60 | 38.50 | 44.87 | 32.67 | 32.37 | 24.65 | 30.34 | 29.77 | 33.51 | 32.41 | 49.85 | 25.50 | 42.10 | 32.27 |
+| google | 33.64 (+1.27, +3.91%) | 42.05 (+4.45, +11.84%) | 41.30 (+2.80, +7.27%) | 46.60 (+1.73, +3.86%) | 34.50 (+1.83, +5.59%) | 35.80 (+3.43, +10.61%) | 26.73 (+2.09, +8.47%) | 32.40 (+2.06, +6.80%) | 28.97 (-0.80, -2.69%) | 35.98 (+2.48, +7.39%) | 34.74 (+2.32, +7.17%) | 53.75 (+3.90, +7.82%) | 28.60 (+3.10, +12.16%) | 44.60 (+2.50, +5.94%) | 33.05 (+0.77, +2.40%) |
+| microsoft | 32.93 (+0.56, +1.72%) | 42.90 (+5.30, +14.10%) | 41.20 (+2.70, +7.01%) | 46.47 (+1.60, +3.57%) | 34.55 (+1.88, +5.74%) | 36.17 (+3.80, +11.74%) | 27.75 (+3.11, +12.60%) | 33.53 (+3.19, +10.53%) | 32.30 (+2.53, +8.51%) | 38.21 (+4.70, +14.03%) | 33.76 (+1.35, +4.17%) | 50.15 (+0.30, +0.60%) | 28.47 (+2.97, +11.63%) | 38.55 (-3.55, -8.43%) | 33.54 (+1.27, +3.93%) |
 
 ![Results](img/avg.png)
 
@@ -145,6 +145,16 @@ Both absolute and relative differences in BLEU scores between Bergamot and other
 
 ![Results](img/cs-en.png)
 
+## en-it
+
+| Translator/Dataset | flores-dev | flores-test | wmt09 |
+| --- | --- | --- | --- |
+| bergamot | 29.20 | 29.30 | 30.80 |
+| google | 28.50 (-0.70, -2.40%) | 29.60 (+0.30, +1.02%) | 28.80 (-2.00, -6.49%) |
+| microsoft | 31.10 (+1.90, +6.51%) | 32.10 (+2.80, +9.56%) | 33.70 (+2.90, +9.42%) |
+
+![Results](img/en-it.png)
+
 ## de-en
 
 | Translator/Dataset | wmt17 | wmt10 | wmt18 | wmt09 | wmt14 | wmt11 | wmt16 | wmt13 | wmt19 | wmt20 | wmt08 | flores-dev | flores-test | wmt12 | wmt15 | iwslt17 |

diff --git a/models/dev/enit/lex.50.50.enit.s2t.bin.gz b/models/dev/enit/lex.50.50.enit.s2t.bin.gz
diff --git a/models/dev/enit/model.enit.intgemm.alphas.bin.gz b/models/dev/enit/model.enit.intgemm.alphas.bin.gz
diff --git a/models/dev/enit/vocab.enit.spm.gz b/models/dev/enit/vocab.enit.spm.gz
diff --git a/models/prod/enit/lex.50.50.enit.s2t.bin.gz b/models/prod/enit/lex.50.50.enit.s2t.bin.gz
diff --git a/models/prod/enit/model.enit.intgemm.alphas.bin.gz b/models/prod/enit/model.enit.intgemm.alphas.bin.gz
diff --git a/models/prod/enit/vocab.enit.spm.gz b/models/prod/enit/vocab.enit.spm.gz
diff --git a/registry.json b/registry.json
@@ -106,19 +106,19 @@
       "expectedSha256Hash": "e19c77231bf977988e31ff8db15fe79966b5170564bd3e10613f239e7f461d97",
       "modelType": "prod"
     },
-    "vocab": {
-      "name": "vocab.csen.spm",
-      "size": 769763,
-      "estimatedCompressedSize": 366392,
-      "expectedSha256Hash": "f71cc5d045e479607078e079884f44032f5a0b82547fb96eefa29cd1eb47c6f3",
-      "modelType": "prod"
-    },
     "qualityModel": {
       "name": "qualityModel.encs.bin",
       "size": 68,
       "estimatedCompressedSize": 108,
       "expectedSha256Hash": "d7eba90036a065e4a1e93e889befe09f93a7d9a3417f3edffdb09a0db88fe83a",
       "modelType": "prod"
+    },
+    "vocab": {
+      "name": "vocab.csen.spm",
+      "size": 769763,
+      "estimatedCompressedSize": 366392,
+      "expectedSha256Hash": "f71cc5d045e479607078e079884f44032f5a0b82547fb96eefa29cd1eb47c6f3",
+      "modelType": "prod"
     }
   },
   "ende": {
@@ -204,6 +204,29 @@
       "modelType": "prod"
     }
   },
+  "enit": {
+    "model": {
+      "name": "model.enit.intgemm.alphas.bin",
+      "size": 17140899,
+      "estimatedCompressedSize": 13267138,
+      "expectedSha256Hash": "3d7bbc4d7977e10b35f53faa79f5d5de8211f4f04baed9e7cd9dee1dcceda917",
+      "modelType": "prod"
+    },
+    "lex": {
+      "name": "lex.50.50.enit.s2t.bin",
+      "size": 4495004,
+      "estimatedCompressedSize": 2441969,
+      "expectedSha256Hash": "351ea80fb9f366f07533c7c4836248e72d9d4aa4eb7a05b5d74891a7abb4208c",
+      "modelType": "prod"
+    },
+    "vocab": {
+      "name": "vocab.enit.spm",
+      "size": 812781,
+      "estimatedCompressedSize": 408834,
+      "expectedSha256Hash": "603f3349657c3deb9736a0c567452d102a5a03c377dfdf1d32c428608f2cff1b",
+      "modelType": "prod"
+    }
+  },
   "enpt": {
     "model": {
       "name": "model.enpt.intgemm.alphas.bin",
@@ -365,29 +388,6 @@
       "modelType": "dev"
     }
   },
-  "enit": {
-    "model": {
-      "name": "model.enit.intgemm.alphas.bin",
-      "size": 17140836,
-      "estimatedCompressedSize": 13283223,
-      "expectedSha256Hash": "a5ce3723f62ead92a0e0373b6df0ad8e3e6d22963adb1333984206e33b8b6c61",
-      "modelType": "dev"
-    },
-    "lex": {
-      "name": "lex.50.50.enit.s2t.bin",
-      "size": 4489920,
-      "estimatedCompressedSize": 2409986,
-      "expectedSha256Hash": "bb1fad3b3f6a13ebce1698cf7f39ca736c4dea4525f3dab5e1a78436f07445e6",
-      "modelType": "dev"
-    },
-    "vocab": {
-      "name": "vocab.enit.spm",
-      "size": 814128,
-      "estimatedCompressedSize": 405338,
-      "expectedSha256Hash": "de8cbeb79e0139304bfa47e8559f2447016bf9906225a97d3df1baed4de8f3a3",
-      "modelType": "dev"
-    }
-  },
   "enru": {
     "model": {
       "name": "model.enru.intgemm.alphas.bin",

diff --git a/scripts/upload.sh b/scripts/upload.sh
@@ -3,5 +3,8 @@
 echo ${GCLOUD_UPLOADER_KEY} | gcloud auth activate-service-account --key-file=-
 gcloud --quiet config set project ${GOOGLE_PROJECT_ID}
 gzip -dr models/*/*/*.gz
+echo ${CIRCLE_TAG} > latest.txt
 gsutil -m cp -rZn models/prod/* gs://bergamot-models-sandbox/${CIRCLE_TAG}/
-gsutil -m cp -rZn models/dev/* gs://bergamot-models-sandbox/${CIRCLE_TAG}/
+gsutil -m cp -rZn models/dev/* gs://bergamot-models-sandbox/${CIRCLE_TAG}/
+gsutil -m cp -n registry.json gs://bergamot-models-sandbox/${CIRCLE_TAG}/
+gsutil -m cp -n latest.txt gs://bergamot-models-sandbox/