Add Polish models (#45)

* Add Polish models * Fake fix readme * Update evaluation results [skip ci] * Update model registry [skip ci] Co-authored-by: CircleCI evaluation job <ci-models-evaluation@firefox-translations>
mozilla · May 31, 2022 · 5ac9ad1 · 5ac9ad1
1 parent 6c519de
commit 5ac9ad1
Show file tree

Hide file tree

Showing 30 changed files with 118 additions and 13 deletions.
diff --git a/README.md b/README.md
@@ -55,6 +55,7 @@ Create a new release with a version tag `x.y.z` following semantic versioning.
 The models will be automatically uploaded to GCS bucket `gs://bergamot-models-sandbox/x.y.z/`. 
 
 # Currently supported Languages
+
 ## Prod
 - Spanish <-> English
 - Estonian <-> English
@@ -64,11 +65,13 @@ The models will be automatically uploaded to GCS bucket `gs://bergamot-models-sa
 - Norwegian Bokmål -> English
 - Portuguese <-> English
 - Italian <-> English
+- Polish <-> English
+
 ## Dev
 - Russian <-> English
 - Persian (Farsi) <-> English
 - Icelandic -> English
 - Norwegian Nynorsk -> English
+
 ## Upcoming
 -  French <-> English
--  Polish <-> English
diff --git a/evaluation/prod/en-pl/flores-dev.bergamot.pl.bleu b/evaluation/prod/en-pl/flores-dev.bergamot.pl.bleu
@@ -0,0 +1 @@
+20.7
diff --git a/evaluation/prod/en-pl/flores-dev.google.pl.bleu b/evaluation/prod/en-pl/flores-dev.google.pl.bleu
@@ -0,0 +1 @@
+24.2
diff --git a/evaluation/prod/en-pl/flores-dev.microsoft.pl.bleu b/evaluation/prod/en-pl/flores-dev.microsoft.pl.bleu
@@ -0,0 +1 @@
+23.0
diff --git a/evaluation/prod/en-pl/flores-test.bergamot.pl.bleu b/evaluation/prod/en-pl/flores-test.bergamot.pl.bleu
@@ -0,0 +1 @@
+21.0
diff --git a/evaluation/prod/en-pl/flores-test.google.pl.bleu b/evaluation/prod/en-pl/flores-test.google.pl.bleu
@@ -0,0 +1 @@
+24.4
diff --git a/evaluation/prod/en-pl/flores-test.microsoft.pl.bleu b/evaluation/prod/en-pl/flores-test.microsoft.pl.bleu
@@ -0,0 +1 @@
+23.8
diff --git a/evaluation/prod/en-pl/wmt20.bergamot.pl.bleu b/evaluation/prod/en-pl/wmt20.bergamot.pl.bleu
@@ -0,0 +1 @@
+25.1
diff --git a/evaluation/prod/en-pl/wmt20.google.pl.bleu b/evaluation/prod/en-pl/wmt20.google.pl.bleu
@@ -0,0 +1 @@
+27.9
diff --git a/evaluation/prod/en-pl/wmt20.microsoft.pl.bleu b/evaluation/prod/en-pl/wmt20.microsoft.pl.bleu
@@ -0,0 +1 @@
+27.7
diff --git a/evaluation/prod/img/avg.png b/evaluation/prod/img/avg.png
diff --git a/evaluation/prod/img/en-pl.png b/evaluation/prod/img/en-pl.png
diff --git a/evaluation/prod/img/pl-en.png b/evaluation/prod/img/pl-en.png
diff --git a/evaluation/prod/pl-en/flores-dev.bergamot.en.bleu b/evaluation/prod/pl-en/flores-dev.bergamot.en.bleu
@@ -0,0 +1 @@
+26.8
diff --git a/evaluation/prod/pl-en/flores-dev.google.en.bleu b/evaluation/prod/pl-en/flores-dev.google.en.bleu
@@ -0,0 +1 @@
+30.0
diff --git a/evaluation/prod/pl-en/flores-dev.microsoft.en.bleu b/evaluation/prod/pl-en/flores-dev.microsoft.en.bleu
@@ -0,0 +1 @@
+30.1
diff --git a/evaluation/prod/pl-en/flores-test.bergamot.en.bleu b/evaluation/prod/pl-en/flores-test.bergamot.en.bleu
@@ -0,0 +1 @@
+25.8
diff --git a/evaluation/prod/pl-en/flores-test.google.en.bleu b/evaluation/prod/pl-en/flores-test.google.en.bleu
@@ -0,0 +1 @@
+29.6
diff --git a/evaluation/prod/pl-en/flores-test.microsoft.en.bleu b/evaluation/prod/pl-en/flores-test.microsoft.en.bleu
@@ -0,0 +1 @@
+29.9
diff --git a/evaluation/prod/pl-en/wmt20.bergamot.en.bleu b/evaluation/prod/pl-en/wmt20.bergamot.en.bleu
@@ -0,0 +1 @@
+31.0
diff --git a/evaluation/prod/pl-en/wmt20.google.en.bleu b/evaluation/prod/pl-en/wmt20.google.en.bleu
@@ -0,0 +1 @@
+34.1
diff --git a/evaluation/prod/pl-en/wmt20.microsoft.en.bleu b/evaluation/prod/pl-en/wmt20.microsoft.en.bleu
@@ -0,0 +1 @@
+35.5
diff --git a/evaluation/prod/results.md b/evaluation/prod/results.md
@@ -57,11 +57,11 @@ Both absolute and relative differences in BLEU scores between Bergamot and other
 
 ## avg
 
-| Translator/Dataset | es-en | nb-en | bg-en | pt-en | it-en | et-en | en-cs | cs-en | en-it | de-en | en-es | en-pt | en-et | en-bg | en-de |
-| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |
-| bergamot | 32.38 | 37.60 | 38.50 | 44.87 | 32.67 | 32.37 | 24.65 | 30.34 | 29.77 | 33.51 | 32.41 | 49.85 | 25.50 | 42.10 | 32.27 |
-| google | 33.64 (+1.27, +3.91%) | 42.05 (+4.45, +11.84%) | 41.30 (+2.80, +7.27%) | 46.60 (+1.73, +3.86%) | 34.50 (+1.83, +5.59%) | 35.80 (+3.43, +10.61%) | 26.73 (+2.09, +8.47%) | 32.40 (+2.06, +6.80%) | 28.97 (-0.80, -2.69%) | 35.98 (+2.48, +7.39%) | 34.74 (+2.32, +7.17%) | 53.75 (+3.90, +7.82%) | 28.60 (+3.10, +12.16%) | 44.60 (+2.50, +5.94%) | 33.05 (+0.77, +2.40%) |
-| microsoft | 32.93 (+0.56, +1.72%) | 42.90 (+5.30, +14.10%) | 41.20 (+2.70, +7.01%) | 46.47 (+1.60, +3.57%) | 34.55 (+1.88, +5.74%) | 36.17 (+3.80, +11.74%) | 27.75 (+3.11, +12.60%) | 33.53 (+3.19, +10.53%) | 32.30 (+2.53, +8.51%) | 38.21 (+4.70, +14.03%) | 33.76 (+1.35, +4.17%) | 50.15 (+0.30, +0.60%) | 28.47 (+2.97, +11.63%) | 38.55 (-3.55, -8.43%) | 33.54 (+1.27, +3.93%) |
+| Translator/Dataset | es-en | en-pl | nb-en | bg-en | pt-en | it-en | et-en | en-cs | cs-en | en-it | pl-en | de-en | en-es | en-pt | en-et | en-bg | en-de |
+| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |
+| bergamot | 32.38 | 22.27 | 37.60 | 38.50 | 44.87 | 32.67 | 32.37 | 24.65 | 30.34 | 29.77 | 27.87 | 33.51 | 32.41 | 49.85 | 25.50 | 42.10 | 32.27 |
+| google | 33.64 (+1.27, +3.91%) | 25.50 (+3.23, +14.52%) | 42.05 (+4.45, +11.84%) | 41.30 (+2.80, +7.27%) | 46.60 (+1.73, +3.86%) | 34.50 (+1.83, +5.59%) | 35.80 (+3.43, +10.61%) | 26.73 (+2.09, +8.47%) | 32.40 (+2.06, +6.80%) | 28.97 (-0.80, -2.69%) | 31.23 (+3.37, +12.08%) | 35.98 (+2.48, +7.39%) | 34.74 (+2.32, +7.17%) | 53.75 (+3.90, +7.82%) | 28.60 (+3.10, +12.16%) | 44.60 (+2.50, +5.94%) | 33.05 (+0.77, +2.40%) |
+| microsoft | 32.93 (+0.56, +1.72%) | 24.83 (+2.57, +11.53%) | 42.90 (+5.30, +14.10%) | 41.20 (+2.70, +7.01%) | 46.47 (+1.60, +3.57%) | 34.55 (+1.88, +5.74%) | 36.17 (+3.80, +11.74%) | 27.75 (+3.11, +12.60%) | 33.53 (+3.19, +10.53%) | 32.30 (+2.53, +8.51%) | 31.83 (+3.97, +14.23%) | 38.21 (+4.70, +14.03%) | 33.76 (+1.35, +4.17%) | 50.15 (+0.30, +0.60%) | 28.47 (+2.97, +11.63%) | 38.55 (-3.55, -8.43%) | 33.54 (+1.27, +3.93%) |
 
 ![Results](img/avg.png)
 
@@ -75,6 +75,16 @@ Both absolute and relative differences in BLEU scores between Bergamot and other
 
 ![Results](img/es-en.png)
 
+## en-pl
+
+| Translator/Dataset | wmt20 | flores-test | flores-dev |
+| --- | --- | --- | --- |
+| bergamot | 25.10 | 21.00 | 20.70 |
+| google | 27.90 (+2.80, +11.16%) | 24.40 (+3.40, +16.19%) | 24.20 (+3.50, +16.91%) |
+| microsoft | 27.70 (+2.60, +10.36%) | 23.80 (+2.80, +13.33%) | 23.00 (+2.30, +11.11%) |
+
+![Results](img/en-pl.png)
+
 ## nb-en
 
 | Translator/Dataset | flores-dev | flores-test |
@@ -155,6 +165,16 @@ Both absolute and relative differences in BLEU scores between Bergamot and other
 
 ![Results](img/en-it.png)
 
+## pl-en
+
+| Translator/Dataset | wmt20 | flores-dev | flores-test |
+| --- | --- | --- | --- |
+| bergamot | 31.00 | 26.80 | 25.80 |
+| google | 34.10 (+3.10, +10.00%) | 30.00 (+3.20, +11.94%) | 29.60 (+3.80, +14.73%) |
+| microsoft | 35.50 (+4.50, +14.52%) | 30.10 (+3.30, +12.31%) | 29.90 (+4.10, +15.89%) |
+
+![Results](img/pl-en.png)
+
 ## de-en
 
 | Translator/Dataset | wmt17 | wmt10 | wmt18 | wmt09 | wmt14 | wmt11 | wmt16 | wmt13 | wmt19 | wmt20 | wmt08 | flores-dev | flores-test | wmt12 | wmt15 | iwslt17 |

diff --git a/models/prod/enpl/lex.50.50.enpl.s2t.bin.gz b/models/prod/enpl/lex.50.50.enpl.s2t.bin.gz
diff --git a/models/prod/enpl/model.enpl.intgemm.alphas.bin.gz b/models/prod/enpl/model.enpl.intgemm.alphas.bin.gz
diff --git a/models/prod/enpl/vocab.enpl.spm.gz b/models/prod/enpl/vocab.enpl.spm.gz
diff --git a/models/prod/plen/lex.50.50.plen.s2t.bin.gz b/models/prod/plen/lex.50.50.plen.s2t.bin.gz
diff --git a/models/prod/plen/model.plen.intgemm.alphas.bin.gz b/models/prod/plen/model.plen.intgemm.alphas.bin.gz
diff --git a/models/prod/plen/vocab.plen.spm.gz b/models/prod/plen/vocab.plen.spm.gz
diff --git a/registry.json b/registry.json
@@ -106,19 +106,19 @@
       "expectedSha256Hash": "e19c77231bf977988e31ff8db15fe79966b5170564bd3e10613f239e7f461d97",
       "modelType": "prod"
     },
-    "qualityModel": {
-      "name": "qualityModel.encs.bin",
-      "size": 68,
-      "estimatedCompressedSize": 108,
-      "expectedSha256Hash": "d7eba90036a065e4a1e93e889befe09f93a7d9a3417f3edffdb09a0db88fe83a",
-      "modelType": "prod"
-    },
     "vocab": {
       "name": "vocab.csen.spm",
       "size": 769763,
       "estimatedCompressedSize": 366392,
       "expectedSha256Hash": "f71cc5d045e479607078e079884f44032f5a0b82547fb96eefa29cd1eb47c6f3",
       "modelType": "prod"
+    },
+    "qualityModel": {
+      "name": "qualityModel.encs.bin",
+      "size": 68,
+      "estimatedCompressedSize": 108,
+      "expectedSha256Hash": "d7eba90036a065e4a1e93e889befe09f93a7d9a3417f3edffdb09a0db88fe83a",
+      "modelType": "prod"
     }
   },
   "ende": {
@@ -227,6 +227,29 @@
       "modelType": "prod"
     }
   },
+  "enpl": {
+    "model": {
+      "name": "model.enpl.intgemm.alphas.bin",
+      "size": 17140899,
+      "estimatedCompressedSize": 12797631,
+      "expectedSha256Hash": "60d45f43a5ac869a80f899751d2d1f0f456da9815d26db70e4d2e0fd18ed4a8f",
+      "modelType": "prod"
+    },
+    "lex": {
+      "name": "lex.50.50.enpl.s2t.bin",
+      "size": 3642112,
+      "estimatedCompressedSize": 1945174,
+      "expectedSha256Hash": "409fbf5856cec372dffe0a3aa3c89462e2efbd783557272af84800a67195c38c",
+      "modelType": "prod"
+    },
+    "vocab": {
+      "name": "vocab.enpl.spm",
+      "size": 822587,
+      "estimatedCompressedSize": 415308,
+      "expectedSha256Hash": "a1d27e6f5c0d29f406364ebf0382949d1c0affc750cec4380f3173150552f43e",
+      "modelType": "prod"
+    }
+  },
   "enpt": {
     "model": {
       "name": "model.enpt.intgemm.alphas.bin",
@@ -342,6 +365,29 @@
       "modelType": "prod"
     }
   },
+  "plen": {
+    "model": {
+      "name": "model.plen.intgemm.alphas.bin",
+      "size": 17140899,
+      "estimatedCompressedSize": 13421783,
+      "expectedSha256Hash": "172a5f1d44bf8dd6a6eec3868b13b33ce265f3530e898fe11a80b739b821726e",
+      "modelType": "prod"
+    },
+    "lex": {
+      "name": "lex.50.50.plen.s2t.bin",
+      "size": 4898024,
+      "estimatedCompressedSize": 2629586,
+      "expectedSha256Hash": "863afade0ba058fb0173fedef3d1fb14d0dcabc24c3b4584cb1fed8f84d6d879",
+      "modelType": "prod"
+    },
+    "vocab": {
+      "name": "vocab.plen.spm",
+      "size": 822587,
+      "estimatedCompressedSize": 415308,
+      "expectedSha256Hash": "a1d27e6f5c0d29f406364ebf0382949d1c0affc750cec4380f3173150552f43e",
+      "modelType": "prod"
+    }
+  },
   "pten": {
     "model": {
       "name": "model.pten.intgemm.alphas.bin",