diff --git a/README.md b/README.md index 9f74c520..034871bd 100644 --- a/README.md +++ b/README.md @@ -82,6 +82,7 @@ Suffix of the model file in the registry: - Portuguese <-> English - Italian <-> English - Polish <-> English +- French <-> English ## Dev - Russian <-> English @@ -91,4 +92,4 @@ Suffix of the model file in the registry: - Ukrainian <-> English ## Upcoming -- French <-> English +- Dutch <-> English diff --git a/evaluation/prod/en-fr/flores-dev.bergamot.fr.bleu b/evaluation/prod/en-fr/flores-dev.bergamot.fr.bleu new file mode 100644 index 00000000..0b320ba1 --- /dev/null +++ b/evaluation/prod/en-fr/flores-dev.bergamot.fr.bleu @@ -0,0 +1 @@ +48.5 diff --git a/evaluation/prod/en-fr/flores-dev.google.fr.bleu b/evaluation/prod/en-fr/flores-dev.google.fr.bleu new file mode 100644 index 00000000..d633c2f5 --- /dev/null +++ b/evaluation/prod/en-fr/flores-dev.google.fr.bleu @@ -0,0 +1 @@ +41.3 diff --git a/evaluation/prod/en-fr/flores-dev.microsoft.fr.bleu b/evaluation/prod/en-fr/flores-dev.microsoft.fr.bleu new file mode 100644 index 00000000..279d1d72 --- /dev/null +++ b/evaluation/prod/en-fr/flores-dev.microsoft.fr.bleu @@ -0,0 +1 @@ +52.5 diff --git a/evaluation/prod/en-fr/flores-test.bergamot.fr.bleu b/evaluation/prod/en-fr/flores-test.bergamot.fr.bleu new file mode 100644 index 00000000..be65a9b4 --- /dev/null +++ b/evaluation/prod/en-fr/flores-test.bergamot.fr.bleu @@ -0,0 +1 @@ +48.7 diff --git a/evaluation/prod/en-fr/flores-test.google.fr.bleu b/evaluation/prod/en-fr/flores-test.google.fr.bleu new file mode 100644 index 00000000..685e6c96 --- /dev/null +++ b/evaluation/prod/en-fr/flores-test.google.fr.bleu @@ -0,0 +1 @@ +41.8 diff --git a/evaluation/prod/en-fr/flores-test.microsoft.fr.bleu b/evaluation/prod/en-fr/flores-test.microsoft.fr.bleu new file mode 100644 index 00000000..3ed5cc5c --- /dev/null +++ b/evaluation/prod/en-fr/flores-test.microsoft.fr.bleu @@ -0,0 +1 @@ +52.7 diff --git a/evaluation/prod/en-fr/iwslt17.bergamot.fr.bleu b/evaluation/prod/en-fr/iwslt17.bergamot.fr.bleu new file mode 100644 index 00000000..31978e34 --- /dev/null +++ b/evaluation/prod/en-fr/iwslt17.bergamot.fr.bleu @@ -0,0 +1 @@ +38.6 diff --git a/evaluation/prod/en-fr/iwslt17.google.fr.bleu b/evaluation/prod/en-fr/iwslt17.google.fr.bleu new file mode 100644 index 00000000..ba70ecb3 --- /dev/null +++ b/evaluation/prod/en-fr/iwslt17.google.fr.bleu @@ -0,0 +1 @@ +28.0 diff --git a/evaluation/prod/en-fr/iwslt17.microsoft.fr.bleu b/evaluation/prod/en-fr/iwslt17.microsoft.fr.bleu new file mode 100644 index 00000000..9f47a225 --- /dev/null +++ b/evaluation/prod/en-fr/iwslt17.microsoft.fr.bleu @@ -0,0 +1 @@ +36.5 diff --git a/evaluation/prod/en-fr/wmt08.bergamot.fr.bleu b/evaluation/prod/en-fr/wmt08.bergamot.fr.bleu new file mode 100644 index 00000000..acae4ca4 --- /dev/null +++ b/evaluation/prod/en-fr/wmt08.bergamot.fr.bleu @@ -0,0 +1 @@ +25.5 diff --git a/evaluation/prod/en-fr/wmt08.google.fr.bleu b/evaluation/prod/en-fr/wmt08.google.fr.bleu new file mode 100644 index 00000000..b1b396bc --- /dev/null +++ b/evaluation/prod/en-fr/wmt08.google.fr.bleu @@ -0,0 +1 @@ +20.7 diff --git a/evaluation/prod/en-fr/wmt08.microsoft.fr.bleu b/evaluation/prod/en-fr/wmt08.microsoft.fr.bleu new file mode 100644 index 00000000..8b9bee5b --- /dev/null +++ b/evaluation/prod/en-fr/wmt08.microsoft.fr.bleu @@ -0,0 +1 @@ +25.1 diff --git a/evaluation/prod/en-fr/wmt09.bergamot.fr.bleu b/evaluation/prod/en-fr/wmt09.bergamot.fr.bleu new file mode 100644 index 00000000..e55ffd7e --- /dev/null +++ b/evaluation/prod/en-fr/wmt09.bergamot.fr.bleu @@ -0,0 +1 @@ +28.8 diff --git a/evaluation/prod/en-fr/wmt09.google.fr.bleu b/evaluation/prod/en-fr/wmt09.google.fr.bleu new file mode 100644 index 00000000..ccf2a368 --- /dev/null +++ b/evaluation/prod/en-fr/wmt09.google.fr.bleu @@ -0,0 +1 @@ +23.5 diff --git a/evaluation/prod/en-fr/wmt09.microsoft.fr.bleu b/evaluation/prod/en-fr/wmt09.microsoft.fr.bleu new file mode 100644 index 00000000..2862be46 --- /dev/null +++ b/evaluation/prod/en-fr/wmt09.microsoft.fr.bleu @@ -0,0 +1 @@ +28.2 diff --git a/evaluation/prod/en-fr/wmt10.bergamot.fr.bleu b/evaluation/prod/en-fr/wmt10.bergamot.fr.bleu new file mode 100644 index 00000000..72d50cd4 --- /dev/null +++ b/evaluation/prod/en-fr/wmt10.bergamot.fr.bleu @@ -0,0 +1 @@ +31.0 diff --git a/evaluation/prod/en-fr/wmt10.google.fr.bleu b/evaluation/prod/en-fr/wmt10.google.fr.bleu new file mode 100644 index 00000000..7363977a --- /dev/null +++ b/evaluation/prod/en-fr/wmt10.google.fr.bleu @@ -0,0 +1 @@ +26.6 diff --git a/evaluation/prod/en-fr/wmt10.microsoft.fr.bleu b/evaluation/prod/en-fr/wmt10.microsoft.fr.bleu new file mode 100644 index 00000000..244a88ae --- /dev/null +++ b/evaluation/prod/en-fr/wmt10.microsoft.fr.bleu @@ -0,0 +1 @@ +33.0 diff --git a/evaluation/prod/en-fr/wmt11.bergamot.fr.bleu b/evaluation/prod/en-fr/wmt11.bergamot.fr.bleu new file mode 100644 index 00000000..2bf965e0 --- /dev/null +++ b/evaluation/prod/en-fr/wmt11.bergamot.fr.bleu @@ -0,0 +1 @@ +33.6 diff --git a/evaluation/prod/en-fr/wmt11.google.fr.bleu b/evaluation/prod/en-fr/wmt11.google.fr.bleu new file mode 100644 index 00000000..1bb9ac15 --- /dev/null +++ b/evaluation/prod/en-fr/wmt11.google.fr.bleu @@ -0,0 +1 @@ +27.5 diff --git a/evaluation/prod/en-fr/wmt11.microsoft.fr.bleu b/evaluation/prod/en-fr/wmt11.microsoft.fr.bleu new file mode 100644 index 00000000..3e5ca532 --- /dev/null +++ b/evaluation/prod/en-fr/wmt11.microsoft.fr.bleu @@ -0,0 +1 @@ +32.1 diff --git a/evaluation/prod/en-fr/wmt12.bergamot.fr.bleu b/evaluation/prod/en-fr/wmt12.bergamot.fr.bleu new file mode 100644 index 00000000..cfedf521 --- /dev/null +++ b/evaluation/prod/en-fr/wmt12.bergamot.fr.bleu @@ -0,0 +1 @@ +31.4 diff --git a/evaluation/prod/en-fr/wmt12.google.fr.bleu b/evaluation/prod/en-fr/wmt12.google.fr.bleu new file mode 100644 index 00000000..8b9bee5b --- /dev/null +++ b/evaluation/prod/en-fr/wmt12.google.fr.bleu @@ -0,0 +1 @@ +25.1 diff --git a/evaluation/prod/en-fr/wmt12.microsoft.fr.bleu b/evaluation/prod/en-fr/wmt12.microsoft.fr.bleu new file mode 100644 index 00000000..ac17b240 --- /dev/null +++ b/evaluation/prod/en-fr/wmt12.microsoft.fr.bleu @@ -0,0 +1 @@ +29.6 diff --git a/evaluation/prod/en-fr/wmt13.bergamot.fr.bleu b/evaluation/prod/en-fr/wmt13.bergamot.fr.bleu new file mode 100644 index 00000000..f823d47a --- /dev/null +++ b/evaluation/prod/en-fr/wmt13.bergamot.fr.bleu @@ -0,0 +1 @@ +33.4 diff --git a/evaluation/prod/en-fr/wmt13.google.fr.bleu b/evaluation/prod/en-fr/wmt13.google.fr.bleu new file mode 100644 index 00000000..58eede2a --- /dev/null +++ b/evaluation/prod/en-fr/wmt13.google.fr.bleu @@ -0,0 +1 @@ +26.5 diff --git a/evaluation/prod/en-fr/wmt13.microsoft.fr.bleu b/evaluation/prod/en-fr/wmt13.microsoft.fr.bleu new file mode 100644 index 00000000..2caea500 --- /dev/null +++ b/evaluation/prod/en-fr/wmt13.microsoft.fr.bleu @@ -0,0 +1 @@ +31.5 diff --git a/evaluation/prod/en-fr/wmt14.bergamot.fr.bleu b/evaluation/prod/en-fr/wmt14.bergamot.fr.bleu new file mode 100644 index 00000000..f940b05c --- /dev/null +++ b/evaluation/prod/en-fr/wmt14.bergamot.fr.bleu @@ -0,0 +1 @@ +39.7 diff --git a/evaluation/prod/en-fr/wmt14.google.fr.bleu b/evaluation/prod/en-fr/wmt14.google.fr.bleu new file mode 100644 index 00000000..02a964cf --- /dev/null +++ b/evaluation/prod/en-fr/wmt14.google.fr.bleu @@ -0,0 +1 @@ +32.6 diff --git a/evaluation/prod/en-fr/wmt14.microsoft.fr.bleu b/evaluation/prod/en-fr/wmt14.microsoft.fr.bleu new file mode 100644 index 00000000..6d971ddc --- /dev/null +++ b/evaluation/prod/en-fr/wmt14.microsoft.fr.bleu @@ -0,0 +1 @@ +40.4 diff --git a/evaluation/prod/en-fr/wmt15.bergamot.fr.bleu b/evaluation/prod/en-fr/wmt15.bergamot.fr.bleu new file mode 100644 index 00000000..4282804f --- /dev/null +++ b/evaluation/prod/en-fr/wmt15.bergamot.fr.bleu @@ -0,0 +1 @@ +36.9 diff --git a/evaluation/prod/en-fr/wmt15.google.fr.bleu b/evaluation/prod/en-fr/wmt15.google.fr.bleu new file mode 100644 index 00000000..60008a3e --- /dev/null +++ b/evaluation/prod/en-fr/wmt15.google.fr.bleu @@ -0,0 +1 @@ +30.6 diff --git a/evaluation/prod/en-fr/wmt15.microsoft.fr.bleu b/evaluation/prod/en-fr/wmt15.microsoft.fr.bleu new file mode 100644 index 00000000..f940b05c --- /dev/null +++ b/evaluation/prod/en-fr/wmt15.microsoft.fr.bleu @@ -0,0 +1 @@ +39.7 diff --git a/evaluation/prod/fr-en/flores-dev.bergamot.en.bleu b/evaluation/prod/fr-en/flores-dev.bergamot.en.bleu new file mode 100644 index 00000000..b9f48ef8 --- /dev/null +++ b/evaluation/prod/fr-en/flores-dev.bergamot.en.bleu @@ -0,0 +1 @@ +43.8 diff --git a/evaluation/prod/fr-en/flores-dev.google.en.bleu b/evaluation/prod/fr-en/flores-dev.google.en.bleu new file mode 100644 index 00000000..be65a9b4 --- /dev/null +++ b/evaluation/prod/fr-en/flores-dev.google.en.bleu @@ -0,0 +1 @@ +48.7 diff --git a/evaluation/prod/fr-en/flores-dev.microsoft.en.bleu b/evaluation/prod/fr-en/flores-dev.microsoft.en.bleu new file mode 100644 index 00000000..7c7f6b9a --- /dev/null +++ b/evaluation/prod/fr-en/flores-dev.microsoft.en.bleu @@ -0,0 +1 @@ +48.9 diff --git a/evaluation/prod/fr-en/flores-test.bergamot.en.bleu b/evaluation/prod/fr-en/flores-test.bergamot.en.bleu new file mode 100644 index 00000000..40e63b52 --- /dev/null +++ b/evaluation/prod/fr-en/flores-test.bergamot.en.bleu @@ -0,0 +1 @@ +42.1 diff --git a/evaluation/prod/fr-en/flores-test.google.en.bleu b/evaluation/prod/fr-en/flores-test.google.en.bleu new file mode 100644 index 00000000..f2287590 --- /dev/null +++ b/evaluation/prod/fr-en/flores-test.google.en.bleu @@ -0,0 +1 @@ +46.7 diff --git a/evaluation/prod/fr-en/flores-test.microsoft.en.bleu b/evaluation/prod/fr-en/flores-test.microsoft.en.bleu new file mode 100644 index 00000000..98bc1f4e --- /dev/null +++ b/evaluation/prod/fr-en/flores-test.microsoft.en.bleu @@ -0,0 +1 @@ +47.0 diff --git a/evaluation/prod/fr-en/iwslt17.bergamot.en.bleu b/evaluation/prod/fr-en/iwslt17.bergamot.en.bleu new file mode 100644 index 00000000..9f78f6a2 --- /dev/null +++ b/evaluation/prod/fr-en/iwslt17.bergamot.en.bleu @@ -0,0 +1 @@ +39.8 diff --git a/evaluation/prod/fr-en/iwslt17.google.en.bleu b/evaluation/prod/fr-en/iwslt17.google.en.bleu new file mode 100644 index 00000000..5e81218d --- /dev/null +++ b/evaluation/prod/fr-en/iwslt17.google.en.bleu @@ -0,0 +1 @@ +40.6 diff --git a/evaluation/prod/fr-en/iwslt17.microsoft.en.bleu b/evaluation/prod/fr-en/iwslt17.microsoft.en.bleu new file mode 100644 index 00000000..685e6c96 --- /dev/null +++ b/evaluation/prod/fr-en/iwslt17.microsoft.en.bleu @@ -0,0 +1 @@ +41.8 diff --git a/evaluation/prod/fr-en/mtedx_test.bergamot.en.bleu b/evaluation/prod/fr-en/mtedx_test.bergamot.en.bleu new file mode 100644 index 00000000..fffcd081 --- /dev/null +++ b/evaluation/prod/fr-en/mtedx_test.bergamot.en.bleu @@ -0,0 +1 @@ +42.8 diff --git a/evaluation/prod/fr-en/mtedx_test.google.en.bleu b/evaluation/prod/fr-en/mtedx_test.google.en.bleu new file mode 100644 index 00000000..8744ebcd --- /dev/null +++ b/evaluation/prod/fr-en/mtedx_test.google.en.bleu @@ -0,0 +1 @@ +42.7 diff --git a/evaluation/prod/fr-en/mtedx_test.microsoft.en.bleu b/evaluation/prod/fr-en/mtedx_test.microsoft.en.bleu new file mode 100644 index 00000000..fa0bf02a --- /dev/null +++ b/evaluation/prod/fr-en/mtedx_test.microsoft.en.bleu @@ -0,0 +1 @@ +46.4 diff --git a/evaluation/prod/fr-en/wmt08.bergamot.en.bleu b/evaluation/prod/fr-en/wmt08.bergamot.en.bleu new file mode 100644 index 00000000..f5eb6934 --- /dev/null +++ b/evaluation/prod/fr-en/wmt08.bergamot.en.bleu @@ -0,0 +1 @@ +24.5 diff --git a/evaluation/prod/fr-en/wmt08.google.en.bleu b/evaluation/prod/fr-en/wmt08.google.en.bleu new file mode 100644 index 00000000..7363977a --- /dev/null +++ b/evaluation/prod/fr-en/wmt08.google.en.bleu @@ -0,0 +1 @@ +26.6 diff --git a/evaluation/prod/fr-en/wmt08.microsoft.en.bleu b/evaluation/prod/fr-en/wmt08.microsoft.en.bleu new file mode 100644 index 00000000..2ca295b4 --- /dev/null +++ b/evaluation/prod/fr-en/wmt08.microsoft.en.bleu @@ -0,0 +1 @@ +27.4 diff --git a/evaluation/prod/fr-en/wmt09.bergamot.en.bleu b/evaluation/prod/fr-en/wmt09.bergamot.en.bleu new file mode 100644 index 00000000..f5988535 --- /dev/null +++ b/evaluation/prod/fr-en/wmt09.bergamot.en.bleu @@ -0,0 +1 @@ +29.1 diff --git a/evaluation/prod/fr-en/wmt09.google.en.bleu b/evaluation/prod/fr-en/wmt09.google.en.bleu new file mode 100644 index 00000000..dd170dab --- /dev/null +++ b/evaluation/prod/fr-en/wmt09.google.en.bleu @@ -0,0 +1 @@ +31.2 diff --git a/evaluation/prod/fr-en/wmt09.microsoft.en.bleu b/evaluation/prod/fr-en/wmt09.microsoft.en.bleu new file mode 100644 index 00000000..4d40a2c7 --- /dev/null +++ b/evaluation/prod/fr-en/wmt09.microsoft.en.bleu @@ -0,0 +1 @@ +32.2 diff --git a/evaluation/prod/fr-en/wmt10.bergamot.en.bleu b/evaluation/prod/fr-en/wmt10.bergamot.en.bleu new file mode 100644 index 00000000..cfedf521 --- /dev/null +++ b/evaluation/prod/fr-en/wmt10.bergamot.en.bleu @@ -0,0 +1 @@ +31.4 diff --git a/evaluation/prod/fr-en/wmt10.google.en.bleu b/evaluation/prod/fr-en/wmt10.google.en.bleu new file mode 100644 index 00000000..efb8b294 --- /dev/null +++ b/evaluation/prod/fr-en/wmt10.google.en.bleu @@ -0,0 +1 @@ +34.1 diff --git a/evaluation/prod/fr-en/wmt10.microsoft.en.bleu b/evaluation/prod/fr-en/wmt10.microsoft.en.bleu new file mode 100644 index 00000000..f0b6ebab --- /dev/null +++ b/evaluation/prod/fr-en/wmt10.microsoft.en.bleu @@ -0,0 +1 @@ +35.0 diff --git a/evaluation/prod/fr-en/wmt11.bergamot.en.bleu b/evaluation/prod/fr-en/wmt11.bergamot.en.bleu new file mode 100644 index 00000000..593224b3 --- /dev/null +++ b/evaluation/prod/fr-en/wmt11.bergamot.en.bleu @@ -0,0 +1 @@ +32.0 diff --git a/evaluation/prod/fr-en/wmt11.google.en.bleu b/evaluation/prod/fr-en/wmt11.google.en.bleu new file mode 100644 index 00000000..e2b901b5 --- /dev/null +++ b/evaluation/prod/fr-en/wmt11.google.en.bleu @@ -0,0 +1 @@ +34.3 diff --git a/evaluation/prod/fr-en/wmt11.microsoft.en.bleu b/evaluation/prod/fr-en/wmt11.microsoft.en.bleu new file mode 100644 index 00000000..005c721d --- /dev/null +++ b/evaluation/prod/fr-en/wmt11.microsoft.en.bleu @@ -0,0 +1 @@ +35.2 diff --git a/evaluation/prod/fr-en/wmt12.bergamot.en.bleu b/evaluation/prod/fr-en/wmt12.bergamot.en.bleu new file mode 100644 index 00000000..e34742bb --- /dev/null +++ b/evaluation/prod/fr-en/wmt12.bergamot.en.bleu @@ -0,0 +1 @@ +31.9 diff --git a/evaluation/prod/fr-en/wmt12.google.en.bleu b/evaluation/prod/fr-en/wmt12.google.en.bleu new file mode 100644 index 00000000..01232ed7 --- /dev/null +++ b/evaluation/prod/fr-en/wmt12.google.en.bleu @@ -0,0 +1 @@ +33.8 diff --git a/evaluation/prod/fr-en/wmt12.microsoft.en.bleu b/evaluation/prod/fr-en/wmt12.microsoft.en.bleu new file mode 100644 index 00000000..0d7882e6 --- /dev/null +++ b/evaluation/prod/fr-en/wmt12.microsoft.en.bleu @@ -0,0 +1 @@ +34.6 diff --git a/evaluation/prod/fr-en/wmt13.bergamot.en.bleu b/evaluation/prod/fr-en/wmt13.bergamot.en.bleu new file mode 100644 index 00000000..2097bae3 --- /dev/null +++ b/evaluation/prod/fr-en/wmt13.bergamot.en.bleu @@ -0,0 +1 @@ +33.3 diff --git a/evaluation/prod/fr-en/wmt13.google.en.bleu b/evaluation/prod/fr-en/wmt13.google.en.bleu new file mode 100644 index 00000000..49215d61 --- /dev/null +++ b/evaluation/prod/fr-en/wmt13.google.en.bleu @@ -0,0 +1 @@ +34.5 diff --git a/evaluation/prod/fr-en/wmt13.microsoft.en.bleu b/evaluation/prod/fr-en/wmt13.microsoft.en.bleu new file mode 100644 index 00000000..2b194df7 --- /dev/null +++ b/evaluation/prod/fr-en/wmt13.microsoft.en.bleu @@ -0,0 +1 @@ +36.1 diff --git a/evaluation/prod/fr-en/wmt14.bergamot.en.bleu b/evaluation/prod/fr-en/wmt14.bergamot.en.bleu new file mode 100644 index 00000000..6bc73938 --- /dev/null +++ b/evaluation/prod/fr-en/wmt14.bergamot.en.bleu @@ -0,0 +1 @@ +37.3 diff --git a/evaluation/prod/fr-en/wmt14.google.en.bleu b/evaluation/prod/fr-en/wmt14.google.en.bleu new file mode 100644 index 00000000..5e81218d --- /dev/null +++ b/evaluation/prod/fr-en/wmt14.google.en.bleu @@ -0,0 +1 @@ +40.6 diff --git a/evaluation/prod/fr-en/wmt14.microsoft.en.bleu b/evaluation/prod/fr-en/wmt14.microsoft.en.bleu new file mode 100644 index 00000000..e2852b2d --- /dev/null +++ b/evaluation/prod/fr-en/wmt14.microsoft.en.bleu @@ -0,0 +1 @@ +42.3 diff --git a/evaluation/prod/fr-en/wmt15.bergamot.en.bleu b/evaluation/prod/fr-en/wmt15.bergamot.en.bleu new file mode 100644 index 00000000..ad4af79d --- /dev/null +++ b/evaluation/prod/fr-en/wmt15.bergamot.en.bleu @@ -0,0 +1 @@ +37.2 diff --git a/evaluation/prod/fr-en/wmt15.google.en.bleu b/evaluation/prod/fr-en/wmt15.google.en.bleu new file mode 100644 index 00000000..45d0c841 --- /dev/null +++ b/evaluation/prod/fr-en/wmt15.google.en.bleu @@ -0,0 +1 @@ +39.9 diff --git a/evaluation/prod/fr-en/wmt15.microsoft.en.bleu b/evaluation/prod/fr-en/wmt15.microsoft.en.bleu new file mode 100644 index 00000000..8744ebcd --- /dev/null +++ b/evaluation/prod/fr-en/wmt15.microsoft.en.bleu @@ -0,0 +1 @@ +42.7 diff --git a/evaluation/prod/img/avg.png b/evaluation/prod/img/avg.png index 8a8fd170..84bbd4c8 100644 Binary files a/evaluation/prod/img/avg.png and b/evaluation/prod/img/avg.png differ diff --git a/evaluation/prod/img/en-fr.png b/evaluation/prod/img/en-fr.png new file mode 100644 index 00000000..1bcd531a Binary files /dev/null and b/evaluation/prod/img/en-fr.png differ diff --git a/evaluation/prod/img/fr-en.png b/evaluation/prod/img/fr-en.png new file mode 100644 index 00000000..55ff7bdc Binary files /dev/null and b/evaluation/prod/img/fr-en.png differ diff --git a/evaluation/prod/results.md b/evaluation/prod/results.md index cf6c96ec..562caf5f 100644 --- a/evaluation/prod/results.md +++ b/evaluation/prod/results.md @@ -57,14 +57,24 @@ Both absolute and relative differences in BLEU scores between Bergamot and other ## avg -| Translator/Dataset | es-en | en-pl | nb-en | bg-en | pt-en | it-en | et-en | en-cs | cs-en | en-it | pl-en | de-en | en-es | en-pt | en-et | en-bg | en-de | -| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | -| bergamot | 32.38 | 22.27 | 37.60 | 38.50 | 44.87 | 32.67 | 32.37 | 24.65 | 30.34 | 29.77 | 27.87 | 33.51 | 32.41 | 49.85 | 25.50 | 42.10 | 32.27 | -| google | 33.64 (+1.27, +3.91%) | 25.50 (+3.23, +14.52%) | 42.05 (+4.45, +11.84%) | 41.30 (+2.80, +7.27%) | 46.60 (+1.73, +3.86%) | 34.50 (+1.83, +5.59%) | 35.80 (+3.43, +10.61%) | 26.73 (+2.09, +8.47%) | 32.40 (+2.06, +6.80%) | 28.97 (-0.80, -2.69%) | 31.23 (+3.37, +12.08%) | 35.98 (+2.48, +7.39%) | 34.74 (+2.32, +7.17%) | 53.75 (+3.90, +7.82%) | 28.60 (+3.10, +12.16%) | 44.60 (+2.50, +5.94%) | 33.05 (+0.77, +2.40%) | -| microsoft | 32.93 (+0.56, +1.72%) | 24.83 (+2.57, +11.53%) | 42.90 (+5.30, +14.10%) | 41.20 (+2.70, +7.01%) | 46.47 (+1.60, +3.57%) | 34.55 (+1.88, +5.74%) | 36.17 (+3.80, +11.74%) | 27.75 (+3.11, +12.60%) | 33.53 (+3.19, +10.53%) | 32.30 (+2.53, +8.51%) | 31.83 (+3.97, +14.23%) | 38.21 (+4.70, +14.03%) | 33.76 (+1.35, +4.17%) | 50.15 (+0.30, +0.60%) | 28.47 (+2.97, +11.63%) | 38.55 (-3.55, -8.43%) | 33.54 (+1.27, +3.93%) | +| Translator/Dataset | en-fr | es-en | en-pl | nb-en | bg-en | pt-en | it-en | et-en | fr-en | en-cs | cs-en | en-it | pl-en | de-en | en-es | en-pt | en-et | en-bg | en-de | +| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | +| bergamot | 36.01 | 32.38 | 22.27 | 37.60 | 38.50 | 44.87 | 32.67 | 32.37 | 35.43 | 24.65 | 30.34 | 29.77 | 27.87 | 33.51 | 32.41 | 49.85 | 25.50 | 42.10 | 32.27 | +| google | 29.47 (-6.54, -18.15%) | 33.64 (+1.27, +3.91%) | 25.50 (+3.23, +14.52%) | 42.05 (+4.45, +11.84%) | 41.30 (+2.80, +7.27%) | 46.60 (+1.73, +3.86%) | 34.50 (+1.83, +5.59%) | 35.80 (+3.43, +10.61%) | 37.81 (+2.38, +6.70%) | 26.73 (+2.09, +8.47%) | 32.40 (+2.06, +6.80%) | 28.97 (-0.80, -2.69%) | 31.23 (+3.37, +12.08%) | 35.98 (+2.48, +7.39%) | 34.74 (+2.32, +7.17%) | 53.75 (+3.90, +7.82%) | 28.60 (+3.10, +12.16%) | 44.60 (+2.50, +5.94%) | 33.05 (+0.77, +2.40%) | +| microsoft | 36.48 (+0.47, +1.31%) | 32.93 (+0.56, +1.72%) | 24.83 (+2.57, +11.53%) | 42.90 (+5.30, +14.10%) | 41.20 (+2.70, +7.01%) | 46.47 (+1.60, +3.57%) | 34.55 (+1.88, +5.74%) | 36.17 (+3.80, +11.74%) | 39.13 (+3.70, +10.44%) | 27.75 (+3.11, +12.60%) | 33.53 (+3.19, +10.53%) | 32.30 (+2.53, +8.51%) | 31.83 (+3.97, +14.23%) | 38.21 (+4.70, +14.03%) | 33.76 (+1.35, +4.17%) | 50.15 (+0.30, +0.60%) | 28.47 (+2.97, +11.63%) | 38.55 (-3.55, -8.43%) | 33.54 (+1.27, +3.93%) | ![Results](img/avg.png) +## en-fr + +| Translator/Dataset | wmt15 | wmt13 | wmt12 | wmt09 | flores-test | wmt10 | iwslt17 | wmt14 | flores-dev | wmt08 | wmt11 | +| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | +| bergamot | 36.90 | 33.40 | 31.40 | 28.80 | 48.70 | 31.00 | 38.60 | 39.70 | 48.50 | 25.50 | 33.60 | +| google | 30.60 (-6.30, -17.07%) | 26.50 (-6.90, -20.66%) | 25.10 (-6.30, -20.06%) | 23.50 (-5.30, -18.40%) | 41.80 (-6.90, -14.17%) | 26.60 (-4.40, -14.19%) | 28.00 (-10.60, -27.46%) | 32.60 (-7.10, -17.88%) | 41.30 (-7.20, -14.85%) | 20.70 (-4.80, -18.82%) | 27.50 (-6.10, -18.15%) | +| microsoft | 39.70 (+2.80, +7.59%) | 31.50 (-1.90, -5.69%) | 29.60 (-1.80, -5.73%) | 28.20 (-0.60, -2.08%) | 52.70 (+4.00, +8.21%) | 33.00 (+2.00, +6.45%) | 36.50 (-2.10, -5.44%) | 40.40 (+0.70, +1.76%) | 52.50 (+4.00, +8.25%) | 25.10 (-0.40, -1.57%) | 32.10 (-1.50, -4.46%) | + +![Results](img/en-fr.png) + ## es-en | Translator/Dataset | wmt10 | wmt09 | wmt11 | mtedx_test | wmt13 | wmt08 | flores-dev | flores-test | wmt12 | @@ -135,6 +145,16 @@ Both absolute and relative differences in BLEU scores between Bergamot and other ![Results](img/et-en.png) +## fr-en + +| Translator/Dataset | wmt10 | wmt09 | wmt14 | wmt11 | mtedx_test | wmt13 | wmt08 | flores-dev | flores-test | wmt12 | wmt15 | iwslt17 | +| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | +| bergamot | 31.40 | 29.10 | 37.30 | 32.00 | 42.80 | 33.30 | 24.50 | 43.80 | 42.10 | 31.90 | 37.20 | 39.80 | +| google | 34.10 (+2.70, +8.60%) | 31.20 (+2.10, +7.22%) | 40.60 (+3.30, +8.85%) | 34.30 (+2.30, +7.19%) | 42.70 (-0.10, -0.23%) | 34.50 (+1.20, +3.60%) | 26.60 (+2.10, +8.57%) | 48.70 (+4.90, +11.19%) | 46.70 (+4.60, +10.93%) | 33.80 (+1.90, +5.96%) | 39.90 (+2.70, +7.26%) | 40.60 (+0.80, +2.01%) | +| microsoft | 35.00 (+3.60, +11.46%) | 32.20 (+3.10, +10.65%) | 42.30 (+5.00, +13.40%) | 35.20 (+3.20, +10.00%) | 46.40 (+3.60, +8.41%) | 36.10 (+2.80, +8.41%) | 27.40 (+2.90, +11.84%) | 48.90 (+5.10, +11.64%) | 47.00 (+4.90, +11.64%) | 34.60 (+2.70, +8.46%) | 42.70 (+5.50, +14.78%) | 41.80 (+2.00, +5.03%) | + +![Results](img/fr-en.png) + ## en-cs | Translator/Dataset | wmt10 | flores-dev | wmt11 | wmt18 | flores-test | wmt13 | wmt15 | wmt08 | wmt09 | wmt19 | wmt14 | wmt17 | wmt20 | wmt16 | wmt12 | diff --git a/models/prod/enfr/lex.50.50.enfr.s2t.bin.gz b/models/prod/enfr/lex.50.50.enfr.s2t.bin.gz new file mode 100644 index 00000000..32ba76e8 --- /dev/null +++ b/models/prod/enfr/lex.50.50.enfr.s2t.bin.gz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ed5eaebf198b787b718b81948ddb184780a559e6eefdb7416fdf405ef3e50576 +size 4177155 diff --git a/models/prod/enfr/model.enfr.intgemm.alphas.bin.gz b/models/prod/enfr/model.enfr.intgemm.alphas.bin.gz new file mode 100644 index 00000000..b6b88a15 --- /dev/null +++ b/models/prod/enfr/model.enfr.intgemm.alphas.bin.gz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f39efbb6ab154967e2762f7c4baa0ff2f2fa08f32bfe5f6d29b787726476e828 +size 12293754 diff --git a/models/prod/enfr/vocab.fren.spm.gz b/models/prod/enfr/vocab.fren.spm.gz new file mode 100644 index 00000000..d2ba3631 --- /dev/null +++ b/models/prod/enfr/vocab.fren.spm.gz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a6822a172454fcf6acba0c926cde59022e4d525b0052dd8c89fb7bc76a1542e0 +size 419721 diff --git a/models/prod/fren/lex.50.50.fren.s2t.bin.gz b/models/prod/fren/lex.50.50.fren.s2t.bin.gz new file mode 100644 index 00000000..c469fe2b --- /dev/null +++ b/models/prod/fren/lex.50.50.fren.s2t.bin.gz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:17b933f2b42516ed0d325dd862ee7c50acc74ffc5d2a66059b357b931f788df7 +size 4761904 diff --git a/models/prod/fren/model.fren.intgemm.alphas.bin.gz b/models/prod/fren/model.fren.intgemm.alphas.bin.gz new file mode 100644 index 00000000..ff5af0a6 --- /dev/null +++ b/models/prod/fren/model.fren.intgemm.alphas.bin.gz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4f5618b93d08bd82bdafc096b5cdfe459e24b263f0a5a23dcf641a070ebb60b5 +size 12641501 diff --git a/models/prod/fren/vocab.fren.spm.gz b/models/prod/fren/vocab.fren.spm.gz new file mode 100644 index 00000000..a7a99fc5 --- /dev/null +++ b/models/prod/fren/vocab.fren.spm.gz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e90e0e635234445df4defafdea8aff23f8a0d68c73744462c021b5dbff36e55f +size 419721 diff --git a/registry.json b/registry.json index 06175dce..60d7120c 100644 --- a/registry.json +++ b/registry.json @@ -106,19 +106,19 @@ "expectedSha256Hash": "e19c77231bf977988e31ff8db15fe79966b5170564bd3e10613f239e7f461d97", "modelType": "prod" }, - "qualityModel": { - "name": "qualityModel.encs.bin", - "size": 68, - "estimatedCompressedSize": 108, - "expectedSha256Hash": "d7eba90036a065e4a1e93e889befe09f93a7d9a3417f3edffdb09a0db88fe83a", - "modelType": "prod" - }, "vocab": { "name": "vocab.csen.spm", "size": 769763, "estimatedCompressedSize": 366392, "expectedSha256Hash": "f71cc5d045e479607078e079884f44032f5a0b82547fb96eefa29cd1eb47c6f3", "modelType": "prod" + }, + "qualityModel": { + "name": "qualityModel.encs.bin", + "size": 68, + "estimatedCompressedSize": 108, + "expectedSha256Hash": "d7eba90036a065e4a1e93e889befe09f93a7d9a3417f3edffdb09a0db88fe83a", + "modelType": "prod" } }, "ende": { @@ -204,6 +204,29 @@ "modelType": "prod" } }, + "enfr": { + "model": { + "name": "model.enfr.intgemm.alphas.bin", + "size": 17140961, + "estimatedCompressedSize": 12293754, + "expectedSha256Hash": "0678019c4d74c8c81d2de17e3e58d3aba5f5eb48f5595d9240c17f69d30461de", + "modelType": "prod" + }, + "lex": { + "name": "lex.50.50.enfr.s2t.bin", + "size": 7886500, + "estimatedCompressedSize": 4177155, + "expectedSha256Hash": "38fb44bad1fd5f1e6bfdcf15cc8baa09d61aad2a4f9c587914e24e7b5c25c32c", + "modelType": "prod" + }, + "vocab": { + "name": "vocab.fren.spm", + "size": 831382, + "estimatedCompressedSize": 419721, + "expectedSha256Hash": "4c84b95b62c930f0791466d73eb996841eef474c96d0c2f0e6c6d80640f2005a", + "modelType": "prod" + } + }, "enit": { "model": { "name": "model.enit.intgemm.alphas.bin", @@ -319,6 +342,29 @@ "modelType": "prod" } }, + "fren": { + "model": { + "name": "model.fren.intgemm.alphas.bin", + "size": 17140961, + "estimatedCompressedSize": 12641501, + "expectedSha256Hash": "185f76d24c2d400fe4ea0cb2487df77722641b97a3ef10633872e8a7fdf40e09", + "modelType": "prod" + }, + "lex": { + "name": "lex.50.50.fren.s2t.bin", + "size": 8818768, + "estimatedCompressedSize": 4761904, + "expectedSha256Hash": "3148abf21ea98a4d69d0e4504e0d68a6c060204a9b9a39b76855aee1d5b2c8ea", + "modelType": "prod" + }, + "vocab": { + "name": "vocab.fren.spm", + "size": 831382, + "estimatedCompressedSize": 419721, + "expectedSha256Hash": "4c84b95b62c930f0791466d73eb996841eef474c96d0c2f0e6c6d80640f2005a", + "modelType": "prod" + } + }, "iten": { "model": { "name": "model.iten.intgemm.alphas.bin", diff --git a/scripts/import-statmt.sh b/scripts/import-statmt.sh index 965b7983..db0574b3 100644 --- a/scripts/import-statmt.sh +++ b/scripts/import-statmt.sh @@ -2,13 +2,12 @@ pair=$1 -mkdir -P models/prod/$pair +mkdir -p models/prod/$pair # directory structure is not consistent on statmt -wget -nc -P models/prod/$pair/ http://data.statmt.org/bergamot/models/$pair/$pair.student.tiny11/lex.s2t.bin -wget -nc -P models/prod/$pair/ http://data.statmt.org/bergamot/models/$pair/$pair.student.tiny11/model.intgemm.alphas.bin -#wget -nc -P models/prod/$pair/ http://data.statmt.org/bergamot/models/$pair/$pair.student.tiny11/vocab.$pair.spm -wget -nc -P models/prod/$pair/ http://data.statmt.org/bergamot/models/$pair/vocab.nben.spm +wget -nc -P models/prod/$pair/ http://data.statmt.org/bergamot/models/fren/$pair/tiny11/lex.s2t.bin +wget -nc -P models/prod/$pair/ http://data.statmt.org/bergamot/models/fren/$pair/tiny11/model.intgemm.alphas.bin +wget -nc -P models/prod/$pair/ http://data.statmt.org/bergamot/models/fren/$pair/tiny11/vocab.fren.spm # follow our naming convention mv models/prod/$pair/lex.s2t.bin models/prod/$pair/lex.50.50.$pair.s2t.bin