Merge branch 'master' into feature/python-bindings

marian-nmt · Aug 21, 2023 · 2c5863c · 2c5863c
2 parents 4661fa9 + 961a728
commit 2c5863c
Show file tree

Hide file tree

Showing 122 changed files with 7,637 additions and 769 deletions.
diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml
@@ -30,7 +30,7 @@ jobs:
           -DCOMPILE_CPU=on \
           -DCOMPILE_CUDA=off \
           -DCOMPILE_EXAMPLES=on \
-          -DCOMPILE_SERVER=on \
+          -DCOMPILE_SERVER=off \
           -DCOMPILE_TESTS=on \
           -DUSE_FBGEMM=on \
           -DUSE_SENTENCEPIECE=on

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -5,15 +5,44 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
 and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
 
-
 ## [Unreleased]
 
+### Added
+- Added `--no-spm-encode` option, allowing the model to use vocabulary IDs directly to train/decode.
+- Added --custom-fallbacks option that allows to specify a list of option sets that get traversed for subsequent fallbacks upon divergence
+- Added --overwrite-checkpoint option that (when set to false) can be used to dump checkpoints with iteration numbers.   
+- Implementations of COMET-20 (reference-based) and BLEURT-20 for inference with conversion scripts.
+- `./marian evaluate` sub command for evaluation with COMET-QE-20, COMET-20 and BLEURT-20
+- A bunch of scripts for metrics use and early MBR experiments
+- LSH vocab filtering for GPU. Speed is not competitive with non-LSH. Checking in for completeness and possible future use of LSH on GPU for non-filtering stuff
+- Added --throw-on-divergence and --fp16-fallback-to-fp32 options to detect (fp16 and fp32) and recover (only fp16) 
+  diverged runs. If not recoverable, exception gets rethrown and goes unhandled to force fatal error and shutdown.
+- Re-implementation of COMET-QE for inference and training; conversion scripts from Unbabel-Comet to Marian.
+- Validator that generates embeddings and can be used during COMET training with an external script.
+- New experimental layer framework for Transformer-like models.
+
+### Fixed
+- Fixed wrong paramter name for norm in new layer framework
+- Fixed unit test for LayerNorm
+- Only collect batch statistics during mini-batch-fit up to actual max-length.
+- Implemented fully correct version of GELU instead of using bad approximatin via Swish.
+- Handle copying from fp32 or fp16 embeddings in embedder mode correctly.
+- Correct defaults for factored embeddings such that shared library use works (move out of config.h/cpp).
+
+### Changed
+- Removed --num-devices N option that wasn't really used by anyone (I assume).
+
+
+## [1.12.0] - 2023-02-20
+
 ### Added
 - Fused inplace-dropout in FFN layer in Transformer
 - `--force-decode` option for marian-decoder
 - `--output-sampling` now works with ensembles (requires proper normalization via e.g `--weights 0.5 0.5`)
+- `--valid-reset-all` option
 
 ### Fixed
+- Make concat factors not break old vector implementation
 - Use allocator in hashing
 - Read/restore checkpoints from main process only when training with MPI
 - Multi-loss casts type to first loss-type before accumulation (aborted before due to missing cast)

diff --git a/VERSION b/VERSION
@@ -1 +1 @@
-v1.11.13
+v1.12.12
diff --git a/azure-pipelines.yml b/azure-pipelines.yml
@@ -13,16 +13,36 @@ parameters:
   type: boolean
   default: true
 
-# The pipeline CI trigger is set on the branch master only and PR trigger on a
-# (non-draft) pull request to any branch
+# Warning: the current branch policies disable the automatic triggering to
+# minimize VM usage!
+# The configuration below specifies that the pipeline CI trigger is set on the
+# branch master only and a PR trigger is on a (non-draft) pull request to any
+# branch.
 trigger:
-- master
+  # This minimizes the number of parallel pipeline runs. When a pipeline is
+  # running, the CI waits until it is completed before starting another one.
+  batch: true
+  branches:
+    include:
+    - master
+  paths:
+    exclude:
+    - azure-regression-tests.yml
+    - contrib
+    - doc
+    - examples
+    - regression-tests
+    - scripts
+    - VERSION
+    - vs
+    - '**/*.md'
+    - '**/*.txt'
 
 pool:
   name: Azure Pipelines
 
 variables:
-  - group: marian-prod-tests
+  - group: marian-regression-tests
   - name: BOOST_ROOT_WINDOWS
     value: "C:/hostedtoolcache/windows/Boost/1.72.0/x86_64"
   - name: BOOST_URL
@@ -32,7 +52,7 @@ variables:
   - name: MKL_DIR
     value: "$(Build.SourcesDirectory)/mkl"
   - name: MKL_URL
-    value: "https://romang.blob.core.windows.net/mariandev/ci/mkl-2020.1-windows-static.zip"
+    value: "https://data.statmt.org/romang/marian-regression-tests/ci/mkl-2020.1-windows-static.zip"
   - name: VCPKG_COMMIT
     value: 2022.03.10
   - name: VCPKG_DIR
@@ -52,6 +72,7 @@ stages:
 
   ######################################################################
   - job: BuildWindows
+    cancelTimeoutInMinutes: 1
     condition: eq(${{ parameters.runBuilds }}, true)
     displayName: Windows
 
@@ -188,6 +209,7 @@ stages:
 
   ######################################################################
   - job: BuildUbuntu
+    cancelTimeoutInMinutes: 1
     condition: eq(${{ parameters.runBuilds }}, true)
     displayName: Ubuntu
     timeoutInMinutes: 120
@@ -324,6 +346,7 @@ stages:
 
   ######################################################################
   - job: BuildMacOS
+    cancelTimeoutInMinutes: 1
     condition: eq(${{ parameters.runBuilds }}, true)
     displayName: macOS CPU clang
 
@@ -348,7 +371,7 @@ stages:
           -DCOMPILE_CPU=on \
           -DCOMPILE_CUDA=off \
           -DCOMPILE_EXAMPLES=on \
-          -DCOMPILE_SERVER=on \
+          -DCOMPILE_SERVER=off \
           -DCOMPILE_TESTS=on \
           -DUSE_FBGEMM=on \
           -DUSE_SENTENCEPIECE=on \
@@ -373,6 +396,7 @@ stages:
 
   ######################################################################
   - job: BuildInstall
+    cancelTimeoutInMinutes: 1
     condition: eq(${{ parameters.runBuilds }}, true)
     displayName: Linux CPU library install
 
@@ -435,6 +459,7 @@ stages:
 
   ######################################################################
   - job: TestWindows
+    cancelTimeoutInMinutes: 1
     displayName: Windows CPU+FBGEMM
 
     pool:
@@ -528,14 +553,14 @@ stages:
       displayName: Machine statistics
       workingDirectory: marian-prod-tests
 
-    # The current SAS token will expire on 8/30/2023 and a new one will need to be set in Marian > Pipelines > Library
+    # The current SAS token will expire on 12/31/2023 and a new one will need to be set in Marian > Pipelines > Library
     - bash: |
         cd models
         bash download-models.sh
         ls
       displayName: Prepare tests
       env:
-          AWS_SECRET_SAS_TOKEN: $(blob-sas-token)
+        AZURE_STORAGE_SAS_TOKEN: $(marian-prod-tests-blob-sas-token)
       workingDirectory: marian-prod-tests
 
     # Avoid using $(Build.SourcesDirectory) in bash tasks because on Windows pools it uses '\'
@@ -560,6 +585,7 @@ stages:
 
   ######################################################################
   - job: TestLinux
+    cancelTimeoutInMinutes: 1
     displayName: Linux CPU+FBGEMM
 
     pool:
@@ -572,7 +598,10 @@ stages:
 
     # The following packages are already installed on Azure-hosted runners: build-essential openssl libssl-dev
     # No need to install libprotobuf{17,10,9v5} on Ubuntu {20,18,16}.04 because it is installed together with libprotobuf-dev
-    - bash: sudo apt-get install -y libgoogle-perftools-dev libprotobuf-dev protobuf-compiler gcc-9 g++-9
+    # Installing libunwind-dev fixes a bug in 2204 (the libunwind-14 and libunwind-dev conflict)
+    - bash: |
+        sudo apt-get install -y libunwind-dev
+        sudo apt-get install -y libgoogle-perftools-dev libprotobuf-dev protobuf-compiler gcc-9 g++-9
       displayName: Install packages
 
     # https://software.intel.com/content/www/us/en/develop/articles/installing-intel-free-libs-and-python-apt-repo.html
@@ -629,14 +658,14 @@ stages:
       displayName: Machine statistics
       workingDirectory: marian-prod-tests
 
-    # The current SAS token will expire on 8/30/2023 and a new one will need to be set in Marian > Pipelines > Library
+    # The current SAS token will expire on 12/31/2023 and a new one will need to be set in Marian > Pipelines > Library
     - bash: |
         cd models
         bash download-models.sh
         ls
       displayName: Prepare tests
       env:
-          AWS_SECRET_SAS_TOKEN: $(blob-sas-token)
+        AZURE_STORAGE_SAS_TOKEN: $(marian-prod-tests-blob-sas-token)
       workingDirectory: marian-prod-tests
 
     - bash: MARIAN=../marian-dev/build bash ./run_mrt.sh '#cpu' '#basics' '#devops'

diff --git a/azure-regression-tests.yml b/azure-regression-tests.yml
@@ -14,12 +14,16 @@ trigger: none
 # Hosted Azure DevOps Pool determining OS, CUDA version and available GPUs
 pool: mariandevops-pool-m60-eus
 
+variables:
+  - group: marian-regression-tests
+
 stages:
 - stage: TestsGPU
   jobs:
 
   ######################################################################
   - job: TestsGPULinux
+    cancelTimeoutInMinutes: 1
     displayName: Linux GPU tests
     timeoutInMinutes: 120
 
@@ -60,6 +64,24 @@ stages:
       displayName: Collect system info
       workingDirectory: regression-tests
 
+    # Always run regression tests from the master branch
+    # The current SAS token will expire on 12/31/2023 and a new one will need to be set in Marian > Pipelines > Library
+    # This is run at the beginning for easier debugging of the Python environment
+    - bash: |
+        set -x
+        git checkout master
+        git pull origin master
+        # Uninstall Cython because the newest 3.0.0 is incompatible with newest available versions of pyyaml and numpy as of July 2023
+        python3 -m pip uninstall -y cython
+        python3 -m pip install 'cython<3'
+        # These modules will be installed via `make install` below, but Cython needs to be installed before
+        python3 -m pip install 'pyyaml<6.0.1' 'numpy>=1.22,<2' websocket-client
+        make install
+      displayName: Prepare regression tests
+      env:
+        AZURE_STORAGE_SAS_TOKEN: $(marian-pub-tests-blob-sas-token)
+      workingDirectory: regression-tests
+
     # https://software.intel.com/content/www/us/en/develop/articles/installing-intel-free-libs-and-python-apt-repo.html
     - bash: |
         wget -qO- "https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS-2019.PUB" | sudo apt-key add -
@@ -102,14 +124,6 @@ stages:
       displayName: Run unit tests
       workingDirectory: build
 
-    # Always run regression tests from the master branch
-    - bash: |
-        git checkout master
-        git pull origin master
-        make install
-      displayName: Prepare regression tests
-      workingDirectory: regression-tests
-
     # Continue on error to be able to collect outputs and publish them as an artifact
     - bash: MARIAN=../build ./run_mrt.sh
       continueOnError: true

diff --git a/cmake/FindSSE.cmake b/cmake/FindSSE.cmake
@@ -4,7 +4,7 @@
 IF(CMAKE_SYSTEM_NAME MATCHES "Linux")
    EXEC_PROGRAM(cat ARGS "/proc/cpuinfo" OUTPUT_VARIABLE CPUINFO)
 
-   STRING(REGEX REPLACE "^.*(sse2).*$" "\\1" SSE_THERE ${CPUINFO})
+   STRING(REGEX REPLACE "^.*(sse2).*$" "\\1" SSE_THERE "${CPUINFO}")
    STRING(COMPARE EQUAL "sse2" "${SSE_THERE}" SSE2_TRUE)
    IF (SSE2_TRUE)
       set(SSE2_FOUND true CACHE BOOL "SSE2 available on host")
@@ -13,14 +13,14 @@ IF(CMAKE_SYSTEM_NAME MATCHES "Linux")
    ENDIF (SSE2_TRUE)
 
    # /proc/cpuinfo apparently omits sse3 :(
-   STRING(REGEX REPLACE "^.*[^s](sse3).*$" "\\1" SSE_THERE ${CPUINFO})
+   STRING(REGEX REPLACE "^.*[^s](sse3).*$" "\\1" SSE_THERE "${CPUINFO}")
    STRING(COMPARE EQUAL "sse3" "${SSE_THERE}" SSE3_TRUE)
    IF (NOT SSE3_TRUE)
-      STRING(REGEX REPLACE "^.*(T2300).*$" "\\1" SSE_THERE ${CPUINFO})
+      STRING(REGEX REPLACE "^.*(T2300).*$" "\\1" SSE_THERE "${CPUINFO}")
       STRING(COMPARE EQUAL "T2300" "${SSE_THERE}" SSE3_TRUE)
    ENDIF (NOT SSE3_TRUE)
 
-   STRING(REGEX REPLACE "^.*(ssse3).*$" "\\1" SSE_THERE ${CPUINFO})
+   STRING(REGEX REPLACE "^.*(ssse3).*$" "\\1" SSE_THERE "${CPUINFO}")
    STRING(COMPARE EQUAL "ssse3" "${SSE_THERE}" SSSE3_TRUE)
    IF (SSE3_TRUE OR SSSE3_TRUE)
       set(SSE3_FOUND true CACHE BOOL "SSE3 available on host")
@@ -33,39 +33,39 @@ IF(CMAKE_SYSTEM_NAME MATCHES "Linux")
       set(SSSE3_FOUND false CACHE BOOL "SSSE3 available on host")
    ENDIF (SSSE3_TRUE)
 
-   STRING(REGEX REPLACE "^.*(sse4_1).*$" "\\1" SSE_THERE ${CPUINFO})
+   STRING(REGEX REPLACE "^.*(sse4_1).*$" "\\1" SSE_THERE "${CPUINFO}")
    STRING(COMPARE EQUAL "sse4_1" "${SSE_THERE}" SSE41_TRUE)
    IF (SSE41_TRUE)
       set(SSE4_1_FOUND true CACHE BOOL "SSE4.1 available on host")
    ELSE (SSE41_TRUE)
       set(SSE4_1_FOUND false CACHE BOOL "SSE4.1 available on host")
    ENDIF (SSE41_TRUE)
 
-   STRING(REGEX REPLACE "^.*(sse4_2).*$" "\\1" SSE_THERE ${CPUINFO})
+   STRING(REGEX REPLACE "^.*(sse4_2).*$" "\\1" SSE_THERE "${CPUINFO}")
    STRING(COMPARE EQUAL "sse4_2" "${SSE_THERE}" SSE42_TRUE)
    IF (SSE42_TRUE)
       set(SSE4_2_FOUND true CACHE BOOL "SSE4.2 available on host")
    ELSE (SSE42_TRUE)
       set(SSE4_2_FOUND false CACHE BOOL "SSE4.2 available on host")
    ENDIF (SSE42_TRUE)
 
-   STRING(REGEX REPLACE "^.*(avx).*$" "\\1" SSE_THERE ${CPUINFO})
+   STRING(REGEX REPLACE "^.*(avx).*$" "\\1" SSE_THERE "${CPUINFO}")
    STRING(COMPARE EQUAL "avx" "${SSE_THERE}" AVX_TRUE)
    IF (AVX_TRUE)
       set(AVX_FOUND true CACHE BOOL "AVX available on host")
    ELSE (AVX_TRUE)
       set(AVX_FOUND false CACHE BOOL "AVX available on host")
    ENDIF (AVX_TRUE)
 
-   STRING(REGEX REPLACE "^.*(avx2).*$" "\\1" SSE_THERE ${CPUINFO})
+   STRING(REGEX REPLACE "^.*(avx2).*$" "\\1" SSE_THERE "${CPUINFO}")
    STRING(COMPARE EQUAL "avx2" "${SSE_THERE}" AVX2_TRUE)
    IF (AVX2_TRUE)
       set(AVX2_FOUND true CACHE BOOL "AVX2 available on host")
    ELSE (AVX2_TRUE)
       set(AVX2_FOUND false CACHE BOOL "AVX2 available on host")
    ENDIF (AVX2_TRUE)
 
-   STRING(REGEX REPLACE "^.*(avx512).*$" "\\1" SSE_THERE ${CPUINFO})
+   STRING(REGEX REPLACE "^.*(avx512).*$" "\\1" SSE_THERE "${CPUINFO}")
    STRING(COMPARE EQUAL "avx512" "${SSE_THERE}" AVX512_TRUE)
    IF (AVX512_TRUE)
       set(AVX512_FOUND true CACHE BOOL "AVX512 available on host")
@@ -76,15 +76,15 @@ IF(CMAKE_SYSTEM_NAME MATCHES "Linux")
 ELSEIF(CMAKE_SYSTEM_NAME MATCHES "Darwin")
    EXEC_PROGRAM("/usr/sbin/sysctl -n machdep.cpu.features machdep.cpu.leaf7_features" OUTPUT_VARIABLE CPUINFO)
 
-   STRING(REGEX REPLACE "^.*[^S](SSE2).*$" "\\1" SSE_THERE ${CPUINFO})
+   STRING(REGEX REPLACE "^.*[^S](SSE2).*$" "\\1" SSE_THERE "${CPUINFO}")
    STRING(COMPARE EQUAL "SSE2" "${SSE_THERE}" SSE2_TRUE)
    IF (SSE2_TRUE)
       set(SSE2_FOUND true CACHE BOOL "SSE2 available on host")
    ELSE (SSE2_TRUE)
       set(SSE2_FOUND false CACHE BOOL "SSE2 available on host")
    ENDIF (SSE2_TRUE)
 
-   STRING(REGEX REPLACE "^.*[^S](SSE3).*$" "\\1" SSE_THERE ${CPUINFO})
+   STRING(REGEX REPLACE "^.*[^S](SSE3).*$" "\\1" SSE_THERE "${CPUINFO}")
    STRING(COMPARE EQUAL "SSE3" "${SSE_THERE}" SSE3_TRUE)
    IF (SSE3_TRUE)
       set(SSE3_FOUND true CACHE BOOL "SSE3 available on host")
@@ -100,31 +100,31 @@ ELSEIF(CMAKE_SYSTEM_NAME MATCHES "Darwin")
       set(SSSE3_FOUND false CACHE BOOL "SSSE3 available on host")
    ENDIF (SSSE3_TRUE)
 
-   STRING(REGEX REPLACE "^.*(SSE4.1).*$" "\\1" SSE_THERE ${CPUINFO})
+   STRING(REGEX REPLACE "^.*(SSE4.1).*$" "\\1" SSE_THERE "${CPUINFO}")
    STRING(COMPARE EQUAL "SSE4.1" "${SSE_THERE}" SSE41_TRUE)
    IF (SSE41_TRUE)
       set(SSE4_1_FOUND true CACHE BOOL "SSE4.1 available on host")
    ELSE (SSE41_TRUE)
       set(SSE4_1_FOUND false CACHE BOOL "SSE4.1 available on host")
    ENDIF (SSE41_TRUE)
 
-   STRING(REGEX REPLACE "^.*(AVX).*$" "\\1" SSE_THERE ${CPUINFO})
+   STRING(REGEX REPLACE "^.*(AVX).*$" "\\1" SSE_THERE "${CPUINFO}")
    STRING(COMPARE EQUAL "AVX" "${SSE_THERE}" AVX_TRUE)
    IF (AVX_TRUE)
       set(AVX_FOUND true CACHE BOOL "AVX available on host")
    ELSE (AVX_TRUE)
       set(AVX_FOUND false CACHE BOOL "AVX available on host")
    ENDIF (AVX_TRUE)
 
-   STRING(REGEX REPLACE "^.*(AVX2).*$" "\\1" SSE_THERE ${CPUINFO})
+   STRING(REGEX REPLACE "^.*(AVX2).*$" "\\1" SSE_THERE "${CPUINFO}")
    STRING(COMPARE EQUAL "AVX2" "${SSE_THERE}" AVX2_TRUE)
    IF (AVX2_TRUE)
       set(AVX2_FOUND true CACHE BOOL "AVX2 available on host")
    ELSE (AVX2_TRUE)
       set(AVX2_FOUND false CACHE BOOL "AVX2 available on host")
    ENDIF (AVX2_TRUE)
 
-   STRING(REGEX REPLACE "^.*(avx512).*$" "\\1" SSE_THERE ${CPUINFO})
+   STRING(REGEX REPLACE "^.*(avx512).*$" "\\1" SSE_THERE "${CPUINFO}")
    STRING(COMPARE EQUAL "avx512" "${SSE_THERE}" AVX512_TRUE)
    IF (AVX512_TRUE)
       set(AVX512_FOUND true CACHE BOOL "AVX512 available on host")