.gitlab-ci.yml

stages:
  - setup
  - builds
  - run
  - tests
  - cleanup

# JOB_DIR points to a persistent working space used for most stages in this pipeline but
# that is unique to this pipeline.
# We use the "fetch" strategy to speed up the startup of stages
variables:
  JOB_DIR: "/lustre/f2/scratch/oar.gfdl.ogrp-account/runner/builds/$CI_PIPELINE_ID"
  WORKSPACE: "/lustre/f2/scratch/oar.gfdl.ogrp-account/runner/$CI_RUNNER_ID"
  GIT_STRATEGY: fetch

# Always eport value of $JOB_DIR
before_script:
  - echo Job directory set to $JOB_DIR

# Test that merge with dev/gfdl works.
p:merge:
  stage: setup
  tags:
    - ncrc4
  script:
    - git pull --no-edit https://github.com/NOAA-GFDL/MOM6.git dev/gfdl

# Setup the persistent JOB_DIR for all subsequent stages
#
# This basically setups up a complete tree much as a user would in their workflow
p:clone:
  stage: setup
  tags:
    - ncrc4
  script:
    - .gitlab/pipeline-ci-tool.sh create-job-dir
#.gitlab/pipeline-ci-tool.sh clean-job-dir

# Make work spaces for running simultaneously in parallel jobs
#
# Each work space is a clone of MOM6-examples with symbolic links for the build and data directories
# so they can share executables which can run simultaneously without interfering with each other

s:work-space:pgi:
  stage: setup
  tags:
    - ncrc4
  needs: ["p:clone"]
  script:
    - .gitlab/pipeline-ci-tool.sh copy-test-space pgi

s:work-space:intel:
  stage: setup
  tags:
    - ncrc4
  needs: ["p:clone"]
  script:
    - .gitlab/pipeline-ci-tool.sh copy-test-space intel

s:work-space:gnu:
  stage: setup
  tags:
    - ncrc4
  needs: ["p:clone"]
  script:
    - .gitlab/pipeline-ci-tool.sh copy-test-space gnu

s:work-space:gnu-restarts:
  stage: setup
  tags:
    - ncrc4
  needs: ["p:clone"]
  script:
    - .gitlab/pipeline-ci-tool.sh copy-test-space gnu-rst

# Compile executables
#
# gnu:repro, gnu:debug, intel:repro and pgi:repro are used by their respective run:* jobs
# gnu:ice-only-nolib and gnu:ocean-only-nolibs are not used but simply test that the model compiles without libraries

compile:pgi:repro:
  stage: builds
  needs: ["p:clone"]
  tags:
    - ncrc4
  script:
    - .gitlab/pipeline-ci-tool.sh mrs-compile repro_pgi

compile:intel:repro:
  stage: builds
  needs: ["p:clone"]
  tags:
    - ncrc4
  script:
    - .gitlab/pipeline-ci-tool.sh mrs-compile repro_intel

compile:gnu:repro:
  stage: builds
  needs: ["p:clone"]
  tags:
    - ncrc4
  script:
    - .gitlab/pipeline-ci-tool.sh mrs-compile repro_gnu mrs-compile static_gnu

compile:gnu:debug:
  stage: builds
  needs: ["p:clone"]
  tags:
    - ncrc4
  script:
    - .gitlab/pipeline-ci-tool.sh mrs-compile debug_gnu

compile:gnu:ocean-only-nolibs:
  stage: builds
  needs: ["p:clone"]
  tags:
    - ncrc4
  script:
    - .gitlab/pipeline-ci-tool.sh nolibs-ocean-only-compile gnu

compile:gnu:ice-ocean-nolibs:
  stage: builds
  needs: ["p:clone"]
  tags:
    - ncrc4
  script:
    - .gitlab/pipeline-ci-tool.sh nolibs-ocean-ice-compile gnu

# Runs

run:pgi:
  stage: run
  needs: ["s:work-space:pgi","compile:pgi:repro"]
  tags:
    - ncrc4
  script:
    - sbatch --clusters=c3,c4 --nodes=30 --time=0:20:00 --account=gfdl_o --qos=debug --job-name=mom6_pgi_tests --output=log.$CI_JOB_ID --wait .gitlab/pipeline-ci-tool.sh run-suite pgi SNL && ( egrep -v 'pagefaults|HiWaterMark=' log.$CI_JOB_ID ; echo Job returned normally ) || ( cat log.$CI_JOB_ID ; echo Job failed ; exit 911 )
    - test -f $JOB_DIR/CI-BATCH-SUCCESS-pgi-SNL || ( echo Batch job did not complete ; exit 911 )

run:intel:
  stage: run
  needs: ["s:work-space:intel","compile:intel:repro"]
  tags:
    - ncrc4
  script:
    - sbatch --clusters=c3,c4 --nodes=30 --time=0:20:00 --account=gfdl_o --qos=debug --job-name=mom6_intel_tests --output=log.$CI_JOB_ID --wait .gitlab/pipeline-ci-tool.sh run-suite intel SNL && ( egrep -v 'pagefaults|HiWaterMark=' log.$CI_JOB_ID ; echo Job returned normally ) || ( cat log.$CI_JOB_ID ; echo Job failed ; exit 911 )
    - test -f $JOB_DIR/CI-BATCH-SUCCESS-intel-SNL || ( echo Batch job did not complete ; exit 911 )

run:gnu:
  stage: run
  needs: ["s:work-space:gnu","compile:gnu:repro","compile:gnu:debug"]
  tags:
    - ncrc4
  script:
    - sbatch --clusters=c3,c4 --nodes=30 --time=0:20:00 --account=gfdl_o --qos=debug --job-name=mom6_gnu_tests --output=log.$CI_JOB_ID --wait .gitlab/pipeline-ci-tool.sh run-suite gnu SNLDT && ( egrep -v 'pagefaults|HiWaterMark=' log.$CI_JOB_ID ; echo Job returned normally ) || ( cat log.$CI_JOB_ID ; echo Job failed ; exit 911 )
    - test -f $JOB_DIR/CI-BATCH-SUCCESS-gnu-SNLDT || ( echo Batch job did not complete ; exit 911 )

run:gnu-restarts:
  stage: run
  needs: ["s:work-space:gnu-restarts","compile:gnu:repro"]
  tags:
    - ncrc4
  script:
    - sbatch --clusters=c3,c4 --nodes=30 --time=0:20:00 --account=gfdl_o --qos=debug --job-name=mom6_gnu_restarts --output=log.$CI_JOB_ID --wait .gitlab/pipeline-ci-tool.sh run-suite gnu R && ( egrep -v 'pagefaults|HiWaterMark=' log.$CI_JOB_ID ; echo Job returned normally ) || ( cat log.$CI_JOB_ID ; echo Job failed ; exit 911 )
    - test -f $JOB_DIR/CI-BATCH-SUCCESS-gnu-R || ( echo Batch job did not complete ; exit 911 )

# GH/autoconf tests (duplicates the GH actions tests)
#
# These stages replace the "before_script" and so start in the transient work-space provided by gitlab.
# We work here to avoid collisions with parallel jobs

actions:gnu:
  stage: tests
  needs: []
  tags:
    - ncrc4
  before_script:
    - echo -e "\e[0Ksection_start:`date +%s`:submodules[collapsed=true]\r\e[0KCloning submodules"
    - git submodule init ; git submodule update
    - echo -e "\e[0Ksection_end:`date +%s`:submodules\r\e[0K"
  script:
    - echo -e "\e[0Ksection_start:`date +%s`:compile[collapsed=true]\r\e[0KCompiling executables"
    - cd .testing
    - module unload PrgEnv-pgi PrgEnv-intel PrgEnv-gnu darshan ; module load PrgEnv-gnu ; module unload netcdf gcc ; module load gcc/7.3.0 cray-hdf5 cray-netcdf
    - make -s -j
    - MPIRUN= make preproc -s -j
    - echo -e "\e[0Ksection_end:`date +%s`:compile\r\e[0K"
    - (echo '#!/bin/bash';echo 'make MPIRUN="srun -mblock --exclusive" WORKSPACE=$WORKSPACE test -s -j') > job.sh
    - sbatch --clusters=c3,c4 --nodes=5 --time=0:05:00 --account=gfdl_o --qos=debug --job-name=MOM6.gnu.testing --output=log.$CI_JOB_ID --wait job.sh || ( cat log.$CI_JOB_ID ; exit 911 ) && make WORKSPACE=$WORKSPACE test -s
    - make WORKSPACE=$WORKSPACE test.summary

actions:intel:
  stage: tests
  needs: []
  tags:
    - ncrc4
  before_script:
    - echo -e "\e[0Ksection_start:`date +%s`:submodules[collapsed=true]\r\e[0KCloning submodules"
    - git submodule init ; git submodule update
    - echo -e "\e[0Ksection_end:`date +%s`:submodules\r\e[0K"
  script:
    - echo -e "\e[0Ksection_start:`date +%s`:compile[collapsed=true]\r\e[0KCompiling executables"
    - cd .testing
    - module unload PrgEnv-pgi PrgEnv-intel PrgEnv-gnu darshan; module load PrgEnv-intel; module unload netcdf intel; module load intel/18.0.6.288 cray-hdf5 cray-netcdf
    - make -s -j
    - MPIRUN= make preproc -s -j
    - echo -e "\e[0Ksection_end:`date +%s`:compile\r\e[0K"
    - (echo '#!/bin/bash';echo 'make MPIRUN="srun -mblock --exclusive" WORKSPACE=$WORKSPACE test -s -j') > job.sh
    - sbatch --clusters=c3,c4 --nodes=5 --time=0:05:00 --account=gfdl_o --qos=debug --job-name=MOM6.intel.testing --output=log.$CI_JOB_ID --wait job.sh || ( cat log.$CI_JOB_ID ; exit 911 ) && make WORKSPACE=$WORKSPACE test -s
    - make WORKSPACE=$WORKSPACE test.summary

# Tests
#
# stats file tests involve comparing the check sums of the generated files against the check sums in the stats-repo
# log file tests involve comparing the check sums of the generated files against the check sums in MOM6-examples

t:pgi:symmetric:
  stage: tests
  needs: ["run:pgi"]
  tags:
    - ncrc4
  script:
    - .gitlab/pipeline-ci-tool.sh check-stats pgi S

t:pgi:non-symmetric:
  stage: tests
  needs: ["run:pgi"]
  tags:
    - ncrc4
  script:
    - .gitlab/pipeline-ci-tool.sh check-stats pgi N

t:pgi:layout:
  stage: tests
  needs: ["run:pgi"]
  tags:
    - ncrc4
  script:
    - .gitlab/pipeline-ci-tool.sh check-stats pgi L

t:pgi:params:
  stage: tests
  needs: ["run:pgi"]
  tags:
    - ncrc4
  script:
    - .gitlab/pipeline-ci-tool.sh check-params pgi
  allow_failure: true

t:intel:symmetric:
  stage: tests
  needs: ["run:intel"]
  tags:
    - ncrc4
  script:
    - .gitlab/pipeline-ci-tool.sh check-stats intel S

t:intel:non-symmetric:
  stage: tests
  needs: ["run:intel"]
  tags:
    - ncrc4
  script:
    - .gitlab/pipeline-ci-tool.sh check-stats intel N

t:intel:layout:
  stage: tests
  needs: ["run:intel"]
  tags:
    - ncrc4
  script:
    - .gitlab/pipeline-ci-tool.sh check-stats intel L

t:intel:params:
  stage: tests
  needs: ["run:intel"]
  tags:
    - ncrc4
  script:
    - .gitlab/pipeline-ci-tool.sh check-params intel
  allow_failure: true

t:gnu:symmetric:
  stage: tests
  needs: ["run:gnu"]
  tags:
    - ncrc4
  script:
    - .gitlab/pipeline-ci-tool.sh check-stats gnu S

t:gnu:non-symmetric:
  stage: tests
  needs: ["run:gnu"]
  tags:
    - ncrc4
  script:
    - .gitlab/pipeline-ci-tool.sh check-stats gnu N

t:gnu:layout:
  stage: tests
  needs: ["run:gnu"]
  tags:
    - ncrc4
  script:
    - .gitlab/pipeline-ci-tool.sh check-stats gnu L

t:gnu:static:
  stage: tests
  needs: ["run:gnu"]
  tags:
    - ncrc4
  script:
    - .gitlab/pipeline-ci-tool.sh check-stats gnu T

t:gnu:symmetric-debug:
  stage: tests
  needs: ["run:gnu"]
  tags:
    - ncrc4
  script:
    - .gitlab/pipeline-ci-tool.sh check-stats gnu D

t:gnu:restart:
  stage: tests
  needs: ["run:gnu-restarts"]
  tags:
    - ncrc4
  script:
    - .gitlab/pipeline-ci-tool.sh check-stats gnu R

t:gnu:params:
  stage: tests
  needs: ["run:gnu"]
  tags:
    - ncrc4
  script:
    - .gitlab/pipeline-ci-tool.sh check-params gnu
  allow_failure: true

t:gnu:diags:
  stage: tests
  needs: ["run:gnu"]
  tags:
    - ncrc4
  script:
    - .gitlab/pipeline-ci-tool.sh check-diags gnu
  allow_failure: true

# We cleanup ONLY if the preceding stages were completed successfully
cleanup:
  stage: cleanup
  tags:
    - ncrc4
  before_script:
    - echo Skipping usual preamble
  script:
    - rm -rf $JOB_DIR