.gitlab-ci.yml

variables:
  CRAN_MIRROR: "https://cloud.r-project.org"
  _R_CHECK_FORCE_SUGGESTS_: "false"
  _R_CHECK_NO_STOP_ON_TEST_ERROR_: "true"

stages:
  - dependencies
  - build
  - test
  - integration
  - deploy

.artifacts-template: &artifacts
  artifacts:
    expire_in: 2 weeks
    when: always
    paths:
      - bus

mirror-packages: # download all recursive dependencies of data.table suggests and integration suggests from inst/tests/tests-DESCRIPTION
  stage: dependencies
  tags:
    - linux
  image: registry.gitlab.com/jangorecki/dockerfiles/r-base-dev
  cache:
    paths:
    - bus/$CI_BUILD_NAME/cran
  variables:
    R_BIN_VERSION: "3.6"
    R_DEVEL_BIN_VERSION: "3.7"
  script:
    - echo 'source(".ci/ci.R")' >> .Rprofile
    - mkdir -p bus/$CI_BUILD_NAME/cran/src/contrib
    # mirror R dependencies: source, win.binary
    - Rscript -e 'mirror.packages(dcf.dependencies(c("DESCRIPTION","inst/tests/tests-DESCRIPTION"), "all"), repos=c(Sys.getenv("CRAN_MIRROR"), dcf.repos("inst/tests/tests-DESCRIPTION")), repodir="bus/mirror-packages/cran")'
    - Rscript -e 'sapply(simplify=FALSE, setNames(nm=Sys.getenv(c("R_BIN_VERSION","R_DEVEL_BIN_VERSION"))), function(binary.ver) mirror.packages(type="win.binary", dcf.dependencies("DESCRIPTION", "all"), repos=Sys.getenv("CRAN_MIRROR"), repodir="bus/mirror-packages/cran", binary.ver=binary.ver))'
  <<: *artifacts

build: # build data.table sources as tar.gz archive
  stage: build
  tags:
    - linux
  image: registry.gitlab.com/jangorecki/dockerfiles/r-builder
  dependencies:
  - mirror-packages
  script:
    - Rscript -e 'install.packages("knitr", repos=file.path("file:",normalizePath("bus/mirror-packages/cran")), quiet=TRUE)'
    - rm -r bus
    - echo "Revision:" $CI_BUILD_REF >> ./DESCRIPTION
    - R CMD build .
    - mkdir -p bus/$CI_BUILD_NAME/cran/src/contrib
    - mv $(ls -1t data.table_*.tar.gz | head -n 1) bus/$CI_BUILD_NAME/cran/src/contrib/.
    - Rscript -e 'tools::write_PACKAGES(contrib.url("bus/build/cran"), fields="Revision", addFiles=TRUE)'
  <<: *artifacts

.test-copy-src: &copy-src
  - &copy-src
    cp $(ls -1t bus/build/cran/src/contrib/data.table_*.tar.gz | head -n 1) .

.test-move-src: &move-src
  - &move-src
    mkdir -p bus/$CI_BUILD_NAME && mv $(ls -1t data.table_*.tar.gz | head -n 1) bus/$CI_BUILD_NAME

.test-cleanup-src: &cleanup-src
  - &cleanup-src
    rm $(ls -1t data.table_*.tar.gz | head -n 1)

.test-template: &test
  stage: test
  dependencies:
  - mirror-packages
  - build
  <<: *artifacts

.test-lin-template: &test-lin
  <<: *test
  tags:
    - linux

.test-win-template: &test-win
  <<: *test
  tags:
    - windows
  before_script:
    - export PATH="/c/$R_DIR/bin:/c/Rtools/bin:$PATH"
    - rm -rf /tmp/$R_DIR/library && mkdir -p /tmp/$R_DIR/library
    - export R_LIBS_USER="/tmp/$R_DIR/library"

.test-osx-template: &test-osx
  <<: *test
  tags:
    - macosx

test-rel-lin: #  most comprehensive tests, force all suggests, also integration tests, using gcc -O3 -flto
  <<: *test-lin
  image: registry.gitlab.com/jangorecki/dockerfiles/r-builder
  variables: # unlike CRAN
    _R_CHECK_CRAN_INCOMING_: "FALSE"
    _R_CHECK_CRAN_INCOMING_REMOTE_: "FALSE"
    _R_CHECK_FORCE_SUGGESTS_: "TRUE"
    OPENBLAS_MAIN_FREE: "1"
    TEST_DATA_TABLE_WITH_OTHER_PACKAGES: "TRUE"
  script:
    - Rscript -e 'source(".ci/ci.R"); install.packages(dcf.dependencies(c("DESCRIPTION","inst/tests/tests-DESCRIPTION"), which="all"), quiet=TRUE)'
    - *copy-src
    - rm -r bus
    - *move-src
    - mkdir -p ~/.R
    - echo 'CFLAGS=-g -O3 -flto -fopenmp -Wall -pedantic -fstack-protector-strong -D_FORTIFY_SOURCE=2' > ~/.R/Makevars
    - echo 'CXXFLAGS=-g -O3 -flto -fopenmp -Wall -pedantic -fstack-protector-strong -D_FORTIFY_SOURCE=2' >> ~/.R/Makevars
    - cd bus/$CI_BUILD_NAME
    - R CMD check $(ls -1t data.table_*.tar.gz | head -n 1)
    - *cleanup-src
    - (! grep "warning:" data.table.Rcheck/00install.out)

test-rel-vanilla-lin: # minimal installation, no suggested deps, no vignettes or manuals, measure memory, using gcc -O0 -fno-openmp
  <<: *test-lin
  image: registry.gitlab.com/jangorecki/dockerfiles/r-base-dev
  variables:
    TEST_DATA_TABLE_MEMTEST: "TRUE"
  before_script:
    - mkdir -p ~/.R
    - echo 'CFLAGS=-g -O0 -fno-openmp -Wall -pedantic -fstack-protector-strong -D_FORTIFY_SOURCE=2' > ~/.R/Makevars
    - echo 'CXXFLAGS=-g -O0 -fno-openmp -Wall -pedantic -fstack-protector-strong -D_FORTIFY_SOURCE=2' >> ~/.R/Makevars
  script:
    - *copy-src
    - rm -r bus
    - *move-src
    - cd bus/$CI_BUILD_NAME
    - R CMD check --no-manual --ignore-vignettes $(ls -1t data.table_*.tar.gz | head -n 1)
    - *cleanup-src

test-rel-cran-lin: # currently released R on Linux
  <<: *test-lin
  image: registry.gitlab.com/jangorecki/dockerfiles/r-builder
  variables:
    _R_CHECK_CRAN_INCOMING_: "TRUE"           # stricter --as-cran checks should run in dev pipelines continuously (not sure what they are though)
    _R_CHECK_CRAN_INCOMING_REMOTE_: "FALSE"   # Other than no URL checking (takes many minutes) or 'Days since last update 0' NOTEs needed, #3284
  before_script:
    - mkdir -p ~/.R
    - echo 'CFLAGS=-g0 -O2 -fopenmp -Wall -pedantic -fstack-protector-strong -D_FORTIFY_SOURCE=2'> ~/.R/Makevars # -g0 because -g increases datatable.so size from 0.5MB to 1.5MB and breaches 'installed package size <= 5MB' note
    - echo 'CXXFLAGS=-g0 -O2 -fopenmp -Wall -pedantic -fstack-protector-strong -D_FORTIFY_SOURCE=2' >> ~/.R/Makevars
  script:
    - Rscript -e 'source(".ci/ci.R"); install.packages(dcf.dependencies("DESCRIPTION", which="most"), quiet=TRUE)'
    - *copy-src
    - rm -r bus
    - *move-src
    - cd bus/$CI_BUILD_NAME
    - R CMD check --as-cran $(ls -1t data.table_*.tar.gz | head -n 1)
    - *cleanup-src
    - >-
        Rscript -e 'l<-readLines("data.table.Rcheck/00check.log"); if (!identical(l[length(l)], "Status: OK")) stop("Last line of ", shQuote("00check.log"), " is not ", shQuote("Status: OK"), " but ", shQuote(toString(l[length(l)]))) else q("no")'

test-dev-cran-lin: # R-devel on Linux, --enable-strict-barrier --disable-long-double
  <<: *test-lin
  image: registry.gitlab.com/jangorecki/dockerfiles/r-devel
  allow_failure: false
  variables:
    _R_CHECK_CRAN_INCOMING_: "TRUE"
    _R_CHECK_CRAN_INCOMING_REMOTE_: "FALSE"
  script:
    - Rscript -e 'source(".ci/ci.R"); install.packages(dcf.dependencies("DESCRIPTION", which="most"), quiet=TRUE)'
    - *copy-src
    - rm -r bus
    - *move-src
    - cd bus/$CI_BUILD_NAME
    - R CMD check --as-cran $(ls -1t data.table_*.tar.gz | head -n 1)
    - *cleanup-src

test-310-cran-lin: # test stated R dependency (3.1.0) using Linux
  <<: *test-lin
  image: registry.gitlab.com/jangorecki/dockerfiles/r-3.1.0
  variables:
    _R_CHECK_CRAN_INCOMING_: "TRUE"
    _R_CHECK_CRAN_INCOMING_REMOTE_: "FALSE"
  script:
    - Rscript -e 'source(".ci/ci.R"); install.packages(dcf.dependencies("DESCRIPTION", which="most"), quiet=TRUE)'
    - *copy-src
    - rm -r bus
    - *move-src
    - cd bus/$CI_BUILD_NAME
    - R CMD check --as-cran $(ls -1t data.table_*.tar.gz | head -n 1)
    - *cleanup-src

test-rel-win: # windows test and build binaries
  <<: *test-win
  variables:
    R_BIN_VERSION: "3.6"
    R_DIR: "R-3.6.0"
  script:
    - Rscript -e "source('.ci/ci.R'); install.packages(dcf.dependencies('DESCRIPTION', which='all'), quiet=TRUE)"
    - *copy-src
    - rm -r bus
    - *move-src
    - cd bus/$CI_BUILD_NAME
    - R CMD check --no-manual $(ls -1t data.table_*.tar.gz | head -n 1)
    - R CMD INSTALL --build $(ls -1t data.table_*.tar.gz | head -n 1)
    - mkdir -p cran/bin/windows/contrib/$R_BIN_VERSION
    - mv $(ls -1t data.table_*.zip | head -n 1) cran/bin/windows/contrib/$R_BIN_VERSION
    - *cleanup-src

test-dev-win: # R-devel on windows
  <<: *test-win
  variables:
    R_BIN_VERSION: "3.7"
    R_DIR: "R-devel"
    TEST_DATA_TABLE_MEMTEST: "FALSE" # disabled as described in #3147
  allow_failure: false
  script:
    - Rscript -e "source('.ci/ci.R'); install.packages(dcf.dependencies('DESCRIPTION', which='all'), quiet=TRUE)"
    - *copy-src
    - rm -r bus
    - *move-src
    - cd bus/$CI_BUILD_NAME
    - R CMD check --no-manual --ignore-vignettes $(ls -1t data.table_*.tar.gz | head -n 1)
    - R CMD INSTALL --build $(ls -1t data.table_*.tar.gz | head -n 1)
    - mkdir -p cran/bin/windows/contrib/$R_BIN_VERSION
    - mv $(ls -1t data.table_*.zip | head -n 1) cran/bin/windows/contrib/$R_BIN_VERSION
    - *cleanup-src

.test-rel-osx: # macosx test and build binaries
  <<: *test-osx
  variables:
    R_BIN_VERSION: "3.6"
  script:
    - Rscript -e 'source(".ci/ci.R"); install.packages(dcf.dependencies("DESCRIPTION", which="all"), quiet=TRUE)'
    - *copy-src
    - rm -r bus
    - *move-src
    - cd bus/$CI_BUILD_NAME
    - R CMD check $(ls -1t data.table_*.tar.gz | head -n 1)
    - R CMD INSTALL --build $(ls -1t data.table_*.tar.gz | head -n 1)
    - mkdir -p cran/bin/macosx/el-capitan/contrib/$R_BIN_VERSION
    - mv $(ls -1t data.table_*.tgz | head -n 1) cran/bin/macosx/el-capitan/contrib/$R_BIN_VERSION
    - *cleanup-src

integration: # merging all artifacts to produce single R repository and summaries
  stage: integration
  image: registry.gitlab.com/jangorecki/dockerfiles/r-builder
  tags:
    - linux
  only:
    - master
  dependencies:
  - mirror-packages
  - build
  - test-rel-lin
  - test-rel-cran-lin
  - test-dev-cran-lin
  - test-rel-vanilla-lin
  - test-310-cran-lin
  - test-rel-win
  - test-dev-win
  #- test-rel-osx
  variables:
    R_BIN_VERSION: "3.6"
    R_DEVEL_BIN_VERSION: "3.7"
  script:
    # pkgdown installs pkgs from "." so run at start to have clean root dir
    - apt-get update -qq && apt-get install -y libxml2-dev
    - mkdir -p /tmp/pkgdown/library
    - R_LIBS_USER=/tmp/pkgdown/library Rscript -e 'install.packages("remotes", repos=Sys.getenv("CRAN_MIRROR"), quiet=TRUE); remotes::install_github("r-lib/pkgdown", repos=Sys.getenv("CRAN_MIRROR"), quiet=TRUE); pkgdown::build_site(override=list(destination="./pkgdown"))'
    # html manual, vignettes, repos, cran_web, cran_checks
    - echo 'source(".ci/ci.R"); source(".ci/publish.R")' >> .Rprofile
    # list of available test-* jobs dynamically based on bus/test-* directories
    - Rscript -e 'cat("\ntest.jobs <- c(\n"); cat(paste0("  \"",list.files("bus",pattern="^test-"),"\" = \"data.table\""), sep=",\n"); cat(")\n")' >> .Rprofile
    - Rscript -e 'sapply(names(test.jobs), check.test, pkg="data.table", simplify=FALSE)'
    - mkdir -p bus/$CI_BUILD_NAME
    # delete any existing non-dev version of data.table
    - rm -f bus/mirror-packages/cran/src/contrib/data.table_*.tar.gz
    - rm -f bus/mirror-packages/cran/bin/windows/contrib/$R_BIN_VERSION/data.table_*.zip
    - rm -f bus/mirror-packages/cran/bin/windows/contrib/$R_DEVEL_BIN_VERSION/data.table_*.zip
    #- rm -f bus/mirror-packages/cran/bin/macosx/el-capitan/contrib/$R_BIN_VERSION/data.table_*.tgz
    #- rm -f bus/mirror-packages/cran/bin/macosx/el-capitan/contrib/$R_DEVEL_BIN_VERSION/data.table_*.tgz
    # merge mirror-packages and R devel packages
    - mv bus/mirror-packages/cran bus/$CI_BUILD_NAME/
    # publish package sources
    - mkdir -p bus/$CI_BUILD_NAME/cran/library bus/$CI_BUILD_NAME/cran/doc
    - mv $(ls -1t bus/build/cran/src/contrib/data.table_*.tar.gz | head -n 1) bus/$CI_BUILD_NAME/cran/src/contrib
    - Rscript -e 'tools::write_PACKAGES(contrib.url("bus/integration/cran", type="source"), type="source", fields="Revision", addFiles=TRUE)'
    # publish binaries
    - Rscript -e 'move.bin("test-rel-win", Sys.getenv("R_BIN_VERSION"), os.type="windows")'
    - Rscript -e 'move.bin("test-dev-win", Sys.getenv("R_DEVEL_BIN_VERSION"), os.type="windows", silent=TRUE)'
    - Rscript -e 'tools::write_PACKAGES(contrib.url("bus/integration/cran", type="win.binary", ver=Sys.getenv("R_BIN_VERSION")), type="win.binary", fields="Revision", addFiles=TRUE)'
    - Rscript -e 'tools::write_PACKAGES(contrib.url("bus/integration/cran", type="win.binary", ver=Sys.getenv("R_DEVEL_BIN_VERSION")), type="win.binary", fields="Revision", addFiles=TRUE)'
    #- Rscript -e 'move.bin("test-rel-osx", Sys.getenv("R_BIN_VERSION"), os.type="macosx")'
    #- Rscript -e 'move.bin("test-dev-osx", Sys.getenv("R_DEVEL_BIN_VERSION"), os.type="macosx", silent=TRUE)'
    #- Rscript -e 'tools::write_PACKAGES(contrib.url("bus/integration/cran", type="mac.binary.el-capitan", ver=Sys.getenv("R_BIN_VERSION")), type="mac.binary.el-capitan", fields="Revision", addFiles=TRUE)'
    #- Rscript -e 'tools::write_PACKAGES(contrib.url("bus/integration/cran", type="mac.binary.el-capitan", ver=Sys.getenv("R_DEVEL_BIN_VERSION")), type="mac.binary.el-capitan", fields="Revision", addFiles=TRUE)'
    # install all pkgs to render html and double check successful installation of all devel packages
    - mkdir -p /tmp/opencran/library /tmp/opencran/doc/html
    - Rscript -e 'install.packages("data.table", dependencies=TRUE, lib="/tmp/opencran/library", repos=file.path("file:",normalizePath("bus/integration/cran")), INSTALL_opts="--html", quiet=TRUE)'
    - Rscript -e 'packageVersion("data.table", lib.loc="/tmp/opencran/library")'
    # CRAN style web/CRAN_web.css
    - wget -q -P bus/integration/cran/web https://cran.r-project.org/web/CRAN_web.css
    # web/packages/$pkg/index.html
    - Rscript -e 'sapply(rownames(installed.packages(lib.loc="/tmp/opencran/library", priority="NA")), package.index, lib.loc="/tmp/opencran/library")'
    # R docs, html, css, icons
    - Rscript -e 'doc.copy(repodir="/tmp/opencran")'
    # Update packages.html, rewrite file:/ to relative path
    - Rscript -e 'setwd("/tmp/opencran/doc/html"); make.packages.html(lib.loc="../../library", docdir="/tmp/opencran/doc"); tmp<-readLines(f<-"/tmp/opencran/doc/html/packages.html"); writeLines(gsub("file:///../../library","../../library", tmp, fixed=TRUE), f)'
    - mv /tmp/opencran/doc bus/integration/cran/
    # library html manual, vignettes
    - Rscript -e 'lib.copy(lib.from="/tmp/opencran/library")'
    # web/checks/$pkg/$job: 00install.out, 00check.log, *.Rout, memtest.csv, memtest.png
    - Rscript -e 'sapply(names(test.jobs), check.copy, simplify=FALSE)'
    # web/packages/$pkg/$pkg.pdf
    - Rscript -e 'pdf.copy("data.table", "test-rel-lin")'
    # web/checks/check_results_$pkg.html
    - Rscript -e 'check.index("data.table", names(test.jobs))'
    # pkgdown merge
    - Rscript -e 'common_files<-function(path1, path2) intersect(list.files(path1, all.files=TRUE, no..=TRUE), list.files(path2, all.files=TRUE, no..=TRUE)); if (length(f<-common_files("pkgdown","bus/integration/cran"))) message(paste(c("Following artifacts will be overwritten by pkgdown artifacts:", paste0("  ", f)), collapse="\n")); q("no")'
    - mv pkgdown/* bus/integration/cran/
    # cleanup artifacts from other jobs
    - mkdir tmpbus
    - mv bus/$CI_BUILD_NAME tmpbus
    - rm -r bus
    - mv tmpbus bus
  <<: *artifacts

.docker-template: &docker
  stage: deploy
  tags:
    - linux
  image: docker
  services:
  - docker:dind
  dependencies:
  - build
  before_script:
    - sed "s/SRC_IMAGE_NAME/$SRC_IMAGE_NAME/" < .ci/Dockerfile.in > Dockerfile
    - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
  script:
    - docker build --pull -t "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG" -f Dockerfile .
    - docker run --rm "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG" Rscript -e 'cat(R.version.string, "\ndata.table revision", read.dcf(system.file("DESCRIPTION", package="data.table"), fields="Revision")[[1L]], "\n"); require(data.table); test.data.table()'
    - docker push "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG"

docker-r-release: # publish docker image of data.table on R-release
  only:
    - master
  variables:
    SRC_IMAGE_NAME: "r-base-dev"
    IMAGE_NAME: "r-release"
    IMAGE_TAG: "latest"
  <<: *docker

docker-r-release-builder: # publish on R-release and OS dependencies for building Rmd vignettes
  only:
    - master
  variables:
    SRC_IMAGE_NAME: "r-builder"
    IMAGE_NAME: "r-release-builder"
    IMAGE_TAG: "latest"
  <<: *docker

docker-r-devel: # publish docker image of data.table on R-devel
  only:
    - master
  variables:
    SRC_IMAGE_NAME: "r-devel"
    IMAGE_NAME: "r-devel"
    IMAGE_TAG: "latest"
  <<: *docker

docker-tags: # publish only on tagged commits, we use tags for version
  only:
    - tags
  variables:
    SRC_IMAGE_NAME: "r-base-dev"
    IMAGE_NAME: "r-release"
    IMAGE_TAG: $CI_COMMIT_TAG
  <<: *docker

pages: # publish R repository, test jobs summaries, html documentation of all packages in repo, pkgdown
  stage: deploy
  environment: production
  tags:
    - linux
  only:
    - master
  image: ubuntu
  dependencies:
  - integration
  script:
    - mkdir -p public
    - cp -r bus/integration/cran/* public
    - cat public/src/contrib/PACKAGES
  artifacts: # publish when no failure
    expire_in: 2 weeks
    paths:
      - public