Bug fixes, continuous integration, logical variable with variable typ…

…e, simplified internal build, Python scripts for RFMIP examples

Accumulated results from ICON GPU Hackathon 2019.  Closes (#28).

-- Several small bug fixes (argument intent, maximum interpolation indices, thanks to Sebastian Rast)
-- Parameterized checking for out-of-range values (works also on GPU)
-- Continuous integration with Travis (thanks to Valentin Clement)
-- Logical type defaults to Fortran; can be set to use c_bool with -DUSE_CBOOL
-- Internal build system can use environmental variables instead of specified files (Makefile.conf etc.) to define compilers, flags, choose kernel directory
-- Python scripts to automate running and testing of RFMIP examples
-- Update RFMIP examples to use version 1.2 of atmospheres file
-- End-to-end RFMIP examples on GPU are broken; fixes pending

.gitignore

@@ -13,3 +13,5 @@
 # Don't commit specific Makefile variants
 Makefile.libs
 Makefile.conf
+
+src/

.travis.yml

@@ -0,0 +1,120 @@
+sudo: required
+dist: xenial
+language: cpp
+# Could run both sets of kernels in parallel but setup is the majority of the time
+#env:
+#  - RTE_KERNELS=""
+#  - RTE_KERNELS="openacc"
+matrix:
+  include:
+  - compiler: gcc # Using
+    env:
+    - FC=pgfortran
+    - FCFLAGS="-Mallocatable=03 -Mstandard"
+    - TEST_PGI=yes
+    addons:
+      apt:
+        sources:
+          - ubuntu-toolchain-r-test
+        packages:
+          - gcc-8
+          - gfortran-8
+          - libnetcdf-dev
+  - compiler: gcc
+    env:
+    - FC=gfortran-9
+    - FCFLAGS="-ffree-line-length-none -m64 -std=f2003 -march=native -DUSE_CBOOL"
+    - TEST_PGI=no
+    - NCHOME=/usr/
+    addons:
+      apt:
+        sources:
+          - ubuntu-toolchain-r-test
+        packages:
+          - gcc-9
+          - gfortran-9
+          - libnetcdf-dev
+  - compiler: gcc
+    env:
+    - FC=gfortran-8
+    - FCFLAGS="-ffree-line-length-none -m64 -std=f2003 -march=native -DUSE_CBOOL"
+    - TEST_PGI=no
+    - NCHOME=/usr/
+    addons:
+      apt:
+        sources:
+          - ubuntu-toolchain-r-test
+        packages:
+          - gcc-8
+          - gfortran-8
+          - libnetcdf-dev
+# cache:
+#   timeout: 1000
+#   directories:
+#     - "$HOME/pgi/"
+before_install:
+# Install PGI Community edition if not present in cache
+- if [[ "${TEST_PGI}" == "yes" ]]; then
+  wget -q 'https://raw.githubusercontent.com/nemequ/pgi-travis/master/install-pgi.sh';
+  chmod +x install-pgi.sh;
+  ./install-pgi.sh --nvidia;
+  pgfortran --version;
+  echo "switch -pthread is" > .userrc;
+  echo "append(LDLIB1=-lpthread);" >> .userrc;
+  fi
+
+# Install netcdf-fortran for PGI
+- git clone https://github.com/Unidata/netcdf-fortran.git --branch v4.4.4
+- cd netcdf-fortran
+- FC=$FC ./configure --prefix=${HOME}/netcdff
+- make
+- make install
+- cd .. || exit 1
+
+# Install CONDA
+- wget http://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh
+- chmod +x miniconda.sh
+- ./miniconda.sh -b
+- export PATH=/home/travis/miniconda3/bin:$PATH
+- conda update --yes conda
+
+install:
+- conda create --yes -n test python=$TRAVIS_PYTHON_VERSION
+- source activate test
+- conda install --yes urllib3 netcdf4 xarray dask
+
+before_script:
+- cd examples/rfmip-clear-sky
+- python3 ./stage_files.py
+- cd ../..
+- export RRTMGP_DIR=$PWD/build
+- export NFHOME=${HOME}/netcdff
+- export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$NFHOME/lib
+script:
+# Build library - default kernels
+- cd build || exit 1
+- make || exit 1
+- cd .. || exit 1
+# Build drivers
+- cd examples/rfmip-clear-sky
+- make
+# Run drivers
+- python3 ./run-rfmip-examples.py
+- python3 ./compare-to-reference.py
+- cd ../..
+# Build library - Open-acc kernels
+- export RTE_KERNELS="openacc"
+- cd build
+- make clean
+- make || exit 1
+- cd ../examples/rfmip-clear-sky
+- make clean
+- make || exit 1
+# Run drivers
+- python3 ./run-rfmip-examples.py
+- python3 ./compare-to-reference.py
+- cd ../..
+notifications:
+  email:
+    on_success: never
+    on_failure: never

README.md

@@ -1,6 +1,6 @@
 # RTE+RRTMGP
 
-This is the repository for RTE+RRTMGP, a set of codes for computing radiative fluxes in planetary atmospheres. RTE+RRTMGP is described in a [manuscript submitted Jan 17, 2019](https://owncloud.gwdg.de/index.php/s/JQo9AeRu6uIwVyR) to [Journal of Advances in Modeling Earth Systems](http://james.agu.org). 
+This is the repository for RTE+RRTMGP, a set of codes for computing radiative fluxes in planetary atmospheres. RTE+RRTMGP is described in a [manuscript revised May 10, 2019](https://doi.org/10.1002/essoar.10500964.1) to [Journal of Advances in Modeling Earth Systems](http://james.agu.org). 
 
 RRTMGP uses a k-distribution to provide an optical description (absorption and possibly Rayleigh optical depth) of the gaseous atmosphere, along with the relevant source functions, on a pre-determined spectral grid given temperatures, pressures, and gas concentration. The k-distribution currently distributed with this package is applicable to the Earth's atmosphere under present-day, pre-industrial, and 4xCO2 conditions.
 
@@ -11,5 +11,13 @@ Example programs and documenation are evolving - please see examples/ in the rep
 ## Building the libraries.
 
 1. `cd build`
-2. Create a file `Makefile.conf` defining make variables `FC` (the Fortran 2003 compiler) and `FCFLAGS` (compiler flags). Alternately  link to an existing file or set these as environment variables.
+2. Set environment variables `FC` (the Fortran 2003 compiler) and `FCFLAGS` (compiler flags). Alternately create a Makefile.conf that sets these variables. You could also link to an existing file. 
+3. Set environment variable `RTE_KERNELS` to `openacc` if you want the OpenACC kernels rather than the default. 
+4. `make`
+
+## Building and running the examples.
+
+1. From the root RTE+RRTMGP directory: `cd examples/rfmip-clear-sky`. 
+2. Set environment variables `NCHOME` and `NFHOME` to the root directories of the Netcdf C and Fortran libraries respectively. Set environment variable `RRTMGP_DIR` to the location of the libraries (`../../build`) in the default layout). 
 3. `make`
+4. Python scripts are provided to stage the files needed (`stage_files.py`),  run the examples `run-rfmip-examples.py`), and compare to results computed on an example host (`compare-to-reference.py`). The python scripts require modules xarray and netCDF.  

build/Makefile

@@ -1,15 +1,28 @@
-#!/bin/make
+#!/usr/bin/env make
+
 RTE_DIR = ../rte
-RTE_KERNEL_DIR = ../rte/kernels
 RRTMGP_DIR = ../rrtmgp
-RRTMGP_KERNEL_DIR = ../rrtmgp/kernels
+#
+# Choose kernels depending on environment variable
+#
+ifeq ($(RTE_KERNELS), openacc)
+  RTE_KERNEL_DIR    = ../rte/kernels-openacc
+  RRTMGP_KERNEL_DIR = ../rrtmgp/kernels-openacc
+endif
+RTE_KERNEL_DIR += ../rte/kernels
+RRTMGP_KERNEL_DIR += ../rrtmgp/kernels
 VPATH = $(RTE_DIR):$(RTE_KERNEL_DIR):$(RRTMGP_DIR):$(RRTMGP_KERNEL_DIR)
-
-include Makefile.conf
-include Makefile.rules
+#
+# Compiler variables FC, FCFLAGS can be set in the environment or in Makefile.conf
+#
+-include Makefile.conf
 
 all: librte.a librrtmgp.a
 
+COMPILE = $(FC) $(FCFLAGS) $(FCINCLUDE) -c
+%.o: %.F90
+	$(COMPILE) $<
+
 include $(RTE_DIR)/Make.depends
 include $(RRTMGP_DIR)/Make.depends
 

build/Makefile.conf.olcf_pgi

@@ -16,4 +16,4 @@ export FC = mpif90
 # answers against baselines, but will need to be fixed before running big
 # expensive jobs probably.
 #
-export FCFLAGS = -O0 -g -ta=nvidia,cc70,ptxinfo,managed -Minfo=accel
+export FCFLAGS = -O0 -g -ta=nvidia,cc70,ptxinfo -Minfo=accel

build/Makefile.conf.pgfortran-cscs

@@ -1,10 +1,10 @@
 #!/bin/make
 
-# Load the following modules
+# Load the following modules to compile with PGI for CPU
 #
 #  module swap PrgEnv-cray PrgEnv-pgi
+#  module swap pgi pgi/18.10.0
 #  module load cray-netcdf cray-hdf5
-#  module load craype-accel-nvidia35
 #  module unload cray-libsci_acc
 #
 #
@@ -13,9 +13,7 @@ FC = ftn
 NCHOME =
 
 # Fortran compiler flags
-#FCFLAGS = -g -Minfo -Mbounds -Mchkptr -Mstandard -Kieee -Mchkstk -Mipa=fast,inline -acc -ta=tesla:6.5
-#FCFLAGS = -g -Minfo -Mbounds -Mchkptr -Mstandard -Kieee -Mchkstk -Mipa=fast,inline -ta=tesla
-#FCFLAGS = -g -Minfo -Mbounds -Mchkptr -Mstandard -Kieee -Mchkstk -Mipa=fast,inline -Mallocatable=03 -D__MAP_NLAY_NLON_NLAT__
+#FCFLAGS = -g -Minfo -Mbounds -Mchkptr -Mstandard -Kieee -Mchkstk -Mipa=fast,inline -Mallocatable=03
 FCFLAGS = -g -Minfo -Mbounds -Mchkptr -Mstandard -Kieee -Mchkstk                   -Mallocatable=03
 
 # Fortran .mod files, e.g. -I<include dir> if you have headers in a nonstandard directory <include dir>

build/Makefile.conf.pgfortran-cscs-gpu

@@ -14,9 +14,7 @@ NCHOME =
 
 # Fortran compiler flags
 #FCFLAGS = -g -Minfo -Mbounds -Mchkptr -Mstandard -Kieee -Mchkstk -Mipa=fast,inline -acc -ta=tesla:6.5
-#FCFLAGS = -g -Minfo -Mbounds -Mchkptr -Mstandard -Kieee -Mchkstk -Mipa=fast,inline -ta=tesla
-#FCFLAGS = -g -Minfo -Mbounds -Mchkptr -Mstandard -Kieee -Mchkstk -Mipa=fast,inline -Mallocatable=03 -D__MAP_NLAY_NLON_NLAT__
-FCFLAGS = -g -ta=tesla:cc60,cuda9.0 -Minfo -Mbounds -Mchkptr -Mstandard -Kieee -Mchkstk                   -Mallocatable=03
+FCFLAGS = -g -ta=tesla:cc60,cuda9.1 -Minfo -Mbounds -Mchkptr -Mstandard -Kieee -Mchkstk                   -Mallocatable=03
 
 # Fortran .mod files, e.g. -I<include dir> if you have headers in a nonstandard directory <include dir>
 FCINCLUDE =

examples/rfmip-clear-sky/Makefile

@@ -1,8 +1,9 @@
 #
 # Here set variables RRTMGP_DIR, NCHOME, NFHOME, TIME_DIR (for GPTL)
+#   or have those variables set in the environment
 #
-include Makefile.libs
-include $(RRTMGP_DIR)/Makefile.conf
+-include Makefile.libs
+-include $(RRTMGP_DIR)/Makefile.conf
 #
 # RRTMGP library, module files
 #
@@ -52,11 +53,11 @@ ADDITIONS = mo_simple_netcdf.o mo_rfmip_io.o mo_load_coefficients.o
 
 all: rrtmgp_rfmip_lw rrtmgp_rfmip_sw
 
-rrtmgp_rfmip_lw:   rrtmgp_rfmip_lw.o   $(ADDITIONS)
+rrtmgp_rfmip_lw:   rrtmgp_rfmip_lw.o   $(ADDITIONS) $(RRTMGP_DIR)/librte.a $(RRTMGP_DIR)/librrtmgp.a
 
 rrtmgp_rfmip_lw.o: rrtmgp_rfmip_lw.F90 $(ADDITIONS)
 
-rrtmgp_rfmip_sw:   rrtmgp_rfmip_sw.o   $(ADDITIONS)
+rrtmgp_rfmip_sw:   rrtmgp_rfmip_sw.o   $(ADDITIONS) $(RRTMGP_DIR)/librte.a $(RRTMGP_DIR)/librrtmgp.a
 
 rrtmgp_rfmip_sw.o: rrtmgp_rfmip_sw.F90 $(ADDITIONS)
 

examples/rfmip-clear-sky/Makefile.libs.pgfortran-cscs

@@ -0,0 +1,17 @@
+# Load the following modules
+#
+#  module swap PrgEnv-cray PrgEnv-pgi
+#  module load cray-netcdf cray-hdf5
+#  module unload cray-libsci_acc
+
+export FC = ftn
+export FCFLAGS = -g -Minfo -Mbounds -Mchkptr -Mstandard -Kieee -Mchkstk -Mipa=fast,inline -Mallocatable=03
+
+# Where to find Fortran .mod files
+export FCINCLUDE =
+
+# Libraries including netCDF and Fortran interface to netCDF
+export LDFLAGS =
+
+#export LIBS = -lnetcdff -lnetcdf   # Not necessary if ftn wrapper is used
+export LIBS =

examples/rfmip-clear-sky/compare-to-reference.py

@@ -0,0 +1,33 @@
+#! /usr/bin/env python
+#
+# This script compares RFMIP results from RTE+RRTMGP against a benchmark
+#
+import os
+import numpy as np
+import xarray as xr
+
+ref_dir  = "reference"
+tst_dir = "."
+
+rrtmg_suffix = "_Efx_RTE-RRTMGP-181204_rad-irf_r1i1p1f1_gn.nc"
+
+tst = xr.open_mfdataset(os.path.join(tst_dir, "r??" + rrtmg_suffix))
+ref = xr.open_mfdataset(os.path.join(ref_dir, "r??" + rrtmg_suffix))
+
+for v in ['rlu', 'rld', 'rsu', 'rsd']:
+  if np.all(np.isnan(tst.variables[v].values)):
+    raise Exception("All test values are missing. Were the tests run?")
+  if np.any(np.isnan(tst.variables[v].values)):
+    raise Exception("Some test values are missing. Now that is strange.")
+
+  diff = abs((tst-ref).variables[v].values)
+  avg  = 0.5*(tst+ref).variables[v].values
+  # Division raises a runtime warning when we divide by zero even if the
+  #   values in those locations will be ignored.
+  with np.errstate(divide='ignore', invalid='ignore'):
+    frac_diff = np.where((avg > 2.*np.finfo(float).eps), diff/avg, 0)
+
+  if diff.max() > 0:
+    print('Variable %s differs (max abs difference: %e; max frac. difference: %e%%)'%(v, diff.max(), 100.0 * frac_diff.max()))
+  else:
+    print('Variable %s: No diffs'%(v))

examples/rfmip-clear-sky/mo_rfmip_io.F90

@@ -256,8 +256,8 @@ subroutine determine_gas_names(concentrationFile, kdistFile, forcing_index, name
       ! Not part of the RFMIP specification, but oxygen is included because it's a major
       !    gas in some bands in the SW
       !
-      names_in_kdist = ['co2   ', 'ch4   ', 'n2o   ', 'o2    ', 'cfc12  ', &
-                       'hfc134a']
+      names_in_kdist = ['co2    ', 'ch4    ', 'n2o    ', 'o2     ', 'cfc12  ', &
+                        'hfc134a']
       names_in_file =  ['carbon_dioxide', &
                         'methane       ', &
                         'nitrous_oxide ', &

examples/rfmip-clear-sky/rrtmgp_rfmip_lw.F90

@@ -225,10 +225,10 @@ program rrtmgp_rfmip_lw
   ! NOTE: these are causing problems right now, most likely due to a compiler
   ! bug related to the use of Fortran classes on the GPU.
   !
-  !$acc enter data copyin(sfc_emis_spec)
-  !$acc enter data copyin(optical_props, optical_props%tau)
-  !$acc enter data copyin(source, source%lay_source, source%lev_source_inc, source%lev_source_dec)
-  !$acc enter data copyin(source%sfc_source, source%band2gpt, source%gpt2band, source%band_lims_wvn)
+  !!!$acc enter data create(sfc_emis_spec)
+  !!$acc enter data create(optical_props, optical_props%tau)
+  !!$acc enter data create(source, source%lay_source, source%lev_source_inc, source%lev_source_dec)
+  !!$acc enter data create(source%sfc_source, source%band2gpt, source%gpt2band, source%band_lims_wvn)
   ! --------------------------------------------------
 #ifdef USE_TIMING
   !
@@ -251,7 +251,7 @@ program rrtmgp_rfmip_lw
     ! Expand the spectrally-constant surface emissivity to a per-band emissivity for each column
     !   (This is partly to show how to keep work on GPUs using OpenACC)
     !
-    !$acc parallel loop collapse(2)
+    !!$acc parallel loop collapse(2) copyin(sfc_emis)
     do icol = 1, block_size
       do ibnd = 1, nbnd
         sfc_emis_spec(ibnd,icol) = sfc_emis(icol,b)
@@ -299,10 +299,10 @@ program rrtmgp_rfmip_lw
   ret = gptlpr(block_size)
   ret = gptlfinalize()
 #endif
-  !$acc exit data delete(sfc_emis_spec)
-  !$acc exit data delete(optical_props%tau, optical_props)
-  !$acc exit data delete(source%lay_source, source%lev_source_inc, source%lev_source_dec, source%sfc_source)
-  !$acc exit data delete(source%band2gpt, source%gpt2band, source%band_lims_wvn, source)
+  !!!$acc exit data delete(sfc_emis_spec)
+  !!$acc exit data delete(optical_props%tau, optical_props)
+  !!$acc exit data delete(source%lay_source, source%lev_source_inc, source%lev_source_dec, source%sfc_source)
+  !!$acc exit data delete(source%band2gpt, source%gpt2band, source%band_lims_wvn, source)
   ! --------------------------------------------------m
   call unblock_and_write(trim(flxup_file), 'rlu', flux_up)
   call unblock_and_write(trim(flxdn_file), 'rld', flux_dn)