Added FTP fetch and test (NOAA-EMC#120)

* Added FTP fetch and test

* Update CMakeLists.txt

* Updated CMakeLists.txt for FTP fetch

* Added test for large data file

* Fixed Comment

* Added FTP tests to developer workflow

* Removing large test file from test script for now

.github/workflows/developer.yml

@@ -90,6 +90,13 @@ jobs:
       with: 
         path: wgrib2
 
+    - name: cache-data
+      id: cache-data
+      uses: actions/cache@v3
+      with:
+        path: ~/data
+        key: data-1
+
     - name: build
       run: |
         cd wgrib2
@@ -98,10 +105,16 @@ jobs:
         export CFLAGS='-Wall -g -fprofile-abs-path -fprofile-arcs -ftest-coverage -O0'        
         export FCFLAGS='-Wall -g -fprofile-abs-path -fprofile-arcs -ftest-coverage -O0'        
         export FFLAGS='-Wall -g -fprofile-abs-path -fprofile-arcs -ftest-coverage -O0'        
-        cmake ..
+        cmake .. -DFTP_TEST_FILES=ON -DTEST_FILE_DIR=/home/runner/data
         make VERBOSE=1
         ctest --verbose --output-on-failure --rerun-failed
         gcovr --root .. -v  --html-details --exclude ../tests --exclude CMakeFiles --print-summary -o test-coverage.html &> /dev/null
+    
+    - name: cache-data
+      if: steps.cache-data.outputs != 'true'
+      run: |
+        mkdir ~/data
+        cp $GITHUB_WORKSPACE/wgrib2/build/tests/data/* ~/data
         
     - name: upload-test-coverage
       uses: actions/upload-artifact@v4

CMakeLists.txt

@@ -30,13 +30,22 @@ option(USE_PNG "Use PNG?" off)
 option(USE_JASPER "Use Jasper?" off)
 option(USE_OPENJPEG "Use OpenJPEG?" off)
 option(USE_AEC "Use AEC?" off)
+option(FTP_TEST_FILES "Fetch and test with files on FTP site." OFF)
+option(FTP_LARGE_TEST_FILES "Fetch and test with very large files on FTP site." OFF)
+option(FTP_EXTRA_TEST_FILES "Fetch even more large files from FTP and test them." OFF)
 # MAKE_FTN_API should only be on when building library
 option(MAKE_FTN_API "add ftn api?" off)
 option(DISABLE_STAT "disable posix feature" off)
 set(BUILD_COMMENTS "stock build")
 option(BUILD_LIB "Build wgrib2 library?" on)
 option(BUILD_SHARED_LIB "Build shared library?" off)
 
+# Developers can use this option to specify a local directory which
+# holds the test files. They will be copied instead of fetching the
+# files via FTP.
+SET(TEST_FILE_DIR "." CACHE STRING "Check this directory for test files before using FTP.")
+message(STATUS "Finding test data files in directory ${TEST_FILE_DIR}.")
+
 list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake")
 
 # Set default install path if not provided.

tests/CMakeLists.txt

@@ -1,7 +1,7 @@
 # This is the CMake file for the test directory in the wgrib2
 # project. 
 #
-# Ed Hartnett 3/27/24
+# Ed Hartnett, Alyson Stahl 3/27/24
 
 # Run each shell test.
 function(shell_test name)
@@ -22,10 +22,73 @@ function(copy_test_data name)
     FILE_PERMISSIONS OWNER_READ OWNER_WRITE GROUP_READ WORLD_READ)
 endfunction()
 
+
+# Some test files are large and are kept on the NOAA EMC FTP
+# site. This function is used to download such test data. It takes two
+# arguments, the URL and the file to be downloaded.
+function(PULL_DATA THE_URL THE_FILE)
+  # If the TEST_FILE_DIR was specified, look for our test data files
+  # there before FTPing them. Developers can keep all test files on
+  # their machines, and save the time of downloading them every time.
+  if(NOT ${TEST_FILE_DIR} STREQUAL ".")
+    if (EXISTS ${TEST_FILE_DIR}/${THE_FILE})
+      message(STATUS "Copying file ${TEST_FILE_DIR}/${THE_FILE} to test data directory.")
+      FILE(COPY ${TEST_FILE_DIR}/${THE_FILE}
+        DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/data)
+    endif()
+  endif()
+  if(NOT EXISTS "${CMAKE_CURRENT_BINARY_DIR}/data/${THE_FILE}")
+    message(STATUS "Downloading file ${THE_URL}/${THE_FILE} to test data directory.")    
+    file(DOWNLOAD
+      ${THE_URL}/${THE_FILE}
+      ${CMAKE_CURRENT_BINARY_DIR}/data/${THE_FILE}
+      SHOW_PROGRESS
+      STATUS status
+      INACTIVITY_TIMEOUT 30
+      )
+    list(GET status 0 status_num)
+    if(NOT status_num EQUAL 0 OR NOT EXISTS "${CMAKE_CURRENT_BINARY_DIR}/data/${THE_FILE}")
+      message(FATAL_ERROR "Could not download ${THE_FILE}")
+    endif()
+  endif()
+endfunction()
+
+# Does the user want to get extra test files from the FTP site, and
+# run extra tests on them?
+if(FTP_TEST_FILES)
+  # This is the FTP site.
+  set(G2_FTP_URL "https://ftp.emc.ncep.noaa.gov/static_files/public/NCEPLIBS-g2")
+
+  # These are the test data files.
+  set(FTP_FILES WW3_Regional_US_West_Coast_20220718_0000.grib2)
+
+  # User may also ask for large test file.
+  if(FTP_LARGE_TEST_FILES)
+    set(FTP_FILES ${FTP_FILES} fv3lam.t00z.prslev.f000.grib2)
+  endif()
+
+  # User may also ask for extra large test files.
+  if(FTP_EXTRA_TEST_FILES)
+    set(FTP_FILES ${FTP_FILES} rrfs.t18z.prslev.f000.grib2)
+  endif()
+  message(STATUS "Getting these files from FTP: ${FTP_FILES}.")      
+
+  # Get each of the test data files.
+  foreach(THE_FILE IN LISTS FTP_FILES) 
+    PULL_DATA(${G2_FTP_URL} ${THE_FILE})
+  endforeach()
+
+endif()
+
 # Copy test data file into buiild directory for testing.
 copy_test_data(gdaswave.t00z.wcoast.0p16.f000.grib2)
 copy_test_data(ref_gdaswave.t00z.wcoast.0p16.f000.grib2.inv)
 copy_test_data(ref_npts_gdaswave.t00z.wcoast.0p16.f000.grib2.txt)
+copy_test_data(ref_WW3_Regional_US_West_Coast_20220718_0000.grib2.inv)
 
 # Run these shell tests.
 shell_test(run_wgrib2_tests)
+
+if (FTP_TEST_FILES)
+  shell_test(run_wgrib2_ftp_tests)
+endif()