Merge pull request #6 from GEOS-ESM/feature/vruizxom/odas_milan

ODAS updates to allow runing reanalysis on Milan

src/Applications/GEOSgcm_App/ocean_das_config.tmpl

@@ -41,6 +41,13 @@ setenv ODAS_do_reinit       False             # True: input to anaice2rst.x is f
                                               # False: input to anaice2rst.x is from M2 ice frac
                                               #       for continuous analysis to be north pole
                                               #       hole free 
+setenv IN_LINE              False             # False: runs multiple observer jobs. NOT recomended for MILAN nodes
+                                              # True: runs observers within model job. Model pauses until complete. Recomended for MILAN nodes
+setenv ODAS_CONSTR          "[sky|cas]"       # Must be same node constraint as in gcm_run.j
+setenv BOMB_ON_SIGO         False             # True: BOMBS the model due to not meeting sigo ADT requirements (as before)
+                                              # False: Skips ADT observations that fail sigo requirements. Does not bomb.
+
+
 # Varibles used for salinity relaxation
 setenv SSSVAR                LEVITUSSC        # use SMAP seasonal cycle for SSS relaxation
                                               #   other options 

src/Applications/UMD_Etc/README

@@ -0,0 +1,62 @@
+ODAS UPDATE FOR MILAN
+=====================
+October, 2024
+[email protected]
+
+
+Problems:
+===========
+1. Given the pagepool memory limitation in SLES17,
+python observer code overloaded the memory resulting
+in a staling of the job with no progress. The problem
+was caused by the code loading full datafiles in memory
+via numpy arrays.
+
+2. Users without prioritty could not complete jobs
+as observer jobs in queue often cuased the main job 
+run out of time.
+
+3. Expected behavior of model BOMBING when ADT sigo variance
+is bellow a threshold. The user would have to either correct
+altimeter observations, or remove the problematic altimeter
+for the cycle, and resubmit for one cycle.
+
+4. Hardcoded path dependencies for utilities like ncap2 and
+mpi4py require code modification every time a software change
+is implemented.
+
+Solutions:
+===========
+1. Xarray paralelized lazy operations were implemented
+to replace numpy array operations:
+
+UMD_utils/ocean_obs.py replaces old code stored as 
+UMD_utils/ocean_obs.legacy.py
+
+Legacy is conserved because new xarray reader where not 
+implemented for some legacy observer readers.
+New reader might be implemented as needed.
+
+2. The IN_LINE environmental variable set in
+ocean_das_config when set to True, allows for observers to
+be executed within the same job. This option is slower than
+spawning multiple observer jobs, but in the long run minimizes
+queue wait times for users.
+NOTE: IN_LINE=False (old configuration) should not be used 
+on Milan nodes due to cpu per node quota.
+
+scripts/oda_run.j replaces old code stored as
+scripts/oda_run.legacy.j
+
+scripts/ocean_observer.py replaces old code stored as
+scripts/ocean_observer.legacy.py
+
+3. An option BOMB_ON_SIGO was implemented in ocean_das_config
+so when set to false, it will skip the altimeter data for
+the single observer, while conserving the ADT obs for the 
+rest of the cycle. BOMB_ON_SIGO default is set to True,
+to be kept for observation people to correct observation
+files.
+
+4. Hard coded paths where removed and new modules were
+aded to g5_modules.

src/Applications/UMD_Etc/UMD_oletkf/obsop.f90

@@ -165,7 +165,7 @@ PROGRAM obsop
   !write(guesfile(1:7),'(A4,I3.3)') 'bkg/',myrank+1
   !print *,'guesfile=',guesfile
 
-  write(guesfile(1:7),'(A4,I3.3)') guesfile(1:4),myrank+1
+  write(guesfile(5:7),'(I3.3)') myrank+1
   !print *,'fname=',fname
 
   CALL read_grd4(guesfile,v3ds,v2ds)

src/Applications/UMD_Etc/UMD_rc/ocean_das_config

@@ -41,6 +41,12 @@ setenv ODAS_do_reinit       False             # True: input to anaice2rst.x is f
                                               # False: input to anaice2rst.x is from M2 ice frac
                                               #       for continuous analysis to be north pole
                                               #       hole free 
+setenv IN_LINE              True              # False: runs multiple observer jobs. NOT recomended for MILAN nodes
+                                              # True: runs observers in line (within model job, model pauses to wait until complete). Recomended for MILAN nodes
+setenv ODAS_CONSTR          "[sky|cas]"       # Only used when IN_LINE is False. Must be same node constraint as in gcm_run.j
+                                              # Not recomended for use in Milan.
+setenv BOMB_ON_SIGO         False             # True: BOMBS the model due to not meeting sigo ADT requirements (as before)
+                                              # False: Skips ADT observations that fail sigo requirements. Does not bomb.
 # Varibles used for salinity relaxation
 setenv SSSVAR                LEVITUSSC        # use SMAP seasonal cycle for SSS relaxation
                                               #   other options 

src/Applications/UMD_Etc/UMD_utils/dynamic_height.py

@@ -7,7 +7,6 @@
 import glob
 import sys
 import utils
-#sys.path.append('/usr/local/other/SLES11/mpi4py-1.3/lib/python/')
 from mpi4py import MPI
 #Ne=int(sys.argv[1])
 

src/Applications/UMD_Etc/UMD_utils/get_slv_offset.py

@@ -26,23 +26,13 @@
 
 print('Adding offset to barotropic restart')
 # Add offset to eta_t
-#command = BASEDIR+'/Linux/bin/ncap -O -s "eta_t=(eta_t+'+str(offset)+')" '+BARO_RESTART+' '+BARO_RESTART
-#command = BASEDIR+'/Linux/bin/ncap2 -O -s "eta_t=(eta_t+'+str(offset)+')" '+BARO_RESTART+' '+BARO_RESTART
-#   next line appear to execute
-#command = '/discover/swdev/mathomp4/Baselibs/ESMA-Baselibs-4.0.11-SLES12/x86_64-unknown-linux-gnu/ifort_18.0.5.274-intelmpi_19.1.0.166-nco_2020Apr09_snapshot/Linux/bin/ncap2 -O -s "eta_t=(eta_t+'+str(offset)+')" '+BARO_RESTART+' '+BARO_RESTART
-#command = BASEDIR+'/Linux/bin/ncap2 -O -s "eta_t=(eta_t+'+str(offset)+')" '+BARO_RESTART+' '+BARO_RESTART
-command = '/discover/swdev/gmao_SIteam/Baselibs/latest-mpiuni-SLES12/Linux/bin/ncap2 -O -s "eta_t=(eta_t+'+str(offset)+')" '+BARO_RESTART+' '+BARO_RESTART
+command = 'ncap2 -O -s "eta_t=(eta_t+'+str(offset)+')" '+BARO_RESTART+' '+BARO_RESTART
 print(command)
 print('PAST NCAP2 in get_slv_offset.py')
 os.system(command)
 
 # Add offset to eta_u
-#command = BASEDIR+'/Linux/bin/ncap -O -s "eta_u=(eta_u+'+str(offset)+')" '+BARO_RESTART+' '+BARO_RESTART
-#command = BASEDIR+'/Linux/bin/ncap2 -O -s "eta_u=(eta_u+'+str(offset)+')" '+BARO_RESTART+' '+BARO_RESTART
-#   next line appear to execute
-#command = '/discover/swdev/mathomp4/Baselibs/ESMA-Baselibs-4.0.11-SLES12/x86_64-unknown-linux-gnu/ifort_18.0.5.274-intelmpi_19.1.0.166-nco_2020Apr09_snapshot/Linux/bin/ncap2 -O -s "eta_u=(eta_u+'+str(offset)+')" '+BARO_RESTART+' '+BARO_RESTART
-#command = BASEDIR+'/Linux/bin/ncap2 -O -s "eta_u=(eta_u+'+str(offset)+')" '+BARO_RESTART+' '+BARO_RESTART
-command = '/discover/swdev/gmao_SIteam/Baselibs/latest-mpiuni-SLES12/Linux/bin/ncap2 -O -s "eta_u=(eta_u+'+str(offset)+')" '+BARO_RESTART+' '+BARO_RESTART
+command = 'ncap2 -O -s "eta_u=(eta_u+'+str(offset)+')" '+BARO_RESTART+' '+BARO_RESTART
 print(command)
 print('PAST 2nd NCAP2 in get_slv_offset.py')
 os.system(command)