Improve support for Intel oneAPI

.github/workflows/build_test.yaml

@@ -64,7 +64,7 @@ jobs:
       working-directory: libs
       run: >
         ${{ matrix.config.python }} ../src/utilities/build_libs.py --no_dl_progress --nthread=2 --ref_blas
-        --build_umfpack=1 --build_arpack=1 --oft_build_python --oft_build_tests
+        --build_umfpack=1 --build_arpack=1 --oft_build_tests=1
 
     - uses: actions/upload-artifact@v3
       if: failure()

src/CMakeLists.txt

@@ -34,6 +34,10 @@ if ("${CMAKE_Fortran_COMPILER_ID}" MATCHES "Intel")
   if(NOT OFT_PACKAGE_BUILD)
     set( CMAKE_Fortran_FLAGS_DEBUG "${CMAKE_Fortran_FLAGS_DEBUG} -check bounds,pointers,shape,uninit" )
   endif()
+  # Needed for new oneAPI Intel compilers
+  if ("${CMAKE_CXX_COMPILER_ID}" MATCHES "IntelLLVM")
+    set( EXT_LIBS "-lstdc++" )
+  endif()
 elseif("${CMAKE_Fortran_COMPILER_ID}" MATCHES "GNU")
   set( CMAKE_C_FLAGS_DEBUG "-g -O0" )
   set( CMAKE_CXX_FLAGS_DEBUG ${CMAKE_C_FLAGS_DEBUG} )
@@ -236,17 +240,28 @@ if(OFT_PACKAGE_BUILD)
   file(WRITE null.c "")
   add_library( oft SHARED null.c )
   set_target_properties( oft PROPERTIES LINKER_LANGUAGE Fortran )
+  set( EXT_STATIC_LIBS "" )
+  set( EXT_SHARED_LIBS "" )
+  foreach(LIB IN LISTS EXT_LIBS)
+    if( LIB MATCHES ".a$" )
+      set( EXT_STATIC_LIBS ${LIB} ${EXT_STATIC_LIBS} )
+      #message(STATUS "Static: ${LIB}")
+    else()
+      set( EXT_SHARED_LIBS ${LIB} ${EXT_SHARED_LIBS} )
+      #message(STATUS "Shared: ${LIB}")
+    endif()
+  endforeach()
   if (WIN32)
-    target_link_libraries( oft PRIVATE ${OFT_LIBS} ${EXT_LIBS} )
+    target_link_libraries( oft PRIVATE ${OFT_LIBS} ${EXT_STATIC_LIBS} )
     set_target_properties( oft PROPERTIES
         LINK_FLAGS "/WHOLEARCHIVE"
     )
   elseif (APPLE)
-    foreach(LIB IN LISTS OFT_LIBS EXT_LIBS)
+    foreach(LIB IN LISTS OFT_LIBS OFT_STATIC_EXT)
       target_link_libraries( oft PRIVATE "-Wl,-force_load" ${LIB} )
     endforeach()
   else ()
-    target_link_libraries( oft PRIVATE "-Wl,-whole-archive" ${OFT_LIBS} ${EXT_LIBS} "-Wl,-no-whole-archive" )
+    target_link_libraries( oft PRIVATE "-Wl,-whole-archive" ${OFT_LIBS} ${EXT_STATIC_LIBS} "-Wl,-no-whole-archive" )
     # set_target_properties(oft PROPERTIES
     #     LINK_FLAGS "-Wl,--allow-multiple-definition"
     # )
@@ -258,7 +273,7 @@ if(OFT_PACKAGE_BUILD)
   )
   set( OFT_SHARED_LIBS oft ${OFT_SHARED_LIBS} CACHE INTERNAL "package_shared_libs" )
   set( OFT_LIBS oft )
-  set( EXT_LIBS "" )
+  set( EXT_LIBS ${EXT_SHARED_LIBS} )
 endif()
 
 ########################

src/cmake/patch_package.py

@@ -9,31 +9,48 @@ def run_command(command):
     return outs.decode(), errs.decode(), errcode
 
 def get_prereqs(filename):
-    output, error, errcode = run_command('readelf -d {0}'.format(filename))
-    # print(filename)
     preqs = []
+    known_libs = {}
+    output, error, errcode = run_command('readelf -d {0}'.format(filename))
+    if errcode != 0:
+        return preqs, known_libs
+    #print(filename)
     for line in output.splitlines():
         if line.find('Shared library:') > 0:
             preqs.append(line.split('[')[1].split(']')[0])
-    # for preq in preqs:
-    #     print('  '+preq)
-    return preqs
+    #for preq in preqs:
+    #    print('  '+preq)
+    ldd_output, error, errcode = run_command('ldd -d {0}'.format(filename))
+    if errcode == 0:
+        for line in ldd_output.splitlines():
+            if line.find('=>') > 0:
+                known_libs[line.split('=>')[0].strip()] = line.split('=>')[1].split('(')[0].strip()
+    return preqs, known_libs
 
+print('Copying required dynamic libraries')
 output, error, errcode = run_command('ldconfig -p')
 ld_confoutput = output.splitlines()
 
+MKL_ROOT = None
 for i in range(2):
     full_prequisites = {}
+    known_resolutions = {}
     # Gather prerequisites
     patch_files = [f for f in os.listdir() if os.path.isfile(f)]
     for filename in patch_files:
-        preqs = get_prereqs(filename)
+        preqs, new_known = get_prereqs(filename)
         for preq in preqs:
             if preq not in full_prequisites:
                 full_prequisites[preq] = None
+        for known_lib in new_known:
+            if known_lib not in known_resolutions:
+                known_resolutions[known_lib] = new_known[known_lib]
 
     # Find system libraries that are not typically in a base install
-    filter_list = ['libgomp.','libgfortran.','libquadmath.']
+    filter_list = [
+        'libgomp.','libgfortran.','libquadmath.',
+        'libmkl','libifport','libifcoremt','libimf','libsvml','libintlc','libirng','libiomp5'
+    ]
     keep_keys = []
     for key in full_prequisites:
         for known_lib in filter_list:
@@ -42,18 +59,24 @@ def get_prereqs(filename):
     prequisites = {}
     for key in keep_keys:
         prequisites[key] = full_prequisites[key]
-
     # Find actual files
     for line in ld_confoutput:
         stripped_line = line.strip()
         for key in prequisites:
-            if stripped_line.startswith(key):
+            if key in known_resolutions:
+                prequisites[key] = known_resolutions[key]
+            elif stripped_line.startswith(key):
                 prequisites[key] = stripped_line.split('=>')[1].strip()
     for key in prequisites:
         path = prequisites[key]
         realpath = os.path.realpath(prequisites[key])
         if i == 1:
-            print('{0} -> {1} ({2})'.format(key, path, realpath))
+            if path == realpath:
+                print('  {0} -> {1}'.format(key, path))
+            else:
+                print('  {0} -> {1} ({2})'.format(key, path, realpath))
+            if (MKL_ROOT is None) and key.startswith('libmkl'):
+                MKL_ROOT = os.path.dirname(realpath)
         filename = os.path.basename(realpath)
         symname = os.path.basename(key)
         if filename == symname:
@@ -66,6 +89,32 @@ def get_prereqs(filename):
                 pass
             os.symlink(filename, symname)
 
+# Copy MKL computational kernels if linked with MKL
+if MKL_ROOT is not None:
+    print('Detected oneAPI MKL usage: {0}'.format(MKL_ROOT))
+    mkl_comp_libs = ('libmkl_def', 'libmkl_avx2')
+    mkl_libs = []
+    for mkl_lib in os.listdir(MKL_ROOT):
+        for comp_lib in mkl_comp_libs:
+            if mkl_lib.startswith(comp_lib):
+                mkl_libs.append(mkl_lib)
+                break
+    for mkl_lib in mkl_libs:
+        path = os.path.join(MKL_ROOT,mkl_lib)
+        realpath = os.path.realpath(path)
+        filename = os.path.basename(realpath)
+        symname = os.path.basename(mkl_lib)
+        print('  Copying MKL kernel lib: {0}'.format(mkl_lib))
+        if filename == symname:
+            shutil.copy(path, mkl_lib)
+        else:
+            shutil.copy(realpath, filename)
+            try:
+                os.remove(symname)
+            except FileNotFoundError:
+                pass
+            os.symlink(filename, symname)
+
 # Patch files with $ORIGIN runpath
 for filename in patch_files:
     run_command("patchelf --set-rpath '$ORIGIN' {0}".format(filename))

src/lin_alg/oft_lu.F90

@@ -747,8 +747,10 @@ subroutine lusolver_delete(self)
 REAL(r8), ALLOCATABLE, DIMENSION(:) :: rvals
 self%refactor=.TRUE.
 self%update_graph=.TRUE.
-NULLIFY(self%A)
-IF(.NOT.self%initialized)RETURN
+IF(.NOT.self%initialized)THEN
+  NULLIFY(self%A)
+  RETURN
+END IF
 ALLOCATE(ivals(1),rvals(1))
 SELECT CASE(TRIM(self%package))
 #ifdef HAVE_SUPERLU
@@ -797,6 +799,7 @@ subroutine lusolver_delete(self)
     DEALLOCATE(self%ipiv,self%atmp)
 END SELECT
 DEALLOCATE(ivals,rvals)
+NULLIFY(self%A)
 self%initialized=.FALSE.
 end subroutine lusolver_delete
 !------------------------------------------------------------------------------

src/physics/grad_shaf.F90

@@ -5334,6 +5334,7 @@ subroutine compute_bcmat(self)
     END IF
   END DO
 END DO
+DEALLOCATE(el1)
 !---
 allocate(self%bc_rhs_list(self%bc_nrhs))
 allocate(self%bc_lmat(oft_blagrange%nbe,oft_blagrange%nbe))
@@ -5358,7 +5359,8 @@ subroutine compute_bcmat(self)
 !$omp parallel private(j,jr,k,kk,rop1,gop1,loc_map1,cell1,el1,f1,ed1,dl1,dn1,dl1_mag,pts1,val, &
 !$omp rop2,gop2,loc_map2,el2,f2,dl2,dn2,dl2_mag,pt2,pts2,work,neval,ierr,iwork,last,ltmp,goptmp1) &
 !$omp reduction(+:nfail)
-ALLOCATE(rop1(oft_blagrange%nce),gop1(3,oft_blagrange%nce),loc_map1(oft_blagrange%nce))
+ALLOCATE(el1(oft_blagrange%nce),loc_map1(oft_blagrange%nce))
+ALLOCATE(rop1(oft_blagrange%nce),gop1(3,oft_blagrange%nce))
 ALLOCATE(el2(oft_blagrange%nce),loc_map2(oft_blagrange%nce))
 ALLOCATE(rop2(oft_blagrange%nce),gop2(3,oft_blagrange%nce))
 !$omp do schedule(static,10)

src/tests/CMakeLists.txt

@@ -12,6 +12,16 @@ function(oft_add_test)
   endif()
 endfunction(oft_add_test)
 
+# Find pytest
+if( NOT OFT_PYTEST_EXE )
+  find_program( OFT_PYTEST_EXE NAMES pytest py.test pytest-3 py.test-3 )
+endif()
+if( OFT_PYTEST_EXE )
+  message( STATUS "Found pytest: ${OFT_PYTEST_EXE}" )
+else()
+  message( FATAL_ERROR "pytest not found but required for automated tests" )
+endif()
+
 # Add sub-directories and copy testing script
 file( COPY oft_testing.py DESTINATION ${CMAKE_CURRENT_BINARY_DIR} )
 add_subdirectory( grid )
@@ -36,13 +46,13 @@ endif()
 add_custom_target(test)
 add_custom_command(
     TARGET  test
-    COMMAND PATH=${OFT_TEST_PATH} OFT_HAVE_MPI=${OFT_MPI_TEST} OFT_DEBUG_TEST=${OFT_DEBUG_TEST} pytest -m \"not slow\" grid lin_alg fem physics
+    COMMAND PATH=${OFT_TEST_PATH} OFT_HAVE_MPI=${OFT_MPI_TEST} OFT_DEBUG_TEST=${OFT_DEBUG_TEST} ${OFT_PYTEST_EXE} -m \"not slow\" grid lin_alg fem physics
     WORKING_DIRECTORY ${CMAKE_BINARY_DIR}/tests
 )
 # Add "full" testing command to Makefile
 add_custom_target(test_full)
 add_custom_command(
     TARGET  test_full
-    COMMAND PATH=${OFT_TEST_PATH} OFT_HAVE_MPI=${OFT_MPI_TEST} OFT_DEBUG_TEST=${OFT_DEBUG_TEST} pytest grid lin_alg fem physics
+    COMMAND PATH=${OFT_TEST_PATH} OFT_HAVE_MPI=${OFT_MPI_TEST} OFT_DEBUG_TEST=${OFT_DEBUG_TEST} ${OFT_PYTEST_EXE} grid lin_alg fem physics
     WORKING_DIRECTORY ${CMAKE_BINARY_DIR}/tests
-)
+)

src/tests/lin_alg/test_native_bjacobi.F90

@@ -70,6 +70,12 @@ PROGRAM test_native_bjacobi
   CALL oft_finalize
 END IF
 #endif
+#if !defined( HAVE_MKL )
+IF(sol_type==5)THEN
+  WRITE(*,*)'SKIP TEST: 5'
+  CALL oft_finalize
+END IF
+#endif
 !---Setup grid
 CALL multigrid_construct
 IF(mesh%cad_type/=mesh_cube_id)CALL oft_abort('Wrong mesh type, test for CUBE only.','main',__FILE__)
@@ -135,6 +141,8 @@ SUBROUTINE test_lap(nlocal,sol_type)
     linv_pre_lu%package='mumps'
   CASE(4)
     linv_pre_lu%package='umfpack'
+  CASE(5)
+    linv_pre_lu%package='pardiso'
 END SELECT
 !---Solve
 CALL u%set(1.d0)

src/tests/lin_alg/test_native_bjacobi.py

@@ -6,7 +6,7 @@
 sys.path.append(os.path.abspath(os.path.join(test_dir, '..')))
 from oft_testing import run_OFT
 
-nSolvers = 4
+nSolvers = 5
 
 # Basic template for input file
 oft_in_template = """