Zstd compressor (#3)

.github/workflows/ossfuzz_workflow.yml

@@ -25,6 +25,12 @@ on:
       - '!src/wrappers/**'
       - '!.github/workflows/**'
       - '.github/workflows/ossfuzz_workflow.yml'
+    # FIXME: Fuzzing build fails because we need to add libblosc2 to the list 
+    # of linked libs in:
+    # https://github.com/google/oss-fuzz/blob/master/projects/openexr/build.sh
+    # ... but we can only do it once this PR is merged.
+    branches-ignore:  # FIXME: REMOVE THIS CONDITION BEFORE MERGING !
+      - 'main'
 
 permissions:
   contents: read

BUILD.bazel

@@ -202,6 +202,7 @@ cc_library(
         "src/lib/OpenEXRCore/internal_win32_file_impl.h",
         "src/lib/OpenEXRCore/internal_xdr.h",
         "src/lib/OpenEXRCore/internal_zip.c",
+        "src/lib/OpenEXRCore/internal_zstd.c",
         "src/lib/OpenEXRCore/memory.c",
         "src/lib/OpenEXRCore/opaque.c",
         "src/lib/OpenEXRCore/openexr_version.h",
@@ -253,10 +254,12 @@ cc_library(
             ":windows": [],
             "//conditions:default": [
                 "-pthread",
+                "-ldl",
             ],
         }),
     visibility = ["//visibility:public"],
     deps = [
+        "@c-blosc2//:c-blosc2",
         "@imath//:Imath",
         "@libdeflate//:deflate",
     ],
@@ -357,6 +360,7 @@ cc_library(
         "src/lib/OpenEXR/ImfWav.cpp",
         "src/lib/OpenEXR/ImfZip.cpp",
         "src/lib/OpenEXR/ImfZipCompressor.cpp",
+        "src/lib/OpenEXR/ImfZstdCompressor.cpp",
         "src/lib/OpenEXR/b44ExpLogTable.h",
         "src/lib/OpenEXR/dwaLookups.h",
     ],
@@ -476,6 +480,7 @@ cc_library(
         "src/lib/OpenEXR/ImfXdr.h",
         "src/lib/OpenEXR/ImfZip.h",
         "src/lib/OpenEXR/ImfZipCompressor.h",
+        "src/lib/OpenEXR/ImfZstdCompressor.h",
         "src/lib/OpenEXR/OpenEXRConfig.h",
         "src/lib/OpenEXR/OpenEXRConfigInternal.h",
     ],
@@ -495,10 +500,12 @@ cc_library(
             ":windows": [],
             "//conditions:default": [
                 "-pthread",
+                "-ldl",
             ],
         }),
     visibility = ["//visibility:public"],
     deps = [
+        "@c-blosc2//:c-blosc2",
         ":IlmThread",
         ":OpenEXRCore",
         "@imath//:Imath",

MODULE.bazel

@@ -10,3 +10,4 @@ bazel_dep(name = "bazel_skylib", version = "1.6.1")
 bazel_dep(name = "imath", version = "3.1.11")
 bazel_dep(name = "libdeflate", version = "1.20.bcr.1")
 bazel_dep(name = "platforms", version = "0.0.10")
+bazel_dep(name = "c-blosc2", version = "2.12.0.bcr.2")

cmake/CMakeLists.txt

@@ -192,6 +192,7 @@ if(OPENEXR_INSTALL_PKG_CONFIG)
   function(openexr_pkg_config_help pcinfile)
     string(TOUPPER "${CMAKE_BUILD_TYPE}" uppercase_CMAKE_BUILD_TYPE)
     set(LIB_SUFFIX_DASH ${OPENEXR_LIB_SUFFIX}${CMAKE_${uppercase_CMAKE_BUILD_TYPE}_POSTFIX})
+    set(LIB_BUILD_SUFFIX ${CMAKE_${uppercase_CMAKE_BUILD_TYPE}_POSTFIX})
     if(OPENEXR_ENABLE_THREADING AND TARGET Threads::Threads)
       # hrm, can't use properties as they end up as generator expressions
       # which don't seem to evaluate

cmake/LibraryDefine.cmake

@@ -30,6 +30,20 @@ function(OPENEXR_DEFINE_LIBRARY libname)
   # we are embedding libdeflate
   target_include_directories(${objlib} PRIVATE ${EXR_DEFLATE_INCLUDE_DIR})
 
+  # we are statically linking blosc2
+  if(${objlib} STREQUAL "OpenEXR" OR ${objlib} STREQUAL "OpenEXRCore")
+    message(STATUS "Blosc2: setting up for ${objlib}...")
+    message(STATUS ">> BLOSC2_INCLUDE_DIRS: ${BLOSC2_INCLUDE_DIRS}")
+    message(STATUS ">> BLOSC2_LIB_DIR: ${BLOSC2_LIB_DIR}")
+    target_include_directories(${objlib} PRIVATE ${BLOSC2_INCLUDE_DIRS})
+    target_link_directories(${objlib} PRIVATE ${BLOSC2_LIB_DIR})
+    target_link_libraries(${objlib} PRIVATE Blosc2::blosc2_static ${CMAKE_DL_LIBS})
+    # install the static library if not using the installed lib.
+    if(TARGET blosc2_static AND NOT Blosc2_FOUND)
+      install(TARGETS blosc2_static EXPORT ${objlib})
+    endif()
+  endif()
+
   if(OPENEXR_CURLIB_PRIV_EXPORT AND BUILD_SHARED_LIBS)
     target_compile_definitions(${objlib} PRIVATE ${OPENEXR_CURLIB_PRIV_EXPORT})
     if(WIN32)

cmake/OpenEXR.pc.in

@@ -9,12 +9,13 @@ libdir=@PKG_CONFIG_INSTALL_LIBDIR@
 includedir=@PKG_CONFIG_INSTALL_INCLUDEDIR@
 OpenEXR_includedir=${includedir}/OpenEXR
 libsuffix=@LIB_SUFFIX_DASH@
+libbuildsuffix=@LIB_BUILD_SUFFIX@
 
 Name: OpenEXR
 Description: OpenEXR image library
 Version: @OPENEXR_VERSION@
 
-Libs: @exr_pthread_libs@ -L${libdir} -lOpenEXR${libsuffix} -lOpenEXRUtil${libsuffix} -lOpenEXRCore${libsuffix} -lIex${libsuffix} -lIlmThread${libsuffix}
+Libs: @exr_pthread_libs@ -L${libdir} -lOpenEXR${libsuffix} -lOpenEXRUtil${libsuffix} -lOpenEXRCore${libsuffix} -lIex${libsuffix} -lIlmThread${libsuffix} -lblosc2${libbuildsuffix} -ldl
 Cflags: -I${includedir} -I${OpenEXR_includedir} @exr_pthread_cflags@
 Requires: Imath
 Requires.private: @EXR_DEFLATE_PKGCONFIG_REQUIRES@

cmake/OpenEXRConfig.cmake.in

@@ -5,12 +5,10 @@
 
 include(CMakeFindDependencyMacro)
 
-set(openexr_needthreads @OPENEXR_ENABLE_THREADING@)
-if (openexr_needthreads)
-  set(THREADS_PREFER_PTHREAD_FLAG ON)
-  find_dependency(Threads)
-endif()
-unset(openexr_needthreads)
+# blosc2 needs threads, so we set it irrespective of OPENEXR_ENABLE_THREADING 
+# which enables threaded processing of requests.
+set(THREADS_PREFER_PTHREAD_FLAG ON)
+find_dependency(Threads)
 
 find_dependency(Imath)
 

cmake/OpenEXRSetup.cmake

@@ -1,6 +1,21 @@
 # SPDX-License-Identifier: BSD-3-Clause
 # Copyright (c) Contributors to the OpenEXR Project.
 
+function(_error_if_not_found prop var fallback)
+  message(STATUS "Blosc2: ${prop}  ${var}  '${fallback}'")
+  string(FIND "${var}" "-NOTFOUND" pos)
+  if(NOT pos EQUAL -1)
+    if(fallback STREQUAL "")
+      message(FATAL_ERROR "Blosc2: Property ${prop} not found: ${var}")
+    else()
+      string(SUBSTRING "${var}" 0 ${pos} var_name)
+      message(STATUS "Blosc2: Property ${prop} not found: ${var_name} falling back to '${fallback}'")
+      set(${var_name} "${fallback}" PARENT_SCOPE) 
+    endif()
+  endif()
+endfunction(_error_if_not_found)
+
+
 include(GNUInstallDirs)
 
 if(NOT "${CMAKE_PROJECT_NAME}" STREQUAL "${PROJECT_NAME}")
@@ -324,6 +339,87 @@ else()
   endif()
 endif()
 
+#######################################
+# Find or install Blosc2
+#######################################
+
+set(MINIMUM_BLOSC2_VERSION 2.11.0)
+option(OPENEXR_FORCE_INTERNAL_BLOSC2 [=[Force using installed Blosc2.]=] OFF)
+
+set(BLOSC2_REPO "https://github.com/Blosc/c-blosc2.git" CACHE STRING "Repo path for blosc2 source")
+set(BLOSC2_TAG "v${MINIMUM_BLOSC2_VERSION}" CACHE STRING "Tag to use for blosc2 source repo")
+
+# Try to find a local bloc2 install if allowed to.
+if(NOT OPENEXR_FORCE_INTERNAL_BLOSC2)
+  message(STATUS "Blosc2: Looking for local install...")
+  set(CMAKE_IGNORE_PATH "${CMAKE_CURRENT_BINARY_DIR}/_deps/blosc2-src/config;${CMAKE_CURRENT_BINARY_DIR}/_deps/blosc2-build/config")
+  find_package(Blosc2 ${MINIMUM_BLOSC2_VERSION})
+  set(CMAKE_IGNORE_PATH)
+endif()
+
+if(NOT TARGET Blosc2::blosc2_static AND NOT Blosc2_FOUND)
+  # we didn't find a local install: let's get it from its repository.
+  if(OPENEXR_FORCE_INTERNAL_BLOSC2)
+    message(STATUS "Blosc2: forced internal, installing from ${BLOSC2_REPO} (${BLOSC2_TAG})")
+  else()
+    message(STATUS "Blosc2: no local blosc2 found, installing from ${BLOSC2_REPO} (${BLOSC2_TAG})")
+  endif()
+
+  # configure the blosc2 build
+  set(BUILD_BENCHMARKS OFF CACHE INTERNAL "no benchmarks")
+  set(BUILD_EXAMPLES OFF CACHE INTERNAL "no examples")
+  set(BUILD_FUZZERS OFF CACHE INTERNAL "no fuzzer")
+  set(BUILD_SHARED OFF CACHE INTERNAL "no shared library")
+  set(BUILD_TESTS OFF CACHE INTERNAL "no tests")
+
+  include(FetchContent)
+  FetchContent_Declare(Blosc2
+    GIT_REPOSITORY "${BLOSC2_REPO}"
+    GIT_TAG "${BLOSC2_TAG}"
+    GIT_SHALLOW ON
+    GIT_PROGRESS ON)
+
+  FetchContent_GetProperties(Blosc2)
+  if(NOT Blosc2_POPULATED)
+    message(STATUS "Blosc2: Downloading ${BLOSC2_TAG} from ${BLOSC2_REPO}...")
+    FetchContent_Populate(Blosc2)
+    add_subdirectory(${blosc2_SOURCE_DIR} ${blosc2_BINARY_DIR})
+  else()
+    message(STATUS "Blosc2: repo code has already been downloaded.")
+  endif()
+
+  # the install creates this but if we're using the library locally we
+  # haven't installed the header files yet, so need to extract those
+  # and make a variable for header only usage
+  if(TARGET Blosc2::blosc2_static)
+    message(STATUS "Blosc2: Setting up blosc directories")
+
+    get_target_property(blosc2inc Blosc2::blosc2_static INCLUDE_DIRECTORIES)
+    set(BLOSC2_INCLUDE_DIRS ${blosc2inc})
+
+    get_target_property(blosc2libdir Blosc2::blosc2_static BINARY_DIR)
+    set(BLOSC2_LIB_DIR ${blosc2libdir})
+
+    if(OPENEXR_RUN_FUZZ_TESTS)
+      target_compile_options(blosc2_static PUBLIC "-gdwarf-4")
+    endif()
+  endif()
+else()
+  message(STATUS "Blosc2: Using installed Blosc2 ${Blosc2_VERSION} from ${Blosc2_DIR}")
+  # local build
+  if(TARGET Blosc2::blosc2_static)
+    message(STATUS "Blosc2: Setting up installed blosc directories")
+
+    get_target_property(blosc2inc Blosc2::blosc2_static INTERFACE_INCLUDE_DIRECTORIES)
+    _error_if_not_found("INTERFACE_INCLUDE_DIRECTORIES" ${blosc2inc} "")
+    set(BLOSC2_INCLUDE_DIRS ${blosc2inc})
+
+    get_target_property(blosc2libdir Blosc2::blosc2_static BINARY_DIR)
+    _error_if_not_found("BINARY_DIR" ${blosc2libdir} "")
+    set(BLOSC2_LIB_DIR ${blosc2libdir})
+  endif()
+endif()
+
 ###########################################
 # Check if we need to emulate vld1q_f32_x2
 ###########################################

src/examples/BUILD.bazel

@@ -24,5 +24,7 @@ cc_test(
         "rgbaInterfaceTiledExamples.cpp",
         "rgbaInterfaceTiledExamples.h",
     ],
-    deps = ["//:OpenEXR"],
+    deps = [
+        "//:OpenEXR",
+    ],
 )

src/examples/deepExamples.cpp

@@ -47,7 +47,7 @@ readDeepScanlineFile (
     //    - allocate memory for the pixels
     //    - describe the layout of the A, and Z pixel buffers
     //    - read the sample counts from the file
-    //    - allocate the memory requred to store the samples
+    //    - allocate the memory required to store the samples
     //    - read the pixels from the file
     //
 
@@ -148,7 +148,8 @@ writeDeepScanlineFile (
 
     Array2D<half*>& dataA,
 
-    Array2D<unsigned int>& sampleCount)
+    Array2D<unsigned int>& sampleCount,
+    Compression            compression = Compression::ZIPS_COMPRESSION)
 
 {
     //
@@ -170,7 +171,7 @@ writeDeepScanlineFile (
     header.channels ().insert ("Z", Channel (FLOAT));
     header.channels ().insert ("A", Channel (HALF));
     header.setType (DEEPSCANLINE);
-    header.compression () = ZIPS_COMPRESSION;
+    header.compression () = compression;
 
     DeepScanLineOutputFile file (filename, header);
 
@@ -255,8 +256,32 @@ deepExamples ()
     testDataZ.resizeErase (h, w);
     drawImage2 (testDataA, testDataZ, w, h);
 
-    writeDeepScanlineFile (
-        "test.deep.exr", window, window, dataZ, dataA, sampleCount);
-    readDeepScanlineFile (
-        "test.deep.exr", window, window, dataZ, dataA, sampleCount);
+    {
+        writeDeepScanlineFile (
+            "test.deep.exr",
+            window,
+            window,
+            dataZ,
+            dataA,
+            sampleCount,
+            Compression::ZSTD_COMPRESSION);
+    }
+    {
+        writeDeepScanlineFile (
+            "test.zips.exr",
+            window,
+            window,
+            dataZ,
+            dataA,
+            sampleCount,
+            Compression::ZIPS_COMPRESSION);
+    }
+    {
+        readDeepScanlineFile (
+            "test.deep.exr", window, window, dataZ, dataA, sampleCount);
+    }
+    {
+        readDeepScanlineFile (
+            "test.zips.exr", window, window, dataZ, dataA, sampleCount);
+    }
 }

src/examples/deepTiledExamples.cpp

@@ -158,11 +158,12 @@ getSampleDataForTile (
 
 void
 writeDeepTiledFile (
-    const char filename[],
-    Box2i      displayWindow,
-    Box2i      dataWindow,
-    int        tileSizeX,
-    int        tileSizeY)
+    const char  filename[],
+    Box2i       displayWindow,
+    Box2i       dataWindow,
+    int         tileSizeX,
+    int         tileSizeY,
+    Compression compression = Compression::ZIPS_COMPRESSION)
 {
     //
     // Write a deep image with only a A (alpha) and a Z (depth) channel,
@@ -182,7 +183,7 @@ writeDeepTiledFile (
     header.channels ().insert ("Z", Channel (FLOAT));
     header.channels ().insert ("A", Channel (HALF));
     header.setType (DEEPTILE);
-    header.compression () = ZIPS_COMPRESSION;
+    header.compression () = compression;
 
     header.setTileDescription (
         TileDescription (tileSizeX, tileSizeY, ONE_LEVEL));
@@ -281,4 +282,13 @@ deepTiledExamples ()
         "testTiled.deep.exr", window, window, tileSizeX, tileSizeY);
     readDeepTiledFile (
         "testTiled.deep.exr", window, window, dataZ, dataA, sampleCount);
+    writeDeepTiledFile (
+        "testTiled.deep.zstd.exr",
+        window,
+        window,
+        tileSizeX,
+        tileSizeY,
+        Compression::ZSTD_COMPRESSION);
+    readDeepTiledFile (
+        "testTiled.deep.zstd.exr", window, window, dataZ, dataA, sampleCount);
 }

src/lib/OpenEXR/CMakeLists.txt

@@ -30,6 +30,7 @@ openexr_define_library(OpenEXR
     ImfTiledMisc.h
     ImfZip.h
     ImfZipCompressor.h
+    ImfZstdCompressor.h
     b44ExpLogTable.h
     dwaLookups.h
     ImfAcesFile.cpp
@@ -124,6 +125,7 @@ openexr_define_library(OpenEXR
     ImfWav.cpp
     ImfZip.cpp
     ImfZipCompressor.cpp
+    ImfZstdCompressor.cpp
   HEADERS
     ImfAcesFile.h
     ImfArray.h
@@ -217,6 +219,7 @@ openexr_define_library(OpenEXR
     ImfXdr.h
   DEPENDENCIES
     Imath::Imath
+    Blosc2::blosc2_static
     OpenEXR::Config
     OpenEXR::Iex
     OpenEXR::IlmThread

src/lib/OpenEXR/ImfCRgbaFile.h

@@ -80,7 +80,8 @@ typedef struct ImfRgba ImfRgba;
 #define IMF_B44A_COMPRESSION 7
 #define IMF_DWAA_COMPRESSION 8
 #define IMF_DWAB_COMPRESSION 9
-#define IMF_NUM_COMPRESSION_METHODS 10
+#define IMF_ZSTD_COMPRESSION 10
+#define IMF_NUM_COMPRESSION_METHODS 11
 
 /*
 ** Channels; values must be the same as in Imf::RgbaChannels.