Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Zstd compressor (#3) #1604

Open
wants to merge 38 commits into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
38 commits
Select commit Hold shift + click to select a range
0689612
Zstd compressor (#3)
clusty Dec 21, 2023
574988e
Fix single blob
clusty Jan 8, 2024
0e90ff1
Add license headers. Clang-format example file.
pleprince Jan 10, 2024
057dd80
Build Blosc as a part of the regular build (#4)
clusty Jan 15, 2024
78ca6c6
Implement the compressor inside OpenEXRCore (#5)
clusty Jan 22, 2024
41afe2e
Fix single blob
clusty Jan 8, 2024
f52266d
Add license headers. Clang-format example file.
pleprince Jan 10, 2024
d0a24d4
Build Blosc as a part of the regular build (#4)
clusty Jan 15, 2024
1e6698e
Implement the compressor inside OpenEXRCore (#5)
clusty Jan 22, 2024
1d2cd0c
Merge branch 'main' of https://github.com/clusty/openexr into main
pleprince Feb 20, 2024
48fa372
Merge remote-tracking branch 'aswf/main' into main
pleprince Mar 13, 2024
ccac098
Adopt new compression API
pleprince Mar 13, 2024
2341fb4
Fix API tests
pleprince Mar 13, 2024
6cfc3cc
Fix cmake build
pleprince Mar 13, 2024
21c60db
Merge latest from aswf main
pleprince Apr 2, 2024
88c0348
fix: debug build was not linking against debug blosc lib.
pleprince Apr 2, 2024
05d9457
fix: debug build was not linking against debug blosc lib, for older c…
pleprince Apr 2, 2024
9799bf9
fix: yet another variation on the same theme
pleprince Apr 2, 2024
a06daec
fix: debug and static builds should work.
pleprince Apr 3, 2024
2d678ae
fix: update bazel build
pleprince Apr 3, 2024
2ad98fe
fix: potential bazel typo
pleprince Apr 3, 2024
f2bdfde
fix: refactored linking and fixed validation
pleprince Apr 4, 2024
a887716
fix: update bazel blosc version
pleprince Apr 4, 2024
688d0d6
fix: tentative: always enable threads to build examples as it is need…
pleprince Apr 5, 2024
387d2ca
fix: Replacing "dl" with CMAKE_DL_LIBS to fix windows build
pleprince Apr 5, 2024
06f30a8
fix: update linking for CI fuzzing
pleprince Apr 5, 2024
903c2c4
fix(bazel): add missing source files to BUILD.bazel
pleprince Apr 5, 2024
40b1937
fix(bazel): tentative: add blosc3 deps to example build
pleprince Apr 5, 2024
bb8754a
fix(bazel): tentative: add linkopts to example build
pleprince Apr 5, 2024
15844b5
fix(bazel): tentative: add linkopts+deps to example build
pleprince Apr 5, 2024
a9ebf07
fix(bazel): everything but the examples build. Can't make it work loc…
pleprince Apr 5, 2024
55f37b0
fix(fuzz): tentative: use target name to link
pleprince Apr 5, 2024
4c714a2
build(bazel): switch c-blosc2 tp 2.12.0.bcr.2
pleprince Apr 23, 2024
81e1701
build(fuzz): build fuzz with dwarf-4. (test)
pleprince Apr 23, 2024
5b0be1d
build(fuzz): build fuzz with dwarf-4. (test #2)
pleprince Apr 23, 2024
cb6a1e0
build(fuzz): build fuzz with dwarf-4. (test #3)
pleprince Apr 23, 2024
17a3fdf
Merge branch 'main' of https://github.com/AcademySoftwareFoundation/o…
pleprince Jun 5, 2024
5047f7f
ci: temporarily disable fuzz test
pleprince Jun 19, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -57,3 +57,7 @@ docs/_test_images/

# Ignore Bazel generated files
bazel-*

cmake-build-debug/

.idea/
2 changes: 2 additions & 0 deletions src/bin/exrheader/main.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,8 @@ printCompression (Compression c)

case DWAB_COMPRESSION: cout << "dwa, medium scanline blocks"; break;

case ZSTD_COMPRESSION: cout << "zstd"; break;

default: cout << int (c); break;
}
}
Expand Down
109 changes: 70 additions & 39 deletions src/examples/deepExamples.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -47,10 +47,10 @@ readDeepScanlineFile (
// - allocate memory for the pixels
// - describe the layout of the A, and Z pixel buffers
// - read the sample counts from the file
// - allocate the memory requred to store the samples
// - allocate the memory required to store the samples
// - read the pixels from the file
//
cary-ilm marked this conversation as resolved.
Show resolved Hide resolved

DeepScanLineInputFile file (filename);

const Header& header = file.header ();
Expand All @@ -69,15 +69,17 @@ readDeepScanlineFile (

frameBuffer.insertSampleCountSlice (Slice (
UINT,
(char*) (&sampleCount[0][0] - dataWindow.min.x - dataWindow.min.y * width),
(char*) (&sampleCount[0][0] - dataWindow.min.x -
dataWindow.min.y * width),
sizeof (unsigned int) * 1, // xStride
sizeof (unsigned int) * width)); // yStride

frameBuffer.insert (
"dataZ",
DeepSlice (
FLOAT,
(char*) (&dataZ[0][0] - dataWindow.min.x - dataWindow.min.y * width),
(char*) (&dataZ[0][0] - dataWindow.min.x -
dataWindow.min.y * width),

sizeof (float*) * 1, // xStride for pointer array
sizeof (float*) * width, // yStride for pointer array
Expand All @@ -87,7 +89,8 @@ readDeepScanlineFile (
"dataA",
DeepSlice (
HALF,
(char*) (&dataA[0][0] - dataWindow.min.x - dataWindow.min.y * width),
(char*) (&dataA[0][0] - dataWindow.min.x -
dataWindow.min.y * width),
sizeof (half*) * 1, // xStride for pointer array
sizeof (half*) * width, // yStride for pointer array
sizeof (half) * 1)); // stride for O data sample
Expand Down Expand Up @@ -118,20 +121,18 @@ readDeepScanlineFile (
}
}

unsigned int getPixelSampleCount (int i, int j)
cary-ilm marked this conversation as resolved.
Show resolved Hide resolved
unsigned int
getPixelSampleCount (int i, int j)
{
// Dummy code creating deep data from a flat image
return 1;
}

Array2D<float> testDataZ;
Array2D<half> testDataA;
cary-ilm marked this conversation as resolved.
Show resolved Hide resolved
Array2D<half> testDataA;

void getPixelSampleData(
int i,
int j,
Array2D<float*>& dataZ,
Array2D<half*>& dataA)
cary-ilm marked this conversation as resolved.
Show resolved Hide resolved
void
getPixelSampleData (int i, int j, Array2D<float*>& dataZ, Array2D<half*>& dataA)
{
// Dummy code creating deep data from a flat image
dataZ[i][j][0] = testDataZ[i][j];
Expand All @@ -147,7 +148,8 @@ writeDeepScanlineFile (

Array2D<half*>& dataA,

Array2D<unsigned int>& sampleCount)
Array2D<unsigned int>& sampleCount,
Compression compression = Compression::ZIPS_COMPRESSION)

{
//
Expand All @@ -160,7 +162,7 @@ writeDeepScanlineFile (
// - describe the memory layout of the A and Z pixels
// - store the pixels in the file
//

int height = dataWindow.max.y - dataWindow.min.y + 1;
int width = dataWindow.max.x - dataWindow.min.x + 1;

Expand All @@ -169,15 +171,16 @@ writeDeepScanlineFile (
header.channels ().insert ("Z", Channel (FLOAT));
header.channels ().insert ("A", Channel (HALF));
header.setType (DEEPSCANLINE);
header.compression () = ZIPS_COMPRESSION;
header.compression () = compression;

DeepScanLineOutputFile file (filename, header);

DeepFrameBuffer frameBuffer;

frameBuffer.insertSampleCountSlice (Slice (
UINT,
(char*) (&sampleCount[0][0] - dataWindow.min.x - dataWindow.min.y * width),
(char*) (&sampleCount[0][0] - dataWindow.min.x -
dataWindow.min.y * width),
sizeof (unsigned int) * 1, // xS

sizeof (unsigned int) * width)); // yStride
Expand All @@ -186,7 +189,8 @@ writeDeepScanlineFile (
"Z",
DeepSlice (
FLOAT,
(char*) (&dataZ[0][0] - dataWindow.min.x - dataWindow.min.y * width),
(char*) (&dataZ[0][0] - dataWindow.min.x -
dataWindow.min.y * width),
sizeof (float*) * 1, // xStride for pointer

sizeof (float*) * width, // yStride for pointer array
Expand All @@ -196,7 +200,8 @@ writeDeepScanlineFile (
"A",
DeepSlice (
HALF,
(char*) (&dataA[0][0] - dataWindow.min.x - dataWindow.min.y * width),
(char*) (&dataA[0][0] - dataWindow.min.x -
dataWindow.min.y * width),
sizeof (half*) * 1, // xStride for pointer array
sizeof (half*) * width, // yStride for pointer array
sizeof (half) * 1)); // stride for A data sample
Expand All @@ -211,7 +216,7 @@ writeDeepScanlineFile (
dataZ[i][j] = new float[sampleCount[i][j]];
dataA[i][j] = new half[sampleCount[i][j]];
// Generate data for dataZ and dataA.
getPixelSampleData(i, j, dataZ, dataA);
getPixelSampleData (i, j, dataZ, dataA);
}

file.writePixels (1);
Expand All @@ -227,30 +232,56 @@ writeDeepScanlineFile (
}
}


cary-ilm marked this conversation as resolved.
Show resolved Hide resolved
void deepExamples()
void
deepExamples ()
{
int w = 800;
int h = 600;

Box2i window;
window.min.setValue(0, 0);
window.max.setValue(w - 1, h - 1);
Array2D<float *> dataZ;
dataZ.resizeErase(h, w);
Array2D<half *> dataA;
dataA.resizeErase(h, w);
window.min.setValue (0, 0);
window.max.setValue (w - 1, h - 1);

Array2D<float*> dataZ;
dataZ.resizeErase (h, w);

Array2D<half*> dataA;
dataA.resizeErase (h, w);

Array2D<unsigned int> sampleCount;
sampleCount.resizeErase(h, w);
sampleCount.resizeErase (h, w);

// Create an image to be used as a source for deep data
testDataA.resizeErase(h, w);
testDataZ.resizeErase(h, w);
drawImage2(testDataA, testDataZ, w, h);

writeDeepScanlineFile("test.deep.exr", window, window, dataZ, dataA, sampleCount);
readDeepScanlineFile ("test.deep.exr", window, window, dataZ, dataA, sampleCount);
testDataA.resizeErase (h, w);
testDataZ.resizeErase (h, w);
drawImage2 (testDataA, testDataZ, w, h);

{
writeDeepScanlineFile (
"test.deep.exr",
window,
window,
dataZ,
dataA,
sampleCount,
Compression::ZSTD_COMPRESSION);
}
{
writeDeepScanlineFile (
"test.zips.exr",
window,
window,
dataZ,
dataA,
sampleCount,
Compression::ZIPS_COMPRESSION);
}
{
readDeepScanlineFile (
"test.deep.exr", window, window, dataZ, dataA, sampleCount);
}
{
readDeepScanlineFile (
"test.zips.exr", window, window, dataZ, dataA, sampleCount);
}
}
Loading
Loading