ObjectReader/Writer support .cobz extension for compressed .cob files

SConstruct

@@ -495,6 +495,27 @@ o.Add(
 	"/usr/local/appleseed/lib",
 )
 
+# Blosc options
+
+o.Add(
+	"BLOSC_INCLUDE_PATH",
+	"The path to the blosc include directory.",
+	"/usr/local/include",
+)
+
+o.Add(
+	"BLOSC_LIB_PATH",
+	"The path to the blosc lib directory.",
+	"/usr/local/lib",
+)
+
+o.Add(
+	"BLOSC_LIB_SUFFIX",
+	"The suffix appended to the names of the blosc libraries. You can modify this "
+	"to link against libraries installed with non-default names",
+	"",
+)
+
 # Build options
 
 o.Add(
@@ -1051,7 +1072,8 @@ dependencyIncludes = [
 	"-isystem", "$PNG_INCLUDE_PATH",
 	"-isystem", "$JPEG_INCLUDE_PATH",
 	"-isystem", "$TIFF_INCLUDE_PATH",
-	"-isystem", "$FREETYPE_INCLUDE_PATH",		
+	"-isystem", "$FREETYPE_INCLUDE_PATH",
+	"-isystem", "$BLOSC_INCLUDE_PATH",
 ]
 
 env.Prepend(
@@ -1069,9 +1091,11 @@ env.Prepend(
 		"$JPEG_LIB_PATH",
 		"$TIFF_LIB_PATH",
 		"$FREETYPE_LIB_PATH",
+		"$BLOSC_LIB_PATH",
 	],
 	LIBS = [
 		"pthread",
+		"blosc" + env["BLOSC_LIB_SUFFIX"],
 	]
 )
 
@@ -1149,9 +1173,13 @@ if doConfigure :
 	if not c.CheckLibWithHeader( "tbb" + env["TBB_LIB_SUFFIX"], "tbb/tbb.h", "C++" ) :
 		sys.stderr.write( "ERROR : unable to find the TBB libraries - check TBB_INCLUDE_PATH and TBB_LIB_PATH.\n" )
 		Exit( 1 )	
-
+
+	if c.CheckLibWithHeader( env.subst( "blosc" + env["BLOSC_LIB_SUFFIX"] ), "blosc.h", "CXX" ) :
+
+		env.Append( CPPFLAGS = "-DIECORE_WITH_BLOSC" )
+
 	c.Finish()
-		
+
 env.Append( LIBS = [
 		"Half" + env["OPENEXR_LIB_SUFFIX"],
 		"Iex" + env["OPENEXR_LIB_SUFFIX"],

config/ie/options

@@ -83,6 +83,7 @@ if targetApp :
 	compilerVersion = targetAppReg["compilerVersion"]
 	openEXRVersion = targetAppReg.get( "OpenEXRVersion", cortexReg["OpenEXRVersion"] )
 	alembicVersion = targetAppReg.get( "AlembicVersion", cortexReg["AlembicVersion"] )
+	bloscVersion = targetAppReg.get( "bloscVersion", cortexReg["bloscVersion"] )
 	hdf5Version = targetAppReg.get( "hdf5Version", cortexReg["hdf5Version"] )
 	glewVersion = targetAppReg.get( "glewVersion", cortexReg["glewVersion"] )
 	pythonVersion = targetAppReg["pythonVersion"]
@@ -99,6 +100,7 @@ else :
 		pythonVersion = cortexReg["preferredPythonVersion"]
 	openEXRVersion = cortexReg["OpenEXRVersion"]
 	alembicVersion = cortexReg["AlembicVersion"]
+	bloscVersion = cortexReg["bloscVersion"]
 	hdf5Version = cortexReg["hdf5Version"]
 	glewVersion = cortexReg["glewVersion"]
 	tbbVersion = cortexReg["tbbVersion"]
@@ -372,6 +374,10 @@ ALEMBIC_LIB_SUFFIX = "-" + alembicVersion
 HDF5_INCLUDE_PATH = os.path.join( "/software/apps/hdf5", hdf5Version, platform, compiler, compilerVersion, "include" )
 HDF5_LIB_PATH = os.path.join( "/software/apps/hdf5", hdf5Version, platform, compiler, compilerVersion, "lib" )
 
+# find blosc:
+BLOSC_INCLUDE_PATH = os.path.join( "/software/tools/include/blosc/", bloscVersion )
+BLOSC_LIB_SUFFIX = "-" + bloscVersion
+
 # find doxygen
 DOXYGEN = os.path.join( "/software/apps/doxygen", os.environ["DOXYGEN_VERSION"], platform, "bin", "doxygen" )
 

include/IECore/IECore.h

@@ -59,6 +59,8 @@ IECORE_API bool withASIO();
 IECORE_API bool withSignals();
 /// Returns true if IECore was built with boost::math::factorial support
 IECORE_API bool withBoostFactorial();
+/// Returns true if IECore was built with blosc support
+IECORE_API bool withBlosc();
 /// Returns true if IECore was built with TIFF support
 IECORE_API bool withTIFF();
 /// Returns true if IECore was built with JPEG support

include/IECore/MemoryIndexedIO.h

@@ -66,6 +66,9 @@ class IECORE_API MemoryIndexedIO : public StreamIndexedIO
 		IndexedIO *duplicate(StreamIndexedIO::Node &rootNode) const;
 
 		class StreamFile;
+		class IStreamFile;
+		class OStreamFile;
+		class CharVectorDataDevice;
 };
 
 IE_CORE_DECLAREPTR( MemoryIndexedIO )

include/IECore/ObjectReader.h

@@ -1,6 +1,6 @@
 //////////////////////////////////////////////////////////////////////////
 //
-//  Copyright (c) 2007-2011, Image Engine Design Inc. All rights reserved.
+//  Copyright (c) 2007-2015, Image Engine Design Inc. All rights reserved.
 //
 //  Redistribution and use in source and binary forms, with or without
 //  modification, are permitted provided that the following conditions are

src/IECore/IECore.cpp

@@ -92,6 +92,15 @@ bool withBoostFactorial()
 #endif
 }
 
+bool withBlosc()
+{
+#ifdef IECORE_WITH_BLOSC
+	return true;
+#else
+	return false;
+#endif
+}
+
 bool withTIFF()
 {
 #ifdef IECORE_WITH_TIFF

src/IECore/MemoryIndexedIO.cpp

@@ -36,106 +36,195 @@
 #include "IECore/FileIndexedIO.h"
 #include "IECore/VectorTypedData.h"
 
+#include <iosfwd>
+#include <boost/iostreams/stream.hpp>
+#include <boost/iostreams/positioning.hpp>
+
 using namespace IECore;
 
 IE_CORE_DEFINERUNTIMETYPEDDESCRIPTION( MemoryIndexedIO )
 
-///////////////////////////////////////////////
+//////////////////////////////////////////////////////
 //
-// FileIndexedIO::StreamFile (begin)
+// FileIndexedIO::StreamFile subclass implementations
 //
-///////////////////////////////////////////////
+//////////////////////////////////////////////////////
 
-class MemoryIndexedIO::StreamFile : public StreamIndexedIO::StreamFile
+class MemoryIndexedIO::CharVectorDataDevice
 {
 	public:
-		StreamFile( const char *buf, size_t size, IndexedIO::OpenMode mode );
 
-		CharVectorDataPtr buffer();
+		typedef char char_type;
+		typedef boost::iostreams::seekable_device_tag  category;
+
+		CharVectorDataDevice( CharVectorData *container ) 
+			: m_container(container), m_pos(0)
+		{}
+
+		std::streamsize read(char* s, std::streamsize n)
+		{
+			std::streamsize amt = static_cast<std::streamsize>(m_container->readable().size() - m_pos);
+			std::streamsize result = std::min(n, amt);
+			if (result != 0)
+			{
+				std::copy( m_container->readable().begin() + m_pos, m_container->readable().begin() + m_pos + result, s );
+				m_pos += result;
+				return result;
+			}
+			else
+			{
+				return -1; // EOF
+			}
+		}
 
-		virtual ~StreamFile();
+		std::streamsize write(const char* s, std::streamsize n)
+		{
+			using namespace std;
+			streamsize result = 0;
+			if (m_pos != m_container->writable().size())
+			{
+				streamsize amt = static_cast<streamsize>(m_container->writable().size() - m_pos);
+				result = (min)(n, amt);
+				std::copy(s, s + result, m_container->writable().begin() + m_pos);
+				m_pos += result;
+			}
+			if (result < n)
+			{
+				m_container->writable().insert(m_container->writable().end(), s, s + n);
+				m_pos = m_container->writable().size();
+			}
+			return n;
+		}
 
-		void flush( size_t endPosition );
+		boost::iostreams::stream_offset seek(boost::iostreams::stream_offset off, std::ios_base::seekdir way )
+		{
+			using namespace std;
+			// Determine new value of m_pos
+			boost::iostreams::stream_offset next;
+			if (way == std::ios_base::beg)
+			{
+				next = off;
+			}
+			else if (way == std::ios_base::cur)
+			{
+				next = m_pos + off;
+			}
+			else if (way == std::ios_base::end)
+			{
+				next = m_container->readable().size() + off;
+			}
+			else
+			{
+				throw ios_base::failure("bad seek direction");
+			}
+
+			// Check for errors
+			if (next < 0 || (size_t)next > m_container->readable().size())
+			{
+				throw ios_base::failure("bad seek offset");
+			}
+
+			m_pos = next;
+			return m_pos;
+		}
 
 	private:
 
-		size_t m_endPosition;
+		CharVectorData* m_container;
+		size_t   m_pos;
 };
 
-MemoryIndexedIO::StreamFile::StreamFile( const char *buf, size_t size, IndexedIO::OpenMode mode ) : StreamIndexedIO::StreamFile(mode), m_endPosition(0)
+class MemoryIndexedIO::StreamFile : public StreamIndexedIO::StreamFile
 {
-	if (mode & IndexedIO::Write)
-	{
-		std::stringstream *f = new std::stringstream( std::ios::trunc | std::ios::binary | std::ios::in | std::ios::out );
-		setStream( f, true );
-	}
-	else if (mode & IndexedIO::Append)
-	{
-		if ( !buf || !size )
+	public:
+		StreamFile( IndexedIO::OpenMode mode ) : StreamIndexedIO::StreamFile(mode)
 		{
-			/// Create new file
-			std::stringstream *f = new std::stringstream(  std::ios::trunc | std::ios::binary | std::ios::in | std::ios::out );
-			setStream( f, true );
 		}
-		else
-		{
-			/// Read existing file
-			assert( buf );
 
-			/// Read existing file
-			std::stringstream *f = new std::stringstream( std::string(buf, size), std::ios::binary | std::ios::in | std::ios::out );
-			setStream( f, false );
+		virtual CharVectorDataPtr buffer() = 0;
+
+		virtual ~StreamFile()
+		{
 		}
-	}
-	else
-	{
-		assert( buf );
-		assert( mode & IndexedIO::Read );
-		std::stringstream *f = new std::stringstream( std::string(buf, size), std::ios::binary | std::ios::in | std::ios::out );
-		setStream( f, false );
-	}
-}
 
-void MemoryIndexedIO::StreamFile::flush( size_t endPosition )
-{
-	m_endPosition = endPosition;
-}
+};
 
-CharVectorDataPtr MemoryIndexedIO::StreamFile::buffer()
+class MemoryIndexedIO::IStreamFile : public MemoryIndexedIO::StreamFile
 {
-	std::stringstream *s = static_cast< std::stringstream *>( m_stream );
-	assert( s );
+	public:
+		IStreamFile( ConstCharVectorDataPtr buffer, IndexedIO::OpenMode mode ) : MemoryIndexedIO::StreamFile(mode), m_buffer( buffer )
+		{
+			setStream( new boost::iostreams::stream<CharVectorDataDevice>( const_cast<CharVectorData*>( buffer.get() ) ), buffer->readable().empty() );
+		}
+
+		virtual CharVectorDataPtr buffer()
+		{
+			return m_buffer->copy();
+		}
 
-	CharVectorData::ValueType d;
-	const std::string &str = s->str();
+		virtual ~IStreamFile()
+		{
+		}
 
-	d.assign( str.begin(), str.end() );
-	d.resize( m_endPosition );
-	assert( d.size() ==  m_endPosition );
-	return new CharVectorData( d );
-}
+	private:
+		ConstCharVectorDataPtr m_buffer;
+
+};
 
-MemoryIndexedIO::StreamFile::~StreamFile()
+class MemoryIndexedIO::OStreamFile : public MemoryIndexedIO::StreamFile
 {
-}
+	public:
+		OStreamFile( CharVectorDataPtr buffer, IndexedIO::OpenMode mode ) : MemoryIndexedIO::StreamFile(mode), m_buffer( buffer )
+		{
+			setStream( new boost::iostreams::stream<CharVectorDataDevice>( buffer.get() ), buffer->readable().empty() );
+		}
+
+		virtual CharVectorDataPtr buffer()
+		{
+			return m_buffer;
+		}
+
+		virtual ~OStreamFile()
+		{
+		}
+
+	private:
+		CharVectorDataPtr m_buffer;
+
+};
 
-///////////////////////////////////////////////
+//////////////////////////////////////////////////////
 //
-// MemoryIndexedIO::StreamFile (end)
+// FileIndexedIO::StreamFile subclasses end
 //
-///////////////////////////////////////////////
+//////////////////////////////////////////////////////
 
 
 MemoryIndexedIO::MemoryIndexedIO( ConstCharVectorDataPtr buf, const IndexedIO::EntryIDList &root, IndexedIO::OpenMode mode)
 {
-	const char *bufPtr = 0;
-	size_t size = 0;
-	if ( buf )
+	if (mode & IndexedIO::Write)
+	{
+		open( new OStreamFile( new CharVectorData, mode ), root );
+	}
+	else if (mode & IndexedIO::Append)
 	{
-		bufPtr = &(buf->readable()[0]);
-		size = buf->readable().size();
+		if ( !buf )
+		{
+			/// Create new file
+			open( new OStreamFile( new CharVectorData, mode ), root );
+		}
+		else
+		{
+			/// Read existing file
+			open( new OStreamFile( buf->copy(), mode ), root );
+		}
+	}
+	else
+	{
+		assert( buf );
+		assert( mode & IndexedIO::Read );
+		open( new IStreamFile( buf, mode ), root );
 	}
-	open( new StreamFile( bufPtr, size, mode ), root );
 }
 
 MemoryIndexedIO::MemoryIndexedIO( StreamIndexedIO::Node &rootNode ) : StreamIndexedIO( rootNode )