ObjectReader/Writer now support .cobz extension for compressed .cob files

Author: davidsminor

include/IECore/ObjectReader.h

@@ -1,6 +1,6 @@
 //////////////////////////////////////////////////////////////////////////
 //
-//  Copyright (c) 2007-2011, Image Engine Design Inc. All rights reserved.
+//  Copyright (c) 2007-2015, Image Engine Design Inc. All rights reserved.
 //
 //  Redistribution and use in source and binary forms, with or without
 //  modification, are permitted provided that the following conditions are
@@ -74,6 +74,7 @@ class IECORE_API ObjectReader : public Reader
 	private :
 
 		static const ReaderDescription<ObjectReader> g_readerDescription;
+		static const ReaderDescription<ObjectReader> g_compressedReaderDescription;
 };
 
 IE_CORE_DECLAREPTR( ObjectReader );

include/IECore/ObjectWriter.h

@@ -70,6 +70,7 @@ class IECORE_API ObjectWriter : public Writer
 		void constructParameters();
 
 		static const WriterDescription<ObjectWriter> g_writerDescription;
+		static const WriterDescription<ObjectWriter> g_compressedWriterDescription;
 
 };
 

src/IECore/ObjectReader.cpp

@@ -1,6 +1,6 @@
 //////////////////////////////////////////////////////////////////////////
 //
-//  Copyright (c) 2007-2010, Image Engine Design Inc. All rights reserved.
+//  Copyright (c) 2007-2015, Image Engine Design Inc. All rights reserved.
 //
 //  Redistribution and use in source and binary forms, with or without
 //  modification, are permitted provided that the following conditions are
@@ -32,11 +32,16 @@
 //
 //////////////////////////////////////////////////////////////////////////
 
+#include "IECore/ObjectWriter.h"
 #include "IECore/ObjectReader.h"
 #include "IECore/FileIndexedIO.h"
+#include "IECore/MemoryIndexedIO.h"
 #include "IECore/FileNameParameter.h"
 #include "IECore/CompoundData.h"
 
+#include "boost/filesystem/convenience.hpp"
+#include "boost/iostreams/filter/gzip.hpp"
+
 #include <cassert>
 
 using namespace IECore;
@@ -45,14 +50,15 @@ using namespace boost;
 IE_CORE_DEFINERUNTIMETYPED( ObjectReader );
 
 const Reader::ReaderDescription<ObjectReader> ObjectReader::g_readerDescription( "cob" );
+const Reader::ReaderDescription<ObjectReader> ObjectReader::g_compressedReaderDescription( "cobz" );
 
 ObjectReader::ObjectReader() :
-		Reader( "Reads instances of a single Object from a file with a .cob extension" )
+		Reader( "Reads instances of a single Object from a file with a .cob or .cobz extension" )
 {
 }
 
 ObjectReader::ObjectReader( const std::string &fileName ) :
-		Reader( "Reads instances of a single Object from a file with a .cob extension" )
+		Reader( "Reads instances of a single Object from a file with a .cob or .cobz extension" )
 {
 	m_fileNameParameter->setTypedValue( fileName );
 }
@@ -91,7 +97,48 @@ bool ObjectReader::canRead( const std::string &fileName )
 ObjectPtr ObjectReader::doOperation( const CompoundObject * operands )
 {
 	IndexedIOPtr io = open(fileName());
-	return Object::load( io, "object" );
+
+	// is this file compressed?
+	if( boost::filesystem::extension( boost::filesystem::path( fileName() ) ) == ".cobz" )
+	{
+		IndexedIOPtr objectCompressed = io->subdirectory( "objectCompressed" );
+
+		// read size of compressed data:
+		size_t compressedSize;
+		objectCompressed->read( "size", compressedSize );
+
+		// read the actual data
+		std::vector<char> compressedData( compressedSize );
+		char* cd = compressedData.data();
+		objectCompressed->read( InternedString( "data" ), cd, compressedSize * sizeof(char) );
+
+		// decompress into a buffer for a MemoryIndexedIO:
+		CharVectorDataPtr memBufferData = new CharVectorData;
+
+		// read compression type:
+		std::string compressionType;
+		objectCompressed->read( "compressionType", compressionType );
+
+		if( compressionType == "gzip" )
+		{
+			boost::iostreams::filtering_ostream decompressingStream;
+			decompressingStream.push( boost::iostreams::gzip_decompressor() );
+			decompressingStream.push( boost::iostreams::back_inserter( memBufferData->writable() ) );
+			decompressingStream.write( compressedData.data(), compressedData.size() * sizeof(char) );
+		}
+		else
+		{
+			throw IECore::Exception( "ObjectReader::doOperation: unrecognized compression type " + compressionType );
+		}
+
+		MemoryIndexedIOPtr mio = new MemoryIndexedIO( memBufferData, IndexedIO::rootPath, IndexedIO::Read );
+
+		return Object::load( mio, "object" );
+	}
+	else
+	{
+		return Object::load( io, "object" );
+	}
 }
 
 CompoundObjectPtr ObjectReader::readHeader()

src/IECore/ObjectWriter.cpp

@@ -1,6 +1,6 @@
 //////////////////////////////////////////////////////////////////////////
 //
-//  Copyright (c) 2007-2010, Image Engine Design Inc. All rights reserved.
+//  Copyright (c) 2007-2015, Image Engine Design Inc. All rights reserved.
 //
 //  Redistribution and use in source and binary forms, with or without
 //  modification, are permitted provided that the following conditions are
@@ -34,9 +34,14 @@
 
 #include <cassert>
 
+#include "boost/filesystem/convenience.hpp"
+#include "boost/iostreams/filter/gzip.hpp"
+
 #include "IECore/ObjectWriter.h"
 #include "IECore/FileIndexedIO.h"
+#include "IECore/MemoryIndexedIO.h"
 #include "IECore/FileNameParameter.h"
+#include "IECore/NumericParameter.h"
 #include "IECore/CompoundParameter.h"
 #include "IECore/ObjectParameter.h"
 #include "IECore/CompoundData.h"
@@ -51,15 +56,16 @@ using namespace boost;
 IE_CORE_DEFINERUNTIMETYPED( ObjectWriter )
 
 const Writer::WriterDescription<ObjectWriter> ObjectWriter::g_writerDescription( "cob" );
+const Writer::WriterDescription<ObjectWriter> ObjectWriter::g_compressedWriterDescription( "cobz" );
 
 ObjectWriter::ObjectWriter()
-	:	Writer( "Writes instances of a single Object to a file with a .cob extension", ObjectTypeId )
+	:	Writer( "Writes instances of a single Object to a file with a .cob or .cobz extension", ObjectTypeId )
 {
 	constructParameters();
 }
 
 ObjectWriter::ObjectWriter( ObjectPtr object, const std::string &fileName )
-	:	Writer( "Writes instances of a single Object to a file with a .cob extension", ObjectTypeId )
+	:	Writer( "Writes instances of a single Object to a file with a .cob or .cobz extension", ObjectTypeId )
 {
 	constructParameters();
 	m_objectParameter->setValue( object );
@@ -95,8 +101,32 @@ void ObjectWriter::doWrite( const CompoundObject *operands )
 
 	((ObjectPtr)header)->save( io, "header" );
 
-	// write the object
-	object()->save( io, "object" );
+	if( boost::filesystem::extension( boost::filesystem::path( fileName() ) ) == ".cobz" )
+	{
+		// write the object to a memory buffer:
+		MemoryIndexedIOPtr bodyIO = new MemoryIndexedIO( ConstCharVectorDataPtr(), IndexedIO::rootPath, IndexedIO::Exclusive | IndexedIO::Write );
+		object()->save( bodyIO, "object" );
+
+		// compress:
+		std::vector<char> compressedData;
+		boost::iostreams::filtering_ostream compressingStream;
+		compressingStream.push( boost::iostreams::gzip_compressor() );
+		compressingStream.push(boost::iostreams::back_inserter(compressedData));
+		compressingStream.write( bodyIO->buffer()->readable().data(), bodyIO->buffer()->readable().size() * sizeof(char) );
+		boost::iostreams::close( compressingStream );
+
+		IndexedIOPtr objectCompressed = io->createSubdirectory( "objectCompressed" );
+		objectCompressed->write( "size", compressedData.size() );
+		objectCompressed->write( "data", compressedData.data(), compressedData.size() * sizeof(char) );
+
+		// write the compression type in case we want to change it later on and retain backward compatitility:
+		objectCompressed->write( "compressionType", "gzip" );
+	}
+	else
+	{
+		// no compression, just write the object:
+		object()->save( io, "object" );
+	}
 }
 
 void ObjectWriter::constructParameters()

src/IECoreHoudini/GEO_CobIOTranslator.cpp

@@ -67,7 +67,7 @@ int GEO_CobIOTranslator::checkExtension( const char *fileName )
 	UT_String sname( fileName );
 
 	/// \todo: support all extensions that can read/write any object supported by the To/FromHoudiniGeometryConverters
-	if ( sname.fileExtension() && ( !strcmp( sname.fileExtension(), ".cob" ) || !strcmp( sname.fileExtension(), ".pdc" ) || !strcmp( sname.fileExtension(), ".ptc" ) ) )
+	if ( sname.fileExtension() && ( !strcmp( sname.fileExtension(), ".cob" ) || !strcmp( sname.fileExtension(), ".cobz" ) || !strcmp( sname.fileExtension(), ".pdc" ) || !strcmp( sname.fileExtension(), ".ptc" ) ) )
 	{
 		return true;
 	}

src/IECoreHoudini/plugin/Plugin.cpp

@@ -214,6 +214,10 @@ void newGeometryIO( void * )
 	{
 		geoextension->addExtension( "cob" );
 	}
+	if ( !geoextension->findExtension( "cobz" ) )
+	{
+		geoextension->addExtension( "cobz" );
+	}
 	if ( !geoextension->findExtension( "pdc" ) )
 	{
 		geoextension->addExtension( "pdc" );

test/IECore/ObjectWriter.py

@@ -69,10 +69,43 @@ def testHeader( self ) :
 		self.assertEqual( h["host"]["nodeName"].value, socket.gethostname() )
 		self.assertEqual( h["ieCoreVersion"].value, IECore.versionString() )
 		self.assertEqual( h["typeName"].value, "IntData" )
-	
+
+	def testCompressed( self ) :
+		
+		floats = IECore.FloatVectorData( range(0,10000) )
+		w = IECore.Writer.create( floats, "test/compressed.cobz" )
+
+		w.write()
+
+		# check FileIndexedIO contents:
+		fio = IECore.FileIndexedIO( "test/compressed.cobz", IECore.IndexedIO.OpenMode.Read )
+		fio.subdirectory("header")
+		fio.subdirectory("objectCompressed")
+
+		# read back and check object:
+		self.assertEqual( IECore.Reader.create( "test/compressed.cobz" ).read(), floats )
+
+	def testCompressedHeader( self ) :
+		
+		o = IECore.IntData()
+
+		w = IECore.Writer.create( o, "test/compressed.cobz" )
+		w["header"].getValue()["testHeaderData"] = IECore.StringData( "i am part of a header" )
+		w["header"].getValue()["testHeaderData2"] = IECore.IntData( 100 )
+		w.write()
+
+		h = IECore.Reader.create( "test/compressed.cobz" ).readHeader()
+
+		for k in w["header"].getValue().keys() :
+			self.assertEqual( w["header"].getValue()[k], h[k] )
+
+		self.assertEqual( h["host"]["nodeName"].value, socket.gethostname() )
+		self.assertEqual( h["ieCoreVersion"].value, IECore.versionString() )
+		self.assertEqual( h["typeName"].value, "IntData" )
+
 	def tearDown( self ) :
 
-		for f in ( "test/compoundData.cob", "test/intData.cob" ) :
+		for f in ( "test/compoundData.cob", "test/intData.cob", "test/compressed.cobz" ) :
 			if os.path.isfile( f ) :
 				os.remove( f )