WIP: ZIP Streaming

src/ASiC_E.cpp

@@ -67,7 +67,7 @@ ASiC_E::ASiC_E(const string &path)
     : ASiContainer(MIMETYPE_ASIC_E)
     , d(make_unique<Private>())
 {
-    auto zip = load(path, true, {MIMETYPE_ASIC_E, MIMETYPE_ADOC});
+    auto *zip = load(path, true, {MIMETYPE_ASIC_E, MIMETYPE_ADOC});
     parseManifestAndLoadFiles(*zip);
 }
 
@@ -229,12 +229,11 @@ void ASiC_E::parseManifestAndLoadFiles(const ZipSerialize &z)
 
     try
     {
-        stringstream manifestdata;
-        z.extract("META-INF/manifest.xml", manifestdata);
+        unique_ptr<istream> manifestdata = z.stream("META-INF/manifest.xml");
         xml_schema::Properties p;
         p.schema_location(ASiC_E::MANIFEST_NAMESPACE,
             File::fullPathUrl(Conf::instance()->xsdPath() + "/OpenDocument_manifest.xsd"));
-        unique_ptr<xercesc::DOMDocument> doc = SecureDOMParser(p.schema_location(), true).parseIStream(manifestdata);
+        unique_ptr<xercesc::DOMDocument> doc = SecureDOMParser(p.schema_location(), true).parseIStream(*manifestdata);
         unique_ptr<manifest::Manifest> manifest = manifest::manifest(*doc, {}, p);
 
         set<string> manifestFiles;
@@ -267,9 +266,9 @@ void ASiC_E::parseManifestAndLoadFiles(const ZipSerialize &z)
             if(mediaType() == MIMETYPE_ADOC &&
                (file.full_path().compare(0, 9, "META-INF/") == 0 ||
                 file.full_path().compare(0, 9, "metadata/") == 0))
-                d->metadata.push_back(new DataFilePrivate(dataStream(file.full_path(), z), file.full_path(), file.media_type()));
+                d->metadata.push_back(dataFile(file.full_path(), file.media_type()));
             else
-                addDataFilePrivate(dataStream(file.full_path(), z), file.full_path(), file.media_type());
+                addDataFilePrivate(file.full_path(), file.media_type());
         }
         if(!mimeFound)
             THROW("Manifest is missing mediatype file entry.");
@@ -288,9 +287,7 @@ void ASiC_E::parseManifestAndLoadFiles(const ZipSerialize &z)
                     THROW("Multiple signature files with same name found '%s'", file.c_str());
                 try
                 {
-                    stringstream data;
-                    z.extract(file, data);
-                    auto signatures = make_shared<Signatures>(data, this);
+                    auto signatures = make_shared<Signatures>(*z.stream(file), this);
                     for(size_t i = 0, count = signatures->count(); i < count; ++i)
                         addSignature(make_unique<SignatureXAdES_LTA>(signatures, i, this));
                 }

src/ASiC_S.cpp

@@ -43,7 +43,7 @@ ASiC_S::ASiC_S(): ASiContainer(MIMETYPE_ASIC_S)
  */
 ASiC_S::ASiC_S(const string &path): ASiContainer(MIMETYPE_ASIC_S)
 {
-    auto z = load(path, false, {mediaType()});
+    auto *z = load(path, false, {mediaType()});
     static const string_view metaInf = "META-INF/";
 
     for(const string &file: z->list())
@@ -55,17 +55,13 @@ ASiC_S::ASiC_S(const string &path): ASiContainer(MIMETYPE_ASIC_S)
             {
                 if(!signatures().empty())
                     THROW("Can not add signature to ASiC-S container which already contains a signature.");
-                stringstream data;
-                z->extract(file, data);
-                addSignature(make_unique<SignatureTST>(data, this));
+                addSignature(make_unique<SignatureTST>(*z->stream(file), this));
             }
             if(file == "META-INF/signatures.xml")
             {
                 if(!signatures().empty())
                     THROW("Can not add signature to ASiC-S container which already contains a signature.");
-                stringstream data;
-                z->extract(file, data);
-                auto signatures = make_shared<Signatures>(data, this);
+                auto signatures = make_shared<Signatures>(*z->stream(file), this);
                 for(size_t i = 0, count = signatures->count(); i < count; ++i)
                     addSignature(make_unique<SignatureXAdES_LTA>(signatures, i, this));
             }
@@ -77,7 +73,7 @@ ASiC_S::ASiC_S(const string &path): ASiContainer(MIMETYPE_ASIC_S)
         {
             if(!dataFiles().empty())
                 THROW("Can not add document to ASiC-S container which already contains a document.");
-            addDataFile(dataStream(file, *z), file, "application/octet-stream");
+            addDataFilePrivate(file, "application/octet-stream");
         }
     }
 

src/ASiContainer.cpp

@@ -46,6 +46,7 @@ class ASiContainer::Private
     vector<DataFile*> documents;
     vector<Signature*> signatures;
     map<string, ZipSerialize::Properties> properties;
+    unique_ptr<ZipSerialize> z;
 };
 
 const string_view ASiContainer::ASICE_EXTENSION = "asice";
@@ -76,12 +77,12 @@ ASiContainer::ASiContainer(const string &mimetype)
  * @param supported supported mimetypes.
  * @return returns zip serializer for the container.
  */
-unique_ptr<ZipSerialize> ASiContainer::load(const string &path, bool mimetypeRequired, const set<string> &supported)
+ZipSerialize* ASiContainer::load(const string &path, bool mimetypeRequired, const set<string> &supported)
 {
     DEBUG("ASiContainer::ASiContainer(path = '%s')", path.c_str());
-    auto z = make_unique<ZipSerialize>(d->path = path, false);
+    d->z = make_unique<ZipSerialize>(d->path = path, false);
 
-    vector<string> list = z->list();
+    vector<string> list = d->z->list();
     if(list.empty())
         THROW("Failed to parse container");
 
@@ -91,16 +92,16 @@ unique_ptr<ZipSerialize> ASiContainer::load(const string &path, bool mimetypeReq
     // ETSI TS 102 918: mimetype has to be the first in the archive
     if(list.front() == "mimetype")
     {
-        d->mimetype = readMimetype(*z);
+        d->mimetype = readMimetype(*d->z);
         DEBUG("mimetype = '%s'", d->mimetype.c_str());
         if(supported.find(d->mimetype) == supported.cend())
             THROW("Incorrect mimetype '%s'", d->mimetype.c_str());
     }
 
     for(const string &file: list)
-        d->properties[file] = z->properties(file);
+        d->properties[file] = d->z->properties(file);
 
-    return z;
+    return d->z.get();
 }
 
 string ASiContainer::mediaType() const
@@ -138,25 +139,9 @@ vector<Signature *> ASiContainer::signatures() const
     return d->signatures;
 }
 
-/**
- * <p>
- * Read a datafile from container.
- * </p>
- * If expected size of the data is too big, then stream is written to temp file.
- *
- * @param path name of the file in zip container stream is used to read from.
- * @param z Zip container.
- * @return returns data as a stream.
- */
-unique_ptr<iostream> ASiContainer::dataStream(const string &path, const ZipSerialize &z) const
+DataFilePrivate* ASiContainer::dataFile(const std::string &path, const std::string &mediaType) const
 {
-    unique_ptr<iostream> data;
-    if(d->properties[path].size > MAX_MEM_FILE)
-        data = make_unique<fstream>(File::tempFileName(), fstream::in|fstream::out|fstream::binary|fstream::trunc);
-    else
-        data = make_unique<stringstream>();
-    z.extract(path, *data);
-    return data;
+    return new DataFilePrivate(d->z->stream(path), path, mediaType, d->properties[path].size);
 }
 
 /**
@@ -191,15 +176,16 @@ void ASiContainer::addDataFile(const string &path, const string &mediaType)
             *data << file.rdbuf();
         is = std::move(data);
     }
-    addDataFilePrivate(std::move(is), fileName, mediaType);
+    d->documents.push_back(new DataFilePrivate(std::move(is), fileName, mediaType, prop.size));
 }
 
 void ASiContainer::addDataFile(unique_ptr<istream> is, const string &fileName, const string &mediaType)
 {
     addDataFileChecks(fileName, mediaType);
     if(fileName.find_last_of("/\\") != string::npos)
         THROW("Document file '%s' cannot contain directory path.", fileName.c_str());
-    addDataFilePrivate(std::move(is), fileName, mediaType);
+    istream::pos_type pos = is->tellg();
+    d->documents.push_back(new DataFilePrivate(std::move(is), fileName, mediaType, pos < 0 ? 0 : (unsigned long)pos));
 }
 
 void ASiContainer::addDataFileChecks(const string &fileName, const string &mediaType)
@@ -214,9 +200,9 @@ void ASiContainer::addDataFileChecks(const string &fileName, const string &media
         THROW("MediaType does not meet format requirements (RFC2045, section 5.1) '%s'.", mediaType.c_str());
 }
 
-void ASiContainer::addDataFilePrivate(unique_ptr<istream> is, const string &fileName, const string &mediaType)
+void ASiContainer::addDataFilePrivate(const string &fileName, const string &mediaType)
 {
-    d->documents.push_back(new DataFilePrivate(std::move(is), fileName, mediaType));
+    d->documents.push_back(dataFile(fileName, mediaType));
 }
 
 /**
@@ -298,17 +284,16 @@ void ASiContainer::zproperty(const string &file, ZipSerialize::Properties &&prop
 string ASiContainer::readMimetype(const ZipSerialize &z)
 {
     DEBUG("ASiContainer::readMimetype()");
-    stringstream is;
-    z.extract("mimetype", is);
+    auto is = z.stream("mimetype");
     string text;
-    is >> text;
+    *is >> text;
     if(!is)
         THROW("Failed to read mimetype.");
     // Contains UTF-16 BOM
-    if(text.find("\xFF\xEF") == 0 || text.find("\xEF\xFF") == 0)
+    if(text.rfind("\xFF\xEF", 0) == 0 || text.rfind("\xEF\xFF", 0) == 0)
         THROW("Mimetype file must be UTF-8 format.");
     // contains UTF-8 BOM, remove
-    if(text.find("\xEF\xBB\xBF") == 0)
+    if(text.rfind("\xEF\xBB\xBF", 0) == 0)
         text.erase(text.cbegin(), text.cbegin() + 3);
     return text;
 }

src/ASiContainer.h

@@ -26,6 +26,7 @@
 
 namespace digidoc
 {
+    class DataFilePrivate;
     /**
      * Base class for the ASiC (Associated Signature Container) documents.
      * Implements the operations and data structures common for more specific ASiC 
@@ -63,10 +64,10 @@ namespace digidoc
       protected:
           ASiContainer(const std::string &mimetype);
 
-          void addDataFilePrivate(std::unique_ptr<std::istream> is, const std::string &fileName, const std::string &mediaType);
+          void addDataFilePrivate(const std::string &fileName, const std::string &mediaType);
           Signature* addSignature(std::unique_ptr<Signature> &&signature);
-          std::unique_ptr<std::iostream> dataStream(const std::string &path, const ZipSerialize &z) const;
-          std::unique_ptr<ZipSerialize> load(const std::string &path, bool requireMimetype, const std::set<std::string> &supported);
+          DataFilePrivate *dataFile(const std::string &path, const std::string &mediaType) const;
+          ZipSerialize* load(const std::string &path, bool requireMimetype, const std::set<std::string> &supported);
           void deleteSignature(Signature* s);
 
           void zpath(const std::string &file);

src/DataFile.cpp

@@ -87,16 +87,13 @@ DataFile::DataFile() = default;
 DataFile::~DataFile() = default;
 
 
-DataFilePrivate::DataFilePrivate(unique_ptr<istream> &&is, string filename, string mediatype, string id)
+DataFilePrivate::DataFilePrivate(unique_ptr<istream> &&is, string filename, string mediatype, unsigned long size, string id)
     : m_is(std::move(is))
     , m_id(std::move(id))
     , m_filename(std::move(filename))
     , m_mediatype(std::move(mediatype))
-{
-    m_is->seekg(0, istream::end);
-    istream::pos_type pos = m_is->tellg();
-    m_size = pos < 0 ? 0 : (unsigned long)pos;
-}
+    , m_size(size)
+{}
 
 vector<unsigned char> DataFilePrivate::calcDigest(const string &method) const
 {

src/DataFile_p.h

@@ -30,7 +30,7 @@ namespace digidoc
 class DataFilePrivate final: public DataFile
 {
 public:
-	DataFilePrivate(std::unique_ptr<std::istream> &&is, std::string filename, std::string mediatype, std::string id = {});
+    DataFilePrivate(std::unique_ptr<std::istream> &&is, std::string filename, std::string mediatype, unsigned long size, std::string id = {});
 
 	std::string id() const final { return m_id.empty() ? m_filename : m_id; }
 	std::string fileName() const final { return m_filename; }

src/SiVaContainer.cpp

@@ -163,7 +163,7 @@ SiVaContainer::SiVaContainer(const string &path, bool useHashCode)
     else if(ext == "pdf")
     {
         d->mediaType = "application/pdf";
-        d->dataFiles.push_back(new DataFilePrivate(std::move(ifs), fileName, "application/pdf"));
+        d->dataFiles.push_back(new DataFilePrivate(std::move(ifs), fileName, "application/pdf", File::fileSize(d->path)));
     }
     else if(find(asic.cbegin(), asic.cend(), ext) != asic.cend())
     {
@@ -185,9 +185,8 @@ SiVaContainer::SiVaContainer(const string &path, bool useHashCode)
             const auto directory = File::directory(file);
             if(directory.empty() || directory == "/" || directory == "./")
             {
-                auto data = make_unique<stringstream>();
-                z.extract(file, *data);
-                d->dataFiles.push_back(new DataFilePrivate(std::move(data), file, "application/octet-stream"));
+                auto properties = z.properties(file);
+                d->dataFiles.push_back(new DataFilePrivate(z.stream(file), file, "application/octet-stream", properties.size));
             }
         }
     }
@@ -381,6 +380,7 @@ unique_ptr<istream> SiVaContainer::parseDDoc(bool useHashCode)
                 d->dataFiles.push_back(new DataFilePrivate(make_unique<stringstream>(base64_decode(b64)),
                     xml::transcode<char>(item->getAttribute(cpXMLCh(u"Filename"))),
                     xml::transcode<char>(item->getAttribute(cpXMLCh(u"MimeType"))),
+                    0,
                     xml::transcode<char>(item->getAttribute(cpXMLCh(u"Id")))));
             }
 

src/SignatureTST.cpp

@@ -26,21 +26,16 @@
 #include "util/DateTime.h"
 #include "util/log.h"
 
+#include <sstream>
+
 using namespace digidoc;
 using namespace std;
 
 SignatureTST::SignatureTST(istream &is, ASiC_S *asicSDoc): asicSDoc(asicSDoc)
 {
-    is.seekg(0, istream::end);
-    istream::pos_type pos = is.tellg();
-    const auto size = pos < 0 ? 0 : (unsigned long)pos;
-    is.clear();
-    is.seekg(0, istream::beg);
-
-    vector<unsigned char> buf(size, 0);
-    is.read((char*)buf.data(), streamsize(buf.size()));
-
-    timestampToken = make_unique<TS>(buf.data(), buf.size());
+    stringstream data;
+    data << is.rdbuf();
+    timestampToken = make_unique<TS>(data.str());
 }
 
 SignatureTST::~SignatureTST() = default;

src/crypto/TS.h

@@ -31,6 +31,8 @@ class TS
 {
 public:
     TS(const std::string &url, const Digest &digest);
+    template <class Container>
+    inline TS(const Container &data): TS((const unsigned char*)data.data(), data.size()) {}
     TS(const unsigned char *data = nullptr, size_t size = 0);
 
     X509Cert cert() const;