WIP: Increase item limit in each tab to 100,000

Saves large data in separate files.

TODO:
- Clean up empty data directories

plugins/itemencrypted/itemencrypted.cpp

@@ -38,7 +38,7 @@ const QLatin1String dataFileHeaderV2("CopyQ_encrypted_tab v2");
 
 const QLatin1String configEncryptTabs("encrypt_tabs");
 
-const int maxItemCount = 10000;
+const int maxItemCount = 100'000;
 
 bool waitOrTerminate(QProcess *p, int timeoutMs)
 {

src/app/app.cpp

@@ -7,6 +7,7 @@
 #include "common/log.h"
 #include "common/settings.h"
 #include "common/textdata.h"
+#include "item/serialize.h"
 #include "platform/platformnativeinterface.h"
 #ifdef Q_OS_UNIX
 #   include "platform/unix/unixsignalhandler.h"
@@ -120,6 +121,8 @@ App::App(QCoreApplication *application,
     , m_started(false)
     , m_closed(false)
 {
+    registerDataFileConverter();
+
     QObject::connect(m_app, &QCoreApplication::aboutToQuit, [this]() { exit(); });
 
 #ifdef Q_OS_UNIX

src/common/appconfig.h

@@ -13,7 +13,7 @@ QString defaultClipboardTabName();
 
 namespace Config {
 
-const int maxItems = 10000;
+const int maxItems = 100'000;
 
 template<typename ValueType>
 struct Config {

src/item/serialize.cpp

@@ -8,16 +8,67 @@
 
 #include <QAbstractItemModel>
 #include <QByteArray>
+#include <QCryptographicHash>
 #include <QDataStream>
+#include <QDir>
 #include <QIODevice>
 #include <QList>
 #include <QPair>
+#include <QSaveFile>
+#include <QStandardPaths>
 #include <QStringList>
 
 #include <unordered_map>
 
+class DataFile {
+public:
+    DataFile() {}
+
+    explicit DataFile(const QString &path)
+        : m_path(path)
+    {}
+
+    const QString &path() const { return m_path; }
+    void setPath(const QString &path)
+    {
+        m_path = path;
+    }
+
+    QByteArray readAll() const
+    {
+        QFile f(m_path);
+        if ( !f.open(QIODevice::ReadOnly) ) {
+            log( QStringLiteral("Failed to read data file \"%1\": %2")
+                    .arg(m_path, f.errorString()),
+                    LogError );
+            return QByteArray();
+        }
+        return f.readAll();
+    }
+
+private:
+    QString m_path;
+};
+Q_DECLARE_METATYPE(DataFile)
+
+QDataStream &operator<<(QDataStream &out, DataFile value)
+{
+    return out << value.path();
+}
+
+QDataStream &operator>>(QDataStream &in, DataFile &value)
+{
+    QString path;
+    in >> path;
+    value.setPath(path);
+    return in;
+}
+
 namespace {
 
+const QString mimeFilePrefix = QStringLiteral("FILE:");
+constexpr int itemDataTreshold = 10'000;
+
 template <typename T>
 bool readOrError(QDataStream *out, T *value, const char *error)
 {
@@ -101,11 +152,13 @@ bool deserializeDataV2(QDataStream *out, QVariantMap *data)
     QByteArray tmpBytes;
     bool compress;
     for (qint32 i = 0; i < size; ++i) {
-        const QString mime = decompressMime(out);
+        QString mime = decompressMime(out);
         if ( out->status() != QDataStream::Ok )
             return false;
 
-        if ( !readOrError(out, &compress, "Failed to read compression flag (v2)") )
+        const bool hasDataFile = mime.startsWith(mimeFilePrefix);
+
+        if ( !hasDataFile && !readOrError(out, &compress, "Failed to read compression flag (v2)") )
             return false;
 
         if ( !readOrError(out, &tmpBytes, "Failed to read item data (v2)") )
@@ -119,14 +172,56 @@ bool deserializeDataV2(QDataStream *out, QVariantMap *data)
                 return false;
             }
         }
-        data->insert(mime, tmpBytes);
+
+        if (hasDataFile) {
+            mime = mime.mid( mimeFilePrefix.length() );
+            const QString path = QString::fromUtf8(tmpBytes);
+            const QVariant value = QVariant::fromValue(DataFile(path));
+            Q_ASSERT(value.canConvert<QByteArray>());
+            Q_ASSERT(value.value<DataFile>().path() == path);
+            data->insert(mime, value);
+        } else {
+            data->insert(mime, tmpBytes);
+        }
     }
 
     return out->status() == QDataStream::Ok;
 }
 
+QString dataFilePath(const QByteArray &bytes, bool create = false)
+{
+    QDir dir( QStandardPaths::writableLocation(QStandardPaths::AppDataLocation) );
+    QCryptographicHash hash(QCryptographicHash::Sha256);
+    hash.addData(QByteArrayLiteral("copyq_salt"));
+    hash.addData(bytes);
+    const QString sha = QString::fromUtf8( hash.result().toHex() );
+    const QString subpath = QStringLiteral("%1/%2/%3").arg(
+            sha.mid(0, 16),
+            sha.mid(16, 16),
+            sha.mid(32, 16)
+    );
+    if (create && !dir.mkpath(subpath)) {
+        log( QStringLiteral("Failed to create data directory: %1")
+                .arg(dir.absoluteFilePath(subpath)),
+                LogError );
+        return QString();
+    }
+    return dir.absoluteFilePath(
+            QStringLiteral("%1/%2.dat").arg(subpath, sha.mid(48)) );
+}
+
 } // namespace
 
+void registerDataFileConverter()
+{
+    QMetaType::registerConverter(&DataFile::readAll);
+#if QT_VERSION < QT_VERSION_CHECK(6,0,0)
+    qRegisterMetaTypeStreamOperators<DataFile>("DataFile");
+#else
+    qRegisterMetaType<DataFile>("DataFile");
+#endif
+}
+
 void serializeData(QDataStream *stream, const QVariantMap &data)
 {
     *stream << static_cast<qint32>(-2);
@@ -138,9 +233,38 @@ void serializeData(QDataStream *stream, const QVariantMap &data)
     for (auto it = data.constBegin(); it != data.constEnd(); ++it) {
         const auto &mime = it.key();
         bytes = data[mime].toByteArray();
-        *stream << compressMime(mime)
-                << /* compressData = */ false
-                << bytes;
+        const QString path = dataFilePath(bytes, true);
+        if ( bytes.length() > itemDataTreshold || mime.startsWith(mimeFilePrefix) ) {
+            if ( path.isEmpty() ) {
+                stream->setStatus(QDataStream::WriteFailed);
+                return;
+            }
+
+            if ( !QFile::exists(path) ) {
+                QSaveFile f(path);
+                f.setDirectWriteFallback(false);
+                if ( !f.open(QIODevice::WriteOnly) || !f.write(bytes) || !f.commit() ) {
+                    log( QStringLiteral("Failed to create data file \"%1\": %2")
+                            .arg(path, f.errorString()),
+                            LogError );
+                    stream->setStatus(QDataStream::WriteFailed);
+                    f.cancelWriting();
+                    return;
+                }
+            }
+
+            if ( mime.startsWith(mimeFilePrefix) )
+                *stream << compressMime(mime) << path.toUtf8();
+            else
+                *stream << compressMime(mimeFilePrefix + mime) << path.toUtf8();
+        } else {
+            if ( !path.isEmpty() && QFile::exists(path) ) {
+                QFile::remove(path);
+            }
+            *stream << compressMime(mime)
+                    << /* compressData = */ false
+                    << bytes;
+        }
     }
 }
 

src/item/serialize.h

@@ -10,6 +10,11 @@ class QByteArray;
 class QDataStream;
 class QIODevice;
 
+class DataFile;
+QDataStream &operator<<(QDataStream &out, DataFile value);
+QDataStream &operator>>(QDataStream &in, DataFile &value);
+void registerDataFileConverter();
+
 void serializeData(QDataStream *stream, const QVariantMap &data);
 bool deserializeData(QDataStream *stream, QVariantMap *data);
 QByteArray serializeData(const QVariantMap &data);

src/ui/configtabhistory.ui

@@ -77,7 +77,7 @@
              <string>Maximum number of items in each tab</string>
             </property>
             <property name="maximum">
-             <number>10000</number>
+             <number>100000</number>
             </property>
             <property name="value">
              <number>200</number>

src/ui/tabpropertieswidget.ui

@@ -56,7 +56,7 @@
         <string>default</string>
        </property>
        <property name="maximum">
-        <number>10000</number>
+        <number>100000</number>
        </property>
        <property name="value">
         <number>200</number>