Increase item limit in each tab to 100,000

Saves larger data in separate files.

Data from these files are loaded into memory only when needed.

Cleans up data directories on idle and exit.

Files in synchronized directories now also work the same.

plugins/itemencrypted/itemencrypted.cpp

@@ -38,7 +38,7 @@ const QLatin1String dataFileHeaderV2("CopyQ_encrypted_tab v2");
 
 const QLatin1String configEncryptTabs("encrypt_tabs");
 
-const int maxItemCount = 10000;
+const int maxItemCount = 100'000;
 
 bool waitOrTerminate(QProcess *p, int timeoutMs)
 {

plugins/itemfakevim/itemfakevim.cpp

@@ -754,7 +754,7 @@ QVariant ItemFakeVimLoader::icon() const
 
 void ItemFakeVimLoader::setEnabled(bool enabled)
 {
-    m_enabled = enabled;
+    ItemLoaderInterface::setEnabled(enabled);
     updateCurrentlyEnabledState();
 }
 
@@ -811,7 +811,7 @@ void ItemFakeVimLoader::updateCurrentlyEnabledState()
     if ( qobject_cast<QGuiApplication*>(qApp) == nullptr )
         return;
 
-    const bool enable = m_enabled && m_reallyEnabled;
+    const bool enable = isEnabled() && m_reallyEnabled;
     if (m_currentlyEnabled == enable)
         return;
 

plugins/itemfakevim/itemfakevim.h

@@ -49,7 +49,6 @@ class ItemFakeVimLoader final : public QObject, public ItemLoaderInterface
 
     void wrapEditWidget(QObject *obj);
 
-    bool m_enabled = true;
     bool m_reallyEnabled = false;
     bool m_currentlyEnabled = false;
     QString m_sourceFileName;

plugins/itemsync/filewatcher.cpp

@@ -15,6 +15,7 @@
 #include <QMimeData>
 #include <QRegularExpression>
 #include <QUrl>
+#include <QDebug>
 
 #include <array>
 #include <vector>
@@ -29,6 +30,77 @@ const QLatin1String mimePrivatePrefix(COPYQ_MIME_PREFIX_ITEMSYNC_PRIVATE);
 const QLatin1String mimeOldBaseName(COPYQ_MIME_PREFIX_ITEMSYNC_PRIVATE "old-basename");
 const QLatin1String mimeHashPrefix(COPYQ_MIME_PREFIX_ITEMSYNC_PRIVATE "hash");
 
+constexpr int itemDataTreshold = 10'000;
+
+class SyncDataFile {
+public:
+    SyncDataFile() {}
+
+    explicit SyncDataFile(const QString &path, const QString &format = QString())
+        : m_path(path)
+        , m_format(format)
+    {}
+
+    const QString &path() const { return m_path; }
+    void setPath(const QString &path) { m_path = path; }
+
+    const QString &format() const { return m_format; }
+    void setFormat(const QString &format) { m_format = format; }
+
+    QByteArray readAll() const
+    {
+        COPYQ_LOG( QStringLiteral("ItemSync: Reading data file: %1").arg(m_path) );
+
+        QFile f(m_path);
+        if ( !f.open(QIODevice::ReadOnly) )
+            return QByteArray();
+
+        if ( m_format.isEmpty() )
+            return f.readAll();
+
+        QDataStream stream(&f);
+        QVariantMap dataMap;
+        if ( !deserializeData(&stream, &dataMap) ) {
+            log( QStringLiteral("ItemSync: Failed to read data file \"%1\": %2")
+                    .arg(m_path, f.errorString()),
+                    LogError );
+            return QByteArray();
+        }
+
+        return dataMap.value(m_format).toByteArray();
+    }
+
+private:
+    QString m_path;
+    QString m_format;
+};
+Q_DECLARE_METATYPE(SyncDataFile)
+
+QDataStream &operator<<(QDataStream &out, SyncDataFile value)
+{
+    return out << value.path() << value.format();
+}
+
+QDataStream &operator>>(QDataStream &in, SyncDataFile &value)
+{
+    QString path;
+    QString format;
+    in >> path >> format;
+    value.setPath(path);
+    value.setFormat(format);
+    return in;
+}
+
+void registerSyncDataFileConverter()
+{
+    QMetaType::registerConverter(&SyncDataFile::readAll);
+#if QT_VERSION < QT_VERSION_CHECK(6,0,0)
+    qRegisterMetaTypeStreamOperators<SyncDataFile>("SyncDataFile");
+#else
+    qRegisterMetaType<SyncDataFile>("SyncDataFile");
+#endif
+}
+
 struct Ext {
     Ext() : extension(), format() {}
 
@@ -61,7 +133,7 @@ const QLatin1String noteFileSuffix("_note.txt");
 const int defaultUpdateFocusItemsIntervalMs = 10000;
 const int batchItemUpdateIntervalMs = 100;
 
-const qint64 sizeLimit = 10 << 20;
+const qint64 sizeLimit = 50'000'000;
 
 FileFormat getFormatSettingsFromFileName(const QString &fileName,
                                          const QList<FileFormat> &formatSettings,
@@ -943,18 +1015,34 @@ void FileWatcher::updateDataAndWatchFile(const QDir &dir, const BaseNameExtensio
 
         const QString fileName = basePath + ext.extension;
 
-        QFile f( dir.absoluteFilePath(fileName) );
+        const QString path = dir.absoluteFilePath(fileName);
+        QFile f(path);
         if ( !f.open(QIODevice::ReadOnly) )
             continue;
 
         if ( ext.extension == dataFileSuffix ) {
             QDataStream stream(&f);
-            if ( deserializeData(&stream, dataMap) )
+            QVariantMap dataMap2;
+            if ( deserializeData(&stream, &dataMap2) ) {
+                for (auto it = dataMap2.constBegin(); it != dataMap2.constEnd(); ++it) {
+                    if (it.value().toByteArray().length() > itemDataTreshold) {
+                        const QVariant value = QVariant::fromValue(SyncDataFile(path, it.key()));
+                        dataMap->insert(it.key(), value);
+                    } else {
+                        dataMap->insert(it.key(), it.value());
+                    }
+                }
+
                 mimeToExtension->insert(mimeUnknownFormats, dataFileSuffix);
+            }
         } else if ( f.size() > sizeLimit || ext.format.startsWith(mimeNoFormat)
                     || dataMap->contains(ext.format) )
         {
             mimeToExtension->insert(mimeNoFormat + ext.extension, ext.extension);
+        } else if ( f.size() > itemDataTreshold ) {
+            const QVariant value = QVariant::fromValue(SyncDataFile(path));
+            dataMap->insert(ext.format, value);
+            mimeToExtension->insert(ext.format, ext.extension);
         } else {
             dataMap->insert(ext.format, f.readAll());
             mimeToExtension->insert(ext.format, ext.extension);

plugins/itemsync/filewatcher.h

@@ -46,6 +46,11 @@ using BaseNameExtensionsList = QList<BaseNameExtensions>;
 
 using Hash = QByteArray;
 
+class SyncDataFile;
+QDataStream &operator<<(QDataStream &out, SyncDataFile value);
+QDataStream &operator>>(QDataStream &in, SyncDataFile &value);
+void registerSyncDataFileConverter();
+
 class FileWatcher final : public QObject {
 public:
     static QString getBaseName(const QModelIndex &index);

plugins/itemsync/itemsync.cpp

@@ -531,6 +531,7 @@ QString ItemSyncScriptable::selectedTabPath()
 
 ItemSyncLoader::ItemSyncLoader()
 {
+    registerSyncDataFileConverter();
 }
 
 ItemSyncLoader::~ItemSyncLoader() = default;

plugins/itemsync/tests/itemsynctests.cpp

@@ -3,6 +3,7 @@
 #include "itemsynctests.h"
 
 #include "common/mimetypes.h"
+#include "common/sleeptimer.h"
 #include "tests/test_utils.h"
 
 #include <QDir>
@@ -768,3 +769,43 @@ void ItemSyncTests::avoidDuplicateItemsAddedFromClipboard()
     TEST( m_test->setClipboard("one") );
     WAIT_ON_OUTPUT(args << "read(0,1,2,3)", "one,two,,");
 }
+
+void ItemSyncTests::saveLargeItem()
+{
+    const auto tab = testTab(1);
+    const auto args = Args("tab") << tab;
+
+    const auto script = R"(
+        write(0, [{
+            'text/plain': '1234567890'.repeat(10000),
+            'application/x-copyq-test-data': 'abcdefghijklmnopqrstuvwxyz'.repeat(10000),
+        }])
+        )";
+    RUN(args << script, "");
+
+    for (int i = 0; i < 2; ++i) {
+        RUN(args << "read(0).left(20)", "12345678901234567890");
+        RUN(args << "read(0).length", "100000\n");
+        RUN(args << "getItem(0)[mimeText].left(20)", "12345678901234567890");
+        RUN(args << "getItem(0)[mimeText].length", "100000\n");
+        RUN(args << "getItem(0)['application/x-copyq-test-data'].left(26)", "abcdefghijklmnopqrstuvwxyz");
+        RUN(args << "getItem(0)['application/x-copyq-test-data'].length", "260000\n");
+        RUN("unload" << tab, tab + "\n");
+    }
+
+    RUN("show" << tab, "");
+    RUN("keys" << clipboardBrowserId << keyNameFor(QKeySequence::Copy), "");
+    WAIT_ON_OUTPUT("clipboard().left(20)", "12345678901234567890");
+    RUN("clipboard('application/x-copyq-test-data').left(26)", "abcdefghijklmnopqrstuvwxyz");
+    RUN("clipboard('application/x-copyq-test-data').length", "260000\n");
+
+    const auto tab2 = testTab(2);
+    const auto args2 = Args("tab") << tab2;
+    RUN("show" << tab2, "");
+    waitFor(waitMsPasteClipboard);
+    RUN("keys" << clipboardBrowserId << keyNameFor(QKeySequence::Paste), "");
+    RUN(args2 << "read(0).left(20)", "12345678901234567890");
+    RUN(args2 << "read(0).length", "100000\n");
+    RUN(args << "getItem(0)['application/x-copyq-test-data'].left(26)", "abcdefghijklmnopqrstuvwxyz");
+    RUN(args << "getItem(0)['application/x-copyq-test-data'].length", "260000\n");
+}

plugins/itemsync/tests/itemsynctests.h

@@ -52,6 +52,8 @@ private slots:
 
     void avoidDuplicateItemsAddedFromClipboard();
 
+    void saveLargeItem();
+
 private:
     TestInterfacePtr m_test;
 };

src/app/app.cpp

@@ -7,6 +7,7 @@
 #include "common/log.h"
 #include "common/settings.h"
 #include "common/textdata.h"
+#include "item/serialize.h"
 #include "platform/platformnativeinterface.h"
 #ifdef Q_OS_UNIX
 #   include "platform/unix/unixsignalhandler.h"
@@ -120,6 +121,8 @@ App::App(QCoreApplication *application,
     , m_started(false)
     , m_closed(false)
 {
+    registerDataFileConverter();
+
     QObject::connect(m_app, &QCoreApplication::aboutToQuit, [this]() { exit(); });
 
 #ifdef Q_OS_UNIX

src/app/clipboardclient.cpp

@@ -9,6 +9,7 @@
 #include "common/common.h"
 #include "common/log.h"
 #include "common/textdata.h"
+#include "item/itemfactory.h"
 #include "platform/platformnativeinterface.h"
 #include "scriptable/scriptable.h"
 #include "scriptable/scriptableproxy.h"
@@ -114,9 +115,14 @@ void ClipboardClient::onConnectionFailed()
 
 void ClipboardClient::start(const QStringList &arguments)
 {
+    ItemFactory itemFactory;
+    itemFactory.loadPlugins();
+    QSettings settings;
+    itemFactory.loadItemFactorySettings(&settings);
+
     QJSEngine engine;
     ScriptableProxy scriptableProxy(nullptr, nullptr);
-    Scriptable scriptable(&engine, &scriptableProxy);
+    Scriptable scriptable(&engine, &scriptableProxy, &itemFactory);
 
     const auto serverName = clipboardServerName();
     ClientSocket socket(serverName);

src/app/clipboardserver.cpp

@@ -23,6 +23,7 @@
 #include "gui/notificationbutton.h"
 #include "gui/notificationdaemon.h"
 #include "item/itemfactory.h"
+#include "item/itemstore.h"
 #include "item/serialize.h"
 #include "scriptable/scriptableproxy.h"
 
@@ -185,6 +186,8 @@ ClipboardServer::ClipboardServer(QApplication *app, const QString &sessionName)
         m_actionDataToSend.clear();
     });
 
+    initSingleShotTimer(&m_timerCleanItemFiles, 120000, this, &ClipboardServer::cleanDataFiles);
+
     initSingleShotTimer(&m_updateThemeTimer, 1000, this, [this](){
         AppConfig appConfig;
         loadSettings(&appConfig);
@@ -294,6 +297,7 @@ void ClipboardServer::onAboutToQuit()
     terminateClients(5000);
 
     m_wnd->saveTabs();
+    cleanDataFiles();
 }
 
 void ClipboardServer::onCommitData(QSessionManager &sessionManager)
@@ -464,6 +468,12 @@ void ClipboardServer::sendActionData(int actionId, const QByteArray &bytes)
     }
 }
 
+void ClipboardServer::cleanDataFiles()
+{
+    COPYQ_LOG(QStringLiteral("Cleaning unused item files"));
+    ::cleanDataFiles( m_wnd->tabs() );
+}
+
 void ClipboardServer::onClientNewConnection(const ClientSocketPtr &client)
 {
     auto proxy = new ScriptableProxy(m_wnd);
@@ -632,6 +642,7 @@ bool ClipboardServer::eventFilter(QObject *object, QEvent *ev)
         if (state != Qt::ApplicationActive) {
             COPYQ_LOG( QString("Saving items on application state change (%1)").arg(state) );
             m_wnd->saveTabs();
+            m_timerCleanItemFiles.start();
         }
     } else if (type == QEvent::ThemeChange) {
         if ( !m_updateThemeTimer.isActive() )

src/app/clipboardserver.h

@@ -118,6 +118,8 @@ class ClipboardServer final : public QObject, public App
 
     void sendActionData(int actionId, const QByteArray &bytes);
 
+    void cleanDataFiles();
+
     Server *m_server = nullptr;
     MainWindow* m_wnd = nullptr;
     QPointer<Action> m_monitor;
@@ -135,6 +137,7 @@ class ClipboardServer final : public QObject, public App
 
     QMap<int, QByteArray> m_actionDataToSend;
     QTimer m_timerClearUnsentActionData;
+    QTimer m_timerCleanItemFiles;
 
     struct ClientData {
         ClientData() = default;

src/common/appconfig.h

@@ -13,7 +13,7 @@ QString defaultClipboardTabName();
 
 namespace Config {
 
-const int maxItems = 10000;
+const int maxItems = 100'000;
 
 template<typename ValueType>
 struct Config {

src/gui/configurationmanager.cpp

@@ -175,7 +175,7 @@ void ConfigurationManager::initPluginWidgets(ItemFactory *itemFactory)
     for ( const auto &loader : itemFactory->loaders() ) {
         ItemOrderList::ItemPtr pluginItem(new PluginItem(loader));
         const QIcon icon = getIcon(loader->icon());
-        const auto state = itemFactory->isLoaderEnabled(loader)
+        const auto state = loader->isEnabled()
                 ? ItemOrderList::Checked
                 : ItemOrderList::Unchecked;
         m_tabItems->appendItem( loader->name(), icon, pluginItem, state );
@@ -542,7 +542,8 @@ void ConfigurationManager::apply(AppConfig *appConfig)
 
     for (int i = 0; i < m_tabItems->itemCount(); ++i) {
         const QString loaderId = m_tabItems->data(i).toString();
-        Q_ASSERT(!loaderId.isEmpty());
+        if ( loaderId.isEmpty() )
+            continue;
 
         pluginPriority.append(loaderId);