Add --download-archive support. Addresses #4

hstr0100 · Nov 25, 2024 · 96e797c · 96e797c
1 parent 0a52f8e
commit 96e797c
Show file tree

Hide file tree

Showing 17 changed files with 142 additions and 38 deletions.
diff --git a/core/src/main/java/net/brlns/gdownloader/downloader/AbstractDownloader.java b/core/src/main/java/net/brlns/gdownloader/downloader/AbstractDownloader.java
@@ -24,6 +24,7 @@
 import net.brlns.gdownloader.GDownloader;
 import net.brlns.gdownloader.downloader.enums.DownloaderIdEnum;
 import net.brlns.gdownloader.downloader.structs.DownloadResult;
+import net.brlns.gdownloader.settings.Settings;
 import net.brlns.gdownloader.settings.enums.DownloadTypeEnum;
 import net.brlns.gdownloader.ui.menu.IMenuEntry;
 
@@ -61,6 +62,8 @@ public AbstractDownloader(DownloadManager managerIn) {
 
     public abstract boolean isMainDownloader();
 
+    public abstract DownloadTypeEnum getFirstArchivableType(Settings configIn);
+
     public abstract DownloaderIdEnum getDownloaderId();
 
     public List<DownloadTypeEnum> getDownloadTypes() {

diff --git a/core/src/main/java/net/brlns/gdownloader/downloader/DirectHttpDownloader.java b/core/src/main/java/net/brlns/gdownloader/downloader/DirectHttpDownloader.java
@@ -47,6 +47,7 @@
 import net.brlns.gdownloader.downloader.enums.DownloadStatusEnum;
 import net.brlns.gdownloader.downloader.enums.DownloaderIdEnum;
 import net.brlns.gdownloader.downloader.structs.DownloadResult;
+import net.brlns.gdownloader.settings.Settings;
 import net.brlns.gdownloader.settings.enums.DownloadTypeEnum;
 import net.brlns.gdownloader.ui.menu.IMenuEntry;
 import net.brlns.gdownloader.ui.menu.RunnableMenuEntry;
@@ -106,6 +107,11 @@ public boolean isMainDownloader() {
         return false;
     }
 
+    @Override
+    public DownloadTypeEnum getFirstArchivableType(Settings configIn) {
+        return DIRECT;
+    }
+
     @Override
     protected boolean canConsumeUrl(String inputUrl) {
         return isEnabled()

diff --git a/core/src/main/java/net/brlns/gdownloader/downloader/DownloadManager.java b/core/src/main/java/net/brlns/gdownloader/downloader/DownloadManager.java
@@ -27,7 +27,6 @@
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicReference;
 import java.util.stream.Collectors;
-import lombok.AccessLevel;
 import lombok.Getter;
 import lombok.extern.slf4j.Slf4j;
 import net.brlns.gdownloader.GDownloader;
@@ -66,7 +65,7 @@ public class DownloadManager implements IEvent {
 
     private static final int MAX_DOWNLOAD_RETRIES = 10;
 
-    @Getter(AccessLevel.PROTECTED)
+    @Getter
     private final GDownloader main;
 
     private final ExecutorService processMonitor;

diff --git a/core/src/main/java/net/brlns/gdownloader/downloader/GalleryDlDownloader.java b/core/src/main/java/net/brlns/gdownloader/downloader/GalleryDlDownloader.java
@@ -33,6 +33,7 @@
 import net.brlns.gdownloader.downloader.enums.DownloadStatusEnum;
 import net.brlns.gdownloader.downloader.enums.DownloaderIdEnum;
 import net.brlns.gdownloader.downloader.structs.DownloadResult;
+import net.brlns.gdownloader.settings.Settings;
 import net.brlns.gdownloader.settings.enums.DownloadTypeEnum;
 import net.brlns.gdownloader.settings.filters.AbstractUrlFilter;
 import net.brlns.gdownloader.ui.menu.IMenuEntry;
@@ -81,6 +82,11 @@ public boolean isMainDownloader() {
         return false;
     }
 
+    @Override
+    public DownloadTypeEnum getFirstArchivableType(Settings configIn) {
+        return GALLERY;
+    }
+
     @Override
     protected boolean canConsumeUrl(String inputUrl) {
         return isEnabled()
@@ -129,7 +135,7 @@ protected DownloadResult tryDownload(QueueEntry entry) throws Exception {
             ));
         }
 
-        genericArguments.addAll(filter.getArguments(getDownloaderId(), ALL, main, tmpPath, entry.getUrl()));
+        genericArguments.addAll(filter.getArguments(this, ALL, manager, tmpPath, entry.getUrl()));
 
         boolean success = false;
         String lastOutput = "";
@@ -146,7 +152,7 @@ protected DownloadResult tryDownload(QueueEntry entry) throws Exception {
 
             List<String> arguments = new ArrayList<>(genericArguments);
 
-            List<String> downloadArguments = filter.getArguments(getDownloaderId(), type, main, tmpPath, entry.getUrl());
+            List<String> downloadArguments = filter.getArguments(this, type, manager, tmpPath, entry.getUrl());
             arguments.addAll(downloadArguments);
 
             if (main.getConfig().isDebugMode()) {

diff --git a/core/src/main/java/net/brlns/gdownloader/downloader/YtDlpDownloader.java b/core/src/main/java/net/brlns/gdownloader/downloader/YtDlpDownloader.java
@@ -35,6 +35,7 @@
 import net.brlns.gdownloader.downloader.structs.DownloadResult;
 import net.brlns.gdownloader.downloader.structs.MediaInfo;
 import net.brlns.gdownloader.settings.QualitySettings;
+import net.brlns.gdownloader.settings.Settings;
 import net.brlns.gdownloader.settings.enums.AudioBitrateEnum;
 import net.brlns.gdownloader.settings.enums.AudioContainerEnum;
 import net.brlns.gdownloader.settings.enums.DownloadTypeEnum;
@@ -84,6 +85,25 @@ public boolean isMainDownloader() {
         return true;
     }
 
+    @Nullable
+    @Override
+    public DownloadTypeEnum getFirstArchivableType(Settings configIn) {
+        for (DownloadTypeEnum type : DownloadTypeEnum.values()) {
+            boolean supported = getDownloadTypes().contains(type);
+
+            if (!supported) {
+                continue;
+            }
+
+            if (type == VIDEO && configIn.isDownloadVideo()
+                || type == AUDIO && configIn.isDownloadAudio()) {
+                return type;
+            }
+        }
+
+        return null;
+    }
+
     @Override
     protected boolean canConsumeUrl(String inputUrl) {
         return isEnabled()
@@ -187,7 +207,7 @@ protected DownloadResult tryDownload(QueueEntry entry) throws Exception {
             genericArguments.add("--ignore-config");
         }
 
-        genericArguments.addAll(filter.getArguments(getDownloaderId(), ALL, main, tmpPath, entry.getUrl()));
+        genericArguments.addAll(filter.getArguments(this, ALL, manager, tmpPath, entry.getUrl()));
 
         boolean success = false;
         String lastOutput = "";
@@ -207,7 +227,7 @@ protected DownloadResult tryDownload(QueueEntry entry) throws Exception {
 
             List<String> arguments = new ArrayList<>(genericArguments);
 
-            List<String> downloadArguments = filter.getArguments(getDownloaderId(), type, main, tmpPath, entry.getUrl());
+            List<String> downloadArguments = filter.getArguments(this, type, manager, tmpPath, entry.getUrl());
             arguments.addAll(downloadArguments);
 
             if (main.getConfig().isDebugMode()) {
@@ -240,6 +260,10 @@ protected DownloadResult tryDownload(QueueEntry entry) throws Exception {
                     log.error("Failed to download {}: {}", type, lastOutput);
                 }
             } else {
+                if (lastOutput.contains("recorded in the archive")) {
+                    return new DownloadResult(FLAG_SUCCESS, lastOutput);
+                }
+
                 success = true;
             }
         }

diff --git a/core/src/main/java/net/brlns/gdownloader/settings/Settings.java b/core/src/main/java/net/brlns/gdownloader/settings/Settings.java
@@ -69,6 +69,9 @@ public class Settings {
     @JsonProperty("DownloadYoutubeChannels")
     private boolean downloadYoutubeChannels = false;
 
+    @JsonProperty("RecordToDownloadArchive")
+    private boolean recordToDownloadArchive = false;
+
     @JsonProperty("RespectYtDlpConfigFile")
     private boolean respectYtDlpConfigFile = false;
 

diff --git a/core/src/main/java/net/brlns/gdownloader/settings/filters/AbstractUrlFilter.java b/core/src/main/java/net/brlns/gdownloader/settings/filters/AbstractUrlFilter.java
@@ -31,6 +31,8 @@
 import lombok.Data;
 import lombok.extern.slf4j.Slf4j;
 import net.brlns.gdownloader.GDownloader;
+import net.brlns.gdownloader.downloader.AbstractDownloader;
+import net.brlns.gdownloader.downloader.DownloadManager;
 import net.brlns.gdownloader.downloader.enums.DownloaderIdEnum;
 import net.brlns.gdownloader.settings.QualitySettings;
 import net.brlns.gdownloader.settings.enums.DownloadTypeEnum;
@@ -210,13 +212,13 @@ public boolean matches(String url) {
     }
 
     @JsonIgnore
-    public List<String> getArguments(DownloaderIdEnum downloaderId, DownloadTypeEnum typeEnum, GDownloader main, File savePath, String inputUrl) {
+    public List<String> getArguments(AbstractDownloader downloader, DownloadTypeEnum typeEnum, DownloadManager manager, File savePath, String inputUrl) {
         List<String> arguments = new ArrayList<>();
 
-        arguments.addAll(buildArguments(downloaderId, typeEnum, main, savePath, inputUrl));
+        arguments.addAll(buildArguments(downloader, typeEnum, manager, savePath, inputUrl));
 
         // TODO: Map<DonwloaderIdEnum, Map<DownloadTypeEnum, List<String>>> or a struct extending that.
-        switch (downloaderId) {
+        switch (downloader.getDownloaderId()) {
             case YT_DLP -> {
                 if (extraYtDlpArguments.containsKey(typeEnum)) {
                     arguments.addAll(extraYtDlpArguments.get(typeEnum));
@@ -228,15 +230,15 @@ public List<String> getArguments(DownloaderIdEnum downloaderId, DownloadTypeEnum
                 }
             }
             default -> {
-                log.warn("Unhandled downloader id {}", downloaderId);
+                log.warn("Unhandled downloader id {}", downloader.getDownloaderId());
             }
         }
 
         return arguments;
     }
 
     @JsonIgnore
-    protected abstract List<String> buildArguments(DownloaderIdEnum downloaderId, DownloadTypeEnum typeEnum, GDownloader main, File savePath, String inputUrl);
+    protected abstract List<String> buildArguments(AbstractDownloader downloader, DownloadTypeEnum typeEnum, DownloadManager manager, File savePath, String inputUrl);
 
     @JsonIgnore
     public abstract boolean areCookiesRequired();

diff --git a/core/src/main/java/net/brlns/gdownloader/settings/filters/FacebookFilter.java b/core/src/main/java/net/brlns/gdownloader/settings/filters/FacebookFilter.java
@@ -22,8 +22,8 @@
 import java.util.List;
 import lombok.Data;
 import lombok.EqualsAndHashCode;
-import net.brlns.gdownloader.GDownloader;
-import net.brlns.gdownloader.downloader.enums.DownloaderIdEnum;
+import net.brlns.gdownloader.downloader.AbstractDownloader;
+import net.brlns.gdownloader.downloader.DownloadManager;
 import net.brlns.gdownloader.settings.Settings;
 import net.brlns.gdownloader.settings.enums.DownloadTypeEnum;
 
@@ -51,12 +51,12 @@ public FacebookFilter() {
 
     @JsonIgnore
     @Override
-    protected List<String> buildArguments(DownloaderIdEnum downloaderId, DownloadTypeEnum typeEnum, GDownloader main, File savePath, String inputUrl) {
-        Settings config = main.getConfig();
+    protected List<String> buildArguments(AbstractDownloader downloader, DownloadTypeEnum typeEnum, DownloadManager manager, File savePath, String inputUrl) {
+        Settings config = manager.getMain().getConfig();
 
-        List<String> arguments = super.buildArguments(downloaderId, typeEnum, main, savePath, inputUrl);
+        List<String> arguments = super.buildArguments(downloader, typeEnum, manager, savePath, inputUrl);
 
-        switch (downloaderId) {
+        switch (downloader.getDownloaderId()) {
             case YT_DLP -> {
                 switch (typeEnum) {
                     case ALL -> {

diff --git a/core/src/main/java/net/brlns/gdownloader/settings/filters/GenericFilter.java b/core/src/main/java/net/brlns/gdownloader/settings/filters/GenericFilter.java
@@ -23,14 +23,17 @@
 import java.util.List;
 import lombok.Data;
 import lombok.EqualsAndHashCode;
+import lombok.extern.slf4j.Slf4j;
 import net.brlns.gdownloader.GDownloader;
-import net.brlns.gdownloader.downloader.enums.DownloaderIdEnum;
+import net.brlns.gdownloader.downloader.AbstractDownloader;
+import net.brlns.gdownloader.downloader.DownloadManager;
 import net.brlns.gdownloader.settings.QualitySettings;
 import net.brlns.gdownloader.settings.Settings;
 import net.brlns.gdownloader.settings.enums.AudioBitrateEnum;
 import net.brlns.gdownloader.settings.enums.AudioCodecEnum;
 import net.brlns.gdownloader.settings.enums.DownloadTypeEnum;
 import net.brlns.gdownloader.settings.enums.VideoContainerEnum;
+import net.brlns.gdownloader.util.FileUtils;
 import net.brlns.gdownloader.util.URLUtils;
 
 import static net.brlns.gdownloader.lang.Language.*;
@@ -39,6 +42,7 @@
  * @author Gabriel / hstr0100 / vertx010
  */
 @Data
+@Slf4j
 @EqualsAndHashCode(callSuper = true)
 @JsonIgnoreProperties(ignoreUnknown = true)
 public class GenericFilter extends AbstractUrlFilter {
@@ -66,14 +70,20 @@ public String getDisplayName() {
 
     @JsonIgnore
     @Override
-    protected List<String> buildArguments(DownloaderIdEnum downloaderId, DownloadTypeEnum typeEnum, GDownloader main, File savePath, String inputUrl) {
-        Settings config = main.getConfig();
+    protected List<String> buildArguments(AbstractDownloader downloader, DownloadTypeEnum typeEnum, DownloadManager manager, File savePath, String inputUrl) {
+        Settings config = manager.getMain().getConfig();
         QualitySettings quality = getQualitySettings();
         AudioBitrateEnum audioBitrate = quality.getAudioBitrate();
 
+        File archiveFile = null;
+        if (config.isRecordToDownloadArchive()) {
+            archiveFile = FileUtils.getOrCreate(GDownloader.getWorkDirectory(),
+                downloader.getDownloaderId().getDisplayName() + "_archive.txt");
+        }
+
         List<String> arguments = new ArrayList<>();
 
-        switch (downloaderId) {
+        switch (downloader.getDownloaderId()) {
             case YT_DLP -> {
                 switch (typeEnum) {
                     case ALL -> {
@@ -103,7 +113,7 @@ protected List<String> buildArguments(DownloaderIdEnum downloaderId, DownloadTyp
                         if (config.isReadCookiesFromBrowser()) {
                             arguments.addAll(List.of(
                                 "--cookies-from-browser",
-                                main.getBrowserForCookies().getName()
+                                manager.getMain().getBrowserForCookies().getName()
                             ));
                         }
 
@@ -166,6 +176,13 @@ protected List<String> buildArguments(DownloaderIdEnum downloaderId, DownloadTyp
                                 + (audioBitrate == AudioBitrateEnum.NO_AUDIO ? 320 : audioBitrate.getValue()) + "k"
                             ));
                         }
+
+                        if (archiveFile != null && downloader.getFirstArchivableType(config) == typeEnum) {
+                            arguments.addAll(List.of(
+                                "--download-archive",
+                                archiveFile.getAbsolutePath()
+                            ));
+                        }
                     }
                     case AUDIO -> {
                         if (audioBitrate != AudioBitrateEnum.NO_AUDIO) {
@@ -190,6 +207,13 @@ protected List<String> buildArguments(DownloaderIdEnum downloaderId, DownloadTyp
                                     "--embed-metadata"
                                 ));
                             }
+
+                            if (archiveFile != null && downloader.getFirstArchivableType(config) == typeEnum) {
+                                arguments.addAll(List.of(
+                                    "--download-archive",
+                                    archiveFile.getAbsolutePath()
+                                ));
+                            }
                         }
                     }
                     case THUMBNAILS -> {
@@ -245,7 +269,7 @@ protected List<String> buildArguments(DownloaderIdEnum downloaderId, DownloadTyp
                         if (config.isReadCookiesFromBrowser()) {
                             arguments.addAll(List.of(
                                 "--cookies-from-browser",
-                                main.getBrowserForCookies().getName()
+                                manager.getMain().getBrowserForCookies().getName()
                             ));
                         }
                     }
@@ -256,6 +280,13 @@ protected List<String> buildArguments(DownloaderIdEnum downloaderId, DownloadTyp
                             "-D",
                             savePath.getAbsolutePath() + (fileName != null ? File.separator + fileName : "")
                         ));
+
+                        if (archiveFile != null && downloader.getFirstArchivableType(config) == typeEnum) {
+                            arguments.addAll(List.of(
+                                "--download-archive",
+                                archiveFile.getAbsolutePath()
+                            ));
+                        }
                     }
                     default ->
                         throw new IllegalArgumentException();

diff --git a/core/src/main/java/net/brlns/gdownloader/settings/filters/TwitchFilter.java b/core/src/main/java/net/brlns/gdownloader/settings/filters/TwitchFilter.java
@@ -22,8 +22,8 @@
 import java.util.List;
 import lombok.Data;
 import lombok.EqualsAndHashCode;
-import net.brlns.gdownloader.GDownloader;
-import net.brlns.gdownloader.downloader.enums.DownloaderIdEnum;
+import net.brlns.gdownloader.downloader.AbstractDownloader;
+import net.brlns.gdownloader.downloader.DownloadManager;
 import net.brlns.gdownloader.settings.QualitySettings;
 import net.brlns.gdownloader.settings.enums.DownloadTypeEnum;
 import net.brlns.gdownloader.settings.enums.QualitySelectorEnum;
@@ -58,10 +58,10 @@ public TwitchFilter() {
 
     @JsonIgnore
     @Override
-    protected List<String> buildArguments(DownloaderIdEnum downloaderId, DownloadTypeEnum typeEnum, GDownloader main, File savePath, String inputUrl) {
-        List<String> arguments = super.buildArguments(downloaderId, typeEnum, main, savePath, inputUrl);
+    protected List<String> buildArguments(AbstractDownloader downloader, DownloadTypeEnum typeEnum, DownloadManager manager, File savePath, String inputUrl) {
+        List<String> arguments = super.buildArguments(downloader, typeEnum, manager, savePath, inputUrl);
 
-        switch (downloaderId) {
+        switch (downloader.getDownloaderId()) {
             case YT_DLP -> {
                 switch (typeEnum) {
                     case ALL -> {

diff --git a/core/src/main/java/net/brlns/gdownloader/settings/filters/YoutubeFilter.java b/core/src/main/java/net/brlns/gdownloader/settings/filters/YoutubeFilter.java
@@ -23,7 +23,8 @@
 import lombok.Data;
 import lombok.EqualsAndHashCode;
 import net.brlns.gdownloader.GDownloader;
-import net.brlns.gdownloader.downloader.enums.DownloaderIdEnum;
+import net.brlns.gdownloader.downloader.AbstractDownloader;
+import net.brlns.gdownloader.downloader.DownloadManager;
 import net.brlns.gdownloader.settings.Settings;
 import net.brlns.gdownloader.settings.enums.DownloadTypeEnum;
 
@@ -51,12 +52,12 @@ public YoutubeFilter() {
 
     @JsonIgnore
     @Override
-    protected List<String> buildArguments(DownloaderIdEnum downloaderId, DownloadTypeEnum typeEnum, GDownloader main, File savePath, String inputUrl) {
-        Settings config = main.getConfig();
+    protected List<String> buildArguments(AbstractDownloader downloader, DownloadTypeEnum typeEnum, DownloadManager manager, File savePath, String inputUrl) {
+        Settings config = manager.getMain().getConfig();
 
-        List<String> arguments = super.buildArguments(downloaderId, typeEnum, main, savePath, inputUrl);
+        List<String> arguments = super.buildArguments(downloader, typeEnum, manager, savePath, inputUrl);
 
-        switch (downloaderId) {
+        switch (downloader.getDownloaderId()) {
             case YT_DLP -> {
                 switch (typeEnum) {
                     case ALL -> {