From 1f7c5f224a3bfae6de3bfbd45ea29acfcae15170 Mon Sep 17 00:00:00 2001 From: Thompson3142 <115718208+Thompson3142@users.noreply.github.com> Date: Sat, 7 Sep 2024 02:32:13 +0200 Subject: [PATCH 1/5] Fixed persistent exact search + fixed git setup --- .../extractors/YoutubeSearchExtractor.java | 26 +++++++---- .../YoutubeSearchQueryHandlerFactory.java | 46 ++++++++++++------- 2 files changed, 47 insertions(+), 25 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java index a0be72cc8f..2991709ad9 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java @@ -7,6 +7,7 @@ import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareDesktopJsonBuilder; import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.ALL; import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.CHANNELS; +import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.EXACT; import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.PLAYLISTS; import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.VIDEOS; import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.getSearchParameter; @@ -63,7 +64,7 @@ public class YoutubeSearchExtractor extends SearchExtractor { @Nullable - private final String searchType; + private String searchType; private final boolean extractVideoResults; private final boolean extractChannelResults; private final boolean extractPlaylistResults; @@ -77,13 +78,21 @@ public YoutubeSearchExtractor(final StreamingService service, searchType = isNullOrEmpty(contentFilters) ? null : contentFilters.get(0); // Save whether we should extract video, channel and playlist results depending on the // requested search type, as YouTube returns sometimes videos inside channel search results - // If no search type is provided or ALL filter is requested, extract everything + // If no search type is provided or ALL/EXACT (without another search type) filter + // is requested, extract everything + extractVideoResults = searchType == null || ALL.equals(searchType) - || VIDEOS.equals(searchType); + || VIDEOS.equals(searchType) || EXACT.equals(searchType); extractChannelResults = searchType == null || ALL.equals(searchType) - || CHANNELS.equals(searchType); + || CHANNELS.equals(searchType) || EXACT.equals(searchType); extractPlaylistResults = searchType == null || ALL.equals(searchType) - || PLAYLISTS.equals(searchType); + || PLAYLISTS.equals(searchType) || EXACT.equals(searchType); + + // if EXACT is a content filter, it replaces the current search type + // this needs to happen AFTER the extract... params are set + if (!isNullOrEmpty(contentFilters) && contentFilters.contains(EXACT)) { + searchType = EXACT; + } } @Override @@ -92,7 +101,6 @@ public void onFetchPage(@Nonnull final Downloader downloader) throws IOException final String query = super.getSearchString(); final Localization localization = getExtractorLocalization(); final String params = getSearchParameter(searchType); - final JsonBuilder jsonBody = prepareDesktopJsonBuilder(localization, getExtractorContentCountry()) .value("query", query); @@ -199,9 +207,9 @@ public InfoItemsPage getPage(final Page page) throws IOException, // @formatter:off final byte[] json = JsonWriter.string(prepareDesktopJsonBuilder(localization, - getExtractorContentCountry()) - .value("continuation", page.getId()) - .done()) + getExtractorContentCountry()) + .value("continuation", page.getId()) + .done()) .getBytes(StandardCharsets.UTF_8); // @formatter:on diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeSearchQueryHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeSearchQueryHandlerFactory.java index e2d60c6a34..ff938c7654 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeSearchQueryHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeSearchQueryHandlerFactory.java @@ -16,6 +16,7 @@ public final class YoutubeSearchQueryHandlerFactory extends SearchQueryHandlerFa new YoutubeSearchQueryHandlerFactory(); public static final String ALL = "all"; + public static final String EXACT = "exact"; public static final String VIDEOS = "videos"; public static final String CHANNELS = "channels"; public static final String PLAYLISTS = "playlists"; @@ -33,14 +34,23 @@ public final class YoutubeSearchQueryHandlerFactory extends SearchQueryHandlerFa public static YoutubeSearchQueryHandlerFactory getInstance() { return INSTANCE; } - @Override public String getUrl(final String searchString, @Nonnull final List contentFilters, final String sortFilter) throws ParsingException, UnsupportedOperationException { - final String contentFilter = !contentFilters.isEmpty() ? contentFilters.get(0) : ""; + final String contentFilter; + + // EXACT takes precedence over other filters + if (contentFilters.contains(EXACT)) { + contentFilter = EXACT; + } else { + contentFilter = !contentFilters.isEmpty() ? contentFilters.get(0) : ""; + } + switch (contentFilter) { + case EXACT: + return SEARCH_URL + encodeUrlUtf8(searchString) + "&sp=QgIIAQ%253D%253D"; case VIDEOS: return SEARCH_URL + encodeUrlUtf8(searchString) + "&sp=EgIQAfABAQ%253D%253D"; case CHANNELS: @@ -56,12 +66,14 @@ public String getUrl(final String searchString, default: return SEARCH_URL + encodeUrlUtf8(searchString) + "&sp=8AEB"; } + } @Override public String[] getAvailableContentFilter() { return new String[]{ ALL, + //EXACT, VIDEOS, CHANNELS, PLAYLISTS, @@ -80,20 +92,22 @@ public static String getSearchParameter(final String contentFilter) { } switch (contentFilter) { - case VIDEOS: - return "EgIQAfABAQ%3D%3D"; - case CHANNELS: - return "EgIQAvABAQ%3D%3D"; - case PLAYLISTS: - return "EgIQA_ABAQ%3D%3D"; - case MUSIC_SONGS: - case MUSIC_VIDEOS: - case MUSIC_ALBUMS: - case MUSIC_PLAYLISTS: - case MUSIC_ARTISTS: - return ""; - default: - return "8AEB"; + case EXACT: + return "QgIIAQ%3D%3D"; + case VIDEOS: + return "EgIQAfABAQ%3D%3D"; + case CHANNELS: + return "EgIQAvABAQ%3D%3D"; + case PLAYLISTS: + return "EgIQA_ABAQ%3D%3D"; + case MUSIC_SONGS: + case MUSIC_VIDEOS: + case MUSIC_ALBUMS: + case MUSIC_PLAYLISTS: + case MUSIC_ARTISTS: + return ""; + default: + return "8AEB"; } } } From 74a500d0bb96af23d3781822ea90ee7c302888d7 Mon Sep 17 00:00:00 2001 From: Thompson3142 <115718208+Thompson3142@users.noreply.github.com> Date: Sat, 7 Sep 2024 02:41:05 +0200 Subject: [PATCH 2/5] Minor indentation fix --- .../services/youtube/extractors/YoutubeSearchExtractor.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java index 2991709ad9..354a1b8f29 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java @@ -207,9 +207,9 @@ public InfoItemsPage getPage(final Page page) throws IOException, // @formatter:off final byte[] json = JsonWriter.string(prepareDesktopJsonBuilder(localization, - getExtractorContentCountry()) - .value("continuation", page.getId()) - .done()) + getExtractorContentCountry()) + .value("continuation", page.getId()) + .done()) .getBytes(StandardCharsets.UTF_8); // @formatter:on From 26938ff22b1ab8f48f4892e8dfe464599a8a56fe Mon Sep 17 00:00:00 2001 From: Thompson3142 <115718208+Thompson3142@users.noreply.github.com> Date: Sat, 7 Sep 2024 02:42:09 +0200 Subject: [PATCH 3/5] Removed wrong whitespace --- .../services/youtube/extractors/YoutubeSearchExtractor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java index 354a1b8f29..04a108f85a 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java @@ -207,7 +207,7 @@ public InfoItemsPage getPage(final Page page) throws IOException, // @formatter:off final byte[] json = JsonWriter.string(prepareDesktopJsonBuilder(localization, - getExtractorContentCountry()) + getExtractorContentCountry()) .value("continuation", page.getId()) .done()) .getBytes(StandardCharsets.UTF_8); From 488e41d24ecdbca56dcce81116354bc854c03406 Mon Sep 17 00:00:00 2001 From: Thompson3142 <115718208+Thompson3142@users.noreply.github.com> Date: Sun, 8 Sep 2024 19:53:38 +0200 Subject: [PATCH 4/5] Cleanup + bug fix with ALL search --- .../extractors/YoutubeSearchExtractor.java | 14 +++++------ .../YoutubeSearchQueryHandlerFactory.java | 24 +++++++------------ 2 files changed, 15 insertions(+), 23 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java index 04a108f85a..41607b8acb 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java @@ -64,7 +64,8 @@ public class YoutubeSearchExtractor extends SearchExtractor { @Nullable - private String searchType; + private final String searchType; + public final boolean isExactSearch; private final boolean extractVideoResults; private final boolean extractChannelResults; private final boolean extractPlaylistResults; @@ -76,11 +77,11 @@ public YoutubeSearchExtractor(final StreamingService service, super(service, linkHandler); final List contentFilters = linkHandler.getContentFilters(); searchType = isNullOrEmpty(contentFilters) ? null : contentFilters.get(0); + // Save whether we should extract video, channel and playlist results depending on the // requested search type, as YouTube returns sometimes videos inside channel search results // If no search type is provided or ALL/EXACT (without another search type) filter // is requested, extract everything - extractVideoResults = searchType == null || ALL.equals(searchType) || VIDEOS.equals(searchType) || EXACT.equals(searchType); extractChannelResults = searchType == null || ALL.equals(searchType) @@ -88,11 +89,8 @@ public YoutubeSearchExtractor(final StreamingService service, extractPlaylistResults = searchType == null || ALL.equals(searchType) || PLAYLISTS.equals(searchType) || EXACT.equals(searchType); - // if EXACT is a content filter, it replaces the current search type - // this needs to happen AFTER the extract... params are set - if (!isNullOrEmpty(contentFilters) && contentFilters.contains(EXACT)) { - searchType = EXACT; - } + // If EXACT is NOT the search type this is needed to filter for a content filter + EXACT + isExactSearch = !isNullOrEmpty(contentFilters) && contentFilters.contains(EXACT); } @Override @@ -100,7 +98,7 @@ public void onFetchPage(@Nonnull final Downloader downloader) throws IOException ExtractionException { final String query = super.getSearchString(); final Localization localization = getExtractorLocalization(); - final String params = getSearchParameter(searchType); + final String params = getSearchParameter(searchType, isExactSearch); final JsonBuilder jsonBody = prepareDesktopJsonBuilder(localization, getExtractorContentCountry()) .value("query", query); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeSearchQueryHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeSearchQueryHandlerFactory.java index ff938c7654..94ac3267c6 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeSearchQueryHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeSearchQueryHandlerFactory.java @@ -39,24 +39,18 @@ public String getUrl(final String searchString, @Nonnull final List contentFilters, final String sortFilter) throws ParsingException, UnsupportedOperationException { - final String contentFilter; - - // EXACT takes precedence over other filters - if (contentFilters.contains(EXACT)) { - contentFilter = EXACT; - } else { - contentFilter = !contentFilters.isEmpty() ? contentFilters.get(0) : ""; - } + final String contentFilter = !contentFilters.isEmpty() ? contentFilters.get(0) : ""; + final boolean isExactSearch = !contentFilters.isEmpty() && contentFilter.contains(EXACT); switch (contentFilter) { case EXACT: return SEARCH_URL + encodeUrlUtf8(searchString) + "&sp=QgIIAQ%253D%253D"; case VIDEOS: - return SEARCH_URL + encodeUrlUtf8(searchString) + "&sp=EgIQAfABAQ%253D%253D"; + return SEARCH_URL + encodeUrlUtf8(searchString) + (isExactSearch ? "&sp=EgIQAUICCAE%253D" : "&sp=EgIQAfABAQ%253D%253D"); case CHANNELS: - return SEARCH_URL + encodeUrlUtf8(searchString) + "&sp=EgIQAvABAQ%253D%253D"; + return SEARCH_URL + encodeUrlUtf8(searchString) + (isExactSearch ? "&sp=EgIQAkICCAE%253D" : "&sp=EgIQAvABAQ%253D%253D"); case PLAYLISTS: - return SEARCH_URL + encodeUrlUtf8(searchString) + "&sp=EgIQA_ABAQ%253D%253D"; + return SEARCH_URL + encodeUrlUtf8(searchString) + (isExactSearch ? "&sp=EgIQA0ICCAE%253D" : "&sp=EgIQA_ABAQ%253D%253D"); case MUSIC_SONGS: case MUSIC_VIDEOS: case MUSIC_ALBUMS: @@ -86,7 +80,7 @@ public String[] getAvailableContentFilter() { } @Nonnull - public static String getSearchParameter(final String contentFilter) { + public static String getSearchParameter(final String contentFilter, final boolean isExactSearch) { if (isNullOrEmpty(contentFilter)) { return "8AEB"; } @@ -95,11 +89,11 @@ public static String getSearchParameter(final String contentFilter) { case EXACT: return "QgIIAQ%3D%3D"; case VIDEOS: - return "EgIQAfABAQ%3D%3D"; + return isExactSearch ? "EgIQAUICCAE%3D" : "EgIQAfABAQ%3D%3D"; case CHANNELS: - return "EgIQAvABAQ%3D%3D"; + return isExactSearch ? "EgIQAkICCAE%3D" : "EgIQAvABAQ%3D%3D"; case PLAYLISTS: - return "EgIQA_ABAQ%3D%3D"; + return isExactSearch ? "EgIQA0ICCAE%3D" : "EgIQA_ABAQ%3D%3D"; case MUSIC_SONGS: case MUSIC_VIDEOS: case MUSIC_ALBUMS: From fa8746c7c7d7784059703554628d3a1c72541e31 Mon Sep 17 00:00:00 2001 From: Thompson3142 <115718208+Thompson3142@users.noreply.github.com> Date: Sun, 8 Sep 2024 20:05:13 +0200 Subject: [PATCH 5/5] No code changes, just visual cleanup --- .../youtube/linkHandler/YoutubeSearchQueryHandlerFactory.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeSearchQueryHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeSearchQueryHandlerFactory.java index 94ac3267c6..a961b6524d 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeSearchQueryHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeSearchQueryHandlerFactory.java @@ -60,14 +60,13 @@ public String getUrl(final String searchString, default: return SEARCH_URL + encodeUrlUtf8(searchString) + "&sp=8AEB"; } - } @Override public String[] getAvailableContentFilter() { return new String[]{ ALL, - //EXACT, + //EXACT, Not a separate content filter (yet) VIDEOS, CHANNELS, PLAYLISTS,