From 1762a527c9c9d6d552bbee6035d5a3f2397f3894 Mon Sep 17 00:00:00 2001 From: wb9688 Date: Tue, 17 Mar 2020 11:33:39 +0100 Subject: [PATCH 01/17] Add support for YouTube Music search --- .../extractors/YoutubeSearchExtractor.java | 423 +++++++++++++++++- .../linkHandler/YoutubeParsingHelper.java | 18 + .../YoutubeSearchQueryHandlerFactory.java | 41 +- 3 files changed, 450 insertions(+), 32 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java index 27a247096..dbb95f04f 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java @@ -2,23 +2,43 @@ package org.schabi.newpipe.extractor.services.youtube.extractors; import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; +import com.grack.nanojson.JsonParser; +import com.grack.nanojson.JsonParserException; +import com.grack.nanojson.JsonWriter; import org.schabi.newpipe.extractor.InfoItem; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.downloader.Downloader; +import org.schabi.newpipe.extractor.downloader.Response; +import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.exceptions.ReCaptchaException; import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandler; +import org.schabi.newpipe.extractor.localization.DateWrapper; import org.schabi.newpipe.extractor.localization.TimeAgoParser; import org.schabi.newpipe.extractor.search.InfoItemsSearchCollector; import org.schabi.newpipe.extractor.search.SearchExtractor; +import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper; +import org.schabi.newpipe.extractor.utils.Utils; import java.io.IOException; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; import javax.annotation.Nonnull; +import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.fixThumbnailUrl; import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getJsonResponse; import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getTextFromObject; +import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getUrlFromNavigationEndpoint; +import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.MUSIC_ALBUMS; +import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.MUSIC_ARTISTS; +import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.MUSIC_PLAYLISTS; +import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.MUSIC_SONGS; +import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.MUSIC_VIDEOS; /* * Created by Christian Schabesberger on 22.07.2018 @@ -49,21 +69,110 @@ public class YoutubeSearchExtractor extends SearchExtractor { @Override public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException { - final String url = getUrl() + "&pbj=1"; + if (isMusicSearch()) { + final String[] youtubeMusicKeys = YoutubeParsingHelper.getYoutubeMusicKeys(); - final JsonArray ajaxJson = getJsonResponse(url, getExtractorLocalization()); + final String url = "https://music.youtube.com/youtubei/v1/search?alt=json&key=" + youtubeMusicKeys[0]; - initialData = ajaxJson.getObject(1).getObject("response"); + String params = null; + + switch (getLinkHandler().getContentFilters().get(0)) { + case MUSIC_SONGS: + params = "Eg-KAQwIARAAGAAgACgAMABqChAEEAUQAxAKEAk%3D"; + break; + case MUSIC_VIDEOS: + params = "Eg-KAQwIABABGAAgACgAMABqChAEEAUQAxAKEAk%3D"; + break; + case MUSIC_ALBUMS: + params = "Eg-KAQwIABAAGAEgACgAMABqChAEEAUQAxAKEAk%3D"; + break; + case MUSIC_PLAYLISTS: + params = "Eg-KAQwIABAAGAAgACgBMABqChAEEAUQAxAKEAk%3D"; + break; + case MUSIC_ARTISTS: + params = "Eg-KAQwIABAAGAAgASgAMABqChAEEAUQAxAKEAk%3D"; + break; + } + + // @formatter:off + byte[] json = JsonWriter.string() + .object() + .object("context") + .object("client") + .value("clientName", "WEB_REMIX") + .value("clientVersion", youtubeMusicKeys[2]) + .value("hl", "en") + .value("gl", getExtractorContentCountry().getCountryCode()) + .array("experimentIds").end() + .value("experimentsToken", "") + .value("utcOffsetMinutes", 0) + .object("locationInfo").end() + .object("musicAppInfo").end() + .end() + .object("capabilities").end() + .object("request") + .array("internalExperimentFlags").end() + .object("sessionIndex").end() + .end() + .object("activePlayers").end() + .object("user") + .value("enableSafetyMode", false) + .end() + .end() + .value("query", getSearchString()) + .value("params", params) + .end().done().getBytes("UTF-8"); + // @formatter:on + + Map> headers = new HashMap<>(); + headers.put("X-YouTube-Client-Name", Collections.singletonList(youtubeMusicKeys[1])); + headers.put("X-YouTube-Client-Version", Collections.singletonList(youtubeMusicKeys[2])); + headers.put("Origin", Collections.singletonList("https://music.youtube.com")); + headers.put("Content-Type", Collections.singletonList("application/json")); + + Response response = getDownloader().post(url, headers, json); + + if (response.responseCode() == 404) { + throw new ContentNotAvailableException("Not found" + + " (\"" + response.responseCode() + " " + response.responseMessage() + "\")"); + } + + final String responseBody = response.responseBody(); + if (responseBody.length() < 50) { // ensure to have a valid response + throw new ParsingException("JSON response is too short"); + } + + final String responseContentType = response.getHeader("Content-Type"); + if (responseContentType != null && responseContentType.toLowerCase().contains("text/html")) { + throw new ParsingException("Got HTML document, expected JSON response" + + " (latest url was: \"" + response.latestUrl() + "\")"); + } + + try { + initialData = JsonParser.object().from(responseBody); + } catch (JsonParserException e) { + throw new ParsingException("Could not parse JSON", e); + } + } else { + final String url = getUrl() + "&pbj=1"; + + final JsonArray ajaxJson = getJsonResponse(url, getExtractorLocalization()); + + initialData = ajaxJson.getObject(1).getObject("response"); + } } @Nonnull @Override public String getUrl() throws ParsingException { + if (isMusicSearch()) return super.getUrl(); return super.getUrl() + "&gl=" + getExtractorContentCountry().getCountryCode(); } @Override public String getSearchSuggestion() throws ParsingException { + if (isMusicSearch()) return ""; + JsonObject showingResultsForRenderer = initialData.getObject("contents") .getObject("twoColumnSearchResultsRenderer").getObject("primaryContents") .getObject("sectionListRenderer").getArray("contents").getObject(0) @@ -78,23 +187,36 @@ public class YoutubeSearchExtractor extends SearchExtractor { @Nonnull @Override - public InfoItemsPage getInitialPage() throws ExtractionException { + public InfoItemsPage getInitialPage() throws ExtractionException, IOException { final InfoItemsSearchCollector collector = new InfoItemsSearchCollector(getServiceId()); - JsonArray sections = initialData.getObject("contents").getObject("twoColumnSearchResultsRenderer") - .getObject("primaryContents").getObject("sectionListRenderer").getArray("contents"); - for (Object section : sections) { - collectStreamsFrom(collector, ((JsonObject) section).getObject("itemSectionRenderer").getArray("contents")); + if (isMusicSearch()) { + JsonArray sections = initialData.getObject("contents").getObject("sectionListRenderer") + .getArray("contents").getObject(0).getObject("musicShelfRenderer").getArray("contents"); + + collectMusicStreamsFrom(collector, sections); + } else { + JsonArray sections = initialData.getObject("contents").getObject("twoColumnSearchResultsRenderer") + .getObject("primaryContents").getObject("sectionListRenderer").getArray("contents"); + + for (Object section : sections) { + collectStreamsFrom(collector, ((JsonObject) section).getObject("itemSectionRenderer").getArray("contents")); + } } return new InfoItemsPage<>(collector, getNextPageUrl()); } @Override - public String getNextPageUrl() throws ExtractionException { - return getNextPageUrlFrom(initialData.getObject("contents").getObject("twoColumnSearchResultsRenderer") - .getObject("primaryContents").getObject("sectionListRenderer").getArray("contents") - .getObject(0).getObject("itemSectionRenderer").getArray("continuations")); + public String getNextPageUrl() throws ExtractionException, IOException { + if (isMusicSearch()) { + return getNextPageUrlFrom(initialData.getObject("contents").getObject("sectionListRenderer") + .getArray("contents").getObject(0).getObject("musicShelfRenderer").getArray("continuations")); + } else { + return getNextPageUrlFrom(initialData.getObject("contents").getObject("twoColumnSearchResultsRenderer") + .getObject("primaryContents").getObject("sectionListRenderer").getArray("contents") + .getObject(0).getObject("itemSectionRenderer").getArray("continuations")); + } } @Override @@ -104,19 +226,97 @@ public class YoutubeSearchExtractor extends SearchExtractor { } final InfoItemsSearchCollector collector = new InfoItemsSearchCollector(getServiceId()); - final JsonArray ajaxJson = getJsonResponse(pageUrl, getExtractorLocalization()); - JsonObject itemSectionRenderer = ajaxJson.getObject(1).getObject("response") - .getObject("continuationContents").getObject("itemSectionContinuation"); + JsonArray continuations; - collectStreamsFrom(collector, itemSectionRenderer.getArray("contents")); + if (isMusicSearch()) { + final String[] youtubeMusicKeys = YoutubeParsingHelper.getYoutubeMusicKeys(); - return new InfoItemsPage<>(collector, getNextPageUrlFrom(itemSectionRenderer.getArray("continuations"))); + // @formatter:off + byte[] json = JsonWriter.string() + .object() + .object("context") + .object("client") + .value("clientName", "WEB_REMIX") + .value("clientVersion", youtubeMusicKeys[2]) + .value("hl", "en") + .value("gl", getExtractorContentCountry().getCountryCode()) + .array("experimentIds").end() + .value("experimentsToken", "") + .value("utcOffsetMinutes", 0) + .object("locationInfo").end() + .object("musicAppInfo").end() + .end() + .object("capabilities").end() + .object("request") + .array("internalExperimentFlags").end() + .object("sessionIndex").end() + .end() + .object("activePlayers").end() + .object("user") + .value("enableSafetyMode", false) + .end() + .end() + .end().done().getBytes("UTF-8"); + // @formatter:on + + Map> headers = new HashMap<>(); + headers.put("X-YouTube-Client-Name", Collections.singletonList(youtubeMusicKeys[1])); + headers.put("X-YouTube-Client-Version", Collections.singletonList(youtubeMusicKeys[2])); + headers.put("Origin", Collections.singletonList("https://music.youtube.com")); + headers.put("Content-Type", Collections.singletonList("application/json")); + + Response response = getDownloader().post(pageUrl, headers, json); + + if (response.responseCode() == 404) { + throw new ContentNotAvailableException("Not found" + + " (\"" + response.responseCode() + " " + response.responseMessage() + "\")"); + } + + final String responseBody = response.responseBody(); + if (responseBody.length() < 50) { // ensure to have a valid response + throw new ParsingException("JSON response is too short"); + } + + final String responseContentType = response.getHeader("Content-Type"); + if (responseContentType != null && responseContentType.toLowerCase().contains("text/html")) { + throw new ParsingException("Got HTML document, expected JSON response" + + " (latest url was: \"" + response.latestUrl() + "\")"); + } + + final JsonObject ajaxJson; + try { + ajaxJson = JsonParser.object().from(responseBody); + } catch (JsonParserException e) { + throw new ParsingException("Could not parse JSON", e); + } + + if (ajaxJson.getObject("continuationContents") == null) return new InfoItemsPage<>(collector, null); + + JsonObject musicShelfContinuation = ajaxJson.getObject("continuationContents").getObject("musicShelfContinuation"); + + collectMusicStreamsFrom(collector, musicShelfContinuation.getArray("contents")); + continuations = musicShelfContinuation.getArray("continuations"); + } else { + final JsonArray ajaxJson = getJsonResponse(pageUrl, getExtractorLocalization()); + + JsonObject itemSectionRenderer = ajaxJson.getObject(1).getObject("response") + .getObject("continuationContents").getObject("itemSectionContinuation"); + + collectStreamsFrom(collector, itemSectionRenderer.getArray("contents")); + continuations = itemSectionRenderer.getArray("continuations"); + } + + return new InfoItemsPage<>(collector, getNextPageUrlFrom(continuations)); + } + + private boolean isMusicSearch() { + final List contentFilters = getLinkHandler().getContentFilters(); + if (contentFilters.size() > 0 && contentFilters.get(0).startsWith("music_")) return true; + return false; } private void collectStreamsFrom(InfoItemsSearchCollector collector, JsonArray videos) throws NothingFoundException, ParsingException { - collector.reset(); - final TimeAgoParser timeAgoParser = getTimeAgoParser(); for (Object item : videos) { @@ -133,7 +333,180 @@ public class YoutubeSearchExtractor extends SearchExtractor { } } - private String getNextPageUrlFrom(JsonArray continuations) throws ParsingException { + private void collectMusicStreamsFrom(InfoItemsSearchCollector collector, JsonArray videos) { + final TimeAgoParser timeAgoParser = getTimeAgoParser(); + + for (Object item : videos) { + final JsonObject info = ((JsonObject) item).getObject("musicResponsiveListItemRenderer"); + if (info != null) { + final String searchType = getLinkHandler().getContentFilters().get(0); + if (searchType.equals(MUSIC_SONGS) || searchType.equals(MUSIC_VIDEOS)) { + collector.commit(new YoutubeStreamInfoItemExtractor(info, timeAgoParser) { + @Override + public String getUrl() throws ParsingException { + String url = getUrlFromNavigationEndpoint(info.getObject("doubleTapCommand")); + if (url != null && !url.isEmpty()) return url; + throw new ParsingException("Could not get url"); + } + + @Override + public String getName() throws ParsingException { + String name = getTextFromObject(info.getArray("flexColumns").getObject(0) + .getObject("musicResponsiveListItemFlexColumnRenderer").getObject("text")); + if (name != null && !name.isEmpty()) return name; + throw new ParsingException("Could not get name"); + } + + @Override + public long getDuration() throws ParsingException { + String duration = getTextFromObject(info.getArray("flexColumns").getObject(3) + .getObject("musicResponsiveListItemFlexColumnRenderer").getObject("text")); + if (duration != null && !duration.isEmpty()) + return YoutubeParsingHelper.parseDurationString(duration); + throw new ParsingException("Could not get duration"); + } + + @Override + public String getUploaderName() throws ParsingException { + String name = getTextFromObject(info.getArray("flexColumns").getObject(1) + .getObject("musicResponsiveListItemFlexColumnRenderer").getObject("text")); + if (name != null && !name.isEmpty()) return name; + throw new ParsingException("Could not get uploader name"); + } + + @Override + public String getTextualUploadDate() { + return null; + } + + @Override + public DateWrapper getUploadDate() { + return null; + } + + @Override + public long getViewCount() throws ParsingException { + if (searchType.equals(MUSIC_SONGS)) return -1; + String viewCount = getTextFromObject(info.getArray("flexColumns").getObject(2) + .getObject("musicResponsiveListItemFlexColumnRenderer").getObject("text")); + if (viewCount != null && !viewCount.isEmpty()) return Utils.mixedNumberWordToLong(viewCount); + throw new ParsingException("Could not get view count"); + } + + @Override + public String getThumbnailUrl() throws ParsingException { + try { + // TODO: Don't simply get the first item, but look at all thumbnails and their resolution + return fixThumbnailUrl(info.getObject("thumbnail").getObject("musicThumbnailRenderer") + .getObject("thumbnail").getArray("thumbnails").getObject(0).getString("url")); + } catch (Exception e) { + throw new ParsingException("Could not get thumbnail url", e); + } + } + }); + } else if (searchType.equals(MUSIC_ARTISTS)) { + collector.commit(new YoutubeChannelInfoItemExtractor(info) { + @Override + public String getThumbnailUrl() throws ParsingException { + try { + // TODO: Don't simply get the first item, but look at all thumbnails and their resolution + return fixThumbnailUrl(info.getObject("thumbnail").getObject("musicThumbnailRenderer") + .getObject("thumbnail").getArray("thumbnails").getObject(0).getString("url")); + } catch (Exception e) { + throw new ParsingException("Could not get thumbnail url", e); + } + } + + @Override + public String getName() throws ParsingException { + String name = getTextFromObject(info.getArray("flexColumns").getObject(0) + .getObject("musicResponsiveListItemFlexColumnRenderer").getObject("text")); + if (name != null && !name.isEmpty()) return name; + throw new ParsingException("Could not get name"); + } + + @Override + public String getUrl() throws ParsingException { + String url = getUrlFromNavigationEndpoint(info.getObject("navigationEndpoint")); + if (url != null && !url.isEmpty()) return url; + throw new ParsingException("Could not get url"); + } + + @Override + public long getSubscriberCount() throws ParsingException { + String viewCount = getTextFromObject(info.getArray("flexColumns").getObject(2) + .getObject("musicResponsiveListItemFlexColumnRenderer").getObject("text")); + if (viewCount != null && !viewCount.isEmpty()) return Utils.mixedNumberWordToLong(viewCount); + throw new ParsingException("Could not get subscriber count"); + } + + @Override + public long getStreamCount() { + return -1; + } + + @Override + public String getDescription() { + return null; + } + }); + } else if (searchType.equals(MUSIC_ALBUMS) || searchType.equals(MUSIC_PLAYLISTS)) { + collector.commit(new YoutubePlaylistInfoItemExtractor(info) { + @Override + public String getThumbnailUrl() throws ParsingException { + try { + // TODO: Don't simply get the first item, but look at all thumbnails and their resolution + return fixThumbnailUrl(info.getObject("thumbnail").getObject("musicThumbnailRenderer") + .getObject("thumbnail").getArray("thumbnails").getObject(0).getString("url")); + } catch (Exception e) { + throw new ParsingException("Could not get thumbnail url", e); + } + } + + @Override + public String getName() throws ParsingException { + String name = getTextFromObject(info.getArray("flexColumns").getObject(0) + .getObject("musicResponsiveListItemFlexColumnRenderer").getObject("text")); + if (name != null && !name.isEmpty()) return name; + throw new ParsingException("Could not get name"); + } + + @Override + public String getUrl() throws ParsingException { + String url = getUrlFromNavigationEndpoint(info.getObject("doubleTapCommand")); + if (url != null && !url.isEmpty()) return url; + throw new ParsingException("Could not get url"); + } + + @Override + public String getUploaderName() throws ParsingException { + String name; + if (searchType.equals(MUSIC_ALBUMS)) { + name = getTextFromObject(info.getArray("flexColumns").getObject(2) + .getObject("musicResponsiveListItemFlexColumnRenderer").getObject("text")); + } else { + name = getTextFromObject(info.getArray("flexColumns").getObject(1) + .getObject("musicResponsiveListItemFlexColumnRenderer").getObject("text")); + } + if (name != null && !name.isEmpty()) return name; + throw new ParsingException("Could not get uploader name"); + } + + @Override + public long getStreamCount() throws ParsingException { + if (searchType.equals(MUSIC_ALBUMS)) return -1; + String count = getTextFromObject(info.getArray("flexColumns").getObject(2) + .getObject("musicResponsiveListItemFlexColumnRenderer").getObject("text")); + if (count != null && !count.isEmpty()) return Long.parseLong(Utils.removeNonDigitCharacters(count)); + throw new ParsingException("Could not get count"); + } + }); + } + } + } + } + + private String getNextPageUrlFrom(JsonArray continuations) throws ParsingException, IOException, ReCaptchaException { if (continuations == null) { return ""; } @@ -141,7 +514,13 @@ public class YoutubeSearchExtractor extends SearchExtractor { JsonObject nextContinuationData = continuations.getObject(0).getObject("nextContinuationData"); String continuation = nextContinuationData.getString("continuation"); String clickTrackingParams = nextContinuationData.getString("clickTrackingParams"); - return getUrl() + "&pbj=1&ctoken=" + continuation + "&continuation=" + continuation - + "&itct=" + clickTrackingParams; + + if (isMusicSearch()) { + return "https://music.youtube.com/youtubei/v1/search?ctoken=" + continuation + "&continuation=" + continuation + + "&itct=" + clickTrackingParams + "&alt=json&key=" + YoutubeParsingHelper.getYoutubeMusicKeys()[0]; + } else { + return getUrl() + "&pbj=1&ctoken=" + continuation + "&continuation=" + continuation + + "&itct=" + clickTrackingParams; + } } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java index 965325550..8847466fc 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java @@ -62,6 +62,8 @@ public class YoutubeParsingHelper { private static final String HARDCODED_CLIENT_VERSION = "2.20200214.04.00"; private static String clientVersion; + private static String[] youtubeMusicKeys; + private static final String FEED_BASE_CHANNEL_ID = "https://www.youtube.com/feeds/videos.xml?channel_id="; private static final String FEED_BASE_USER = "https://www.youtube.com/feeds/videos.xml?user="; @@ -259,6 +261,19 @@ public class YoutubeParsingHelper { throw new ParsingException("Could not get client version"); } + public static String[] getYoutubeMusicKeys() throws IOException, ReCaptchaException, Parser.RegexException { + if (youtubeMusicKeys != null && youtubeMusicKeys.length == 3) return youtubeMusicKeys; + + final String url = "https://music.youtube.com/"; + final String html = getDownloader().get(url).responseBody(); + + final String key = Parser.matchGroup1("INNERTUBE_API_KEY\":\"([0-9a-zA-Z_-]+?)\"", html); + final String clientName = Parser.matchGroup1("INNERTUBE_CONTEXT_CLIENT_NAME\":([0-9]+?),", html); + final String clientVersion = Parser.matchGroup1("INNERTUBE_CLIENT_VERSION\":\"([0-9\\.]+?)\"", html); + + return youtubeMusicKeys = new String[]{key, clientName, clientVersion}; + } + public static String getUrlFromNavigationEndpoint(JsonObject navigationEndpoint) throws ParsingException { if (navigationEndpoint.getObject("urlEndpoint") != null) { String internUrl = navigationEndpoint.getObject("urlEndpoint").getString("url"); @@ -303,6 +318,9 @@ public class YoutubeParsingHelper { if (navigationEndpoint.getObject("watchEndpoint").has("startTimeSeconds")) url.append("&t=").append(navigationEndpoint.getObject("watchEndpoint").getInt("startTimeSeconds")); return url.toString(); + } else if (navigationEndpoint.getObject("watchPlaylistEndpoint") != null) { + return "https://www.youtube.com/playlist?list=" + + navigationEndpoint.getObject("watchPlaylistEndpoint").getString("playlistId"); } return null; } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeSearchQueryHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeSearchQueryHandlerFactory.java index 13481b345..57829eab0 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeSearchQueryHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeSearchQueryHandlerFactory.java @@ -8,13 +8,21 @@ import java.net.URLEncoder; import java.util.List; public class YoutubeSearchQueryHandlerFactory extends SearchQueryHandlerFactory { - public static final String CHARSET_UTF_8 = "UTF-8"; + public static final String ALL = "all"; public static final String VIDEOS = "videos"; public static final String CHANNELS = "channels"; public static final String PLAYLISTS = "playlists"; - public static final String ALL = "all"; + + public static final String MUSIC_SONGS = "music_songs"; + public static final String MUSIC_VIDEOS = "music_videos"; + public static final String MUSIC_ALBUMS = "music_albums"; + public static final String MUSIC_PLAYLISTS = "music_playlists"; + public static final String MUSIC_ARTISTS = "music_artists"; + + private static final String SEARCH_URL = "https://www.youtube.com/results?search_query="; + private static final String MUSIC_SEARCH_URL = "https://music.youtube.com/search?q="; public static YoutubeSearchQueryHandlerFactory getInstance() { return new YoutubeSearchQueryHandlerFactory(); @@ -23,20 +31,27 @@ public class YoutubeSearchQueryHandlerFactory extends SearchQueryHandlerFactory @Override public String getUrl(String searchString, List contentFilters, String sortFilter) throws ParsingException { try { - final String url = "https://www.youtube.com/results" - + "?search_query=" + URLEncoder.encode(searchString, CHARSET_UTF_8); - if (contentFilters.size() > 0) { switch (contentFilters.get(0)) { - case VIDEOS: return url + "&sp=EgIQAQ%253D%253D"; - case CHANNELS: return url + "&sp=EgIQAg%253D%253D"; - case PLAYLISTS: return url + "&sp=EgIQAw%253D%253D"; case ALL: default: + break; + case VIDEOS: + return SEARCH_URL + URLEncoder.encode(searchString, CHARSET_UTF_8) + "&sp=EgIQAQ%253D%253D"; + case CHANNELS: + return SEARCH_URL + URLEncoder.encode(searchString, CHARSET_UTF_8) + "&sp=EgIQAg%253D%253D"; + case PLAYLISTS: + return SEARCH_URL + URLEncoder.encode(searchString, CHARSET_UTF_8) + "&sp=EgIQAw%253D%253D"; + case MUSIC_SONGS: + case MUSIC_VIDEOS: + case MUSIC_ALBUMS: + case MUSIC_PLAYLISTS: + case MUSIC_ARTISTS: + return MUSIC_SEARCH_URL + URLEncoder.encode(searchString, CHARSET_UTF_8); } } - return url; + return SEARCH_URL + URLEncoder.encode(searchString, CHARSET_UTF_8); } catch (UnsupportedEncodingException e) { throw new ParsingException("Could not encode query", e); } @@ -48,6 +63,12 @@ public class YoutubeSearchQueryHandlerFactory extends SearchQueryHandlerFactory ALL, VIDEOS, CHANNELS, - PLAYLISTS}; + PLAYLISTS, + MUSIC_SONGS, + MUSIC_VIDEOS, + MUSIC_ALBUMS, + MUSIC_PLAYLISTS, + MUSIC_ARTISTS + }; } } From 4ddbdf0aee23b67fb5dd2e806c174d960dbd7ffc Mon Sep 17 00:00:00 2001 From: wb9688 Date: Tue, 17 Mar 2020 13:06:25 +0100 Subject: [PATCH 02/17] Disable artist search for now --- .../youtube/linkHandler/YoutubeSearchQueryHandlerFactory.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeSearchQueryHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeSearchQueryHandlerFactory.java index 57829eab0..7acf7d714 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeSearchQueryHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeSearchQueryHandlerFactory.java @@ -68,7 +68,7 @@ public class YoutubeSearchQueryHandlerFactory extends SearchQueryHandlerFactory MUSIC_VIDEOS, MUSIC_ALBUMS, MUSIC_PLAYLISTS, - MUSIC_ARTISTS +// MUSIC_ARTISTS }; } } From fb9b9691b76ddabd4cc3d5c02231fce28e6b8f17 Mon Sep 17 00:00:00 2001 From: wb9688 Date: Fri, 20 Mar 2020 11:05:19 +0100 Subject: [PATCH 03/17] Improve getYoutubeMusicKeys() --- .../linkHandler/YoutubeParsingHelper.java | 83 ++++++++++++++++--- 1 file changed, 70 insertions(+), 13 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java index 8847466fc..7fd814b9d 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java @@ -5,6 +5,8 @@ import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonParser; import com.grack.nanojson.JsonParserException; +import com.grack.nanojson.JsonWriter; + import org.jsoup.Jsoup; import org.jsoup.nodes.Document; import org.schabi.newpipe.extractor.downloader.Response; @@ -62,6 +64,7 @@ public class YoutubeParsingHelper { private static final String HARDCODED_CLIENT_VERSION = "2.20200214.04.00"; private static String clientVersion; + private static final String[] HARDCODED_YOUTUBE_MUSIC_KEYS = {"AIzaSyC9XL3ZjWddXya6X74dJoCTL-WEYFDNX30", "67", "0.1"}; private static String[] youtubeMusicKeys; private static final String FEED_BASE_CHANNEL_ID = "https://www.youtube.com/feeds/videos.xml?channel_id="; @@ -198,11 +201,7 @@ public class YoutubeParsingHelper { */ public static String getClientVersion() throws IOException, ExtractionException { if (clientVersion != null && !clientVersion.isEmpty()) return clientVersion; - - if (isHardcodedClientVersionValid()) { - clientVersion = HARDCODED_CLIENT_VERSION; - return clientVersion; - } + if (isHardcodedClientVersionValid()) return clientVersion = HARDCODED_CLIENT_VERSION; final String url = "https://www.youtube.com/results?search_query=test"; final String html = getDownloader().get(url).responseBody(); @@ -219,8 +218,7 @@ public class YoutubeParsingHelper { JsonObject p = (JsonObject) param; String key = p.getString("key"); if (key != null && key.equals("cver")) { - clientVersion = p.getString("value"); - return clientVersion; + return clientVersion = p.getString("value"); } } } else if (s.getString("service").equals("ECATCHER")) { @@ -246,30 +244,89 @@ public class YoutubeParsingHelper { try { contextClientVersion = Parser.matchGroup1(pattern, html); if (contextClientVersion != null && !contextClientVersion.isEmpty()) { - clientVersion = contextClientVersion; - return clientVersion; + return clientVersion = contextClientVersion; } } catch (Exception ignored) { } } if (shortClientVersion != null) { - clientVersion = shortClientVersion; - return clientVersion; + return clientVersion = shortClientVersion; } throw new ParsingException("Could not get client version"); } + public static boolean areHardcodedYoutubeMusicKeysValid() throws IOException, ReCaptchaException { + final String url = "https://music.youtube.com/youtubei/v1/search?alt=json&key=" + HARDCODED_YOUTUBE_MUSIC_KEYS[0]; + + // @formatter:off + byte[] json = JsonWriter.string() + .object() + .object("context") + .object("client") + .value("clientName", "WEB_REMIX") + .value("clientVersion", HARDCODED_YOUTUBE_MUSIC_KEYS[2]) + .value("hl", "en") + .value("gl", "GB") + .array("experimentIds").end() + .value("experimentsToken", "") + .value("utcOffsetMinutes", 0) + .object("locationInfo").end() + .object("musicAppInfo").end() + .end() + .object("capabilities").end() + .object("request") + .array("internalExperimentFlags").end() + .object("sessionIndex").end() + .end() + .object("activePlayers").end() + .object("user") + .value("enableSafetyMode", false) + .end() + .end() + .value("query", "test") + .value("params", "Eg-KAQwIARAAGAAgACgAMABqChAEEAUQAxAKEAk%3D") + .end().done().getBytes("UTF-8"); + // @formatter:on + + Map> headers = new HashMap<>(); + headers.put("X-YouTube-Client-Name", Collections.singletonList(HARDCODED_YOUTUBE_MUSIC_KEYS[1])); + headers.put("X-YouTube-Client-Version", Collections.singletonList(HARDCODED_YOUTUBE_MUSIC_KEYS[2])); + headers.put("Origin", Collections.singletonList("https://music.youtube.com")); + headers.put("Content-Type", Collections.singletonList("application/json")); + + String response = getDownloader().post(url, headers, json).responseBody(); + + return response.length() > 50; // ensure to have a valid response + } + public static String[] getYoutubeMusicKeys() throws IOException, ReCaptchaException, Parser.RegexException { if (youtubeMusicKeys != null && youtubeMusicKeys.length == 3) return youtubeMusicKeys; + if (areHardcodedYoutubeMusicKeysValid()) return youtubeMusicKeys = HARDCODED_YOUTUBE_MUSIC_KEYS; final String url = "https://music.youtube.com/"; final String html = getDownloader().get(url).responseBody(); - final String key = Parser.matchGroup1("INNERTUBE_API_KEY\":\"([0-9a-zA-Z_-]+?)\"", html); + String key; + try { + key = Parser.matchGroup1("INNERTUBE_API_KEY\":\"([0-9a-zA-Z_-]+?)\"", html); + } catch (Parser.RegexException e) { + key = Parser.matchGroup1("innertube_api_key\":\"([0-9a-zA-Z_-]+?)\"", html); + } + final String clientName = Parser.matchGroup1("INNERTUBE_CONTEXT_CLIENT_NAME\":([0-9]+?),", html); - final String clientVersion = Parser.matchGroup1("INNERTUBE_CLIENT_VERSION\":\"([0-9\\.]+?)\"", html); + + String clientVersion; + try { + clientVersion = Parser.matchGroup1("INNERTUBE_CONTEXT_CLIENT_VERSION\":\"([0-9\\.]+?)\"", html); + } catch (Parser.RegexException e) { + try { + clientVersion = Parser.matchGroup1("INNERTUBE_CLIENT_VERSION\":\"([0-9\\.]+?)\"", html); + } catch (Parser.RegexException ee) { + clientVersion = Parser.matchGroup1("innertube_context_client_version\":\"([0-9\\.]+?)\"", html); + } + } return youtubeMusicKeys = new String[]{key, clientName, clientVersion}; } From 2b9b2a78e8128f1730456c83f3168a60580de4e9 Mon Sep 17 00:00:00 2001 From: wb9688 Date: Fri, 20 Mar 2020 11:17:12 +0100 Subject: [PATCH 04/17] Handle 100+ items in playlist --- .../extractor/playlist/PlaylistExtractor.java | 3 +++ .../youtube/extractors/YoutubeSearchExtractor.java | 12 ++++++++++-- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistExtractor.java index 2a901dbf0..06330de05 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistExtractor.java @@ -7,6 +7,9 @@ import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; import org.schabi.newpipe.extractor.stream.StreamInfoItem; public abstract class PlaylistExtractor extends ListExtractor { + public final static long UNKNOWN_ITEMS = -1; + public final static long INFINITE_ITEMS = -2; + public final static long MORE_THAN_100_ITEMS = -3; public PlaylistExtractor(StreamingService service, ListLinkHandler linkHandler) { super(service, linkHandler); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java index dbb95f04f..0108e3303 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java @@ -30,6 +30,8 @@ import java.util.Map; import javax.annotation.Nonnull; +import static org.schabi.newpipe.extractor.playlist.PlaylistExtractor.MORE_THAN_100_ITEMS; +import static org.schabi.newpipe.extractor.playlist.PlaylistExtractor.UNKNOWN_ITEMS; import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.fixThumbnailUrl; import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getJsonResponse; import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getTextFromObject; @@ -494,10 +496,16 @@ public class YoutubeSearchExtractor extends SearchExtractor { @Override public long getStreamCount() throws ParsingException { - if (searchType.equals(MUSIC_ALBUMS)) return -1; + if (searchType.equals(MUSIC_ALBUMS)) return UNKNOWN_ITEMS; String count = getTextFromObject(info.getArray("flexColumns").getObject(2) .getObject("musicResponsiveListItemFlexColumnRenderer").getObject("text")); - if (count != null && !count.isEmpty()) return Long.parseLong(Utils.removeNonDigitCharacters(count)); + if (count != null && !count.isEmpty()) { + if (count.contains("100+")) { + return MORE_THAN_100_ITEMS; + } else { + return Long.parseLong(Utils.removeNonDigitCharacters(count)); + } + } throw new ParsingException("Could not get count"); } }); From eb485244119c4338d32dc1468ae9dcf98b81058a Mon Sep 17 00:00:00 2001 From: wb9688 Date: Fri, 20 Mar 2020 12:23:58 +0100 Subject: [PATCH 05/17] Add tests for YouTube Music search --- .../youtube/YoutubeParsingHelperTest.java | 6 ++ .../YoutubeSearchExtractorMusicTest.java | 79 +++++++++++++++++++ .../youtube/search/YoutubeSearchQHTest.java | 18 ++++- 3 files changed, 102 insertions(+), 1 deletion(-) create mode 100644 extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorMusicTest.java diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelperTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelperTest.java index 87dbbd750..669bc3292 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelperTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelperTest.java @@ -22,4 +22,10 @@ public class YoutubeParsingHelperTest { assertTrue("Hardcoded client version is not valid anymore", YoutubeParsingHelper.isHardcodedClientVersionValid()); } + + @Test + public void testAreHardcodedYoutubeMusicKeysValid() throws IOException, ExtractionException { + assertTrue("Hardcoded YouTube Music keys are not valid anymore", + YoutubeParsingHelper.areHardcodedYoutubeMusicKeysValid()); + } } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorMusicTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorMusicTest.java new file mode 100644 index 000000000..aaf6105c4 --- /dev/null +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorMusicTest.java @@ -0,0 +1,79 @@ +package org.schabi.newpipe.extractor.services.youtube.search; + +import org.junit.BeforeClass; +import org.junit.Test; +import org.schabi.newpipe.DownloaderTestImpl; +import org.schabi.newpipe.extractor.InfoItem; +import org.schabi.newpipe.extractor.ListExtractor; +import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeSearchExtractor; +import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory; + +import java.net.URL; +import java.net.URLDecoder; +import java.util.LinkedHashMap; +import java.util.Map; + +import static java.util.Arrays.asList; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.schabi.newpipe.extractor.ServiceList.YouTube; + +public class YoutubeSearchExtractorMusicTest extends YoutubeSearchExtractorBaseTest { + @BeforeClass + public static void setUpClass() throws Exception { + NewPipe.init(DownloaderTestImpl.getInstance()); + extractor = (YoutubeSearchExtractor) YouTube.getSearchExtractor("mocromaniac", + asList(YoutubeSearchQueryHandlerFactory.MUSIC_SONGS), null); + extractor.fetchPage(); + itemsPage = extractor.getInitialPage(); + } + + @Test + public void testGetSecondPage() throws Exception { + YoutubeSearchExtractor secondExtractor = (YoutubeSearchExtractor) YouTube.getSearchExtractor("mocromaniac", + asList(YoutubeSearchQueryHandlerFactory.MUSIC_SONGS), null); + ListExtractor.InfoItemsPage secondPage = secondExtractor.getPage(itemsPage.getNextPageUrl()); + assertTrue(Integer.toString(secondPage.getItems().size()), + secondPage.getItems().size() > 10); + + // check if its the same result + boolean equals = true; + for (int i = 0; i < secondPage.getItems().size() + && i < itemsPage.getItems().size(); i++) { + if (!secondPage.getItems().get(i).getUrl().equals( + itemsPage.getItems().get(i).getUrl())) { + equals = false; + } + } + assertFalse("First and second page are equal", equals); + } + + @Override + @Test + public void testUrl() throws Exception { + assertTrue(extractor.getUrl(), extractor.getUrl().startsWith("https://music.youtube.com/search?q=")); + } + + @Test + public void testGetSecondPageUrl() throws Exception { + URL url = new URL(extractor.getNextPageUrl()); + + assertEquals(url.getHost(), "music.youtube.com"); + assertEquals(url.getPath(), "/youtubei/v1/search"); + + Map queryPairs = new LinkedHashMap<>(); + for (String queryPair : url.getQuery().split("&")) { + int index = queryPair.indexOf("="); + queryPairs.put(URLDecoder.decode(queryPair.substring(0, index), "UTF-8"), + URLDecoder.decode(queryPair.substring(index + 1), "UTF-8")); + } + + assertEquals(queryPairs.get("ctoken"), queryPairs.get("continuation")); + assertTrue(queryPairs.get("continuation").length() > 5); + assertTrue(queryPairs.get("itct").length() > 5); + assertEquals("json", queryPairs.get("alt")); + assertTrue(queryPairs.get("key").length() > 5); + } +} diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchQHTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchQHTest.java index fc6af4c4d..b558ad88d 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchQHTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchQHTest.java @@ -16,6 +16,12 @@ public class YoutubeSearchQHTest { assertEquals("https://www.youtube.com/results?search_query=Poifj%26jaijf", YouTube.getSearchQHFactory().fromQuery("Poifj&jaijf").getUrl()); assertEquals("https://www.youtube.com/results?search_query=G%C3%BCl%C3%BCm", YouTube.getSearchQHFactory().fromQuery("Gülüm").getUrl()); assertEquals("https://www.youtube.com/results?search_query=%3Fj%24%29H%C2%A7B", YouTube.getSearchQHFactory().fromQuery("?j$)H§B").getUrl()); + + assertEquals("https://music.youtube.com/search?q=asdf", YouTube.getSearchQHFactory().fromQuery("asdf", asList(new String[]{MUSIC_SONGS}), "").getUrl()); + assertEquals("https://music.youtube.com/search?q=hans", YouTube.getSearchQHFactory().fromQuery("hans", asList(new String[]{MUSIC_SONGS}), "").getUrl()); + assertEquals("https://music.youtube.com/search?q=Poifj%26jaijf", YouTube.getSearchQHFactory().fromQuery("Poifj&jaijf", asList(new String[]{MUSIC_SONGS}), "").getUrl()); + assertEquals("https://music.youtube.com/search?q=G%C3%BCl%C3%BCm", YouTube.getSearchQHFactory().fromQuery("Gülüm", asList(new String[]{MUSIC_SONGS}), "").getUrl()); + assertEquals("https://music.youtube.com/search?q=%3Fj%24%29H%C2%A7B", YouTube.getSearchQHFactory().fromQuery("?j$)H§B", asList(new String[]{MUSIC_SONGS}), "").getUrl()); } @Test @@ -24,6 +30,9 @@ public class YoutubeSearchQHTest { .fromQuery("", asList(new String[]{VIDEOS}), "").getContentFilters().get(0)); assertEquals(CHANNELS, YouTube.getSearchQHFactory() .fromQuery("asdf", asList(new String[]{CHANNELS}), "").getContentFilters().get(0)); + + assertEquals(MUSIC_SONGS, YouTube.getSearchQHFactory() + .fromQuery("asdf", asList(new String[]{MUSIC_SONGS}), "").getContentFilters().get(0)); } @Test @@ -36,16 +45,23 @@ public class YoutubeSearchQHTest { .fromQuery("asdf", asList(new String[]{PLAYLISTS}), "").getUrl()); assertEquals("https://www.youtube.com/results?search_query=asdf", YouTube.getSearchQHFactory() .fromQuery("asdf", asList(new String[]{"fjiijie"}), "").getUrl()); + + assertEquals("https://music.youtube.com/search?q=asdf", YouTube.getSearchQHFactory() + .fromQuery("asdf", asList(new String[]{MUSIC_SONGS}), "").getUrl()); } @Test public void testGetAvailableContentFilter() { final String[] contentFilter = YouTube.getSearchQHFactory().getAvailableContentFilter(); - assertEquals(4, contentFilter.length); + assertEquals(8, contentFilter.length); assertEquals("all", contentFilter[0]); assertEquals("videos", contentFilter[1]); assertEquals("channels", contentFilter[2]); assertEquals("playlists", contentFilter[3]); + assertEquals("music_songs", contentFilter[4]); + assertEquals("music_videos", contentFilter[5]); + assertEquals("music_albums", contentFilter[6]); + assertEquals("music_playlists", contentFilter[7]); } @Test From c852b13d5a395f696b6d5e93a07dcc3dd26b5497 Mon Sep 17 00:00:00 2001 From: wb9688 Date: Fri, 20 Mar 2020 14:14:02 +0100 Subject: [PATCH 06/17] Add Referer header so that it also works with HttpsUrlConnection --- .../services/youtube/extractors/YoutubeSearchExtractor.java | 2 ++ .../services/youtube/linkHandler/YoutubeParsingHelper.java | 1 + 2 files changed, 3 insertions(+) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java index 0108e3303..85882e1f4 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java @@ -130,6 +130,7 @@ public class YoutubeSearchExtractor extends SearchExtractor { headers.put("X-YouTube-Client-Name", Collections.singletonList(youtubeMusicKeys[1])); headers.put("X-YouTube-Client-Version", Collections.singletonList(youtubeMusicKeys[2])); headers.put("Origin", Collections.singletonList("https://music.youtube.com")); + headers.put("Referer", Collections.singletonList("music.youtube.com")); headers.put("Content-Type", Collections.singletonList("application/json")); Response response = getDownloader().post(url, headers, json); @@ -266,6 +267,7 @@ public class YoutubeSearchExtractor extends SearchExtractor { headers.put("X-YouTube-Client-Name", Collections.singletonList(youtubeMusicKeys[1])); headers.put("X-YouTube-Client-Version", Collections.singletonList(youtubeMusicKeys[2])); headers.put("Origin", Collections.singletonList("https://music.youtube.com")); + headers.put("Referer", Collections.singletonList("music.youtube.com")); headers.put("Content-Type", Collections.singletonList("application/json")); Response response = getDownloader().post(pageUrl, headers, json); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java index 7fd814b9d..4ea6e0967 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java @@ -294,6 +294,7 @@ public class YoutubeParsingHelper { headers.put("X-YouTube-Client-Name", Collections.singletonList(HARDCODED_YOUTUBE_MUSIC_KEYS[1])); headers.put("X-YouTube-Client-Version", Collections.singletonList(HARDCODED_YOUTUBE_MUSIC_KEYS[2])); headers.put("Origin", Collections.singletonList("https://music.youtube.com")); + headers.put("Referer", Collections.singletonList("music.youtube.com")); headers.put("Content-Type", Collections.singletonList("application/json")); String response = getDownloader().post(url, headers, json).responseBody(); From dc29d87962f0f9acd97d7d0bcad6afcc67b127fd Mon Sep 17 00:00:00 2001 From: wb9688 Date: Fri, 20 Mar 2020 16:11:28 +0100 Subject: [PATCH 07/17] Extract YouTube search suggestions --- .../extractors/YoutubeSearchExtractor.java | 45 ++++++++++++------- .../YoutubeSearchExtractorMusicTest.java | 10 +++++ 2 files changed, 38 insertions(+), 17 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java index 85882e1f4..6dd574873 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java @@ -174,16 +174,19 @@ public class YoutubeSearchExtractor extends SearchExtractor { @Override public String getSearchSuggestion() throws ParsingException { - if (isMusicSearch()) return ""; - - JsonObject showingResultsForRenderer = initialData.getObject("contents") - .getObject("twoColumnSearchResultsRenderer").getObject("primaryContents") - .getObject("sectionListRenderer").getArray("contents").getObject(0) - .getObject("itemSectionRenderer").getArray("contents").getObject(0) - .getObject("showingResultsForRenderer"); - if (showingResultsForRenderer == null) { - return ""; + if (isMusicSearch()) { + final JsonObject itemSectionRenderer = initialData.getObject("contents").getObject("sectionListRenderer") + .getArray("contents").getObject(0).getObject("itemSectionRenderer"); + if (itemSectionRenderer == null) return ""; + return getTextFromObject(itemSectionRenderer.getArray("contents").getObject(0) + .getObject("didYouMeanRenderer").getObject("correctedQuery")); } else { + JsonObject showingResultsForRenderer = initialData.getObject("contents") + .getObject("twoColumnSearchResultsRenderer").getObject("primaryContents") + .getObject("sectionListRenderer").getArray("contents").getObject(0) + .getObject("itemSectionRenderer").getArray("contents").getObject(0) + .getObject("showingResultsForRenderer"); + if (showingResultsForRenderer == null) return ""; return getTextFromObject(showingResultsForRenderer.getObject("correctedQuery")); } } @@ -194,10 +197,13 @@ public class YoutubeSearchExtractor extends SearchExtractor { final InfoItemsSearchCollector collector = new InfoItemsSearchCollector(getServiceId()); if (isMusicSearch()) { - JsonArray sections = initialData.getObject("contents").getObject("sectionListRenderer") - .getArray("contents").getObject(0).getObject("musicShelfRenderer").getArray("contents"); + final JsonArray contents = initialData.getObject("contents").getObject("sectionListRenderer").getArray("contents"); - collectMusicStreamsFrom(collector, sections); + for (Object content : contents) { + if (((JsonObject) content).getObject("musicShelfRenderer") != null) { + collectMusicStreamsFrom(collector, ((JsonObject) content).getObject("musicShelfRenderer").getArray("contents")); + } + } } else { JsonArray sections = initialData.getObject("contents").getObject("twoColumnSearchResultsRenderer") .getObject("primaryContents").getObject("sectionListRenderer").getArray("contents"); @@ -213,8 +219,15 @@ public class YoutubeSearchExtractor extends SearchExtractor { @Override public String getNextPageUrl() throws ExtractionException, IOException { if (isMusicSearch()) { - return getNextPageUrlFrom(initialData.getObject("contents").getObject("sectionListRenderer") - .getArray("contents").getObject(0).getObject("musicShelfRenderer").getArray("continuations")); + final JsonArray contents = initialData.getObject("contents").getObject("sectionListRenderer").getArray("contents"); + + for (Object content : contents) { + if (((JsonObject) content).getObject("musicShelfRenderer") != null) { + return getNextPageUrlFrom(((JsonObject) content).getObject("musicShelfRenderer").getArray("continuations")); + } + } + + return ""; } else { return getNextPageUrlFrom(initialData.getObject("contents").getObject("twoColumnSearchResultsRenderer") .getObject("primaryContents").getObject("sectionListRenderer").getArray("contents") @@ -517,9 +530,7 @@ public class YoutubeSearchExtractor extends SearchExtractor { } private String getNextPageUrlFrom(JsonArray continuations) throws ParsingException, IOException, ReCaptchaException { - if (continuations == null) { - return ""; - } + if (continuations == null) return ""; JsonObject nextContinuationData = continuations.getObject(0).getObject("nextContinuationData"); String continuation = nextContinuationData.getString("continuation"); diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorMusicTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorMusicTest.java index aaf6105c4..61c8e04b7 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorMusicTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorMusicTest.java @@ -76,4 +76,14 @@ public class YoutubeSearchExtractorMusicTest extends YoutubeSearchExtractorBaseT assertEquals("json", queryPairs.get("alt")); assertTrue(queryPairs.get("key").length() > 5); } + + @Test + public void testSuggestions() throws Exception { + YoutubeSearchExtractor newExtractor = (YoutubeSearchExtractor) YouTube.getSearchExtractor("megaman x3", + asList(YoutubeSearchQueryHandlerFactory.MUSIC_SONGS), null); + newExtractor.fetchPage(); + + assertTrue(newExtractor.getInitialPage().getItems().size() > 10); + assertEquals("mega man x3", newExtractor.getSearchSuggestion()); + } } From d58c0f230d27cf34c2bf044597c908c519f94a84 Mon Sep 17 00:00:00 2001 From: wb9688 Date: Fri, 20 Mar 2020 19:14:53 +0100 Subject: [PATCH 08/17] Improve code for YouTube Music search --- .../extractors/YoutubeSearchExtractor.java | 44 ++++++++++++++----- .../YoutubeSearchExtractorMusicTest.java | 6 +++ 2 files changed, 38 insertions(+), 12 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java index 6dd574873..7c5adca5d 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java @@ -308,7 +308,9 @@ public class YoutubeSearchExtractor extends SearchExtractor { throw new ParsingException("Could not parse JSON", e); } - if (ajaxJson.getObject("continuationContents") == null) return new InfoItemsPage<>(collector, null); + if (ajaxJson.getObject("continuationContents") == null) { + return InfoItemsPage.emptyPage(); + } JsonObject musicShelfContinuation = ajaxJson.getObject("continuationContents").getObject("musicShelfContinuation"); @@ -329,8 +331,7 @@ public class YoutubeSearchExtractor extends SearchExtractor { private boolean isMusicSearch() { final List contentFilters = getLinkHandler().getContentFilters(); - if (contentFilters.size() > 0 && contentFilters.get(0).startsWith("music_")) return true; - return false; + return contentFilters.size() > 0 && contentFilters.get(0).startsWith("music_"); } private void collectStreamsFrom(InfoItemsSearchCollector collector, JsonArray videos) throws NothingFoundException, ParsingException { @@ -391,6 +392,16 @@ public class YoutubeSearchExtractor extends SearchExtractor { throw new ParsingException("Could not get uploader name"); } + @Override + public String getUploaderUrl() throws ParsingException { + if (searchType.equals(MUSIC_VIDEOS)) return null; + String url = getUrlFromNavigationEndpoint(info.getArray("flexColumns") + .getObject(1).getObject("musicResponsiveListItemFlexColumnRenderer") + .getObject("text").getArray("runs").getObject(0).getObject("navigationEndpoint")); + if (url != null && !url.isEmpty()) return url; + throw new ParsingException("Could not get uploader url"); + } + @Override public String getTextualUploadDate() { return null; @@ -413,9 +424,12 @@ public class YoutubeSearchExtractor extends SearchExtractor { @Override public String getThumbnailUrl() throws ParsingException { try { - // TODO: Don't simply get the first item, but look at all thumbnails and their resolution - return fixThumbnailUrl(info.getObject("thumbnail").getObject("musicThumbnailRenderer") - .getObject("thumbnail").getArray("thumbnails").getObject(0).getString("url")); + JsonArray thumbnails = info.getObject("thumbnail").getObject("musicThumbnailRenderer") + .getObject("thumbnail").getArray("thumbnails"); + // the last thumbnail is the one with the highest resolution + String url = thumbnails.getObject(thumbnails.size() - 1).getString("url"); + + return fixThumbnailUrl(url); } catch (Exception e) { throw new ParsingException("Could not get thumbnail url", e); } @@ -426,9 +440,12 @@ public class YoutubeSearchExtractor extends SearchExtractor { @Override public String getThumbnailUrl() throws ParsingException { try { - // TODO: Don't simply get the first item, but look at all thumbnails and their resolution - return fixThumbnailUrl(info.getObject("thumbnail").getObject("musicThumbnailRenderer") - .getObject("thumbnail").getArray("thumbnails").getObject(0).getString("url")); + JsonArray thumbnails = info.getObject("thumbnail").getObject("musicThumbnailRenderer") + .getObject("thumbnail").getArray("thumbnails"); + // the last thumbnail is the one with the highest resolution + String url = thumbnails.getObject(thumbnails.size() - 1).getString("url"); + + return fixThumbnailUrl(url); } catch (Exception e) { throw new ParsingException("Could not get thumbnail url", e); } @@ -472,9 +489,12 @@ public class YoutubeSearchExtractor extends SearchExtractor { @Override public String getThumbnailUrl() throws ParsingException { try { - // TODO: Don't simply get the first item, but look at all thumbnails and their resolution - return fixThumbnailUrl(info.getObject("thumbnail").getObject("musicThumbnailRenderer") - .getObject("thumbnail").getArray("thumbnails").getObject(0).getString("url")); + JsonArray thumbnails = info.getObject("thumbnail").getObject("musicThumbnailRenderer") + .getObject("thumbnail").getArray("thumbnails"); + // the last thumbnail is the one with the highest resolution + String url = thumbnails.getObject(thumbnails.size() - 1).getString("url"); + + return fixThumbnailUrl(url); } catch (Exception e) { throw new ParsingException("Could not get thumbnail url", e); } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorMusicTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorMusicTest.java index 61c8e04b7..6b2f472bd 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorMusicTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorMusicTest.java @@ -19,6 +19,7 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; import static org.schabi.newpipe.extractor.ServiceList.YouTube; +import static org.schabi.newpipe.extractor.services.DefaultTests.defaultTestRelatedItems; public class YoutubeSearchExtractorMusicTest extends YoutubeSearchExtractorBaseTest { @BeforeClass @@ -30,6 +31,11 @@ public class YoutubeSearchExtractorMusicTest extends YoutubeSearchExtractorBaseT itemsPage = extractor.getInitialPage(); } + @Test + public void testRelatedItems() throws Exception { + defaultTestRelatedItems(extractor); + } + @Test public void testGetSecondPage() throws Exception { YoutubeSearchExtractor secondExtractor = (YoutubeSearchExtractor) YouTube.getSearchExtractor("mocromaniac", From 5a775a4bbe6fc1e06a29a3665c8e8c23234a0536 Mon Sep 17 00:00:00 2001 From: wb9688 Date: Sat, 21 Mar 2020 19:56:17 +0100 Subject: [PATCH 09/17] Use new way of specifying stream count --- .../newpipe/extractor/ListExtractor.java | 19 ++++++++++++++++++- .../extractor/playlist/PlaylistExtractor.java | 3 --- .../extractors/YoutubeSearchExtractor.java | 6 ++---- 3 files changed, 20 insertions(+), 8 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/ListExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/ListExtractor.java index b254adbd8..0e21f04b8 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/ListExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/ListExtractor.java @@ -3,16 +3,33 @@ package org.schabi.newpipe.extractor; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; -import javax.annotation.Nonnull; import java.io.IOException; import java.util.Collections; import java.util.List; +import javax.annotation.Nonnull; + /** * Base class to extractors that have a list (e.g. playlists, users). */ public abstract class ListExtractor extends Extractor { + /** + * Constant that should be returned whenever + * a list has an unknown number of items. + */ + public static final long ITEM_COUNT_UNKNOWN = -1; + /** + * Constant that should be returned whenever a list has an + * infinite number of items. For example a YouTube mix. + */ + public static final long ITEM_COUNT_INFINITE = -2; + /** + * Constant that should be returned whenever a list + * has an unknown number of items bigger than 100. + */ + public static final long ITEM_COUNT_MORE_THAN_100 = -3; + public ListExtractor(StreamingService service, ListLinkHandler linkHandler) { super(service, linkHandler); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistExtractor.java index 06330de05..2a901dbf0 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistExtractor.java @@ -7,9 +7,6 @@ import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; import org.schabi.newpipe.extractor.stream.StreamInfoItem; public abstract class PlaylistExtractor extends ListExtractor { - public final static long UNKNOWN_ITEMS = -1; - public final static long INFINITE_ITEMS = -2; - public final static long MORE_THAN_100_ITEMS = -3; public PlaylistExtractor(StreamingService service, ListLinkHandler linkHandler) { super(service, linkHandler); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java index 7c5adca5d..37c9515d0 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java @@ -30,8 +30,6 @@ import java.util.Map; import javax.annotation.Nonnull; -import static org.schabi.newpipe.extractor.playlist.PlaylistExtractor.MORE_THAN_100_ITEMS; -import static org.schabi.newpipe.extractor.playlist.PlaylistExtractor.UNKNOWN_ITEMS; import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.fixThumbnailUrl; import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getJsonResponse; import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getTextFromObject; @@ -531,12 +529,12 @@ public class YoutubeSearchExtractor extends SearchExtractor { @Override public long getStreamCount() throws ParsingException { - if (searchType.equals(MUSIC_ALBUMS)) return UNKNOWN_ITEMS; + if (searchType.equals(MUSIC_ALBUMS)) return ITEM_COUNT_UNKNOWN; String count = getTextFromObject(info.getArray("flexColumns").getObject(2) .getObject("musicResponsiveListItemFlexColumnRenderer").getObject("text")); if (count != null && !count.isEmpty()) { if (count.contains("100+")) { - return MORE_THAN_100_ITEMS; + return ITEM_COUNT_MORE_THAN_100; } else { return Long.parseLong(Utils.removeNonDigitCharacters(count)); } From dd434cca01d401ffcb236a2e2f3f042fabe0e272 Mon Sep 17 00:00:00 2001 From: wb9688 Date: Sat, 21 Mar 2020 20:11:06 +0100 Subject: [PATCH 10/17] Fix issue when there is no didYouMeanRenderer in itemSectionRenderer --- .../services/youtube/extractors/YoutubeSearchExtractor.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java index 37c9515d0..7e640a1b8 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java @@ -176,8 +176,10 @@ public class YoutubeSearchExtractor extends SearchExtractor { final JsonObject itemSectionRenderer = initialData.getObject("contents").getObject("sectionListRenderer") .getArray("contents").getObject(0).getObject("itemSectionRenderer"); if (itemSectionRenderer == null) return ""; - return getTextFromObject(itemSectionRenderer.getArray("contents").getObject(0) - .getObject("didYouMeanRenderer").getObject("correctedQuery")); + final JsonObject didYouMeanRenderer = itemSectionRenderer.getArray("contents") + .getObject(0).getObject("didYouMeanRenderer"); + if (didYouMeanRenderer == null) return ""; + return getTextFromObject(didYouMeanRenderer.getObject("correctedQuery")); } else { JsonObject showingResultsForRenderer = initialData.getObject("contents") .getObject("twoColumnSearchResultsRenderer").getObject("primaryContents") From aa8cea47f3d6276ab592d18ff550ad9d6e63d81b Mon Sep 17 00:00:00 2001 From: wb9688 Date: Wed, 25 Mar 2020 10:22:32 +0100 Subject: [PATCH 11/17] Refactor YouTube Music search tests --- .../extractors/YoutubeSearchExtractor.java | 6 +- .../extractor/services/DefaultTests.java | 9 +- .../YoutubeSearchExtractorMusicTest.java | 120 +++++++----------- 3 files changed, 54 insertions(+), 81 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java index 7e640a1b8..52df8c449 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java @@ -395,9 +395,11 @@ public class YoutubeSearchExtractor extends SearchExtractor { @Override public String getUploaderUrl() throws ParsingException { if (searchType.equals(MUSIC_VIDEOS)) return null; - String url = getUrlFromNavigationEndpoint(info.getArray("flexColumns") + JsonObject navigationEndpoint = info.getArray("flexColumns") .getObject(1).getObject("musicResponsiveListItemFlexColumnRenderer") - .getObject("text").getArray("runs").getObject(0).getObject("navigationEndpoint")); + .getObject("text").getArray("runs").getObject(0).getObject("navigationEndpoint"); + if (navigationEndpoint == null) return null; + String url = getUrlFromNavigationEndpoint(navigationEndpoint); if (url != null && !url.isEmpty()) return url; throw new ParsingException("Could not get uploader url"); } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/DefaultTests.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/DefaultTests.java index b34bbbff9..a52698415 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/DefaultTests.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/DefaultTests.java @@ -37,11 +37,14 @@ public final class DefaultTests { if (item instanceof StreamInfoItem) { StreamInfoItem streamInfoItem = (StreamInfoItem) item; assertNotEmpty("Uploader name not set: " + item, streamInfoItem.getUploaderName()); - assertNotEmpty("Uploader url not set: " + item, streamInfoItem.getUploaderUrl()); - assertIsSecureUrl(streamInfoItem.getUploaderUrl()); + +// assertNotEmpty("Uploader url not set: " + item, streamInfoItem.getUploaderUrl()); + if (streamInfoItem.getUploaderUrl() != null && !streamInfoItem.getUploaderUrl().isEmpty()) { + assertIsSecureUrl(streamInfoItem.getUploaderUrl()); + assertExpectedLinkType(expectedService, streamInfoItem.getUploaderUrl(), LinkType.CHANNEL); + } assertExpectedLinkType(expectedService, streamInfoItem.getUrl(), LinkType.STREAM); - assertExpectedLinkType(expectedService, streamInfoItem.getUploaderUrl(), LinkType.CHANNEL); final String textualUploadDate = streamInfoItem.getTextualUploadDate(); if (textualUploadDate != null && !textualUploadDate.isEmpty()) { diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorMusicTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorMusicTest.java index 6b2f472bd..fe1743803 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorMusicTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorMusicTest.java @@ -1,95 +1,63 @@ package org.schabi.newpipe.extractor.services.youtube.search; import org.junit.BeforeClass; -import org.junit.Test; import org.schabi.newpipe.DownloaderTestImpl; import org.schabi.newpipe.extractor.InfoItem; -import org.schabi.newpipe.extractor.ListExtractor; import org.schabi.newpipe.extractor.NewPipe; -import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeSearchExtractor; +import org.schabi.newpipe.extractor.StreamingService; +import org.schabi.newpipe.extractor.search.SearchExtractor; +import org.schabi.newpipe.extractor.services.DefaultSearchExtractorTest; import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory; -import java.net.URL; -import java.net.URLDecoder; -import java.util.LinkedHashMap; -import java.util.Map; +import java.net.URLEncoder; -import static java.util.Arrays.asList; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; +import javax.annotation.Nullable; + +import static java.util.Collections.singletonList; import static org.schabi.newpipe.extractor.ServiceList.YouTube; -import static org.schabi.newpipe.extractor.services.DefaultTests.defaultTestRelatedItems; -public class YoutubeSearchExtractorMusicTest extends YoutubeSearchExtractorBaseTest { - @BeforeClass - public static void setUpClass() throws Exception { - NewPipe.init(DownloaderTestImpl.getInstance()); - extractor = (YoutubeSearchExtractor) YouTube.getSearchExtractor("mocromaniac", - asList(YoutubeSearchQueryHandlerFactory.MUSIC_SONGS), null); - extractor.fetchPage(); - itemsPage = extractor.getInitialPage(); - } +public class YoutubeSearchExtractorMusicTest { + public static class MusicSongs extends DefaultSearchExtractorTest { + private static SearchExtractor extractor; + private static final String QUERY = "mocromaniac"; - @Test - public void testRelatedItems() throws Exception { - defaultTestRelatedItems(extractor); - } - - @Test - public void testGetSecondPage() throws Exception { - YoutubeSearchExtractor secondExtractor = (YoutubeSearchExtractor) YouTube.getSearchExtractor("mocromaniac", - asList(YoutubeSearchQueryHandlerFactory.MUSIC_SONGS), null); - ListExtractor.InfoItemsPage secondPage = secondExtractor.getPage(itemsPage.getNextPageUrl()); - assertTrue(Integer.toString(secondPage.getItems().size()), - secondPage.getItems().size() > 10); - - // check if its the same result - boolean equals = true; - for (int i = 0; i < secondPage.getItems().size() - && i < itemsPage.getItems().size(); i++) { - if (!secondPage.getItems().get(i).getUrl().equals( - itemsPage.getItems().get(i).getUrl())) { - equals = false; - } - } - assertFalse("First and second page are equal", equals); - } - - @Override - @Test - public void testUrl() throws Exception { - assertTrue(extractor.getUrl(), extractor.getUrl().startsWith("https://music.youtube.com/search?q=")); - } - - @Test - public void testGetSecondPageUrl() throws Exception { - URL url = new URL(extractor.getNextPageUrl()); - - assertEquals(url.getHost(), "music.youtube.com"); - assertEquals(url.getPath(), "/youtubei/v1/search"); - - Map queryPairs = new LinkedHashMap<>(); - for (String queryPair : url.getQuery().split("&")) { - int index = queryPair.indexOf("="); - queryPairs.put(URLDecoder.decode(queryPair.substring(0, index), "UTF-8"), - URLDecoder.decode(queryPair.substring(index + 1), "UTF-8")); + @BeforeClass + public static void setUp() throws Exception { + NewPipe.init(DownloaderTestImpl.getInstance()); + extractor = YouTube.getSearchExtractor(QUERY, singletonList(YoutubeSearchQueryHandlerFactory.MUSIC_SONGS), ""); + extractor.fetchPage(); } - assertEquals(queryPairs.get("ctoken"), queryPairs.get("continuation")); - assertTrue(queryPairs.get("continuation").length() > 5); - assertTrue(queryPairs.get("itct").length() > 5); - assertEquals("json", queryPairs.get("alt")); - assertTrue(queryPairs.get("key").length() > 5); + @Override public SearchExtractor extractor() { return extractor; } + @Override public StreamingService expectedService() { return YouTube; } + @Override public String expectedName() { return QUERY; } + @Override public String expectedId() { return QUERY; } + @Override public String expectedUrlContains() { return "music.youtube.com/search?q=" + QUERY; } + @Override public String expectedOriginalUrlContains() { return "music.youtube.com/search?q=" + QUERY; } + @Override public String expectedSearchString() { return QUERY; } + @Nullable @Override public String expectedSearchSuggestion() { return null; } + @Override public InfoItem.InfoType expectedInfoItemType() { return InfoItem.InfoType.STREAM; } } - @Test - public void testSuggestions() throws Exception { - YoutubeSearchExtractor newExtractor = (YoutubeSearchExtractor) YouTube.getSearchExtractor("megaman x3", - asList(YoutubeSearchQueryHandlerFactory.MUSIC_SONGS), null); - newExtractor.fetchPage(); + public static class Suggestion extends DefaultSearchExtractorTest { + private static SearchExtractor extractor; + private static final String QUERY = "megaman x3"; - assertTrue(newExtractor.getInitialPage().getItems().size() > 10); - assertEquals("mega man x3", newExtractor.getSearchSuggestion()); + @BeforeClass + public static void setUp() throws Exception { + NewPipe.init(DownloaderTestImpl.getInstance()); + extractor = YouTube.getSearchExtractor(QUERY, singletonList(YoutubeSearchQueryHandlerFactory.MUSIC_SONGS), ""); + extractor.fetchPage(); + } + + @Override public SearchExtractor extractor() { return extractor; } + @Override public StreamingService expectedService() { return YouTube; } + @Override public String expectedName() { return QUERY; } + @Override public String expectedId() { return QUERY; } + @Override public String expectedUrlContains() { return "music.youtube.com/search?q=" + URLEncoder.encode(QUERY); } + @Override public String expectedOriginalUrlContains() { return "music.youtube.com/search?q=" + URLEncoder.encode(QUERY); } + @Override public String expectedSearchString() { return QUERY; } + @Nullable @Override public String expectedSearchSuggestion() { return "mega man x3"; } + @Override public InfoItem.InfoType expectedInfoItemType() { return InfoItem.InfoType.STREAM; } } } From cf0f2aff3e8d7dc539ff662a006329342dd6d55a Mon Sep 17 00:00:00 2001 From: wb9688 Date: Wed, 25 Mar 2020 10:42:34 +0100 Subject: [PATCH 12/17] Extract uploader url from certain YouTube Music videos --- .../extractors/YoutubeSearchExtractor.java | 25 ++++++++++++++----- ...a => YoutubeMusicSearchExtractorTest.java} | 2 +- 2 files changed, 20 insertions(+), 7 deletions(-) rename extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/{YoutubeSearchExtractorMusicTest.java => YoutubeMusicSearchExtractorTest.java} (98%) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java index 52df8c449..ca56283d9 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java @@ -394,12 +394,25 @@ public class YoutubeSearchExtractor extends SearchExtractor { @Override public String getUploaderUrl() throws ParsingException { - if (searchType.equals(MUSIC_VIDEOS)) return null; - JsonObject navigationEndpoint = info.getArray("flexColumns") - .getObject(1).getObject("musicResponsiveListItemFlexColumnRenderer") - .getObject("text").getArray("runs").getObject(0).getObject("navigationEndpoint"); - if (navigationEndpoint == null) return null; - String url = getUrlFromNavigationEndpoint(navigationEndpoint); + String url = null; + + if (searchType.equals(MUSIC_VIDEOS)) { + JsonArray items = info.getObject("menu").getObject("menuRenderer").getArray("items"); + for (Object item : items) { + JsonObject menuNavigationItemRenderer = ((JsonObject) item).getObject("menuNavigationItemRenderer"); + if (menuNavigationItemRenderer != null && menuNavigationItemRenderer.getObject("icon").getString("iconType").equals("ARTIST")) { + url = getUrlFromNavigationEndpoint(menuNavigationItemRenderer.getObject("navigationEndpoint")); + break; + } + } + } else { + JsonObject navigationEndpoint = info.getArray("flexColumns") + .getObject(1).getObject("musicResponsiveListItemFlexColumnRenderer") + .getObject("text").getArray("runs").getObject(0).getObject("navigationEndpoint"); + if (navigationEndpoint == null) return null; + url = getUrlFromNavigationEndpoint(navigationEndpoint); + } + if (url != null && !url.isEmpty()) return url; throw new ParsingException("Could not get uploader url"); } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorMusicTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeMusicSearchExtractorTest.java similarity index 98% rename from extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorMusicTest.java rename to extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeMusicSearchExtractorTest.java index fe1743803..689e6127c 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorMusicTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeMusicSearchExtractorTest.java @@ -16,7 +16,7 @@ import javax.annotation.Nullable; import static java.util.Collections.singletonList; import static org.schabi.newpipe.extractor.ServiceList.YouTube; -public class YoutubeSearchExtractorMusicTest { +public class YoutubeMusicSearchExtractorTest { public static class MusicSongs extends DefaultSearchExtractorTest { private static SearchExtractor extractor; private static final String QUERY = "mocromaniac"; From ac15df45489178002c95db436a717319020a517a Mon Sep 17 00:00:00 2001 From: wb9688 Date: Wed, 1 Apr 2020 10:03:54 +0200 Subject: [PATCH 13/17] Move YouTube Music search to its own class --- .../services/youtube/YoutubeService.java | 41 +- .../YoutubeMusicSearchExtractor.java | 526 ++++++++++++++++++ .../extractors/YoutubeSearchExtractor.java | 508 ++--------------- 3 files changed, 598 insertions(+), 477 deletions(-) create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java index eae3bcb9b..519672141 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java @@ -7,22 +7,45 @@ import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.feed.FeedExtractor; import org.schabi.newpipe.extractor.kiosk.KioskExtractor; import org.schabi.newpipe.extractor.kiosk.KioskList; -import org.schabi.newpipe.extractor.linkhandler.*; +import org.schabi.newpipe.extractor.linkhandler.LinkHandler; +import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory; +import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; +import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory; +import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandler; +import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandlerFactory; import org.schabi.newpipe.extractor.localization.ContentCountry; import org.schabi.newpipe.extractor.localization.Localization; import org.schabi.newpipe.extractor.playlist.PlaylistExtractor; import org.schabi.newpipe.extractor.search.SearchExtractor; -import org.schabi.newpipe.extractor.services.youtube.extractors.*; -import org.schabi.newpipe.extractor.services.youtube.linkHandler.*; +import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeChannelExtractor; +import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeCommentsExtractor; +import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeFeedExtractor; +import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeMusicSearchExtractor; +import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubePlaylistExtractor; +import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeSearchExtractor; +import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeStreamExtractor; +import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeSubscriptionExtractor; +import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeSuggestionExtractor; +import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeTrendingExtractor; +import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeChannelLinkHandlerFactory; +import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeCommentsLinkHandlerFactory; +import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubePlaylistLinkHandlerFactory; +import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory; +import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeStreamLinkHandlerFactory; +import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeTrendingLinkHandlerFactory; import org.schabi.newpipe.extractor.stream.StreamExtractor; import org.schabi.newpipe.extractor.subscription.SubscriptionExtractor; import org.schabi.newpipe.extractor.suggestion.SuggestionExtractor; -import javax.annotation.Nonnull; import java.util.List; +import javax.annotation.Nonnull; + import static java.util.Arrays.asList; -import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.*; +import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.AUDIO; +import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.COMMENTS; +import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.LIVE; +import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.VIDEO; /* * Created by Christian Schabesberger on 23.08.15. @@ -92,7 +115,13 @@ public class YoutubeService extends StreamingService { @Override public SearchExtractor getSearchExtractor(SearchQueryHandler query) { - return new YoutubeSearchExtractor(this, query); + final List contentFilters = query.getContentFilters(); + + if (contentFilters.size() > 0 && contentFilters.get(0).startsWith("music_")) { + return new YoutubeMusicSearchExtractor(this, query); + } else { + return new YoutubeSearchExtractor(this, query); + } } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java new file mode 100644 index 000000000..46e9d93c0 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java @@ -0,0 +1,526 @@ +package org.schabi.newpipe.extractor.services.youtube.extractors; + +import com.grack.nanojson.JsonArray; +import com.grack.nanojson.JsonObject; +import com.grack.nanojson.JsonParser; +import com.grack.nanojson.JsonParserException; +import com.grack.nanojson.JsonWriter; + +import org.schabi.newpipe.extractor.InfoItem; +import org.schabi.newpipe.extractor.StreamingService; +import org.schabi.newpipe.extractor.downloader.Downloader; +import org.schabi.newpipe.extractor.downloader.Response; +import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException; +import org.schabi.newpipe.extractor.exceptions.ExtractionException; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.exceptions.ReCaptchaException; +import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandler; +import org.schabi.newpipe.extractor.localization.DateWrapper; +import org.schabi.newpipe.extractor.localization.TimeAgoParser; +import org.schabi.newpipe.extractor.search.InfoItemsSearchCollector; +import org.schabi.newpipe.extractor.search.SearchExtractor; +import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper; +import org.schabi.newpipe.extractor.utils.Utils; + +import java.io.IOException; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import javax.annotation.Nonnull; + +import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.fixThumbnailUrl; +import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getTextFromObject; +import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getUrlFromNavigationEndpoint; +import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.MUSIC_ALBUMS; +import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.MUSIC_ARTISTS; +import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.MUSIC_PLAYLISTS; +import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.MUSIC_SONGS; +import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.MUSIC_VIDEOS; + +public class YoutubeMusicSearchExtractor extends SearchExtractor { + private JsonObject initialData; + + public YoutubeMusicSearchExtractor(final StreamingService service, final SearchQueryHandler linkHandler) { + super(service, linkHandler); + } + + @Override + public void onFetchPage(@Nonnull final Downloader downloader) throws IOException, ExtractionException { + final String[] youtubeMusicKeys = YoutubeParsingHelper.getYoutubeMusicKeys(); + + final String url = "https://music.youtube.com/youtubei/v1/search?alt=json&key=" + youtubeMusicKeys[0]; + + final String params; + + switch (getLinkHandler().getContentFilters().get(0)) { + case MUSIC_SONGS: + params = "Eg-KAQwIARAAGAAgACgAMABqChAEEAUQAxAKEAk%3D"; + break; + case MUSIC_VIDEOS: + params = "Eg-KAQwIABABGAAgACgAMABqChAEEAUQAxAKEAk%3D"; + break; + case MUSIC_ALBUMS: + params = "Eg-KAQwIABAAGAEgACgAMABqChAEEAUQAxAKEAk%3D"; + break; + case MUSIC_PLAYLISTS: + params = "Eg-KAQwIABAAGAAgACgBMABqChAEEAUQAxAKEAk%3D"; + break; + case MUSIC_ARTISTS: + params = "Eg-KAQwIABAAGAAgASgAMABqChAEEAUQAxAKEAk%3D"; + break; + default: + params = null; + break; + } + + // @formatter:off + byte[] json = JsonWriter.string() + .object() + .object("context") + .object("client") + .value("clientName", "WEB_REMIX") + .value("clientVersion", youtubeMusicKeys[2]) + .value("hl", "en") + .value("gl", getExtractorContentCountry().getCountryCode()) + .array("experimentIds").end() + .value("experimentsToken", "") + .value("utcOffsetMinutes", 0) + .object("locationInfo").end() + .object("musicAppInfo").end() + .end() + .object("capabilities").end() + .object("request") + .array("internalExperimentFlags").end() + .object("sessionIndex").end() + .end() + .object("activePlayers").end() + .object("user") + .value("enableSafetyMode", false) + .end() + .end() + .value("query", getSearchString()) + .value("params", params) + .end().done().getBytes("UTF-8"); + // @formatter:on + + final Map> headers = new HashMap<>(); + headers.put("X-YouTube-Client-Name", Collections.singletonList(youtubeMusicKeys[1])); + headers.put("X-YouTube-Client-Version", Collections.singletonList(youtubeMusicKeys[2])); + headers.put("Origin", Collections.singletonList("https://music.youtube.com")); + headers.put("Referer", Collections.singletonList("music.youtube.com")); + headers.put("Content-Type", Collections.singletonList("application/json")); + + final Response response = getDownloader().post(url, headers, json); + + if (response.responseCode() == 404) { + throw new ContentNotAvailableException("Not found" + + " (\"" + response.responseCode() + " " + response.responseMessage() + "\")"); + } + + final String responseBody = response.responseBody(); + if (responseBody.length() < 50) { // ensure to have a valid response + throw new ParsingException("JSON response is too short"); + } + + final String responseContentType = response.getHeader("Content-Type"); + if (responseContentType != null && responseContentType.toLowerCase().contains("text/html")) { + throw new ParsingException("Got HTML document, expected JSON response" + + " (latest url was: \"" + response.latestUrl() + "\")"); + } + + try { + initialData = JsonParser.object().from(responseBody); + } catch (JsonParserException e) { + throw new ParsingException("Could not parse JSON", e); + } + } + + @Nonnull + @Override + public String getUrl() throws ParsingException { + return super.getUrl(); + } + + @Override + public String getSearchSuggestion() throws ParsingException { + final JsonObject itemSectionRenderer = initialData.getObject("contents").getObject("sectionListRenderer") + .getArray("contents").getObject(0).getObject("itemSectionRenderer"); + if (itemSectionRenderer == null) { + return ""; + } + final JsonObject didYouMeanRenderer = itemSectionRenderer.getArray("contents") + .getObject(0).getObject("didYouMeanRenderer"); + if (didYouMeanRenderer == null) { + return ""; + } + return getTextFromObject(didYouMeanRenderer.getObject("correctedQuery")); + } + + @Nonnull + @Override + public InfoItemsPage getInitialPage() throws ExtractionException, IOException { + final InfoItemsSearchCollector collector = new InfoItemsSearchCollector(getServiceId()); + + final JsonArray contents = initialData.getObject("contents").getObject("sectionListRenderer").getArray("contents"); + + for (Object content : contents) { + if (((JsonObject) content).getObject("musicShelfRenderer") != null) { + collectMusicStreamsFrom(collector, ((JsonObject) content).getObject("musicShelfRenderer").getArray("contents")); + } + } + + return new InfoItemsPage<>(collector, getNextPageUrl()); + } + + @Override + public String getNextPageUrl() throws ExtractionException, IOException { + final JsonArray contents = initialData.getObject("contents").getObject("sectionListRenderer").getArray("contents"); + + for (Object content : contents) { + if (((JsonObject) content).getObject("musicShelfRenderer") != null) { + return getNextPageUrlFrom(((JsonObject) content).getObject("musicShelfRenderer").getArray("continuations")); + } + } + + return ""; + } + + @Override + public InfoItemsPage getPage(final String pageUrl) throws IOException, ExtractionException { + if (pageUrl == null || pageUrl.isEmpty()) { + throw new ExtractionException(new IllegalArgumentException("Page url is empty or null")); + } + + final InfoItemsSearchCollector collector = new InfoItemsSearchCollector(getServiceId()); + + final String[] youtubeMusicKeys = YoutubeParsingHelper.getYoutubeMusicKeys(); + + // @formatter:off + byte[] json = JsonWriter.string() + .object() + .object("context") + .object("client") + .value("clientName", "WEB_REMIX") + .value("clientVersion", youtubeMusicKeys[2]) + .value("hl", "en") + .value("gl", getExtractorContentCountry().getCountryCode()) + .array("experimentIds").end() + .value("experimentsToken", "") + .value("utcOffsetMinutes", 0) + .object("locationInfo").end() + .object("musicAppInfo").end() + .end() + .object("capabilities").end() + .object("request") + .array("internalExperimentFlags").end() + .object("sessionIndex").end() + .end() + .object("activePlayers").end() + .object("user") + .value("enableSafetyMode", false) + .end() + .end() + .end().done().getBytes("UTF-8"); + // @formatter:on + + final Map> headers = new HashMap<>(); + headers.put("X-YouTube-Client-Name", Collections.singletonList(youtubeMusicKeys[1])); + headers.put("X-YouTube-Client-Version", Collections.singletonList(youtubeMusicKeys[2])); + headers.put("Origin", Collections.singletonList("https://music.youtube.com")); + headers.put("Referer", Collections.singletonList("music.youtube.com")); + headers.put("Content-Type", Collections.singletonList("application/json")); + + final Response response = getDownloader().post(pageUrl, headers, json); + + if (response.responseCode() == 404) { + throw new ContentNotAvailableException("Not found" + + " (\"" + response.responseCode() + " " + response.responseMessage() + "\")"); + } + + final String responseBody = response.responseBody(); + if (responseBody.length() < 50) { // ensure to have a valid response + throw new ParsingException("JSON response is too short"); + } + + final String responseContentType = response.getHeader("Content-Type"); + if (responseContentType != null && responseContentType.toLowerCase().contains("text/html")) { + throw new ParsingException("Got HTML document, expected JSON response" + + " (latest url was: \"" + response.latestUrl() + "\")"); + } + + final JsonObject ajaxJson; + try { + ajaxJson = JsonParser.object().from(responseBody); + } catch (JsonParserException e) { + throw new ParsingException("Could not parse JSON", e); + } + + if (ajaxJson.getObject("continuationContents") == null) { + return InfoItemsPage.emptyPage(); + } + + final JsonObject musicShelfContinuation = ajaxJson.getObject("continuationContents").getObject("musicShelfContinuation"); + + collectMusicStreamsFrom(collector, musicShelfContinuation.getArray("contents")); + final JsonArray continuations = musicShelfContinuation.getArray("continuations"); + + return new InfoItemsPage<>(collector, getNextPageUrlFrom(continuations)); + } + + private void collectMusicStreamsFrom(final InfoItemsSearchCollector collector, final JsonArray videos) { + final TimeAgoParser timeAgoParser = getTimeAgoParser(); + + for (Object item : videos) { + final JsonObject info = ((JsonObject) item).getObject("musicResponsiveListItemRenderer"); + if (info != null) { + final String searchType = getLinkHandler().getContentFilters().get(0); + if (searchType.equals(MUSIC_SONGS) || searchType.equals(MUSIC_VIDEOS)) { + collector.commit(new YoutubeStreamInfoItemExtractor(info, timeAgoParser) { + @Override + public String getUrl() throws ParsingException { + final String url = getUrlFromNavigationEndpoint(info.getObject("doubleTapCommand")); + if (url != null && !url.isEmpty()) { + return url; + } + throw new ParsingException("Could not get url"); + } + + @Override + public String getName() throws ParsingException { + final String name = getTextFromObject(info.getArray("flexColumns").getObject(0) + .getObject("musicResponsiveListItemFlexColumnRenderer").getObject("text")); + if (name != null && !name.isEmpty()) { + return name; + } + throw new ParsingException("Could not get name"); + } + + @Override + public long getDuration() throws ParsingException { + final String duration = getTextFromObject(info.getArray("flexColumns").getObject(3) + .getObject("musicResponsiveListItemFlexColumnRenderer").getObject("text")); + if (duration != null && !duration.isEmpty()) { + return YoutubeParsingHelper.parseDurationString(duration); + } + throw new ParsingException("Could not get duration"); + } + + @Override + public String getUploaderName() throws ParsingException { + final String name = getTextFromObject(info.getArray("flexColumns").getObject(1) + .getObject("musicResponsiveListItemFlexColumnRenderer").getObject("text")); + if (name != null && !name.isEmpty()) { + return name; + } + throw new ParsingException("Could not get uploader name"); + } + + @Override + public String getUploaderUrl() throws ParsingException { + String url = null; + + if (searchType.equals(MUSIC_VIDEOS)) { + JsonArray items = info.getObject("menu").getObject("menuRenderer").getArray("items"); + for (Object item : items) { + final JsonObject menuNavigationItemRenderer = ((JsonObject) item).getObject("menuNavigationItemRenderer"); + if (menuNavigationItemRenderer != null && menuNavigationItemRenderer.getObject("icon").getString("iconType").equals("ARTIST")) { + url = getUrlFromNavigationEndpoint(menuNavigationItemRenderer.getObject("navigationEndpoint")); + break; + } + } + } else { + final JsonObject navigationEndpoint = info.getArray("flexColumns") + .getObject(1).getObject("musicResponsiveListItemFlexColumnRenderer") + .getObject("text").getArray("runs").getObject(0).getObject("navigationEndpoint"); + if (navigationEndpoint == null) { + return null; + } + url = getUrlFromNavigationEndpoint(navigationEndpoint); + } + + if (url != null && !url.isEmpty()) { + return url; + } + throw new ParsingException("Could not get uploader url"); + } + + @Override + public String getTextualUploadDate() { + return null; + } + + @Override + public DateWrapper getUploadDate() { + return null; + } + + @Override + public long getViewCount() throws ParsingException { + if (searchType.equals(MUSIC_SONGS)) { + return -1; + } + final String viewCount = getTextFromObject(info.getArray("flexColumns").getObject(2) + .getObject("musicResponsiveListItemFlexColumnRenderer").getObject("text")); + if (viewCount != null && !viewCount.isEmpty()) { + return Utils.mixedNumberWordToLong(viewCount); + } + throw new ParsingException("Could not get view count"); + } + + @Override + public String getThumbnailUrl() throws ParsingException { + try { + final JsonArray thumbnails = info.getObject("thumbnail").getObject("musicThumbnailRenderer") + .getObject("thumbnail").getArray("thumbnails"); + // the last thumbnail is the one with the highest resolution + final String url = thumbnails.getObject(thumbnails.size() - 1).getString("url"); + + return fixThumbnailUrl(url); + } catch (Exception e) { + throw new ParsingException("Could not get thumbnail url", e); + } + } + }); + } else if (searchType.equals(MUSIC_ARTISTS)) { + collector.commit(new YoutubeChannelInfoItemExtractor(info) { + @Override + public String getThumbnailUrl() throws ParsingException { + try { + final JsonArray thumbnails = info.getObject("thumbnail").getObject("musicThumbnailRenderer") + .getObject("thumbnail").getArray("thumbnails"); + // the last thumbnail is the one with the highest resolution + final String url = thumbnails.getObject(thumbnails.size() - 1).getString("url"); + + return fixThumbnailUrl(url); + } catch (Exception e) { + throw new ParsingException("Could not get thumbnail url", e); + } + } + + @Override + public String getName() throws ParsingException { + final String name = getTextFromObject(info.getArray("flexColumns").getObject(0) + .getObject("musicResponsiveListItemFlexColumnRenderer").getObject("text")); + if (name != null && !name.isEmpty()) { + return name; + } + throw new ParsingException("Could not get name"); + } + + @Override + public String getUrl() throws ParsingException { + final String url = getUrlFromNavigationEndpoint(info.getObject("navigationEndpoint")); + if (url != null && !url.isEmpty()) { + return url; + } + throw new ParsingException("Could not get url"); + } + + @Override + public long getSubscriberCount() throws ParsingException { + final String viewCount = getTextFromObject(info.getArray("flexColumns").getObject(2) + .getObject("musicResponsiveListItemFlexColumnRenderer").getObject("text")); + if (viewCount != null && !viewCount.isEmpty()) { + return Utils.mixedNumberWordToLong(viewCount); + } + throw new ParsingException("Could not get subscriber count"); + } + + @Override + public long getStreamCount() { + return -1; + } + + @Override + public String getDescription() { + return null; + } + }); + } else if (searchType.equals(MUSIC_ALBUMS) || searchType.equals(MUSIC_PLAYLISTS)) { + collector.commit(new YoutubePlaylistInfoItemExtractor(info) { + @Override + public String getThumbnailUrl() throws ParsingException { + try { + final JsonArray thumbnails = info.getObject("thumbnail").getObject("musicThumbnailRenderer") + .getObject("thumbnail").getArray("thumbnails"); + // the last thumbnail is the one with the highest resolution + final String url = thumbnails.getObject(thumbnails.size() - 1).getString("url"); + + return fixThumbnailUrl(url); + } catch (Exception e) { + throw new ParsingException("Could not get thumbnail url", e); + } + } + + @Override + public String getName() throws ParsingException { + final String name = getTextFromObject(info.getArray("flexColumns").getObject(0) + .getObject("musicResponsiveListItemFlexColumnRenderer").getObject("text")); + if (name != null && !name.isEmpty()) { + return name; + } + throw new ParsingException("Could not get name"); + } + + @Override + public String getUrl() throws ParsingException { + final String url = getUrlFromNavigationEndpoint(info.getObject("doubleTapCommand")); + if (url != null && !url.isEmpty()) { + return url; + } + throw new ParsingException("Could not get url"); + } + + @Override + public String getUploaderName() throws ParsingException { + final String name; + if (searchType.equals(MUSIC_ALBUMS)) { + name = getTextFromObject(info.getArray("flexColumns").getObject(2) + .getObject("musicResponsiveListItemFlexColumnRenderer").getObject("text")); + } else { + name = getTextFromObject(info.getArray("flexColumns").getObject(1) + .getObject("musicResponsiveListItemFlexColumnRenderer").getObject("text")); + } + if (name != null && !name.isEmpty()) { + return name; + } + throw new ParsingException("Could not get uploader name"); + } + + @Override + public long getStreamCount() throws ParsingException { + if (searchType.equals(MUSIC_ALBUMS)) { + return ITEM_COUNT_UNKNOWN; + } + final String count = getTextFromObject(info.getArray("flexColumns").getObject(2) + .getObject("musicResponsiveListItemFlexColumnRenderer").getObject("text")); + if (count != null && !count.isEmpty()) { + if (count.contains("100+")) { + return ITEM_COUNT_MORE_THAN_100; + } else { + return Long.parseLong(Utils.removeNonDigitCharacters(count)); + } + } + throw new ParsingException("Could not get count"); + } + }); + } + } + } + } + + private String getNextPageUrlFrom(final JsonArray continuations) throws ParsingException, IOException, ReCaptchaException { + if (continuations == null) { + return ""; + } + + final JsonObject nextContinuationData = continuations.getObject(0).getObject("nextContinuationData"); + final String continuation = nextContinuationData.getString("continuation"); + final String clickTrackingParams = nextContinuationData.getString("clickTrackingParams"); + + return "https://music.youtube.com/youtubei/v1/search?ctoken=" + continuation + "&continuation=" + continuation + + "&itct=" + clickTrackingParams + "&alt=json&key=" + YoutubeParsingHelper.getYoutubeMusicKeys()[0]; + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java index ca56283d9..06d83b243 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java @@ -2,43 +2,23 @@ package org.schabi.newpipe.extractor.services.youtube.extractors; import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; -import com.grack.nanojson.JsonParser; -import com.grack.nanojson.JsonParserException; -import com.grack.nanojson.JsonWriter; import org.schabi.newpipe.extractor.InfoItem; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.downloader.Downloader; -import org.schabi.newpipe.extractor.downloader.Response; -import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; -import org.schabi.newpipe.extractor.exceptions.ReCaptchaException; import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandler; -import org.schabi.newpipe.extractor.localization.DateWrapper; import org.schabi.newpipe.extractor.localization.TimeAgoParser; import org.schabi.newpipe.extractor.search.InfoItemsSearchCollector; import org.schabi.newpipe.extractor.search.SearchExtractor; -import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper; -import org.schabi.newpipe.extractor.utils.Utils; import java.io.IOException; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; import javax.annotation.Nonnull; -import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.fixThumbnailUrl; import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getJsonResponse; import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getTextFromObject; -import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getUrlFromNavigationEndpoint; -import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.MUSIC_ALBUMS; -import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.MUSIC_ARTISTS; -import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.MUSIC_PLAYLISTS; -import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.MUSIC_SONGS; -import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.MUSIC_VIDEOS; /* * Created by Christian Schabesberger on 22.07.2018 @@ -63,278 +43,79 @@ import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeS public class YoutubeSearchExtractor extends SearchExtractor { private JsonObject initialData; - public YoutubeSearchExtractor(StreamingService service, SearchQueryHandler linkHandler) { + public YoutubeSearchExtractor(final StreamingService service, final SearchQueryHandler linkHandler) { super(service, linkHandler); } @Override - public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException { - if (isMusicSearch()) { - final String[] youtubeMusicKeys = YoutubeParsingHelper.getYoutubeMusicKeys(); + public void onFetchPage(@Nonnull final Downloader downloader) throws IOException, ExtractionException { + final String url = getUrl() + "&pbj=1"; - final String url = "https://music.youtube.com/youtubei/v1/search?alt=json&key=" + youtubeMusicKeys[0]; + final JsonArray ajaxJson = getJsonResponse(url, getExtractorLocalization()); - String params = null; - - switch (getLinkHandler().getContentFilters().get(0)) { - case MUSIC_SONGS: - params = "Eg-KAQwIARAAGAAgACgAMABqChAEEAUQAxAKEAk%3D"; - break; - case MUSIC_VIDEOS: - params = "Eg-KAQwIABABGAAgACgAMABqChAEEAUQAxAKEAk%3D"; - break; - case MUSIC_ALBUMS: - params = "Eg-KAQwIABAAGAEgACgAMABqChAEEAUQAxAKEAk%3D"; - break; - case MUSIC_PLAYLISTS: - params = "Eg-KAQwIABAAGAAgACgBMABqChAEEAUQAxAKEAk%3D"; - break; - case MUSIC_ARTISTS: - params = "Eg-KAQwIABAAGAAgASgAMABqChAEEAUQAxAKEAk%3D"; - break; - } - - // @formatter:off - byte[] json = JsonWriter.string() - .object() - .object("context") - .object("client") - .value("clientName", "WEB_REMIX") - .value("clientVersion", youtubeMusicKeys[2]) - .value("hl", "en") - .value("gl", getExtractorContentCountry().getCountryCode()) - .array("experimentIds").end() - .value("experimentsToken", "") - .value("utcOffsetMinutes", 0) - .object("locationInfo").end() - .object("musicAppInfo").end() - .end() - .object("capabilities").end() - .object("request") - .array("internalExperimentFlags").end() - .object("sessionIndex").end() - .end() - .object("activePlayers").end() - .object("user") - .value("enableSafetyMode", false) - .end() - .end() - .value("query", getSearchString()) - .value("params", params) - .end().done().getBytes("UTF-8"); - // @formatter:on - - Map> headers = new HashMap<>(); - headers.put("X-YouTube-Client-Name", Collections.singletonList(youtubeMusicKeys[1])); - headers.put("X-YouTube-Client-Version", Collections.singletonList(youtubeMusicKeys[2])); - headers.put("Origin", Collections.singletonList("https://music.youtube.com")); - headers.put("Referer", Collections.singletonList("music.youtube.com")); - headers.put("Content-Type", Collections.singletonList("application/json")); - - Response response = getDownloader().post(url, headers, json); - - if (response.responseCode() == 404) { - throw new ContentNotAvailableException("Not found" + - " (\"" + response.responseCode() + " " + response.responseMessage() + "\")"); - } - - final String responseBody = response.responseBody(); - if (responseBody.length() < 50) { // ensure to have a valid response - throw new ParsingException("JSON response is too short"); - } - - final String responseContentType = response.getHeader("Content-Type"); - if (responseContentType != null && responseContentType.toLowerCase().contains("text/html")) { - throw new ParsingException("Got HTML document, expected JSON response" + - " (latest url was: \"" + response.latestUrl() + "\")"); - } - - try { - initialData = JsonParser.object().from(responseBody); - } catch (JsonParserException e) { - throw new ParsingException("Could not parse JSON", e); - } - } else { - final String url = getUrl() + "&pbj=1"; - - final JsonArray ajaxJson = getJsonResponse(url, getExtractorLocalization()); - - initialData = ajaxJson.getObject(1).getObject("response"); - } + initialData = ajaxJson.getObject(1).getObject("response"); } @Nonnull @Override public String getUrl() throws ParsingException { - if (isMusicSearch()) return super.getUrl(); return super.getUrl() + "&gl=" + getExtractorContentCountry().getCountryCode(); } @Override public String getSearchSuggestion() throws ParsingException { - if (isMusicSearch()) { - final JsonObject itemSectionRenderer = initialData.getObject("contents").getObject("sectionListRenderer") - .getArray("contents").getObject(0).getObject("itemSectionRenderer"); - if (itemSectionRenderer == null) return ""; - final JsonObject didYouMeanRenderer = itemSectionRenderer.getArray("contents") - .getObject(0).getObject("didYouMeanRenderer"); - if (didYouMeanRenderer == null) return ""; - return getTextFromObject(didYouMeanRenderer.getObject("correctedQuery")); - } else { - JsonObject showingResultsForRenderer = initialData.getObject("contents") - .getObject("twoColumnSearchResultsRenderer").getObject("primaryContents") - .getObject("sectionListRenderer").getArray("contents").getObject(0) - .getObject("itemSectionRenderer").getArray("contents").getObject(0) - .getObject("showingResultsForRenderer"); - if (showingResultsForRenderer == null) return ""; - return getTextFromObject(showingResultsForRenderer.getObject("correctedQuery")); + final JsonObject showingResultsForRenderer = initialData.getObject("contents") + .getObject("twoColumnSearchResultsRenderer").getObject("primaryContents") + .getObject("sectionListRenderer").getArray("contents").getObject(0) + .getObject("itemSectionRenderer").getArray("contents").getObject(0) + .getObject("showingResultsForRenderer"); + if (showingResultsForRenderer == null) { + return ""; } + return getTextFromObject(showingResultsForRenderer.getObject("correctedQuery")); } @Nonnull @Override - public InfoItemsPage getInitialPage() throws ExtractionException, IOException { + public InfoItemsPage getInitialPage() throws ExtractionException { final InfoItemsSearchCollector collector = new InfoItemsSearchCollector(getServiceId()); - if (isMusicSearch()) { - final JsonArray contents = initialData.getObject("contents").getObject("sectionListRenderer").getArray("contents"); + final JsonArray sections = initialData.getObject("contents").getObject("twoColumnSearchResultsRenderer") + .getObject("primaryContents").getObject("sectionListRenderer").getArray("contents"); - for (Object content : contents) { - if (((JsonObject) content).getObject("musicShelfRenderer") != null) { - collectMusicStreamsFrom(collector, ((JsonObject) content).getObject("musicShelfRenderer").getArray("contents")); - } - } - } else { - JsonArray sections = initialData.getObject("contents").getObject("twoColumnSearchResultsRenderer") - .getObject("primaryContents").getObject("sectionListRenderer").getArray("contents"); - - for (Object section : sections) { - collectStreamsFrom(collector, ((JsonObject) section).getObject("itemSectionRenderer").getArray("contents")); - } + for (Object section : sections) { + collectStreamsFrom(collector, ((JsonObject) section).getObject("itemSectionRenderer").getArray("contents")); } return new InfoItemsPage<>(collector, getNextPageUrl()); } @Override - public String getNextPageUrl() throws ExtractionException, IOException { - if (isMusicSearch()) { - final JsonArray contents = initialData.getObject("contents").getObject("sectionListRenderer").getArray("contents"); - - for (Object content : contents) { - if (((JsonObject) content).getObject("musicShelfRenderer") != null) { - return getNextPageUrlFrom(((JsonObject) content).getObject("musicShelfRenderer").getArray("continuations")); - } - } - - return ""; - } else { - return getNextPageUrlFrom(initialData.getObject("contents").getObject("twoColumnSearchResultsRenderer") - .getObject("primaryContents").getObject("sectionListRenderer").getArray("contents") - .getObject(0).getObject("itemSectionRenderer").getArray("continuations")); - } + public String getNextPageUrl() throws ExtractionException { + return getNextPageUrlFrom(initialData.getObject("contents").getObject("twoColumnSearchResultsRenderer") + .getObject("primaryContents").getObject("sectionListRenderer").getArray("contents") + .getObject(0).getObject("itemSectionRenderer").getArray("continuations")); } @Override - public InfoItemsPage getPage(String pageUrl) throws IOException, ExtractionException { + public InfoItemsPage getPage(final String pageUrl) throws IOException, ExtractionException { if (pageUrl == null || pageUrl.isEmpty()) { throw new ExtractionException(new IllegalArgumentException("Page url is empty or null")); } final InfoItemsSearchCollector collector = new InfoItemsSearchCollector(getServiceId()); + final JsonArray ajaxJson = getJsonResponse(pageUrl, getExtractorLocalization()); - JsonArray continuations; + final JsonObject itemSectionRenderer = ajaxJson.getObject(1).getObject("response") + .getObject("continuationContents").getObject("itemSectionContinuation"); - if (isMusicSearch()) { - final String[] youtubeMusicKeys = YoutubeParsingHelper.getYoutubeMusicKeys(); - - // @formatter:off - byte[] json = JsonWriter.string() - .object() - .object("context") - .object("client") - .value("clientName", "WEB_REMIX") - .value("clientVersion", youtubeMusicKeys[2]) - .value("hl", "en") - .value("gl", getExtractorContentCountry().getCountryCode()) - .array("experimentIds").end() - .value("experimentsToken", "") - .value("utcOffsetMinutes", 0) - .object("locationInfo").end() - .object("musicAppInfo").end() - .end() - .object("capabilities").end() - .object("request") - .array("internalExperimentFlags").end() - .object("sessionIndex").end() - .end() - .object("activePlayers").end() - .object("user") - .value("enableSafetyMode", false) - .end() - .end() - .end().done().getBytes("UTF-8"); - // @formatter:on - - Map> headers = new HashMap<>(); - headers.put("X-YouTube-Client-Name", Collections.singletonList(youtubeMusicKeys[1])); - headers.put("X-YouTube-Client-Version", Collections.singletonList(youtubeMusicKeys[2])); - headers.put("Origin", Collections.singletonList("https://music.youtube.com")); - headers.put("Referer", Collections.singletonList("music.youtube.com")); - headers.put("Content-Type", Collections.singletonList("application/json")); - - Response response = getDownloader().post(pageUrl, headers, json); - - if (response.responseCode() == 404) { - throw new ContentNotAvailableException("Not found" + - " (\"" + response.responseCode() + " " + response.responseMessage() + "\")"); - } - - final String responseBody = response.responseBody(); - if (responseBody.length() < 50) { // ensure to have a valid response - throw new ParsingException("JSON response is too short"); - } - - final String responseContentType = response.getHeader("Content-Type"); - if (responseContentType != null && responseContentType.toLowerCase().contains("text/html")) { - throw new ParsingException("Got HTML document, expected JSON response" + - " (latest url was: \"" + response.latestUrl() + "\")"); - } - - final JsonObject ajaxJson; - try { - ajaxJson = JsonParser.object().from(responseBody); - } catch (JsonParserException e) { - throw new ParsingException("Could not parse JSON", e); - } - - if (ajaxJson.getObject("continuationContents") == null) { - return InfoItemsPage.emptyPage(); - } - - JsonObject musicShelfContinuation = ajaxJson.getObject("continuationContents").getObject("musicShelfContinuation"); - - collectMusicStreamsFrom(collector, musicShelfContinuation.getArray("contents")); - continuations = musicShelfContinuation.getArray("continuations"); - } else { - final JsonArray ajaxJson = getJsonResponse(pageUrl, getExtractorLocalization()); - - JsonObject itemSectionRenderer = ajaxJson.getObject(1).getObject("response") - .getObject("continuationContents").getObject("itemSectionContinuation"); - - collectStreamsFrom(collector, itemSectionRenderer.getArray("contents")); - continuations = itemSectionRenderer.getArray("continuations"); - } + collectStreamsFrom(collector, itemSectionRenderer.getArray("contents")); + final JsonArray continuations = itemSectionRenderer.getArray("continuations"); return new InfoItemsPage<>(collector, getNextPageUrlFrom(continuations)); } - private boolean isMusicSearch() { - final List contentFilters = getLinkHandler().getContentFilters(); - return contentFilters.size() > 0 && contentFilters.get(0).startsWith("music_"); - } - - private void collectStreamsFrom(InfoItemsSearchCollector collector, JsonArray videos) throws NothingFoundException, ParsingException { + private void collectStreamsFrom(final InfoItemsSearchCollector collector, final JsonArray videos) throws NothingFoundException, ParsingException { final TimeAgoParser timeAgoParser = getTimeAgoParser(); for (Object item : videos) { @@ -351,232 +132,17 @@ public class YoutubeSearchExtractor extends SearchExtractor { } } - private void collectMusicStreamsFrom(InfoItemsSearchCollector collector, JsonArray videos) { - final TimeAgoParser timeAgoParser = getTimeAgoParser(); - for (Object item : videos) { - final JsonObject info = ((JsonObject) item).getObject("musicResponsiveListItemRenderer"); - if (info != null) { - final String searchType = getLinkHandler().getContentFilters().get(0); - if (searchType.equals(MUSIC_SONGS) || searchType.equals(MUSIC_VIDEOS)) { - collector.commit(new YoutubeStreamInfoItemExtractor(info, timeAgoParser) { - @Override - public String getUrl() throws ParsingException { - String url = getUrlFromNavigationEndpoint(info.getObject("doubleTapCommand")); - if (url != null && !url.isEmpty()) return url; - throw new ParsingException("Could not get url"); - } - - @Override - public String getName() throws ParsingException { - String name = getTextFromObject(info.getArray("flexColumns").getObject(0) - .getObject("musicResponsiveListItemFlexColumnRenderer").getObject("text")); - if (name != null && !name.isEmpty()) return name; - throw new ParsingException("Could not get name"); - } - - @Override - public long getDuration() throws ParsingException { - String duration = getTextFromObject(info.getArray("flexColumns").getObject(3) - .getObject("musicResponsiveListItemFlexColumnRenderer").getObject("text")); - if (duration != null && !duration.isEmpty()) - return YoutubeParsingHelper.parseDurationString(duration); - throw new ParsingException("Could not get duration"); - } - - @Override - public String getUploaderName() throws ParsingException { - String name = getTextFromObject(info.getArray("flexColumns").getObject(1) - .getObject("musicResponsiveListItemFlexColumnRenderer").getObject("text")); - if (name != null && !name.isEmpty()) return name; - throw new ParsingException("Could not get uploader name"); - } - - @Override - public String getUploaderUrl() throws ParsingException { - String url = null; - - if (searchType.equals(MUSIC_VIDEOS)) { - JsonArray items = info.getObject("menu").getObject("menuRenderer").getArray("items"); - for (Object item : items) { - JsonObject menuNavigationItemRenderer = ((JsonObject) item).getObject("menuNavigationItemRenderer"); - if (menuNavigationItemRenderer != null && menuNavigationItemRenderer.getObject("icon").getString("iconType").equals("ARTIST")) { - url = getUrlFromNavigationEndpoint(menuNavigationItemRenderer.getObject("navigationEndpoint")); - break; - } - } - } else { - JsonObject navigationEndpoint = info.getArray("flexColumns") - .getObject(1).getObject("musicResponsiveListItemFlexColumnRenderer") - .getObject("text").getArray("runs").getObject(0).getObject("navigationEndpoint"); - if (navigationEndpoint == null) return null; - url = getUrlFromNavigationEndpoint(navigationEndpoint); - } - - if (url != null && !url.isEmpty()) return url; - throw new ParsingException("Could not get uploader url"); - } - - @Override - public String getTextualUploadDate() { - return null; - } - - @Override - public DateWrapper getUploadDate() { - return null; - } - - @Override - public long getViewCount() throws ParsingException { - if (searchType.equals(MUSIC_SONGS)) return -1; - String viewCount = getTextFromObject(info.getArray("flexColumns").getObject(2) - .getObject("musicResponsiveListItemFlexColumnRenderer").getObject("text")); - if (viewCount != null && !viewCount.isEmpty()) return Utils.mixedNumberWordToLong(viewCount); - throw new ParsingException("Could not get view count"); - } - - @Override - public String getThumbnailUrl() throws ParsingException { - try { - JsonArray thumbnails = info.getObject("thumbnail").getObject("musicThumbnailRenderer") - .getObject("thumbnail").getArray("thumbnails"); - // the last thumbnail is the one with the highest resolution - String url = thumbnails.getObject(thumbnails.size() - 1).getString("url"); - - return fixThumbnailUrl(url); - } catch (Exception e) { - throw new ParsingException("Could not get thumbnail url", e); - } - } - }); - } else if (searchType.equals(MUSIC_ARTISTS)) { - collector.commit(new YoutubeChannelInfoItemExtractor(info) { - @Override - public String getThumbnailUrl() throws ParsingException { - try { - JsonArray thumbnails = info.getObject("thumbnail").getObject("musicThumbnailRenderer") - .getObject("thumbnail").getArray("thumbnails"); - // the last thumbnail is the one with the highest resolution - String url = thumbnails.getObject(thumbnails.size() - 1).getString("url"); - - return fixThumbnailUrl(url); - } catch (Exception e) { - throw new ParsingException("Could not get thumbnail url", e); - } - } - - @Override - public String getName() throws ParsingException { - String name = getTextFromObject(info.getArray("flexColumns").getObject(0) - .getObject("musicResponsiveListItemFlexColumnRenderer").getObject("text")); - if (name != null && !name.isEmpty()) return name; - throw new ParsingException("Could not get name"); - } - - @Override - public String getUrl() throws ParsingException { - String url = getUrlFromNavigationEndpoint(info.getObject("navigationEndpoint")); - if (url != null && !url.isEmpty()) return url; - throw new ParsingException("Could not get url"); - } - - @Override - public long getSubscriberCount() throws ParsingException { - String viewCount = getTextFromObject(info.getArray("flexColumns").getObject(2) - .getObject("musicResponsiveListItemFlexColumnRenderer").getObject("text")); - if (viewCount != null && !viewCount.isEmpty()) return Utils.mixedNumberWordToLong(viewCount); - throw new ParsingException("Could not get subscriber count"); - } - - @Override - public long getStreamCount() { - return -1; - } - - @Override - public String getDescription() { - return null; - } - }); - } else if (searchType.equals(MUSIC_ALBUMS) || searchType.equals(MUSIC_PLAYLISTS)) { - collector.commit(new YoutubePlaylistInfoItemExtractor(info) { - @Override - public String getThumbnailUrl() throws ParsingException { - try { - JsonArray thumbnails = info.getObject("thumbnail").getObject("musicThumbnailRenderer") - .getObject("thumbnail").getArray("thumbnails"); - // the last thumbnail is the one with the highest resolution - String url = thumbnails.getObject(thumbnails.size() - 1).getString("url"); - - return fixThumbnailUrl(url); - } catch (Exception e) { - throw new ParsingException("Could not get thumbnail url", e); - } - } - - @Override - public String getName() throws ParsingException { - String name = getTextFromObject(info.getArray("flexColumns").getObject(0) - .getObject("musicResponsiveListItemFlexColumnRenderer").getObject("text")); - if (name != null && !name.isEmpty()) return name; - throw new ParsingException("Could not get name"); - } - - @Override - public String getUrl() throws ParsingException { - String url = getUrlFromNavigationEndpoint(info.getObject("doubleTapCommand")); - if (url != null && !url.isEmpty()) return url; - throw new ParsingException("Could not get url"); - } - - @Override - public String getUploaderName() throws ParsingException { - String name; - if (searchType.equals(MUSIC_ALBUMS)) { - name = getTextFromObject(info.getArray("flexColumns").getObject(2) - .getObject("musicResponsiveListItemFlexColumnRenderer").getObject("text")); - } else { - name = getTextFromObject(info.getArray("flexColumns").getObject(1) - .getObject("musicResponsiveListItemFlexColumnRenderer").getObject("text")); - } - if (name != null && !name.isEmpty()) return name; - throw new ParsingException("Could not get uploader name"); - } - - @Override - public long getStreamCount() throws ParsingException { - if (searchType.equals(MUSIC_ALBUMS)) return ITEM_COUNT_UNKNOWN; - String count = getTextFromObject(info.getArray("flexColumns").getObject(2) - .getObject("musicResponsiveListItemFlexColumnRenderer").getObject("text")); - if (count != null && !count.isEmpty()) { - if (count.contains("100+")) { - return ITEM_COUNT_MORE_THAN_100; - } else { - return Long.parseLong(Utils.removeNonDigitCharacters(count)); - } - } - throw new ParsingException("Could not get count"); - } - }); - } - } + private String getNextPageUrlFrom(final JsonArray continuations) throws ParsingException { + if (continuations == null) { + return ""; } - } - private String getNextPageUrlFrom(JsonArray continuations) throws ParsingException, IOException, ReCaptchaException { - if (continuations == null) return ""; + final JsonObject nextContinuationData = continuations.getObject(0).getObject("nextContinuationData"); + final String continuation = nextContinuationData.getString("continuation"); + final String clickTrackingParams = nextContinuationData.getString("clickTrackingParams"); - JsonObject nextContinuationData = continuations.getObject(0).getObject("nextContinuationData"); - String continuation = nextContinuationData.getString("continuation"); - String clickTrackingParams = nextContinuationData.getString("clickTrackingParams"); - - if (isMusicSearch()) { - return "https://music.youtube.com/youtubei/v1/search?ctoken=" + continuation + "&continuation=" + continuation - + "&itct=" + clickTrackingParams + "&alt=json&key=" + YoutubeParsingHelper.getYoutubeMusicKeys()[0]; - } else { - return getUrl() + "&pbj=1&ctoken=" + continuation + "&continuation=" + continuation - + "&itct=" + clickTrackingParams; - } + return getUrl() + "&pbj=1&ctoken=" + continuation + "&continuation=" + continuation + + "&itct=" + clickTrackingParams; } } From c7f7bd244278d52779b392500d148ccda9c1508c Mon Sep 17 00:00:00 2001 From: wb9688 Date: Wed, 1 Apr 2020 10:17:34 +0200 Subject: [PATCH 14/17] Fix error when YT Music videos has no uploader URL --- .../YoutubeMusicSearchExtractor.java | 19 ++++++++++--------- .../extractors/YoutubeSearchExtractor.java | 1 - 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java index 46e9d93c0..da4dc2b52 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java @@ -319,31 +319,32 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor { @Override public String getUploaderUrl() throws ParsingException { - String url = null; - if (searchType.equals(MUSIC_VIDEOS)) { JsonArray items = info.getObject("menu").getObject("menuRenderer").getArray("items"); for (Object item : items) { final JsonObject menuNavigationItemRenderer = ((JsonObject) item).getObject("menuNavigationItemRenderer"); if (menuNavigationItemRenderer != null && menuNavigationItemRenderer.getObject("icon").getString("iconType").equals("ARTIST")) { - url = getUrlFromNavigationEndpoint(menuNavigationItemRenderer.getObject("navigationEndpoint")); - break; + return getUrlFromNavigationEndpoint(menuNavigationItemRenderer.getObject("navigationEndpoint")); } } + + return null; } else { final JsonObject navigationEndpoint = info.getArray("flexColumns") .getObject(1).getObject("musicResponsiveListItemFlexColumnRenderer") .getObject("text").getArray("runs").getObject(0).getObject("navigationEndpoint"); + if (navigationEndpoint == null) { return null; } - url = getUrlFromNavigationEndpoint(navigationEndpoint); - } - if (url != null && !url.isEmpty()) { - return url; + final String url = getUrlFromNavigationEndpoint(navigationEndpoint); + + if (url != null && !url.isEmpty()) { + return url; + } + throw new ParsingException("Could not get uploader url"); } - throw new ParsingException("Could not get uploader url"); } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java index 06d83b243..70c47f6b1 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java @@ -132,7 +132,6 @@ public class YoutubeSearchExtractor extends SearchExtractor { } } - private String getNextPageUrlFrom(final JsonArray continuations) throws ParsingException { if (continuations == null) { return ""; From 2af610e3e7942b1d28e304fa53cb4d263c8d6403 Mon Sep 17 00:00:00 2001 From: wb9688 Date: Wed, 1 Apr 2020 10:28:55 +0200 Subject: [PATCH 15/17] Add tests for other YT Music search types --- .../YoutubeSearchQueryHandlerFactory.java | 2 +- .../YoutubeMusicSearchExtractorTest.java | 90 +++++++++++++++++++ 2 files changed, 91 insertions(+), 1 deletion(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeSearchQueryHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeSearchQueryHandlerFactory.java index 7acf7d714..866c18569 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeSearchQueryHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeSearchQueryHandlerFactory.java @@ -67,7 +67,7 @@ public class YoutubeSearchQueryHandlerFactory extends SearchQueryHandlerFactory MUSIC_SONGS, MUSIC_VIDEOS, MUSIC_ALBUMS, - MUSIC_PLAYLISTS, + MUSIC_PLAYLISTS // MUSIC_ARTISTS }; } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeMusicSearchExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeMusicSearchExtractorTest.java index 689e6127c..420db0adb 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeMusicSearchExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeMusicSearchExtractorTest.java @@ -1,6 +1,7 @@ package org.schabi.newpipe.extractor.services.youtube.search; import org.junit.BeforeClass; +import org.junit.Ignore; import org.schabi.newpipe.DownloaderTestImpl; import org.schabi.newpipe.extractor.InfoItem; import org.schabi.newpipe.extractor.NewPipe; @@ -39,6 +40,95 @@ public class YoutubeMusicSearchExtractorTest { @Override public InfoItem.InfoType expectedInfoItemType() { return InfoItem.InfoType.STREAM; } } + public static class MusicVideos extends DefaultSearchExtractorTest { + private static SearchExtractor extractor; + private static final String QUERY = "fresku"; + + @BeforeClass + public static void setUp() throws Exception { + NewPipe.init(DownloaderTestImpl.getInstance()); + extractor = YouTube.getSearchExtractor(QUERY, singletonList(YoutubeSearchQueryHandlerFactory.MUSIC_VIDEOS), ""); + extractor.fetchPage(); + } + + @Override public SearchExtractor extractor() { return extractor; } + @Override public StreamingService expectedService() { return YouTube; } + @Override public String expectedName() { return QUERY; } + @Override public String expectedId() { return QUERY; } + @Override public String expectedUrlContains() { return "music.youtube.com/search?q=" + QUERY; } + @Override public String expectedOriginalUrlContains() { return "music.youtube.com/search?q=" + QUERY; } + @Override public String expectedSearchString() { return QUERY; } + @Nullable @Override public String expectedSearchSuggestion() { return null; } + @Override public InfoItem.InfoType expectedInfoItemType() { return InfoItem.InfoType.STREAM; } + } + + public static class MusicAlbums extends DefaultSearchExtractorTest { + private static SearchExtractor extractor; + private static final String QUERY = "johnny sellah"; + + @BeforeClass + public static void setUp() throws Exception { + NewPipe.init(DownloaderTestImpl.getInstance()); + extractor = YouTube.getSearchExtractor(QUERY, singletonList(YoutubeSearchQueryHandlerFactory.MUSIC_ALBUMS), ""); + extractor.fetchPage(); + } + + @Override public SearchExtractor extractor() { return extractor; } + @Override public StreamingService expectedService() { return YouTube; } + @Override public String expectedName() { return QUERY; } + @Override public String expectedId() { return QUERY; } + @Override public String expectedUrlContains() { return "music.youtube.com/search?q=" + URLEncoder.encode(QUERY); } + @Override public String expectedOriginalUrlContains() { return "music.youtube.com/search?q=" + URLEncoder.encode(QUERY); } + @Override public String expectedSearchString() { return QUERY; } + @Nullable @Override public String expectedSearchSuggestion() { return null; } + @Override public InfoItem.InfoType expectedInfoItemType() { return InfoItem.InfoType.PLAYLIST; } + } + + public static class MusicPlaylists extends DefaultSearchExtractorTest { + private static SearchExtractor extractor; + private static final String QUERY = "louivos"; + + @BeforeClass + public static void setUp() throws Exception { + NewPipe.init(DownloaderTestImpl.getInstance()); + extractor = YouTube.getSearchExtractor(QUERY, singletonList(YoutubeSearchQueryHandlerFactory.MUSIC_PLAYLISTS), ""); + extractor.fetchPage(); + } + + @Override public SearchExtractor extractor() { return extractor; } + @Override public StreamingService expectedService() { return YouTube; } + @Override public String expectedName() { return QUERY; } + @Override public String expectedId() { return QUERY; } + @Override public String expectedUrlContains() { return "music.youtube.com/search?q=" + QUERY; } + @Override public String expectedOriginalUrlContains() { return "music.youtube.com/search?q=" + QUERY; } + @Override public String expectedSearchString() { return QUERY; } + @Nullable @Override public String expectedSearchSuggestion() { return null; } + @Override public InfoItem.InfoType expectedInfoItemType() { return InfoItem.InfoType.PLAYLIST; } + } + + @Ignore + public static class MusicArtists extends DefaultSearchExtractorTest { + private static SearchExtractor extractor; + private static final String QUERY = "kevin"; + + @BeforeClass + public static void setUp() throws Exception { + NewPipe.init(DownloaderTestImpl.getInstance()); + extractor = YouTube.getSearchExtractor(QUERY, singletonList(YoutubeSearchQueryHandlerFactory.MUSIC_ARTISTS), ""); + extractor.fetchPage(); + } + + @Override public SearchExtractor extractor() { return extractor; } + @Override public StreamingService expectedService() { return YouTube; } + @Override public String expectedName() { return QUERY; } + @Override public String expectedId() { return QUERY; } + @Override public String expectedUrlContains() { return "music.youtube.com/search?q=" + QUERY; } + @Override public String expectedOriginalUrlContains() { return "music.youtube.com/search?q=" + QUERY; } + @Override public String expectedSearchString() { return QUERY; } + @Nullable @Override public String expectedSearchSuggestion() { return null; } + @Override public InfoItem.InfoType expectedInfoItemType() { return InfoItem.InfoType.CHANNEL; } + } + public static class Suggestion extends DefaultSearchExtractorTest { private static SearchExtractor extractor; private static final String QUERY = "megaman x3"; From 8a9e137385c5e8dac62219ff40088ef5fac6ef95 Mon Sep 17 00:00:00 2001 From: wb9688 Date: Wed, 1 Apr 2020 16:01:21 +0200 Subject: [PATCH 16/17] Extract some code to getValidResponseBody() --- .../YoutubeMusicSearchExtractor.java | 39 ++----------------- .../linkHandler/YoutubeParsingHelper.java | 25 ++++++++---- 2 files changed, 21 insertions(+), 43 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java index da4dc2b52..e4251ffc7 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java @@ -9,8 +9,6 @@ import com.grack.nanojson.JsonWriter; import org.schabi.newpipe.extractor.InfoItem; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.downloader.Downloader; -import org.schabi.newpipe.extractor.downloader.Response; -import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.exceptions.ReCaptchaException; @@ -31,6 +29,7 @@ import java.util.Map; import javax.annotation.Nonnull; import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.fixThumbnailUrl; +import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getValidResponseBody; import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getTextFromObject; import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getUrlFromNavigationEndpoint; import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.MUSIC_ALBUMS; @@ -112,23 +111,7 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor { headers.put("Referer", Collections.singletonList("music.youtube.com")); headers.put("Content-Type", Collections.singletonList("application/json")); - final Response response = getDownloader().post(url, headers, json); - - if (response.responseCode() == 404) { - throw new ContentNotAvailableException("Not found" + - " (\"" + response.responseCode() + " " + response.responseMessage() + "\")"); - } - - final String responseBody = response.responseBody(); - if (responseBody.length() < 50) { // ensure to have a valid response - throw new ParsingException("JSON response is too short"); - } - - final String responseContentType = response.getHeader("Content-Type"); - if (responseContentType != null && responseContentType.toLowerCase().contains("text/html")) { - throw new ParsingException("Got HTML document, expected JSON response" + - " (latest url was: \"" + response.latestUrl() + "\")"); - } + final String responseBody = getValidResponseBody(getDownloader().post(url, headers, json)); try { initialData = JsonParser.object().from(responseBody); @@ -232,23 +215,7 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor { headers.put("Referer", Collections.singletonList("music.youtube.com")); headers.put("Content-Type", Collections.singletonList("application/json")); - final Response response = getDownloader().post(pageUrl, headers, json); - - if (response.responseCode() == 404) { - throw new ContentNotAvailableException("Not found" + - " (\"" + response.responseCode() + " " + response.responseMessage() + "\")"); - } - - final String responseBody = response.responseBody(); - if (responseBody.length() < 50) { // ensure to have a valid response - throw new ParsingException("JSON response is too short"); - } - - final String responseContentType = response.getHeader("Content-Type"); - if (responseContentType != null && responseContentType.toLowerCase().contains("text/html")) { - throw new ParsingException("Got HTML document, expected JSON response" + - " (latest url was: \"" + response.latestUrl() + "\")"); - } + final String responseBody = getValidResponseBody(getDownloader().post(pageUrl, headers, json)); final JsonObject ajaxJson; try { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java index 4ea6e0967..c639261bc 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java @@ -20,6 +20,7 @@ import org.schabi.newpipe.extractor.utils.Utils; import java.io.IOException; import java.io.UnsupportedEncodingException; +import java.net.MalformedURLException; import java.net.URL; import java.net.URLDecoder; import java.text.ParseException; @@ -427,12 +428,8 @@ public class YoutubeParsingHelper { return thumbnailUrl; } - public static JsonArray getJsonResponse(String url, Localization localization) throws IOException, ExtractionException { - Map> headers = new HashMap<>(); - headers.put("X-YouTube-Client-Name", Collections.singletonList("1")); - headers.put("X-YouTube-Client-Version", Collections.singletonList(getClientVersion())); - final Response response = getDownloader().get(url, headers, localization); - + public static String getValidResponseBody(final Response response) + throws ParsingException, MalformedURLException { if (response.responseCode() == 404) { throw new ContentNotAvailableException("Not found" + " (\"" + response.responseCode() + " " + response.responseMessage() + "\")"); @@ -453,11 +450,24 @@ public class YoutubeParsingHelper { } final String responseContentType = response.getHeader("Content-Type"); - if (responseContentType != null && responseContentType.toLowerCase().contains("text/html")) { + if (responseContentType != null + && responseContentType.toLowerCase().contains("text/html")) { throw new ParsingException("Got HTML document, expected JSON response" + " (latest url was: \"" + response.latestUrl() + "\")"); } + return responseBody; + } + + public static JsonArray getJsonResponse(final String url, final Localization localization) + throws IOException, ExtractionException { + Map> headers = new HashMap<>(); + headers.put("X-YouTube-Client-Name", Collections.singletonList("1")); + headers.put("X-YouTube-Client-Version", Collections.singletonList(getClientVersion())); + final Response response = getDownloader().get(url, headers, localization); + + final String responseBody = getValidResponseBody(response); + try { return JsonParser.array().from(responseBody); } catch (JsonParserException e) { @@ -469,6 +479,7 @@ public class YoutubeParsingHelper { * Shared alert detection function, multiple endpoints return the error similarly structured. *

* Will check if the object has an alert of the type "ERROR". + *

* * @param initialData the object which will be checked if an alert is present * @throws ContentNotAvailableException if an alert is detected From bce27a0e22ef124fc4f426c7b3512c40d79f43bb Mon Sep 17 00:00:00 2001 From: wb9688 Date: Fri, 3 Apr 2020 17:23:18 +0200 Subject: [PATCH 17/17] Rename getValidResponseBody() to getValidJsonResponseBody() --- .../youtube/extractors/YoutubeMusicSearchExtractor.java | 6 +++--- .../services/youtube/linkHandler/YoutubeParsingHelper.java | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java index e4251ffc7..51918e9ce 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java @@ -29,7 +29,7 @@ import java.util.Map; import javax.annotation.Nonnull; import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.fixThumbnailUrl; -import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getValidResponseBody; +import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getValidJsonResponseBody; import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getTextFromObject; import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getUrlFromNavigationEndpoint; import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.MUSIC_ALBUMS; @@ -111,7 +111,7 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor { headers.put("Referer", Collections.singletonList("music.youtube.com")); headers.put("Content-Type", Collections.singletonList("application/json")); - final String responseBody = getValidResponseBody(getDownloader().post(url, headers, json)); + final String responseBody = getValidJsonResponseBody(getDownloader().post(url, headers, json)); try { initialData = JsonParser.object().from(responseBody); @@ -215,7 +215,7 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor { headers.put("Referer", Collections.singletonList("music.youtube.com")); headers.put("Content-Type", Collections.singletonList("application/json")); - final String responseBody = getValidResponseBody(getDownloader().post(pageUrl, headers, json)); + final String responseBody = getValidJsonResponseBody(getDownloader().post(pageUrl, headers, json)); final JsonObject ajaxJson; try { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java index c639261bc..54bfe95c5 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java @@ -428,7 +428,7 @@ public class YoutubeParsingHelper { return thumbnailUrl; } - public static String getValidResponseBody(final Response response) + public static String getValidJsonResponseBody(final Response response) throws ParsingException, MalformedURLException { if (response.responseCode() == 404) { throw new ContentNotAvailableException("Not found" + @@ -466,7 +466,7 @@ public class YoutubeParsingHelper { headers.put("X-YouTube-Client-Version", Collections.singletonList(getClientVersion())); final Response response = getDownloader().get(url, headers, localization); - final String responseBody = getValidResponseBody(response); + final String responseBody = getValidJsonResponseBody(response); try { return JsonParser.array().from(responseBody);