From 3d38459cf37ff3d34decec4f01035861706f43b6 Mon Sep 17 00:00:00 2001
From: TiA4f8R <74829229+TiA4f8R@users.noreply.github.com>
Date: Tue, 15 Mar 2022 11:46:12 +0100
Subject: [PATCH] [YouTube] Reduce InnerTube response sizes by adding the
prettyPrint parameter with the false value
InnerTube responses return pretty printed responses, which increase responses' size for nothing.
By using the prettyPrint parameter on requests and setting its value to false, responses are not pretty printed anymore, which reduces responses size, and so data transfer and processing times.
This usage has been recently deployed by YouTube on their websites.
---
.../youtube/YoutubeParsingHelper.java | 20 ++++++++++++++-----
.../extractors/YoutubeChannelExtractor.java | 3 ++-
.../YoutubeMixPlaylistExtractor.java | 4 ++--
.../YoutubeMusicSearchExtractor.java | 2 +-
.../extractors/YoutubePlaylistExtractor.java | 3 ++-
.../extractors/YoutubeSearchExtractor.java | 3 ++-
.../YoutubeMixPlaylistExtractorTest.java | 16 ++++++++-------
7 files changed, 33 insertions(+), 18 deletions(-)
diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java
index a36c511dc..eb67a5bb2 100644
--- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java
+++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java
@@ -81,6 +81,16 @@ public final class YoutubeParsingHelper {
}
public static final String YOUTUBEI_V1_URL = "https://www.youtube.com/youtubei/v1/";
+
+ /**
+ * A parameter to disable pretty-printed response of InnerTube requests, to reduce response
+ * sizes.
+ *
+ *
+ * Sent in query parameters of the requests, after the API key.
+ *
+ **/
+ public static final String DISABLE_PRETTY_PRINT_PARAMETER = "&prettyPrint=false";
public static final String CPN = "cpn";
public static final String VIDEO_ID = "videoId";
@@ -495,7 +505,7 @@ public final class YoutubeParsingHelper {
// This endpoint is fetched by the YouTube website to get the items of its main menu and is
// pretty lightweight (around 30kB)
final Response response = getDownloader().post(YOUTUBEI_V1_URL + "guide?key="
- + HARDCODED_KEY, headers, body);
+ + HARDCODED_KEY + DISABLE_PRETTY_PRINT_PARAMETER, headers, body);
final String responseBody = response.responseBody();
final int responseCode = response.responseCode();
@@ -674,7 +684,7 @@ public final class YoutubeParsingHelper {
ReCaptchaException {
final String url =
"https://music.youtube.com/youtubei/v1/music/get_search_suggestions?alt=json&key="
- + HARDCODED_YOUTUBE_MUSIC_KEY[0];
+ + HARDCODED_YOUTUBE_MUSIC_KEY[0] + DISABLE_PRETTY_PRINT_PARAMETER;
// @formatter:off
final byte[] json = JsonWriter.string()
@@ -953,7 +963,7 @@ public final class YoutubeParsingHelper {
headers.put("Content-Type", Collections.singletonList("application/json"));
final Response response = getDownloader().post(YOUTUBEI_V1_URL + endpoint + "?key="
- + getKey(), headers, body, localization);
+ + getKey() + DISABLE_PRETTY_PRINT_PARAMETER, headers, body, localization);
return JsonUtils.toJsonObject(getValidJsonResponseBody(response));
}
@@ -972,7 +982,7 @@ public final class YoutubeParsingHelper {
headers.put("X-Goog-Api-Format-Version", Collections.singletonList("2"));
final String baseEndpointUrl = "https://youtubei.googleapis.com/youtubei/v1/" + endpoint
- + "?key=" + ANDROID_YOUTUBE_KEY;
+ + "?key=" + ANDROID_YOUTUBE_KEY + DISABLE_PRETTY_PRINT_PARAMETER;
final Response response = getDownloader().post(isNullOrEmpty(endPartOfUrlRequest)
? baseEndpointUrl : baseEndpointUrl + endPartOfUrlRequest,
@@ -995,7 +1005,7 @@ public final class YoutubeParsingHelper {
headers.put("X-Goog-Api-Format-Version", Collections.singletonList("2"));
final String baseEndpointUrl = "https://youtubei.googleapis.com/youtubei/v1/" + endpoint
- + "?key=" + IOS_YOUTUBE_KEY;
+ + "?key=" + IOS_YOUTUBE_KEY + DISABLE_PRETTY_PRINT_PARAMETER;
final Response response = getDownloader().post(isNullOrEmpty(endPartOfUrlRequest)
? baseEndpointUrl : baseEndpointUrl + endPartOfUrlRequest,
diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java
index ec04e3583..6209317da 100644
--- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java
+++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java
@@ -395,7 +395,8 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
.done())
.getBytes(UTF_8);
- return new Page(YOUTUBEI_V1_URL + "browse?key=" + getKey(), null, channelIds, null, body);
+ return new Page(YOUTUBEI_V1_URL + "browse?key=" + getKey()
+ + DISABLE_PRETTY_PRINT_PARAMETER, null, channelIds, null, body);
}
/**
diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java
index 327c10772..788d0b4db 100644
--- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java
+++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java
@@ -90,8 +90,8 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor {
final Map> headers = new HashMap<>();
addClientInfoHeaders(headers);
- final Response response = getDownloader().post(YOUTUBEI_V1_URL + "next?key=" + getKey(),
- headers, body, localization);
+ final Response response = getDownloader().post(YOUTUBEI_V1_URL + "next?key=" + getKey()
+ + DISABLE_PRETTY_PRINT_PARAMETER, headers, body, localization);
initialData = JsonUtils.toJsonObject(getValidJsonResponseBody(response));
playlistData = initialData.getObject("contents").getObject("twoColumnWatchNextResults")
diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java
index 5f45a3356..022ea1672 100644
--- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java
+++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java
@@ -60,7 +60,7 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor {
final String[] youtubeMusicKeys = YoutubeParsingHelper.getYoutubeMusicKey();
final String url = "https://music.youtube.com/youtubei/v1/search?alt=json&key="
- + youtubeMusicKeys[0];
+ + youtubeMusicKeys[0] + DISABLE_PRETTY_PRINT_PARAMETER;
final String params;
diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java
index f52608dfd..d2f2c877c 100644
--- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java
+++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java
@@ -317,7 +317,8 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
.done())
.getBytes(StandardCharsets.UTF_8);
- return new Page(YOUTUBEI_V1_URL + "browse?key=" + getKey(), body);
+ return new Page(YOUTUBEI_V1_URL + "browse?key=" + getKey()
+ + DISABLE_PRETTY_PRINT_PARAMETER, body);
} else {
return null;
}
diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java
index 895a85946..b985f7bc3 100644
--- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java
+++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java
@@ -239,7 +239,8 @@ public class YoutubeSearchExtractor extends SearchExtractor {
final String token = continuationItemRenderer.getObject("continuationEndpoint")
.getObject("continuationCommand").getString("token");
- final String url = YOUTUBEI_V1_URL + "search?key=" + getKey();
+ final String url = YOUTUBEI_V1_URL + "search?key=" + getKey()
+ + DISABLE_PRETTY_PRINT_PARAMETER;
return new Page(url, token);
}
diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeMixPlaylistExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeMixPlaylistExtractorTest.java
index 27d7d98e3..b562d6594 100644
--- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeMixPlaylistExtractorTest.java
+++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeMixPlaylistExtractorTest.java
@@ -6,9 +6,7 @@ import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl;
import static org.schabi.newpipe.extractor.ServiceList.YouTube;
-import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.YOUTUBEI_V1_URL;
-import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getKey;
-import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareDesktopJsonBuilder;
+import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.*;
import com.grack.nanojson.JsonWriter;
@@ -93,7 +91,8 @@ public class YoutubeMixPlaylistExtractorTest {
.getBytes(StandardCharsets.UTF_8);
final InfoItemsPage streams = extractor.getPage(new Page(
- YOUTUBEI_V1_URL + "next?key=" + getKey(), null, null, dummyCookie, body));
+ YOUTUBEI_V1_URL + "next?key=" + getKey() + DISABLE_PRETTY_PRINT_PARAMETER,
+ null, null, dummyCookie, body));
assertFalse(streams.getItems().isEmpty());
assertTrue(streams.hasNextPage());
}
@@ -183,7 +182,8 @@ public class YoutubeMixPlaylistExtractorTest {
.getBytes(StandardCharsets.UTF_8);
final InfoItemsPage streams = extractor.getPage(new Page(
- YOUTUBEI_V1_URL + "next?key=" + getKey(), null, null, dummyCookie, body));
+ YOUTUBEI_V1_URL + "next?key=" + getKey() + DISABLE_PRETTY_PRINT_PARAMETER,
+ null, null, dummyCookie, body));
assertFalse(streams.getItems().isEmpty());
assertTrue(streams.hasNextPage());
}
@@ -270,7 +270,8 @@ public class YoutubeMixPlaylistExtractorTest {
.getBytes(StandardCharsets.UTF_8);
final InfoItemsPage streams = extractor.getPage(new Page(
- YOUTUBEI_V1_URL + "next?key=" + getKey(), null, null, dummyCookie, body));
+ YOUTUBEI_V1_URL + "next?key=" + getKey() + DISABLE_PRETTY_PRINT_PARAMETER,
+ null, null, dummyCookie, body));
assertFalse(streams.getItems().isEmpty());
assertTrue(streams.hasNextPage());
}
@@ -389,7 +390,8 @@ public class YoutubeMixPlaylistExtractorTest {
.getBytes(StandardCharsets.UTF_8);
final InfoItemsPage streams = extractor.getPage(new Page(
- YOUTUBEI_V1_URL + "next?key=" + getKey(), null, null, dummyCookie, body));
+ YOUTUBEI_V1_URL + "next?key=" + getKey() + DISABLE_PRETTY_PRINT_PARAMETER,
+ null, null, dummyCookie, body));
assertFalse(streams.getItems().isEmpty());
assertTrue(streams.hasNextPage());
}