mirror of
https://github.com/TeamNewPipe/NewPipeExtractor
synced 2025-08-29 13:27:38 +00:00
[YouTube] Reduce InnerTube response sizes by adding the prettyPrint parameter with the false value
InnerTube responses return pretty printed responses, which increase responses' size for nothing. By using the prettyPrint parameter on requests and setting its value to false, responses are not pretty printed anymore, which reduces responses size, and so data transfer and processing times. This usage has been recently deployed by YouTube on their websites.
This commit is contained in:
parent
349ba8db7f
commit
3d38459cf3
@ -81,6 +81,16 @@ public final class YoutubeParsingHelper {
|
|||||||
}
|
}
|
||||||
|
|
||||||
public static final String YOUTUBEI_V1_URL = "https://www.youtube.com/youtubei/v1/";
|
public static final String YOUTUBEI_V1_URL = "https://www.youtube.com/youtubei/v1/";
|
||||||
|
|
||||||
|
/**
|
||||||
|
* A parameter to disable pretty-printed response of InnerTube requests, to reduce response
|
||||||
|
* sizes.
|
||||||
|
*
|
||||||
|
* <p>
|
||||||
|
* Sent in query parameters of the requests, <b>after</b> the API key.
|
||||||
|
* </p>
|
||||||
|
**/
|
||||||
|
public static final String DISABLE_PRETTY_PRINT_PARAMETER = "&prettyPrint=false";
|
||||||
public static final String CPN = "cpn";
|
public static final String CPN = "cpn";
|
||||||
public static final String VIDEO_ID = "videoId";
|
public static final String VIDEO_ID = "videoId";
|
||||||
|
|
||||||
@ -495,7 +505,7 @@ public final class YoutubeParsingHelper {
|
|||||||
// This endpoint is fetched by the YouTube website to get the items of its main menu and is
|
// This endpoint is fetched by the YouTube website to get the items of its main menu and is
|
||||||
// pretty lightweight (around 30kB)
|
// pretty lightweight (around 30kB)
|
||||||
final Response response = getDownloader().post(YOUTUBEI_V1_URL + "guide?key="
|
final Response response = getDownloader().post(YOUTUBEI_V1_URL + "guide?key="
|
||||||
+ HARDCODED_KEY, headers, body);
|
+ HARDCODED_KEY + DISABLE_PRETTY_PRINT_PARAMETER, headers, body);
|
||||||
final String responseBody = response.responseBody();
|
final String responseBody = response.responseBody();
|
||||||
final int responseCode = response.responseCode();
|
final int responseCode = response.responseCode();
|
||||||
|
|
||||||
@ -674,7 +684,7 @@ public final class YoutubeParsingHelper {
|
|||||||
ReCaptchaException {
|
ReCaptchaException {
|
||||||
final String url =
|
final String url =
|
||||||
"https://music.youtube.com/youtubei/v1/music/get_search_suggestions?alt=json&key="
|
"https://music.youtube.com/youtubei/v1/music/get_search_suggestions?alt=json&key="
|
||||||
+ HARDCODED_YOUTUBE_MUSIC_KEY[0];
|
+ HARDCODED_YOUTUBE_MUSIC_KEY[0] + DISABLE_PRETTY_PRINT_PARAMETER;
|
||||||
|
|
||||||
// @formatter:off
|
// @formatter:off
|
||||||
final byte[] json = JsonWriter.string()
|
final byte[] json = JsonWriter.string()
|
||||||
@ -953,7 +963,7 @@ public final class YoutubeParsingHelper {
|
|||||||
headers.put("Content-Type", Collections.singletonList("application/json"));
|
headers.put("Content-Type", Collections.singletonList("application/json"));
|
||||||
|
|
||||||
final Response response = getDownloader().post(YOUTUBEI_V1_URL + endpoint + "?key="
|
final Response response = getDownloader().post(YOUTUBEI_V1_URL + endpoint + "?key="
|
||||||
+ getKey(), headers, body, localization);
|
+ getKey() + DISABLE_PRETTY_PRINT_PARAMETER, headers, body, localization);
|
||||||
|
|
||||||
return JsonUtils.toJsonObject(getValidJsonResponseBody(response));
|
return JsonUtils.toJsonObject(getValidJsonResponseBody(response));
|
||||||
}
|
}
|
||||||
@ -972,7 +982,7 @@ public final class YoutubeParsingHelper {
|
|||||||
headers.put("X-Goog-Api-Format-Version", Collections.singletonList("2"));
|
headers.put("X-Goog-Api-Format-Version", Collections.singletonList("2"));
|
||||||
|
|
||||||
final String baseEndpointUrl = "https://youtubei.googleapis.com/youtubei/v1/" + endpoint
|
final String baseEndpointUrl = "https://youtubei.googleapis.com/youtubei/v1/" + endpoint
|
||||||
+ "?key=" + ANDROID_YOUTUBE_KEY;
|
+ "?key=" + ANDROID_YOUTUBE_KEY + DISABLE_PRETTY_PRINT_PARAMETER;
|
||||||
|
|
||||||
final Response response = getDownloader().post(isNullOrEmpty(endPartOfUrlRequest)
|
final Response response = getDownloader().post(isNullOrEmpty(endPartOfUrlRequest)
|
||||||
? baseEndpointUrl : baseEndpointUrl + endPartOfUrlRequest,
|
? baseEndpointUrl : baseEndpointUrl + endPartOfUrlRequest,
|
||||||
@ -995,7 +1005,7 @@ public final class YoutubeParsingHelper {
|
|||||||
headers.put("X-Goog-Api-Format-Version", Collections.singletonList("2"));
|
headers.put("X-Goog-Api-Format-Version", Collections.singletonList("2"));
|
||||||
|
|
||||||
final String baseEndpointUrl = "https://youtubei.googleapis.com/youtubei/v1/" + endpoint
|
final String baseEndpointUrl = "https://youtubei.googleapis.com/youtubei/v1/" + endpoint
|
||||||
+ "?key=" + IOS_YOUTUBE_KEY;
|
+ "?key=" + IOS_YOUTUBE_KEY + DISABLE_PRETTY_PRINT_PARAMETER;
|
||||||
|
|
||||||
final Response response = getDownloader().post(isNullOrEmpty(endPartOfUrlRequest)
|
final Response response = getDownloader().post(isNullOrEmpty(endPartOfUrlRequest)
|
||||||
? baseEndpointUrl : baseEndpointUrl + endPartOfUrlRequest,
|
? baseEndpointUrl : baseEndpointUrl + endPartOfUrlRequest,
|
||||||
|
@ -395,7 +395,8 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
|
|||||||
.done())
|
.done())
|
||||||
.getBytes(UTF_8);
|
.getBytes(UTF_8);
|
||||||
|
|
||||||
return new Page(YOUTUBEI_V1_URL + "browse?key=" + getKey(), null, channelIds, null, body);
|
return new Page(YOUTUBEI_V1_URL + "browse?key=" + getKey()
|
||||||
|
+ DISABLE_PRETTY_PRINT_PARAMETER, null, channelIds, null, body);
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -90,8 +90,8 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor {
|
|||||||
final Map<String, List<String>> headers = new HashMap<>();
|
final Map<String, List<String>> headers = new HashMap<>();
|
||||||
addClientInfoHeaders(headers);
|
addClientInfoHeaders(headers);
|
||||||
|
|
||||||
final Response response = getDownloader().post(YOUTUBEI_V1_URL + "next?key=" + getKey(),
|
final Response response = getDownloader().post(YOUTUBEI_V1_URL + "next?key=" + getKey()
|
||||||
headers, body, localization);
|
+ DISABLE_PRETTY_PRINT_PARAMETER, headers, body, localization);
|
||||||
|
|
||||||
initialData = JsonUtils.toJsonObject(getValidJsonResponseBody(response));
|
initialData = JsonUtils.toJsonObject(getValidJsonResponseBody(response));
|
||||||
playlistData = initialData.getObject("contents").getObject("twoColumnWatchNextResults")
|
playlistData = initialData.getObject("contents").getObject("twoColumnWatchNextResults")
|
||||||
|
@ -60,7 +60,7 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor {
|
|||||||
final String[] youtubeMusicKeys = YoutubeParsingHelper.getYoutubeMusicKey();
|
final String[] youtubeMusicKeys = YoutubeParsingHelper.getYoutubeMusicKey();
|
||||||
|
|
||||||
final String url = "https://music.youtube.com/youtubei/v1/search?alt=json&key="
|
final String url = "https://music.youtube.com/youtubei/v1/search?alt=json&key="
|
||||||
+ youtubeMusicKeys[0];
|
+ youtubeMusicKeys[0] + DISABLE_PRETTY_PRINT_PARAMETER;
|
||||||
|
|
||||||
final String params;
|
final String params;
|
||||||
|
|
||||||
|
@ -317,7 +317,8 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
|
|||||||
.done())
|
.done())
|
||||||
.getBytes(StandardCharsets.UTF_8);
|
.getBytes(StandardCharsets.UTF_8);
|
||||||
|
|
||||||
return new Page(YOUTUBEI_V1_URL + "browse?key=" + getKey(), body);
|
return new Page(YOUTUBEI_V1_URL + "browse?key=" + getKey()
|
||||||
|
+ DISABLE_PRETTY_PRINT_PARAMETER, body);
|
||||||
} else {
|
} else {
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
@ -239,7 +239,8 @@ public class YoutubeSearchExtractor extends SearchExtractor {
|
|||||||
final String token = continuationItemRenderer.getObject("continuationEndpoint")
|
final String token = continuationItemRenderer.getObject("continuationEndpoint")
|
||||||
.getObject("continuationCommand").getString("token");
|
.getObject("continuationCommand").getString("token");
|
||||||
|
|
||||||
final String url = YOUTUBEI_V1_URL + "search?key=" + getKey();
|
final String url = YOUTUBEI_V1_URL + "search?key=" + getKey()
|
||||||
|
+ DISABLE_PRETTY_PRINT_PARAMETER;
|
||||||
|
|
||||||
return new Page(url, token);
|
return new Page(url, token);
|
||||||
}
|
}
|
||||||
|
@ -6,9 +6,7 @@ import static org.junit.jupiter.api.Assertions.assertThrows;
|
|||||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||||
import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl;
|
import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl;
|
||||||
import static org.schabi.newpipe.extractor.ServiceList.YouTube;
|
import static org.schabi.newpipe.extractor.ServiceList.YouTube;
|
||||||
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.YOUTUBEI_V1_URL;
|
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.*;
|
||||||
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getKey;
|
|
||||||
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareDesktopJsonBuilder;
|
|
||||||
|
|
||||||
import com.grack.nanojson.JsonWriter;
|
import com.grack.nanojson.JsonWriter;
|
||||||
|
|
||||||
@ -93,7 +91,8 @@ public class YoutubeMixPlaylistExtractorTest {
|
|||||||
.getBytes(StandardCharsets.UTF_8);
|
.getBytes(StandardCharsets.UTF_8);
|
||||||
|
|
||||||
final InfoItemsPage<StreamInfoItem> streams = extractor.getPage(new Page(
|
final InfoItemsPage<StreamInfoItem> streams = extractor.getPage(new Page(
|
||||||
YOUTUBEI_V1_URL + "next?key=" + getKey(), null, null, dummyCookie, body));
|
YOUTUBEI_V1_URL + "next?key=" + getKey() + DISABLE_PRETTY_PRINT_PARAMETER,
|
||||||
|
null, null, dummyCookie, body));
|
||||||
assertFalse(streams.getItems().isEmpty());
|
assertFalse(streams.getItems().isEmpty());
|
||||||
assertTrue(streams.hasNextPage());
|
assertTrue(streams.hasNextPage());
|
||||||
}
|
}
|
||||||
@ -183,7 +182,8 @@ public class YoutubeMixPlaylistExtractorTest {
|
|||||||
.getBytes(StandardCharsets.UTF_8);
|
.getBytes(StandardCharsets.UTF_8);
|
||||||
|
|
||||||
final InfoItemsPage<StreamInfoItem> streams = extractor.getPage(new Page(
|
final InfoItemsPage<StreamInfoItem> streams = extractor.getPage(new Page(
|
||||||
YOUTUBEI_V1_URL + "next?key=" + getKey(), null, null, dummyCookie, body));
|
YOUTUBEI_V1_URL + "next?key=" + getKey() + DISABLE_PRETTY_PRINT_PARAMETER,
|
||||||
|
null, null, dummyCookie, body));
|
||||||
assertFalse(streams.getItems().isEmpty());
|
assertFalse(streams.getItems().isEmpty());
|
||||||
assertTrue(streams.hasNextPage());
|
assertTrue(streams.hasNextPage());
|
||||||
}
|
}
|
||||||
@ -270,7 +270,8 @@ public class YoutubeMixPlaylistExtractorTest {
|
|||||||
.getBytes(StandardCharsets.UTF_8);
|
.getBytes(StandardCharsets.UTF_8);
|
||||||
|
|
||||||
final InfoItemsPage<StreamInfoItem> streams = extractor.getPage(new Page(
|
final InfoItemsPage<StreamInfoItem> streams = extractor.getPage(new Page(
|
||||||
YOUTUBEI_V1_URL + "next?key=" + getKey(), null, null, dummyCookie, body));
|
YOUTUBEI_V1_URL + "next?key=" + getKey() + DISABLE_PRETTY_PRINT_PARAMETER,
|
||||||
|
null, null, dummyCookie, body));
|
||||||
assertFalse(streams.getItems().isEmpty());
|
assertFalse(streams.getItems().isEmpty());
|
||||||
assertTrue(streams.hasNextPage());
|
assertTrue(streams.hasNextPage());
|
||||||
}
|
}
|
||||||
@ -389,7 +390,8 @@ public class YoutubeMixPlaylistExtractorTest {
|
|||||||
.getBytes(StandardCharsets.UTF_8);
|
.getBytes(StandardCharsets.UTF_8);
|
||||||
|
|
||||||
final InfoItemsPage<StreamInfoItem> streams = extractor.getPage(new Page(
|
final InfoItemsPage<StreamInfoItem> streams = extractor.getPage(new Page(
|
||||||
YOUTUBEI_V1_URL + "next?key=" + getKey(), null, null, dummyCookie, body));
|
YOUTUBEI_V1_URL + "next?key=" + getKey() + DISABLE_PRETTY_PRINT_PARAMETER,
|
||||||
|
null, null, dummyCookie, body));
|
||||||
assertFalse(streams.getItems().isEmpty());
|
assertFalse(streams.getItems().isEmpty());
|
||||||
assertTrue(streams.hasNextPage());
|
assertTrue(streams.hasNextPage());
|
||||||
}
|
}
|
||||||
|
Loading…
x
Reference in New Issue
Block a user