Merge pull request #1104 from AudricV/yt_continuations-playlists-shorts-ui

build.gradle

@@ -40,12 +40,20 @@ dependencies {
 }
 
 subprojects {
-    task sourcesJar(type: Jar, dependsOn: classes) {
+    tasks.register('sourcesJar', Jar) {
+        dependsOn classes
         archiveClassifier.set('sources')
         from sourceSets.main.allSource
     }
 
-    tasks.withType(Test) {
+    // Protobuf files would uselessly end up in the JAR otherwise, see
+    // https://github.com/google/protobuf-gradle-plugin/issues/390
+    tasks.withType(Jar).configureEach {
+        exclude '**/*.proto'
+        includeEmptyDirs false
+    }
+
+    tasks.withType(Test).configureEach {
         testLogging {
             events "skipped", "failed"
             showStandardStreams = true
@@ -59,8 +67,8 @@ subprojects {
 }
 
 // https://discuss.gradle.org/t/best-approach-gradle-multi-module-project-generate-just-one-global-javadoc/18657/21
-task aggregatedJavadocs(type: Javadoc, group: 'Documentation') {
+tasks.register('aggregatedJavadocs', Javadoc) {
-    destinationDir = file("$buildDir/docs/javadoc")
+    destinationDir = file("${layout.buildDirectory}/docs/javadoc")
     title = "$project.name $version"
     // options.memberLevel = JavadocMemberLevel.PRIVATE
     options.links 'https://docs.oracle.com/javase/8/docs/api/'

extractor/build.gradle

@@ -1,5 +1,6 @@
 plugins {
-    id 'checkstyle'
+    id "checkstyle"
+    id "com.google.protobuf" version "0.9.5"
 }
 
 test {
@@ -18,12 +19,16 @@ checkstyle {
     toolVersion checkstyleVersion
 }
 
+// Exclude Protobuf generated files from Checkstyle
+checkstyleMain.exclude("org/schabi/newpipe/extractor/services/youtube/protos")
+
 checkstyleTest {
     enabled false // do not checkstyle test files
 }
 
 ext {
     rhinoVersion = '1.8.0'
+    protobufVersion = '4.30.2'
 }
 
 dependencies {
@@ -32,6 +37,7 @@ dependencies {
     implementation "com.github.TeamNewPipe:nanojson:$nanojsonVersion"
     implementation 'org.jsoup:jsoup:1.21.1'
     implementation "com.google.code.findbugs:jsr305:$jsr305Version"
+    implementation "com.google.protobuf:protobuf-javalite:$protobufVersion"
 
     implementation "org.mozilla:rhino:$rhinoVersion"
     implementation "org.mozilla:rhino-engine:$rhinoVersion"
@@ -47,3 +53,19 @@ dependencies {
     testImplementation "com.squareup.okhttp3:okhttp:4.12.0"
     testImplementation 'com.google.code.gson:gson:2.13.1'
 }
+
+protobuf {
+    protoc {
+        artifact = "com.google.protobuf:protoc:$protobufVersion"
+    }
+
+    generateProtoTasks {
+        all().configureEach { task ->
+            task.builtins {
+                java {
+                    option "lite"
+                }
+            }
+        }
+    }
+}

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java

@@ -1017,9 +1017,9 @@ public final class YoutubeParsingHelper {
         return responseBody;
     }
 
-    public static JsonObject getJsonPostResponse(final String endpoint,
+    public static JsonObject getJsonPostResponse(@Nonnull final String endpoint,
                                                  final byte[] body,
-                                                 final Localization localization)
+                                                 @Nonnull final Localization localization)
             throws IOException, ExtractionException {
         final var headers = getYouTubeHeaders();
 
@@ -1028,6 +1028,26 @@ public final class YoutubeParsingHelper {
                         + DISABLE_PRETTY_PRINT_PARAMETER, headers, body, localization)));
     }
 
+    public static JsonObject getJsonPostResponse(@Nonnull final String endpoint,
+                                                 @Nonnull final List<String> queryParameters,
+                                                 final byte[] body,
+                                                 @Nonnull final Localization localization)
+            throws IOException, ExtractionException {
+        final var headers = getYouTubeHeaders();
+
+        final String queryParametersString;
+        if (queryParameters.isEmpty()) {
+            queryParametersString = "?" + DISABLE_PRETTY_PRINT_PARAMETER;
+        } else {
+            queryParametersString = "?" + String.join("&", queryParameters)
+                    + "&" + DISABLE_PRETTY_PRINT_PARAMETER;
+        }
+
+        return JsonUtils.toJsonObject(getValidJsonResponseBody(
+                getDownloader().postWithContentTypeJson(YOUTUBEI_V1_URL + endpoint
+                        + queryParametersString, headers, body, localization)));
+    }
+
     @Nonnull
     public static JsonBuilder<JsonObject> prepareDesktopJsonBuilder(
             @Nonnull final Localization localization,

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java

@@ -8,6 +8,8 @@ import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper
 import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getImagesFromThumbnailsArray;
 import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getUrlFromNavigationEndpoint;
 import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareDesktopJsonBuilder;
+import static org.schabi.newpipe.extractor.services.youtube.protos.playlist.PlaylistProtobufContinuation.ContinuationParams;
+import static org.schabi.newpipe.extractor.services.youtube.protos.playlist.PlaylistProtobufContinuation.PlaylistContinuation;
 import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
 
 import com.grack.nanojson.JsonArray;
@@ -33,6 +35,7 @@ import org.schabi.newpipe.extractor.utils.Utils;
 
 import java.io.IOException;
 import java.nio.charset.StandardCharsets;
+import java.util.Base64;
 import java.util.List;
 
 import javax.annotation.Nonnull;
@@ -41,14 +44,17 @@ import javax.annotation.Nullable;
 public class YoutubePlaylistExtractor extends PlaylistExtractor {
     // Names of some objects in JSON response frequently used in this class
     private static final String PLAYLIST_VIDEO_RENDERER = "playlistVideoRenderer";
-    private static final String PLAYLIST_VIDEO_LIST_RENDERER = "playlistVideoListRenderer";
-    private static final String RICH_GRID_RENDERER = "richGridRenderer";
     private static final String RICH_ITEM_RENDERER = "richItemRenderer";
     private static final String REEL_ITEM_RENDERER = "reelItemRenderer";
     private static final String SIDEBAR = "sidebar";
+    private static final String HEADER = "header";
     private static final String VIDEO_OWNER_RENDERER = "videoOwnerRenderer";
+    private static final String MICROFORMAT = "microformat";
+    // Continuation properties requesting first page and showing unavailable videos
+    private static final String PLAYLIST_CONTINUATION_PROPERTIES_BASE64 = "CADCBgIIAA%3D%3D";
 
-    private JsonObject browseResponse;
+    private JsonObject browseMetadataResponse;
+    private JsonObject initialBrowseContinuationResponse;
 
     private JsonObject playlistInfo;
     private JsonObject uploaderInfo;
@@ -64,17 +70,40 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
     @Override
     public void onFetchPage(@Nonnull final Downloader downloader) throws IOException,
             ExtractionException {
+        final String playlistId = getId();
+
         final Localization localization = getExtractorLocalization();
         final byte[] body = JsonWriter.string(prepareDesktopJsonBuilder(localization,
                         getExtractorContentCountry())
-                        .value("browseId", "VL" + getId())
+                        .value("browseId", "VL" + playlistId)
                         .value("params", "wgYCCAA%3D") // Show unavailable videos
                         .done())
                 .getBytes(StandardCharsets.UTF_8);
 
-        browseResponse = getJsonPostResponse("browse", body, localization);
+        browseMetadataResponse = getJsonPostResponse("browse",
-        YoutubeParsingHelper.defaultAlertsCheck(browseResponse);
+                List.of("$fields=" + SIDEBAR + "," + HEADER + "," + MICROFORMAT + ",alerts"),
+                body,
+                localization);
+
+        YoutubeParsingHelper.defaultAlertsCheck(browseMetadataResponse);
         isNewPlaylistInterface = checkIfResponseIsNewPlaylistInterface();
+
+        final PlaylistContinuation playlistContinuation = PlaylistContinuation.newBuilder()
+                .setParameters(ContinuationParams.newBuilder()
+                        .setBrowseId("VL" + playlistId)
+                        .setPlaylistId(playlistId)
+                        .setContinuationProperties(PLAYLIST_CONTINUATION_PROPERTIES_BASE64)
+                        .build())
+                .build();
+
+        initialBrowseContinuationResponse = getJsonPostResponse("browse",
+                JsonWriter.string(prepareDesktopJsonBuilder(localization,
+                        getExtractorContentCountry())
+                        .value("continuation", Utils.encodeUrlUtf8(Base64.getUrlEncoder()
+                                .encodeToString(playlistContinuation.toByteArray())))
+                        .done())
+                        .getBytes(StandardCharsets.UTF_8),
+                localization);
     }
 
     /**
@@ -93,13 +122,13 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
      */
     private boolean checkIfResponseIsNewPlaylistInterface() {
         // The "old" playlist UI can be also returned with the new one
-        return browseResponse.has("header") && !browseResponse.has(SIDEBAR);
+        return browseMetadataResponse.has(HEADER) && !browseMetadataResponse.has(SIDEBAR);
     }
 
     @Nonnull
     private JsonObject getUploaderInfo() throws ParsingException {
         if (uploaderInfo == null) {
-            uploaderInfo = browseResponse.getObject(SIDEBAR)
+            uploaderInfo = browseMetadataResponse.getObject(SIDEBAR)
                     .getObject("playlistSidebarRenderer")
                     .getArray("items")
                     .stream()
@@ -121,7 +150,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
     @Nonnull
     private JsonObject getPlaylistInfo() throws ParsingException {
         if (playlistInfo == null) {
-            playlistInfo = browseResponse.getObject(SIDEBAR)
+            playlistInfo = browseMetadataResponse.getObject(SIDEBAR)
                     .getObject("playlistSidebarRenderer")
                     .getArray("items")
                     .stream()
@@ -139,7 +168,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
     @Nonnull
     private JsonObject getPlaylistHeader() {
         if (playlistHeader == null) {
-            playlistHeader = browseResponse.getObject("header")
+            playlistHeader = browseMetadataResponse.getObject(HEADER)
                     .getObject("playlistHeaderRenderer");
         }
 
@@ -154,7 +183,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
             return name;
         }
 
-        return browseResponse.getObject("microformat")
+        return browseMetadataResponse.getObject(MICROFORMAT)
                 .getObject("microformatDataRenderer")
                 .getString("title");
     }
@@ -180,7 +209,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
         }
 
         // This data structure is returned in both layouts
-        final JsonArray microFormatThumbnailsArray = browseResponse.getObject("microformat")
+        final JsonArray microFormatThumbnailsArray = browseMetadataResponse.getObject(MICROFORMAT)
                     .getObject("microformatDataRenderer")
                     .getObject("thumbnail")
                     .getArray("thumbnails");
@@ -302,45 +331,16 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
     @Override
     public InfoItemsPage<StreamInfoItem> getInitialPage() throws IOException, ExtractionException {
         final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
-        Page nextPage = null;
 
-        final JsonArray contents = browseResponse.getObject("contents")
+        final JsonArray initialItems = initialBrowseContinuationResponse
-                .getObject("twoColumnBrowseResultsRenderer")
+                .getArray("onResponseReceivedActions")
-                .getArray("tabs")
                 .getObject(0)
-                .getObject("tabRenderer")
+                .getObject("reloadContinuationItemsCommand")
-                .getObject("content")
+                .getArray("continuationItems");
-                .getObject("sectionListRenderer")
-                .getArray("contents");
 
-        final JsonObject videoPlaylistObject = contents.stream()
+        collectStreamsFrom(collector, initialItems);
-                .filter(JsonObject.class::isInstance)
-                .map(JsonObject.class::cast)
-                .map(content -> content.getObject("itemSectionRenderer")
-                        .getArray("contents")
-                        .getObject(0))
-                .filter(content -> content.has(PLAYLIST_VIDEO_LIST_RENDERER)
-                        || content.has(RICH_GRID_RENDERER))
-                .findFirst()
-                .orElse(null);
 
-        if (videoPlaylistObject != null) {
+        return new InfoItemsPage<>(collector, getNextPageFrom(initialItems));
-            final JsonObject renderer;
-            if (videoPlaylistObject.has(PLAYLIST_VIDEO_LIST_RENDERER)) {
-                renderer = videoPlaylistObject.getObject(PLAYLIST_VIDEO_LIST_RENDERER);
-            } else if (videoPlaylistObject.has(RICH_GRID_RENDERER)) {
-                renderer = videoPlaylistObject.getObject(RICH_GRID_RENDERER);
-            } else {
-                return new InfoItemsPage<>(collector, null);
-            }
-
-            final JsonArray videosArray = renderer.getArray("contents");
-            collectStreamsFrom(collector, videosArray);
-
-            nextPage = getNextPageFrom(videosArray);
-        }
-
-        return new InfoItemsPage<>(collector, nextPage);
     }
 
     @Override

extractor/src/main/proto/youtube/playlist/playlist_continuation_schema.proto

@@ -0,0 +1,20 @@
+syntax = "proto3";
+
+package youtube.playlists;
+
+option java_outer_classname = "PlaylistProtobufContinuation";
+option java_multiple_files = false;
+option java_package = "org.schabi.newpipe.extractor.services.youtube.protos.playlist";
+option optimize_for = LITE_RUNTIME;
+
+message PlaylistContinuation {
+  ContinuationParams parameters = 80226972;
+}
+
+message ContinuationParams {
+  // The playlist ID as a browse one (it should be "VL" + playlist ID)
+  string browseId = 2;
+  // A PlaylistContinuationProperties message safe-encoded as a Base64 string
+  string continuationProperties = 3;
+  string playlistId = 35;
+}

extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractorTest.java

@@ -481,17 +481,12 @@ public class YoutubePlaylistExtractorTest {
                     extractor.getOriginalUrl());
         }
 
-        @Disabled("Known problem, see https://github.com/TeamNewPipe/NewPipeExtractor/issues/1273")
         @Test
         @Override
         public void testRelatedItems() throws Exception {
             defaultTestRelatedItems(extractor);
         }
 
-        // TODO: enable test when continuations are available
-        @Disabled("Shorts UI doesn't return any continuation, even if when there are more than 100 "
-                + "items: this is a bug on YouTube's side, which is not related to the requirement "
-                + "of a valid visitorData like it is for Shorts channel tab")
         @Test
         @Override
         public void testMoreRelatedItems() throws Exception {

extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/channelTabs/ageRestrictedTabsShorts/generated_mock_0.json

@@ -44,10 +44,10 @@
         "same-origin; report-to\u003d\"youtube_main\""
       ],
       "date": [
-        "Sun, 10 Nov 2024 17:48:21 GMT"
+        "Fri, 18 Apr 2025 14:18:26 GMT"
       ],
       "expires": [
-        "Sun, 10 Nov 2024 17:48:21 GMT"
+        "Fri, 18 Apr 2025 14:18:26 GMT"
       ],
       "origin-trial": [
         "AmhMBR6zCLzDDxpW+HfpP67BqwIknWnyMOXOQGfzYswFmJe+fgaI6XZgAzcxOrzNtP7hEDsOo1jdjFnVr2IdxQ4AAAB4eyJvcmlnaW4iOiJodHRwczovL3lvdXR1YmUuY29tOjQ0MyIsImZlYXR1cmUiOiJXZWJWaWV3WFJlcXVlc3RlZFdpdGhEZXByZWNhdGlvbiIsImV4cGlyeSI6MTc1ODA2NzE5OSwiaXNTdWJkb21haW4iOnRydWV9"
@@ -65,8 +65,8 @@
         "ESF"
       ],
       "set-cookie": [
-        "YSC\u003d4hCTcf7rUXA; Domain\u003d.youtube.com; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone",
+        "YSC\u003dQCKW-aKaIuY; Domain\u003d.youtube.com; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone",
-        "VISITOR_INFO1_LIVE\u003d; Domain\u003d.youtube.com; Expires\u003dMon, 14-Feb-2022 17:48:21 GMT; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone"
+        "VISITOR_INFO1_LIVE\u003d; Domain\u003d.youtube.com; Expires\u003dSat, 23-Jul-2022 14:18:26 GMT; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone"
       ],
       "strict-transport-security": [
         "max-age\u003d31536000"

extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/channelTabs/ageRestrictedTabsVideos/generated_mock_0.json

@@ -3,10 +3,10 @@
     "httpMethod": "GET",
     "url": "https://www.youtube.com/sw.js",
     "headers": {
-      "Origin": [
+      "Referer": [
         "https://www.youtube.com"
       ],
-      "Referer": [
+      "Origin": [
         "https://www.youtube.com"
       ],
       "Accept-Language": [
@@ -44,10 +44,10 @@
         "same-origin; report-to\u003d\"youtube_main\""
       ],
       "date": [
-        "Sun, 10 Nov 2024 17:48:58 GMT"
+        "Fri, 18 Apr 2025 14:18:37 GMT"
       ],
       "expires": [
-        "Sun, 10 Nov 2024 17:48:58 GMT"
+        "Fri, 18 Apr 2025 14:18:37 GMT"
       ],
       "origin-trial": [
         "AmhMBR6zCLzDDxpW+HfpP67BqwIknWnyMOXOQGfzYswFmJe+fgaI6XZgAzcxOrzNtP7hEDsOo1jdjFnVr2IdxQ4AAAB4eyJvcmlnaW4iOiJodHRwczovL3lvdXR1YmUuY29tOjQ0MyIsImZlYXR1cmUiOiJXZWJWaWV3WFJlcXVlc3RlZFdpdGhEZXByZWNhdGlvbiIsImV4cGlyeSI6MTc1ODA2NzE5OSwiaXNTdWJkb21haW4iOnRydWV9"
@@ -65,8 +65,8 @@
         "ESF"
       ],
       "set-cookie": [
-        "YSC\u003dXTJBIfCZjcU; Domain\u003d.youtube.com; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone",
+        "YSC\u003dwJABBPlcVn0; Domain\u003d.youtube.com; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone",
-        "VISITOR_INFO1_LIVE\u003d; Domain\u003d.youtube.com; Expires\u003dMon, 14-Feb-2022 17:48:58 GMT; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone"
+        "VISITOR_INFO1_LIVE\u003d; Domain\u003d.youtube.com; Expires\u003dSat, 23-Jul-2022 14:18:37 GMT; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone"
       ],
       "strict-transport-security": [
         "max-age\u003d31536000"

extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/playlist/TimelessPopHits/generated_mock_0.json

@@ -44,10 +44,10 @@
         "same-origin; report-to\u003d\"youtube_main\""
       ],
       "date": [
-        "Sun, 10 Nov 2024 17:54:27 GMT"
+        "Fri, 18 Apr 2025 14:06:42 GMT"
       ],
       "expires": [
-        "Sun, 10 Nov 2024 17:54:27 GMT"
+        "Fri, 18 Apr 2025 14:06:42 GMT"
       ],
       "origin-trial": [
         "AmhMBR6zCLzDDxpW+HfpP67BqwIknWnyMOXOQGfzYswFmJe+fgaI6XZgAzcxOrzNtP7hEDsOo1jdjFnVr2IdxQ4AAAB4eyJvcmlnaW4iOiJodHRwczovL3lvdXR1YmUuY29tOjQ0MyIsImZlYXR1cmUiOiJXZWJWaWV3WFJlcXVlc3RlZFdpdGhEZXByZWNhdGlvbiIsImV4cGlyeSI6MTc1ODA2NzE5OSwiaXNTdWJkb21haW4iOnRydWV9"
@@ -65,8 +65,8 @@
         "ESF"
       ],
       "set-cookie": [
-        "YSC\u003dT6Ir-R5w6Xs; Domain\u003d.youtube.com; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone",
+        "YSC\u003dXLTJQnQA_yI; Domain\u003d.youtube.com; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone",
-        "VISITOR_INFO1_LIVE\u003d; Domain\u003d.youtube.com; Expires\u003dMon, 14-Feb-2022 17:54:27 GMT; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone"
+        "VISITOR_INFO1_LIVE\u003d; Domain\u003d.youtube.com; Expires\u003dSat, 23-Jul-2022 14:06:42 GMT; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone"
       ],
       "strict-transport-security": [
         "max-age\u003d31536000"

extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/playlist/continuations/generated_mock_0.json

@@ -44,10 +44,10 @@
         "same-origin; report-to\u003d\"youtube_main\""
       ],
       "date": [
-        "Sun, 10 Nov 2024 17:53:34 GMT"
+        "Fri, 18 Apr 2025 14:07:37 GMT"
       ],
       "expires": [
-        "Sun, 10 Nov 2024 17:53:34 GMT"
+        "Fri, 18 Apr 2025 14:07:37 GMT"
       ],
       "origin-trial": [
         "AmhMBR6zCLzDDxpW+HfpP67BqwIknWnyMOXOQGfzYswFmJe+fgaI6XZgAzcxOrzNtP7hEDsOo1jdjFnVr2IdxQ4AAAB4eyJvcmlnaW4iOiJodHRwczovL3lvdXR1YmUuY29tOjQ0MyIsImZlYXR1cmUiOiJXZWJWaWV3WFJlcXVlc3RlZFdpdGhEZXByZWNhdGlvbiIsImV4cGlyeSI6MTc1ODA2NzE5OSwiaXNTdWJkb21haW4iOnRydWV9"
@@ -65,8 +65,8 @@
         "ESF"
       ],
       "set-cookie": [
-        "YSC\u003dIQFgVpm5PIE; Domain\u003d.youtube.com; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone",
+        "YSC\u003dAglz5BMfZGU; Domain\u003d.youtube.com; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone",
-        "VISITOR_INFO1_LIVE\u003d; Domain\u003d.youtube.com; Expires\u003dMon, 14-Feb-2022 17:53:34 GMT; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone"
+        "VISITOR_INFO1_LIVE\u003d; Domain\u003d.youtube.com; Expires\u003dSat, 23-Jul-2022 14:07:37 GMT; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone"
       ],
       "strict-transport-security": [
         "max-age\u003d31536000"

extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/playlist/huge/generated_mock_0.json

@@ -3,10 +3,10 @@
     "httpMethod": "GET",
     "url": "https://www.youtube.com/sw.js",
     "headers": {
-      "Origin": [
+      "Referer": [
         "https://www.youtube.com"
       ],
-      "Referer": [
+      "Origin": [
         "https://www.youtube.com"
       ],
       "Accept-Language": [
@@ -44,10 +44,10 @@
         "same-origin; report-to\u003d\"youtube_main\""
       ],
       "date": [
-        "Sun, 10 Nov 2024 17:53:35 GMT"
+        "Fri, 18 Apr 2025 14:06:52 GMT"
       ],
       "expires": [
-        "Sun, 10 Nov 2024 17:53:35 GMT"
+        "Fri, 18 Apr 2025 14:06:52 GMT"
       ],
       "origin-trial": [
         "AmhMBR6zCLzDDxpW+HfpP67BqwIknWnyMOXOQGfzYswFmJe+fgaI6XZgAzcxOrzNtP7hEDsOo1jdjFnVr2IdxQ4AAAB4eyJvcmlnaW4iOiJodHRwczovL3lvdXR1YmUuY29tOjQ0MyIsImZlYXR1cmUiOiJXZWJWaWV3WFJlcXVlc3RlZFdpdGhEZXByZWNhdGlvbiIsImV4cGlyeSI6MTc1ODA2NzE5OSwiaXNTdWJkb21haW4iOnRydWV9"
@@ -65,8 +65,8 @@
         "ESF"
       ],
       "set-cookie": [
-        "YSC\u003dCVf2q0qaHt4; Domain\u003d.youtube.com; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone",
+        "YSC\u003deJou_uqKNwk; Domain\u003d.youtube.com; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone",
-        "VISITOR_INFO1_LIVE\u003d; Domain\u003d.youtube.com; Expires\u003dMon, 14-Feb-2022 17:53:35 GMT; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone"
+        "VISITOR_INFO1_LIVE\u003d; Domain\u003d.youtube.com; Expires\u003dSat, 23-Jul-2022 14:06:52 GMT; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone"
       ],
       "strict-transport-security": [
         "max-age\u003d31536000"

extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/playlist/learning/generated_mock_0.json

@@ -3,10 +3,10 @@
     "httpMethod": "GET",
     "url": "https://www.youtube.com/sw.js",
     "headers": {
-      "Origin": [
+      "Referer": [
         "https://www.youtube.com"
       ],
-      "Referer": [
+      "Origin": [
         "https://www.youtube.com"
       ],
       "Accept-Language": [
@@ -44,10 +44,10 @@
         "same-origin; report-to\u003d\"youtube_main\""
       ],
       "date": [
-        "Sun, 10 Nov 2024 17:54:05 GMT"
+        "Fri, 18 Apr 2025 14:07:10 GMT"
       ],
       "expires": [
-        "Sun, 10 Nov 2024 17:54:05 GMT"
+        "Fri, 18 Apr 2025 14:07:10 GMT"
       ],
       "origin-trial": [
         "AmhMBR6zCLzDDxpW+HfpP67BqwIknWnyMOXOQGfzYswFmJe+fgaI6XZgAzcxOrzNtP7hEDsOo1jdjFnVr2IdxQ4AAAB4eyJvcmlnaW4iOiJodHRwczovL3lvdXR1YmUuY29tOjQ0MyIsImZlYXR1cmUiOiJXZWJWaWV3WFJlcXVlc3RlZFdpdGhEZXByZWNhdGlvbiIsImV4cGlyeSI6MTc1ODA2NzE5OSwiaXNTdWJkb21haW4iOnRydWV9"
@@ -65,8 +65,8 @@
         "ESF"
       ],
       "set-cookie": [
-        "YSC\u003dvqHCAJkmPgU; Domain\u003d.youtube.com; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone",
+        "YSC\u003dzryyiwCAVoo; Domain\u003d.youtube.com; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone",
-        "VISITOR_INFO1_LIVE\u003d; Domain\u003d.youtube.com; Expires\u003dMon, 14-Feb-2022 17:54:05 GMT; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone"
+        "VISITOR_INFO1_LIVE\u003d; Domain\u003d.youtube.com; Expires\u003dSat, 23-Jul-2022 14:07:10 GMT; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone"
       ],
       "strict-transport-security": [
         "max-age\u003d31536000"

extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/playlist/notAvailable/generated_mock_0.json

@@ -3,10 +3,10 @@
     "httpMethod": "GET",
     "url": "https://www.youtube.com/sw.js",
     "headers": {
-      "Origin": [
+      "Referer": [
         "https://www.youtube.com"
       ],
-      "Referer": [
+      "Origin": [
         "https://www.youtube.com"
       ],
       "Accept-Language": [
@@ -44,10 +44,10 @@
         "same-origin; report-to\u003d\"youtube_main\""
       ],
       "date": [
-        "Sun, 10 Nov 2024 17:54:06 GMT"
+        "Fri, 18 Apr 2025 14:06:26 GMT"
       ],
       "expires": [
-        "Sun, 10 Nov 2024 17:54:06 GMT"
+        "Fri, 18 Apr 2025 14:06:26 GMT"
       ],
       "origin-trial": [
         "AmhMBR6zCLzDDxpW+HfpP67BqwIknWnyMOXOQGfzYswFmJe+fgaI6XZgAzcxOrzNtP7hEDsOo1jdjFnVr2IdxQ4AAAB4eyJvcmlnaW4iOiJodHRwczovL3lvdXR1YmUuY29tOjQ0MyIsImZlYXR1cmUiOiJXZWJWaWV3WFJlcXVlc3RlZFdpdGhEZXByZWNhdGlvbiIsImV4cGlyeSI6MTc1ODA2NzE5OSwiaXNTdWJkb21haW4iOnRydWV9"
@@ -65,8 +65,8 @@
         "ESF"
       ],
       "set-cookie": [
-        "YSC\u003d5IYREKOqWBI; Domain\u003d.youtube.com; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone",
+        "YSC\u003dy6ur9A1xR5s; Domain\u003d.youtube.com; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone",
-        "VISITOR_INFO1_LIVE\u003d; Domain\u003d.youtube.com; Expires\u003dMon, 14-Feb-2022 17:54:06 GMT; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone"
+        "VISITOR_INFO1_LIVE\u003d; Domain\u003d.youtube.com; Expires\u003dSat, 23-Jul-2022 14:06:26 GMT; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone"
       ],
       "strict-transport-security": [
         "max-age\u003d31536000"

extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/playlist/notAvailable/generated_mock_2.json

@@ -1,19 +1,19 @@
 {
   "request": {
     "httpMethod": "POST",
-    "url": "https://www.youtube.com/youtubei/v1/browse?prettyPrint\u003dfalse",
+    "url": "https://www.youtube.com/youtubei/v1/browse?$fields\u003dsidebar,header,microformat,alerts\u0026prettyPrint\u003dfalse",
     "headers": {
-      "Origin": [
+      "Referer": [
         "https://www.youtube.com"
       ],
-      "Referer": [
+      "Origin": [
         "https://www.youtube.com"
       ],
       "Cookie": [
         "SOCS\u003dCAE\u003d"
       ],
       "X-YouTube-Client-Version": [
-        "2.20241107.11.00"
+        "2.20250417.01.00"
       ],
       "X-YouTube-Client-Name": [
         "1"
@@ -254,13 +254,13 @@
       50,
       48,
       50,
+      53,
+      48,
       52,
       49,
-      49,
-      48,
       55,
       46,
-      49,
+      48,
       49,
       46,
       48,
@@ -359,14 +359,11 @@
       "alt-svc": [
         "h3\u003d\":443\"; ma\u003d2592000,h3-29\u003d\":443\"; ma\u003d2592000"
       ],
-      "cache-control": [
-        "private"
-      ],
       "content-type": [
         "application/json; charset\u003dUTF-8"
       ],
       "date": [
-        "Sun, 10 Nov 2024 17:54:06 GMT"
+        "Fri, 18 Apr 2025 14:06:28 GMT"
       ],
       "server": [
         "scaffolding on HTTPServer2"
@@ -386,7 +383,7 @@
         "0"
       ]
     },
-    "responseBody": "{\"responseContext\":{\"visitorData\":\"Cgs4SXRCWTFxTnZIZyi-5sO5BjIKCgJERRIEEgAgEw%3D%3D\",\"serviceTrackingParams\":[{\"service\":\"GFEEDBACK\",\"params\":[{\"key\":\"browse_id\",\"value\":\"VLINVALID_ID\"},{\"key\":\"browse_id_prefix\",\"value\":\"\"},{\"key\":\"logged_in\",\"value\":\"0\"},{\"key\":\"e\",\"value\":\"9406122,23804281,23880831,23880837,23930656,23966208,24004644,24007613,24077241,24166867,24181174,24241378,24439361,24453989,24457856,24466624,24499533,24542367,24547316,24548629,24566687,24699899,39325854,39326986,51009781,51010235,51017346,51020570,51025415,51030103,51037346,51037349,51050361,51053689,51057842,51057857,51063643,51064835,51072748,51091058,51095478,51098299,51101169,51111738,51115184,51117319,51124104,51129210,51133103,51138233,51141472,51151424,51152050,51157411,51157841,51158514,51160545,51165467,51169118,51176511,51178320,51178331,51178344,51178355,51178982,51182851,51183910,51184990,51194137,51195231,51204329,51213773,51217504,51221148,51222382,51222973,51223962,51227037,51227292,51227774,51228350,51228849,51230241,51230478,51231814,51237842,51239093,51241028,51242448,51243940,51248255,51248734,51249751,51251836,51255676,51255680,51255743,51256074,51256084,51257902,51257911,51257918,51258066,51259135,51263449,51265339,51265362,51265373,51266454,51272458,51272663,51273608,51274583,51275785,51276557,51276565,51281227,51282069,51282088,51282792,51285052,51285421,51285717,51287196,51287500,51287509,51287510,51289924,51289935,51289938,51289956,51289963,51289972,51290034,51292054,51294322,51295132,51295578,51296439,51298019,51298021,51299627,51299710,51299724,51299971,51299997,51300014,51300176,51300241,51300532,51300699,51302492,51302680,51303667,51303670,51303789,51304155,51305531,51305839,51306259,51307502,51308045,51308060,51308199,51309314,51310323,51310742,51311029,51311038,51311505,51312146,51312688,51313149,51313767,51314158,51314677,51314692,51314707,51314714,51314729,51315041,51315912,51315921,51315926,51315931,51315940,51315949,51315952,51315963,51315970,51315975,51317748,51318207,51318844,51323366,51325576,51326207,51326641,51326932,51327138,51327167,51327182,51327636,51328144,51329227,51329505,51330194,51331487,51331502,51331518,51331533,51331540,51331547,51331552,51331561,51332896,51333541,51333739,51333879,51335366,51335646,51336953,51337187,51337349,51337456,51338523,51339118,51339163,51339517,51339747,51340611,51341226,51342093,51342147,51342741,51343109,51343368,51344412,51344672\"},{\"key\":\"visitor_data\",\"value\":\"Cgs4SXRCWTFxTnZIZyi-5sO5BjIKCgJERRIEEgAgEw%3D%3D\"}]},{\"service\":\"GOOGLE_HELP\",\"params\":[{\"key\":\"browse_id\",\"value\":\"VLINVALID_ID\"},{\"key\":\"browse_id_prefix\",\"value\":\"\"}]},{\"service\":\"CSI\",\"params\":[{\"key\":\"c\",\"value\":\"WEB\"},{\"key\":\"cver\",\"value\":\"2.20241107.11.00\"},{\"key\":\"yt_li\",\"value\":\"0\"},{\"key\":\"GetPlaylist_rid\",\"value\":\"0x83b6c228fe54a2be\"}]},{\"service\":\"GUIDED_HELP\",\"params\":[{\"key\":\"logged_in\",\"value\":\"0\"}]},{\"service\":\"ECATCHER\",\"params\":[{\"key\":\"client.version\",\"value\":\"2.20241107\"},{\"key\":\"client.name\",\"value\":\"WEB\"}]}],\"mainAppWebResponseContext\":{\"loggedOut\":true,\"trackingParam\":\"kx_fmPxhoPZRLF3BvyekUnTEMFagcMtxocsRMtk9YkCx4MwRgkuswmIBwOcCE59TDtslLKPQ-SS\"},\"webResponseContextExtensionData\":{\"hasDecorated\":true}},\"alerts\":[{\"alertRenderer\":{\"type\":\"ERROR\",\"text\":{\"runs\":[{\"text\":\"This playlist type is unviewable.\"}]}}}],\"trackingParams\":\"CAAQhGciEwjijIjdqtKJAxWxfkECHfNoO94\u003d\",\"microformat\":{\"microformatDataRenderer\":{\"noindex\":true}}}",
+    "responseBody": "{\"alerts\":[{\"alertRenderer\":{\"type\":\"ERROR\",\"text\":{\"runs\":[{\"text\":\"This playlist type is unviewable.\"}]}}}],\"microformat\":{\"microformatDataRenderer\":{\"noindex\":true}}}",
-    "latestUrl": "https://www.youtube.com/youtubei/v1/browse?prettyPrint\u003dfalse"
+    "latestUrl": "https://www.youtube.com/youtubei/v1/browse?$fields\u003dsidebar,header,microformat,alerts\u0026prettyPrint\u003dfalse"
   }
 }

extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/playlist/shortsUI/generated_mock_0.json

@@ -44,10 +44,10 @@
         "same-origin; report-to\u003d\"youtube_main\""
       ],
       "date": [
-        "Sun, 10 Nov 2024 17:54:26 GMT"
+        "Fri, 18 Apr 2025 14:07:27 GMT"
       ],
       "expires": [
-        "Sun, 10 Nov 2024 17:54:26 GMT"
+        "Fri, 18 Apr 2025 14:07:27 GMT"
       ],
       "origin-trial": [
         "AmhMBR6zCLzDDxpW+HfpP67BqwIknWnyMOXOQGfzYswFmJe+fgaI6XZgAzcxOrzNtP7hEDsOo1jdjFnVr2IdxQ4AAAB4eyJvcmlnaW4iOiJodHRwczovL3lvdXR1YmUuY29tOjQ0MyIsImZlYXR1cmUiOiJXZWJWaWV3WFJlcXVlc3RlZFdpdGhEZXByZWNhdGlvbiIsImV4cGlyeSI6MTc1ODA2NzE5OSwiaXNTdWJkb21haW4iOnRydWV9"
@@ -65,8 +65,8 @@
         "ESF"
       ],
       "set-cookie": [
-        "YSC\u003d14OiXUBwaQw; Domain\u003d.youtube.com; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone",
+        "YSC\u003dxBHU9P8-EM8; Domain\u003d.youtube.com; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone",
-        "VISITOR_INFO1_LIVE\u003d; Domain\u003d.youtube.com; Expires\u003dMon, 14-Feb-2022 17:54:27 GMT; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone"
+        "VISITOR_INFO1_LIVE\u003d; Domain\u003d.youtube.com; Expires\u003dSat, 23-Jul-2022 14:07:27 GMT; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone"
       ],
       "strict-transport-security": [
         "max-age\u003d31536000"