From bf017bf5b9d75c40ef5a1b1f9cdba3bb1f7f5b81 Mon Sep 17 00:00:00 2001 From: Stypox Date: Wed, 11 Sep 2019 19:05:41 +0200 Subject: [PATCH 1/6] Fix TeamNewPipe/NewPipe#2615 --- .../extractors/YoutubeStreamExtractor.java | 66 ++++++++----------- 1 file changed, 29 insertions(+), 37 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index 44e77b01e..08bf50582 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -85,6 +85,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { private JsonObject playerArgs; @Nonnull private final Map videoInfoPage = new HashMap<>(); + private JsonObject playerResponse; @Nonnull private List subtitlesInfos = new ArrayList<>(); @@ -486,7 +487,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { assertPageFetched(); List videoStreams = new ArrayList<>(); try { - for (Map.Entry entry : getItags(URL_ENCODED_FMT_STREAM_MAP, ItagItem.ItagType.VIDEO).entrySet()) { + for (Map.Entry entry : getItags(ADAPTIVE_FMTS, ItagItem.ItagType.VIDEO).entrySet()) { ItagItem itag = entry.getValue(); VideoStream videoStream = new VideoStream(entry.getKey(), itag.getMediaFormat(), itag.resolutionString); @@ -620,7 +621,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { //////////////////////////////////////////////////////////////////////////*/ private static final String URL_ENCODED_FMT_STREAM_MAP = "url_encoded_fmt_stream_map"; - private static final String ADAPTIVE_FMTS = "adaptive_fmts"; + private static final String ADAPTIVE_FMTS = "adaptiveFormats"; private static final String HTTPS = "https:"; private static final String CONTENT = "content"; private static final String DECRYPTION_FUNC_NAME = "decrypt"; @@ -667,6 +668,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { playerUrl = getPlayerUrl(ytPlayerConfig); isAgeRestricted = false; } + playerResponse = getPlayerResponse(); if (decryptionCode.isEmpty()) { decryptionCode = loadDecryptionCode(playerUrl); @@ -728,6 +730,20 @@ public class YoutubeStreamExtractor extends StreamExtractor { } } + private JsonObject getPlayerResponse() throws ParsingException { + try { + String playerResponseStr; + if(playerArgs != null) { + playerResponseStr = playerArgs.getString("player_response"); + } else { + playerResponseStr = videoInfoPage.get("player_response"); + } + return JsonParser.object().from(playerResponseStr); + } catch (Exception e) { + throw new ParsingException("Could not parse yt player response", e); + } + } + @Nonnull private EmbeddedInfo getEmbeddedInfo() throws ParsingException, ReCaptchaException { try { @@ -924,45 +940,21 @@ public class YoutubeStreamExtractor extends StreamExtractor { "&sts=" + sts + "&ps=default&gl=US&hl=en"; } - private Map getItags(String encodedUrlMapKey, ItagItem.ItagType itagTypeWanted) throws ParsingException { + private Map getItags(String streamingDataKey, ItagItem.ItagType itagTypeWanted) throws ParsingException { Map urlAndItags = new LinkedHashMap<>(); - String encodedUrlMap = ""; - if (playerArgs != null && playerArgs.isString(encodedUrlMapKey)) { - encodedUrlMap = playerArgs.getString(encodedUrlMapKey, ""); - } else if (videoInfoPage.containsKey(encodedUrlMapKey)) { - encodedUrlMap = videoInfoPage.get(encodedUrlMapKey); - } + JsonArray formats = playerResponse.getObject("streamingData").getArray(streamingDataKey); + for (int i = 0; i != formats.size(); ++i) { + JsonObject formatData = formats.getObject(i); + int itag = formatData.getInt("itag"); - for (String url_data_str : encodedUrlMap.split(",")) { - try { - // This loop iterates through multiple streams, therefore tags - // is related to one and the same stream at a time. - Map tags = Parser.compatParseMap( - org.jsoup.parser.Parser.unescapeEntities(url_data_str, true)); - - int itag = Integer.parseInt(tags.get("itag")); - - if (ItagItem.isSupported(itag)) { - ItagItem itagItem = ItagItem.getItag(itag); - if (itagItem.itagType == itagTypeWanted) { - String streamUrl = tags.get("url"); - // if video has a signature: decrypt it and add it to the url - if (tags.get("s") != null) { - if (tags.get("sp") == null) { - // fallback for urls not conaining the "sp" tag - streamUrl = streamUrl + "&signature=" + decryptSignature(tags.get("s"), decryptionCode); - } - else { - streamUrl = streamUrl + "&" + tags.get("sp") + "=" + decryptSignature(tags.get("s"), decryptionCode); - } - } - urlAndItags.put(streamUrl, itagItem); - } + if (ItagItem.isSupported(itag)) { + ItagItem itagItem = ItagItem.getItag(itag); + if (itagItem.itagType == itagTypeWanted) { + String streamUrl = formatData.getString("url"); + System.out.println(streamUrl); + urlAndItags.put(streamUrl, itagItem); } - } catch (DecryptException e) { - throw e; - } catch (Exception ignored) { } } From 63a37c48e3bfce4f2a0bb2472fda4afd75f8903a Mon Sep 17 00:00:00 2001 From: Stypox Date: Wed, 11 Sep 2019 19:31:39 +0200 Subject: [PATCH 2/6] Remove println left behind --- .../services/youtube/extractors/YoutubeStreamExtractor.java | 1 - 1 file changed, 1 deletion(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index 08bf50582..32fe8d8bb 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -952,7 +952,6 @@ public class YoutubeStreamExtractor extends StreamExtractor { ItagItem itagItem = ItagItem.getItag(itag); if (itagItem.itagType == itagTypeWanted) { String streamUrl = formatData.getString("url"); - System.out.println(streamUrl); urlAndItags.put(streamUrl, itagItem); } } From d9570d8634754ca03755268bac6f25c54ac37010 Mon Sep 17 00:00:00 2001 From: Stypox Date: Wed, 11 Sep 2019 19:35:08 +0200 Subject: [PATCH 3/6] Use pre-generated playerResponse field everywhere in YtStreamExtractor --- .../extractors/YoutubeStreamExtractor.java | 28 +++---------------- 1 file changed, 4 insertions(+), 24 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index 32fe8d8bb..b4c8058e8 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -254,20 +254,6 @@ public class YoutubeStreamExtractor extends StreamExtractor { public long getLength() throws ParsingException { assertPageFetched(); - final JsonObject playerResponse; - try { - final String pr; - if(playerArgs != null) { - pr = playerArgs.getString("player_response"); - } else { - pr = videoInfoPage.get("player_response"); - } - playerResponse = JsonParser.object() - .from(pr); - } catch (Exception e) { - throw new ParsingException("Could not get playerResponse", e); - } - // try getting duration from playerargs try { String durationMs = playerResponse @@ -859,19 +845,13 @@ public class YoutubeStreamExtractor extends StreamExtractor { } catch (IOException | ExtractionException e) { throw new SubtitlesException("Unable to download player configs", e); } - final String playerResponse = playerConfig.getObject("args", new JsonObject()) - .getString("player_response"); final JsonObject captions; - try { - if (playerResponse == null || !JsonParser.object().from(playerResponse).has("captions")) { - // Captions does not exist - return Collections.emptyList(); - } - captions = JsonParser.object().from(playerResponse).getObject("captions"); - } catch (JsonParserException e) { - throw new SubtitlesException("Unable to parse subtitles listing", e); + if (!playerResponse.has("captions")) { + // Captions does not exist + return Collections.emptyList(); } + captions = playerResponse.getObject("captions"); final JsonObject renderer = captions.getObject("playerCaptionsTracklistRenderer", new JsonObject()); final JsonArray captionsArray = renderer.getArray("captionTracks", new JsonArray()); From e5e8c66686589de0c452fa75c68143152a3e1c4f Mon Sep 17 00:00:00 2001 From: Stypox Date: Wed, 11 Sep 2019 19:56:16 +0200 Subject: [PATCH 4/6] Readd signature decryption in YtStreamExtractor --- .../extractors/YoutubeStreamExtractor.java | 20 +++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index b4c8058e8..82836ce8d 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -929,10 +929,22 @@ public class YoutubeStreamExtractor extends StreamExtractor { int itag = formatData.getInt("itag"); if (ItagItem.isSupported(itag)) { - ItagItem itagItem = ItagItem.getItag(itag); - if (itagItem.itagType == itagTypeWanted) { - String streamUrl = formatData.getString("url"); - urlAndItags.put(streamUrl, itagItem); + try { + ItagItem itagItem = ItagItem.getItag(itag); + if (itagItem.itagType == itagTypeWanted) { + String streamUrl; + if (formatData.has("url")) { + streamUrl = formatData.getString("url"); + } else { + // this url has an encrypted signature + Map cipher = Parser.compatParseMap(formatData.getString("cipher")); + streamUrl = cipher.get("url") + "&" + cipher.get("sp") + "=" + decryptSignature(cipher.get("s"), decryptionCode); + } + + urlAndItags.put(streamUrl, itagItem); + } + } catch (UnsupportedEncodingException ignored) { + } } } From 9c423a0a4084d5497409798221a0871b648fd36f Mon Sep 17 00:00:00 2001 From: Stypox Date: Wed, 11 Sep 2019 20:04:28 +0200 Subject: [PATCH 5/6] Use FORMATS to get video+audio streams on yt Not ADAPTIVE_FORMATS --- .../extractors/YoutubeStreamExtractor.java | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index 82836ce8d..8f5c0c8cd 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -449,11 +449,11 @@ public class YoutubeStreamExtractor extends StreamExtractor { } @Override - public List getAudioStreams() throws IOException, ExtractionException { + public List getAudioStreams() throws ExtractionException { assertPageFetched(); List audioStreams = new ArrayList<>(); try { - for (Map.Entry entry : getItags(ADAPTIVE_FMTS, ItagItem.ItagType.AUDIO).entrySet()) { + for (Map.Entry entry : getItags(ADAPTIVE_FORMATS, ItagItem.ItagType.AUDIO).entrySet()) { ItagItem itag = entry.getValue(); AudioStream audioStream = new AudioStream(entry.getKey(), itag.getMediaFormat(), itag.avgBitrate); @@ -469,11 +469,11 @@ public class YoutubeStreamExtractor extends StreamExtractor { } @Override - public List getVideoStreams() throws IOException, ExtractionException { + public List getVideoStreams() throws ExtractionException { assertPageFetched(); List videoStreams = new ArrayList<>(); try { - for (Map.Entry entry : getItags(ADAPTIVE_FMTS, ItagItem.ItagType.VIDEO).entrySet()) { + for (Map.Entry entry : getItags(FORMATS, ItagItem.ItagType.VIDEO).entrySet()) { ItagItem itag = entry.getValue(); VideoStream videoStream = new VideoStream(entry.getKey(), itag.getMediaFormat(), itag.resolutionString); @@ -493,7 +493,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { assertPageFetched(); List videoOnlyStreams = new ArrayList<>(); try { - for (Map.Entry entry : getItags(ADAPTIVE_FMTS, ItagItem.ItagType.VIDEO_ONLY).entrySet()) { + for (Map.Entry entry : getItags(ADAPTIVE_FORMATS, ItagItem.ItagType.VIDEO_ONLY).entrySet()) { ItagItem itag = entry.getValue(); VideoStream videoStream = new VideoStream(entry.getKey(), itag.getMediaFormat(), itag.resolutionString, true); @@ -530,7 +530,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { assertPageFetched(); try { if (playerArgs != null && (playerArgs.has("ps") && playerArgs.get("ps").toString().equals("live") || - playerArgs.get(URL_ENCODED_FMT_STREAM_MAP).toString().isEmpty())) { + playerResponse.getObject("streamingData").getArray(FORMATS).isEmpty())) { return StreamType.LIVE_STREAM; } } catch (Exception e) { @@ -606,8 +606,8 @@ public class YoutubeStreamExtractor extends StreamExtractor { // Fetch page //////////////////////////////////////////////////////////////////////////*/ - private static final String URL_ENCODED_FMT_STREAM_MAP = "url_encoded_fmt_stream_map"; - private static final String ADAPTIVE_FMTS = "adaptiveFormats"; + private static final String FORMATS = "formats"; + private static final String ADAPTIVE_FORMATS = "adaptiveFormats"; private static final String HTTPS = "https:"; private static final String CONTENT = "content"; private static final String DECRYPTION_FUNC_NAME = "decrypt"; From 24a37b88a9b59519c606f15e11f5e2c2164c2fdc Mon Sep 17 00:00:00 2001 From: Stypox Date: Wed, 11 Sep 2019 20:12:30 +0200 Subject: [PATCH 6/6] Use pre-generated playerResponse field in yt's getHlsUrl() Also refactored code to always throw exception when the url can't be found --- .../extractors/YoutubeStreamExtractor.java | 21 +++++++------------ 1 file changed, 7 insertions(+), 14 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index 8f5c0c8cd..8983c1a2f 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -429,22 +429,15 @@ public class YoutubeStreamExtractor extends StreamExtractor { @Override public String getHlsUrl() throws ParsingException { assertPageFetched(); - try { - String hlsvp = ""; - if (playerArgs != null) { - if( playerArgs.isString("hlsvp") ) { - hlsvp = playerArgs.getString("hlsvp", ""); - }else { - hlsvp = JsonParser.object() - .from(playerArgs.getString("player_response", "{}")) - .getObject("streamingData", new JsonObject()) - .getString("hlsManifestUrl", ""); - } - } - return hlsvp; + try { + return playerResponse.getObject("streamingData").getString("hlsManifestUrl"); } catch (Exception e) { - throw new ParsingException("Could not get hls manifest url", e); + if (playerArgs != null && playerArgs.isString("hlsvp")) { + return playerArgs.getString("hlsvp"); + } else { + throw new ParsingException("Could not get hls manifest url", e); + } } }