[yandexdisk] fix extraction(closes #17861)(closes #27131)

test/test_utils.py

@@ -554,6 +554,11 @@ class TestUtil(unittest.TestCase):
         self.assertEqual(url_or_none('http$://foo.de'), None)
         self.assertEqual(url_or_none('http://foo.de'), 'http://foo.de')
         self.assertEqual(url_or_none('//foo.de'), '//foo.de')
+        self.assertEqual(url_or_none('s3://foo.de'), None)
+        self.assertEqual(url_or_none('rtmpte://foo.de'), 'rtmpte://foo.de')
+        self.assertEqual(url_or_none('mms://foo.de'), 'mms://foo.de')
+        self.assertEqual(url_or_none('rtspu://foo.de'), 'rtspu://foo.de')
+        self.assertEqual(url_or_none('ftps://foo.de'), 'ftps://foo.de')
 
     def test_parse_age_limit(self):
         self.assertEqual(parse_age_limit(None), None)

youtube_dl/extractor/yandexdisk.py

@@ -1,19 +1,40 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
+import json
+
 from .common import InfoExtractor
-from ..compat import compat_str
+from ..compat import compat_HTTPError
 from ..utils import (
     determine_ext,
+    ExtractorError,
     float_or_none,
     int_or_none,
-    try_get,
+    mimetype2ext,
-    urlencode_postdata,
+    parse_iso8601,
+    urljoin,
 )
 
 
 class YandexDiskIE(InfoExtractor):
-    _VALID_URL = r'https?://yadi\.sk/[di]/(?P<id>[^/?#&]+)'
+    _VALID_URL = r'''(?x)https?://
+        (?:
+            (?:www\.)?yadi\.sk|
+            disk\.yandex\.
+                (?:
+                    az|
+                    by|
+                    co(?:m(?:\.(?:am|ge|tr))?|\.il)|
+                    ee|
+                    fr|
+                    k[gz]|
+                    l[tv]|
+                    md|
+                    t[jm]|
+                    u[az]|
+                    ru
+                )
+        )/(?:[di]/|public.*?\bhash=)(?P<id>[^/?#&]+)'''
 
     _TESTS = [{
         'url': 'https://yadi.sk/i/VdOeDou8eZs6Y',
@@ -25,94 +46,97 @@ class YandexDiskIE(InfoExtractor):
             'duration': 168.6,
             'uploader': 'y.botova',
             'uploader_id': '300043621',
+            'timestamp': 1421396809,
+            'upload_date': '20150116',
             'view_count': int,
         },
     }, {
         'url': 'https://yadi.sk/d/h3WAXvDS3Li3Ce',
         'only_matching': True,
+    }, {
+        'url': 'https://yadi.sk/public?hash=5DZ296JK9GWCLp02f6jrObjnctjRxMs8L6%2B%2FuhNqk38%3D',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
-        status = self._download_webpage(
+        try:
-            'https://disk.yandex.com/auth/status', video_id, query={
+            resource = self._download_json(
-                'urlOrigin': url,
+                'https://cloud-api.yandex.net/v1/disk/public/resources',
-                'source': 'public',
+                video_id, query={'public_key': url})
-                'md5': 'false',
+        except ExtractorError as e:
-            })
+            if isinstance(e.cause, compat_HTTPError) and e.cause.code == 403:
+                error_description = self._parse_json(
+                    e.cause.read().decode(), video_id)['description']
+                raise ExtractorError(error_description, expected=True)
+            raise
 
-        sk = self._search_regex(
+        title = resource['name']
-            r'(["\'])sk(?:External)?\1\s*:\s*(["\'])(?P<value>(?:(?!\2).)+)\2',
+        public_url = resource.get('public_url')
-            status, 'sk', group='value')
+        if public_url:
+            video_id = self._match_id(public_url)
 
-        webpage = self._download_webpage(url, video_id)
+        self._set_cookie('yadi.sk', 'yandexuid', '0')
 
-        models = self._parse_json(
+        def call_api(action):
-            self._search_regex(
+            return (self._download_json(
-                r'<script[^>]+id=["\']models-client[^>]+>\s*(\[.+?\])\s*</script',
+                urljoin(url, '/public/api/') + action, video_id, data=json.dumps({
-                webpage, 'video JSON'),
+                    'hash': url,
-            video_id)
+                    # obtain sk if needed from call_api('check-auth') while
-
+                    # the yandexuid cookie is set and sending an empty JSON object
-        data = next(
+                    'sk': 'ya6b52f8c6b12abe91a66d22d3a31084b'
-            model['data'] for model in models
+                }).encode(), headers={
-            if model.get('model') == 'resource')
+                    'Content-Type': 'text/plain',
-
+                }, fatal=False) or {}).get('data') or {}
-        video_hash = data['id']
-        title = data['name']
-
-        models = self._download_json(
-            'https://disk.yandex.com/models/', video_id,
-            data=urlencode_postdata({
-                '_model.0': 'videoInfo',
-                'id.0': video_hash,
-                '_model.1': 'do-get-resource-url',
-                'id.1': video_hash,
-                'version': '13.6',
-                'sk': sk,
-            }), query={'_m': 'videoInfo'})['models']
-
-        videos = try_get(models, lambda x: x[0]['data']['videos'], list) or []
-        source_url = try_get(
-            models, lambda x: x[1]['data']['file'], compat_str)
 
         formats = []
+        source_url = resource.get('file')
+        if not source_url:
+            source_url = call_api('download-url').get('url')
         if source_url:
             formats.append({
                 'url': source_url,
                 'format_id': 'source',
-                'ext': determine_ext(title, 'mp4'),
+                'ext': determine_ext(title, mimetype2ext(resource.get('mime_type')) or 'mp4'),
                 'quality': 1,
+                'filesize': int_or_none(resource.get('size'))
             })
-        for video in videos:
+
+        video_streams = call_api('get-video-streams')
+        for video in (video_streams.get('videos') or []):
             format_url = video.get('url')
             if not format_url:
                 continue
-            if determine_ext(format_url) == 'm3u8':
+            if video.get('dimension') == 'adaptive':
                 formats.extend(self._extract_m3u8_formats(
-                    format_url, video_id, 'mp4', entry_protocol='m3u8_native',
+                    format_url, video_id, 'mp4', 'm3u8_native',
                     m3u8_id='hls', fatal=False))
             else:
+                size = video.get('size') or {}
+                height = int_or_none(size.get('height'))
+                format_id = 'hls'
+                if height:
+                    format_id += '-%dp' % height
                 formats.append({
+                    'ext': 'mp4',
+                    'format_id': format_id,
+                    'height': height,
+                    'protocol': 'm3u8_native',
                     'url': format_url,
+                    'width': int_or_none(size.get('width')),
                 })
         self._sort_formats(formats)
 
-        duration = float_or_none(try_get(
+        owner = resource.get('owner') or {}
-            models, lambda x: x[0]['data']['duration']), 1000)
-        uploader = try_get(
-            data, lambda x: x['user']['display_name'], compat_str)
-        uploader_id = try_get(
-            data, lambda x: x['user']['uid'], compat_str)
-        view_count = int_or_none(try_get(
-            data, lambda x: x['meta']['views_counter']))
 
         return {
             'id': video_id,
             'title': title,
-            'duration': duration,
+            'duration': float_or_none(video_streams.get('duration'), 1000),
-            'uploader': uploader,
+            'uploader': owner.get('display_name'),
-            'uploader_id': uploader_id,
+            'uploader_id': owner.get('uid'),
-            'view_count': view_count,
+            'view_count': int_or_none(resource.get('views_count')),
+            'timestamp': parse_iso8601(resource.get('created')),
             'formats': formats,
         }

youtube_dl/extractor/yandexvideo.py

@@ -5,6 +5,7 @@ from .common import InfoExtractor
 from ..utils import (
     determine_ext,
     int_or_none,
+    try_get,
     url_or_none,
 )
 
@@ -13,26 +14,30 @@ class YandexVideoIE(InfoExtractor):
     _VALID_URL = r'''(?x)
                     https?://
                         (?:
-                            yandex\.ru(?:/portal/(?:video|efir))?/?\?.*?stream_id=|
+                            yandex\.ru(?:/(?:portal/(?:video|efir)|efir))?/?\?.*?stream_id=|
                             frontend\.vh\.yandex\.ru/player/
                         )
-                        (?P<id>[\da-f]+)
+                        (?P<id>(?:[\da-f]{32}|[\w-]{12}))
                     '''
     _TESTS = [{
-        'url': 'https://yandex.ru/portal/video?stream_id=4dbb262b4fe5cf15a215de4f34eee34d',
+        'url': 'https://yandex.ru/portal/video?stream_id=4dbb36ec4e0526d58f9f2dc8f0ecf374',
-        'md5': '33955d7ae052f15853dc41f35f17581c',
+        'md5': 'e02a05bfaf0d9615ef07ae3a10f4faf4',
         'info_dict': {
-            'id': '4dbb262b4fe5cf15a215de4f34eee34d',
+            'id': '4dbb36ec4e0526d58f9f2dc8f0ecf374',
             'ext': 'mp4',
-            'title': 'В Нью-Йорке баржи и теплоход оторвались от причала и расплылись по Гудзону',
+            'title': 'Русский Вудсток - главный рок-фест в истории СССР / вДудь',
-            'description': '',
+            'description': 'md5:7d6b8d4bc4a3b9a56499916c1ea5b5fa',
-            'thumbnail': r're:^https?://.*\.jpg$',
+            'thumbnail': r're:^https?://',
-            'timestamp': 0,
+            'timestamp': 1549972939,
-            'duration': 30,
+            'duration': 5575,
             'age_limit': 18,
+            'upload_date': '20190212',
+            'view_count': int,
+            'like_count': int,
+            'dislike_count': int,
         },
     }, {
-        'url': 'https://yandex.ru/portal/efir?stream_id=4dbb36ec4e0526d58f9f2dc8f0ecf374&from=morda',
+        'url': 'https://yandex.ru/portal/efir?stream_id=4dbb262b4fe5cf15a215de4f34eee34d&from=morda',
         'only_matching': True,
     }, {
         'url': 'https://yandex.ru/?stream_id=4dbb262b4fe5cf15a215de4f34eee34d',
@@ -52,53 +57,88 @@ class YandexVideoIE(InfoExtractor):
         # DASH with DRM
         'url': 'https://yandex.ru/portal/video?from=morda&stream_id=485a92d94518d73a9d0ff778e13505f8',
         'only_matching': True,
+    }, {
+        'url': 'https://yandex.ru/efir?stream_active=watching&stream_id=v7a2dZ-v5mSI&from_block=efir_newtab',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
-        content = self._download_json(
+        player = try_get((self._download_json(
-            'https://frontend.vh.yandex.ru/v22/player/%s.json' % video_id,
+            'https://frontend.vh.yandex.ru/graphql', video_id, data=b'''{
-            video_id, query={
+  player(content_id: "%s") {
-                'stream_options': 'hires',
+    computed_title
-                'disable_trackings': 1,
+    content_url
-            })['content']
+    description
+    dislikes
+    duration
+    likes
+    program_title
+    release_date
+    release_date_ut
+    release_year
+    restriction_age
+    season
+    start_time
+    streams
+    thumbnail
+    title
+    views_count
+  }
+}''' % video_id.encode(), fatal=False)), lambda x: x['player']['content'])
+        if not player or player.get('error'):
+            player = self._download_json(
+                'https://frontend.vh.yandex.ru/v23/player/%s.json' % video_id,
+                video_id, query={
+                    'stream_options': 'hires',
+                    'disable_trackings': 1,
+                })
+        content = player['content']
 
-        content_url = url_or_none(content.get('content_url')) or url_or_none(
+        title = content.get('title') or content['computed_title']
-            content['streams'][0]['url'])
-        title = content.get('title') or content.get('computed_title')
 
-        ext = determine_ext(content_url)
+        formats = []
-
+        streams = content.get('streams') or []
-        if ext == 'm3u8':
+        streams.append({'url': content.get('content_url')})
-            formats = self._extract_m3u8_formats(
+        for stream in streams:
-                content_url, video_id, 'mp4', entry_protocol='m3u8_native',
+            content_url = url_or_none(stream.get('url'))
-                m3u8_id='hls')
+            if not content_url:
-        elif ext == 'mpd':
+                continue
-            formats = self._extract_mpd_formats(
+            ext = determine_ext(content_url)
-                content_url, video_id, mpd_id='dash')
+            if ext == 'ismc':
-        else:
+                continue
-            formats = [{'url': content_url}]
+            elif ext == 'm3u8':
+                formats.extend(self._extract_m3u8_formats(
+                    content_url, video_id, 'mp4',
+                    'm3u8_native', m3u8_id='hls', fatal=False))
+            elif ext == 'mpd':
+                formats.extend(self._extract_mpd_formats(
+                    content_url, video_id, mpd_id='dash', fatal=False))
+            else:
+                formats.append({'url': content_url})
 
         self._sort_formats(formats)
 
-        description = content.get('description')
-        thumbnail = content.get('thumbnail')
         timestamp = (int_or_none(content.get('release_date'))
                      or int_or_none(content.get('release_date_ut'))
                      or int_or_none(content.get('start_time')))
-        duration = int_or_none(content.get('duration'))
+        season = content.get('season') or {}
-        series = content.get('program_title')
-        age_limit = int_or_none(content.get('restriction_age'))
 
         return {
             'id': video_id,
             'title': title,
-            'description': description,
+            'description': content.get('description'),
-            'thumbnail': thumbnail,
+            'thumbnail': content.get('thumbnail'),
             'timestamp': timestamp,
-            'duration': duration,
+            'duration': int_or_none(content.get('duration')),
-            'series': series,
+            'series': content.get('program_title'),
-            'age_limit': age_limit,
+            'age_limit': int_or_none(content.get('restriction_age')),
+            'view_count': int_or_none(content.get('views_count')),
+            'like_count': int_or_none(content.get('likes')),
+            'dislike_count': int_or_none(content.get('dislikes')),
+            'season_number': int_or_none(season.get('season_number')),
+            'season_id': season.get('id'),
+            'release_year': int_or_none(content.get('release_year')),
             'formats': formats,
         }

youtube_dl/utils.py

@@ -3640,7 +3640,7 @@ def url_or_none(url):
     if not url or not isinstance(url, compat_str):
         return None
     url = url.strip()
-    return url if re.match(r'^(?:[a-zA-Z][\da-zA-Z.+-]*:)?//', url) else None
+    return url if re.match(r'^(?:(?:https?|rt(?:m(?:pt?[es]?|fp)|sp[su]?)|mms|ftps?):)?//', url) else None
 
 
 def parse_duration(s):