release 2014.02.06.2

An error like https://travis-ci.org/rg3/youtube-dl/jobs/18317799#L449 is almost certainly the server's fault.

test/test_download.py

@@ -22,6 +22,7 @@ import socket
 
 import youtube_dl.YoutubeDL
 from youtube_dl.utils import (
+    compat_http_client,
     compat_str,
     compat_urllib_error,
     compat_HTTPError,
@@ -110,7 +111,7 @@ def generator(test_case):
                     ydl.download([test_case['url']])
                 except (DownloadError, ExtractorError) as err:
                     # Check if the exception is not a network related one
-                    if not err.exc_info[0] in (compat_urllib_error.URLError, socket.timeout, UnavailableVideoError) or (err.exc_info[0] == compat_HTTPError and err.exc_info[1].code == 503):
+                    if not err.exc_info[0] in (compat_urllib_error.URLError, socket.timeout, UnavailableVideoError, compat_http_client.BadStatusLine) or (err.exc_info[0] == compat_HTTPError and err.exc_info[1].code == 503):
                         raise
 
                     if try_num == RETRIES:

youtube_dl/extractor/__init__.py

@@ -54,6 +54,7 @@ from .ebaumsworld import EbaumsWorldIE
 from .ehow import EHowIE
 from .eighttracks import EightTracksIE
 from .eitb import EitbIE
+from .elpais import ElPaisIE
 from .escapist import EscapistIE
 from .everyonesmixtape import EveryonesMixtapeIE
 from .exfm import ExfmIE
@@ -143,6 +144,7 @@ from .nba import NBAIE
 from .nbc import NBCNewsIE
 from .ndtv import NDTVIE
 from .newgrounds import NewgroundsIE
+from .nfb import NFBIE
 from .nhl import NHLIE, NHLVideocenterIE
 from .niconico import NiconicoIE
 from .ninegag import NineGagIE

youtube_dl/extractor/collegehumor.py

@@ -4,6 +4,7 @@ import json
 import re
 
 from .common import InfoExtractor
+from ..utils import int_or_none
 
 
 class CollegeHumorIE(InfoExtractor):
@@ -11,22 +12,25 @@ class CollegeHumorIE(InfoExtractor):
 
     _TESTS = [{
         'url': 'http://www.collegehumor.com/video/6902724/comic-con-cosplay-catastrophe',
-        'file': '6902724.mp4',
         'md5': 'dcc0f5c1c8be98dc33889a191f4c26bd',
         'info_dict': {
+            'id': '6902724',
+            'ext': 'mp4',
             'title': 'Comic-Con Cosplay Catastrophe',
-            'description': 'Fans get creative this year at San Diego.  Too',
+            'description': 'Fans get creative this year',
             'age_limit': 13,
         },
     },
     {
         'url': 'http://www.collegehumor.com/video/3505939/font-conference',
-        'file': '3505939.mp4',
         'md5': '72fa701d8ef38664a4dbb9e2ab721816',
         'info_dict': {
+            'id': '3505939',
+            'ext': 'mp4',
             'title': 'Font Conference',
-            'description': 'This video wasn\'t long enough, so we made it double-spaced.',
+            'description': 'This video wasn\'t long enough,',
             'age_limit': 10,
+            'duration': 179,
         },
     },
     # embedded youtube video
@@ -82,6 +86,8 @@ class CollegeHumorIE(InfoExtractor):
                 })
         self._sort_formats(formats)
 
+        duration = int_or_none(vdata.get('duration'), 1000)
+
         return {
             'id': video_id,
             'title': vdata['title'],
@@ -89,4 +95,5 @@ class CollegeHumorIE(InfoExtractor):
             'thumbnail': vdata.get('thumbnail'),
             'formats': formats,
             'age_limit': age_limit,
+            'duration': duration,
         }

youtube_dl/extractor/elpais.py

@@ -0,0 +1,58 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import unified_strdate
+
+
+class ElPaisIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:[^.]+\.)?elpais\.com/.*/(?P<id>[^/#?]+)\.html(?:$|[?#])'
+    IE_DESCR = 'El País'
+
+    _TEST = {
+        'url': 'http://blogs.elpais.com/la-voz-de-inaki/2014/02/tiempo-nuevo-recetas-viejas.html',
+        'md5': '98406f301f19562170ec071b83433d55',
+        'info_dict': {
+            'id': 'tiempo-nuevo-recetas-viejas',
+            'ext': 'mp4',
+            'title': 'Tiempo nuevo, recetas viejas',
+            'description': 'De lunes a viernes, a partir de las ocho de la mañana, Iñaki Gabilondo nos cuenta su visión de la actualidad nacional e internacional.',
+            'upload_date': '20140206',
+        }
+    }
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        video_id = mobj.group('id')
+
+        webpage = self._download_webpage(url, video_id)
+
+        prefix = self._html_search_regex(
+            r'var url_cache = "([^"]+)";', webpage, 'URL prefix')
+        video_suffix = self._search_regex(
+            r"URLMediaFile = url_cache \+ '([^']+)'", webpage, 'video URL')
+        video_url = prefix + video_suffix
+        thumbnail_suffix = self._search_regex(
+            r"URLMediaStill = url_cache \+ '([^']+)'", webpage, 'thumbnail URL',
+            fatal=False)
+        thumbnail = (
+            None if thumbnail_suffix is None
+            else prefix + thumbnail_suffix)
+        title = self._html_search_regex(
+            '<h2 class="entry-header entry-title.*?>(.*?)</h2>',
+            webpage, 'title')
+        date_str = self._search_regex(
+            r'<p class="date-header date-int updated"\s+title="([^"]+)">',
+            webpage, 'upload date', fatal=False)
+        upload_date = (None if date_str is None else unified_strdate(date_str))
+
+        return {
+            'id': video_id,
+            'url': video_url,
+            'title': title,
+            'description': self._og_search_description(webpage),
+            'thumbnail': thumbnail,
+            'upload_date': upload_date,
+        }

youtube_dl/extractor/mooshare.py

@@ -61,7 +61,7 @@ class MooshareIE(InfoExtractor):
         }
 
         request = compat_urllib_request.Request(
-            'http://mooshare.biz/8dqtk4bjbp8g', compat_urllib_parse.urlencode(download_form))
+            'http://mooshare.biz/%s' % video_id, compat_urllib_parse.urlencode(download_form))
         request.add_header('Content-Type', 'application/x-www-form-urlencoded')
 
         self.to_screen('%s: Waiting for timeout' % video_id)
@@ -111,4 +111,4 @@ class MooshareIE(InfoExtractor):
             'thumbnail': thumbnail,
             'duration': duration,
             'formats': formats,
-        }
+        }

youtube_dl/extractor/mtv.py

@@ -82,10 +82,13 @@ class MTVServicesInfoExtractor(InfoExtractor):
             title_el = find_xpath_attr(
                 itemdoc, './/{http://search.yahoo.com/mrss/}category',
                 'scheme', 'urn:mtvn:video_title')
-        if title_el is None:
-            title_el = itemdoc.find('.//{http://search.yahoo.com/mrss/}title')
         if title_el is None:
             title_el = itemdoc.find('.//title')
+            if title_el.text is None:
+                title_el = None
+        if title_el is None:
+            title_el = itemdoc.find('.//{http://search.yahoo.com/mrss/}title')
+
         title = title_el.text
         if title is None:
             raise ExtractorError('Could not find video title')

youtube_dl/extractor/nfb.py

@@ -0,0 +1,76 @@
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+    compat_urllib_request,
+    compat_urllib_parse,
+)
+
+
+class NFBIE(InfoExtractor):
+    IE_NAME = 'nfb'
+    IE_DESC = 'National Film Board of Canada'
+    _VALID_URL = r'https?://(?:www\.)?(nfb|onf)\.ca/film/(?P<id>[\da-z_-]+)'
+
+    _TEST = {
+        'url': 'https://www.nfb.ca/film/qallunaat_why_white_people_are_funny',
+        'info_dict': {
+            'id': 'qallunaat_why_white_people_are_funny',
+            'ext': 'mp4',
+            'title': 'Qallunaat! Why White People Are Funny ',
+            'description': 'md5:836d8aff55e087d04d9f6df554d4e038',
+            'duration': 3128,
+            'uploader': 'Mark Sandiford',
+            'uploader_id': 'mark-sandiford',
+        },
+        'params': {
+            # rtmp download
+            'skip_download': True,
+        }
+    }
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        video_id = mobj.group('id')
+
+        page = self._download_webpage('https://www.nfb.ca/film/%s' % video_id, video_id, 'Downloading film page')
+
+        uploader_id = self._html_search_regex(r'<a class="director-link" href="/explore-all-directors/([^/]+)/"',
+            page, 'director id', fatal=False)
+        uploader = self._html_search_regex(r'<em class="director-name" itemprop="name">([^<]+)</em>',
+            page, 'director name', fatal=False)
+
+        request = compat_urllib_request.Request('https://www.nfb.ca/film/%s/player_config' % video_id,
+            compat_urllib_parse.urlencode({'getConfig': 'true'}))
+        request.add_header('Content-Type', 'application/x-www-form-urlencoded')
+        request.add_header('X-NFB-Referer', 'http://www.nfb.ca/medias/flash/NFBVideoPlayer.swf')
+
+        config = self._download_xml(request, video_id, 'Downloading player config XML')
+
+        thumbnail = config.find("./player/stream/media[@type='posterImage']/assets/asset[@quality='high']/default/url").text
+        video = config.find("./player/stream/media[@type='video']")
+        duration = int(video.get('duration'))
+        title = video.find('title').text
+        description = video.find('description').text
+
+        # It seems assets always go from lower to better quality, so no need to sort
+        formats = [{
+            'url': x.find('default/streamerURI').text + '/',
+            'play_path': x.find('default/url').text,
+            'rtmp_live': False,
+            'ext': 'mp4',
+            'format_id': x.get('quality'),
+        } for x in video.findall('assets/asset')]
+
+        return {
+            'id': video_id,
+            'title': title,
+            'description': description,
+            'thumbnail': thumbnail,
+            'duration': duration,
+            'uploader': uploader,
+            'uploader_id': uploader_id,
+            'formats': formats,
+        }

youtube_dl/extractor/pbs.py

@@ -1,7 +1,6 @@
 from __future__ import unicode_literals
 
 import re
-import json
 
 from .common import InfoExtractor
 

youtube_dl/utils.py

@@ -751,13 +751,14 @@ class YoutubeDLHandler(compat_urllib_request.HTTPHandler):
     https_request = http_request
     https_response = http_response
 
+
 def unified_strdate(date_str):
     """Return a string with the date in the format YYYYMMDD"""
     upload_date = None
     #Replace commas
     date_str = date_str.replace(',',' ')
     # %z (UTC offset) is only supported in python>=3.2
-    date_str = re.sub(r' (\+|-)[\d]*$', '', date_str)
+    date_str = re.sub(r' ?(\+|-)[0-9:]*$', '', date_str)
     format_expressions = [
         '%d %B %Y',
         '%B %d %Y',
@@ -771,11 +772,12 @@ def unified_strdate(date_str):
         '%Y-%m-%dT%H:%M:%S.%fZ',
         '%Y-%m-%dT%H:%M:%S.%f0Z',
         '%Y-%m-%dT%H:%M:%S',
+        '%Y-%m-%dT%H:%M',
     ]
     for expression in format_expressions:
         try:
             upload_date = datetime.datetime.strptime(date_str, expression).strftime('%Y%m%d')
-        except:
+        except ValueError:
             pass
     if upload_date is None:
         timetuple = email.utils.parsedate_tz(date_str)

youtube_dl/version.py

@@ -1,2 +1,2 @@
 
-__version__ = '2014.02.06'
+__version__ = '2014.02.06.2'