commit e8ce2375e0851e65c4882002297404825fe1045e
parent 6fdb39ded15c6276b49fa67cb517bf1fed63af35
Author: Sergey M? <dstftw@gmail.com>
Date: Sun, 8 Nov 2015 06:54:27 +0600
[viidea] Improve and cleanup (Closes #7390)
* Optimize requests for multipart videos
* Fix cfg regex
* Improve titles and identifiers
Diffstat:
1 file changed, 72 insertions(+), 27 deletions(-)
diff --git a/youtube_dl/extractor/viidea.py b/youtube_dl/extractor/viidea.py
@@ -35,35 +35,42 @@ class ViideaIE(InfoExtractor):
_TESTS = [{
'url': 'http://videolectures.net/promogram_igor_mekjavic_eng/',
'info_dict': {
- 'id': '20171_part1',
+ 'id': '20171',
+ 'display_id': 'promogram_igor_mekjavic_eng',
'ext': 'mp4',
'title': 'Automatics, robotics and biocybernetics',
'description': 'md5:815fc1deb6b3a2bff99de2d5325be482',
+ 'thumbnail': 're:http://.*\.jpg',
+ 'timestamp': 1372349289,
'upload_date': '20130627',
'duration': 565,
- 'thumbnail': 're:http://.*\.jpg',
},
}, {
# video with invalid direct format links (HTTP 403)
'url': 'http://videolectures.net/russir2010_filippova_nlp/',
'info_dict': {
- 'id': '14891_part1',
+ 'id': '14891',
+ 'display_id': 'russir2010_filippova_nlp',
'ext': 'flv',
'title': 'NLP at Google',
'description': 'md5:fc7a6d9bf0302d7cc0e53f7ca23747b3',
- 'duration': 5352,
'thumbnail': 're:http://.*\.jpg',
+ 'timestamp': 1284375600,
+ 'upload_date': '20100913',
+ 'duration': 5352,
},
'params': {
# rtmp download
'skip_download': True,
},
}, {
+ # event playlist
'url': 'http://videolectures.net/deeplearning2015_montreal/',
'info_dict': {
'id': '23181',
'title': 'Deep Learning Summer School, Montreal 2015',
'description': 'md5:0533a85e4bd918df52a01f0e1ebe87b7',
+ 'thumbnail': 're:http://.*\.jpg',
'timestamp': 1438560000,
},
'playlist_count': 30,
@@ -72,37 +79,54 @@ class ViideaIE(InfoExtractor):
'url': 'http://videolectures.net/mlss09uk_bishop_ibi/',
'info_dict': {
'id': '9737',
+ 'display_id': 'mlss09uk_bishop_ibi',
'title': 'Introduction To Bayesian Inference',
+ 'thumbnail': 're:http://.*\.jpg',
'timestamp': 1251622800,
},
'playlist': [{
'info_dict': {
'id': '9737_part1',
+ 'display_id': 'mlss09uk_bishop_ibi_part1',
'ext': 'wmv',
- 'title': 'Introduction To Bayesian Inference',
+ 'title': 'Introduction To Bayesian Inference (Part 1)',
+ 'thumbnail': 're:http://.*\.jpg',
+ 'duration': 4622,
+ 'timestamp': 1251622800,
+ 'upload_date': '20090830',
},
}, {
'info_dict': {
'id': '9737_part2',
+ 'display_id': 'mlss09uk_bishop_ibi_part2',
'ext': 'wmv',
- 'title': 'Introduction To Bayesian Inference',
+ 'title': 'Introduction To Bayesian Inference (Part 2)',
+ 'thumbnail': 're:http://.*\.jpg',
+ 'duration': 5641,
+ 'timestamp': 1251622800,
+ 'upload_date': '20090830',
},
}],
'playlist_count': 2,
}]
def _real_extract(self, url):
- lecture_slug, part = re.match(self._VALID_URL, url).groups()
+ lecture_slug, explicit_part_id = re.match(self._VALID_URL, url).groups()
webpage = self._download_webpage(url, lecture_slug)
- cfg = self._parse_json(self._search_regex([r'cfg\s*:\s*({.+?}),[\da-zA-Z_]:\(?function', r'cfg\s*:\s*({[^}]+})'], webpage, 'cfg'), lecture_slug, js_to_json)
+ cfg = self._parse_json(self._search_regex(
+ [r'cfg\s*:\s*({.+?})\s*,\s*[\da-zA-Z_]+\s*:\s*\(?\s*function',
+ r'cfg\s*:\s*({[^}]+})'],
+ webpage, 'cfg'), lecture_slug, js_to_json)
lecture_id = compat_str(cfg['obj_id'])
base_url = self._proto_relative_url(cfg['livepipe'], 'http:')
- lecture_data = self._download_json('%s/site/api/lecture/%s?format=json' % (base_url, lecture_id), lecture_id)['lecture'][0]
+ lecture_data = self._download_json(
+ '%s/site/api/lecture/%s?format=json' % (base_url, lecture_id),
+ lecture_id)['lecture'][0]
lecture_info = {
'id': lecture_id,
@@ -113,31 +137,52 @@ class ViideaIE(InfoExtractor):
'thumbnail': lecture_data.get('thumb'),
}
- entries = []
- parts = cfg.get('videos')
+ playlist_entries = []
+ lecture_type = lecture_data.get('type')
+ parts = [compat_str(video) for video in cfg.get('videos', [])]
if parts:
- if len(parts) == 1:
- part = compat_str(parts[0])
- if part:
- smil_url = '%s/%s/video/%s/smil.xml' % (base_url, lecture_slug, part)
+ multipart = len(parts) > 1
+
+ def extract_part(part_id):
+ smil_url = '%s/%s/video/%s/smil.xml' % (base_url, lecture_slug, part_id)
smil = self._download_smil(smil_url, lecture_id)
info = self._parse_smil(smil, smil_url, lecture_id)
- info['id'] = '%s_part%s' % (lecture_id, part)
+ info['id'] = lecture_id if not multipart else '%s_part%s' % (lecture_id, part_id)
+ info['display_id'] = lecture_slug if not multipart else '%s_part%s' % (lecture_slug, part_id)
+ if multipart:
+ info['title'] += ' (Part %s)' % part_id
switch = smil.find('.//switch')
if switch is not None:
info['duration'] = parse_duration(switch.attrib.get('dur'))
- return info
+ item_info = lecture_info.copy()
+ item_info.update(info)
+ return item_info
+
+ if explicit_part_id or not multipart:
+ result = extract_part(explicit_part_id or parts[0])
else:
- for part in parts:
- entries.append(self.url_result('%s/%s/video/%s' % (base_url, lecture_slug, part), 'Viidea'))
- lecture_info['_type'] = 'multi_video'
- if not parts or lecture_data.get('type') == 'evt':
- # Probably a playlist
- playlist_webpage = self._download_webpage('%s/site/ajax/drilldown/?id=%s' % (base_url, lecture_id), lecture_id)
+ result = {
+ '_type': 'multi_video',
+ 'entries': [extract_part(part) for part in parts],
+ }
+ result.update(lecture_info)
+
+ # Immediately return explicitly requested part or non event item
+ if explicit_part_id or lecture_type != 'evt':
+ return result
+
+ playlist_entries.append(result)
+
+ # It's probably a playlist
+ if not parts or lecture_type == 'evt':
+ playlist_webpage = self._download_webpage(
+ '%s/site/ajax/drilldown/?id=%s' % (base_url, lecture_id), lecture_id)
entries = [
self.url_result(compat_urlparse.urljoin(url, video_url), 'Viidea')
- for _, video_url in re.findall(r'<a[^>]+href=(["\'])(.+?)\1[^>]+id=["\']lec=\d+', playlist_webpage)]
- lecture_info['_type'] = 'playlist'
+ for _, video_url in re.findall(
+ r'<a[^>]+href=(["\'])(.+?)\1[^>]+id=["\']lec=\d+', playlist_webpage)]
+ playlist_entries.extend(entries)
- lecture_info['entries'] = entries
- return lecture_info
+ playlist = self.playlist_result(playlist_entries, lecture_id)
+ playlist.update(lecture_info)
+ return playlist
