mit.py (5210B)
1 from __future__ import unicode_literals 2 3 import re 4 import json 5 6 from .common import InfoExtractor 7 from .youtube import YoutubeIE 8 from ..utils import ( 9 clean_html, 10 ExtractorError, 11 get_element_by_id, 12 ) 13 14 15 class TechTVMITIE(InfoExtractor): 16 IE_NAME = 'techtv.mit.edu' 17 _VALID_URL = r'https?://techtv\.mit\.edu/(?:videos|embeds)/(?P<id>\d+)' 18 19 _TEST = { 20 'url': 'http://techtv.mit.edu/videos/25418-mit-dna-learning-center-set', 21 'md5': '00a3a27ee20d44bcaa0933ccec4a2cf7', 22 'info_dict': { 23 'id': '25418', 24 'ext': 'mp4', 25 'title': 'MIT DNA and Protein Sets', 26 'description': 'md5:46f5c69ce434f0a97e7c628cc142802d', 27 }, 28 } 29 30 def _real_extract(self, url): 31 video_id = self._match_id(url) 32 raw_page = self._download_webpage( 33 'http://techtv.mit.edu/videos/%s' % video_id, video_id) 34 clean_page = re.compile(r'<!--.*?-->', re.S).sub('', raw_page) 35 36 base_url = self._proto_relative_url(self._search_regex( 37 r'ipadUrl: \'(.+?cloudfront.net/)', raw_page, 'base url'), 'http:') 38 formats_json = self._search_regex( 39 r'bitrates: (\[.+?\])', raw_page, 'video formats') 40 formats_mit = json.loads(formats_json) 41 formats = [ 42 { 43 'format_id': f['label'], 44 'url': base_url + f['url'].partition(':')[2], 45 'ext': f['url'].partition(':')[0], 46 'format': f['label'], 47 'width': f['width'], 48 'vbr': f['bitrate'], 49 } 50 for f in formats_mit 51 ] 52 53 title = get_element_by_id('edit-title', clean_page) 54 description = clean_html(get_element_by_id('edit-description', clean_page)) 55 thumbnail = self._search_regex( 56 r'playlist:.*?url: \'(.+?)\'', 57 raw_page, 'thumbnail', flags=re.DOTALL) 58 59 return { 60 'id': video_id, 61 'title': title, 62 'formats': formats, 63 'description': description, 64 'thumbnail': thumbnail, 65 } 66 67 68 class OCWMITIE(InfoExtractor): 69 IE_NAME = 'ocw.mit.edu' 70 _VALID_URL = r'^https?://ocw\.mit\.edu/courses/(?P<topic>[a-z0-9\-]+)' 71 _BASE_URL = 'http://ocw.mit.edu/' 72 73 _TESTS = [ 74 { 75 'url': 'http://ocw.mit.edu/courses/electrical-engineering-and-computer-science/6-041-probabilistic-systems-analysis-and-applied-probability-fall-2010/video-lectures/lecture-7-multiple-variables-expectations-independence/', 76 'info_dict': { 77 'id': 'EObHWIEKGjA', 78 'ext': 'webm', 79 'title': 'Lecture 7: Multiple Discrete Random Variables: Expectations, Conditioning, Independence', 80 'description': 'In this lecture, the professor discussed multiple random variables, expectations, and binomial distribution.', 81 'upload_date': '20121109', 82 'uploader_id': 'MIT', 83 'uploader': 'MIT OpenCourseWare', 84 } 85 }, 86 { 87 'url': 'http://ocw.mit.edu/courses/mathematics/18-01sc-single-variable-calculus-fall-2010/1.-differentiation/part-a-definition-and-basic-rules/session-1-introduction-to-derivatives/', 88 'info_dict': { 89 'id': '7K1sB05pE0A', 90 'ext': 'mp4', 91 'title': 'Session 1: Introduction to Derivatives', 92 'upload_date': '20090818', 93 'uploader_id': 'MIT', 94 'uploader': 'MIT OpenCourseWare', 95 'description': 'This section contains lecture video excerpts, lecture notes, an interactive mathlet with supporting documents, and problem solving videos.', 96 } 97 } 98 ] 99 100 def _real_extract(self, url): 101 mobj = re.match(self._VALID_URL, url) 102 topic = mobj.group('topic') 103 104 webpage = self._download_webpage(url, topic) 105 title = self._html_search_meta('WT.cg_s', webpage) 106 description = self._html_search_meta('Description', webpage) 107 108 # search for call to ocw_embed_chapter_media(container_id, media_url, provider, page_url, image_url, start, stop, captions_file) 109 embed_chapter_media = re.search(r'ocw_embed_chapter_media\((.+?)\)', webpage) 110 if embed_chapter_media: 111 metadata = re.sub(r'[\'"]', '', embed_chapter_media.group(1)) 112 metadata = re.split(r', ?', metadata) 113 yt = metadata[1] 114 else: 115 # search for call to ocw_embed_chapter_media(container_id, media_url, provider, page_url, image_url, captions_file) 116 embed_media = re.search(r'ocw_embed_media\((.+?)\)', webpage) 117 if embed_media: 118 metadata = re.sub(r'[\'"]', '', embed_media.group(1)) 119 metadata = re.split(r', ?', metadata) 120 yt = metadata[1] 121 else: 122 raise ExtractorError('Unable to find embedded YouTube video.') 123 video_id = YoutubeIE.extract_id(yt) 124 125 return { 126 '_type': 'url_transparent', 127 'id': video_id, 128 'title': title, 129 'description': description, 130 'url': yt, 131 'ie_key': 'Youtube', 132 }