frontendmasters.py (8807B)
1 # coding: utf-8 2 from __future__ import unicode_literals 3 4 import re 5 6 from .common import InfoExtractor 7 from ..compat import ( 8 compat_str, 9 compat_urlparse, 10 ) 11 from ..utils import ( 12 ExtractorError, 13 parse_duration, 14 url_or_none, 15 urlencode_postdata, 16 ) 17 18 19 class FrontendMastersBaseIE(InfoExtractor): 20 _API_BASE = 'https://api.frontendmasters.com/v1/kabuki' 21 _LOGIN_URL = 'https://frontendmasters.com/login/' 22 23 _NETRC_MACHINE = 'frontendmasters' 24 25 _QUALITIES = { 26 'low': {'width': 480, 'height': 360}, 27 'mid': {'width': 1280, 'height': 720}, 28 'high': {'width': 1920, 'height': 1080} 29 } 30 31 def _real_initialize(self): 32 self._login() 33 34 def _login(self): 35 (username, password) = self._get_login_info() 36 if username is None: 37 return 38 39 login_page = self._download_webpage( 40 self._LOGIN_URL, None, 'Downloading login page') 41 42 login_form = self._hidden_inputs(login_page) 43 44 login_form.update({ 45 'username': username, 46 'password': password 47 }) 48 49 post_url = self._search_regex( 50 r'<form[^>]+action=(["\'])(?P<url>.+?)\1', login_page, 51 'post_url', default=self._LOGIN_URL, group='url') 52 53 if not post_url.startswith('http'): 54 post_url = compat_urlparse.urljoin(self._LOGIN_URL, post_url) 55 56 response = self._download_webpage( 57 post_url, None, 'Logging in', data=urlencode_postdata(login_form), 58 headers={'Content-Type': 'application/x-www-form-urlencoded'}) 59 60 # Successful login 61 if any(p in response for p in ( 62 'wp-login.php?action=logout', '>Logout')): 63 return 64 65 error = self._html_search_regex( 66 r'class=(["\'])(?:(?!\1).)*\bMessageAlert\b(?:(?!\1).)*\1[^>]*>(?P<error>[^<]+)<', 67 response, 'error message', default=None, group='error') 68 if error: 69 raise ExtractorError('Unable to login: %s' % error, expected=True) 70 raise ExtractorError('Unable to log in') 71 72 73 class FrontendMastersPageBaseIE(FrontendMastersBaseIE): 74 def _download_course(self, course_name, url): 75 return self._download_json( 76 '%s/courses/%s' % (self._API_BASE, course_name), course_name, 77 'Downloading course JSON', headers={'Referer': url}) 78 79 @staticmethod 80 def _extract_chapters(course): 81 chapters = [] 82 lesson_elements = course.get('lessonElements') 83 if isinstance(lesson_elements, list): 84 chapters = [url_or_none(e) for e in lesson_elements if url_or_none(e)] 85 return chapters 86 87 @staticmethod 88 def _extract_lesson(chapters, lesson_id, lesson): 89 title = lesson.get('title') or lesson_id 90 display_id = lesson.get('slug') 91 description = lesson.get('description') 92 thumbnail = lesson.get('thumbnail') 93 94 chapter_number = None 95 index = lesson.get('index') 96 element_index = lesson.get('elementIndex') 97 if (isinstance(index, int) and isinstance(element_index, int) 98 and index < element_index): 99 chapter_number = element_index - index 100 chapter = (chapters[chapter_number - 1] 101 if chapter_number - 1 < len(chapters) else None) 102 103 duration = None 104 timestamp = lesson.get('timestamp') 105 if isinstance(timestamp, compat_str): 106 mobj = re.search( 107 r'(?P<start>\d{1,2}:\d{1,2}:\d{1,2})\s*-(?P<end>\s*\d{1,2}:\d{1,2}:\d{1,2})', 108 timestamp) 109 if mobj: 110 duration = parse_duration(mobj.group('end')) - parse_duration( 111 mobj.group('start')) 112 113 return { 114 '_type': 'url_transparent', 115 'url': 'frontendmasters:%s' % lesson_id, 116 'ie_key': FrontendMastersIE.ie_key(), 117 'id': lesson_id, 118 'display_id': display_id, 119 'title': title, 120 'description': description, 121 'thumbnail': thumbnail, 122 'duration': duration, 123 'chapter': chapter, 124 'chapter_number': chapter_number, 125 } 126 127 128 class FrontendMastersIE(FrontendMastersBaseIE): 129 _VALID_URL = r'(?:frontendmasters:|https?://api\.frontendmasters\.com/v\d+/kabuki/video/)(?P<id>[^/]+)' 130 _TESTS = [{ 131 'url': 'https://api.frontendmasters.com/v1/kabuki/video/a2qogef6ba', 132 'md5': '7f161159710d6b7016a4f4af6fcb05e2', 133 'info_dict': { 134 'id': 'a2qogef6ba', 135 'ext': 'mp4', 136 'title': 'a2qogef6ba', 137 }, 138 'skip': 'Requires FrontendMasters account credentials', 139 }, { 140 'url': 'frontendmasters:a2qogef6ba', 141 'only_matching': True, 142 }] 143 144 def _real_extract(self, url): 145 lesson_id = self._match_id(url) 146 147 source_url = '%s/video/%s/source' % (self._API_BASE, lesson_id) 148 149 formats = [] 150 for ext in ('webm', 'mp4'): 151 for quality in ('low', 'mid', 'high'): 152 resolution = self._QUALITIES[quality].copy() 153 format_id = '%s-%s' % (ext, quality) 154 format_url = self._download_json( 155 source_url, lesson_id, 156 'Downloading %s source JSON' % format_id, query={ 157 'f': ext, 158 'r': resolution['height'], 159 }, headers={ 160 'Referer': url, 161 }, fatal=False)['url'] 162 163 if not format_url: 164 continue 165 166 f = resolution.copy() 167 f.update({ 168 'url': format_url, 169 'ext': ext, 170 'format_id': format_id, 171 }) 172 formats.append(f) 173 self._sort_formats(formats) 174 175 subtitles = { 176 'en': [{ 177 'url': '%s/transcripts/%s.vtt' % (self._API_BASE, lesson_id), 178 }] 179 } 180 181 return { 182 'id': lesson_id, 183 'title': lesson_id, 184 'formats': formats, 185 'subtitles': subtitles 186 } 187 188 189 class FrontendMastersLessonIE(FrontendMastersPageBaseIE): 190 _VALID_URL = r'https?://(?:www\.)?frontendmasters\.com/courses/(?P<course_name>[^/]+)/(?P<lesson_name>[^/]+)' 191 _TEST = { 192 'url': 'https://frontendmasters.com/courses/web-development/tools', 193 'info_dict': { 194 'id': 'a2qogef6ba', 195 'display_id': 'tools', 196 'ext': 'mp4', 197 'title': 'Tools', 198 'description': 'md5:82c1ea6472e88ed5acd1829fe992e4f7', 199 'thumbnail': r're:^https?://.*\.jpg$', 200 'chapter': 'Introduction', 201 'chapter_number': 1, 202 }, 203 'params': { 204 'skip_download': True, 205 }, 206 'skip': 'Requires FrontendMasters account credentials', 207 } 208 209 def _real_extract(self, url): 210 mobj = re.match(self._VALID_URL, url) 211 course_name, lesson_name = mobj.group('course_name', 'lesson_name') 212 213 course = self._download_course(course_name, url) 214 215 lesson_id, lesson = next( 216 (video_id, data) 217 for video_id, data in course['lessonData'].items() 218 if data.get('slug') == lesson_name) 219 220 chapters = self._extract_chapters(course) 221 return self._extract_lesson(chapters, lesson_id, lesson) 222 223 224 class FrontendMastersCourseIE(FrontendMastersPageBaseIE): 225 _VALID_URL = r'https?://(?:www\.)?frontendmasters\.com/courses/(?P<id>[^/]+)' 226 _TEST = { 227 'url': 'https://frontendmasters.com/courses/web-development/', 228 'info_dict': { 229 'id': 'web-development', 230 'title': 'Introduction to Web Development', 231 'description': 'md5:9317e6e842098bf725d62360e52d49a6', 232 }, 233 'playlist_count': 81, 234 'skip': 'Requires FrontendMasters account credentials', 235 } 236 237 @classmethod 238 def suitable(cls, url): 239 return False if FrontendMastersLessonIE.suitable(url) else super( 240 FrontendMastersBaseIE, cls).suitable(url) 241 242 def _real_extract(self, url): 243 course_name = self._match_id(url) 244 245 course = self._download_course(course_name, url) 246 247 chapters = self._extract_chapters(course) 248 249 lessons = sorted( 250 course['lessonData'].values(), key=lambda data: data['index']) 251 252 entries = [] 253 for lesson in lessons: 254 lesson_name = lesson.get('slug') 255 if not lesson_name: 256 continue 257 lesson_id = lesson.get('hash') or lesson.get('statsId') 258 entries.append(self._extract_lesson(chapters, lesson_id, lesson)) 259 260 title = course.get('title') 261 description = course.get('description') 262 263 return self.playlist_result(entries, course_name, title, description)