wdr.py (13233B)
1 # coding: utf-8 2 from __future__ import unicode_literals 3 4 import re 5 6 from .common import InfoExtractor 7 from ..compat import ( 8 compat_str, 9 compat_urlparse, 10 ) 11 from ..utils import ( 12 determine_ext, 13 ExtractorError, 14 js_to_json, 15 strip_jsonp, 16 try_get, 17 unified_strdate, 18 update_url_query, 19 urlhandle_detect_ext, 20 url_or_none, 21 ) 22 23 24 class WDRIE(InfoExtractor): 25 _VALID_URL = r'https?://deviceids-medp\.wdr\.de/ondemand/\d+/(?P<id>\d+)\.js' 26 _GEO_COUNTRIES = ['DE'] 27 _TEST = { 28 'url': 'http://deviceids-medp.wdr.de/ondemand/155/1557833.js', 29 'info_dict': { 30 'id': 'mdb-1557833', 31 'ext': 'mp4', 32 'title': 'Biathlon-Staffel verpasst Podest bei Olympia-Generalprobe', 33 'upload_date': '20180112', 34 }, 35 } 36 37 def _real_extract(self, url): 38 video_id = self._match_id(url) 39 40 metadata = self._download_json( 41 url, video_id, transform_source=strip_jsonp) 42 43 is_live = metadata.get('mediaType') == 'live' 44 45 tracker_data = metadata['trackerData'] 46 title = tracker_data['trackerClipTitle'] 47 48 media_resource = metadata['mediaResource'] 49 50 formats = [] 51 52 # check if the metadata contains a direct URL to a file 53 for kind, media in media_resource.items(): 54 if not isinstance(media, dict): 55 continue 56 if kind not in ('dflt', 'alt'): 57 continue 58 59 for tag_name, medium_url in media.items(): 60 if tag_name not in ('videoURL', 'audioURL'): 61 continue 62 63 ext = determine_ext(medium_url) 64 if ext == 'm3u8': 65 formats.extend(self._extract_m3u8_formats( 66 medium_url, video_id, 'mp4', 'm3u8_native', 67 m3u8_id='hls')) 68 elif ext == 'f4m': 69 manifest_url = update_url_query( 70 medium_url, {'hdcore': '3.2.0', 'plugin': 'aasp-3.2.0.77.18'}) 71 formats.extend(self._extract_f4m_formats( 72 manifest_url, video_id, f4m_id='hds', fatal=False)) 73 elif ext == 'smil': 74 formats.extend(self._extract_smil_formats( 75 medium_url, 'stream', fatal=False)) 76 else: 77 a_format = { 78 'url': medium_url 79 } 80 if ext == 'unknown_video': 81 urlh = self._request_webpage( 82 medium_url, video_id, note='Determining extension') 83 ext = urlhandle_detect_ext(urlh) 84 a_format['ext'] = ext 85 formats.append(a_format) 86 87 self._sort_formats(formats) 88 89 subtitles = {} 90 caption_url = media_resource.get('captionURL') 91 if caption_url: 92 subtitles['de'] = [{ 93 'url': caption_url, 94 'ext': 'ttml', 95 }] 96 captions_hash = media_resource.get('captionsHash') 97 if isinstance(captions_hash, dict): 98 for ext, format_url in captions_hash.items(): 99 format_url = url_or_none(format_url) 100 if not format_url: 101 continue 102 subtitles.setdefault('de', []).append({ 103 'url': format_url, 104 'ext': determine_ext(format_url, None) or ext, 105 }) 106 107 return { 108 'id': tracker_data.get('trackerClipId', video_id), 109 'title': self._live_title(title) if is_live else title, 110 'alt_title': tracker_data.get('trackerClipSubcategory'), 111 'formats': formats, 112 'subtitles': subtitles, 113 'upload_date': unified_strdate(tracker_data.get('trackerClipAirTime')), 114 'is_live': is_live, 115 } 116 117 118 class WDRPageIE(InfoExtractor): 119 _CURRENT_MAUS_URL = r'https?://(?:www\.)wdrmaus.de/(?:[^/]+/){1,2}[^/?#]+\.php5' 120 _PAGE_REGEX = r'/(?:mediathek/)?(?:[^/]+/)*(?P<display_id>[^/]+)\.html' 121 _VALID_URL = r'https?://(?:www\d?\.)?(?:(?:kinder\.)?wdr\d?|sportschau)\.de' + _PAGE_REGEX + '|' + _CURRENT_MAUS_URL 122 123 _TESTS = [ 124 { 125 'url': 'http://www1.wdr.de/mediathek/video/sendungen/doku-am-freitag/video-geheimnis-aachener-dom-100.html', 126 # HDS download, MD5 is unstable 127 'info_dict': { 128 'id': 'mdb-1058683', 129 'ext': 'flv', 130 'display_id': 'doku-am-freitag/video-geheimnis-aachener-dom-100', 131 'title': 'Geheimnis Aachener Dom', 132 'alt_title': 'Doku am Freitag', 133 'upload_date': '20160304', 134 'description': 'md5:87be8ff14d8dfd7a7ee46f0299b52318', 135 'is_live': False, 136 'subtitles': {'de': [{ 137 'url': 'http://ondemand-ww.wdr.de/medp/fsk0/105/1058683/1058683_12220974.xml', 138 'ext': 'ttml', 139 }]}, 140 }, 141 'skip': 'HTTP Error 404: Not Found', 142 }, 143 { 144 'url': 'http://www1.wdr.de/mediathek/audio/wdr3/wdr3-gespraech-am-samstag/audio-schriftstellerin-juli-zeh-100.html', 145 'md5': 'f4c1f96d01cf285240f53ea4309663d8', 146 'info_dict': { 147 'id': 'mdb-1072000', 148 'ext': 'mp3', 149 'display_id': 'wdr3-gespraech-am-samstag/audio-schriftstellerin-juli-zeh-100', 150 'title': 'Schriftstellerin Juli Zeh', 151 'alt_title': 'WDR 3 Gespräch am Samstag', 152 'upload_date': '20160312', 153 'description': 'md5:e127d320bc2b1f149be697ce044a3dd7', 154 'is_live': False, 155 'subtitles': {} 156 }, 157 'skip': 'HTTP Error 404: Not Found', 158 }, 159 { 160 'url': 'http://www1.wdr.de/mediathek/video/live/index.html', 161 'info_dict': { 162 'id': 'mdb-1406149', 163 'ext': 'mp4', 164 'title': r're:^WDR Fernsehen im Livestream \(nur in Deutschland erreichbar\) [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$', 165 'alt_title': 'WDR Fernsehen Live', 166 'upload_date': '20150101', 167 'is_live': True, 168 }, 169 'params': { 170 'skip_download': True, # m3u8 download 171 }, 172 }, 173 { 174 'url': 'http://www1.wdr.de/mediathek/video/sendungen/aktuelle-stunde/aktuelle-stunde-120.html', 175 'playlist_mincount': 7, 176 'info_dict': { 177 'id': 'aktuelle-stunde-120', 178 }, 179 }, 180 { 181 'url': 'http://www.wdrmaus.de/aktuelle-sendung/index.php5', 182 'info_dict': { 183 'id': 'mdb-1552552', 184 'ext': 'mp4', 185 'upload_date': 're:^[0-9]{8}$', 186 'title': 're:^Die Sendung mit der Maus vom [0-9.]{10}$', 187 }, 188 'skip': 'The id changes from week to week because of the new episode' 189 }, 190 { 191 'url': 'http://www.wdrmaus.de/filme/sachgeschichten/achterbahn.php5', 192 'md5': '803138901f6368ee497b4d195bb164f2', 193 'info_dict': { 194 'id': 'mdb-186083', 195 'ext': 'mp4', 196 'upload_date': '20130919', 197 'title': 'Sachgeschichte - Achterbahn ', 198 }, 199 }, 200 { 201 'url': 'http://www1.wdr.de/radio/player/radioplayer116~_layout-popupVersion.html', 202 # Live stream, MD5 unstable 203 'info_dict': { 204 'id': 'mdb-869971', 205 'ext': 'mp4', 206 'title': r're:^COSMO Livestream [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$', 207 'upload_date': '20160101', 208 }, 209 'params': { 210 'skip_download': True, # m3u8 download 211 } 212 }, 213 { 214 'url': 'http://www.sportschau.de/handballem2018/handball-nationalmannschaft-em-stolperstein-vorrunde-100.html', 215 'info_dict': { 216 'id': 'mdb-1556012', 217 'ext': 'mp4', 218 'title': 'DHB-Vizepräsident Bob Hanning - "Die Weltspitze ist extrem breit"', 219 'upload_date': '20180111', 220 }, 221 'params': { 222 'skip_download': True, 223 }, 224 }, 225 { 226 'url': 'http://www.sportschau.de/handballem2018/audio-vorschau---die-handball-em-startet-mit-grossem-favoritenfeld-100.html', 227 'only_matching': True, 228 }, 229 { 230 'url': 'https://kinder.wdr.de/tv/die-sendung-mit-dem-elefanten/av/video-folge---astronaut-100.html', 231 'only_matching': True, 232 }, 233 ] 234 235 def _real_extract(self, url): 236 mobj = re.match(self._VALID_URL, url) 237 display_id = mobj.group('display_id') 238 webpage = self._download_webpage(url, display_id) 239 240 entries = [] 241 242 # Article with several videos 243 244 # for wdr.de the data-extension is in a tag with the class "mediaLink" 245 # for wdr.de radio players, in a tag with the class "wdrrPlayerPlayBtn" 246 # for wdrmaus, in a tag with the class "videoButton" (previously a link 247 # to the page in a multiline "videoLink"-tag) 248 for mobj in re.finditer( 249 r'''(?sx)class= 250 (?: 251 (["\'])(?:mediaLink|wdrrPlayerPlayBtn|videoButton)\b.*?\1[^>]+| 252 (["\'])videoLink\b.*?\2[\s]*>\n[^\n]* 253 )data-extension=(["\'])(?P<data>(?:(?!\3).)+)\3 254 ''', webpage): 255 media_link_obj = self._parse_json( 256 mobj.group('data'), display_id, transform_source=js_to_json, 257 fatal=False) 258 if not media_link_obj: 259 continue 260 jsonp_url = try_get( 261 media_link_obj, lambda x: x['mediaObj']['url'], compat_str) 262 if jsonp_url: 263 entries.append(self.url_result(jsonp_url, ie=WDRIE.ie_key())) 264 265 # Playlist (e.g. https://www1.wdr.de/mediathek/video/sendungen/aktuelle-stunde/aktuelle-stunde-120.html) 266 if not entries: 267 entries = [ 268 self.url_result( 269 compat_urlparse.urljoin(url, mobj.group('href')), 270 ie=WDRPageIE.ie_key()) 271 for mobj in re.finditer( 272 r'<a[^>]+\bhref=(["\'])(?P<href>(?:(?!\1).)+)\1[^>]+\bdata-extension=', 273 webpage) if re.match(self._PAGE_REGEX, mobj.group('href')) 274 ] 275 276 return self.playlist_result(entries, playlist_id=display_id) 277 278 279 class WDRElefantIE(InfoExtractor): 280 _VALID_URL = r'https?://(?:www\.)wdrmaus\.de/elefantenseite/#(?P<id>.+)' 281 _TEST = { 282 'url': 'http://www.wdrmaus.de/elefantenseite/#folge_ostern_2015', 283 'info_dict': { 284 'title': 'Folge Oster-Spezial 2015', 285 'id': 'mdb-1088195', 286 'ext': 'mp4', 287 'age_limit': None, 288 'upload_date': '20150406' 289 }, 290 'params': { 291 'skip_download': True, 292 }, 293 } 294 295 def _real_extract(self, url): 296 display_id = self._match_id(url) 297 298 # Table of Contents seems to always be at this address, so fetch it directly. 299 # The website fetches configurationJS.php5, which links to tableOfContentsJS.php5. 300 table_of_contents = self._download_json( 301 'https://www.wdrmaus.de/elefantenseite/data/tableOfContentsJS.php5', 302 display_id) 303 if display_id not in table_of_contents: 304 raise ExtractorError( 305 'No entry in site\'s table of contents for this URL. ' 306 'Is the fragment part of the URL (after the #) correct?', 307 expected=True) 308 xml_metadata_path = table_of_contents[display_id]['xmlPath'] 309 xml_metadata = self._download_xml( 310 'https://www.wdrmaus.de/elefantenseite/' + xml_metadata_path, 311 display_id) 312 zmdb_url_element = xml_metadata.find('./movie/zmdb_url') 313 if zmdb_url_element is None: 314 raise ExtractorError( 315 '%s is not a video' % display_id, expected=True) 316 return self.url_result(zmdb_url_element.text, ie=WDRIE.ie_key()) 317 318 319 class WDRMobileIE(InfoExtractor): 320 _VALID_URL = r'''(?x) 321 https?://mobile-ondemand\.wdr\.de/ 322 .*?/fsk(?P<age_limit>[0-9]+) 323 /[0-9]+/[0-9]+/ 324 (?P<id>[0-9]+)_(?P<title>[0-9]+)''' 325 IE_NAME = 'wdr:mobile' 326 _TEST = { 327 'url': 'http://mobile-ondemand.wdr.de/CMS2010/mdb/ondemand/weltweit/fsk0/42/421735/421735_4283021.mp4', 328 'info_dict': { 329 'title': '4283021', 330 'id': '421735', 331 'ext': 'mp4', 332 'age_limit': 0, 333 }, 334 'skip': 'Problems with loading data.' 335 } 336 337 def _real_extract(self, url): 338 mobj = re.match(self._VALID_URL, url) 339 return { 340 'id': mobj.group('id'), 341 'title': mobj.group('title'), 342 'age_limit': int(mobj.group('age_limit')), 343 'url': url, 344 'http_headers': { 345 'User-Agent': 'mobile', 346 }, 347 }