ard.py (18428B)
1 # coding: utf-8 2 from __future__ import unicode_literals 3 4 import json 5 import re 6 7 from .common import InfoExtractor 8 from .generic import GenericIE 9 from ..utils import ( 10 determine_ext, 11 ExtractorError, 12 int_or_none, 13 parse_duration, 14 qualities, 15 str_or_none, 16 try_get, 17 unified_strdate, 18 unified_timestamp, 19 update_url_query, 20 url_or_none, 21 xpath_text, 22 ) 23 from ..compat import compat_etree_fromstring 24 25 26 class ARDMediathekBaseIE(InfoExtractor): 27 _GEO_COUNTRIES = ['DE'] 28 29 def _extract_media_info(self, media_info_url, webpage, video_id): 30 media_info = self._download_json( 31 media_info_url, video_id, 'Downloading media JSON') 32 return self._parse_media_info(media_info, video_id, '"fsk"' in webpage) 33 34 def _parse_media_info(self, media_info, video_id, fsk): 35 formats = self._extract_formats(media_info, video_id) 36 37 if not formats: 38 if fsk: 39 raise ExtractorError( 40 'This video is only available after 20:00', expected=True) 41 elif media_info.get('_geoblocked'): 42 self.raise_geo_restricted( 43 'This video is not available due to geoblocking', 44 countries=self._GEO_COUNTRIES) 45 46 self._sort_formats(formats) 47 48 subtitles = {} 49 subtitle_url = media_info.get('_subtitleUrl') 50 if subtitle_url: 51 subtitles['de'] = [{ 52 'ext': 'ttml', 53 'url': subtitle_url, 54 }] 55 56 return { 57 'id': video_id, 58 'duration': int_or_none(media_info.get('_duration')), 59 'thumbnail': media_info.get('_previewImage'), 60 'is_live': media_info.get('_isLive') is True, 61 'formats': formats, 62 'subtitles': subtitles, 63 } 64 65 def _extract_formats(self, media_info, video_id): 66 type_ = media_info.get('_type') 67 media_array = media_info.get('_mediaArray', []) 68 formats = [] 69 for num, media in enumerate(media_array): 70 for stream in media.get('_mediaStreamArray', []): 71 stream_urls = stream.get('_stream') 72 if not stream_urls: 73 continue 74 if not isinstance(stream_urls, list): 75 stream_urls = [stream_urls] 76 quality = stream.get('_quality') 77 server = stream.get('_server') 78 for stream_url in stream_urls: 79 if not url_or_none(stream_url): 80 continue 81 ext = determine_ext(stream_url) 82 if quality != 'auto' and ext in ('f4m', 'm3u8'): 83 continue 84 if ext == 'f4m': 85 formats.extend(self._extract_f4m_formats( 86 update_url_query(stream_url, { 87 'hdcore': '3.1.1', 88 'plugin': 'aasp-3.1.1.69.124' 89 }), video_id, f4m_id='hds', fatal=False)) 90 elif ext == 'm3u8': 91 formats.extend(self._extract_m3u8_formats( 92 stream_url, video_id, 'mp4', 'm3u8_native', 93 m3u8_id='hls', fatal=False)) 94 else: 95 if server and server.startswith('rtmp'): 96 f = { 97 'url': server, 98 'play_path': stream_url, 99 'format_id': 'a%s-rtmp-%s' % (num, quality), 100 } 101 else: 102 f = { 103 'url': stream_url, 104 'format_id': 'a%s-%s-%s' % (num, ext, quality) 105 } 106 m = re.search( 107 r'_(?P<width>\d+)x(?P<height>\d+)\.mp4$', 108 stream_url) 109 if m: 110 f.update({ 111 'width': int(m.group('width')), 112 'height': int(m.group('height')), 113 }) 114 if type_ == 'audio': 115 f['vcodec'] = 'none' 116 formats.append(f) 117 return formats 118 119 120 class ARDMediathekIE(ARDMediathekBaseIE): 121 IE_NAME = 'ARD:mediathek' 122 _VALID_URL = r'^https?://(?:(?:(?:www|classic)\.)?ardmediathek\.de|mediathek\.(?:daserste|rbb-online)\.de|one\.ard\.de)/(?:.*/)(?P<video_id>[0-9]+|[^0-9][^/\?]+)[^/\?]*(?:\?.*)?' 123 124 _TESTS = [{ 125 # available till 26.07.2022 126 'url': 'http://www.ardmediathek.de/tv/S%C3%9CDLICHT/Was-ist-die-Kunst-der-Zukunft-liebe-Ann/BR-Fernsehen/Video?bcastId=34633636&documentId=44726822', 127 'info_dict': { 128 'id': '44726822', 129 'ext': 'mp4', 130 'title': 'Was ist die Kunst der Zukunft, liebe Anna McCarthy?', 131 'description': 'md5:4ada28b3e3b5df01647310e41f3a62f5', 132 'duration': 1740, 133 }, 134 'params': { 135 # m3u8 download 136 'skip_download': True, 137 } 138 }, { 139 'url': 'https://one.ard.de/tv/Mord-mit-Aussicht/Mord-mit-Aussicht-6-39-T%C3%B6dliche-Nach/ONE/Video?bcastId=46384294&documentId=55586872', 140 'only_matching': True, 141 }, { 142 # audio 143 'url': 'http://www.ardmediathek.de/tv/WDR-H%C3%B6rspiel-Speicher/Tod-eines-Fu%C3%9Fballers/WDR-3/Audio-Podcast?documentId=28488308&bcastId=23074086', 144 'only_matching': True, 145 }, { 146 'url': 'http://mediathek.daserste.de/sendungen_a-z/328454_anne-will/22429276_vertrauen-ist-gut-spionieren-ist-besser-geht', 147 'only_matching': True, 148 }, { 149 # audio 150 'url': 'http://mediathek.rbb-online.de/radio/Hörspiel/Vor-dem-Fest/kulturradio/Audio?documentId=30796318&topRessort=radio&bcastId=9839158', 151 'only_matching': True, 152 }, { 153 'url': 'https://classic.ardmediathek.de/tv/Panda-Gorilla-Co/Panda-Gorilla-Co-Folge-274/Das-Erste/Video?bcastId=16355486&documentId=58234698', 154 'only_matching': True, 155 }] 156 157 @classmethod 158 def suitable(cls, url): 159 return False if ARDBetaMediathekIE.suitable(url) else super(ARDMediathekIE, cls).suitable(url) 160 161 def _real_extract(self, url): 162 # determine video id from url 163 m = re.match(self._VALID_URL, url) 164 165 document_id = None 166 167 numid = re.search(r'documentId=([0-9]+)', url) 168 if numid: 169 document_id = video_id = numid.group(1) 170 else: 171 video_id = m.group('video_id') 172 173 webpage = self._download_webpage(url, video_id) 174 175 ERRORS = ( 176 ('>Leider liegt eine Störung vor.', 'Video %s is unavailable'), 177 ('>Der gewünschte Beitrag ist nicht mehr verfügbar.<', 178 'Video %s is no longer available'), 179 ) 180 181 for pattern, message in ERRORS: 182 if pattern in webpage: 183 raise ExtractorError(message % video_id, expected=True) 184 185 if re.search(r'[\?&]rss($|[=&])', url): 186 doc = compat_etree_fromstring(webpage.encode('utf-8')) 187 if doc.tag == 'rss': 188 return GenericIE()._extract_rss(url, video_id, doc) 189 190 title = self._og_search_title(webpage, default=None) or self._html_search_regex( 191 [r'<h1(?:\s+class="boxTopHeadline")?>(.*?)</h1>', 192 r'<meta name="dcterms\.title" content="(.*?)"/>', 193 r'<h4 class="headline">(.*?)</h4>', 194 r'<title[^>]*>(.*?)</title>'], 195 webpage, 'title') 196 description = self._og_search_description(webpage, default=None) or self._html_search_meta( 197 'dcterms.abstract', webpage, 'description', default=None) 198 if description is None: 199 description = self._html_search_meta( 200 'description', webpage, 'meta description', default=None) 201 if description is None: 202 description = self._html_search_regex( 203 r'<p\s+class="teasertext">(.+?)</p>', 204 webpage, 'teaser text', default=None) 205 206 # Thumbnail is sometimes not present. 207 # It is in the mobile version, but that seems to use a different URL 208 # structure altogether. 209 thumbnail = self._og_search_thumbnail(webpage, default=None) 210 211 media_streams = re.findall(r'''(?x) 212 mediaCollection\.addMediaStream\([0-9]+,\s*[0-9]+,\s*"[^"]*",\s* 213 "([^"]+)"''', webpage) 214 215 if media_streams: 216 QUALITIES = qualities(['lo', 'hi', 'hq']) 217 formats = [] 218 for furl in set(media_streams): 219 if furl.endswith('.f4m'): 220 fid = 'f4m' 221 else: 222 fid_m = re.match(r'.*\.([^.]+)\.[^.]+$', furl) 223 fid = fid_m.group(1) if fid_m else None 224 formats.append({ 225 'quality': QUALITIES(fid), 226 'format_id': fid, 227 'url': furl, 228 }) 229 self._sort_formats(formats) 230 info = { 231 'formats': formats, 232 } 233 else: # request JSON file 234 if not document_id: 235 video_id = self._search_regex( 236 r'/play/(?:config|media)/(\d+)', webpage, 'media id') 237 info = self._extract_media_info( 238 'http://www.ardmediathek.de/play/media/%s' % video_id, 239 webpage, video_id) 240 241 info.update({ 242 'id': video_id, 243 'title': self._live_title(title) if info.get('is_live') else title, 244 'description': description, 245 'thumbnail': thumbnail, 246 }) 247 248 return info 249 250 251 class ARDIE(InfoExtractor): 252 _VALID_URL = r'(?P<mainurl>https?://(?:www\.)?daserste\.de/(?:[^/?#&]+/)+(?P<id>[^/?#&]+))\.html' 253 _TESTS = [{ 254 # available till 7.01.2022 255 'url': 'https://www.daserste.de/information/talk/maischberger/videos/maischberger-die-woche-video100.html', 256 'md5': '867d8aa39eeaf6d76407c5ad1bb0d4c1', 257 'info_dict': { 258 'id': 'maischberger-die-woche-video100', 259 'display_id': 'maischberger-die-woche-video100', 260 'ext': 'mp4', 261 'duration': 3687.0, 262 'title': 'maischberger. die woche vom 7. Januar 2021', 263 'upload_date': '20210107', 264 'thumbnail': r're:^https?://.*\.jpg$', 265 }, 266 }, { 267 'url': 'https://www.daserste.de/information/politik-weltgeschehen/morgenmagazin/videosextern/dominik-kahun-aus-der-nhl-direkt-zur-weltmeisterschaft-100.html', 268 'only_matching': True, 269 }, { 270 'url': 'https://www.daserste.de/information/nachrichten-wetter/tagesthemen/videosextern/tagesthemen-17736.html', 271 'only_matching': True, 272 }, { 273 'url': 'http://www.daserste.de/information/reportage-dokumentation/dokus/videos/die-story-im-ersten-mission-unter-falscher-flagge-100.html', 274 'only_matching': True, 275 }, { 276 'url': 'https://www.daserste.de/unterhaltung/serie/in-aller-freundschaft-die-jungen-aerzte/Drehpause-100.html', 277 'only_matching': True, 278 }, { 279 'url': 'https://www.daserste.de/unterhaltung/film/filmmittwoch-im-ersten/videos/making-ofwendezeit-video-100.html', 280 'only_matching': True, 281 }] 282 283 def _real_extract(self, url): 284 mobj = re.match(self._VALID_URL, url) 285 display_id = mobj.group('id') 286 287 player_url = mobj.group('mainurl') + '~playerXml.xml' 288 doc = self._download_xml(player_url, display_id) 289 video_node = doc.find('./video') 290 upload_date = unified_strdate(xpath_text( 291 video_node, './broadcastDate')) 292 thumbnail = xpath_text(video_node, './/teaserImage//variant/url') 293 294 formats = [] 295 for a in video_node.findall('.//asset'): 296 file_name = xpath_text(a, './fileName', default=None) 297 if not file_name: 298 continue 299 format_type = a.attrib.get('type') 300 format_url = url_or_none(file_name) 301 if format_url: 302 ext = determine_ext(file_name) 303 if ext == 'm3u8': 304 formats.extend(self._extract_m3u8_formats( 305 format_url, display_id, 'mp4', entry_protocol='m3u8_native', 306 m3u8_id=format_type or 'hls', fatal=False)) 307 continue 308 elif ext == 'f4m': 309 formats.extend(self._extract_f4m_formats( 310 update_url_query(format_url, {'hdcore': '3.7.0'}), 311 display_id, f4m_id=format_type or 'hds', fatal=False)) 312 continue 313 f = { 314 'format_id': format_type, 315 'width': int_or_none(xpath_text(a, './frameWidth')), 316 'height': int_or_none(xpath_text(a, './frameHeight')), 317 'vbr': int_or_none(xpath_text(a, './bitrateVideo')), 318 'abr': int_or_none(xpath_text(a, './bitrateAudio')), 319 'vcodec': xpath_text(a, './codecVideo'), 320 'tbr': int_or_none(xpath_text(a, './totalBitrate')), 321 } 322 server_prefix = xpath_text(a, './serverPrefix', default=None) 323 if server_prefix: 324 f.update({ 325 'url': server_prefix, 326 'playpath': file_name, 327 }) 328 else: 329 if not format_url: 330 continue 331 f['url'] = format_url 332 formats.append(f) 333 self._sort_formats(formats) 334 335 return { 336 'id': xpath_text(video_node, './videoId', default=display_id), 337 'formats': formats, 338 'display_id': display_id, 339 'title': video_node.find('./title').text, 340 'duration': parse_duration(video_node.find('./duration').text), 341 'upload_date': upload_date, 342 'thumbnail': thumbnail, 343 } 344 345 346 class ARDBetaMediathekIE(ARDMediathekBaseIE): 347 _VALID_URL = r'https://(?:(?:beta|www)\.)?ardmediathek\.de/(?:[^/]+/)?(?:player|live|video)/(?:[^/]+/)*(?P<id>Y3JpZDovL[a-zA-Z0-9]+)' 348 _TESTS = [{ 349 'url': 'https://www.ardmediathek.de/mdr/video/die-robuste-roswita/Y3JpZDovL21kci5kZS9iZWl0cmFnL2Ntcy84MWMxN2MzZC0wMjkxLTRmMzUtODk4ZS0wYzhlOWQxODE2NGI/', 350 'md5': 'a1dc75a39c61601b980648f7c9f9f71d', 351 'info_dict': { 352 'display_id': 'die-robuste-roswita', 353 'id': '78566716', 354 'title': 'Die robuste Roswita', 355 'description': r're:^Der Mord.*totgeglaubte Ehefrau Roswita', 356 'duration': 5316, 357 'thumbnail': 'https://img.ardmediathek.de/standard/00/78/56/67/84/575672121/16x9/960?mandant=ard', 358 'timestamp': 1596658200, 359 'upload_date': '20200805', 360 'ext': 'mp4', 361 }, 362 }, { 363 'url': 'https://beta.ardmediathek.de/ard/video/Y3JpZDovL2Rhc2Vyc3RlLmRlL3RhdG9ydC9mYmM4NGM1NC0xNzU4LTRmZGYtYWFhZS0wYzcyZTIxNGEyMDE', 364 'only_matching': True, 365 }, { 366 'url': 'https://ardmediathek.de/ard/video/saartalk/saartalk-gesellschaftsgift-haltung-gegen-hass/sr-fernsehen/Y3JpZDovL3NyLW9ubGluZS5kZS9TVF84MTY4MA/', 367 'only_matching': True, 368 }, { 369 'url': 'https://www.ardmediathek.de/ard/video/trailer/private-eyes-s01-e01/one/Y3JpZDovL3dkci5kZS9CZWl0cmFnLTE1MTgwYzczLWNiMTEtNGNkMS1iMjUyLTg5MGYzOWQxZmQ1YQ/', 370 'only_matching': True, 371 }, { 372 'url': 'https://www.ardmediathek.de/ard/player/Y3JpZDovL3N3ci5kZS9hZXgvbzEwNzE5MTU/', 373 'only_matching': True, 374 }, { 375 'url': 'https://www.ardmediathek.de/swr/live/Y3JpZDovL3N3ci5kZS8xMzQ4MTA0Mg', 376 'only_matching': True, 377 }, { 378 'url': 'https://www.ardmediathek.de/video/coronavirus-update-ndr-info/astrazeneca-kurz-lockdown-und-pims-syndrom-81/ndr/Y3JpZDovL25kci5kZS84NzE0M2FjNi0wMWEwLTQ5ODEtOTE5NS1mOGZhNzdhOTFmOTI/', 379 'only_matching': True, 380 }, { 381 'url': 'https://www.ardmediathek.de/ard/player/Y3JpZDovL3dkci5kZS9CZWl0cmFnLWQ2NDJjYWEzLTMwZWYtNGI4NS1iMTI2LTU1N2UxYTcxOGIzOQ/tatort-duo-koeln-leipzig-ihr-kinderlein-kommet', 382 'only_matching': True, 383 }] 384 385 def _real_extract(self, url): 386 video_id = self._match_id(url) 387 388 player_page = self._download_json( 389 'https://api.ardmediathek.de/public-gateway', 390 video_id, data=json.dumps({ 391 'query': '''{ 392 playerPage(client: "ard", clipId: "%s") { 393 blockedByFsk 394 broadcastedOn 395 maturityContentRating 396 mediaCollection { 397 _duration 398 _geoblocked 399 _isLive 400 _mediaArray { 401 _mediaStreamArray { 402 _quality 403 _server 404 _stream 405 } 406 } 407 _previewImage 408 _subtitleUrl 409 _type 410 } 411 show { 412 title 413 } 414 synopsis 415 title 416 tracking { 417 atiCustomVars { 418 contentId 419 } 420 } 421 } 422 }''' % video_id, 423 }).encode(), headers={ 424 'Content-Type': 'application/json' 425 })['data']['playerPage'] 426 title = player_page['title'] 427 content_id = str_or_none(try_get( 428 player_page, lambda x: x['tracking']['atiCustomVars']['contentId'])) 429 media_collection = player_page.get('mediaCollection') or {} 430 if not media_collection and content_id: 431 media_collection = self._download_json( 432 'https://www.ardmediathek.de/play/media/' + content_id, 433 content_id, fatal=False) or {} 434 info = self._parse_media_info( 435 media_collection, content_id or video_id, 436 player_page.get('blockedByFsk')) 437 age_limit = None 438 description = player_page.get('synopsis') 439 maturity_content_rating = player_page.get('maturityContentRating') 440 if maturity_content_rating: 441 age_limit = int_or_none(maturity_content_rating.lstrip('FSK')) 442 if not age_limit and description: 443 age_limit = int_or_none(self._search_regex( 444 r'\(FSK\s*(\d+)\)\s*$', description, 'age limit', default=None)) 445 info.update({ 446 'age_limit': age_limit, 447 'title': title, 448 'description': description, 449 'timestamp': unified_timestamp(player_page.get('broadcastedOn')), 450 'series': try_get(player_page, lambda x: x['show']['title']), 451 }) 452 return info