niconico.py (20357B)
1 # coding: utf-8 2 from __future__ import unicode_literals 3 4 import datetime 5 import functools 6 import json 7 import math 8 9 from .common import InfoExtractor 10 from ..compat import ( 11 compat_parse_qs, 12 compat_urllib_parse_urlparse, 13 ) 14 from ..utils import ( 15 determine_ext, 16 dict_get, 17 ExtractorError, 18 float_or_none, 19 InAdvancePagedList, 20 int_or_none, 21 parse_duration, 22 parse_iso8601, 23 remove_start, 24 try_get, 25 unified_timestamp, 26 urlencode_postdata, 27 xpath_text, 28 ) 29 30 31 class NiconicoIE(InfoExtractor): 32 IE_NAME = 'niconico' 33 IE_DESC = 'ニコニコ動画' 34 35 _TESTS = [{ 36 'url': 'http://www.nicovideo.jp/watch/sm22312215', 37 'md5': 'd1a75c0823e2f629128c43e1212760f9', 38 'info_dict': { 39 'id': 'sm22312215', 40 'ext': 'mp4', 41 'title': 'Big Buck Bunny', 42 'thumbnail': r're:https?://.*', 43 'uploader': 'takuya0301', 44 'uploader_id': '2698420', 45 'upload_date': '20131123', 46 'timestamp': int, # timestamp is unstable 47 'description': '(c) copyright 2008, Blender Foundation / www.bigbuckbunny.org', 48 'duration': 33, 49 'view_count': int, 50 'comment_count': int, 51 }, 52 'skip': 'Requires an account', 53 }, { 54 # File downloaded with and without credentials are different, so omit 55 # the md5 field 56 'url': 'http://www.nicovideo.jp/watch/nm14296458', 57 'info_dict': { 58 'id': 'nm14296458', 59 'ext': 'swf', 60 'title': '【鏡音リン】Dance on media【オリジナル】take2!', 61 'description': 'md5:689f066d74610b3b22e0f1739add0f58', 62 'thumbnail': r're:https?://.*', 63 'uploader': 'りょうた', 64 'uploader_id': '18822557', 65 'upload_date': '20110429', 66 'timestamp': 1304065916, 67 'duration': 209, 68 }, 69 'skip': 'Requires an account', 70 }, { 71 # 'video exists but is marked as "deleted" 72 # md5 is unstable 73 'url': 'http://www.nicovideo.jp/watch/sm10000', 74 'info_dict': { 75 'id': 'sm10000', 76 'ext': 'unknown_video', 77 'description': 'deleted', 78 'title': 'ドラえもんエターナル第3話「決戦第3新東京市」＜前編＞', 79 'thumbnail': r're:https?://.*', 80 'upload_date': '20071224', 81 'timestamp': int, # timestamp field has different value if logged in 82 'duration': 304, 83 'view_count': int, 84 }, 85 'skip': 'Requires an account', 86 }, { 87 'url': 'http://www.nicovideo.jp/watch/so22543406', 88 'info_dict': { 89 'id': '1388129933', 90 'ext': 'mp4', 91 'title': '【第1回】RADIOアニメロミックス ラブライブ！～のぞえりRadio Garden～', 92 'description': 'md5:b27d224bb0ff53d3c8269e9f8b561cf1', 93 'thumbnail': r're:https?://.*', 94 'timestamp': 1388851200, 95 'upload_date': '20140104', 96 'uploader': 'アニメロチャンネル', 97 'uploader_id': '312', 98 }, 99 'skip': 'The viewing period of the video you were searching for has expired.', 100 }, { 101 # video not available via `getflv`; "old" HTML5 video 102 'url': 'http://www.nicovideo.jp/watch/sm1151009', 103 'md5': '8fa81c364eb619d4085354eab075598a', 104 'info_dict': { 105 'id': 'sm1151009', 106 'ext': 'mp4', 107 'title': 'マスターシステム本体内蔵のスペハリのメインテーマ（ＰＳＧ版）', 108 'description': 'md5:6ee077e0581ff5019773e2e714cdd0b7', 109 'thumbnail': r're:https?://.*', 110 'duration': 184, 111 'timestamp': 1190868283, 112 'upload_date': '20070927', 113 'uploader': 'denden2', 114 'uploader_id': '1392194', 115 'view_count': int, 116 'comment_count': int, 117 }, 118 'skip': 'Requires an account', 119 }, { 120 # "New" HTML5 video 121 # md5 is unstable 122 'url': 'http://www.nicovideo.jp/watch/sm31464864', 123 'info_dict': { 124 'id': 'sm31464864', 125 'ext': 'mp4', 126 'title': '新作TVアニメ「戦姫絶唱シンフォギアAXZ」PV 最高画質', 127 'description': 'md5:e52974af9a96e739196b2c1ca72b5feb', 128 'timestamp': 1498514060, 129 'upload_date': '20170626', 130 'uploader': 'ゲスト', 131 'uploader_id': '40826363', 132 'thumbnail': r're:https?://.*', 133 'duration': 198, 134 'view_count': int, 135 'comment_count': int, 136 }, 137 'skip': 'Requires an account', 138 }, { 139 # Video without owner 140 'url': 'http://www.nicovideo.jp/watch/sm18238488', 141 'md5': 'd265680a1f92bdcbbd2a507fc9e78a9e', 142 'info_dict': { 143 'id': 'sm18238488', 144 'ext': 'mp4', 145 'title': '【実写版】ミュータントタートルズ', 146 'description': 'md5:15df8988e47a86f9e978af2064bf6d8e', 147 'timestamp': 1341160408, 148 'upload_date': '20120701', 149 'uploader': None, 150 'uploader_id': None, 151 'thumbnail': r're:https?://.*', 152 'duration': 5271, 153 'view_count': int, 154 'comment_count': int, 155 }, 156 'skip': 'Requires an account', 157 }, { 158 'url': 'http://sp.nicovideo.jp/watch/sm28964488?ss_pos=1&cp_in=wt_tg', 159 'only_matching': True, 160 }] 161 162 _VALID_URL = r'https?://(?:www\.|secure\.|sp\.)?nicovideo\.jp/watch/(?P<id>(?:[a-z]{2})?[0-9]+)' 163 _NETRC_MACHINE = 'niconico' 164 165 def _real_initialize(self): 166 self._login() 167 168 def _login(self): 169 username, password = self._get_login_info() 170 # No authentication to be performed 171 if not username: 172 return True 173 174 # Log in 175 login_ok = True 176 login_form_strs = { 177 'mail_tel': username, 178 'password': password, 179 } 180 urlh = self._request_webpage( 181 'https://account.nicovideo.jp/api/v1/login', None, 182 note='Logging in', errnote='Unable to log in', 183 data=urlencode_postdata(login_form_strs)) 184 if urlh is False: 185 login_ok = False 186 else: 187 parts = compat_urllib_parse_urlparse(urlh.geturl()) 188 if compat_parse_qs(parts.query).get('message', [None])[0] == 'cant_login': 189 login_ok = False 190 if not login_ok: 191 self._downloader.report_warning('unable to log in: bad username or password') 192 return login_ok 193 194 def _extract_format_for_quality(self, api_data, video_id, audio_quality, video_quality): 195 def yesno(boolean): 196 return 'yes' if boolean else 'no' 197 198 session_api_data = api_data['video']['dmcInfo']['session_api'] 199 session_api_endpoint = session_api_data['urls'][0] 200 201 format_id = '-'.join(map(lambda s: remove_start(s['id'], 'archive_'), [video_quality, audio_quality])) 202 203 session_response = self._download_json( 204 session_api_endpoint['url'], video_id, 205 query={'_format': 'json'}, 206 headers={'Content-Type': 'application/json'}, 207 note='Downloading JSON metadata for %s' % format_id, 208 data=json.dumps({ 209 'session': { 210 'client_info': { 211 'player_id': session_api_data['player_id'], 212 }, 213 'content_auth': { 214 'auth_type': session_api_data['auth_types'][session_api_data['protocols'][0]], 215 'content_key_timeout': session_api_data['content_key_timeout'], 216 'service_id': 'nicovideo', 217 'service_user_id': session_api_data['service_user_id'] 218 }, 219 'content_id': session_api_data['content_id'], 220 'content_src_id_sets': [{ 221 'content_src_ids': [{ 222 'src_id_to_mux': { 223 'audio_src_ids': [audio_quality['id']], 224 'video_src_ids': [video_quality['id']], 225 } 226 }] 227 }], 228 'content_type': 'movie', 229 'content_uri': '', 230 'keep_method': { 231 'heartbeat': { 232 'lifetime': session_api_data['heartbeat_lifetime'] 233 } 234 }, 235 'priority': session_api_data['priority'], 236 'protocol': { 237 'name': 'http', 238 'parameters': { 239 'http_parameters': { 240 'parameters': { 241 'http_output_download_parameters': { 242 'use_ssl': yesno(session_api_endpoint['is_ssl']), 243 'use_well_known_port': yesno(session_api_endpoint['is_well_known_port']), 244 } 245 } 246 } 247 } 248 }, 249 'recipe_id': session_api_data['recipe_id'], 250 'session_operation_auth': { 251 'session_operation_auth_by_signature': { 252 'signature': session_api_data['signature'], 253 'token': session_api_data['token'], 254 } 255 }, 256 'timing_constraint': 'unlimited' 257 } 258 }).encode()) 259 260 resolution = video_quality.get('resolution', {}) 261 262 return { 263 'url': session_response['data']['session']['content_uri'], 264 'format_id': format_id, 265 'ext': 'mp4', # Session API are used in HTML5, which always serves mp4 266 'abr': float_or_none(audio_quality.get('bitrate'), 1000), 267 'vbr': float_or_none(video_quality.get('bitrate'), 1000), 268 'height': resolution.get('height'), 269 'width': resolution.get('width'), 270 } 271 272 def _real_extract(self, url): 273 video_id = self._match_id(url) 274 275 # Get video webpage. We are not actually interested in it for normal 276 # cases, but need the cookies in order to be able to download the 277 # info webpage 278 webpage, handle = self._download_webpage_handle( 279 'http://www.nicovideo.jp/watch/' + video_id, video_id) 280 if video_id.startswith('so'): 281 video_id = self._match_id(handle.geturl()) 282 283 api_data = self._parse_json(self._html_search_regex( 284 'data-api-data="([^"]+)"', webpage, 285 'API data', default='{}'), video_id) 286 287 def _format_id_from_url(video_url): 288 return 'economy' if video_real_url.endswith('low') else 'normal' 289 290 try: 291 video_real_url = api_data['video']['smileInfo']['url'] 292 except KeyError: # Flash videos 293 # Get flv info 294 flv_info_webpage = self._download_webpage( 295 'http://flapi.nicovideo.jp/api/getflv/' + video_id + '?as3=1', 296 video_id, 'Downloading flv info') 297 298 flv_info = compat_parse_qs(flv_info_webpage) 299 if 'url' not in flv_info: 300 if 'deleted' in flv_info: 301 raise ExtractorError('The video has been deleted.', 302 expected=True) 303 elif 'closed' in flv_info: 304 raise ExtractorError('Niconico videos now require logging in', 305 expected=True) 306 elif 'error' in flv_info: 307 raise ExtractorError('%s reports error: %s' % ( 308 self.IE_NAME, flv_info['error'][0]), expected=True) 309 else: 310 raise ExtractorError('Unable to find video URL') 311 312 video_info_xml = self._download_xml( 313 'http://ext.nicovideo.jp/api/getthumbinfo/' + video_id, 314 video_id, note='Downloading video info page') 315 316 def get_video_info(items): 317 if not isinstance(items, list): 318 items = [items] 319 for item in items: 320 ret = xpath_text(video_info_xml, './/' + item) 321 if ret: 322 return ret 323 324 video_real_url = flv_info['url'][0] 325 326 extension = get_video_info('movie_type') 327 if not extension: 328 extension = determine_ext(video_real_url) 329 330 formats = [{ 331 'url': video_real_url, 332 'ext': extension, 333 'format_id': _format_id_from_url(video_real_url), 334 }] 335 else: 336 formats = [] 337 338 dmc_info = api_data['video'].get('dmcInfo') 339 if dmc_info: # "New" HTML5 videos 340 quality_info = dmc_info['quality'] 341 for audio_quality in quality_info['audios']: 342 for video_quality in quality_info['videos']: 343 if not audio_quality['available'] or not video_quality['available']: 344 continue 345 formats.append(self._extract_format_for_quality( 346 api_data, video_id, audio_quality, video_quality)) 347 348 self._sort_formats(formats) 349 else: # "Old" HTML5 videos 350 formats = [{ 351 'url': video_real_url, 352 'ext': 'mp4', 353 'format_id': _format_id_from_url(video_real_url), 354 }] 355 356 def get_video_info(items): 357 return dict_get(api_data['video'], items) 358 359 # Start extracting information 360 title = get_video_info('title') 361 if not title: 362 title = self._og_search_title(webpage, default=None) 363 if not title: 364 title = self._html_search_regex( 365 r'<span[^>]+class="videoHeaderTitle"[^>]*>([^<]+)</span>', 366 webpage, 'video title') 367 368 watch_api_data_string = self._html_search_regex( 369 r'<div[^>]+id="watchAPIDataContainer"[^>]+>([^<]+)</div>', 370 webpage, 'watch api data', default=None) 371 watch_api_data = self._parse_json(watch_api_data_string, video_id) if watch_api_data_string else {} 372 video_detail = watch_api_data.get('videoDetail', {}) 373 374 thumbnail = ( 375 get_video_info(['thumbnail_url', 'thumbnailURL']) 376 or self._html_search_meta('image', webpage, 'thumbnail', default=None) 377 or video_detail.get('thumbnail')) 378 379 description = get_video_info('description') 380 381 timestamp = (parse_iso8601(get_video_info('first_retrieve')) 382 or unified_timestamp(get_video_info('postedDateTime'))) 383 if not timestamp: 384 match = self._html_search_meta('datePublished', webpage, 'date published', default=None) 385 if match: 386 timestamp = parse_iso8601(match.replace('+', ':00+')) 387 if not timestamp and video_detail.get('postedAt'): 388 timestamp = parse_iso8601( 389 video_detail['postedAt'].replace('/', '-'), 390 delimiter=' ', timezone=datetime.timedelta(hours=9)) 391 392 view_count = int_or_none(get_video_info(['view_counter', 'viewCount'])) 393 if not view_count: 394 match = self._html_search_regex( 395 r'>Views: <strong[^>]*>([^<]+)</strong>', 396 webpage, 'view count', default=None) 397 if match: 398 view_count = int_or_none(match.replace(',', '')) 399 view_count = view_count or video_detail.get('viewCount') 400 401 comment_count = (int_or_none(get_video_info('comment_num')) 402 or video_detail.get('commentCount') 403 or try_get(api_data, lambda x: x['thread']['commentCount'])) 404 if not comment_count: 405 match = self._html_search_regex( 406 r'>Comments: <strong[^>]*>([^<]+)</strong>', 407 webpage, 'comment count', default=None) 408 if match: 409 comment_count = int_or_none(match.replace(',', '')) 410 411 duration = (parse_duration( 412 get_video_info('length') 413 or self._html_search_meta( 414 'video:duration', webpage, 'video duration', default=None)) 415 or video_detail.get('length') 416 or get_video_info('duration')) 417 418 webpage_url = get_video_info('watch_url') or url 419 420 # Note: cannot use api_data.get('owner', {}) because owner may be set to "null" 421 # in the JSON, which will cause None to be returned instead of {}. 422 owner = try_get(api_data, lambda x: x.get('owner'), dict) or {} 423 uploader_id = get_video_info(['ch_id', 'user_id']) or owner.get('id') 424 uploader = get_video_info(['ch_name', 'user_nickname']) or owner.get('nickname') 425 426 return { 427 'id': video_id, 428 'title': title, 429 'formats': formats, 430 'thumbnail': thumbnail, 431 'description': description, 432 'uploader': uploader, 433 'timestamp': timestamp, 434 'uploader_id': uploader_id, 435 'view_count': view_count, 436 'comment_count': comment_count, 437 'duration': duration, 438 'webpage_url': webpage_url, 439 } 440 441 442 class NiconicoPlaylistIE(InfoExtractor): 443 _VALID_URL = r'https?://(?:www\.)?nicovideo\.jp/(?:user/\d+/)?mylist/(?P<id>\d+)' 444 445 _TESTS = [{ 446 'url': 'http://www.nicovideo.jp/mylist/27411728', 447 'info_dict': { 448 'id': '27411728', 449 'title': 'AKB48のオールナイトニッポン', 450 'description': 'md5:d89694c5ded4b6c693dea2db6e41aa08', 451 'uploader': 'のっく', 452 'uploader_id': '805442', 453 }, 454 'playlist_mincount': 225, 455 }, { 456 'url': 'https://www.nicovideo.jp/user/805442/mylist/27411728', 457 'only_matching': True, 458 }] 459 _PAGE_SIZE = 100 460 461 def _call_api(self, list_id, resource, query): 462 return self._download_json( 463 'https://nvapi.nicovideo.jp/v2/mylists/' + list_id, list_id, 464 'Downloading %s JSON metatdata' % resource, query=query, 465 headers={'X-Frontend-Id': 6})['data']['mylist'] 466 467 def _parse_owner(self, item): 468 owner = item.get('owner') or {} 469 if owner: 470 return { 471 'uploader': owner.get('name'), 472 'uploader_id': owner.get('id'), 473 } 474 return {} 475 476 def _fetch_page(self, list_id, page): 477 page += 1 478 items = self._call_api(list_id, 'page %d' % page, { 479 'page': page, 480 'pageSize': self._PAGE_SIZE, 481 })['items'] 482 for item in items: 483 video = item.get('video') or {} 484 video_id = video.get('id') 485 if not video_id: 486 continue 487 count = video.get('count') or {} 488 get_count = lambda x: int_or_none(count.get(x)) 489 info = { 490 '_type': 'url', 491 'id': video_id, 492 'title': video.get('title'), 493 'url': 'https://www.nicovideo.jp/watch/' + video_id, 494 'description': video.get('shortDescription'), 495 'duration': int_or_none(video.get('duration')), 496 'view_count': get_count('view'), 497 'comment_count': get_count('comment'), 498 'ie_key': NiconicoIE.ie_key(), 499 } 500 info.update(self._parse_owner(video)) 501 yield info 502 503 def _real_extract(self, url): 504 list_id = self._match_id(url) 505 mylist = self._call_api(list_id, 'list', { 506 'pageSize': 1, 507 }) 508 entries = InAdvancePagedList( 509 functools.partial(self._fetch_page, list_id), 510 math.ceil(mylist['totalItemCount'] / self._PAGE_SIZE), 511 self._PAGE_SIZE) 512 result = self.playlist_result( 513 entries, list_id, mylist.get('name'), mylist.get('description')) 514 result.update(self._parse_owner(mylist)) 515 return result