asiancrush.py (7807B)
1 # coding: utf-8 2 from __future__ import unicode_literals 3 4 import functools 5 import re 6 7 from .common import InfoExtractor 8 from .kaltura import KalturaIE 9 from ..utils import ( 10 extract_attributes, 11 int_or_none, 12 OnDemandPagedList, 13 parse_age_limit, 14 strip_or_none, 15 try_get, 16 ) 17 18 19 class AsianCrushBaseIE(InfoExtractor): 20 _VALID_URL_BASE = r'https?://(?:www\.)?(?P<host>(?:(?:asiancrush|yuyutv|midnightpulp)\.com|(?:cocoro|retrocrush)\.tv))' 21 _KALTURA_KEYS = [ 22 'video_url', 'progressive_url', 'download_url', 'thumbnail_url', 23 'widescreen_thumbnail_url', 'screencap_widescreen', 24 ] 25 _API_SUFFIX = {'retrocrush.tv': '-ott'} 26 27 def _call_api(self, host, endpoint, video_id, query, resource): 28 return self._download_json( 29 'https://api%s.%s/%s' % (self._API_SUFFIX.get(host, ''), host, endpoint), video_id, 30 'Downloading %s JSON metadata' % resource, query=query, 31 headers=self.geo_verification_headers())['objects'] 32 33 def _download_object_data(self, host, object_id, resource): 34 return self._call_api( 35 host, 'search', object_id, {'id': object_id}, resource)[0] 36 37 def _get_object_description(self, obj): 38 return strip_or_none(obj.get('long_description') or obj.get('short_description')) 39 40 def _parse_video_data(self, video): 41 title = video['name'] 42 43 entry_id, partner_id = [None] * 2 44 for k in self._KALTURA_KEYS: 45 k_url = video.get(k) 46 if k_url: 47 mobj = re.search(r'/p/(\d+)/.+?/entryId/([^/]+)/', k_url) 48 if mobj: 49 partner_id, entry_id = mobj.groups() 50 break 51 52 meta_categories = try_get(video, lambda x: x['meta']['categories'], list) or [] 53 categories = list(filter(None, [c.get('name') for c in meta_categories])) 54 55 show_info = video.get('show_info') or {} 56 57 return { 58 '_type': 'url_transparent', 59 'url': 'kaltura:%s:%s' % (partner_id, entry_id), 60 'ie_key': KalturaIE.ie_key(), 61 'id': entry_id, 62 'title': title, 63 'description': self._get_object_description(video), 64 'age_limit': parse_age_limit(video.get('mpaa_rating') or video.get('tv_rating')), 65 'categories': categories, 66 'series': show_info.get('show_name'), 67 'season_number': int_or_none(show_info.get('season_num')), 68 'season_id': show_info.get('season_id'), 69 'episode_number': int_or_none(show_info.get('episode_num')), 70 } 71 72 73 class AsianCrushIE(AsianCrushBaseIE): 74 _VALID_URL = r'%s/video/(?:[^/]+/)?0+(?P<id>\d+)v\b' % AsianCrushBaseIE._VALID_URL_BASE 75 _TESTS = [{ 76 'url': 'https://www.asiancrush.com/video/004289v/women-who-flirt', 77 'md5': 'c3b740e48d0ba002a42c0b72857beae6', 78 'info_dict': { 79 'id': '1_y4tmjm5r', 80 'ext': 'mp4', 81 'title': 'Women Who Flirt', 82 'description': 'md5:b65c7e0ae03a85585476a62a186f924c', 83 'timestamp': 1496936429, 84 'upload_date': '20170608', 85 'uploader_id': 'craig@crifkin.com', 86 'age_limit': 13, 87 'categories': 'count:5', 88 'duration': 5812, 89 }, 90 }, { 91 'url': 'https://www.asiancrush.com/video/she-was-pretty/011886v-pretty-episode-3/', 92 'only_matching': True, 93 }, { 94 'url': 'https://www.yuyutv.com/video/013886v/the-act-of-killing/', 95 'only_matching': True, 96 }, { 97 'url': 'https://www.yuyutv.com/video/peep-show/013922v-warring-factions/', 98 'only_matching': True, 99 }, { 100 'url': 'https://www.midnightpulp.com/video/010400v/drifters/', 101 'only_matching': True, 102 }, { 103 'url': 'https://www.midnightpulp.com/video/mononoke/016378v-zashikiwarashi-part-1/', 104 'only_matching': True, 105 }, { 106 'url': 'https://www.cocoro.tv/video/the-wonderful-wizard-of-oz/008878v-the-wonderful-wizard-of-oz-ep01/', 107 'only_matching': True, 108 }, { 109 'url': 'https://www.retrocrush.tv/video/true-tears/012328v-i...gave-away-my-tears', 110 'only_matching': True, 111 }] 112 113 def _real_extract(self, url): 114 host, video_id = re.match(self._VALID_URL, url).groups() 115 116 if host == 'cocoro.tv': 117 webpage = self._download_webpage(url, video_id) 118 embed_vars = self._parse_json(self._search_regex( 119 r'iEmbedVars\s*=\s*({.+?})', webpage, 'embed vars', 120 default='{}'), video_id, fatal=False) or {} 121 video_id = embed_vars.get('entry_id') or video_id 122 123 video = self._download_object_data(host, video_id, 'video') 124 return self._parse_video_data(video) 125 126 127 class AsianCrushPlaylistIE(AsianCrushBaseIE): 128 _VALID_URL = r'%s/series/0+(?P<id>\d+)s\b' % AsianCrushBaseIE._VALID_URL_BASE 129 _TESTS = [{ 130 'url': 'https://www.asiancrush.com/series/006447s/fruity-samurai', 131 'info_dict': { 132 'id': '6447', 133 'title': 'Fruity Samurai', 134 'description': 'md5:7535174487e4a202d3872a7fc8f2f154', 135 }, 136 'playlist_count': 13, 137 }, { 138 'url': 'https://www.yuyutv.com/series/013920s/peep-show/', 139 'only_matching': True, 140 }, { 141 'url': 'https://www.midnightpulp.com/series/016375s/mononoke/', 142 'only_matching': True, 143 }, { 144 'url': 'https://www.cocoro.tv/series/008549s/the-wonderful-wizard-of-oz/', 145 'only_matching': True, 146 }, { 147 'url': 'https://www.retrocrush.tv/series/012355s/true-tears', 148 'only_matching': True, 149 }] 150 _PAGE_SIZE = 1000000000 151 152 def _fetch_page(self, domain, parent_id, page): 153 videos = self._call_api( 154 domain, 'getreferencedobjects', parent_id, { 155 'max': self._PAGE_SIZE, 156 'object_type': 'video', 157 'parent_id': parent_id, 158 'start': page * self._PAGE_SIZE, 159 }, 'page %d' % (page + 1)) 160 for video in videos: 161 yield self._parse_video_data(video) 162 163 def _real_extract(self, url): 164 host, playlist_id = re.match(self._VALID_URL, url).groups() 165 166 if host == 'cocoro.tv': 167 webpage = self._download_webpage(url, playlist_id) 168 169 entries = [] 170 171 for mobj in re.finditer( 172 r'<a[^>]+href=(["\'])(?P<url>%s.*?)\1[^>]*>' % AsianCrushIE._VALID_URL, 173 webpage): 174 attrs = extract_attributes(mobj.group(0)) 175 if attrs.get('class') == 'clearfix': 176 entries.append(self.url_result( 177 mobj.group('url'), ie=AsianCrushIE.ie_key())) 178 179 title = self._html_search_regex( 180 r'(?s)<h1\b[^>]\bid=["\']movieTitle[^>]+>(.+?)</h1>', webpage, 181 'title', default=None) or self._og_search_title( 182 webpage, default=None) or self._html_search_meta( 183 'twitter:title', webpage, 'title', 184 default=None) or self._search_regex( 185 r'<title>([^<]+)</title>', webpage, 'title', fatal=False) 186 if title: 187 title = re.sub(r'\s*\|\s*.+?$', '', title) 188 189 description = self._og_search_description( 190 webpage, default=None) or self._html_search_meta( 191 'twitter:description', webpage, 'description', fatal=False) 192 else: 193 show = self._download_object_data(host, playlist_id, 'show') 194 title = show.get('name') 195 description = self._get_object_description(show) 196 entries = OnDemandPagedList( 197 functools.partial(self._fetch_page, host, playlist_id), 198 self._PAGE_SIZE) 199 200 return self.playlist_result(entries, playlist_id, title, description)