bandcamp.py (14460B)
1 # coding: utf-8 2 from __future__ import unicode_literals 3 4 import random 5 import re 6 import time 7 8 from .common import InfoExtractor 9 from ..compat import compat_str 10 from ..utils import ( 11 ExtractorError, 12 float_or_none, 13 int_or_none, 14 KNOWN_EXTENSIONS, 15 parse_filesize, 16 str_or_none, 17 try_get, 18 update_url_query, 19 unified_strdate, 20 unified_timestamp, 21 url_or_none, 22 urljoin, 23 ) 24 25 26 class BandcampIE(InfoExtractor): 27 _VALID_URL = r'https?://[^/]+\.bandcamp\.com/track/(?P<id>[^/?#&]+)' 28 _TESTS = [{ 29 'url': 'http://youtube-dl.bandcamp.com/track/youtube-dl-test-song', 30 'md5': 'c557841d5e50261777a6585648adf439', 31 'info_dict': { 32 'id': '1812978515', 33 'ext': 'mp3', 34 'title': "youtube-dl \"'/\\ä↭ - youtube-dl \"'/\\ä↭ - youtube-dl test song \"'/\\ä↭", 35 'duration': 9.8485, 36 'uploader': 'youtube-dl "\'/\\ä↭', 37 'upload_date': '20121129', 38 'timestamp': 1354224127, 39 }, 40 '_skip': 'There is a limit of 200 free downloads / month for the test song' 41 }, { 42 # free download 43 'url': 'http://benprunty.bandcamp.com/track/lanius-battle', 44 'info_dict': { 45 'id': '2650410135', 46 'ext': 'aiff', 47 'title': 'Ben Prunty - Lanius (Battle)', 48 'thumbnail': r're:^https?://.*\.jpg$', 49 'uploader': 'Ben Prunty', 50 'timestamp': 1396508491, 51 'upload_date': '20140403', 52 'release_timestamp': 1396483200, 53 'release_date': '20140403', 54 'duration': 260.877, 55 'track': 'Lanius (Battle)', 56 'track_number': 1, 57 'track_id': '2650410135', 58 'artist': 'Ben Prunty', 59 'album': 'FTL: Advanced Edition Soundtrack', 60 }, 61 }, { 62 # no free download, mp3 128 63 'url': 'https://relapsealumni.bandcamp.com/track/hail-to-fire', 64 'md5': 'fec12ff55e804bb7f7ebeb77a800c8b7', 65 'info_dict': { 66 'id': '2584466013', 67 'ext': 'mp3', 68 'title': 'Mastodon - Hail to Fire', 69 'thumbnail': r're:^https?://.*\.jpg$', 70 'uploader': 'Mastodon', 71 'timestamp': 1322005399, 72 'upload_date': '20111122', 73 'release_timestamp': 1076112000, 74 'release_date': '20040207', 75 'duration': 120.79, 76 'track': 'Hail to Fire', 77 'track_number': 5, 78 'track_id': '2584466013', 79 'artist': 'Mastodon', 80 'album': 'Call of the Mastodon', 81 }, 82 }] 83 84 def _extract_data_attr(self, webpage, video_id, attr='tralbum', fatal=True): 85 return self._parse_json(self._html_search_regex( 86 r'data-%s=(["\'])({.+?})\1' % attr, webpage, 87 attr + ' data', group=2), video_id, fatal=fatal) 88 89 def _real_extract(self, url): 90 title = self._match_id(url) 91 webpage = self._download_webpage(url, title) 92 tralbum = self._extract_data_attr(webpage, title) 93 thumbnail = self._og_search_thumbnail(webpage) 94 95 track_id = None 96 track = None 97 track_number = None 98 duration = None 99 100 formats = [] 101 track_info = try_get(tralbum, lambda x: x['trackinfo'][0], dict) 102 if track_info: 103 file_ = track_info.get('file') 104 if isinstance(file_, dict): 105 for format_id, format_url in file_.items(): 106 if not url_or_none(format_url): 107 continue 108 ext, abr_str = format_id.split('-', 1) 109 formats.append({ 110 'format_id': format_id, 111 'url': self._proto_relative_url(format_url, 'http:'), 112 'ext': ext, 113 'vcodec': 'none', 114 'acodec': ext, 115 'abr': int_or_none(abr_str), 116 }) 117 track = track_info.get('title') 118 track_id = str_or_none( 119 track_info.get('track_id') or track_info.get('id')) 120 track_number = int_or_none(track_info.get('track_num')) 121 duration = float_or_none(track_info.get('duration')) 122 123 embed = self._extract_data_attr(webpage, title, 'embed', False) 124 current = tralbum.get('current') or {} 125 artist = embed.get('artist') or current.get('artist') or tralbum.get('artist') 126 timestamp = unified_timestamp( 127 current.get('publish_date') or tralbum.get('album_publish_date')) 128 129 download_link = tralbum.get('freeDownloadPage') 130 if download_link: 131 track_id = compat_str(tralbum['id']) 132 133 download_webpage = self._download_webpage( 134 download_link, track_id, 'Downloading free downloads page') 135 136 blob = self._extract_data_attr(download_webpage, track_id, 'blob') 137 138 info = try_get( 139 blob, (lambda x: x['digital_items'][0], 140 lambda x: x['download_items'][0]), dict) 141 if info: 142 downloads = info.get('downloads') 143 if isinstance(downloads, dict): 144 if not track: 145 track = info.get('title') 146 if not artist: 147 artist = info.get('artist') 148 if not thumbnail: 149 thumbnail = info.get('thumb_url') 150 151 download_formats = {} 152 download_formats_list = blob.get('download_formats') 153 if isinstance(download_formats_list, list): 154 for f in blob['download_formats']: 155 name, ext = f.get('name'), f.get('file_extension') 156 if all(isinstance(x, compat_str) for x in (name, ext)): 157 download_formats[name] = ext.strip('.') 158 159 for format_id, f in downloads.items(): 160 format_url = f.get('url') 161 if not format_url: 162 continue 163 # Stat URL generation algorithm is reverse engineered from 164 # download_*_bundle_*.js 165 stat_url = update_url_query( 166 format_url.replace('/download/', '/statdownload/'), { 167 '.rand': int(time.time() * 1000 * random.random()), 168 }) 169 format_id = f.get('encoding_name') or format_id 170 stat = self._download_json( 171 stat_url, track_id, 'Downloading %s JSON' % format_id, 172 transform_source=lambda s: s[s.index('{'):s.rindex('}') + 1], 173 fatal=False) 174 if not stat: 175 continue 176 retry_url = url_or_none(stat.get('retry_url')) 177 if not retry_url: 178 continue 179 formats.append({ 180 'url': self._proto_relative_url(retry_url, 'http:'), 181 'ext': download_formats.get(format_id), 182 'format_id': format_id, 183 'format_note': f.get('description'), 184 'filesize': parse_filesize(f.get('size_mb')), 185 'vcodec': 'none', 186 }) 187 188 self._sort_formats(formats) 189 190 title = '%s - %s' % (artist, track) if artist else track 191 192 if not duration: 193 duration = float_or_none(self._html_search_meta( 194 'duration', webpage, default=None)) 195 196 return { 197 'id': track_id, 198 'title': title, 199 'thumbnail': thumbnail, 200 'uploader': artist, 201 'timestamp': timestamp, 202 'release_timestamp': unified_timestamp(tralbum.get('album_release_date')), 203 'duration': duration, 204 'track': track, 205 'track_number': track_number, 206 'track_id': track_id, 207 'artist': artist, 208 'album': embed.get('album_title'), 209 'formats': formats, 210 } 211 212 213 class BandcampAlbumIE(BandcampIE): 214 IE_NAME = 'Bandcamp:album' 215 _VALID_URL = r'https?://(?:(?P<subdomain>[^.]+)\.)?bandcamp\.com(?:/album/(?P<id>[^/?#&]+))?' 216 217 _TESTS = [{ 218 'url': 'http://blazo.bandcamp.com/album/jazz-format-mixtape-vol-1', 219 'playlist': [ 220 { 221 'md5': '39bc1eded3476e927c724321ddf116cf', 222 'info_dict': { 223 'id': '1353101989', 224 'ext': 'mp3', 225 'title': 'Blazo - Intro', 226 'timestamp': 1311756226, 227 'upload_date': '20110727', 228 'uploader': 'Blazo', 229 } 230 }, 231 { 232 'md5': '1a2c32e2691474643e912cc6cd4bffaa', 233 'info_dict': { 234 'id': '38097443', 235 'ext': 'mp3', 236 'title': 'Blazo - Kero One - Keep It Alive (Blazo remix)', 237 'timestamp': 1311757238, 238 'upload_date': '20110727', 239 'uploader': 'Blazo', 240 } 241 }, 242 ], 243 'info_dict': { 244 'title': 'Jazz Format Mixtape vol.1', 245 'id': 'jazz-format-mixtape-vol-1', 246 'uploader_id': 'blazo', 247 }, 248 'params': { 249 'playlistend': 2 250 }, 251 'skip': 'Bandcamp imposes download limits.' 252 }, { 253 'url': 'http://nightbringer.bandcamp.com/album/hierophany-of-the-open-grave', 254 'info_dict': { 255 'title': 'Hierophany of the Open Grave', 256 'uploader_id': 'nightbringer', 257 'id': 'hierophany-of-the-open-grave', 258 }, 259 'playlist_mincount': 9, 260 }, { 261 'url': 'http://dotscale.bandcamp.com', 262 'info_dict': { 263 'title': 'Loom', 264 'id': 'dotscale', 265 'uploader_id': 'dotscale', 266 }, 267 'playlist_mincount': 7, 268 }, { 269 # with escaped quote in title 270 'url': 'https://jstrecords.bandcamp.com/album/entropy-ep', 271 'info_dict': { 272 'title': '"Entropy" EP', 273 'uploader_id': 'jstrecords', 274 'id': 'entropy-ep', 275 'description': 'md5:0ff22959c943622972596062f2f366a5', 276 }, 277 'playlist_mincount': 3, 278 }, { 279 # not all tracks have songs 280 'url': 'https://insulters.bandcamp.com/album/we-are-the-plague', 281 'info_dict': { 282 'id': 'we-are-the-plague', 283 'title': 'WE ARE THE PLAGUE', 284 'uploader_id': 'insulters', 285 'description': 'md5:b3cf845ee41b2b1141dc7bde9237255f', 286 }, 287 'playlist_count': 2, 288 }] 289 290 @classmethod 291 def suitable(cls, url): 292 return (False 293 if BandcampWeeklyIE.suitable(url) or BandcampIE.suitable(url) 294 else super(BandcampAlbumIE, cls).suitable(url)) 295 296 def _real_extract(self, url): 297 uploader_id, album_id = re.match(self._VALID_URL, url).groups() 298 playlist_id = album_id or uploader_id 299 webpage = self._download_webpage(url, playlist_id) 300 tralbum = self._extract_data_attr(webpage, playlist_id) 301 track_info = tralbum.get('trackinfo') 302 if not track_info: 303 raise ExtractorError('The page doesn\'t contain any tracks') 304 # Only tracks with duration info have songs 305 entries = [ 306 self.url_result( 307 urljoin(url, t['title_link']), BandcampIE.ie_key(), 308 str_or_none(t.get('track_id') or t.get('id')), t.get('title')) 309 for t in track_info 310 if t.get('duration')] 311 312 current = tralbum.get('current') or {} 313 314 return { 315 '_type': 'playlist', 316 'uploader_id': uploader_id, 317 'id': playlist_id, 318 'title': current.get('title'), 319 'description': current.get('about'), 320 'entries': entries, 321 } 322 323 324 class BandcampWeeklyIE(BandcampIE): 325 IE_NAME = 'Bandcamp:weekly' 326 _VALID_URL = r'https?://(?:www\.)?bandcamp\.com/?\?(?:.*?&)?show=(?P<id>\d+)' 327 _TESTS = [{ 328 'url': 'https://bandcamp.com/?show=224', 329 'md5': 'b00df799c733cf7e0c567ed187dea0fd', 330 'info_dict': { 331 'id': '224', 332 'ext': 'opus', 333 'title': 'BC Weekly April 4th 2017 - Magic Moments', 334 'description': 'md5:5d48150916e8e02d030623a48512c874', 335 'duration': 5829.77, 336 'release_date': '20170404', 337 'series': 'Bandcamp Weekly', 338 'episode': 'Magic Moments', 339 'episode_id': '224', 340 }, 341 'params': { 342 'format': 'opus-lo', 343 }, 344 }, { 345 'url': 'https://bandcamp.com/?blah/blah@&show=228', 346 'only_matching': True 347 }] 348 349 def _real_extract(self, url): 350 show_id = self._match_id(url) 351 webpage = self._download_webpage(url, show_id) 352 353 blob = self._extract_data_attr(webpage, show_id, 'blob') 354 355 show = blob['bcw_data'][show_id] 356 357 formats = [] 358 for format_id, format_url in show['audio_stream'].items(): 359 if not url_or_none(format_url): 360 continue 361 for known_ext in KNOWN_EXTENSIONS: 362 if known_ext in format_id: 363 ext = known_ext 364 break 365 else: 366 ext = None 367 formats.append({ 368 'format_id': format_id, 369 'url': format_url, 370 'ext': ext, 371 'vcodec': 'none', 372 }) 373 self._sort_formats(formats) 374 375 title = show.get('audio_title') or 'Bandcamp Weekly' 376 subtitle = show.get('subtitle') 377 if subtitle: 378 title += ' - %s' % subtitle 379 380 return { 381 'id': show_id, 382 'title': title, 383 'description': show.get('desc') or show.get('short_desc'), 384 'duration': float_or_none(show.get('audio_duration')), 385 'is_live': False, 386 'release_date': unified_strdate(show.get('published_date')), 387 'series': 'Bandcamp Weekly', 388 'episode': show.get('subtitle'), 389 'episode_id': show_id, 390 'formats': formats 391 }