From: Jaime Marquínez Ferrándiz Date: Fri, 13 Sep 2013 17:55:49 +0000 (+0200) Subject: Merge pull request #1413 from tewe/master X-Git-Url: http://git.oshgnacknak.de/?a=commitdiff_plain;h=74ac9bdd82b8a625ea9782251258ab7da1463877;p=youtube-dl Merge pull request #1413 from tewe/master Add Ustream channel support --- 74ac9bdd82b8a625ea9782251258ab7da1463877 diff --cc test/test_playlists.py index 65de3a55c,65de3a55c..4a2e00b01 --- a/test/test_playlists.py +++ b/test/test_playlists.py @@@ -8,7 -8,7 +8,7 @@@ import jso import os sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) --from youtube_dl.extractor import DailymotionPlaylistIE, VimeoChannelIE ++from youtube_dl.extractor import DailymotionPlaylistIE, VimeoChannelIE, UstreamChannelIE from youtube_dl.utils import * from helper import FakeYDL @@@ -34,5 -34,5 +34,13 @@@ class TestPlaylists(unittest.TestCase) self.assertEqual(result['title'], u'Vimeo Tributes') self.assertTrue(len(result['entries']) > 24) ++ def test_ustream_channel(self): ++ dl = FakeYDL() ++ ie = UstreamChannelIE(dl) ++ result = ie.extract('http://www.ustream.tv/channel/young-americans-for-liberty') ++ self.assertIsPlaylist(result) ++ self.assertEqual(result['id'], u'5124905') ++ self.assertTrue(len(result['entries']) >= 11) ++ if __name__ == '__main__': unittest.main() diff --cc youtube_dl/extractor/ustream.py index 5f423870a,16cdcc765..f69b27d44 --- a/youtube_dl/extractor/ustream.py +++ b/youtube_dl/extractor/ustream.py @@@ -1,6 -1,9 +1,11 @@@ -from HTMLParser import HTMLParser + import json import re -from urlparse import urljoin from .common import InfoExtractor ++from ..utils import ( ++ compat_urlparse, ++ compat_html_parser, ++) class UstreamIE(InfoExtractor): @@@ -43,3 -46,70 +48,64 @@@ 'thumbnail': thumbnail, } return info + + # More robust than regular expressions + -class ChannelParser(HTMLParser): ++class ChannelParser(compat_html_parser.HTMLParser): + """ + + """ + channel_id = None + + def handle_starttag(self, tag, attrs): + if tag != 'meta': + return + values = dict(attrs) + if values.get('name') != 'ustream:channel_id': + return + value = values.get('content', '') + if value.isdigit(): + self.channel_id = value + -class SocialstreamParser(HTMLParser): ++class SocialstreamParser(compat_html_parser.HTMLParser): + """ +
  • + """ + def __init__(self): - HTMLParser.__init__(self) ++ compat_html_parser.HTMLParser.__init__(self) + self.content_ids = [] + + def handle_starttag(self, tag, attrs): + if tag != 'li': + return + for (attr, value) in attrs: + if attr == 'data-content-id' and value.isdigit(): + self.content_ids.append(value) + + class UstreamChannelIE(InfoExtractor): + _VALID_URL = r'https?://www\.ustream\.tv/channel/(?P.+)' + IE_NAME = u'ustream:channel' + + def _real_extract(self, url): + m = re.match(self._VALID_URL, url) + slug = m.group('slug') - # Slugs can be non-ascii, but youtube-dl can't handle non-ascii command lines, - # so if we got this far it's probably percent encoded and we needn't worry. + + p = ChannelParser() + p.feed(self._download_webpage(url, slug)) + p.close() + channel_id = p.channel_id + + p = SocialstreamParser() + BASE = 'http://www.ustream.tv' + next_url = '/ajax/socialstream/videos/%s/1.json' % channel_id + while next_url: - reply = json.loads(self._download_webpage(urljoin(BASE, next_url), channel_id)) ++ reply = json.loads(self._download_webpage(compat_urlparse.urljoin(BASE, next_url), channel_id)) + p.feed(reply['data']) + next_url = reply['nextUrl'] + p.close() + video_ids = p.content_ids + - # From YoutubeChannelIE - - self._downloader.to_screen(u'[ustream] Channel %s: Found %i videos' % (channel_id, len(video_ids))) - + urls = ['http://www.ustream.tv/recorded/' + vid for vid in video_ids] + url_entries = [self.url_result(eurl, 'Ustream') for eurl in urls] - return [self.playlist_result(url_entries, channel_id)] ++ return self.playlist_result(url_entries, channel_id)