Add metacafe.com support and minor changse

diff --git a/youtube-dl b/youtube-dl
index fe64819c75a7c481ee75c243c02b80d881b624f8..c0d85aa569d0a6e1441a7e4e58394d4d6d1ffde0 100755 (executable)
--- a/youtube-dl
+++ b/youtube-dl
@@ -253,7 +253,6 @@ class FileDownloader(object):
                                        raise SameFileError(self._params['outtmpl'])
 
                                for result in results:
-
                                        # Forced printings
                                        if self._params.get('forcetitle', False):
                                                print result['title']
@@ -363,7 +362,7 @@ class InfoExtractor(object):
        @staticmethod
        def suitable(url):
                """Receives a URL and returns True if suitable for this IE."""
-               return True
+               return False
 
        def initialize(self):
                """Initializes an instance (authentication, etc)."""
@@ -400,10 +399,15 @@ class InfoExtractor(object):
 class YoutubeIE(InfoExtractor):
        """Information extractor for youtube.com."""
 
+       _VALID_URL = r'^((?:http://)?(?:\w+\.)?youtube\.com/(?:(?:v/)|(?:(?:watch(?:\.php)?)?\?(?:.+&)?v=)))?([0-9A-Za-z_-]+)(?(1).+)?$'
        _LOGIN_URL = 'http://www.youtube.com/login?next=/'
        _AGE_URL = 'http://www.youtube.com/verify_age?next_url=/'
        _NETRC_MACHINE = 'youtube'
 
+       @staticmethod
+       def suitable(url):
+               return (re.match(YoutubeIE._VALID_URL, url) is not None)
+
        def report_login(self):
                """Report attempt to log in."""
                self.to_stdout('[youtube] Logging in')
@@ -486,7 +490,7 @@ class YoutubeIE(InfoExtractor):
 
        def _real_extract(self, url):
                # Extract video id from URL
-               mobj = re.match(r'^((?:http://)?(?:\w+\.)?youtube\.com/(?:(?:v/)|(?:(?:watch(?:\.php)?)?\?(?:.+&)?v=)))?([0-9A-Za-z_-]+)(?(1).+)?$', url)
+               mobj = re.match(self._VALID_URL, url)
                if mobj is None:
                        self.to_stderr('ERROR: invalid URL: %s' % url)
                        return [None]
@@ -554,6 +558,124 @@ class YoutubeIE(InfoExtractor):
                        'ext':          video_extension,
                        }]
 
+class MetacafeIE(InfoExtractor):
+       """Information Extractor for metacafe.com."""
+
+       _VALID_URL = r'(?:http://)?(?:www\.)?metacafe\.com/watch/([^/]+)/([^/]+)/.*'
+       _DISCLAIMER = 'http://www.metacafe.com/disclaimer'
+       _youtube_ie = None
+
+       def __init__(self, youtube_ie, downloader=None):
+               InfoExtractor.__init__(self, downloader)
+               self._youtube_ie = youtube_ie
+
+       @staticmethod
+       def suitable(url):
+               return (re.match(MetacafeIE._VALID_URL, url) is not None)
+
+       def report_disclaimer(self):
+               """Report disclaimer retrieval."""
+               self.to_stdout('[metacafe] Retrieving disclaimer')
+
+       def report_age_confirmation(self):
+               """Report attempt to confirm age."""
+               self.to_stdout('[metacafe] Confirming age')
+       
+       def report_download_webpage(self, video_id):
+               """Report webpage download."""
+               self.to_stdout('[metacafe] %s: Downloading webpage' % video_id)
+       
+       def report_extraction(self, video_id):
+               """Report information extraction."""
+               self.to_stdout('[metacafe] %s: Extracting information' % video_id)
+
+       def _real_initialize(self):
+               # Retrieve disclaimer
+               request = urllib2.Request(self._DISCLAIMER, None, std_headers)
+               try:
+                       self.report_disclaimer()
+                       disclaimer = urllib2.urlopen(request).read()
+               except (urllib2.URLError, httplib.HTTPException, socket.error), err:
+                       self.to_stderr('ERROR: unable to retrieve disclaimer: %s' % str(err))
+                       return
+
+               # Confirm age
+               disclaimer_form = {
+                       'allowAdultContent': '1',
+                       'submit': "Continue - I'm over 18",
+                       }
+               request = urllib2.Request('http://www.metacafe.com/watch/', urllib.urlencode(disclaimer_form), std_headers)
+               try:
+                       self.report_age_confirmation()
+                       disclaimer = urllib2.urlopen(request).read()
+               except (urllib2.URLError, httplib.HTTPException, socket.error), err:
+                       self.to_stderr('ERROR: unable to confirm age: %s' % str(err))
+                       return
+       
+       def _real_extract(self, url):
+               # Extract id and simplified title from URL
+               mobj = re.match(self._VALID_URL, url)
+               if mobj is None:
+                       self.to_stderr('ERROR: invalid URL: %s' % url)
+                       return [None]
+
+               video_id = mobj.group(1)
+
+               # Check if video comes from YouTube
+               mobj2 = re.match(r'^yt-(.*)$', video_id)
+               if mobj2 is not None:
+                       return self._youtube_ie.extract('http://www.youtube.com/watch?v=%s' % mobj2.group(1))
+
+               simple_title = mobj.group(2).decode('utf-8')
+               video_extension = 'flv'
+
+               # Retrieve video webpage to extract further information
+               request = urllib2.Request('http://www.metacafe.com/watch/%s/' % video_id)
+               try:
+                       self.report_download_webpage(video_id)
+                       webpage = urllib2.urlopen(request).read()
+               except (urllib2.URLError, httplib.HTTPException, socket.error), err:
+                       self.to_stderr('ERROR: unable retrieve video webpage: %s' % str(err))
+                       return [None]
+
+               # Extract URL, uploader and title from webpage
+               self.report_extraction(video_id)
+               mobj = re.search(r'(?m)"mediaURL":"(http.*?\.flv)"', webpage)
+               if mobj is None:
+                       self.to_stderr('ERROR: unable to extract media URL')
+                       return [None]
+               mediaURL = mobj.group(1).replace('\\', '')
+
+               mobj = re.search(r'(?m)"gdaKey":"(.*?)"', webpage)
+               if mobj is None:
+                       self.to_stderr('ERROR: unable to extract gdaKey')
+                       return [None]
+               gdaKey = mobj.group(1)
+
+               video_url = '%s?__gda__=%s' % (mediaURL, gdaKey)
+
+               mobj = re.search(r'(?im)<meta name="title" content="Metacafe - ([^"]+)"', webpage)
+               if mobj is None:
+                       self.to_stderr('ERROR: unable to extract title')
+                       return [None]
+               video_title = mobj.group(1).decode('utf-8')
+
+               mobj = re.search(r'(?m)<li id="ChnlUsr">.*?Submitter:<br />(.*?)</li>', webpage)
+               if mobj is None:
+                       self.to_stderr('ERROR: unable to extract uploader nickname')
+                       return [None]
+               video_uploader = re.sub(r'<.*?>', '', mobj.group(1))
+
+               # Return information
+               return [{
+                       'id':           video_id,
+                       'url':          video_url,
+                       'uploader':     video_uploader,
+                       'title':        video_title,
+                       'stitle':       simple_title,
+                       'ext':          video_extension,
+                       }]
+
 if __name__ == '__main__':
        try:
                # Modules needed only when running the main program
@@ -628,6 +750,7 @@ if __name__ == '__main__':
 
                # Information extractors
                youtube_ie = YoutubeIE()
+               metacafe_ie = MetacafeIE(youtube_ie)
 
                # File downloader
                fd = FileDownloader({
@@ -646,6 +769,7 @@ if __name__ == '__main__':
                        'ignoreerrors': opts.ignoreerrors,
                        'ratelimit': opts.ratelimit,
                        })
+               fd.add_info_extractor(metacafe_ie)
                fd.add_info_extractor(youtube_ie)
                retcode = fd.download(args)
                sys.exit(retcode)