commit 222516d97d5ff9e62f3a9860fe2e65aa99c001b3
parent a055469fafe088b6aa0e569d989cbf7f70535951
Author: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 24 Jan 2015 01:38:48 +0100
[downloader] Lay groundwork for external downloaders.
This comes with a very simply implementation for wget; the real work is in setting up the infrastructure.
Diffstat:
7 files changed, 169 insertions(+), 14 deletions(-)
diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
@@ -219,6 +219,7 @@ class YoutubeDL(object):
call_home: Boolean, true iff we are allowed to contact the
youtube-dl servers for debugging.
sleep_interval: Number of seconds to sleep before each download.
+ external_downloader: Executable of the external downloader to call.
The following parameters are not used by YoutubeDL itself, they are used by
diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
@@ -330,6 +330,7 @@ def _real_main(argv=None):
'source_address': opts.source_address,
'call_home': opts.call_home,
'sleep_interval': opts.sleep_interval,
+ 'external_downloader': opts.external_downloader,
}
with YoutubeDL(ydl_opts) as ydl:
diff --git a/youtube_dl/downloader/__init__.py b/youtube_dl/downloader/__init__.py
@@ -1,12 +1,13 @@
from __future__ import unicode_literals
from .common import FileDownloader
+from .external import get_external_downloader
+from .f4m import F4mFD
from .hls import HlsFD
from .hls import NativeHlsFD
from .http import HttpFD
from .mplayer import MplayerFD
from .rtmp import RtmpFD
-from .f4m import F4mFD
from ..utils import (
determine_protocol,
@@ -27,6 +28,12 @@ def get_suitable_downloader(info_dict, params={}):
protocol = determine_protocol(info_dict)
info_dict['protocol'] = protocol
+ external_downloader = params.get('external_downloader')
+ if external_downloader is not None:
+ ed = get_external_downloader(external_downloader)
+ if ed.supports(info_dict):
+ return ed
+
return PROTOCOL_MAP.get(protocol, HttpFD)
diff --git a/youtube_dl/downloader/common.py b/youtube_dl/downloader/common.py
@@ -325,3 +325,24 @@ class FileDownloader(object):
# See YoutubeDl.py (search for progress_hooks) for a description of
# this interface
self._progress_hooks.append(ph)
+
+ def _debug_cmd(self, args, subprocess_encoding, exe=None):
+ if not self.params.get('verbose', False):
+ return
+
+ if exe is None:
+ exe = os.path.basename(args[0])
+
+ if subprocess_encoding:
+ str_args = [
+ a.decode(subprocess_encoding) if isinstance(a, bytes) else a
+ for a in args]
+ else:
+ str_args = args
+ try:
+ import pipes
+ shell_quote = lambda args: ' '.join(map(pipes.quote, str_args))
+ except ImportError:
+ shell_quote = repr
+ self.to_screen('[debug] %s command line: %s' % (
+ exe, shell_quote(str_args)))
diff --git a/youtube_dl/downloader/external.py b/youtube_dl/downloader/external.py
@@ -0,0 +1,131 @@
+from __future__ import unicode_literals
+
+import os.path
+import subprocess
+import sys
+
+from .common import FileDownloader
+from ..utils import (
+ encodeFilename,
+ std_headers,
+)
+
+
+class ExternalFD(FileDownloader):
+ def real_download(self, filename, info_dict):
+ self.report_destination(filename)
+ tmpfilename = self.temp_name(filename)
+
+ retval = self._call_downloader(tmpfilename, info_dict)
+ if retval == 0:
+ fsize = os.path.getsize(encodeFilename(tmpfilename))
+ self.to_screen('\r[%s] Downloaded %s bytes' % (self.get_basename(), fsize))
+ self.try_rename(tmpfilename, filename)
+ self._hook_progress({
+ 'downloaded_bytes': fsize,
+ 'total_bytes': fsize,
+ 'filename': filename,
+ 'status': 'finished',
+ })
+ return True
+ else:
+ self.to_stderr('\n')
+ self.report_error('%s exited with code %d' % (
+ self.get_basename(), retval))
+ return False
+
+ @classmethod
+ def get_basename(cls):
+ return cls.__name__[:-2].lower()
+
+ @property
+ def exe(self):
+ return self.params.get('external_downloader')
+
+ @classmethod
+ def supports(cls, info_dict):
+ return info_dict['protocol'] in ('http', 'https', 'ftp', 'ftps')
+
+ def _calc_headers(self, info_dict):
+ res = std_headers.copy()
+
+ ua = info_dict.get('user_agent')
+ if ua is not None:
+ res['User-Agent'] = ua
+
+ cookies = self._calc_cookies(info_dict)
+ if cookies:
+ res['Cookie'] = cookies
+
+ return res
+
+ def _calc_cookies(self, info_dict):
+ class _PseudoRequest(object):
+ def __init__(self, url):
+ self.url = url
+ self.headers = {}
+ self.unverifiable = False
+
+ def add_unredirected_header(self, k, v):
+ self.headers[k] = v
+
+ def get_full_url(self):
+ return self.url
+
+ def is_unverifiable(self):
+ return self.unverifiable
+
+ def has_header(self, h):
+ return h in self.headers
+
+ pr = _PseudoRequest(info_dict['url'])
+ self.ydl.cookiejar.add_cookie_header(pr)
+ return pr.headers.get('Cookie')
+
+ def _call_downloader(self, tmpfilename, info_dict):
+ """ Either overwrite this or implement _make_cmd """
+ cmd = self._make_cmd(tmpfilename, info_dict)
+
+ if sys.platform == 'win32' and sys.version_info < (3, 0):
+ # Windows subprocess module does not actually support Unicode
+ # on Python 2.x
+ # See http://stackoverflow.com/a/9951851/35070
+ subprocess_encoding = sys.getfilesystemencoding()
+ cmd = [a.encode(subprocess_encoding, 'ignore') for a in cmd]
+ else:
+ subprocess_encoding = None
+ self._debug_cmd(cmd, subprocess_encoding)
+
+ p = subprocess.Popen(
+ cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+ stdout, stderr = p.communicate()
+ if p.returncode != 0:
+ self.to_stderr(stderr)
+ return p.returncode
+
+
+class WgetFD(ExternalFD):
+ def _make_cmd(self, tmpfilename, info_dict):
+ cmd = [self.exe, '-O', tmpfilename, '-nv', '--no-cookies']
+ for key, val in self._calc_headers(info_dict).items():
+ cmd += ['--header', '%s: %s' % (key, val)]
+ cmd += ['--', info_dict['url']]
+ return cmd
+
+
+_BY_NAME = dict(
+ (klass.get_basename(), klass)
+ for name, klass in globals().items()
+ if name.endswith('FD') and name != 'ExternalFD'
+)
+
+
+def list_external_downloaders():
+ return sorted(_BY_NAME.keys())
+
+
+def get_external_downloader(external_downloader):
+ """ Given the name of the executable, see whether we support the given
+ downloader . """
+ bn = os.path.basename(external_downloader)
+ return _BY_NAME[bn]
diff --git a/youtube_dl/downloader/rtmp.py b/youtube_dl/downloader/rtmp.py
@@ -152,19 +152,7 @@ class RtmpFD(FileDownloader):
else:
subprocess_encoding = None
- if self.params.get('verbose', False):
- if subprocess_encoding:
- str_args = [
- a.decode(subprocess_encoding) if isinstance(a, bytes) else a
- for a in args]
- else:
- str_args = args
- try:
- import pipes
- shell_quote = lambda args: ' '.join(map(pipes.quote, str_args))
- except ImportError:
- shell_quote = repr
- self.to_screen('[debug] rtmpdump command line: ' + shell_quote(str_args))
+ self._debug_cmd(args, subprocess_encoding, exe='rtmpdump')
RD_SUCCESS = 0
RD_FAILED = 1
diff --git a/youtube_dl/options.py b/youtube_dl/options.py
@@ -5,6 +5,7 @@ import optparse
import shlex
import sys
+from .downloader.external import list_external_downloaders
from .compat import (
compat_expanduser,
compat_getenv,
@@ -389,6 +390,11 @@ def parseOpts(overrideArguments=None):
'--playlist-reverse',
action='store_true',
help='Download playlist videos in reverse order')
+ downloader.add_option(
+ '--external-downloader',
+ dest='external_downloader', metavar='COMMAND',
+ help='(experimental) Use the specified external downloader. '
+ 'Currently supports %s' % ','.join(list_external_downloaders()))
workarounds = optparse.OptionGroup(parser, 'Workarounds')
workarounds.add_option(
