commit 4333d56494a48929303ce306330ee3cded989d48
parent 882c6992967914c245e086ddaacde9d595cd6ed9
Author: Sergey M <dstftw@gmail.com>
Date: Wed, 23 Mar 2016 20:12:32 +0500
Merge pull request #8898 from dstftw/fragment-retries
Add --fragment-retries option (Fixes #8466)
Diffstat:
5 files changed, 64 insertions(+), 16 deletions(-)
diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
@@ -144,14 +144,20 @@ def _real_main(argv=None):
if numeric_limit is None:
parser.error('invalid max_filesize specified')
opts.max_filesize = numeric_limit
- if opts.retries is not None:
- if opts.retries in ('inf', 'infinite'):
- opts_retries = float('inf')
+
+ def parse_retries(retries):
+ if retries in ('inf', 'infinite'):
+ parsed_retries = float('inf')
else:
try:
- opts_retries = int(opts.retries)
+ parsed_retries = int(retries)
except (TypeError, ValueError):
parser.error('invalid retry count specified')
+ return parsed_retries
+ if opts.retries is not None:
+ opts.retries = parse_retries(opts.retries)
+ if opts.fragment_retries is not None:
+ opts.fragment_retries = parse_retries(opts.fragment_retries)
if opts.buffersize is not None:
numeric_buffersize = FileDownloader.parse_bytes(opts.buffersize)
if numeric_buffersize is None:
@@ -299,7 +305,8 @@ def _real_main(argv=None):
'force_generic_extractor': opts.force_generic_extractor,
'ratelimit': opts.ratelimit,
'nooverwrites': opts.nooverwrites,
- 'retries': opts_retries,
+ 'retries': opts.retries,
+ 'fragment_retries': opts.fragment_retries,
'buffersize': opts.buffersize,
'noresizebuffer': opts.noresizebuffer,
'continuedl': opts.continue_dl,
diff --git a/youtube_dl/downloader/common.py b/youtube_dl/downloader/common.py
@@ -116,6 +116,10 @@ class FileDownloader(object):
return '%10s' % ('%s/s' % format_bytes(speed))
@staticmethod
+ def format_retries(retries):
+ return 'inf' if retries == float('inf') else '%.0f' % retries
+
+ @staticmethod
def best_block_size(elapsed_time, bytes):
new_min = max(bytes / 2.0, 1.0)
new_max = min(max(bytes * 2.0, 1.0), 4194304) # Do not surpass 4 MB
@@ -297,7 +301,9 @@ class FileDownloader(object):
def report_retry(self, count, retries):
"""Report retry in case of HTTP error 5xx"""
- self.to_screen('[download] Got server HTTP error. Retrying (attempt %d of %.0f)...' % (count, retries))
+ self.to_screen(
+ '[download] Got server HTTP error. Retrying (attempt %d of %s)...'
+ % (count, self.format_retries(retries)))
def report_file_already_downloaded(self, file_name):
"""Report file has already been fully downloaded."""
diff --git a/youtube_dl/downloader/dash.py b/youtube_dl/downloader/dash.py
@@ -4,6 +4,7 @@ import os
import re
from .fragment import FragmentFD
+from ..compat import compat_urllib_error
from ..utils import (
sanitize_open,
encodeFilename,
@@ -36,20 +37,41 @@ class DashSegmentsFD(FragmentFD):
segments_filenames = []
- def append_url_to_file(target_url, target_filename):
- success = ctx['dl'].download(target_filename, {'url': combine_url(base_url, target_url)})
- if not success:
+ fragment_retries = self.params.get('fragment_retries', 0)
+
+ def append_url_to_file(target_url, tmp_filename, segment_name):
+ target_filename = '%s-%s' % (tmp_filename, segment_name)
+ count = 0
+ while count <= fragment_retries:
+ try:
+ success = ctx['dl'].download(target_filename, {'url': combine_url(base_url, target_url)})
+ if not success:
+ return False
+ down, target_sanitized = sanitize_open(target_filename, 'rb')
+ ctx['dest_stream'].write(down.read())
+ down.close()
+ segments_filenames.append(target_sanitized)
+ break
+ except (compat_urllib_error.HTTPError, ) as err:
+ # YouTube may often return 404 HTTP error for a fragment causing the
+ # whole download to fail. However if the same fragment is immediately
+ # retried with the same request data this usually succeeds (1-2 attemps
+ # is usually enough) thus allowing to download the whole file successfully.
+ # So, we will retry all fragments that fail with 404 HTTP error for now.
+ if err.code != 404:
+ raise
+ # Retry fragment
+ count += 1
+ if count <= fragment_retries:
+ self.report_retry_fragment(segment_name, count, fragment_retries)
+ if count > fragment_retries:
+ self.report_error('giving up after %s fragment retries' % fragment_retries)
return False
- down, target_sanitized = sanitize_open(target_filename, 'rb')
- ctx['dest_stream'].write(down.read())
- down.close()
- segments_filenames.append(target_sanitized)
if initialization_url:
- append_url_to_file(initialization_url, ctx['tmpfilename'] + '-Init')
+ append_url_to_file(initialization_url, ctx['tmpfilename'], 'Init')
for i, segment_url in enumerate(segment_urls):
- segment_filename = '%s-Seg%d' % (ctx['tmpfilename'], i)
- append_url_to_file(segment_url, segment_filename)
+ append_url_to_file(segment_url, ctx['tmpfilename'], 'Seg%d' % i)
self._finish_frag_download(ctx)
diff --git a/youtube_dl/downloader/fragment.py b/youtube_dl/downloader/fragment.py
@@ -19,8 +19,17 @@ class HttpQuietDownloader(HttpFD):
class FragmentFD(FileDownloader):
"""
A base file downloader class for fragmented media (e.g. f4m/m3u8 manifests).
+
+ Available options:
+
+ fragment_retries: Number of times to retry a fragment for HTTP error (DASH only)
"""
+ def report_retry_fragment(self, fragment_name, count, retries):
+ self.to_screen(
+ '[download] Got server HTTP error. Retrying fragment %s (attempt %d of %s)...'
+ % (fragment_name, count, self.format_retries(retries)))
+
def _prepare_and_start_frag_download(self, ctx):
self._prepare_frag_download(ctx)
self._start_frag_download(ctx)
diff --git a/youtube_dl/options.py b/youtube_dl/options.py
@@ -400,6 +400,10 @@ def parseOpts(overrideArguments=None):
dest='retries', metavar='RETRIES', default=10,
help='Number of retries (default is %default), or "infinite".')
downloader.add_option(
+ '--fragment-retries',
+ dest='fragment_retries', metavar='RETRIES', default=10,
+ help='Number of retries for a fragment (default is %default), or "infinite" (DASH only)')
+ downloader.add_option(
'--buffer-size',
dest='buffersize', metavar='SIZE', default='1024',
help='Size of download buffer (e.g. 1024 or 16K) (default is %default)')
