commit 25afc2a7830e281e849609202b4f70728664bdb7
parent 9603b6601208333bc49e0c69199f0e652a7aaea3
Author: Sergey M․ <dstftw@gmail.com>
Date: Sat, 27 Aug 2016 04:55:55 +0700
[downloader/dash:hls] Respect --fragment-retries and --skip-unavailable-fragments (Closes #10165, closes #10448)
Diffstat:
2 files changed, 41 insertions(+), 12 deletions(-)
diff --git a/youtube_dl/downloader/dash.py b/youtube_dl/downloader/dash.py
@@ -38,6 +38,7 @@ class DashSegmentsFD(FragmentFD):
segments_filenames = []
fragment_retries = self.params.get('fragment_retries', 0)
+ skip_unavailable_fragments = self.params.get('skip_unavailable_fragments', True)
def append_url_to_file(target_url, tmp_filename, segment_name):
target_filename = '%s-%s' % (tmp_filename, segment_name)
@@ -52,19 +53,20 @@ class DashSegmentsFD(FragmentFD):
down.close()
segments_filenames.append(target_sanitized)
break
- except (compat_urllib_error.HTTPError, ) as err:
+ except compat_urllib_error.HTTPError:
# YouTube may often return 404 HTTP error for a fragment causing the
# whole download to fail. However if the same fragment is immediately
# retried with the same request data this usually succeeds (1-2 attemps
# is usually enough) thus allowing to download the whole file successfully.
- # So, we will retry all fragments that fail with 404 HTTP error for now.
- if err.code != 404:
- raise
- # Retry fragment
+ # To be future-proof we will retry all fragments that fail with any
+ # HTTP error.
count += 1
if count <= fragment_retries:
self.report_retry_fragment(segment_name, count, fragment_retries)
if count > fragment_retries:
+ if skip_unavailable_fragments:
+ self.report_skip_fragment(segment_name)
+ return
self.report_error('giving up after %s fragment retries' % fragment_retries)
return False
diff --git a/youtube_dl/downloader/hls.py b/youtube_dl/downloader/hls.py
@@ -13,6 +13,7 @@ from .fragment import FragmentFD
from .external import FFmpegFD
from ..compat import (
+ compat_urllib_error,
compat_urlparse,
compat_struct_pack,
)
@@ -83,6 +84,10 @@ class HlsFD(FragmentFD):
self._prepare_and_start_frag_download(ctx)
+ fragment_retries = self.params.get('fragment_retries', 0)
+ skip_unavailable_fragments = self.params.get('skip_unavailable_fragments', True)
+ test = self.params.get('test', False)
+
extra_query = None
extra_param_to_segment_url = info_dict.get('extra_param_to_segment_url')
if extra_param_to_segment_url:
@@ -99,15 +104,37 @@ class HlsFD(FragmentFD):
line
if re.match(r'^https?://', line)
else compat_urlparse.urljoin(man_url, line))
- frag_filename = '%s-Frag%d' % (ctx['tmpfilename'], i)
+ frag_name = 'Frag%d' % i
+ frag_filename = '%s-%s' % (ctx['tmpfilename'], frag_name)
if extra_query:
frag_url = update_url_query(frag_url, extra_query)
- success = ctx['dl'].download(frag_filename, {'url': frag_url})
- if not success:
+ count = 0
+ while count <= fragment_retries:
+ try:
+ success = ctx['dl'].download(frag_filename, {'url': frag_url})
+ if not success:
+ return False
+ down, frag_sanitized = sanitize_open(frag_filename, 'rb')
+ frag_content = down.read()
+ down.close()
+ break
+ except compat_urllib_error.HTTPError:
+ # Unavailable (possibly temporary) fragments may be served.
+ # First we try to retry then either skip or abort.
+ # See https://github.com/rg3/youtube-dl/issues/10165,
+ # https://github.com/rg3/youtube-dl/issues/10448).
+ count += 1
+ if count <= fragment_retries:
+ self.report_retry_fragment(frag_name, count, fragment_retries)
+ if count > fragment_retries:
+ if skip_unavailable_fragments:
+ i += 1
+ media_sequence += 1
+ self.report_skip_fragment(frag_name)
+ continue
+ self.report_error(
+ 'giving up after %s fragment retries' % fragment_retries)
return False
- down, frag_sanitized = sanitize_open(frag_filename, 'rb')
- frag_content = down.read()
- down.close()
if decrypt_info['METHOD'] == 'AES-128':
iv = decrypt_info.get('IV') or compat_struct_pack('>8xq', media_sequence)
frag_content = AES.new(
@@ -115,7 +142,7 @@ class HlsFD(FragmentFD):
ctx['dest_stream'].write(frag_content)
frags_filenames.append(frag_sanitized)
# We only download the first fragment during the test
- if self.params.get('test', False):
+ if test:
break
i += 1
media_sequence += 1
