make_lazy_extractors.py (2872B)
1 from __future__ import unicode_literals, print_function 2 3 from inspect import getsource 4 import io 5 import os 6 from os.path import dirname as dirn 7 import sys 8 9 print('WARNING: Lazy loading extractors is an experimental feature that may not always work', file=sys.stderr) 10 11 sys.path.insert(0, dirn(dirn((os.path.abspath(__file__))))) 12 13 lazy_extractors_filename = sys.argv[1] 14 if os.path.exists(lazy_extractors_filename): 15 os.remove(lazy_extractors_filename) 16 17 from youtube_dl.extractor import _ALL_CLASSES 18 from youtube_dl.extractor.common import InfoExtractor, SearchInfoExtractor 19 20 with open('devscripts/lazy_load_template.py', 'rt') as f: 21 module_template = f.read() 22 23 module_contents = [ 24 module_template + '\n' + getsource(InfoExtractor.suitable) + '\n', 25 'class LazyLoadSearchExtractor(LazyLoadExtractor):\n pass\n'] 26 27 ie_template = ''' 28 class {name}({bases}): 29 _VALID_URL = {valid_url!r} 30 _module = '{module}' 31 ''' 32 33 make_valid_template = ''' 34 @classmethod 35 def _make_valid_url(cls): 36 return {valid_url!r} 37 ''' 38 39 40 def get_base_name(base): 41 if base is InfoExtractor: 42 return 'LazyLoadExtractor' 43 elif base is SearchInfoExtractor: 44 return 'LazyLoadSearchExtractor' 45 else: 46 return base.__name__ 47 48 49 def build_lazy_ie(ie, name): 50 valid_url = getattr(ie, '_VALID_URL', None) 51 s = ie_template.format( 52 name=name, 53 bases=', '.join(map(get_base_name, ie.__bases__)), 54 valid_url=valid_url, 55 module=ie.__module__) 56 if ie.suitable.__func__ is not InfoExtractor.suitable.__func__: 57 s += '\n' + getsource(ie.suitable) 58 if hasattr(ie, '_make_valid_url'): 59 # search extractors 60 s += make_valid_template.format(valid_url=ie._make_valid_url()) 61 return s 62 63 64 # find the correct sorting and add the required base classes so that subclasses 65 # can be correctly created 66 classes = _ALL_CLASSES[:-1] 67 ordered_cls = [] 68 while classes: 69 for c in classes[:]: 70 bases = set(c.__bases__) - set((object, InfoExtractor, SearchInfoExtractor)) 71 stop = False 72 for b in bases: 73 if b not in classes and b not in ordered_cls: 74 if b.__name__ == 'GenericIE': 75 exit() 76 classes.insert(0, b) 77 stop = True 78 if stop: 79 break 80 if all(b in ordered_cls for b in bases): 81 ordered_cls.append(c) 82 classes.remove(c) 83 break 84 ordered_cls.append(_ALL_CLASSES[-1]) 85 86 names = [] 87 for ie in ordered_cls: 88 name = ie.__name__ 89 src = build_lazy_ie(ie, name) 90 module_contents.append(src) 91 if ie in _ALL_CLASSES: 92 names.append(name) 93 94 module_contents.append( 95 '_ALL_CLASSES = [{0}]'.format(', '.join(names))) 96 97 module_src = '\n'.join(module_contents) + '\n' 98 99 with io.open(lazy_extractors_filename, 'wt', encoding='utf-8') as f: 100 f.write(module_src)