buildserver.py (13935B)
1 #!/usr/bin/python3 2 3 import argparse 4 import ctypes 5 import functools 6 import shutil 7 import subprocess 8 import sys 9 import tempfile 10 import threading 11 import traceback 12 import os.path 13 14 sys.path.insert(0, os.path.dirname(os.path.dirname((os.path.abspath(__file__))))) 15 from youtube_dl.compat import ( 16 compat_input, 17 compat_http_server, 18 compat_str, 19 compat_urlparse, 20 ) 21 22 # These are not used outside of buildserver.py thus not in compat.py 23 24 try: 25 import winreg as compat_winreg 26 except ImportError: # Python 2 27 import _winreg as compat_winreg 28 29 try: 30 import socketserver as compat_socketserver 31 except ImportError: # Python 2 32 import SocketServer as compat_socketserver 33 34 35 class BuildHTTPServer(compat_socketserver.ThreadingMixIn, compat_http_server.HTTPServer): 36 allow_reuse_address = True 37 38 39 advapi32 = ctypes.windll.advapi32 40 41 SC_MANAGER_ALL_ACCESS = 0xf003f 42 SC_MANAGER_CREATE_SERVICE = 0x02 43 SERVICE_WIN32_OWN_PROCESS = 0x10 44 SERVICE_AUTO_START = 0x2 45 SERVICE_ERROR_NORMAL = 0x1 46 DELETE = 0x00010000 47 SERVICE_STATUS_START_PENDING = 0x00000002 48 SERVICE_STATUS_RUNNING = 0x00000004 49 SERVICE_ACCEPT_STOP = 0x1 50 51 SVCNAME = 'youtubedl_builder' 52 53 LPTSTR = ctypes.c_wchar_p 54 START_CALLBACK = ctypes.WINFUNCTYPE(None, ctypes.c_int, ctypes.POINTER(LPTSTR)) 55 56 57 class SERVICE_TABLE_ENTRY(ctypes.Structure): 58 _fields_ = [ 59 ('lpServiceName', LPTSTR), 60 ('lpServiceProc', START_CALLBACK) 61 ] 62 63 64 HandlerEx = ctypes.WINFUNCTYPE( 65 ctypes.c_int, # return 66 ctypes.c_int, # dwControl 67 ctypes.c_int, # dwEventType 68 ctypes.c_void_p, # lpEventData, 69 ctypes.c_void_p, # lpContext, 70 ) 71 72 73 def _ctypes_array(c_type, py_array): 74 ar = (c_type * len(py_array))() 75 ar[:] = py_array 76 return ar 77 78 79 def win_OpenSCManager(): 80 res = advapi32.OpenSCManagerW(None, None, SC_MANAGER_ALL_ACCESS) 81 if not res: 82 raise Exception('Opening service manager failed - ' 83 'are you running this as administrator?') 84 return res 85 86 87 def win_install_service(service_name, cmdline): 88 manager = win_OpenSCManager() 89 try: 90 h = advapi32.CreateServiceW( 91 manager, service_name, None, 92 SC_MANAGER_CREATE_SERVICE, SERVICE_WIN32_OWN_PROCESS, 93 SERVICE_AUTO_START, SERVICE_ERROR_NORMAL, 94 cmdline, None, None, None, None, None) 95 if not h: 96 raise OSError('Service creation failed: %s' % ctypes.FormatError()) 97 98 advapi32.CloseServiceHandle(h) 99 finally: 100 advapi32.CloseServiceHandle(manager) 101 102 103 def win_uninstall_service(service_name): 104 manager = win_OpenSCManager() 105 try: 106 h = advapi32.OpenServiceW(manager, service_name, DELETE) 107 if not h: 108 raise OSError('Could not find service %s: %s' % ( 109 service_name, ctypes.FormatError())) 110 111 try: 112 if not advapi32.DeleteService(h): 113 raise OSError('Deletion failed: %s' % ctypes.FormatError()) 114 finally: 115 advapi32.CloseServiceHandle(h) 116 finally: 117 advapi32.CloseServiceHandle(manager) 118 119 120 def win_service_report_event(service_name, msg, is_error=True): 121 with open('C:/sshkeys/log', 'a', encoding='utf-8') as f: 122 f.write(msg + '\n') 123 124 event_log = advapi32.RegisterEventSourceW(None, service_name) 125 if not event_log: 126 raise OSError('Could not report event: %s' % ctypes.FormatError()) 127 128 try: 129 type_id = 0x0001 if is_error else 0x0004 130 event_id = 0xc0000000 if is_error else 0x40000000 131 lines = _ctypes_array(LPTSTR, [msg]) 132 133 if not advapi32.ReportEventW( 134 event_log, type_id, 0, event_id, None, len(lines), 0, 135 lines, None): 136 raise OSError('Event reporting failed: %s' % ctypes.FormatError()) 137 finally: 138 advapi32.DeregisterEventSource(event_log) 139 140 141 def win_service_handler(stop_event, *args): 142 try: 143 raise ValueError('Handler called with args ' + repr(args)) 144 TODO 145 except Exception as e: 146 tb = traceback.format_exc() 147 msg = str(e) + '\n' + tb 148 win_service_report_event(service_name, msg, is_error=True) 149 raise 150 151 152 def win_service_set_status(handle, status_code): 153 svcStatus = SERVICE_STATUS() 154 svcStatus.dwServiceType = SERVICE_WIN32_OWN_PROCESS 155 svcStatus.dwCurrentState = status_code 156 svcStatus.dwControlsAccepted = SERVICE_ACCEPT_STOP 157 158 svcStatus.dwServiceSpecificExitCode = 0 159 160 if not advapi32.SetServiceStatus(handle, ctypes.byref(svcStatus)): 161 raise OSError('SetServiceStatus failed: %r' % ctypes.FormatError()) 162 163 164 def win_service_main(service_name, real_main, argc, argv_raw): 165 try: 166 # args = [argv_raw[i].value for i in range(argc)] 167 stop_event = threading.Event() 168 handler = HandlerEx(functools.partial(stop_event, win_service_handler)) 169 h = advapi32.RegisterServiceCtrlHandlerExW(service_name, handler, None) 170 if not h: 171 raise OSError('Handler registration failed: %s' % 172 ctypes.FormatError()) 173 174 TODO 175 except Exception as e: 176 tb = traceback.format_exc() 177 msg = str(e) + '\n' + tb 178 win_service_report_event(service_name, msg, is_error=True) 179 raise 180 181 182 def win_service_start(service_name, real_main): 183 try: 184 cb = START_CALLBACK( 185 functools.partial(win_service_main, service_name, real_main)) 186 dispatch_table = _ctypes_array(SERVICE_TABLE_ENTRY, [ 187 SERVICE_TABLE_ENTRY( 188 service_name, 189 cb 190 ), 191 SERVICE_TABLE_ENTRY(None, ctypes.cast(None, START_CALLBACK)) 192 ]) 193 194 if not advapi32.StartServiceCtrlDispatcherW(dispatch_table): 195 raise OSError('ctypes start failed: %s' % ctypes.FormatError()) 196 except Exception as e: 197 tb = traceback.format_exc() 198 msg = str(e) + '\n' + tb 199 win_service_report_event(service_name, msg, is_error=True) 200 raise 201 202 203 def main(args=None): 204 parser = argparse.ArgumentParser() 205 parser.add_argument('-i', '--install', 206 action='store_const', dest='action', const='install', 207 help='Launch at Windows startup') 208 parser.add_argument('-u', '--uninstall', 209 action='store_const', dest='action', const='uninstall', 210 help='Remove Windows service') 211 parser.add_argument('-s', '--service', 212 action='store_const', dest='action', const='service', 213 help='Run as a Windows service') 214 parser.add_argument('-b', '--bind', metavar='<host:port>', 215 action='store', default='0.0.0.0:8142', 216 help='Bind to host:port (default %default)') 217 options = parser.parse_args(args=args) 218 219 if options.action == 'install': 220 fn = os.path.abspath(__file__).replace('v:', '\\\\vboxsrv\\vbox') 221 cmdline = '%s %s -s -b %s' % (sys.executable, fn, options.bind) 222 win_install_service(SVCNAME, cmdline) 223 return 224 225 if options.action == 'uninstall': 226 win_uninstall_service(SVCNAME) 227 return 228 229 if options.action == 'service': 230 win_service_start(SVCNAME, main) 231 return 232 233 host, port_str = options.bind.split(':') 234 port = int(port_str) 235 236 print('Listening on %s:%d' % (host, port)) 237 srv = BuildHTTPServer((host, port), BuildHTTPRequestHandler) 238 thr = threading.Thread(target=srv.serve_forever) 239 thr.start() 240 compat_input('Press ENTER to shut down') 241 srv.shutdown() 242 thr.join() 243 244 245 def rmtree(path): 246 for name in os.listdir(path): 247 fname = os.path.join(path, name) 248 if os.path.isdir(fname): 249 rmtree(fname) 250 else: 251 os.chmod(fname, 0o666) 252 os.remove(fname) 253 os.rmdir(path) 254 255 256 class BuildError(Exception): 257 def __init__(self, output, code=500): 258 self.output = output 259 self.code = code 260 261 def __str__(self): 262 return self.output 263 264 265 class HTTPError(BuildError): 266 pass 267 268 269 class PythonBuilder(object): 270 def __init__(self, **kwargs): 271 python_version = kwargs.pop('python', '3.4') 272 python_path = None 273 for node in ('Wow6432Node\\', ''): 274 try: 275 key = compat_winreg.OpenKey( 276 compat_winreg.HKEY_LOCAL_MACHINE, 277 r'SOFTWARE\%sPython\PythonCore\%s\InstallPath' % (node, python_version)) 278 try: 279 python_path, _ = compat_winreg.QueryValueEx(key, '') 280 finally: 281 compat_winreg.CloseKey(key) 282 break 283 except Exception: 284 pass 285 286 if not python_path: 287 raise BuildError('No such Python version: %s' % python_version) 288 289 self.pythonPath = python_path 290 291 super(PythonBuilder, self).__init__(**kwargs) 292 293 294 class GITInfoBuilder(object): 295 def __init__(self, **kwargs): 296 try: 297 self.user, self.repoName = kwargs['path'][:2] 298 self.rev = kwargs.pop('rev') 299 except ValueError: 300 raise BuildError('Invalid path') 301 except KeyError as e: 302 raise BuildError('Missing mandatory parameter "%s"' % e.args[0]) 303 304 path = os.path.join(os.environ['APPDATA'], 'Build archive', self.repoName, self.user) 305 if not os.path.exists(path): 306 os.makedirs(path) 307 self.basePath = tempfile.mkdtemp(dir=path) 308 self.buildPath = os.path.join(self.basePath, 'build') 309 310 super(GITInfoBuilder, self).__init__(**kwargs) 311 312 313 class GITBuilder(GITInfoBuilder): 314 def build(self): 315 try: 316 subprocess.check_output(['git', 'clone', 'git://github.com/%s/%s.git' % (self.user, self.repoName), self.buildPath]) 317 subprocess.check_output(['git', 'checkout', self.rev], cwd=self.buildPath) 318 except subprocess.CalledProcessError as e: 319 raise BuildError(e.output) 320 321 super(GITBuilder, self).build() 322 323 324 class YoutubeDLBuilder(object): 325 authorizedUsers = ['fraca7', 'phihag', 'rg3', 'FiloSottile', 'ytdl-org'] 326 327 def __init__(self, **kwargs): 328 if self.repoName != 'youtube-dl': 329 raise BuildError('Invalid repository "%s"' % self.repoName) 330 if self.user not in self.authorizedUsers: 331 raise HTTPError('Unauthorized user "%s"' % self.user, 401) 332 333 super(YoutubeDLBuilder, self).__init__(**kwargs) 334 335 def build(self): 336 try: 337 proc = subprocess.Popen([os.path.join(self.pythonPath, 'python.exe'), 'setup.py', 'py2exe'], stdin=subprocess.PIPE, cwd=self.buildPath) 338 proc.wait() 339 #subprocess.check_output([os.path.join(self.pythonPath, 'python.exe'), 'setup.py', 'py2exe'], 340 # cwd=self.buildPath) 341 except subprocess.CalledProcessError as e: 342 raise BuildError(e.output) 343 344 super(YoutubeDLBuilder, self).build() 345 346 347 class DownloadBuilder(object): 348 def __init__(self, **kwargs): 349 self.handler = kwargs.pop('handler') 350 self.srcPath = os.path.join(self.buildPath, *tuple(kwargs['path'][2:])) 351 self.srcPath = os.path.abspath(os.path.normpath(self.srcPath)) 352 if not self.srcPath.startswith(self.buildPath): 353 raise HTTPError(self.srcPath, 401) 354 355 super(DownloadBuilder, self).__init__(**kwargs) 356 357 def build(self): 358 if not os.path.exists(self.srcPath): 359 raise HTTPError('No such file', 404) 360 if os.path.isdir(self.srcPath): 361 raise HTTPError('Is a directory: %s' % self.srcPath, 401) 362 363 self.handler.send_response(200) 364 self.handler.send_header('Content-Type', 'application/octet-stream') 365 self.handler.send_header('Content-Disposition', 'attachment; filename=%s' % os.path.split(self.srcPath)[-1]) 366 self.handler.send_header('Content-Length', str(os.stat(self.srcPath).st_size)) 367 self.handler.end_headers() 368 369 with open(self.srcPath, 'rb') as src: 370 shutil.copyfileobj(src, self.handler.wfile) 371 372 super(DownloadBuilder, self).build() 373 374 375 class CleanupTempDir(object): 376 def build(self): 377 try: 378 rmtree(self.basePath) 379 except Exception as e: 380 print('WARNING deleting "%s": %s' % (self.basePath, e)) 381 382 super(CleanupTempDir, self).build() 383 384 385 class Null(object): 386 def __init__(self, **kwargs): 387 pass 388 389 def start(self): 390 pass 391 392 def close(self): 393 pass 394 395 def build(self): 396 pass 397 398 399 class Builder(PythonBuilder, GITBuilder, YoutubeDLBuilder, DownloadBuilder, CleanupTempDir, Null): 400 pass 401 402 403 class BuildHTTPRequestHandler(compat_http_server.BaseHTTPRequestHandler): 404 actionDict = {'build': Builder, 'download': Builder} # They're the same, no more caching. 405 406 def do_GET(self): 407 path = compat_urlparse.urlparse(self.path) 408 paramDict = dict([(key, value[0]) for key, value in compat_urlparse.parse_qs(path.query).items()]) 409 action, _, path = path.path.strip('/').partition('/') 410 if path: 411 path = path.split('/') 412 if action in self.actionDict: 413 try: 414 builder = self.actionDict[action](path=path, handler=self, **paramDict) 415 builder.start() 416 try: 417 builder.build() 418 finally: 419 builder.close() 420 except BuildError as e: 421 self.send_response(e.code) 422 msg = compat_str(e).encode('UTF-8') 423 self.send_header('Content-Type', 'text/plain; charset=UTF-8') 424 self.send_header('Content-Length', len(msg)) 425 self.end_headers() 426 self.wfile.write(msg) 427 else: 428 self.send_response(500, 'Unknown build method "%s"' % action) 429 else: 430 self.send_response(500, 'Malformed URL') 431 432 if __name__ == '__main__': 433 main()