From 4257cef04cad5af0f2b779ff6a380dca079601f5 Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Sun, 21 Jan 2018 19:20:24 +0200 Subject: Change: download and unpack as a pipeline --- worker_manager | 33 ++++++++++++++++----------------- 1 file changed, 16 insertions(+), 17 deletions(-) (limited to 'worker_manager') diff --git a/worker_manager b/worker_manager index 4cb0635..a9d61a5 100755 --- a/worker_manager +++ b/worker_manager @@ -515,30 +515,29 @@ class SystreePopulator(WorkerBase): return 1 self.make_directory_empty(self.systree_dir) - self.report(b'Downloading systree blob\n') - tarball = self._api.download_blob(systree_name) - if tarball is None: - self.report(b'Download failed\n') - return 1 - - self.report(b'Unpacking systree blob\n') - self.unpack_systree(tarball, self.systree_dir) - self.report(b'Systree has been populated\n') - return 0 + self.report(b'Downloading and unpacking systree blob\n') + return self.download_and_unpack_systree(systree_name, self.systree_dir) def make_directory_empty(self, dirname): return self.execute_argv(['sudo', 'find', '-delete'], cwd=dirname) - def unpack_systree(self, tarball, dirname): - return self.execute_argv( - ['sudo', 'tar', '-zxf', '-', '-C', dirname], - feed_stdin=tarball, - ) + def download_and_unpack_systree(self, systree_name, dirname): + url = self._api.url('/blobs/{}'.format(systree_name)) + headers = self._api.get_auth_headers() + download = ['curl', '-k'] + for name, value in headers.items(): + header = '{}: {}'.format(name, value) + download.extend(['-H', header]) + download.append(url) - def execute_argv(self, argv, **kwargs): + unpack = ['sudo', 'tar', '-zxf', '-', '-C', dirname] + + return self.execute_argv(download, unpack) + + def execute_argv(self, *argvs, **kwargs): exit_code, _, _ = cliapp.runcmd_unchecked( - argv, + *argvs, stdout_callback=self.report, stderr_callback=self.report, **kwargs, -- cgit v1.2.1