diff options
author | Lars Wirzenius <liw@liw.fi> | 2016-03-06 22:47:25 +0200 |
---|---|---|
committer | Lars Wirzenius <liw@liw.fi> | 2016-03-06 22:47:25 +0200 |
commit | c682869e6d3f0faa8eb96b6e4d08774840bb988c (patch) | |
tree | 7035607601801f31094f7340b3684bf9d8a3f506 | |
parent | 85e1778a83b6dbc1557c9dded363f6f43531b80e (diff) | |
download | ick-c682869e6d3f0faa8eb96b6e4d08774840bb988c.tar.gz |
Move LineLogger, Debian CI upload to modules
-rw-r--r-- | icklib/__init__.py | 2 | ||||
-rw-r--r-- | icklib/linelogger.py | 60 | ||||
-rw-r--r-- | icklib/project.py | 96 | ||||
-rw-r--r-- | icklib/step_debian_ci_upload.py | 70 | ||||
-rw-r--r-- | without-tests | 2 |
5 files changed, 138 insertions, 92 deletions
diff --git a/icklib/__init__.py b/icklib/__init__.py index 88dbe96..c6f6540 100644 --- a/icklib/__init__.py +++ b/icklib/__init__.py @@ -18,6 +18,7 @@ from .version import __version__, __version_info__ from .logger import Logger +from .linelogger import LineLogger from .target import Target, create_targets_from_ick from .build_step import BuildStep from .run_state import RunState @@ -31,6 +32,7 @@ from .step_debian_binary import ( CreateDebianBinaryPackagesForRelease, ) from .step_debian_changes import FindDebianChangesFiles +from .step_debian_ci_upload import UploadDebianPackagesToCIRepo from .step_debian_info import CollectDebianInfoAboutTargets from .step_debian_source import ( CreateDebianSourcePackagesForCI, diff --git a/icklib/linelogger.py b/icklib/linelogger.py new file mode 100644 index 0000000..4e53f50 --- /dev/null +++ b/icklib/linelogger.py @@ -0,0 +1,60 @@ +# Copyright 2016 Lars Wirzenius +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +# +# =*= License: GPL-3+ =*= + + +import logging + + +class LineLogger(object): # pragma: no cover + + def __init__(self, progress, logger): + self._progress = progress + self._progress_lines = [] + self._logger = logger + self._buffer = '' + self._lineno = 0 + + def log_lines(self, text): + self._buffer += text + while '\n' in self._buffer: + line, self._buffer = self._buffer.split('\n', 1) + self._log_line(line) + + # Return empty string so that cliapp doesn't collect + # potentially large amounts of text pointlessly. + return '' + + def _log_line(self, line): + self._lineno += 1 + line = '{lineno:>04}: {line}'.format(lineno=self._lineno, line=line) + logging.debug('STDOUT: %s', line) + self._logger.chatty('{line}', line=line) + + lines = self._progress_lines + max_lines = self._progress.max_output_lines + + lines.append(line) + if len(lines) > max_lines: + del lines[:-max_lines] + for i, line in enumerate(lines): + self._progress['line%d' % i] = line + for i in range(len(lines), max_lines + 1): + self._progress['line%d' % i] = '' + + def log_rest(self): + if self._buffer: + self._log_line(self._buffer) diff --git a/icklib/project.py b/icklib/project.py index 66fdf90..4331764 100644 --- a/icklib/project.py +++ b/icklib/project.py @@ -17,16 +17,12 @@ import glob -import logging import os -import re import shutil import StringIO import subprocess -import sys import tempfile import time -import urllib import cliapp @@ -134,7 +130,7 @@ class Project(object): whole_argv = self.create_argv_with_env() + argv - line_logger = LineLogger(self.progress, self.run_state.logger) + line_logger = icklib.LineLogger(self.progress, self.run_state.logger) with self.run_state.logger: self.run_state.logger.chatty('+ %s' % ' '.join(whole_argv)) cliapp.runcmd( @@ -173,7 +169,7 @@ class Project(object): env_argv.append('%s=%s' % (name, value)) whole_argv = self.create_argv_with_env() + argv - line_logger = LineLogger(self.progress, self.run_state.logger) + line_logger = icklib.LineLogger(self.progress, self.run_state.logger) with self.run_state.logger: self.run_state.logger.chatty('+ %s' % ' '.join(whole_argv)) target.ssh_runcmd( @@ -222,49 +218,6 @@ class Project(object): remote_git_dir) -class UploadDebianPackagesToCIRepo(icklib.BuildStep): - - def build(self): - if not self.run_state.changes_files: - return - - self.run_state.progress['step'] = ( - 'Upload built packages to APT repository') - self.run_state.logger.important( - 'Upload built packages to APT repository') - with self.run_state.logger: - repopath = self.statedir.get_apt_directory() - incoming = os.path.join(repopath, 'incoming') - line_logger = LineLogger( - self.run_state.progress, self.run_state.logger) - for changes_url in self.run_state.changes_files: - # dget doesn't URL-decode file:// URLs. So we do that. - changes_url = urllib.unquote(changes_url) - cliapp.runcmd( - ['dget', '-du', changes_url], - cwd=incoming, - stderr=subprocess.STDOUT, - stdout_callback=line_logger.log_lines) - - exit_code, stdout, stderr = cliapp.runcmd_unchecked( - ['reprepro', 'processincoming', 'default'], - cwd=repopath) - line_logger.log_lines(stdout) - line_logger.log_lines(stderr) - if exit_code != 0: - pattern = r"Name '.*-dbgsym' of binary '.*' is not listed" - if re.search(pattern, stderr): - self.run_state.logger.chatty( - 'Ignoring reprepro problem wrt -dbgsym') - cliapp.runcmd( - ['reprepro', 'export'], - cwd=repopath, - stdout_callback=line_logger.log_lines, - stderr=subprocess.STDOUT) - else: - sys.exit(exit_code) - - class PublishDebianPackages(icklib.BuildStep): def build(self): @@ -318,47 +271,6 @@ class PublishDebianPackages(icklib.BuildStep): tempdir) -class LineLogger(object): # pragma: no cover - - def __init__(self, progress, logger): - self._progress = progress - self._progress_lines = [] - self._logger = logger - self._buffer = '' - self._lineno = 0 - - def log_lines(self, text): - self._buffer += text - while '\n' in self._buffer: - line, self._buffer = self._buffer.split('\n', 1) - self._log_line(line) - - # Return empty string so that cliapp doesn't collect - # potentially large amounts of text pointlessly. - return '' - - def _log_line(self, line): - self._lineno += 1 - line = '{lineno:>04}: {line}'.format(lineno=self._lineno, line=line) - logging.debug('STDOUT: %s', line) - self._logger.chatty('{line}', line=line) - - lines = self._progress_lines - max_lines = self._progress.max_output_lines - - lines.append(line) - if len(lines) > max_lines: - del lines[:-max_lines] - for i, line in enumerate(lines): - self._progress['line%d' % i] = line - for i in range(len(lines), max_lines + 1): - self._progress['line%d' % i] = '' - - def log_rest(self): - if self._buffer: - self._log_line(self._buffer) - - def create_projects_from_ick(ick, wanted_names): pipelines = { 'shell': [ @@ -393,7 +305,7 @@ def create_projects_from_ick(ick, wanted_names): icklib.CreateCIDebianBinaryPackages, icklib.FindDebianChangesFiles, icklib.SetupAPTRepository, - UploadDebianPackagesToCIRepo, + icklib.UploadDebianPackagesToCIRepo, icklib.FinishBuildInfo, icklib.SaveProjectInfo, ], @@ -409,7 +321,7 @@ def create_projects_from_ick(ick, wanted_names): icklib.CreateDebianBinaryPackagesForRelease, icklib.FindDebianChangesFiles, icklib.SetupAPTRepository, - UploadDebianPackagesToCIRepo, + icklib.UploadDebianPackagesToCIRepo, PublishDebianPackages, icklib.FinishBuildInfo, icklib.SaveProjectInfo, diff --git a/icklib/step_debian_ci_upload.py b/icklib/step_debian_ci_upload.py new file mode 100644 index 0000000..193687b --- /dev/null +++ b/icklib/step_debian_ci_upload.py @@ -0,0 +1,70 @@ +# Copyright 2016 Lars Wirzenius +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +# +# =*= License: GPL-3+ =*= + + +import os +import re +import subprocess +import sys +import urllib + +import cliapp + +import icklib + + +class UploadDebianPackagesToCIRepo(icklib.BuildStep): + + def build(self): + if not self.run_state.changes_files: + return + + self.run_state.progress['step'] = ( + 'Upload built packages to APT repository') + self.run_state.logger.important( + 'Upload built packages to APT repository') + with self.run_state.logger: + repopath = self.statedir.get_apt_directory() + incoming = os.path.join(repopath, 'incoming') + line_logger = icklib.LineLogger( + self.run_state.progress, self.run_state.logger) + for changes_url in self.run_state.changes_files: + # dget doesn't URL-decode file:// URLs. So we do that. + changes_url = urllib.unquote(changes_url) + cliapp.runcmd( + ['dget', '-du', changes_url], + cwd=incoming, + stderr=subprocess.STDOUT, + stdout_callback=line_logger.log_lines) + + exit_code, stdout, stderr = cliapp.runcmd_unchecked( + ['reprepro', 'processincoming', 'default'], + cwd=repopath) + line_logger.log_lines(stdout) + line_logger.log_lines(stderr) + if exit_code != 0: + pattern = r"Name '.*-dbgsym' of binary '.*' is not listed" + if re.search(pattern, stderr): + self.run_state.logger.chatty( + 'Ignoring reprepro problem wrt -dbgsym') + cliapp.runcmd( + ['reprepro', 'export'], + cwd=repopath, + stdout_callback=line_logger.log_lines, + stderr=subprocess.STDOUT) + else: + sys.exit(exit_code) diff --git a/without-tests b/without-tests index 32a4a2b..ac75b5c 100644 --- a/without-tests +++ b/without-tests @@ -4,11 +4,13 @@ icklib/project.py icklib/info.py icklib/version.py icklib/progress.py +icklib/linelogger.py icklib/step_build_info.py icklib/step_create_tarball.py icklib/step_debian_apt_setup.py icklib/step_debian_binary.py icklib/step_debian_changes.py +icklib/step_debian_ci_upload.py icklib/step_debian_info.py icklib/step_debian_source.py icklib/step_find_current_commit.py |