Move LineLogger, Debian CI upload to modules

4 files changed, 136 insertions, 92 deletions

diff --git a/icklib/__init__.py b/icklib/__init__.py
index 88dbe96..c6f6540 100644
--- a/icklib/__init__.py
+++ b/icklib/__init__.py
@@ -18,6 +18,7 @@
 
 from .version import __version__, __version_info__
 from .logger import Logger
+from .linelogger import LineLogger
 from .target import Target, create_targets_from_ick
 from .build_step import BuildStep
 from .run_state import RunState
@@ -31,6 +32,7 @@ from .step_debian_binary import (
     CreateDebianBinaryPackagesForRelease,
 )
 from .step_debian_changes import FindDebianChangesFiles
+from .step_debian_ci_upload import UploadDebianPackagesToCIRepo
 from .step_debian_info import CollectDebianInfoAboutTargets
 from .step_debian_source import (
     CreateDebianSourcePackagesForCI,
diff --git a/icklib/linelogger.py b/icklib/linelogger.py
new file mode 100644
index 0000000..4e53f50
--- /dev/null
+++ b/icklib/linelogger.py
@@ -0,0 +1,60 @@
+# Copyright 2016  Lars Wirzenius
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# =*= License: GPL-3+ =*=
+
+
+import logging
+
+
+class LineLogger(object):  # pragma: no cover
+
+    def __init__(self, progress, logger):
+        self._progress = progress
+        self._progress_lines = []
+        self._logger = logger
+        self._buffer = ''
+        self._lineno = 0
+
+    def log_lines(self, text):
+        self._buffer += text
+        while '\n' in self._buffer:
+            line, self._buffer = self._buffer.split('\n', 1)
+            self._log_line(line)
+
+        # Return empty string so that cliapp doesn't collect
+        # potentially large amounts of text pointlessly.
+        return ''
+
+    def _log_line(self, line):
+        self._lineno += 1
+        line = '{lineno:>04}: {line}'.format(lineno=self._lineno, line=line)
+        logging.debug('STDOUT: %s', line)
+        self._logger.chatty('{line}', line=line)
+
+        lines = self._progress_lines
+        max_lines = self._progress.max_output_lines
+
+        lines.append(line)
+        if len(lines) > max_lines:
+            del lines[:-max_lines]
+        for i, line in enumerate(lines):
+            self._progress['line%d' % i] = line
+        for i in range(len(lines), max_lines + 1):
+            self._progress['line%d' % i] = ''
+
+    def log_rest(self):
+        if self._buffer:
+            self._log_line(self._buffer)
diff --git a/icklib/project.py b/icklib/project.py
index 66fdf90..4331764 100644
--- a/icklib/project.py
+++ b/icklib/project.py
@@ -17,16 +17,12 @@
 
 
 import glob
-import logging
 import os
-import re
 import shutil
 import StringIO
 import subprocess
-import sys
 import tempfile
 import time
-import urllib
 
 import cliapp
 
@@ -134,7 +130,7 @@ class Project(object):
 
         whole_argv = self.create_argv_with_env() + argv
 
-        line_logger = LineLogger(self.progress, self.run_state.logger)
+        line_logger = icklib.LineLogger(self.progress, self.run_state.logger)
         with self.run_state.logger:
             self.run_state.logger.chatty('+ %s' % ' '.join(whole_argv))
             cliapp.runcmd(
@@ -173,7 +169,7 @@ class Project(object):
             env_argv.append('%s=%s' % (name, value))
         whole_argv = self.create_argv_with_env() + argv
 
-        line_logger = LineLogger(self.progress, self.run_state.logger)
+        line_logger = icklib.LineLogger(self.progress, self.run_state.logger)
         with self.run_state.logger:
             self.run_state.logger.chatty('+ %s' % ' '.join(whole_argv))
             target.ssh_runcmd(
@@ -222,49 +218,6 @@ class Project(object):
             remote_git_dir)
 
 
-class UploadDebianPackagesToCIRepo(icklib.BuildStep):
-
-    def build(self):
-        if not self.run_state.changes_files:
-            return
-
-        self.run_state.progress['step'] = (
-            'Upload built packages to APT repository')
-        self.run_state.logger.important(
-            'Upload built packages to APT repository')
-        with self.run_state.logger:
-            repopath = self.statedir.get_apt_directory()
-            incoming = os.path.join(repopath, 'incoming')
-            line_logger = LineLogger(
-                self.run_state.progress, self.run_state.logger)
-            for changes_url in self.run_state.changes_files:
-                # dget doesn't URL-decode file:// URLs. So we do that.
-                changes_url = urllib.unquote(changes_url)
-                cliapp.runcmd(
-                    ['dget', '-du', changes_url],
-                    cwd=incoming,
-                    stderr=subprocess.STDOUT,
-                    stdout_callback=line_logger.log_lines)
-
-            exit_code, stdout, stderr = cliapp.runcmd_unchecked(
-                ['reprepro', 'processincoming', 'default'],
-                cwd=repopath)
-            line_logger.log_lines(stdout)
-            line_logger.log_lines(stderr)
-            if exit_code != 0:
-                pattern = r"Name '.*-dbgsym' of binary '.*' is not listed"
-                if re.search(pattern, stderr):
-                    self.run_state.logger.chatty(
-                        'Ignoring reprepro problem wrt -dbgsym')
-                    cliapp.runcmd(
-                        ['reprepro', 'export'],
-                        cwd=repopath,
-                        stdout_callback=line_logger.log_lines,
-                        stderr=subprocess.STDOUT)
-                else:
-                    sys.exit(exit_code)
-
-
 class PublishDebianPackages(icklib.BuildStep):
 
     def build(self):
@@ -318,47 +271,6 @@ class PublishDebianPackages(icklib.BuildStep):
                 tempdir)
 
 
-class LineLogger(object):  # pragma: no cover
-
-    def __init__(self, progress, logger):
-        self._progress = progress
-        self._progress_lines = []
-        self._logger = logger
-        self._buffer = ''
-        self._lineno = 0
-
-    def log_lines(self, text):
-        self._buffer += text
-        while '\n' in self._buffer:
-            line, self._buffer = self._buffer.split('\n', 1)
-            self._log_line(line)
-
-        # Return empty string so that cliapp doesn't collect
-        # potentially large amounts of text pointlessly.
-        return ''
-
-    def _log_line(self, line):
-        self._lineno += 1
-        line = '{lineno:>04}: {line}'.format(lineno=self._lineno, line=line)
-        logging.debug('STDOUT: %s', line)
-        self._logger.chatty('{line}', line=line)
-
-        lines = self._progress_lines
-        max_lines = self._progress.max_output_lines
-
-        lines.append(line)
-        if len(lines) > max_lines:
-            del lines[:-max_lines]
-        for i, line in enumerate(lines):
-            self._progress['line%d' % i] = line
-        for i in range(len(lines), max_lines + 1):
-            self._progress['line%d' % i] = ''
-
-    def log_rest(self):
-        if self._buffer:
-            self._log_line(self._buffer)
-
-
 def create_projects_from_ick(ick, wanted_names):
     pipelines = {
         'shell': [
@@ -393,7 +305,7 @@ def create_projects_from_ick(ick, wanted_names):
             icklib.CreateCIDebianBinaryPackages,
             icklib.FindDebianChangesFiles,
             icklib.SetupAPTRepository,
-            UploadDebianPackagesToCIRepo,
+            icklib.UploadDebianPackagesToCIRepo,
             icklib.FinishBuildInfo,
             icklib.SaveProjectInfo,
         ],
@@ -409,7 +321,7 @@ def create_projects_from_ick(ick, wanted_names):
             icklib.CreateDebianBinaryPackagesForRelease,
             icklib.FindDebianChangesFiles,
             icklib.SetupAPTRepository,
-            UploadDebianPackagesToCIRepo,
+            icklib.UploadDebianPackagesToCIRepo,
             PublishDebianPackages,
             icklib.FinishBuildInfo,
             icklib.SaveProjectInfo,
diff --git a/icklib/step_debian_ci_upload.py b/icklib/step_debian_ci_upload.py
new file mode 100644
index 0000000..193687b
--- /dev/null
+++ b/icklib/step_debian_ci_upload.py
@@ -0,0 +1,70 @@
+# Copyright 2016  Lars Wirzenius
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# =*= License: GPL-3+ =*=
+
+
+import os
+import re
+import subprocess
+import sys
+import urllib
+
+import cliapp
+
+import icklib
+
+
+class UploadDebianPackagesToCIRepo(icklib.BuildStep):
+
+    def build(self):
+        if not self.run_state.changes_files:
+            return
+
+        self.run_state.progress['step'] = (
+            'Upload built packages to APT repository')
+        self.run_state.logger.important(
+            'Upload built packages to APT repository')
+        with self.run_state.logger:
+            repopath = self.statedir.get_apt_directory()
+            incoming = os.path.join(repopath, 'incoming')
+            line_logger = icklib.LineLogger(
+                self.run_state.progress, self.run_state.logger)
+            for changes_url in self.run_state.changes_files:
+                # dget doesn't URL-decode file:// URLs. So we do that.
+                changes_url = urllib.unquote(changes_url)
+                cliapp.runcmd(
+                    ['dget', '-du', changes_url],
+                    cwd=incoming,
+                    stderr=subprocess.STDOUT,
+                    stdout_callback=line_logger.log_lines)
+
+            exit_code, stdout, stderr = cliapp.runcmd_unchecked(
+                ['reprepro', 'processincoming', 'default'],
+                cwd=repopath)
+            line_logger.log_lines(stdout)
+            line_logger.log_lines(stderr)
+            if exit_code != 0:
+                pattern = r"Name '.*-dbgsym' of binary '.*' is not listed"
+                if re.search(pattern, stderr):
+                    self.run_state.logger.chatty(
+                        'Ignoring reprepro problem wrt -dbgsym')
+                    cliapp.runcmd(
+                        ['reprepro', 'export'],
+                        cwd=repopath,
+                        stdout_callback=line_logger.log_lines,
+                        stderr=subprocess.STDOUT)
+                else:
+                    sys.exit(exit_code)