From fd3fd66a1db8925e5a9a3d34acdd213f49378877 Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Mon, 30 Apr 2018 17:24:27 +0300 Subject: Add: notification service --- ick2/__init__.py | 3 ++ ick2/actions.py | 63 ++++++++++++++++++++++++++++++++- ick2/actions_tests.py | 5 ++- ick2/client.py | 10 ++++++ ick2/controllerapi.py | 3 ++ ick2/notificationapi.py | 45 +++++++++++++++++++++++ ick2/sendmail.py | 92 ++++++++++++++++++++++++++++++++++++++++++++++++ ick2/versionapi.py | 5 +++ ick2/versionapi_tests.py | 3 ++ ick2/workapi.py | 3 +- 10 files changed, 229 insertions(+), 3 deletions(-) create mode 100644 ick2/notificationapi.py create mode 100644 ick2/sendmail.py (limited to 'ick2') diff --git a/ick2/__init__.py b/ick2/__init__.py index a6a4295..fe8027c 100644 --- a/ick2/__init__.py +++ b/ick2/__init__.py @@ -71,6 +71,7 @@ from .workerapi import WorkerAPI from .controllerapi import ( ControllerAPI, ) +from .notificationapi import NotificationAPI from .blobapi import BlobAPI from .blob_store import BlobStore @@ -101,3 +102,5 @@ from .actions import ( PopulateSystreeAction, GitAction, ) + +from .sendmail import Sendmail diff --git a/ick2/actions.py b/ick2/actions.py index 202bcb6..3d77aa5 100644 --- a/ick2/actions.py +++ b/ick2/actions.py @@ -35,7 +35,9 @@ class ActionFactory: 'container': ick2.ContainerEnvironment, } - def __init__(self, systree, workspace_area, reporter): + def __init__(self, build_id, systree, workspace_area, reporter): + self._cc = None + self._build_id = build_id self._systree = systree self._workspace_area = workspace_area self._reporter = reporter @@ -46,6 +48,9 @@ class ActionFactory: def add_env_var(self, name, value): # pragma: no cover self._extra_env[name] = value + def set_controller_client(self, cc): # pragma: no cover + self._cc = cc + def set_token(self, token): self._token = token @@ -79,6 +84,8 @@ class ActionFactory: def create_action(self, spec, project_name): env = self.create_environment(spec, project_name) action = self._create_action_object(env, spec) + action.set_controller_client(self._cc) + action.set_build_id(self._build_id) action.set_token(self.get_token()) action.set_blob_url_func(self.get_blob_url_func()) return action @@ -103,6 +110,7 @@ class ActionFactory: 'git': GitAction, 'rsync': RsyncAction, 'dput': DputAction, + 'notify': NotifyAction, } kind = spec['action'] klass = rules2.get(kind) @@ -116,9 +124,23 @@ class Action: # pragma: no cover def __init__(self, env): self._env = env + self._cc = None + self._build_id = None self._token = None self._blob_url = None + def set_controller_client(self, cc): + self._cc = cc + + def get_controller_client(self): + return self._cc + + def set_build_id(self, build_id): + self._build_id = build_id + + def get_build_id(self): + return self._build_id + def set_token(self, token): self._token = token @@ -444,6 +466,45 @@ class DputAction(Action): # pragma: no cover return exit_code +class NotifyAction(Action): # pragma: no cover + + def encode_parameters(self, params): + pass + + def execute(self, params, step): + env = self.get_env() + cc = self.get_controller_client() + assert cc is not None + build_id = self.get_build_id() + + env.report(None, 'Notifying about build ending\n') + + build_path = '/builds/{}'.format(build_id) + build = cc.show(build_path) + + params = build.get('parameters', {}) + if 'notify' not in params: + env.report( + 0, + 'NOT notifying about build ending: no "notify" parameter.\n') + return + + recipients = params['notify'] + + log = cc.get_log(build_id) + log = log.decode('utf-8') + + notify = { + 'recipients': recipients, + 'build': build, + 'log': log, + } + + cc.notify(notify) + + env.report(0, 'Notified about build {} ending\n'.format(build_id)) + + def make_directory_empty(env, dirname): return env.runcmd( ['sudo', 'find', dirname, '-mindepth', '1', '-delete']) diff --git a/ick2/actions_tests.py b/ick2/actions_tests.py index 06c07f5..775428f 100644 --- a/ick2/actions_tests.py +++ b/ick2/actions_tests.py @@ -28,7 +28,9 @@ class ActionFactoryTests(unittest.TestCase): self.tempdir = tempfile.mkdtemp() self.workspace_area = self.tempdir self.project = 'magic' - self.af = ick2.ActionFactory('systree', self.workspace_area, None) + self.build_id = 'moomin/42' + self.af = ick2.ActionFactory( + self.build_id, 'systree', self.workspace_area, None) def tearDown(self): shutil.rmtree(self.tempdir) @@ -88,6 +90,7 @@ class ActionFactoryTests(unittest.TestCase): action = self.af.create_action(spec, self.project) self.assertTrue(isinstance(action, ick2.ShellAction)) self.assertTrue(isinstance(action.get_env(), ick2.HostEnvironment)) + self.assertEqual(action.get_build_id(), self.build_id) self.assertEqual(action.get_token(), token) self.assertEqual(action.get_blob_upload_url('foo'), 'foo') diff --git a/ick2/client.py b/ick2/client.py index 47941b5..e654d29 100644 --- a/ick2/client.py +++ b/ick2/client.py @@ -172,6 +172,12 @@ class ControllerClient: logging.info('Authentication URL: %r', url) return url + def get_notify_url(self): # pragma: no cover + version = self.get_version() + url = version.get('notify_url') + logging.info('Notification URL: %r', url) + return url + def get_auth_client(self): url = self.get_auth_url() ac = AuthClient() @@ -249,6 +255,10 @@ class ControllerClient: url = self.url(path) return self._api.get_blob(url) + def notify(self, notify): # pragma: no cover + url = self.get_notify_url() + self._api.post(url, body=notify) + class AuthClient: diff --git a/ick2/controllerapi.py b/ick2/controllerapi.py index c2e7eff..1bc830d 100644 --- a/ick2/controllerapi.py +++ b/ick2/controllerapi.py @@ -28,6 +28,9 @@ class ControllerAPI: def set_auth_url(self, url): # pragma: no cover self._set_url('set_auth_url', url) + def set_notify_url(self, url): # pragma: no cover + self._set_url('set_notify_url', url) + def _set_url(self, what, url): # pragma: no cover api = self._get_version_api() if api: diff --git a/ick2/notificationapi.py b/ick2/notificationapi.py new file mode 100644 index 0000000..ab261bd --- /dev/null +++ b/ick2/notificationapi.py @@ -0,0 +1,45 @@ +# Copyright 2018 Lars Wirzenius +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + + +import ick2 + + +class NotificationAPI: + + def __init__(self, config): + self._config = config + + def find_missing_route(self, missing_path): + return [ + { + 'method': 'POST', + 'path': '/notify', + 'callback': self.notify, + }, + ] + + def notify(self, content_type, body, **kwargs): + ick2.log.log('info', msg_text='Notification requested', kwargs=kwargs) + + recipients = body.get('recipients', []) + build = body.get('build', {}) + log = body.get('log', '') + + sendmail = ick2.Sendmail() + sendmail.set_config(self._config) + sendmail.send(recipients, build, log) + + return ick2.OK('') diff --git a/ick2/sendmail.py b/ick2/sendmail.py new file mode 100644 index 0000000..0b0d34e --- /dev/null +++ b/ick2/sendmail.py @@ -0,0 +1,92 @@ +# Copyright 2018 Lars Wirzenius +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + + +import email.message +import smtplib + + +body_tmpl = """\ +This is a notification from ick, the CI system. + +Project: {project} +Build ID: {build_id} +Log ID: {log} +Worker: {worker} +Status: {status} +Exit code: {exit_code} + +Build log is attached. + +""" + + +class Sendmail: + + config_fields = [ + 'from_addr', + 'smtp_server', + 'smtp_port', + 'smtp_user', + 'smtp_password', + ] + + def __init__(self): + self._config = None + + def set_config(self, config): + for field in self.config_fields: + assert field in config + self._config = config + + def send(self, recipients, build, log): + msg = self.construct_msg(recipients, build, log) + self._send_msg(recipients, msg) + + def _send_msg(self, recipients, msg): + server = smtplib.SMTP() + server.connect( + host=self._config['smtp_server'], port=self._config['smtp_port']) + server.ehlo() + server.starttls() + server.login(self._config['smtp_user'], self._config['smtp_password']) + server.sendmail(self._config['from_addr'], recipients, msg.as_bytes()) + + def construct_msg(self, recipients, build, log): + msg = email.message.Message() + msg.add_header('From', self._config['from_addr']) + msg.add_header('To', ', '.join(recipients)) + msg.add_header('Subject', self.get_subject(build)) + msg.add_header('Content-Type', 'multipart/mixed') + msg.attach(self.get_body(build)) + msg.attach(self.get_log(log)) + return msg + + def get_subject(self, build): + return 'Ick notification: {build_id} {status}'.format(**build) + + def get_body(self, build): + msg = email.message.Message() + msg.set_payload(body_tmpl.format(**build)) + msg.add_header('Content-Type', 'text/plain') + return msg + + def get_log(self, log): + msg = email.message.Message() + msg.set_payload(log) + msg.add_header('Content-Type', 'text/plain') + msg.add_header( + 'Content-Disposition', 'inline', filename='build.log') + return msg diff --git a/ick2/versionapi.py b/ick2/versionapi.py index c688c1b..cd62ce9 100644 --- a/ick2/versionapi.py +++ b/ick2/versionapi.py @@ -22,6 +22,7 @@ class VersionAPI(ick2.APIbase): super().__init__(state) self._artifact_store_url = None self._auth_url = None + self._notify_url = None def set_artifact_store_url(self, url): self._artifact_store_url = url @@ -29,6 +30,9 @@ class VersionAPI(ick2.APIbase): def set_auth_url(self, url): self._auth_url = url + def set_notify_url(self, url): + self._notify_url = url + def get_routes(self, path): # pragma: no cover return [ { @@ -44,6 +48,7 @@ class VersionAPI(ick2.APIbase): 'version': ick2.__version__, 'artifact_store': self._artifact_store_url, 'auth_url': self._auth_url, + 'notify_url': self._notify_url, } def create(self, body, **kwargs): # pragma: no cover diff --git a/ick2/versionapi_tests.py b/ick2/versionapi_tests.py index 32f9808..4f42b29 100644 --- a/ick2/versionapi_tests.py +++ b/ick2/versionapi_tests.py @@ -24,9 +24,11 @@ class VersionAPITests(unittest.TestCase): def test_returns_version_correcly(self): bloburl = 'https://blobs.example.com' idpurl = 'https://idp.example.com' + notifyurl = 'https://notify.example.com' api = ick2.VersionAPI(None) api.set_artifact_store_url(bloburl) api.set_auth_url(idpurl) + api.set_notify_url(notifyurl) response = api.get_version() self.assertEqual( response, @@ -34,5 +36,6 @@ class VersionAPITests(unittest.TestCase): 'version': ick2.__version__, 'artifact_store': bloburl, 'auth_url': idpurl, + 'notify_url': notifyurl, } ) diff --git a/ick2/workapi.py b/ick2/workapi.py index 01f9932..0c885b5 100644 --- a/ick2/workapi.py +++ b/ick2/workapi.py @@ -192,12 +192,13 @@ class WorkAPI(ick2.APIbase): name, doing.get(name), update[name])) def _append_to_build_log(self, log, update): + ick2.log.log('trace', msg_text='appending to build log', update=update) for stream in ['stdout', 'stderr']: text = update.get(stream, '') self._append_text_to_build_log(log, text) def _append_text_to_build_log(self, log, text): - log['log'] = log.get('log', '') + text + log['log'] = log.get('log', '') + (text or '') def create(self, body, **kwargs): # pragma: no cover pass -- cgit v1.2.1