From d91d165e7afa8e93bb773414e821748c1fe61699 Mon Sep 17 00:00:00 2001 From: Distix listener Date: Sun, 19 Jun 2016 19:30:06 +0200 Subject: imported mails --- .../Maildir/new/1466357405.M965256P29655Q1.hrun | 104 +++++++++ .../Maildir/new/1466357406.M372488P29655Q2.hrun | 259 +++++++++++++++++++++ 2 files changed, 363 insertions(+) create mode 100644 tickets/3092ee8cdf8e49fda9937a228d0546fd/Maildir/new/1466357405.M965256P29655Q1.hrun create mode 100644 tickets/3092ee8cdf8e49fda9937a228d0546fd/Maildir/new/1466357406.M372488P29655Q2.hrun (limited to 'tickets/3092ee8cdf8e49fda9937a228d0546fd') diff --git a/tickets/3092ee8cdf8e49fda9937a228d0546fd/Maildir/new/1466357405.M965256P29655Q1.hrun b/tickets/3092ee8cdf8e49fda9937a228d0546fd/Maildir/new/1466357405.M965256P29655Q1.hrun new file mode 100644 index 0000000..e185573 --- /dev/null +++ b/tickets/3092ee8cdf8e49fda9937a228d0546fd/Maildir/new/1466357405.M965256P29655Q1.hrun @@ -0,0 +1,104 @@ +Return-Path: +X-Original-To: distix@pieni.net +Delivered-To: distix@pieni.net +Received: from bagpuss.pepperfish.net (bagpuss.pepperfish.net [148.251.8.16]) + (using TLSv1.2 with cipher AECDH-AES256-SHA (256/256 bits)) + (No client certificate requested) + by pieni.net (Postfix) with ESMTPS id 47BD8248C7 + for ; Sun, 19 Jun 2016 19:27:43 +0200 (CEST) +Received: from platypus.pepperfish.net (unknown [10.112.100.20]) + by bagpuss.pepperfish.net (Postfix) with ESMTP id 8A8CD59E; + Sun, 19 Jun 2016 18:27:42 +0100 (BST) +Received: from ip6-localhost ([::1] helo=platypus.pepperfish.net) + by platypus.pepperfish.net with esmtp (Exim 4.80 #2 (Debian)) + id 1bEgVa-00074o-E2; Sun, 19 Jun 2016 18:27:42 +0100 +Received: from inmail0 ([10.112.100.10] helo=mx0.pepperfish.net) + by platypus.pepperfish.net with esmtp (Exim 4.80 #2 (Debian)) + id 1bEgVY-00074a-2x + for ; Sun, 19 Jun 2016 18:27:40 +0100 +Received: from smtp.gentoo.org ([140.211.166.183]) + by mx0.pepperfish.net with esmtps (TLS1.2:DHE_RSA_AES_256_CBC_SHA256:256) + (Exim 4.80) (envelope-from ) id 1bEgVV-0004Te-PY + for obnam-dev@obnam.org; Sun, 19 Jun 2016 18:27:39 +0100 +Received: from grubbs.orbis-terrarum.net (localhost [127.0.0.1]) + (using TLSv1.2 with cipher ECDHE-RSA-AES256-SHA (256/256 bits)) + (No client certificate requested) + by smtp.gentoo.org (Postfix) with ESMTPS id 5D489340815 + for ; Sun, 19 Jun 2016 17:27:22 +0000 (UTC) +Received: (qmail 5206 invoked by uid 129); 19 Jun 2016 17:27:22 -0000 +X-HELO: bohr-int.orbis-terrarum.net +Authentication-Results: orbis-terrarum.net; auth=pass (plain) + smtp.auth=robbat2-bohr@orbis-terrarum.net +Received: from Unknown (HELO bohr-int.orbis-terrarum.net) (2001:470:e889:1::8) + by orbis-terrarum.net (qpsmtpd/0.95) with ESMTPSA + (ECDHE-RSA-AES256-GCM-SHA384 encrypted); Sun, 19 Jun 2016 17:27:22 +0000 +Received: (nullmailer pid 8212 invoked by uid 10000); + Sun, 19 Jun 2016 17:27:19 -0000 +From: "Robin H. Johnson" +To: obnam-dev@obnam.org +Date: Sun, 19 Jun 2016 10:27:16 -0700 +Message-Id: <20160619172717.18445-1-robbat2@gentoo.org> +X-Mailer: git-send-email 2.9.0 +In-Reply-To: +References: +X-Virus-Checked: Checked by ClamAV on orbis-terrarum.net +X-Spam-Score: -8.3 +X-Spam-Score-int: -82 +X-Spam-Bar: -------- +X-Scanned-By: pepperfish.net, Sun, 19 Jun 2016 18:27:39 +0100 +X-Spam-Report: Content analysis details: (-8.3 points) + pts rule name description + ---- ---------------------- -------------------------------------------------- + -5.0 RCVD_IN_DNSWL_HI RBL: Sender listed at http://www.dnswl.org/, high + trust [140.211.166.183 listed in list.dnswl.org] + -0.0 SPF_PASS SPF: sender matches SPF record + -1.4 RP_MATCHES_RCVD Envelope sender domain matches handover relay domain + -1.9 BAYES_00 BODY: Bayes spam probability is 0 to 1% + [score: 0.0000] +X-ACL-Warn: message may be spam +X-Scan-Signature: e22476795c0c271ab9cf3b1ad5ae1506 +Subject: [PATCH 0/1] encryption: boost GPG performance. +X-BeenThere: obnam-dev@obnam.org +X-Mailman-Version: 2.1.5 +Precedence: list +List-Id: Obnam development discussions +List-Unsubscribe: , + +List-Archive: +List-Post: +List-Help: +List-Subscribe: , + +Sender: obnam-dev-bounces@obnam.org +Errors-To: obnam-dev-bounces@obnam.org + +The following patch (next in the thread), significently boosts the performance +of the encryption plugin by tuning the GPG symmetric parameters. + +See the previous thread of 'GPG & performance' for the nuanced details about +why this works. + +Running 'production.yaml' benchmarks using obnam-bench. This does use the +green-albatross repo format, but the patch is generic, and improves all +symmetric GPG usage. + +Time in seconds. + + many_files one_big_file +Unencrypted 225.4 10.2 +GPG(master) 284.8 272.5 +GPG(patch) 251.3 47.8 + +Yes, that's correct, for the big file case, it's 5.7x faster. + +There is one case where this patch will have a NEGATIVE impact: +Usage of compress-with=None along with highly compressible input. + +This is because GPG was previously compressing the data before encryption, and +faster specifically because it had less data to compress than with this patch. + + +_______________________________________________ +obnam-dev mailing list +obnam-dev@obnam.org +http://listmaster.pepperfish.net/cgi-bin/mailman/listinfo/obnam-dev-obnam.org diff --git a/tickets/3092ee8cdf8e49fda9937a228d0546fd/Maildir/new/1466357406.M372488P29655Q2.hrun b/tickets/3092ee8cdf8e49fda9937a228d0546fd/Maildir/new/1466357406.M372488P29655Q2.hrun new file mode 100644 index 0000000..cabd1f5 --- /dev/null +++ b/tickets/3092ee8cdf8e49fda9937a228d0546fd/Maildir/new/1466357406.M372488P29655Q2.hrun @@ -0,0 +1,259 @@ +Return-Path: +X-Original-To: distix@pieni.net +Delivered-To: distix@pieni.net +Received: from bagpuss.pepperfish.net (bagpuss.pepperfish.net [148.251.8.16]) + (using TLSv1.2 with cipher AECDH-AES256-SHA (256/256 bits)) + (No client certificate requested) + by pieni.net (Postfix) with ESMTPS id B3F4A20F8A + for ; Sun, 19 Jun 2016 19:27:43 +0200 (CEST) +Received: from platypus.pepperfish.net (unknown [10.112.100.20]) + by bagpuss.pepperfish.net (Postfix) with ESMTP id 3A7775BF; + Sun, 19 Jun 2016 18:27:43 +0100 (BST) +Received: from ip6-localhost ([::1] helo=platypus.pepperfish.net) + by platypus.pepperfish.net with esmtp (Exim 4.80 #2 (Debian)) + id 1bEgVb-00074z-0g; Sun, 19 Jun 2016 18:27:43 +0100 +Received: from inmail0 ([10.112.100.10] helo=mx0.pepperfish.net) + by platypus.pepperfish.net with esmtp (Exim 4.80 #2 (Debian)) + id 1bEgVY-00074i-EY + for ; Sun, 19 Jun 2016 18:27:40 +0100 +Received: from smtp.gentoo.org ([140.211.166.183]) + by mx0.pepperfish.net with esmtps (TLS1.2:DHE_RSA_AES_256_CBC_SHA256:256) + (Exim 4.80) (envelope-from ) id 1bEgVW-0004Tf-0E + for obnam-dev@obnam.org; Sun, 19 Jun 2016 18:27:40 +0100 +Received: from grubbs.orbis-terrarum.net (localhost [127.0.0.1]) + (using TLSv1.2 with cipher ECDHE-RSA-AES256-SHA (256/256 bits)) + (No client certificate requested) + by smtp.gentoo.org (Postfix) with ESMTPS id 0755D340831 + for ; Sun, 19 Jun 2016 17:27:25 +0000 (UTC) +Received: (qmail 5243 invoked by uid 129); 19 Jun 2016 17:27:25 -0000 +X-HELO: bohr-int.orbis-terrarum.net +Authentication-Results: orbis-terrarum.net; auth=pass (plain) + smtp.auth=robbat2-bohr@orbis-terrarum.net +Received: from Unknown (HELO bohr-int.orbis-terrarum.net) (2001:470:e889:1::8) + by orbis-terrarum.net (qpsmtpd/0.95) with ESMTPSA + (ECDHE-RSA-AES256-GCM-SHA384 encrypted); Sun, 19 Jun 2016 17:27:25 +0000 +Received: (nullmailer pid 9310 invoked by uid 10000); + Sun, 19 Jun 2016 17:27:22 -0000 +From: "Robin H. Johnson" +To: obnam-dev@obnam.org +Date: Sun, 19 Jun 2016 10:27:17 -0700 +Message-Id: <20160619172717.18445-2-robbat2@gentoo.org> +X-Mailer: git-send-email 2.9.0 +In-Reply-To: <20160619172717.18445-1-robbat2@gentoo.org> +References: + <20160619172717.18445-1-robbat2@gentoo.org> +X-Virus-Checked: Checked by ClamAV on orbis-terrarum.net +X-Spam-Score: -8.2 +X-Spam-Score-int: -81 +X-Spam-Bar: -------- +X-Scanned-By: pepperfish.net, Sun, 19 Jun 2016 18:27:40 +0100 +X-Spam-Report: Content analysis details: (-8.2 points) + pts rule name description + ---- ---------------------- -------------------------------------------------- + -5.0 RCVD_IN_DNSWL_HI RBL: Sender listed at http://www.dnswl.org/, high + trust [140.211.166.183 listed in list.dnswl.org] + -0.0 SPF_PASS SPF: sender matches SPF record + -1.4 RP_MATCHES_RCVD Envelope sender domain matches handover relay domain + -1.9 BAYES_00 BODY: Bayes spam probability is 0 to 1% + [score: 0.0000] + 0.1 PPF_SPLIT_TAG RAW: Body contains a split HTML tag +X-ACL-Warn: message may be spam +X-Scan-Signature: ebfe5e7f0df0b84657034def484e72a4 +Cc: "Robin H. Johnson" +Subject: [PATCH] encryption: boost GPG performance. +X-BeenThere: obnam-dev@obnam.org +X-Mailman-Version: 2.1.5 +Precedence: list +List-Id: Obnam development discussions +List-Unsubscribe: , + +List-Archive: +List-Post: +List-Help: +List-Subscribe: , + +Sender: obnam-dev-bounces@obnam.org +Errors-To: obnam-dev-bounces@obnam.org + +Boost GPG performance: +- disabling compression during symmetric encryption. +- tuning symmetric key handling. + +Also adds configuration options symmetric-cipher and symmetric-digest +for tuning GPG behavior. + +Signed-off-by: Robin H. Johnson +--- + obnamlib/encryption.py | 38 +++++++++++++++++++++++++---- + obnamlib/plugins/encryption_plugin.py | 44 ++++++++++++++++++++++++++++++---- + test-gpghome/random_seed | Bin 600 -> 600 bytes + 3 files changed, 72 insertions(+), 10 deletions(-) + +diff --git a/obnamlib/encryption.py b/obnamlib/encryption.py +index d2c78d0..d79f21f 100644 +--- a/obnamlib/encryption.py ++++ b/obnamlib/encryption.py +@@ -102,13 +102,41 @@ def _gpg_pipe(args, data, passphrase, gpghome=None): + return out + + +-def encrypt_symmetric(cleartext, key, gpghome=None): ++def encrypt_symmetric( ++ cleartext, key, ++ gpghome=None, ++ cipher=None, # pylint: disable=unused-argument ++ digest=None): # pylint: disable=unused-argument + '''Encrypt data with symmetric encryption.''' +- return _gpg_pipe(['-c'], cleartext, key, gpghome=gpghome) +- +- +-def decrypt_symmetric(encrypted, key, gpghome=None): ++ opts = [ ++ # Perform symmetric encryption ++ '-c', ++ # Disable compression as our data will be pre-compressed. ++ '--compress-algo', 'none', ++ # Key stretching is generally required as keys are raw random data. ++ # But, in case other parts of obnam reuse this key, salting it provides ++ # more variation than iteration, as on a single system the iterative ++ # count will generally remain consistent, as well as being being ++ # faster. ++ '--s2k-mode', '1', ++ ] ++ if cipher: ++ opts += ['--s2k-cipher-algo', cipher] ++ if digest: ++ opts += ['--s2k-digest-algo', digest] ++ ++ return _gpg_pipe(opts, cleartext, key, gpghome=gpghome) ++ ++ ++def decrypt_symmetric( ++ encrypted, key, ++ gpghome=None, ++ cipher=None, # pylint: disable=unused-argument ++ digest=None): # pylint: disable=unused-argument + '''Decrypt encrypted data with symmetric encryption.''' ++ # cipher and digest are unused with GPG, as the values used to encrypt the ++ # data are stored in the S2K packet data. ++ # the parameters are here for future interface symmetry. + return _gpg_pipe(['-d'], encrypted, key, gpghome=gpghome) + + +diff --git a/obnamlib/plugins/encryption_plugin.py b/obnamlib/plugins/encryption_plugin.py +index 8c8eecf..a48c329 100644 +--- a/obnamlib/plugins/encryption_plugin.py ++++ b/obnamlib/plugins/encryption_plugin.py +@@ -55,6 +55,18 @@ class EncryptionPlugin(obnamlib.ObnamPlugin): + metavar='HOMEDIR', + group=encryption_group, + default=None) ++ self.app.settings.string( ++ ['symmetric-cipher'], ++ 'GPG symmetric encryption cipher', ++ metavar='CIPHER', ++ group=encryption_group, ++ default=None) ++ self.app.settings.string( ++ ['symmetric-digest'], ++ 'GPG symmetric encryption passphrase digest', ++ metavar='DIGEST', ++ group=encryption_group, ++ default=None) + + self.tag = "encrypt1" + +@@ -112,6 +124,18 @@ class EncryptionPlugin(obnamlib.ObnamPlugin): + def symmetric_key_bits(self): + return int(self.app.settings['symmetric-key-bits'] or '256') + ++ @property ++ def symmetric_cipher(self): ++ '''Get the symmetric cipher from config. ++ Return None for GPG default.''' ++ return self.app.settings['symmetric-cipher'] ++ ++ @property ++ def symmetric_digest(self): ++ '''Get the symmetric digest from config. ++ Return None for GPG default.''' ++ return self.app.settings['symmetric-digest'] ++ + def _write_file(self, repo, pathname, contents): + repo.get_fs().write_file(pathname, contents) + +@@ -133,20 +157,30 @@ class EncryptionPlugin(obnamlib.ObnamPlugin): + self._write_file(repo, os.path.join(toplevel, 'key'), encrypted) + + encoded = str(pubkeys) +- encrypted = obnamlib.encrypt_symmetric(encoded, symmetric_key) ++ encrypted = obnamlib.encrypt_symmetric( ++ encoded, symmetric_key, ++ gpghome=self.gnupghome, ++ cipher=self.symmetric_cipher, ++ digest=self.symmetric_digest) + self._write_file(repo, os.path.join(toplevel, 'userkeys'), encrypted) + + def filter_read(self, encrypted, repo, toplevel): + symmetric_key = self.get_symmetric_key(repo, toplevel) +- return obnamlib.decrypt_symmetric(encrypted, symmetric_key, +- gpghome=self.gnupghome) ++ return obnamlib.decrypt_symmetric( ++ encrypted, symmetric_key, ++ gpghome=self.gnupghome, ++ cipher=self.symmetric_cipher, ++ digest=self.symmetric_digest) + + def filter_write(self, cleartext, repo, toplevel): + if not self.keyid: + return cleartext + symmetric_key = self.get_symmetric_key(repo, toplevel) +- return obnamlib.encrypt_symmetric(cleartext, symmetric_key, +- gpghome=self.gnupghome) ++ return obnamlib.encrypt_symmetric( ++ cleartext, symmetric_key, ++ gpghome=self.gnupghome, ++ cipher=self.symmetric_cipher, ++ digest=self.symmetric_digest) + + def get_symmetric_key(self, repo, toplevel): + key = self._symkeys.get(repo, toplevel) +diff --git a/test-gpghome/random_seed b/test-gpghome/random_seed +index f4ad4794cccb730cf965ead333d93493c4a721e4..421d386505c9f9dc662682137c0f4e3e4a31e917 100644 +GIT binary patch +literal 600 +zcmV-e0;l~%M+tI7i&1?H@__g{3A^S4QdWHxSy4^tdPW5AO6XxaV}F4g3=K|#f~8E( +zG$dGz<^_n0T)IFj%Lq!}y +z6@=($YE82yjJ%b)1-Qd6ce1h@Mjf|NSShPUxU02=gR8VXYyB`CaX|lyA_*ers$|M_RqipCd)DSy^Cr=oF-0MAse>Y1px! +z$ZwMKyUpk!S3!PB?ou%H6iJe^N@3ygk#2{e9efyQ>Ef0Zrg=In)ja37N0V@Keh%^W +zs^I9qS4k(G>Fa|qOM@gT^N6CEh)-qBPB?Pm*yzNZ6YK@lIPh08yoOX5F5%=5bv8#fOmlgEJ+^%wmmhYhoi;^F4L63KdG{q5WpG>crD6fspt_xxX1I@HBZ9Iq;F0S=}&qh?2s-NmrRbx6!;Spp0TP0U5u7;F1y9 +zZ8SrIXtL1NB|#FUU7d(!*7g#eTR4s#>frt|DJ-Kk)&{KMrTQRJ+g+Wu?kHaN06~i} +zz`2kqB*v`rrR_kO7EC#IdpZ=AlEo;O$V#4S%*cf%?J4m4M_v@0prK3=t2hs#*p*R4 +zoQ}lQ1-nLHG_Xw7+zu36jNGWS_Gx3u6D~x&2SY)QtvHRLj$JnJ|0OksXJ=WLo5oMh9{ +z1UyfWe!aA9?cWcTnSwH_^2rHP6?mvJk8S8KQt1}b$_cQ&_#C{To(bos5=|xFwMqG? +mEtumkn|KB_R5k5WtSjRBQOZQzHC@2Ny+3eufrduCa3Xsieky4I + +-- +2.9.0 + + +_______________________________________________ +obnam-dev mailing list +obnam-dev@obnam.org +http://listmaster.pepperfish.net/cgi-bin/mailman/listinfo/obnam-dev-obnam.org -- cgit v1.2.1