From 2a79f5d4afd62f6ffb9f765a498c036cf7f8822e Mon Sep 17 00:00:00 2001 From: Distix listener Date: Sun, 19 Jun 2016 10:40:09 +0200 Subject: imported mails --- .../cur/.this-dir-not-empty/.empty/empty-file | 0 .../new/.this-dir-not-empty/.empty/empty-file | 0 .../Maildir/new/1466325608.M779738P21117Q1.hrun | 230 +++++++++++++++++++++ .../tmp/.this-dir-not-empty/.empty/empty-file | 0 .../cb75a21b4a874f86ba49e06ae8d887fc/ticket.yaml | 4 + 5 files changed, 234 insertions(+) create mode 100644 tickets/cb75a21b4a874f86ba49e06ae8d887fc/Maildir/cur/.this-dir-not-empty/.empty/empty-file create mode 100644 tickets/cb75a21b4a874f86ba49e06ae8d887fc/Maildir/new/.this-dir-not-empty/.empty/empty-file create mode 100644 tickets/cb75a21b4a874f86ba49e06ae8d887fc/Maildir/new/1466325608.M779738P21117Q1.hrun create mode 100644 tickets/cb75a21b4a874f86ba49e06ae8d887fc/Maildir/tmp/.this-dir-not-empty/.empty/empty-file create mode 100644 tickets/cb75a21b4a874f86ba49e06ae8d887fc/ticket.yaml (limited to 'tickets/cb75a21b4a874f86ba49e06ae8d887fc') diff --git a/tickets/cb75a21b4a874f86ba49e06ae8d887fc/Maildir/cur/.this-dir-not-empty/.empty/empty-file b/tickets/cb75a21b4a874f86ba49e06ae8d887fc/Maildir/cur/.this-dir-not-empty/.empty/empty-file new file mode 100644 index 0000000..e69de29 diff --git a/tickets/cb75a21b4a874f86ba49e06ae8d887fc/Maildir/new/.this-dir-not-empty/.empty/empty-file b/tickets/cb75a21b4a874f86ba49e06ae8d887fc/Maildir/new/.this-dir-not-empty/.empty/empty-file new file mode 100644 index 0000000..e69de29 diff --git a/tickets/cb75a21b4a874f86ba49e06ae8d887fc/Maildir/new/1466325608.M779738P21117Q1.hrun b/tickets/cb75a21b4a874f86ba49e06ae8d887fc/Maildir/new/1466325608.M779738P21117Q1.hrun new file mode 100644 index 0000000..24dff8c --- /dev/null +++ b/tickets/cb75a21b4a874f86ba49e06ae8d887fc/Maildir/new/1466325608.M779738P21117Q1.hrun @@ -0,0 +1,230 @@ +Return-Path: +X-Original-To: distix@pieni.net +Delivered-To: distix@pieni.net +Received: from bagpuss.pepperfish.net (bagpuss.pepperfish.net [148.251.8.16]) + (using TLSv1.2 with cipher AECDH-AES256-SHA (256/256 bits)) + (No client certificate requested) + by pieni.net (Postfix) with ESMTPS id CCB862268D + for ; Sun, 19 Jun 2016 10:39:27 +0200 (CEST) +Received: from platypus.pepperfish.net (unknown [10.112.100.20]) + by bagpuss.pepperfish.net (Postfix) with ESMTP id 3AED25BC; + Sun, 19 Jun 2016 09:39:27 +0100 (BST) +Received: from ip6-localhost ([::1] helo=platypus.pepperfish.net) + by platypus.pepperfish.net with esmtp (Exim 4.80 #2 (Debian)) + id 1bEYGN-0002Y5-2C; Sun, 19 Jun 2016 09:39:27 +0100 +Received: from inmail0 ([10.112.100.10] helo=mx0.pepperfish.net) + by platypus.pepperfish.net with esmtp (Exim 4.80 #2 (Debian)) + id 1bEYGL-0002Xn-Q5 + for ; Sun, 19 Jun 2016 09:39:25 +0100 +Received: from smtp.gentoo.org ([140.211.166.183]) + by mx0.pepperfish.net with esmtps (TLS1.2:DHE_RSA_AES_256_CBC_SHA256:256) + (Exim 4.80) (envelope-from ) id 1bEYGH-00068d-FU + for obnam-dev@obnam.org; Sun, 19 Jun 2016 09:39:25 +0100 +Received: from grubbs.orbis-terrarum.net (localhost [127.0.0.1]) + (using TLSv1.2 with cipher ECDHE-RSA-AES256-SHA (256/256 bits)) + (No client certificate requested) + by smtp.gentoo.org (Postfix) with ESMTPS id C008834067D + for ; Sun, 19 Jun 2016 08:39:04 +0000 (UTC) +Received: (qmail 8958 invoked by uid 10000); 19 Jun 2016 08:39:04 -0000 +Date: Sun, 19 Jun 2016 08:39:04 +0000 +From: "Robin H. Johnson" +To: obnam-support@obnam.org, obnam-dev@obnam.org +Message-ID: <20160619083904.GA18768@orbis-terrarum.net> +MIME-Version: 1.0 +User-Agent: Mutt/1.5.24 (2015-08-30) +X-Spam-Score: -9.8 +X-Spam-Score-int: -97 +X-Spam-Bar: --------- +X-Scanned-By: pepperfish.net, Sun, 19 Jun 2016 09:39:25 +0100 +X-Spam-Report: Content analysis details: (-9.8 points) + pts rule name description + ---- ---------------------- -------------------------------------------------- + -5.0 RCVD_IN_DNSWL_HI RBL: Sender listed at http://www.dnswl.org/, high + trust [140.211.166.183 listed in list.dnswl.org] + -1.0 PPF_USER_AGENT_MUTT User-Agent: contains Mutt (Mutt isn't a spam + tool) -0.5 PPF_USER_AGENT User-Agent: exists + -0.0 SPF_PASS SPF: sender matches SPF record + -1.4 RP_MATCHES_RCVD Envelope sender domain matches handover relay domain + -1.9 BAYES_00 BODY: Bayes spam probability is 0 to 1% + [score: 0.0000] +X-ACL-Warn: message may be spam +X-Scan-Signature: 9b2f0c8d06ad4d4ac6b47be2df4622cd +Subject: [1/2] GPG & performance: a deep-dive +X-BeenThere: obnam-dev@obnam.org +X-Mailman-Version: 2.1.5 +Precedence: list +List-Id: Obnam development discussions +List-Unsubscribe: , + +List-Archive: +List-Post: +List-Help: +List-Subscribe: , + +Content-Type: multipart/mixed; boundary="===============8343931285586773396==" +Mime-version: 1.0 +Sender: obnam-dev-bounces@obnam.org +Errors-To: obnam-dev-bounces@obnam.org + + +--===============8343931285586773396== +Content-Type: multipart/signed; micalg=pgp-sha1; + protocol="application/pgp-signature"; boundary="APlYHCtpeOhspHkB" +Content-Disposition: inline + + +--APlYHCtpeOhspHkB +Content-Type: text/plain; charset=us-ascii +Content-Disposition: inline +Content-Transfer-Encoding: quoted-printable + +Hi all, + +I've been looking at backup options for a deployment, and in considering +obnam, I like it's general speed, but found that it dropped unacceptably +when encryption was enabled. + +TL;DR: suggestions +- Right now: set '-z 0' in obnam symmetric crypto call, immediate 10% + performance boost. +- Plan for moving to PyCrypto or other for symmetric crypto + +A first pass examination pointed strongly to obnam's of GPG symmetric +encryption.=20 + +I improved the obnam-benchmark tool to help take these measurements +below, the changes are on GitHub [1]; but first let's look at how GPG +does symmetric encryption. + +GPG symmetric encryption (S2K) does the following: +- takes a passphrase & data input, +- optionally transforms the passphrase. + (see s2k-digest-algo, s2k-mode, s2k-count) +- optionally compresses the input + (compress-level) +- enciphers the output + (see s2k-cipher-algo) +- emit output in the S2K structure + This records all of the above s2k-* parameters, as well. + +Stock obnam simply calls 'gpg -c' for symmetric encryption. +In the absence of any other configuration, this generally has the +following defaults: +- s2k-digest-algo=3DSHA1 +- s2k-mode=3D3 (key stretching by repeated hashing) +- s2k-count=3Dvaries, my systems are 25M..65M +- compress-level=3D6 (ZLIB level 6) +- s2k-cipher-algo=3DAES128 + +It uses the cipher in a modified CFB mode [RFC4880, sec 5.7, ... "Tag 9"] + +Naively, you might think that GPG is fast enough. Sure, take a 1GB +incompressible input, as a single file. + + 0.6s | cat in > out + 0.8s | gpg --store -z 0 +22.3s | gpg --store -z 6 + 5.6s | gpg --symmetric -z 0 +27.4s | gpg --symmetric -z 6 # Default settings! + +S2K packet encoding: 33% slower +Compression, used by default: 5-28x performance hit +Symmetric enciphering: ~7x performance hit +Overall: 45x slower + +The catch is that obnam calls gpg many many times, with much smaller +inputs, so we have to pay the startup costs many times over. + +I set out to measure the cost breakdown of using gpg: +- exec overhead +- S2K packet overhead +- symmetric encryption +- S2K compression + +With the stock codebase, the gpg encryption plugin has this approx +performance effect for me: +- many_files benchmark, it's only a 20% hit, but there are only 256 +unique values.=20 +- On the big_file benchmark, it's ~45x slower (than cat) + +First, the reference/stock runs: +A: rsync -a live backup && rsync -a backup restore +B: stock obnam, run with obnam-benchmark, production.yaml, no encryption +C: stock as B, with gpg encryption (compressed symmetric encryption) + +Now the modified code variants: +W: gpg, symmetric encryption, uncompressed +X: gpg, s/--symmetric/--store/, compressed +Y: gpg, s/--symmetric/--store/, uncompressed +Z: HACK gpg-symmetric to just return the raw block +PYC: quick hack for PyCrypto AES-256-CTR + +B->Z: obnam's overhead in asymmetric encryption only. +Z->Y: this is the overhead added by obnam using GPG symmetric encryption + (on top of the asymmetric management). +Y->X,=20 +C->W: this is the overhead added by S2K compression on stored & + enciphering. +Y->W: this is the overhead added by enciphering, with no compression + +Timing data: +------------ +in seconds, average of 3 runs. +B1 =3D many_files +B2 =3D one_big_file + + Benchmark +Test| B1 | B2 +----+------+------- +A | 15.0| 5.1 +B | 225.4| 10.2 +C | 284.8| 272.5 +----+------+------ +W | 288.3| 246.4 +X | 266.1| 57.7 +Y | 266.4| 33.9 +Z | 249.9| 14.4 +----+------+------ +PYC | 236.4| 33.6 +----+------+------ + +[1] https://github.com/robbat2/obnam-benchmarks/tree/robbat2/flexibility + +--=20 +Robin Hugh Johnson +Gentoo Linux: Dev, Infra Lead, Foundation Trustee & Treasurer +E-Mail : robbat2@gentoo.org +GnuPG FP : 11ACBA4F 4778E3F6 E4EDF38E B27B944E 34884E85 +GnuPG FP : 7D0B3CEB E9B85B1F 825BCECF EE05E6F6 A48F6136 + +--APlYHCtpeOhspHkB +Content-Type: application/pgp-signature; name="signature.asc" +Content-Description: Digital signature + +-----BEGIN PGP SIGNATURE----- +Version: GnuPG v2.1 +Comment: Robbat2 @ Orbis-Terrarum Networks - The text below is a digital signature. If it doesn't make any sense to you, ignore it. + +iKYEARECAGYFAldmWihfFIAAAAAALgAoaXNzdWVyLWZwckBub3RhdGlvbnMub3Bl +bnBncC5maWZ0aGhvcnNlbWFuLm5ldDc1OTQwNEJFQkQ0MUY3MTIzODIzODZFRjNF +OTIyQzIyMzIzM0MyMkMACgkQPpIsIjIzwixPYACgzqgcY2uxMJwMmjBYG/HPXYAu +jo4AoNhPOQjVu1CF7eE8RPG2cndRd2Fk +=ukL+ +-----END PGP SIGNATURE----- + +--APlYHCtpeOhspHkB-- + + +--===============8343931285586773396== +Content-Type: text/plain; charset="us-ascii" +MIME-Version: 1.0 +Content-Transfer-Encoding: 7bit +Content-Disposition: inline + +_______________________________________________ +obnam-dev mailing list +obnam-dev@obnam.org +http://listmaster.pepperfish.net/cgi-bin/mailman/listinfo/obnam-dev-obnam.org + +--===============8343931285586773396==-- + diff --git a/tickets/cb75a21b4a874f86ba49e06ae8d887fc/Maildir/tmp/.this-dir-not-empty/.empty/empty-file b/tickets/cb75a21b4a874f86ba49e06ae8d887fc/Maildir/tmp/.this-dir-not-empty/.empty/empty-file new file mode 100644 index 0000000..e69de29 diff --git a/tickets/cb75a21b4a874f86ba49e06ae8d887fc/ticket.yaml b/tickets/cb75a21b4a874f86ba49e06ae8d887fc/ticket.yaml new file mode 100644 index 0000000..07c5593 --- /dev/null +++ b/tickets/cb75a21b4a874f86ba49e06ae8d887fc/ticket.yaml @@ -0,0 +1,4 @@ +ticket-id: +- cb75a21b4a874f86ba49e06ae8d887fc +title: +- '[1/2] GPG & performance: a deep-dive' -- cgit v1.2.1