From 666cb366f67dc2dd5e6c7733d292547e6a1d37dd Mon Sep 17 00:00:00 2001 From: distix ticketing system Date: Wed, 19 Apr 2017 11:20:13 +0000 Subject: imported mails --- .../cur/.this-dir-not-empty/.empty/empty-file | 0 .../new/.this-dir-not-empty/.empty/empty-file | 0 .../Maildir/new/1492600813.M742275P11728Q1.koom | 197 +++++++++++++++++++++ .../tmp/.this-dir-not-empty/.empty/empty-file | 0 .../6e2a44fcd630472b82cf898f26a7f936/ticket.yaml | 4 + 5 files changed, 201 insertions(+) create mode 100644 tickets/6e2a44fcd630472b82cf898f26a7f936/Maildir/cur/.this-dir-not-empty/.empty/empty-file create mode 100644 tickets/6e2a44fcd630472b82cf898f26a7f936/Maildir/new/.this-dir-not-empty/.empty/empty-file create mode 100644 tickets/6e2a44fcd630472b82cf898f26a7f936/Maildir/new/1492600813.M742275P11728Q1.koom create mode 100644 tickets/6e2a44fcd630472b82cf898f26a7f936/Maildir/tmp/.this-dir-not-empty/.empty/empty-file create mode 100644 tickets/6e2a44fcd630472b82cf898f26a7f936/ticket.yaml diff --git a/tickets/6e2a44fcd630472b82cf898f26a7f936/Maildir/cur/.this-dir-not-empty/.empty/empty-file b/tickets/6e2a44fcd630472b82cf898f26a7f936/Maildir/cur/.this-dir-not-empty/.empty/empty-file new file mode 100644 index 0000000..e69de29 diff --git a/tickets/6e2a44fcd630472b82cf898f26a7f936/Maildir/new/.this-dir-not-empty/.empty/empty-file b/tickets/6e2a44fcd630472b82cf898f26a7f936/Maildir/new/.this-dir-not-empty/.empty/empty-file new file mode 100644 index 0000000..e69de29 diff --git a/tickets/6e2a44fcd630472b82cf898f26a7f936/Maildir/new/1492600813.M742275P11728Q1.koom b/tickets/6e2a44fcd630472b82cf898f26a7f936/Maildir/new/1492600813.M742275P11728Q1.koom new file mode 100644 index 0000000..e93458f --- /dev/null +++ b/tickets/6e2a44fcd630472b82cf898f26a7f936/Maildir/new/1492600813.M742275P11728Q1.koom @@ -0,0 +1,197 @@ +Return-Path: +X-Original-To: distix@pieni.net +Delivered-To: distix@pieni.net +Received: from bagpuss.pepperfish.net (bagpuss.pepperfish.net [148.251.8.16]) + by pieni.net (Postfix) with ESMTPS id 9B8224123D + for ; Wed, 19 Apr 2017 11:18:49 +0000 (UTC) +Received: from platypus.pepperfish.net (unknown [10.112.100.20]) + by bagpuss.pepperfish.net (Postfix) with ESMTP id 0A1F4BD1; + Wed, 19 Apr 2017 12:18:49 +0100 (BST) +Received: from ip6-localhost ([::1] helo=platypus.pepperfish.net) + by platypus.pepperfish.net with esmtp (Exim 4.80 #2 (Debian)) + id 1d0ndI-0005xa-TX; Wed, 19 Apr 2017 12:18:48 +0100 +Received: from inmail0 ([10.112.100.10] helo=mx0.pepperfish.net) + by platypus.pepperfish.net with esmtp (Exim 4.80 #2 (Debian)) + id 1d0nMZ-0002SD-0O + for ; Wed, 19 Apr 2017 12:01:31 +0100 +Received: from mout.gmx.net ([212.227.17.20]) + by mx0.pepperfish.net with esmtps (TLS1.2:DHE_RSA_AES_128_CBC_SHA1:128) + (Exim 4.80) (envelope-from ) id 1d0n6A-0001N9-S7 + for obnam-dev@obnam.org; Wed, 19 Apr 2017 11:44:36 +0100 +Received: from [95.90.239.153] by 3capp-gmx-bs37.server.lan (via HTTP); Wed, + 19 Apr 2017 12:44:21 +0200 +MIME-Version: 1.0 +Message-ID: +From: "Dionisus Torimens" +To: obnam-dev@obnam.org +Content-Type: multipart/mixed; + boundary=knika-9f351c14-7529-49fa-b1a7-f5d233f9a9fe +Date: Wed, 19 Apr 2017 12:44:21 +0200 +Importance: normal +Sensitivity: Normal +X-Priority: 3 +X-Provags-ID: V03:K1:ehlR11JKeHCQRT2ki6HdaAJ1aIO3+YsgbhfLrEYfWPK + L41RuxUNzzOlco9ZzfLsoh23FOdcGg/eBTXCwMMicaEGYNSPXJ + sFpRaq2dIyDFNSPVvNnorR6lzIdUQOx05IAdYJGELR2ZqxXnuc + Rubj5GO/JXtc6x/9GebDgq1upGYLn0+aY0rX5bOHOgpGfCfMtS + 0NK2KOR3UrBuSlVHPuo6WLAsmj+CyUTqEpsu4ayJuRD/LeOGxt + D+Mt6x8aUFhIGhmnxafQ5J3tV/M2SlEc5XOc9Gz6QJnQ0+ojTy kH245U= +X-UI-Out-Filterresults: notjunk:1;V01:K0:5CAQ8FP+IRY=:ScZh0LWaKKUkljVWMQoRjQ + 5sfoyPM9SyUkQeBUTqGRT9DVDj+8/xVKGiJ7kTir8JKf0+OO4N+7qPdKsYJyH6ABRtPJZ+Vgi + X5vfqFPBbgh4tQji+GgZmRRWdKxzxf43HKTBriBSuWnuVqAqe3i7pnkgnmpC9zHLJko2Nv4eX + qm/JtlJzpt1i12lO4S91cwu4q8AHC3rIpWWN6VU17SGokl3tl5R8g8wPIvaiaSW3HErJRIKgm + C0MYY1QV3nLwD/zNDnt79ETAzAxRFn8dctJQiCQtpiwDkE/jBVeGpbdWZ51hXZInAM2VNHioY + p8QHeLVQuZLwP9UKTEzP4eXJWFDh188jJafe2XlOVnJ8EnoNevLUsFc9Hv6lQDDu0j2hi3EuE + 33grGMhNO0cdNT8uK+DaQLpIhunaqlpdF1MKFdzbZE568q89xLE9PrgTloQIYcRqbPmsLfibz + GUo2HkDf2w== +X-Pepperfish-Transaction: 9acf-e0a0-d921-5b6b +X-Spam-Score: 0.5 +X-Spam-Score-int: 5 +X-Spam-Bar: / +X-Scanned-By: pepperfish.net, Wed, 19 Apr 2017 11:44:36 +0100 +X-Spam-Report: Content analysis details: (0.5 points) + pts rule name description + ---- ---------------------- -------------------------------------------------- + -0.7 RCVD_IN_DNSWL_LOW RBL: Sender listed at http://www.dnswl.org/, low + trust [212.227.17.20 listed in list.dnswl.org] + 1.0 PPF_MSGID_DIGITS Message ID has @ in it + 0.4 PPF_XPRIORITY X-Priority header exists + 0.5 PPF_MESSAGEID_NODOTS Message-Id contains no dots after the @ + 0.5 PPF_RECEIVED_HTTP Received header mentions http + 1.2 FREEMAIL_FROM Sender email is commonly abused enduser mail provider + (djtm[at]gmx.net) + -0.0 SPF_PASS SPF: sender matches SPF record + -1.0 RP_MATCHES_RCVD Envelope sender domain matches handover relay domain + -1.9 BAYES_00 BODY: Bayes spam probability is 0 to 1% + [score: 0.0000] + 0.3 HTML_MESSAGE BODY: HTML included in message + 0.2 PPF_NUMERIC_ENTITY RAW: Body contains numeric HTML entities +X-ACL-Warn: message may be spam +X-Scan-Signature: 805eda458b26030fb35277d5bf2b304c +Subject: [rfc] [PATCH] parallelize listdir +X-BeenThere: obnam-dev@obnam.org +X-Mailman-Version: 2.1.5 +Precedence: list +List-Id: Obnam development discussions +List-Unsubscribe: , + +List-Archive: +List-Post: +List-Help: +List-Subscribe: , + +Sender: obnam-dev-bounces@obnam.org +Errors-To: obnam-dev-bounces@obnam.org + +--knika-9f351c14-7529-49fa-b1a7-f5d233f9a9fe +Content-Type: text/html; charset=UTF-8 + +
Hi,
+ +
 
+ +
an initial try to use a bit of multiprocessing. Makes obnam a bit faster in scannig. No change in the algorithms. I'd love to see some benchmarking, it feels a bit snappier. The pathos library is very helpful, but not in most distribution packages afaik.
+ +
 
+ +
I've been thinking and I guess a good solution in the longer term would be to have
+ +
- a scanning process to create a list with changes,
+ +
- another to prepare the uploads and
+ +
- another to actually upload files to the repository.
+ +
 
+ +
Cheers
+--knika-9f351c14-7529-49fa-b1a7-f5d233f9a9fe +Content-Type: text/x-patch +Content-Disposition: attachment; filename=parallel2.patch + +>From fe5333545cda1ac13fb10ae7b93c5f022f518f05 Mon Sep 17 00:00:00 2001 +From: djtm +Date: Wed, 19 Apr 2017 12:28:23 +0200 +Subject: [PATCH] *** SUBJECT HERE *** + +*** BLURB HERE *** + +djtm (1): + parallelize listdir + + obnamlib/vfs_local.py | 25 +++++++++++++++---------- + 1 file changed, 15 insertions(+), 10 deletions(-) + +-- +2.7.4 + +>From fe5333545cda1ac13fb10ae7b93c5f022f518f05 Mon Sep 17 00:00:00 2001 +From: djtm +Date: Tue, 18 Apr 2017 02:54:18 +0200 +Subject: [PATCH] parallelize listdir + +--- + obnamlib/vfs_local.py | 25 +++++++++++++++---------- + 1 file changed, 15 insertions(+), 10 deletions(-) + +diff --git a/obnamlib/vfs_local.py b/obnamlib/vfs_local.py +index e01617e..697ef51 100644 +--- a/obnamlib/vfs_local.py ++++ b/obnamlib/vfs_local.py +@@ -24,6 +24,9 @@ import tempfile + import time + import tracing + ++from pathos.multiprocessing import ProcessingPool ++from functools import partial ++ + import obnamlib + + +@@ -411,17 +414,19 @@ class LocalFS(obnamlib.VirtualFileSystem): + def listdir(self, dirname): + return os.listdir(self.join(dirname)) + ++ def listdir2helper(self, dirname, name): ++ try: ++ st = self.lstat(os.path.join(dirname, name)) ++ except OSError, e: # pragma: no cover ++ st = e ++ ino = -1 ++ else: ++ ino = st.st_ino ++ return (ino, name, st) ++ + def listdir2(self, dirname): +- result = [] +- for name in self.listdir(dirname): +- try: +- st = self.lstat(os.path.join(dirname, name)) +- except OSError, e: # pragma: no cover +- st = e +- ino = -1 +- else: +- ino = st.st_ino +- result.append((ino, name, st)) ++ pool = ProcessingPool() ++ result = pool.map(partial(self.listdir2helper, dirname), self.listdir(dirname)) + + # We sort things in inode order, for speed when doing name lookups + # when backing up. +-- +2.7.4 + + +--knika-9f351c14-7529-49fa-b1a7-f5d233f9a9fe +Content-Type: text/plain; charset="us-ascii" +MIME-Version: 1.0 +Content-Transfer-Encoding: 7bit +Content-Disposition: inline + +_______________________________________________ +obnam-dev mailing list +obnam-dev@obnam.org +http://listmaster.pepperfish.net/cgi-bin/mailman/listinfo/obnam-dev-obnam.org + +--knika-9f351c14-7529-49fa-b1a7-f5d233f9a9fe-- + diff --git a/tickets/6e2a44fcd630472b82cf898f26a7f936/Maildir/tmp/.this-dir-not-empty/.empty/empty-file b/tickets/6e2a44fcd630472b82cf898f26a7f936/Maildir/tmp/.this-dir-not-empty/.empty/empty-file new file mode 100644 index 0000000..e69de29 diff --git a/tickets/6e2a44fcd630472b82cf898f26a7f936/ticket.yaml b/tickets/6e2a44fcd630472b82cf898f26a7f936/ticket.yaml new file mode 100644 index 0000000..29c4424 --- /dev/null +++ b/tickets/6e2a44fcd630472b82cf898f26a7f936/ticket.yaml @@ -0,0 +1,4 @@ +ticket-id: +- 6e2a44fcd630472b82cf898f26a7f936 +title: +- '[rfc] [PATCH] parallelize listdir' -- cgit v1.2.1