summaryrefslogtreecommitdiff
path: root/obnamlib/bag_store.py
blob: 39c4795b2085f577f4304216bbed589191aa6187 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
# Copyright 2015-2016  Lars Wirzenius
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
# =*= License: GPL-3+ =*=


import errno
import logging
import os
import random

import tracing

import obnamlib


# Start with a new random id after this many sequential ones.
# This avoids putting all the ids in one directory.
MAX_IDS_UNTIL_RESET = 512


class BagStore(object):

    def __init__(self):
        self._fs = None
        self._dirname = None
        self._id_inventor = IdInventor()
        self._id_inventor.set_filename_maker(self._make_bag_filename)

    def _make_bag_filename(self, bag_id):
        if isinstance(bag_id, str):
            return os.path.join(self._dirname, '%s.bag' % bag_id)

        basename = '%016x' % bag_id
        return os.path.join(
            self._dirname,
            basename[0:2],
            basename[2:4],
            basename[4:6],
            '%s.bag' % basename)

    def set_location(self, fs, dirname):
        self._fs = fs
        self._dirname = dirname
        self._id_inventor.set_fs(fs)

    def reserve_bag_id(self):
        reserved = self._id_inventor.reserve_id()
        tracing.trace('id=%r', reserved)
        return reserved

    def put_bag(self, bag):
        filename = self._make_bag_filename(bag.get_id())
        serialised = serialise_bag(bag)
        self._fs.overwrite_file(filename, serialised)

    def get_bag(self, bag_id):
        filename = self._make_bag_filename(bag_id)
        serialised = self._fs.cat(filename)
        return deserialise_bag(serialised)

    def has_bag(self, bag_id):
        filename = self._make_bag_filename(bag_id)
        try:
            st = self._fs.lstat(filename)
        except (IOError, OSError):  # pragma: no cover
            return False
        return st.st_size > 0

    def get_bag_ids(self):
        for pathname, _ in self._fs.scan_tree(self._dirname):
            if self._is_bag_filename(pathname):
                yield self._get_bag_id_from_filename(pathname)

    def _is_bag_filename(self, pathname):
        return pathname.endswith('.bag')

    def _get_bag_id_from_filename(self, pathname):
        basename = os.path.basename(pathname)
        return int(basename[:-len('.bag')], 16)

    def remove_bag(self, bag_id):
        filename = self._make_bag_filename(bag_id)
        try:
            self._fs.remove(filename)
        except EnvironmentError as e:
            logging.warning(
                'Tried to delete %s which does not exist:%d:%s',
                filename, e.errno, str(e))


class IdInventor(object):

    def __init__(self):
        self.set_fs(None)
        self._filename_maker = None

    def set_fs(self, fs):
        self._fs = fs
        self._reset_ids()

    def _reset_ids(self):
        self._prev_id = None
        self._ids_since_reset = 0
        self._max_ids_until_reset = MAX_IDS_UNTIL_RESET

    def set_filename_maker(self, maker):
        self._filename_maker = maker

    def reserve_id(self):
        while True:
            if self._time_to_reset_ids():  # pragma: no cover
                self._reset_ids()
            self._next_id()
            if self._reserve_succeeds():
                return self._prev_id
            self._prev_id = None  # pragma: no cover

    def _time_to_reset_ids(self):
        return self._ids_since_reset >= self._max_ids_until_reset

    def _next_id(self):
        if self._prev_id is None:
            self._prev_id = random.randint(0, obnamlib.MAX_ID)
        else:
            self._prev_id += 1  # pragma: no cover
        self._ids_since_reset += 1

    def _reserve_succeeds(self):
        filename = self._filename_maker(self._prev_id)
        try:
            self._fs.write_file(filename, '')
        except OSError as e:  # pragma: no cover
            if e.errno == errno.EEXIST:
                return False
            raise
        return True


def serialise_bag(bag):
    obj = {
        'bag-id': bag.get_id(),
        'blobs': [bag[i] for i in range(len(bag))],
    }
    return obnamlib.serialise_object(obj)


def deserialise_bag(serialised):
    obj = obnamlib.deserialise_object(serialised)
    bag = obnamlib.Bag()
    bag.set_id(obj['bag-id'])
    for blob in obj['blobs']:
        bag.append(blob)
    return bag