1 | """ |
---|
2 | Core data represenation. |
---|
3 | |
---|
4 | |
---|
5 | """ |
---|
6 | from itertools import * |
---|
7 | import cPickle as pickle |
---|
8 | import os, sys, bsddb |
---|
9 | import logger, util, conf |
---|
10 | |
---|
11 | def path_join(*args): |
---|
12 | return os.path.abspath(os.path.join(*args)) |
---|
13 | |
---|
14 | class FileDict(object): |
---|
15 | """ |
---|
16 | File based dictionary that can be pickled and restored |
---|
17 | from a pickle. Built around bsddb.btopen. |
---|
18 | |
---|
19 | >>> fname = conf.tempdata('filedict-testdata.bin') |
---|
20 | >>> fdict = FileDict(fname, mode='n') |
---|
21 | >>> for value in range(5): |
---|
22 | ... fdict[value] = value*10 |
---|
23 | >>> fdict.save() |
---|
24 | |
---|
25 | >>> # still works after unpickling the object |
---|
26 | >>> blob = pickle.dumps(fdict) |
---|
27 | >>> db = pickle.loads(blob) |
---|
28 | |
---|
29 | >>> db['1'] |
---|
30 | 10 |
---|
31 | >>> list(db.keys()) |
---|
32 | ['0', '1', '2', '3', '4'] |
---|
33 | >>> list(db.values()) |
---|
34 | [0, 10, 20, 30, 40] |
---|
35 | >>> list(iter(db)) |
---|
36 | [0, 10, 20, 30, 40] |
---|
37 | >>> list(db.items()) |
---|
38 | [('0', 0), ('1', 10), ('2', 20), ('3', 30), ('4', 40)] |
---|
39 | >>> len(db) |
---|
40 | 5 |
---|
41 | >>> 4 in db |
---|
42 | True |
---|
43 | >>> 5 in db |
---|
44 | False |
---|
45 | """ |
---|
46 | |
---|
47 | def __init__(self, fname, mode='c' ): |
---|
48 | self.fname = path_join( fname ) |
---|
49 | try: |
---|
50 | self.db = bsddb.btopen(self.fname, flag=mode) |
---|
51 | except Exception, exc: |
---|
52 | # default error does not indicate the file name |
---|
53 | logger.error('error opening %s' % self.fname) |
---|
54 | raise exc |
---|
55 | |
---|
56 | def __setitem__(self, key, value): |
---|
57 | self.db[ str(key) ] = pickle.dumps(value, 0) |
---|
58 | |
---|
59 | def __getstate__(self): |
---|
60 | # remove file handle |
---|
61 | cdict = self.__dict__.copy() |
---|
62 | del cdict['db'] |
---|
63 | return cdict |
---|
64 | |
---|
65 | def __setstate__(self, data): |
---|
66 | self.db = bsddb.btopen( data['fname'], flag='c') |
---|
67 | |
---|
68 | def __getitem__(self, key): |
---|
69 | return pickle.loads( self.db[str(key)] ) |
---|
70 | |
---|
71 | def save(self): |
---|
72 | self.db.sync() |
---|
73 | |
---|
74 | def keys( self ): |
---|
75 | return self.db.iterkeys() |
---|
76 | |
---|
77 | def values( self ): |
---|
78 | return imap(pickle.loads, self.db.itervalues()) |
---|
79 | |
---|
80 | def items( self ): |
---|
81 | return izip(self.keys(), self.values() ) |
---|
82 | |
---|
83 | def __iter__(self): |
---|
84 | return self.values() |
---|
85 | |
---|
86 | def __contains__(self, key): |
---|
87 | return self.db.has_key(str(key)) |
---|
88 | |
---|
89 | def __len__(self): |
---|
90 | return len(self.db) |
---|
91 | |
---|
92 | def close(self): |
---|
93 | self.db.close() |
---|
94 | |
---|
95 | def __del__(self): |
---|
96 | self.close() |
---|
97 | |
---|
98 | def test(): |
---|
99 | fname = conf.testdata('filedict-testdata.bin') |
---|
100 | |
---|
101 | |
---|
102 | if __name__ == '__main__': |
---|
103 | import doctest |
---|
104 | doctest.testmod() |
---|
105 | |
---|
106 | test() |
---|