| 1 | """ |
|---|
| 2 | Core data represenation. |
|---|
| 3 | |
|---|
| 4 | |
|---|
| 5 | """ |
|---|
| 6 | from itertools import * |
|---|
| 7 | import cPickle as pickle |
|---|
| 8 | import os, sys, bsddb |
|---|
| 9 | import logger, util, conf |
|---|
| 10 | |
|---|
| 11 | def path_join(*args): |
|---|
| 12 | return os.path.abspath(os.path.join(*args)) |
|---|
| 13 | |
|---|
| 14 | class FileDict(object): |
|---|
| 15 | """ |
|---|
| 16 | File based dictionary that can be pickled and restored |
|---|
| 17 | from a pickle. Built around bsddb.btopen. |
|---|
| 18 | |
|---|
| 19 | >>> fname = conf.tempdata('filedict-testdata.bin') |
|---|
| 20 | >>> fdict = FileDict(fname, mode='n') |
|---|
| 21 | >>> for value in range(5): |
|---|
| 22 | ... fdict[value] = value*10 |
|---|
| 23 | >>> fdict.save() |
|---|
| 24 | |
|---|
| 25 | >>> # still works after unpickling the object |
|---|
| 26 | >>> blob = pickle.dumps(fdict) |
|---|
| 27 | >>> db = pickle.loads(blob) |
|---|
| 28 | |
|---|
| 29 | >>> db['1'] |
|---|
| 30 | 10 |
|---|
| 31 | >>> list(db.keys()) |
|---|
| 32 | ['0', '1', '2', '3', '4'] |
|---|
| 33 | >>> list(db.values()) |
|---|
| 34 | [0, 10, 20, 30, 40] |
|---|
| 35 | >>> list(iter(db)) |
|---|
| 36 | [0, 10, 20, 30, 40] |
|---|
| 37 | >>> list(db.items()) |
|---|
| 38 | [('0', 0), ('1', 10), ('2', 20), ('3', 30), ('4', 40)] |
|---|
| 39 | >>> len(db) |
|---|
| 40 | 5 |
|---|
| 41 | >>> 4 in db |
|---|
| 42 | True |
|---|
| 43 | >>> 5 in db |
|---|
| 44 | False |
|---|
| 45 | """ |
|---|
| 46 | |
|---|
| 47 | def __init__(self, fname, mode='c' ): |
|---|
| 48 | self.fname = path_join( fname ) |
|---|
| 49 | try: |
|---|
| 50 | self.db = bsddb.btopen(self.fname, flag=mode) |
|---|
| 51 | except Exception, exc: |
|---|
| 52 | # default error does not indicate the file name |
|---|
| 53 | logger.error('error opening %s' % self.fname) |
|---|
| 54 | raise exc |
|---|
| 55 | |
|---|
| 56 | def __setitem__(self, key, value): |
|---|
| 57 | self.db[ str(key) ] = pickle.dumps(value, 0) |
|---|
| 58 | |
|---|
| 59 | def __getstate__(self): |
|---|
| 60 | # remove file handle |
|---|
| 61 | cdict = self.__dict__.copy() |
|---|
| 62 | del cdict['db'] |
|---|
| 63 | return cdict |
|---|
| 64 | |
|---|
| 65 | def __setstate__(self, data): |
|---|
| 66 | self.db = bsddb.btopen( data['fname'], flag='c') |
|---|
| 67 | |
|---|
| 68 | def __getitem__(self, key): |
|---|
| 69 | return pickle.loads( self.db[str(key)] ) |
|---|
| 70 | |
|---|
| 71 | def save(self): |
|---|
| 72 | self.db.sync() |
|---|
| 73 | |
|---|
| 74 | def keys( self ): |
|---|
| 75 | return self.db.iterkeys() |
|---|
| 76 | |
|---|
| 77 | def values( self ): |
|---|
| 78 | return imap(pickle.loads, self.db.itervalues()) |
|---|
| 79 | |
|---|
| 80 | def items( self ): |
|---|
| 81 | return izip(self.keys(), self.values() ) |
|---|
| 82 | |
|---|
| 83 | def __iter__(self): |
|---|
| 84 | return self.values() |
|---|
| 85 | |
|---|
| 86 | def __contains__(self, key): |
|---|
| 87 | return self.db.has_key(str(key)) |
|---|
| 88 | |
|---|
| 89 | def __len__(self): |
|---|
| 90 | return len(self.db) |
|---|
| 91 | |
|---|
| 92 | def close(self): |
|---|
| 93 | self.db.close() |
|---|
| 94 | |
|---|
| 95 | def __del__(self): |
|---|
| 96 | self.close() |
|---|
| 97 | |
|---|
| 98 | def test(): |
|---|
| 99 | fname = conf.testdata('filedict-testdata.bin') |
|---|
| 100 | |
|---|
| 101 | |
|---|
| 102 | if __name__ == '__main__': |
|---|
| 103 | import doctest |
|---|
| 104 | doctest.testmod() |
|---|
| 105 | |
|---|
| 106 | test() |
|---|