home *** CD-ROM | disk | FTP | other *** search
- # Source Generated with Decompyle++
- # File: in.pyc (Python 2.6)
-
- from IPython.external.path import path as Path
- import os
- import stat
- import time
- import cPickle as pickle
- import UserDict
- import warnings
- import glob
-
- def gethashfile(key):
- return '%02x' % abs(hash(key) % 256)[-2:]
-
- _sentinel = object()
-
- class PickleShareDB(UserDict.DictMixin):
-
- def __init__(self, root):
- self.root = Path(root).expanduser().abspath()
- if not self.root.isdir():
- self.root.makedirs()
-
- self.cache = { }
-
-
- def __getitem__(self, key):
- fil = self.root / key
-
- try:
- mtime = fil.stat()[stat.ST_MTIME]
- except OSError:
- raise KeyError(key)
-
- if fil in self.cache and mtime == self.cache[fil][1]:
- return self.cache[fil][0]
-
- try:
- obj = pickle.load(fil.open())
- except:
- mtime == self.cache[fil][1]
- raise KeyError(key)
-
- self.cache[fil] = (obj, mtime)
- return obj
-
-
- def __setitem__(self, key, value):
- fil = self.root / key
- parent = fil.parent
- if parent and not parent.isdir():
- parent.makedirs()
-
- pickled = pickle.dump(value, fil.open('w'))
-
- try:
- self.cache[fil] = (value, fil.mtime)
- except OSError:
- e = None
- if e.errno != 2:
- raise
- e.errno != 2
-
-
-
- def hset(self, hashroot, key, value):
- hroot = self.root / hashroot
- if not hroot.isdir():
- hroot.makedirs()
-
- hfile = hroot / gethashfile(key)
- d = self.get(hfile, { })
- d.update({
- key: value })
- self[hfile] = d
-
-
- def hget(self, hashroot, key, default = _sentinel, fast_only = True):
- hroot = self.root / hashroot
- hfile = hroot / gethashfile(key)
- d = self.get(hfile, _sentinel)
- if d is _sentinel:
- if fast_only:
- if default is _sentinel:
- raise KeyError(key)
- default is _sentinel
- return default
- d = self.hdict(hashroot)
-
- return d.get(key, default)
-
-
- def hdict(self, hashroot):
- hfiles = self.keys(hashroot + '/*')
- hfiles.sort()
- if not len(hfiles) or hfiles[-1]:
- pass
- last = ''
- if last.endswith('xx'):
- hfiles = [
- last] + hfiles[:-1]
-
- all = { }
- for f in hfiles:
-
- try:
- all.update(self[f])
- except KeyError:
- print 'Corrupt', f, 'deleted - hset is not threadsafe!'
- del self[f]
-
- self.uncache(f)
-
- return all
-
-
- def hcompress(self, hashroot):
- hfiles = self.keys(hashroot + '/*')
- all = { }
- for f in hfiles:
- all.update(self[f])
- self.uncache(f)
-
- self[hashroot + '/xx'] = all
- for f in hfiles:
- p = self.root / f
- if p.basename() == 'xx':
- continue
-
- p.remove()
-
-
-
- def __delitem__(self, key):
- fil = self.root / key
- self.cache.pop(fil, None)
-
- try:
- fil.remove()
- except OSError:
- pass
-
-
-
- def _normalized(self, p):
- return str(self.root.relpathto(p)).replace('\\', '/')
-
-
- def keys(self, globpat = None):
- return _[2]
-
-
- def uncache(self, *items):
- if not items:
- self.cache = { }
-
- for it in items:
- self.cache.pop(it, None)
-
-
-
- def waitget(self, key, maxwaittime = 60):
- wtimes = [
- 0.2] * 3 + [
- 0.5] * 2 + [
- 1]
- tries = 0
- waited = 0
- while None:
-
- try:
- val = self[key]
- return val
- except KeyError:
- pass
-
- if waited > maxwaittime:
- raise KeyError(key)
- time.sleep(wtimes[tries])
- waited += wtimes[tries]
- if tries < len(wtimes) - 1:
- tries += 1
- continue
- continue
- return None
-
-
- def getlink(self, folder):
- return PickleShareLink(self, folder)
-
-
- def __repr__(self):
- return "PickleShareDB('%s')" % self.root
-
-
-
- class PickleShareLink:
-
- def __init__(self, db, keydir):
- self.__dict__.update(locals())
-
-
- def __getattr__(self, key):
- return self.__dict__['db'][self.__dict__['keydir'] + '/' + key]
-
-
- def __setattr__(self, key, val):
- self.db[self.keydir + '/' + key] = val
-
-
- def __repr__(self):
- db = self.__dict__['db']
- keys = db.keys(self.__dict__['keydir'] + '/*')
- return ';'.join % ([], []([ Path(k).basename() for k in keys ]))
-
-
-
- def test():
- db = PickleShareDB('~/testpickleshare')
- db.clear()
- print 'Should be empty:', db.items()
- db['hello'] = 15
- db['aku ankka'] = [
- 1,
- 2,
- 313]
- db['paths/nest/ok/keyname'] = [
- 1,
- (5, 46)]
- db.hset('hash', 'aku', 12)
- db.hset('hash', 'ankka', 313)
- print '12 =', db.hget('hash', 'aku')
- print '313 =', db.hget('hash', 'ankka')
- print 'all hashed', db.hdict('hash')
- print db.keys()
- print db.keys('paths/nest/ok/k*')
- print dict(db)
- db.uncache()
- lnk = db.getlink('myobjects/test')
- lnk.foo = 2
- lnk.bar = lnk.foo + 5
- print lnk.bar
-
-
- def stress():
- db = PickleShareDB('~/fsdbtest')
- import time
- import sys
- for i in range(1000):
- for j in range(1000):
- if i % 15 == 0 and i < 200:
- if str(j) in db:
- del db[str(j)]
- continue
- continue
-
- if j % 33 == 0:
- time.sleep(0.02)
-
- db[str(j)] = db.get(str(j), []) + [
- (i, j, 'proc %d' % os.getpid())]
- db.hset('hash', j, db.hget('hash', j, 15) + 1)
-
- print i,
- sys.stdout.flush()
- if i % 10 == 0:
- db.uncache()
- continue
-
-
-
- def main():
- import textwrap
- usage = textwrap.dedent(' pickleshare - manage PickleShare databases \n \n Usage:\n \n pickleshare dump /path/to/db > dump.txt\n pickleshare load /path/to/db < dump.txt\n pickleshare test /path/to/db\n ')
- DB = PickleShareDB
- import sys
- if len(sys.argv) < 2:
- print usage
- return None
- cmd = sys.argv[1]
- args = sys.argv[2:]
- if cmd == 'dump':
- if not args:
- args = [
- '.']
-
- db = DB(args[0])
- import pprint
- pprint.pprint(db.items())
- elif cmd == 'load':
- cont = sys.stdin.read()
- db = DB(args[0])
- data = eval(cont)
- db.clear()
- for k, v in db.items():
- db[k] = v
-
- elif cmd == 'testwait':
- db = DB(args[0])
- db.clear()
- print db.waitget('250')
- elif cmd == 'test':
- test()
- stress()
-
-
- if __name__ == '__main__':
- main()
-
-