home *** CD-ROM | disk | FTP | other *** search
/ Maximum CD 2010 November / maximum-cd-2010-11.iso / DiscContents / calibre-0.7.13.msi / file_803 (.txt) < prev    next >
Encoding:
Python Compiled Bytecode  |  2010-08-06  |  21.1 KB  |  623 lines
  1. # Source Generated with Decompyle++
  2. # File: in.pyc (Python 2.6)
  3.  
  4. __license__ = 'GPL v3'
  5. __copyright__ = '2010, Kovid Goyal <kovid@kovidgoyal.net>'
  6. __docformat__ = 'restructuredtext en'
  7. import os
  8. import time
  9. from base64 import b64decode
  10. from uuid import uuid4
  11. from lxml import etree
  12. from calibre import prints, guess_type, isbytestring
  13. from calibre.devices.errors import DeviceError
  14. from calibre.devices.usbms.driver import debug_print
  15. from calibre.constants import DEBUG, preferred_encoding
  16. from calibre.ebooks.chardet import xml_to_unicode
  17. from calibre.ebooks.metadata import authors_to_string, title_sort
  18. EMPTY_CARD_CACHE = '<?xml version="1.0" encoding="UTF-8"?>\n<cache xmlns="http://www.kinoma.com/FskCache/1">\n</cache>\n'
  19. MIME_MAP = {
  20.     'lrf': 'application/x-sony-bbeb',
  21.     'lrx': 'application/x-sony-bbeb',
  22.     'rtf': 'application/rtf',
  23.     'pdf': 'application/pdf',
  24.     'txt': 'text/plain',
  25.     'epub': 'application/epub+zip' }
  26. DAY_MAP = dict(Sun = 0, Mon = 1, Tue = 2, Wed = 3, Thu = 4, Fri = 5, Sat = 6)
  27. MONTH_MAP = dict(Jan = 1, Feb = 2, Mar = 3, Apr = 4, May = 5, Jun = 6, Jul = 7, Aug = 8, Sep = 9, Oct = 10, Nov = 11, Dec = 12)
  28. INVERSE_DAY_MAP = dict(zip(DAY_MAP.values(), DAY_MAP.keys()))
  29. INVERSE_MONTH_MAP = dict(zip(MONTH_MAP.values(), MONTH_MAP.keys()))
  30.  
  31. def strptime(src):
  32.     src = src.strip()
  33.     src = src.split()
  34.     src[0] = str(DAY_MAP[src[0][:-1]]) + ','
  35.     src[2] = str(MONTH_MAP[src[2]])
  36.     return time.strptime(' '.join(src), '%w, %d %m %Y %H:%M:%S %Z')
  37.  
  38.  
  39. def strftime(epoch, zone = time.localtime):
  40.     
  41.     try:
  42.         src = time.strftime('%w, %d %m %Y %H:%M:%S GMT', zone(epoch)).split()
  43.     except:
  44.         src = time.strftime('%w, %d %m %Y %H:%M:%S GMT', zone()).split()
  45.  
  46.     src[0] = INVERSE_DAY_MAP[int(src[0][:-1])] + ','
  47.     src[2] = INVERSE_MONTH_MAP[int(src[2])]
  48.     return ' '.join(src)
  49.  
  50.  
  51. def uuid():
  52.     return str(uuid4()).replace('-', '', 1).upper()
  53.  
  54.  
  55. class XMLCache(object):
  56.     
  57.     def __init__(self, paths, prefixes, use_author_sort):
  58.         if DEBUG:
  59.             debug_print('Building XMLCache...', paths)
  60.         
  61.         self.paths = paths
  62.         self.prefixes = prefixes
  63.         self.use_author_sort = use_author_sort
  64.         parser = etree.XMLParser(recover = True)
  65.         self.roots = { }
  66.         for source_id, path in paths.items():
  67.             self.roots[source_id] = etree.fromstring(xml_to_unicode(raw, strip_encoding_pats = True, assume_utf8 = True, verbose = DEBUG)[0], parser = parser)
  68.         
  69.         recs = self.roots[0].xpath('//*[local-name()="records"]')
  70.         if not recs:
  71.             raise DeviceError('The SONY XML database is corrupted (no <records>). Try disconnecting an reconnecting your reader.')
  72.         recs
  73.         self.record_roots = { }
  74.         self.record_roots.update(self.roots)
  75.         self.record_roots[0] = recs[0]
  76.         self.detect_namespaces()
  77.         debug_print('Done building XMLCache...')
  78.  
  79.     
  80.     def purge_broken_playlist_items(self, root):
  81.         id_map = self.build_id_map(root)
  82.         for pl in root.xpath('//*[local-name()="playlist"]'):
  83.             seen = set([])
  84.             for item in list(pl):
  85.                 id_ = item.get('id', None)
  86.                 if id_ is None and id_ in seen or id_map.get(id_, None) is None:
  87.                     if DEBUG:
  88.                         if id_ is None:
  89.                             cause = 'invalid id'
  90.                         elif id_ in seen:
  91.                             cause = 'duplicate item'
  92.                         else:
  93.                             cause = 'id not found'
  94.                         prints('Purging broken playlist item:', id_, 'from playlist:', pl.get('title', None), 'because:', cause)
  95.                     
  96.                     item.getparent().remove(item)
  97.                     continue
  98.                 
  99.                 seen.add(id_)
  100.             
  101.         
  102.  
  103.     
  104.     def prune_empty_playlists(self):
  105.         for i, root in self.record_roots.items():
  106.             self.purge_broken_playlist_items(root)
  107.             for playlist in root.xpath('//*[local-name()="playlist"]'):
  108.                 if len(playlist) == 0 or not playlist.get('title', None):
  109.                     if DEBUG:
  110.                         debug_print('Removing playlist id:', playlist.get('id', None), playlist.get('title', None))
  111.                     
  112.                     playlist.getparent().remove(playlist)
  113.                     continue
  114.             
  115.         
  116.  
  117.     
  118.     def ensure_unique_playlist_titles(self):
  119.         for i, root in self.record_roots.items():
  120.             seen = set([])
  121.             for playlist in root.xpath('//*[local-name()="playlist"]'):
  122.                 title = playlist.get('title', None)
  123.                 if title is None:
  124.                     title = _('Unnamed')
  125.                     playlist.set('title', title)
  126.                 
  127.                 if title in seen:
  128.                     for i in range(2, 1000):
  129.                         if title + str(i) not in seen:
  130.                             title = title + str(i)
  131.                             playlist.set('title', title)
  132.                             seen.add(title)
  133.                             break
  134.                             continue
  135.                     
  136.                 seen.add(title)
  137.             
  138.         
  139.  
  140.     
  141.     def build_id_playlist_map(self, bl_index):
  142.         debug_print('Start build_id_playlist_map')
  143.         self.ensure_unique_playlist_titles()
  144.         self.prune_empty_playlists()
  145.         debug_print('after cleaning playlists')
  146.         root = self.record_roots[bl_index]
  147.         if root is None:
  148.             return None
  149.         id_map = self.build_id_map(root)
  150.         playlist_map = { }
  151.         for playlist in root.xpath('//*[local-name()="playlist"]'):
  152.             name = playlist.get('title')
  153.             if name is None:
  154.                 debug_print('build_id_playlist_map: unnamed playlist!')
  155.                 continue
  156.             
  157.             for item in playlist:
  158.                 id_ = item.get('id', None)
  159.                 if id_ is None:
  160.                     debug_print('build_id_playlist_map: id_ is None!')
  161.                     continue
  162.                 
  163.                 bk = id_map.get(id_, None)
  164.                 if bk is None:
  165.                     debug_print('build_id_playlist_map: book is None!', id_)
  166.                     continue
  167.                 
  168.                 lpath = bk.get('path', None)
  169.                 if lpath is None:
  170.                     debug_print('build_id_playlist_map: lpath is None!', id_)
  171.                     continue
  172.                 
  173.                 if lpath not in playlist_map:
  174.                     playlist_map[lpath] = []
  175.                 
  176.                 playlist_map[lpath].append(name)
  177.             
  178.         
  179.         debug_print('Finish build_id_playlist_map. Found', len(playlist_map))
  180.         return playlist_map
  181.  
  182.     
  183.     def reset_existing_playlists_map(self):
  184.         self._playlist_to_playlist_id_map = { }
  185.  
  186.     
  187.     def get_or_create_playlist(self, bl_idx, title):
  188.         root = self.record_roots[bl_idx]
  189.         if bl_idx not in self._playlist_to_playlist_id_map:
  190.             self._playlist_to_playlist_id_map[bl_idx] = { }
  191.             for playlist in root.xpath('//*[local-name()="playlist"]'):
  192.                 pl_title = playlist.get('title', None)
  193.                 if pl_title is not None:
  194.                     self._playlist_to_playlist_id_map[bl_idx][pl_title] = playlist
  195.                     continue
  196.             
  197.         
  198.         if title in self._playlist_to_playlist_id_map[bl_idx]:
  199.             return self._playlist_to_playlist_id_map[bl_idx][title]
  200.         debug_print('Creating playlist:', title)
  201.         ans = root.makeelement('{%s}playlist' % self.namespaces[bl_idx], nsmap = root.nsmap, attrib = {
  202.             'uuid': uuid(),
  203.             'title': title,
  204.             'id': str(self.max_id(root) + 1),
  205.             'sourceid': '1' })
  206.         root.append(ans)
  207.         self._playlist_to_playlist_id_map[bl_idx][title] = ans
  208.         return ans
  209.  
  210.     
  211.     def fix_ids(self):
  212.         debug_print('Running fix_ids()')
  213.         
  214.         def ensure_numeric_ids(root):
  215.             idmap = { }
  216.             for x in root.xpath('child::*[@id]'):
  217.                 id_ = x.get('id')
  218.                 
  219.                 try:
  220.                     id_ = int(id_)
  221.                 continue
  222.                 x.set('id', '-1')
  223.                 idmap[id_] = '-1'
  224.                 continue
  225.  
  226.             
  227.             if DEBUG and idmap:
  228.                 debug_print('Found non numeric ids:')
  229.                 debug_print(list(idmap.keys()))
  230.             
  231.             return idmap
  232.  
  233.         
  234.         def remap_playlist_references(root, idmap):
  235.             for playlist in root.xpath('//*[local-name()="playlist"]'):
  236.                 for item in playlist.xpath('descendant::*[@id and local-name()="item"]'):
  237.                     id_ = item.get('id')
  238.                     if id_ in idmap:
  239.                         item.set('id', idmap[id_])
  240.                         if DEBUG:
  241.                             debug_print('Remapping id %s to %s' % (id_, idmap[id_]))
  242.                         
  243.                     DEBUG
  244.                 
  245.             
  246.  
  247.         
  248.         def ensure_media_xml_base_ids(root):
  249.             for num, tag in enumerate(('library', 'watchSpecial')):
  250.                 for x in root.xpath('//*[local-name()="%s"]' % tag):
  251.                     x.set('id', str(num))
  252.                 
  253.             
  254.  
  255.         
  256.         def rebase_ids(root, base, sourceid, pl_sourceid):
  257.             for item in root.xpath('//*[@sourceid]'):
  258.                 sid = None if item.tag.endswith('playlist') else sourceid
  259.                 item.set('sourceid', str(sid))
  260.             
  261.             items = root.xpath('child::*[@id]')
  262.             items.sort(cmp = (lambda x, y: cmp(int(x.get('id')), int(y.get('id')))))
  263.             idmap = { }
  264.             for i, item in enumerate(items):
  265.                 old = int(item.get('id'))
  266.                 new = base + i
  267.                 if old != new:
  268.                     item.set('id', str(new))
  269.                     idmap[str(old)] = str(new)
  270.                     continue
  271.             
  272.             return idmap
  273.  
  274.         self.prune_empty_playlists()
  275.         for i in sorted(self.roots.keys()):
  276.             root = self.record_roots[i]
  277.             if i == 0:
  278.                 ensure_media_xml_base_ids(root)
  279.             
  280.             idmap = ensure_numeric_ids(root)
  281.             if len(idmap) > 0:
  282.                 debug_print('fix_ids: found some non-numeric ids')
  283.                 remap_playlist_references(root, idmap)
  284.             
  285.             if i == 0:
  286.                 (sourceid, playlist_sid) = (1, 0)
  287.                 base = 0
  288.             else:
  289.                 previous = i - 1
  290.                 if previous not in self.roots:
  291.                     previous = 0
  292.                 
  293.                 max_id = self.max_id(self.roots[previous])
  294.                 sourceid = playlist_sid = max_id + 1
  295.                 base = max_id + 2
  296.             idmap = rebase_ids(root, base, sourceid, playlist_sid)
  297.             remap_playlist_references(root, idmap)
  298.         
  299.         last_bl = max(self.roots.keys())
  300.         max_id = self.max_id(self.roots[last_bl])
  301.         self.roots[0].set('nextID', str(max_id + 1))
  302.         debug_print('Finished running fix_ids()')
  303.  
  304.     
  305.     def update_booklist(self, bl, bl_index):
  306.         if bl_index not in self.record_roots:
  307.             return None
  308.         debug_print('Updating JSON cache:', bl_index)
  309.         playlist_map = self.build_id_playlist_map(bl_index)
  310.         root = self.record_roots[bl_index]
  311.         lpath_map = self.build_lpath_map(root)
  312.         for book in bl:
  313.             record = lpath_map.get(book.lpath, None)
  314.             if record is not None:
  315.                 title = record.get('title', None)
  316.                 if title is not None and title != book.title:
  317.                     debug_print('Renaming title', book.title, 'to', title)
  318.                     book.title = title
  319.                 
  320.                 for thumbnail in record.xpath('descendant::*[local-name()="thumbnail"]'):
  321.                     for img in thumbnail.xpath('descendant::*[local-name()="jpeg"]|descendant::*[local-name()="png"]'):
  322.                         if img.text:
  323.                             
  324.                             try:
  325.                                 raw = b64decode(img.text.strip())
  326.                             except:
  327.                                 continue
  328.  
  329.                             book.thumbnail = raw
  330.                             break
  331.                             continue
  332.                     
  333.                 
  334.                 book.device_collections = playlist_map.get(book.lpath, [])
  335.                 continue
  336.         
  337.         debug_print('Finished updating JSON cache:', bl_index)
  338.  
  339.     
  340.     def update(self, booklists, collections_attributes):
  341.         debug_print('Starting update', collections_attributes)
  342.         for i, booklist in booklists.items():
  343.             playlist_map = self.build_id_playlist_map(i)
  344.             debug_print('Updating XML Cache:', i)
  345.             root = self.record_roots[i]
  346.             lpath_map = self.build_lpath_map(root)
  347.             gtz_count = ltz_count = 0
  348.             for book in booklist:
  349.                 path = os.path.join(self.prefixes[i], *book.lpath.split('/'))
  350.                 record = lpath_map.get(book.lpath, None)
  351.                 if record is None:
  352.                     record = self.create_text_record(root, i, book.lpath)
  353.                 
  354.                 (gtz_count, ltz_count) = self.update_text_record(record, book, path, i, gtz_count, ltz_count)
  355.                 if book.device_collections is None:
  356.                     book.device_collections = []
  357.                 
  358.                 book.device_collections = playlist_map.get(book.lpath, [])
  359.             
  360.             debug_print('Timezone votes: %d GMT, %d LTZ' % (gtz_count, ltz_count))
  361.             self.update_playlists(i, root, booklist, collections_attributes)
  362.         
  363.         debug_print('In update/ Starting refresh of device_collections')
  364.         for i, booklist in booklists.items():
  365.             playlist_map = self.build_id_playlist_map(i)
  366.             for book in booklist:
  367.                 book.device_collections = playlist_map.get(book.lpath, [])
  368.             
  369.         
  370.         self.fix_ids()
  371.         debug_print('Finished update')
  372.  
  373.     
  374.     def rebuild_collections(self, booklist, bl_index):
  375.         if bl_index not in self.record_roots:
  376.             return None
  377.         root = self.record_roots[bl_index]
  378.         self.update_playlists(bl_index, root, booklist, [])
  379.         self.fix_ids()
  380.  
  381.     
  382.     def update_playlists(self, bl_index, root, booklist, collections_attributes):
  383.         debug_print('Starting update_playlists', collections_attributes, bl_index)
  384.         self.reset_existing_playlists_map()
  385.         collections = booklist.get_collections(collections_attributes)
  386.         lpath_map = self.build_lpath_map(root)
  387.         debug_print('update_playlists: finished building maps')
  388.         for category, books in collections.items():
  389.             records = [ lpath_map.get(b.lpath, None) for b in books ]
  390.             records = _[2]
  391.             for rec in records:
  392.                 if rec.get('id', None) is None:
  393.                     rec.set('id', str(self.max_id(root) + 1))
  394.                     continue
  395.                 []
  396.             
  397.             ids = [ x.get('id', None) for x in records ]
  398.             playlist = self.get_or_create_playlist(bl_index, category)
  399.             playlist_ids = []
  400.             for item in playlist:
  401.                 id_ = item.get('id', None)
  402.                 if id_ is not None:
  403.                     playlist_ids.append(id_)
  404.                     continue
  405.                 [] if None in ids else []
  406.             
  407.             for item in list(playlist):
  408.                 playlist.remove(item)
  409.             
  410.             extra_ids = _[5]
  411.             for id_ in ids + extra_ids:
  412.                 item = playlist.makeelement('{%s}item' % self.namespaces[bl_index], nsmap = playlist.nsmap, attrib = {
  413.                     'id': id_ })
  414.                 playlist.append(item)
  415.             
  416.         
  417.         for playlist in root.xpath('//*[local-name()="playlist"]'):
  418.             title = playlist.get('title', None)
  419.             books = collections[title]
  420.             records = [ lpath_map.get(b.lpath, None) for b in books ]
  421.             records = _[7]
  422.             ids = [ x.get('id', None) for x in records ]
  423.             ids = _[9]
  424.             for item in list(playlist):
  425.                 if item.get('id', None) not in ids:
  426.                     playlist.remove(item)
  427.                     continue
  428.                 [] if DEBUG else []
  429.             
  430.         
  431.         debug_print('Finishing update_playlists')
  432.  
  433.     
  434.     def create_text_record(self, root, bl_id, lpath):
  435.         namespace = self.namespaces[bl_id]
  436.         id_ = self.max_id(root) + 1
  437.         attrib = {
  438.             'page': '0',
  439.             'part': '0',
  440.             'pageOffset': '0',
  441.             'scale': '0',
  442.             'id': str(id_),
  443.             'sourceid': '1',
  444.             'path': lpath }
  445.         ans = root.makeelement('{%s}text' % namespace, attrib = attrib, nsmap = root.nsmap)
  446.         root.append(ans)
  447.         return ans
  448.  
  449.     
  450.     def update_text_record(self, record, book, path, bl_index, gtz_count, ltz_count):
  451.         timestamp = os.path.getmtime(path)
  452.         rec_date = record.get('date', None)
  453.         
  454.         def clean(x):
  455.             if isbytestring(x):
  456.                 x = x.decode(preferred_encoding, 'replace')
  457.             
  458.             x.replace(u'\x00', '')
  459.             return x
  460.  
  461.         if not getattr(book, '_new_book', False):
  462.             if strftime(timestamp, zone = time.gmtime) == rec_date:
  463.                 gtz_count += 1
  464.             elif strftime(timestamp, zone = time.localtime) == rec_date:
  465.                 ltz_count += 1
  466.             
  467.         elif ltz_count >= gtz_count:
  468.             tz = time.localtime
  469.             debug_print('Using localtime TZ for new book', book.lpath)
  470.         else:
  471.             tz = time.gmtime
  472.             debug_print('Using GMT TZ for new book', book.lpath)
  473.         date = strftime(timestamp, zone = tz)
  474.         record.set('date', clean(date))
  475.         record.set('size', clean(str(os.stat(path).st_size)))
  476.         title = None if book.title else _('Unknown')
  477.         record.set('title', clean(title))
  478.         ts = book.title_sort
  479.         if not ts:
  480.             ts = title_sort(title)
  481.         
  482.         record.set('titleSorter', clean(ts))
  483.         if self.use_author_sort and book.author_sort is not None:
  484.             record.set('author', clean(book.author_sort))
  485.         else:
  486.             record.set('author', clean(authors_to_string(book.authors)))
  487.         ext = os.path.splitext(path)[1]
  488.         if ext:
  489.             ext = ext[1:].lower()
  490.             mime = MIME_MAP.get(ext, None)
  491.             if mime is None:
  492.                 mime = guess_type('a.' + ext)[0]
  493.             
  494.             if mime is not None:
  495.                 record.set('mime', clean(mime))
  496.             
  497.         
  498.         if 'sourceid' not in record.attrib:
  499.             record.set('sourceid', '1')
  500.         
  501.         if 'id' not in record.attrib:
  502.             num = self.max_id(record.getroottree().getroot())
  503.             record.set('id', str(num + 1))
  504.         
  505.         return (gtz_count, ltz_count)
  506.  
  507.     
  508.     def cleanup_whitespace(self, bl_index):
  509.         root = self.record_roots[bl_index]
  510.         level = None if bl_index == 0 else 1
  511.         if len(root) > 0:
  512.             root.text = '\n' + '\t' * level
  513.             for child in root:
  514.                 child.tail = '\n' + '\t' * level
  515.                 if len(child) > 0:
  516.                     child.text = '\n' + '\t' * (level + 1)
  517.                     for gc in child:
  518.                         gc.tail = '\n' + '\t' * (level + 1)
  519.                     
  520.                     child.iterchildren(reversed = True).next().tail = '\n' + '\t' * level
  521.                     continue
  522.             
  523.             root.iterchildren(reversed = True).next().tail = '\n' + '\t' * (level - 1)
  524.         
  525.  
  526.     
  527.     def move_playlists_to_bottom(self):
  528.         for root in self.record_roots.values():
  529.             seen = []
  530.             for pl in root.xpath('//*[local-name()="playlist"]'):
  531.                 pl.getparent().remove(pl)
  532.                 seen.append(pl)
  533.             
  534.             for pl in seen:
  535.                 root.append(pl)
  536.             
  537.         
  538.  
  539.     
  540.     def write(self):
  541.         for i, path in self.paths.items():
  542.             self.move_playlists_to_bottom()
  543.             self.cleanup_whitespace(i)
  544.             raw = etree.tostring(self.roots[i], encoding = 'UTF-8', xml_declaration = True)
  545.             raw = raw.replace("<?xml version='1.0' encoding='UTF-8'?>", '<?xml version="1.0" encoding="UTF-8"?>')
  546.             
  547.             try:
  548.                 f = _[1]
  549.                 f.write(raw)
  550.             finally:
  551.                 pass
  552.  
  553.         
  554.  
  555.     
  556.     def build_lpath_map(self, root):
  557.         m = { }
  558.         for bk in root.xpath('//*[local-name()="text"]'):
  559.             m[bk.get('path')] = bk
  560.         
  561.         return m
  562.  
  563.     
  564.     def build_id_map(self, root):
  565.         m = { }
  566.         for bk in root.xpath('//*[local-name()="text"]'):
  567.             m[bk.get('id')] = bk
  568.         
  569.         return m
  570.  
  571.     
  572.     def book_by_lpath(self, lpath, root):
  573.         matches = root.xpath(u'//*[local-name()="text" and @path="%s"]' % lpath)
  574.         if matches:
  575.             return matches[0]
  576.  
  577.     
  578.     def max_id(self, root):
  579.         ans = -1
  580.         for x in root.xpath('//*[@id]'):
  581.             id_ = x.get('id')
  582.             
  583.             try:
  584.                 num = int(id_)
  585.                 if num > ans:
  586.                     ans = num
  587.             continue
  588.             continue
  589.             continue
  590.  
  591.         
  592.         return ans
  593.  
  594.     
  595.     def detect_namespaces(self):
  596.         self.nsmaps = { }
  597.         for i, root in self.roots.items():
  598.             self.nsmaps[i] = root.nsmap
  599.         
  600.         self.namespaces = { }
  601.         for i in self.roots:
  602.             for c in ('library', 'text', 'image', 'playlist', 'thumbnail', 'watchSpecial'):
  603.                 matches = self.record_roots[i].xpath('//*[local-name()="%s"]' % c)
  604.                 if matches:
  605.                     e = matches[0]
  606.                     self.namespaces[i] = e.nsmap[e.prefix]
  607.                     break
  608.                     continue
  609.             
  610.             if i not in self.namespaces:
  611.                 ns = self.nsmaps[i].get(None, None)
  612.                 for prefix in self.nsmaps[i]:
  613.                     if prefix is not None:
  614.                         ns = self.nsmaps[i][prefix]
  615.                         break
  616.                         continue
  617.                 
  618.                 self.namespaces[i] = ns
  619.                 continue
  620.         
  621.  
  622.  
  623.