Maximum CD 2011 January/maximum-cd-2011-01.iso/DiscContents/calibre-0.7.26.msi/file_1405

Python Compiled Bytecode | 2010-10-31 | 122.6 KB | 3,885 lines

# Source Generated with Decompyle++
# File: in.pyc (Python 2.6)

__license__ = 'GPL v3'
__copyright__ = '2010, Greg Riker <griker at hotmail.com>'
import datetime
import htmlentitydefs
import os
import re
import shutil
import codecs
from collections import namedtuple
from copy import deepcopy
from xml.sax.saxutils import escape
from calibre import prints, prepare_string_for_xml, strftime
from calibre.constants import preferred_encoding
from calibre.customize import CatalogPlugin
from calibre.customize.conversion import OptionRecommendation, DummyReporter
from calibre.ebooks.BeautifulSoup import BeautifulSoup, BeautifulStoneSoup, Tag, NavigableString
from calibre.ptempfile import PersistentTemporaryDirectory
from calibre.utils.date import isoformat, now as nowf
from calibre.utils.logging import default_log as log
FIELDS = [
    'all',
    'author_sort',
    'authors',
    'comments',
    'cover',
    'formats',
    'id',
    'isbn',
    'ondevice',
    'pubdate',
    'publisher',
    'rating',
    'series_index',
    'series',
    'size',
    'tags',
    'timestamp',
    'title',
    'uuid']
TEMPLATE_ALLOWED_FIELDS = [
    'author_sort',
    'authors',
    'id',
    'isbn',
    'pubdate',
    'publisher',
    'series_index',
    'series',
    'tags',
    'timestamp',
    'title',
    'uuid']

class CSV_XML(CatalogPlugin):
    Option = namedtuple('Option', 'option, default, dest, action, help')
    name = 'Catalog_CSV_XML'
    description = 'CSV/XML catalog generator'
    supported_platforms = [
        'windows',
        'osx',
        'linux']
    author = 'Greg Riker'
    version = (1, 0, 0)
    file_types = set([
        'csv',
        'xml'])
    cli_options = [
        Option('--fields', default = 'all', dest = 'fields', action = None, help = _("The fields to output when cataloging books in the database.  Should be a comma-separated list of fields.\nAvailable fields: %s.\nDefault: '%%default'\nApplies to: CSV, XML output formats") % ', '.join(FIELDS)),
        Option('--sort-by', default = 'id', dest = 'sort_by', action = None, help = _("Output field to sort on.\nAvailable fields: author_sort, id, rating, size, timestamp, title.\nDefault: '%default'\nApplies to: CSV, XML output formats"))]

    def run(self, path_to_output, opts, db, notification = DummyReporter()):
        self.fmt = path_to_output.rpartition('.')[2]
        self.notification = notification
        if opts.verbose:
            opts_dict = vars(opts)
            log('%s(): Generating %s' % (self.name, self.fmt))
            if opts.connected_device['is_device_connected']:
                log(' connected_device: %s' % opts.connected_device['name'])

            if opts_dict['search_text']:
                log(" --search='%s'" % opts_dict['search_text'])

            if opts_dict['ids']:
                log(' Book count: %d' % len(opts_dict['ids']))
                if opts_dict['search_text']:
                    log(' (--search ignored when a subset of the database is specified)')


            if opts_dict['fields']:
                if opts_dict['fields'] == 'all':
                    log(' Fields: %s' % ', '.join(FIELDS[1:]))
                else:
                    log(' Fields: %s' % opts_dict['fields'])


        if opts.ids:
            opts.search_text = None

        data = self.search_sort_db(db, opts)
        if not len(data):
            log.error("\nNo matching database entries for search criteria '%s'" % opts.search_text)

        fields = self.get_output_fields(opts)
        if opts.connected_device['is_device_connected'] and 'ondevice' in fields:
            for entry in data:
                entry['ondevice'] = db.catalog_plugin_on_device_temp_mapping[entry['id']]['ondevice']


        if self.fmt == 'csv':
            outfile = codecs.open(path_to_output, 'w', 'utf8')
            outfile.write(u'%s\n' % u','.join(fields))
            for entry in data:
                outstr = []
                for field in fields:
                    item = entry[field]
                    if item is None:
                        outstr.append('""')
                        continue
                    elif field == 'formats':
                        fmt_list = []
                        for format in item:
                            fmt_list.append(format.rpartition('.')[2].lower())

                        item = ', '.join(fmt_list)
                    elif field in ('authors', 'tags'):
                        item = ', '.join(item)
                    elif field == 'isbn':
                        item = u'%s' % re.sub('[\\D]', '', item)
                    elif field in ('pubdate', 'timestamp'):
                        item = isoformat(item)
                    elif field == 'comments':
                        item = item.replace(u'\r\n', u' ')
                        item = item.replace(u'\n', u' ')

                    outstr.append(u'"%s"' % unicode(item).replace('"', '""'))

                outfile.write(u','.join(outstr) + u'\n')

            outfile.close()
        elif self.fmt == 'xml':
            etree = etree
            import lxml
            E = E
            import lxml.builder
            root = E.calibredb()
            for r in data:
                record = E.record()
                root.append(record)
                for field in ('id', 'uuid', 'title', 'publisher', 'rating', 'size', 'isbn', 'ondevice'):
                    if field in fields:
                        val = r[field]
                        if not val:
                            continue

                        if not isinstance(val, (str, unicode)):
                            val = unicode(val)

                        item = getattr(E, field)(val)
                        record.append(item)
                        continue

                if 'authors' in fields:
                    aus = E.authors(sort = r['author_sort'])
                    for au in r['authors']:
                        aus.append(E.author(au))

                    record.append(aus)

                for field in ('timestamp', 'pubdate'):
                    if field in fields:
                        record.append(getattr(E, field)(r[field].isoformat()))
                        continue

                if 'tags' in fields and r['tags']:
                    tags = E.tags()
                    for tag in r['tags']:
                        tags.append(E.tag(tag))

                    record.append(tags)

                if 'comments' in fields and r['comments']:
                    record.append(E.comments(r['comments']))

                if 'series' in fields and r['series']:
                    record.append(E.series(r['series'], index = str(r['series_index'])))

                if 'cover' in fields and r['cover']:
                    record.append(E.cover(r['cover'].replace(os.sep, '/')))

                if 'formats' in fields and r['formats']:
                    fmt = E.formats()
                    for f in r['formats']:
                        fmt.append(E.format(f.replace(os.sep, '/')))

                    record.append(fmt)
                    continue


            try:
                f = _[1]
                f.write(etree.tostring(root, encoding = 'utf-8', xml_declaration = True, pretty_print = True))
            finally:
                pass





class BIBTEX(CatalogPlugin):
    Option = namedtuple('Option', 'option, default, dest, action, help')
    name = 'Catalog_BIBTEX'
    description = 'BIBTEX catalog generator'
    supported_platforms = [
        'windows',
        'osx',
        'linux']
    author = 'Sengian'
    version = (1, 0, 0)
    file_types = set([
        'bib'])
    cli_options = [
        Option('--fields', default = 'all', dest = 'fields', action = None, help = _("The fields to output when cataloging books in the database.  Should be a comma-separated list of fields.\nAvailable fields: %s.\nDefault: '%%default'\nApplies to: BIBTEX output format") % ', '.join(FIELDS)),
        Option('--sort-by', default = 'id', dest = 'sort_by', action = None, help = _("Output field to sort on.\nAvailable fields: author_sort, id, rating, size, timestamp, title.\nDefault: '%default'\nApplies to: BIBTEX output format")),
        Option('--create-citation', default = 'True', dest = 'impcit', action = None, help = _("Create a citation for BibTeX entries.\nBoolean value: True, False\nDefault: '%default'\nApplies to: BIBTEX output format")),
        Option('--citation-template', default = '{authors}{id}', dest = 'bib_cit', action = None, help = _("The template for citation creation from database fields.\n Should be a template with {} enclosed fields.\nAvailable fields: %s.\nDefault: '%%default'\nApplies to: BIBTEX output format") % ', '.join(TEMPLATE_ALLOWED_FIELDS)),
        Option('--choose-encoding', default = 'utf8', dest = 'bibfile_enc', action = None, help = _("BibTeX file encoding output.\nAvailable types: utf8, cp1252, ascii.\nDefault: '%default'\nApplies to: BIBTEX output format")),
        Option('--choose-encoding-configuration', default = 'strict', dest = 'bibfile_enctag', action = None, help = _("BibTeX file encoding flag.\nAvailable types: strict, replace, ignore, backslashreplace.\nDefault: '%default'\nApplies to: BIBTEX output format")),
        Option('--entry-type', default = 'book', dest = 'bib_entry', action = None, help = _("Entry type for BibTeX catalog.\nAvailable types: book, misc, mixed.\nDefault: '%default'\nApplies to: BIBTEX output format"))]

    def run(self, path_to_output, opts, db, notification = DummyReporter()):
        StringType = StringType
        UnicodeType = UnicodeType
        import types
        preprocess_template = preprocess_template
        import calibre.library.save_to_disk
        bibtex_author_format = bibtex_author_format
        utf8ToBibtex = utf8ToBibtex
        ValidateCitationKey = ValidateCitationKey
        import calibre.utils.bibtex

        def create_bibtex_entry(entry, fields, mode, template_citation, asccii_bibtex = None, citation_bibtex = (None, None, True, True)):
            bibtex_entry = []
            if mode != 'misc' and check_entry_book_valid(entry):
                bibtex_entry.append(u'@book{')
            elif mode != 'book':
                bibtex_entry.append(u'@misc{')
            else:
                return ''
            if check_entry_book_valid(entry):
                bibtex_entry.append(make_bibtex_citation(entry, template_citation, asccii_bibtex))
                bibtex_entry = [
                    u' '.join(bibtex_entry)]

            for field in fields:
                item = entry[field]
                if item is None:
                    continue


                try:
                    if len(item) == 0:
                        continue
                except TypeError:
                    pass

                if field == 'authors':
                    bibtex_entry.append(u'author = "%s"' % bibtex_author_format(item))
                    continue
                if field in ('title', 'publisher', 'cover', 'uuid', 'author_sort', 'series'):
                    bibtex_entry.append(u'%s = "%s"' % (field, utf8ToBibtex(item, asccii_bibtex)))
                    continue
                if field == 'id':
                    bibtex_entry.append(u'calibreid = "%s"' % int(item))
                    continue
                if field == 'rating':
                    bibtex_entry.append(u'rating = "%s"' % int(item))
                    continue
                if field == 'size':
                    bibtex_entry.append(u'%s = "%s octets"' % (field, int(item)))
                    continue
                if field == 'tags':
                    bibtex_entry.append(u'tags = "%s"' % utf8ToBibtex(u', '.join(item), asccii_bibtex))
                    continue
                if field == 'comments':
                    item = item.replace(u'\r\n', u' ')
                    item = item.replace(u'\n', u' ')
                    bibtex_entry.append(u'note = "%s"' % utf8ToBibtex(item, asccii_bibtex))
                    continue
                if field == 'isbn':
                    bibtex_entry.append(u'isbn = "%s"' % re.sub(u'[\\D]', u'', item))
                    continue
                if field == 'formats':
                    item = []([ format.rpartition('.')[2].lower() for format in item ])
                    bibtex_entry.append(u'formats = "%s"' % item)
                    continue
                []
                if field == 'series_index':
                    bibtex_entry.append(u'volume = "%s"' % int(item))
                    continue
                u', '.join
                if field == 'timestamp':
                    bibtex_entry.append(u'timestamp = "%s"' % isoformat(item).partition('T')[0])
                    continue
                if field == 'pubdate':
                    bibtex_entry.append(u'year = "%s"' % item.year)
                    bibtex_entry.append(u'month = "%s"' % utf8ToBibtex(strftime('%b', item), asccii_bibtex))
                    continue

            bibtex_entry = u',\n    '.join(bibtex_entry)
            bibtex_entry += u' }\n\n'
            return bibtex_entry


        def check_entry_book_valid(entry):
            for field in [
                'title',
                'authors',
                'publisher']:
                if entry[field] is None or len(entry[field]) == 0:
                    return False

            if entry['pubdate'] is None:
                return False
            return True


        def make_bibtex_citation(entry, template_citation, asccii_bibtex):

            def tpl_replace(objtplname):
                tpl_field = re.sub(u'[\\{\\}]', u'', objtplname.group())
                if tpl_field in TEMPLATE_ALLOWED_FIELDS:
                    if tpl_field in ('pubdate', 'timestamp'):
                        tpl_field = isoformat(entry[tpl_field]).partition('T')[0]
                    elif tpl_field in ('tags', 'authors'):
                        tpl_field = entry[tpl_field][0]
                    elif tpl_field in ('id', 'series_index'):
                        tpl_field = str(entry[tpl_field])
                    else:
                        tpl_field = entry[tpl_field]
                    return tpl_field
                return u''

            if len(entry['isbn']) > 0:
                template_citation = u'%s' % re.sub(u'[\\D]', u'', entry['isbn'])
            else:
                template_citation = u'%s' % str(entry['id'])
            if asccii_bibtex:
                return ValidateCitationKey(template_citation.encode('ascii', 'replace'))
            return ValidateCitationKey(template_citation)

        self.fmt = path_to_output.rpartition('.')[2]
        self.notification = notification
        bibfile_enc = [
            'utf8',
            'cp1252',
            'ascii']
        bibfile_enctag = [
            'strict',
            'replace',
            'ignore',
            'backslashreplace']
        bib_entry = [
            'mixed',
            'misc',
            'book']

        try:
            bibfile_enc = bibfile_enc[opts.bibfile_enc]
            bibfile_enctag = bibfile_enctag[opts.bibfile_enctag]
            bib_entry = bib_entry[opts.bib_entry]
        except:
            (None, None)
            if opts.bibfile_enc in bibfile_enc:
                bibfile_enc = opts.bibfile_enc
            else:
                log(' WARNING: incorrect --choose-encoding flag, revert to default')
                bibfile_enc = bibfile_enc[0]
            if opts.bibfile_enctag in bibfile_enctag:
                bibfile_enctag = opts.bibfile_enctag
            else:
                log(' WARNING: incorrect --choose-encoding-configuration flag, revert to default')
                bibfile_enctag = bibfile_enctag[0]
            if opts.bib_entry in bib_entry:
                bib_entry = opts.bib_entry
            else:
                log(' WARNING: incorrect --entry-type flag, revert to default')
                bib_entry = bib_entry[0]

        if opts.ids:
            opts.search_text = None

        data = self.search_sort_db(db, opts)
        if not len(data):
            log.error("\nNo matching database entries for search criteria '%s'" % opts.search_text)

        fields = self.get_output_fields(opts)
        if not len(data):
            log.error("\nNo matching database entries for search criteria '%s'" % opts.search_text)

        if bibfile_enc != 'ascii':
            asccii_bibtex = False
        else:
            asccii_bibtex = True
        if isinstance(opts.impcit, (StringType, UnicodeType)):
            if opts.impcit == 'False':
                citation_bibtex = False
            elif opts.impcit == 'True':
                citation_bibtex = True
            else:
                log(' WARNING: incorrect --create-citation, revert to default')
                citation_bibtex = True
        else:
            citation_bibtex = opts.impcit
        template_citation = preprocess_template(opts.bib_cit)
        outfile = codecs.open(path_to_output, 'w', bibfile_enc, bibfile_enctag)
        nb_entries = len(data)
        if bib_entry == 'book':
            nb_books = len(filter(check_entry_book_valid, data))
            if nb_books < nb_entries:
                log(' WARNING: only %d entries in %d are book compatible' % (nb_books, nb_entries))
                nb_entries = nb_books


        outfile.write(u'%%%Calibre catalog\n%%%{0} entries in catalog\n\n'.format(nb_entries))
        outfile.write(u'@preamble{"This catalog of %d entries was generated by calibre on %s"}\n\n' % (nb_entries, nowf().strftime('%A, %d. %B %Y %H:%M').decode(preferred_encoding)))
        for entry in data:
            outfile.write(create_bibtex_entry(entry, fields, bib_entry, template_citation, asccii_bibtex, citation_bibtex))

        outfile.close()



class EPUB_MOBI(CatalogPlugin):
    Option = namedtuple('Option', 'option, default, dest, action, help')
    name = 'Catalog_EPUB_MOBI'
    description = 'EPUB/MOBI catalog generator'
    supported_platforms = [
        'windows',
        'osx',
        'linux']
    minimum_calibre_version = (0, 6, 34)
    author = 'Greg Riker'
    version = (0, 0, 1)
    file_types = set([
        'epub',
        'mobi'])
    cli_options = [
        Option('--catalog-title', default = 'My Books', dest = 'catalog_title', action = None, help = _("Title of generated catalog used as title in metadata.\nDefault: '%default'\nApplies to: ePub, MOBI output formats")),
        Option('--debug-pipeline', default = None, dest = 'debug_pipeline', action = None, help = _("Save the output from different stages of the conversion pipeline to the specified directory. Useful if you are unsure at which stage of the conversion process a bug is occurring.\nDefault: '%default'None\nApplies to: ePub, MOBI output formats")),
        Option('--exclude-genre', default = '\\[.+\\]', dest = 'exclude_genre', action = None, help = _("Regex describing tags to exclude as genres.\nDefault: '%default' excludes bracketed tags, e.g. '[<tag>]'\nApplies to: ePub, MOBI output formats")),
        Option('--exclude-tags', default = '~,' + _('Catalog'), dest = 'exclude_tags', action = None, help = _("Comma-separated list of tag words indicating book should be excluded from output.  Case-insensitive.\n--exclude-tags=skip will match 'skip this book' and 'Skip will like this'.\nDefault: '%default'\nApplies to: ePub, MOBI output formats")),
        Option('--generate-titles', default = False, dest = 'generate_titles', action = 'store_true', help = _("Include 'Titles' section in catalog.\nDefault: '%default'\nApplies to: ePub, MOBI output formats")),
        Option('--generate-series', default = False, dest = 'generate_series', action = 'store_true', help = _("Include 'Series' section in catalog.\nDefault: '%default'\nApplies to: ePub, MOBI output formats")),
        Option('--generate-recently-added', default = False, dest = 'generate_recently_added', action = 'store_true', help = _("Include 'Recently Added' section in catalog.\nDefault: '%default'\nApplies to: ePub, MOBI output formats")),
        Option('--note-tag', default = '*', dest = 'note_tag', action = None, help = _("Tag prefix for user notes, e.g. '*Jeff might enjoy reading this'.\nDefault: '%default'\nApplies to: ePub, MOBI output formats")),
        Option('--numbers-as-text', default = False, dest = 'numbers_as_text', action = None, help = _("Sort titles with leading numbers as text, e.g.,\n'2001: A Space Odyssey' sorts as \n'Two Thousand One: A Space Odyssey'.\nDefault: '%default'\nApplies to: ePub, MOBI output formats")),
        Option('--output-profile', default = None, dest = 'output_profile', action = None, help = _("Specifies the output profile.  In some cases, an output profile is required to optimize the catalog for the device.  For example, 'kindle' or 'kindle_dx' creates a structured Table of Contents with Sections and Articles.\nDefault: '%default'\nApplies to: ePub, MOBI output formats")),
        Option('--read-tag', default = '+', dest = 'read_tag', action = None, help = _("Tag indicating book has been read.\nDefault: '%default'\nApplies to: ePub, MOBI output formats"))]

    class NumberToText(object):
        ORDINALS = [
            'zeroth',
            'first',
            'second',
            'third',
            'fourth',
            'fifth',
            'sixth',
            'seventh',
            'eighth',
            'ninth']
        lessThanTwenty = [
            '<zero>',
            'one',
            'two',
            'three',
            'four',
            'five',
            'six',
            'seven',
            'eight',
            'nine',
            'ten',
            'eleven',
            'twelve',
            'thirteen',
            'fourteen',
            'fifteen',
            'sixteen',
            'seventeen',
            'eighteen',
            'nineteen']
        tens = [
            '<zero>',
            '<tens>',
            'twenty',
            'thirty',
            'forty',
            'fifty',
            'sixty',
            'seventy',
            'eighty',
            'ninety']
        hundreds = [
            '<zero>',
            'one',
            'two',
            'three',
            'four',
            'five',
            'six',
            'seven',
            'eight',
            'nine']

        def __init__(self, number, verbose = False):
            self.number = number
            self.number_as_float = 0
            self.text = ''
            self.verbose = verbose
            self.log = log
            self.numberTranslate()


        def stringFromInt(self, intToTranslate):
            tensComponentString = ''
            hundredsComponent = intToTranslate - intToTranslate % 100
            tensComponent = intToTranslate % 100
            if hundredsComponent:
                hundredsComponentString = '%s hundred' % self.hundreds[hundredsComponent / 100]
            else:
                hundredsComponentString = ''
            if tensComponent < 20:
                tensComponentString = self.lessThanTwenty[tensComponent]
            else:
                tensPart = ''
                onesPart = ''
                tensPart = self.tens[tensComponent / 10]
                onesPart = self.lessThanTwenty[tensComponent % 10]
                if intToTranslate % 10:
                    tensComponentString = '%s-%s' % (tensPart, onesPart)
                else:
                    tensComponentString = '%s' % tensPart
            result = ''
            if hundredsComponent and not tensComponent:
                result = hundredsComponentString
            elif not hundredsComponent and tensComponent:
                result = tensComponentString
            elif hundredsComponent and tensComponent:
                result = hundredsComponentString + ' ' + tensComponentString
            else:
                prints(' NumberToText.stringFromInt(): empty result translating %d' % intToTranslate)
            return result


        def numberTranslate(self):
            hundredsNumber = 0
            thousandsNumber = 0
            hundredsString = ''
            thousandsString = ''
            resultString = ''
            self.suffix = ''
            if self.verbose:
                self.log('numberTranslate(): %s' % self.number)

            if re.search('[st|nd|rd|th]', self.number):
                self.number = re.sub(',', '', self.number)
                ordinal_suffix = re.search('[\\D]', self.number)
                ordinal_number = re.sub('\\D', '', re.sub(',', '', self.number))
                if self.verbose:
                    self.log('Ordinal: %s' % ordinal_number)

                self.number_as_float = ordinal_number
                self.suffix = self.number[ordinal_suffix.start():]
                if int(ordinal_number) > 9:
                    self.text = '%s' % EPUB_MOBI.NumberToText(ordinal_number).text
                else:
                    self.text = '%s' % self.ORDINALS[int(ordinal_number)]
            elif re.search(':', self.number):
                if self.verbose:
                    self.log('Time: %s' % self.number)

                self.number_as_float = re.sub(':', '.', self.number)
                time_strings = self.number.split(':')
                hours = EPUB_MOBI.NumberToText(time_strings[0]).text
                minutes = EPUB_MOBI.NumberToText(time_strings[1]).text
                self.text = '%s-%s' % (hours.capitalize(), minutes)
            elif re.search('%', self.number):
                if self.verbose:
                    self.log('Percent: %s' % self.number)

                self.number_as_float = self.number.split('%')[0]
                self.text = EPUB_MOBI.NumberToText(self.number.replace('%', ' percent')).text
            elif re.search('\\.', self.number):
                if self.verbose:
                    self.log('Decimal: %s' % self.number)

                self.number_as_float = self.number
                decimal_strings = self.number.split('.')
                left = EPUB_MOBI.NumberToText(decimal_strings[0]).text
                right = EPUB_MOBI.NumberToText(decimal_strings[1]).text
                self.text = '%s point %s' % (left.capitalize(), right)
            elif re.search('-', self.number):
                if self.verbose:
                    self.log('Hyphenated: %s' % self.number)

                self.number_as_float = self.number.split('-')[0]
                strings = self.number.split('-')
                if re.search('[0-9]+', strings[0]):
                    left = EPUB_MOBI.NumberToText(strings[0]).text
                    right = strings[1]
                else:
                    left = strings[0]
                    right = EPUB_MOBI.NumberToText(strings[1]).text
                self.text = '%s-%s' % (left, right)
            elif re.search(',', self.number) and not re.search('[^0-9,]', self.number):
                if self.verbose:
                    self.log('Comma(s): %s' % self.number)

                self.number_as_float = re.sub(',', '', self.number)
                self.text = EPUB_MOBI.NumberToText(self.number_as_float).text
            elif re.search('[\\D]+', self.number):
                if self.verbose:
                    self.log('Hybrid: %s' % self.number)

                number_position = re.search('\\d', self.number).start()
                text_position = re.search('\\D', self.number).start()
                if number_position < text_position:
                    number = self.number[:text_position]
                    text = self.number[text_position:]
                    self.text = '%s%s' % (EPUB_MOBI.NumberToText(number).text, text)
                else:
                    text = self.number[:number_position]
                    number = self.number[number_position:]
                    self.text = '%s%s' % (text, EPUB_MOBI.NumberToText(number).text)
            elif self.verbose:
                self.log('Clean: %s' % self.number)


            try:
                self.float_as_number = float(self.number)
                number = int(self.number)
            except:
                return None

            if number > 1000000000:
                self.text = '%d out of range' % number
                return None
            if number == 1000000000:
                self.text = 'one billion'
            else:
                millionsNumber = number / 1000000
                thousandsNumber = (number - millionsNumber * 1000000) / 1000
                hundredsNumber = number - millionsNumber * 1000000 - thousandsNumber * 1000
                if self.verbose:
                    print 'Converting %s %s %s' % (millionsNumber, thousandsNumber, hundredsNumber)

                if hundredsNumber:
                    hundredsString = self.stringFromInt(hundredsNumber)

                if thousandsNumber:
                    if number > 1099 and number < 2000:
                        resultString = '%s %s' % (self.lessThanTwenty[number / 100], self.stringFromInt(number % 100))
                        self.text = resultString.strip().capitalize()
                        return None
                    thousandsString = self.stringFromInt(thousandsNumber)

                if millionsNumber:
                    millionsString = self.stringFromInt(millionsNumber)

                resultString = ''
                if millionsNumber:
                    resultString += '%s million ' % millionsString

                if thousandsNumber:
                    resultString += '%s thousand ' % thousandsString

                if hundredsNumber:
                    resultString += '%s' % hundredsString

                if not millionsNumber and not thousandsNumber and not hundredsNumber:
                    resultString = 'zero'

                if self.verbose:
                    self.log(u'resultString: %s' % resultString)

                self.text = resultString.strip().capitalize()



    class CatalogBuilder(object):
        DATE_RANGE = [
            30]

        def __init__(self, db, opts, plugin, report_progress = DummyReporter(), stylesheet = 'content/stylesheet.css'):
            self._CatalogBuilder__opts = opts
            self._CatalogBuilder__authorClip = opts.authorClip
            self._CatalogBuilder__authors = None
            self._CatalogBuilder__basename = opts.basename
            self._CatalogBuilder__bookmarked_books = None
            self._CatalogBuilder__booksByAuthor = None
            self._CatalogBuilder__booksByDateRead = None
            self._CatalogBuilder__booksByTitle = None
            self._CatalogBuilder__booksByTitle_noSeriesPrefix = None
            self._CatalogBuilder__catalogPath = PersistentTemporaryDirectory('_epub_mobi_catalog', prefix = '')
            self._CatalogBuilder__contentDir = os.path.join(self.catalogPath, 'content')
            self._CatalogBuilder__currentStep = 0
            self._CatalogBuilder__creator = opts.creator
            self._CatalogBuilder__db = db
            self._CatalogBuilder__descriptionClip = opts.descriptionClip
            self._CatalogBuilder__error = None
            self._CatalogBuilder__generateForKindle = None if self.opts.fmt == 'mobi' and self.opts.output_profile and self.opts.output_profile.startswith('kindle') else False
            self._CatalogBuilder__generateRecentlyRead = None if self.opts.generate_recently_added and self.opts.connected_kindle and self.generateForKindle else False
            self._CatalogBuilder__genres = None
            self._CatalogBuilder__genre_tags_dict = None
            self._CatalogBuilder__htmlFileList = []
            self._CatalogBuilder__markerTags = self.getMarkerTags()
            self._CatalogBuilder__ncxSoup = None
            self._CatalogBuilder__playOrder = 1
            self._CatalogBuilder__plugin = plugin
            self._CatalogBuilder__progressInt = 0
            self._CatalogBuilder__progressString = ''
            self._CatalogBuilder__reporter = report_progress
            self._CatalogBuilder__stylesheet = stylesheet
            self._CatalogBuilder__thumbs = None
            self._CatalogBuilder__thumbWidth = 0
            self._CatalogBuilder__thumbHeight = 0
            self._CatalogBuilder__title = opts.catalog_title
            self._CatalogBuilder__totalSteps = 11
            self._CatalogBuilder__useSeriesPrefixInTitlesSection = False
            self._CatalogBuilder__verbose = opts.verbose
            if self.opts.generate_titles:
                self._CatalogBuilder__totalSteps += 2

            if self.opts.generate_recently_added:
                self._CatalogBuilder__totalSteps += 2
                if self.generateRecentlyRead:
                    self._CatalogBuilder__totalSteps += 2


            if self.opts.generate_series:
                self._CatalogBuilder__totalSteps += 2


        if True:

            def authorClip(self):

                def fget(self):
                    return self._CatalogBuilder__authorClip


                def fset(self, val):
                    self._CatalogBuilder__authorClip = val

                return property(fget = fget, fset = fset)

            authorClip = dynamic_property(authorClip)

            def authors(self):

                def fget(self):
                    return self._CatalogBuilder__authors


                def fset(self, val):
                    self._CatalogBuilder__authors = val

                return property(fget = fget, fset = fset)

            authors = dynamic_property(authors)

            def basename(self):

                def fget(self):
                    return self._CatalogBuilder__basename


                def fset(self, val):
                    self._CatalogBuilder__basename = val

                return property(fget = fget, fset = fset)

            basename = dynamic_property(basename)

            def bookmarked_books(self):

                def fget(self):
                    return self._CatalogBuilder__bookmarked_books


                def fset(self, val):
                    self._CatalogBuilder__bookmarked_books = val

                return property(fget = fget, fset = fset)

            bookmarked_books = dynamic_property(bookmarked_books)

            def booksByAuthor(self):

                def fget(self):
                    return self._CatalogBuilder__booksByAuthor


                def fset(self, val):
                    self._CatalogBuilder__booksByAuthor = val

                return property(fget = fget, fset = fset)

            booksByAuthor = dynamic_property(booksByAuthor)

            def booksByDateRead(self):

                def fget(self):
                    return self._CatalogBuilder__booksByDateRead


                def fset(self, val):
                    self._CatalogBuilder__booksByDateRead = val

                return property(fget = fget, fset = fset)

            booksByDateRead = dynamic_property(booksByDateRead)

            def booksByTitle(self):

                def fget(self):
                    return self._CatalogBuilder__booksByTitle


                def fset(self, val):
                    self._CatalogBuilder__booksByTitle = val

                return property(fget = fget, fset = fset)

            booksByTitle = dynamic_property(booksByTitle)

            def booksByTitle_noSeriesPrefix(self):

                def fget(self):
                    return self._CatalogBuilder__booksByTitle_noSeriesPrefix


                def fset(self, val):
                    self._CatalogBuilder__booksByTitle_noSeriesPrefix = val

                return property(fget = fget, fset = fset)

            booksByTitle_noSeriesPrefix = dynamic_property(booksByTitle_noSeriesPrefix)

            def catalogPath(self):

                def fget(self):
                    return self._CatalogBuilder__catalogPath


                def fset(self, val):
                    self._CatalogBuilder__catalogPath = val

                return property(fget = fget, fset = fset)

            catalogPath = dynamic_property(catalogPath)

            def contentDir(self):

                def fget(self):
                    return self._CatalogBuilder__contentDir


                def fset(self, val):
                    self._CatalogBuilder__contentDir = val

                return property(fget = fget, fset = fset)

            contentDir = dynamic_property(contentDir)

            def currentStep(self):

                def fget(self):
                    return self._CatalogBuilder__currentStep


                def fset(self, val):
                    self._CatalogBuilder__currentStep = val

                return property(fget = fget, fset = fset)

            currentStep = dynamic_property(currentStep)

            def creator(self):

                def fget(self):
                    return self._CatalogBuilder__creator


                def fset(self, val):
                    self._CatalogBuilder__creator = val

                return property(fget = fget, fset = fset)

            creator = dynamic_property(creator)

            def db(self):

                def fget(self):
                    return self._CatalogBuilder__db

                return property(fget = fget)

            db = dynamic_property(db)

            def descriptionClip(self):

                def fget(self):
                    return self._CatalogBuilder__descriptionClip


                def fset(self, val):
                    self._CatalogBuilder__descriptionClip = val

                return property(fget = fget, fset = fset)

            descriptionClip = dynamic_property(descriptionClip)

            def error(self):

                def fget(self):
                    return self._CatalogBuilder__error

                return property(fget = fget)

            error = dynamic_property(error)

            def generateForKindle(self):

                def fget(self):
                    return self._CatalogBuilder__generateForKindle


                def fset(self, val):
                    self._CatalogBuilder__generateForKindle = val

                return property(fget = fget, fset = fset)

            generateForKindle = dynamic_property(generateForKindle)

            def generateRecentlyRead(self):

                def fget(self):
                    return self._CatalogBuilder__generateRecentlyRead


                def fset(self, val):
                    self._CatalogBuilder__generateRecentlyRead = val

                return property(fget = fget, fset = fset)

            generateRecentlyRead = dynamic_property(generateRecentlyRead)

            def genres(self):

                def fget(self):
                    return self._CatalogBuilder__genres


                def fset(self, val):
                    self._CatalogBuilder__genres = val

                return property(fget = fget, fset = fset)

            genres = dynamic_property(genres)

            def genre_tags_dict(self):

                def fget(self):
                    return self._CatalogBuilder__genre_tags_dict


                def fset(self, val):
                    self._CatalogBuilder__genre_tags_dict = val

                return property(fget = fget, fset = fset)

            genre_tags_dict = dynamic_property(genre_tags_dict)

            def htmlFileList(self):

                def fget(self):
                    return self._CatalogBuilder__htmlFileList


                def fset(self, val):
                    self._CatalogBuilder__htmlFileList = val

                return property(fget = fget, fset = fset)

            htmlFileList = dynamic_property(htmlFileList)

            def libraryPath(self):

                def fget(self):
                    return self._CatalogBuilder__libraryPath


                def fset(self, val):
                    self._CatalogBuilder__libraryPath = val

                return property(fget = fget, fset = fset)

            libraryPath = dynamic_property(libraryPath)

            def markerTags(self):

                def fget(self):
                    return self._CatalogBuilder__markerTags


                def fset(self, val):
                    self._CatalogBuilder__markerTags = val

                return property(fget = fget, fset = fset)

            markerTags = dynamic_property(markerTags)

            def ncxSoup(self):

                def fget(self):
                    return self._CatalogBuilder__ncxSoup


                def fset(self, val):
                    self._CatalogBuilder__ncxSoup = val

                return property(fget = fget, fset = fset)

            ncxSoup = dynamic_property(ncxSoup)

            def opts(self):

                def fget(self):
                    return self._CatalogBuilder__opts

                return property(fget = fget)

            opts = dynamic_property(opts)

            def playOrder(self):

                def fget(self):
                    return self._CatalogBuilder__playOrder


                def fset(self, val):
                    self._CatalogBuilder__playOrder = val

                return property(fget = fget, fset = fset)

            playOrder = dynamic_property(playOrder)

            def plugin(self):

                def fget(self):
                    return self._CatalogBuilder__plugin

                return property(fget = fget)

            plugin = dynamic_property(plugin)

            def progressInt(self):

                def fget(self):
                    return self._CatalogBuilder__progressInt


                def fset(self, val):
                    self._CatalogBuilder__progressInt = val

                return property(fget = fget, fset = fset)

            progressInt = dynamic_property(progressInt)

            def progressString(self):

                def fget(self):
                    return self._CatalogBuilder__progressString


                def fset(self, val):
                    self._CatalogBuilder__progressString = val

                return property(fget = fget, fset = fset)

            progressString = dynamic_property(progressString)

            def reporter(self):

                def fget(self):
                    return self._CatalogBuilder__reporter


                def fset(self, val):
                    self._CatalogBuilder__reporter = val

                return property(fget = fget, fset = fset)

            reporter = dynamic_property(reporter)

            def stylesheet(self):

                def fget(self):
                    return self._CatalogBuilder__stylesheet


                def fset(self, val):
                    self._CatalogBuilder__stylesheet = val

                return property(fget = fget, fset = fset)

            stylesheet = dynamic_property(stylesheet)

            def thumbs(self):

                def fget(self):
                    return self._CatalogBuilder__thumbs


                def fset(self, val):
                    self._CatalogBuilder__thumbs = val

                return property(fget = fget, fset = fset)

            thumbs = dynamic_property(thumbs)

            def thumbWidth(self):

                def fget(self):
                    return self._CatalogBuilder__thumbWidth


                def fset(self, val):
                    self._CatalogBuilder__thumbWidth = val

                return property(fget = fget, fset = fset)


            def thumbHeight(self):

                def fget(self):
                    return self._CatalogBuilder__thumbHeight


                def fset(self, val):
                    self._CatalogBuilder__thumbHeight = val

                return property(fget = fget, fset = fset)


            def title(self):

                def fget(self):
                    return self._CatalogBuilder__title


                def fset(self, val):
                    self._CatalogBuilder__title = val

                return property(fget = fget, fset = fset)

            title = dynamic_property(title)

            def totalSteps(self):

                def fget(self):
                    return self._CatalogBuilder__totalSteps

                return property(fget = fget)

            totalSteps = dynamic_property(totalSteps)

            def useSeriesPrefixInTitlesSection(self):

                def fget(self):
                    return self._CatalogBuilder__useSeriesPrefixInTitlesSection


                def fset(self, val):
                    self._CatalogBuilder__useSeriesPrefixInTitlesSection = val

                return property(fget = fget, fset = fset)

            useSeriesPrefixInTitlesSection = dynamic_property(useSeriesPrefixInTitlesSection)

            def verbose(self):

                def fget(self):
                    return self._CatalogBuilder__verbose


                def fset(self, val):
                    self._CatalogBuilder__verbose = val

                return property(fget = fget, fset = fset)

            verbose = dynamic_property(verbose)

            def NOT_READ_SYMBOL(self):

                def fget(self):
                    if self.generateForKindle:
                        return '<span style="color:white">✓</span>'
                    return '<span style="color:white">%s</span>' % self.opts.read_tag

                return property(fget = fget)

            NOT_READ_SYMBOL = dynamic_property(NOT_READ_SYMBOL)

            def READING_SYMBOL(self):

                def fget(self):
                    if self.generateForKindle:
                        return '<span style="color:black">▷</span>'
                    return '<span style="color:white">%s</span>' % self.opts.read_tag

                return property(fget = fget)

            READING_SYMBOL = dynamic_property(READING_SYMBOL)

            def READ_SYMBOL(self):

                def fget(self):
                    if self.generateForKindle:
                        return '<span style="color:black">✓</span>'
                    return '<span style="color:black">%s</span>' % self.opts.read_tag

                return property(fget = fget)

            READ_SYMBOL = dynamic_property(READ_SYMBOL)

            def FULL_RATING_SYMBOL(self):

                def fget(self):
                    if self.generateForKindle:
                        return '★'
                    return '*'

                return property(fget = fget)

            FULL_RATING_SYMBOL = dynamic_property(FULL_RATING_SYMBOL)

            def EMPTY_RATING_SYMBOL(self):

                def fget(self):
                    if self.generateForKindle:
                        return '☆'
                    return ' '

                return property(fget = fget)

            EMPTY_RATING_SYMBOL = dynamic_property(EMPTY_RATING_SYMBOL)

            def READ_PROGRESS_SYMBOL(self):

                def fget(self):
                    if self.generateForKindle:
                        return '▪'
                    return '+'

                return property(fget = fget)

            READ_PROGRESS_SYMBOL = dynamic_property(READ_PROGRESS_SYMBOL)

            def UNREAD_PROGRESS_SYMBOL(self):

                def fget(self):
                    if self.generateForKindle:
                        return '▫'
                    return '-'

                return property(fget = fget)

            UNREAD_PROGRESS_SYMBOL = dynamic_property(UNREAD_PROGRESS_SYMBOL)


        def buildSources(self):
            if self.booksByTitle is None:
                if not self.fetchBooksByTitle():
                    return False

            self.fetchBooksByAuthor()
            self.fetchBookmarks()
            self.generateHTMLDescriptions()
            self.generateHTMLByAuthor()
            if self.opts.generate_titles:
                self.generateHTMLByTitle()

            if self.opts.generate_series:
                self.generateHTMLBySeries()

            if self.opts.generate_recently_added:
                self.generateHTMLByDateAdded()
                if self.generateRecentlyRead:
                    self.generateHTMLByDateRead()


            self.generateHTMLByTags()
            self.generateThumbnails()
            self.generateOPF()
            self.generateNCXHeader()
            self.generateNCXByAuthor('Authors')
            if self.opts.generate_titles:
                self.generateNCXByTitle('Titles')

            if self.opts.generate_series:
                self.generateNCXBySeries('Series')

            if self.opts.generate_recently_added:
                self.generateNCXByDateAdded('Recently Added')
                if self.generateRecentlyRead:
                    self.generateNCXByDateRead('Recently Read')


            self.generateNCXByGenre('Genres')
            self.generateNCXDescriptions('Descriptions')
            self.writeNCX()
            return True


        def cleanUp(self):
            pass


        def copyResources(self):
            catalog_resources = P('catalog')
            files_to_copy = [
                ('', 'DefaultCover.jpg'),
                ('content', 'stylesheet.css'),
                ('images', 'mastheadImage.gif')]
            for file in files_to_copy:
                if file[0] == '':
                    shutil.copy(os.path.join(catalog_resources, file[1]), self.catalogPath)
                    continue
                shutil.copy(os.path.join(catalog_resources, file[1]), os.path.join(self.catalogPath, file[0]))

            if self.generateForKindle:

                try:
                    self.generateMastheadImage(os.path.join(self.catalogPath, 'images/mastheadImage.gif'))




        def fetchBooksByTitle(self):
            self.updateProgressFullStep('Fetching database')
            self.opts.sort_by = 'title'
            empty_exclude_tags = None if len(self.opts.exclude_tags) else True
            search_phrase = ''
            if not empty_exclude_tags:
                exclude_tags = self.opts.exclude_tags.split(',')
                search_terms = []
                for tag in exclude_tags:
                    search_terms.append('tag:=%s' % tag)

                search_phrase = 'not (%s)' % ' or '.join(search_terms)

            if self.opts.ids:
                self.opts.search_text = search_phrase
            elif self.opts.search_text:
                self.opts.search_text += ' ' + search_phrase
            else:
                self.opts.search_text = search_phrase
            data = self.plugin.search_sort_db(self.db, self.opts)
            titles = []
            for record in data:
                this_title = { }
                this_title['id'] = record['id']
                this_title['title'] = self.convertHTMLEntities(record['title'])
                if record['series']:
                    this_title['series'] = record['series']
                    this_title['series_index'] = record['series_index']
                    this_title['title'] = self.generateSeriesTitle(this_title)
                else:
                    this_title['series'] = None
                    this_title['series_index'] = 0
                this_title['title_sort'] = self.generateSortTitle(this_title['title'])
                if 'authors' in record:
                    this_title['authors'] = record['authors']
                    if record['authors']:
                        this_title['author'] = ' & '.join(record['authors'])
                    else:
                        this_title['author'] = 'Unknown'

                if 'author_sort' in record and record['author_sort'].strip():
                    this_title['author_sort'] = record['author_sort']
                else:
                    this_title['author_sort'] = self.author_to_author_sort(this_title['author'])
                if record['publisher']:
                    this_title['publisher'] = re.sub('&', '&', record['publisher'])

                this_title['rating'] = None if record['rating'] else 0
                this_title['date'] = strftime(u'%B %Y', record['pubdate'].timetuple())
                this_title['timestamp'] = record['timestamp']
                if record['comments']:
                    a_offset = record['comments'].find('<div class="user_annotations">')
                    ad_offset = record['comments'].find('<hr class="annotations_divider" />')
                    if a_offset >= 0:
                        record['comments'] = record['comments'][:a_offset]

                    if ad_offset >= 0:
                        record['comments'] = record['comments'][:ad_offset]

                    this_title['description'] = self.markdownComments(record['comments'])
                    paras = BeautifulSoup(this_title['description']).findAll('p')
                    tokens = []
                    for p in paras:
                        for token in p.contents:
                            if token.string is not None:
                                tokens.append(token.string)
                                continue


                    this_title['short_description'] = self.generateShortDescription(' '.join(tokens), dest = 'description')
                else:
                    this_title['description'] = None
                    this_title['short_description'] = None
                if record['cover']:
                    this_title['cover'] = re.sub('&', '&', record['cover'])

                this_title['read'] = False
                if record['tags']:
                    this_title['tags'] = self.processSpecialTags(record['tags'], this_title, self.opts)

                if record['formats']:
                    formats = []
                    for format in record['formats']:
                        formats.append(self.convertHTMLEntities(format))

                    this_title['formats'] = formats

                titles.append(this_title)

            if len(titles):
                self.booksByTitle = sorted(titles, key = (lambda x: (x['title_sort'].upper(), x['title_sort'].upper())))
                if False and self.verbose:
                    self.opts.log.info('fetchBooksByTitle(): %d books' % len(self.booksByTitle))
                    self.opts.log.info(' %-40s %-40s' % ('title', 'title_sort'))
                    for title in self.booksByTitle:
                        self.opts.log.info((u' %-40s %-40s' % (title['title'][0:40], title['title_sort'][0:40])).decode('mac-roman'))


                return True
            return False


        def fetchBooksByAuthor(self):
            self.updateProgressFullStep('Sorting database')
            self.booksByAuthor = list(self.booksByTitle)
            self.booksByAuthor.sort(self.author_compare)
            if False and self.verbose:
                self.opts.log.info('fetchBooksByAuthor(): %d books' % len(self.booksByAuthor))
                self.opts.log.info(' %-30s %-20s %s' % ('title', 'series', 'series_index'))
                for title in self.booksByAuthor:
                    None((self.opts.log.info % (u' %-30s %-20s%5s ', title['title'][:30] if title['series'] else '', title['series_index'])).encode('utf-8'))

                raise SystemExit
            self.verbose
            authors = [ (record['author'], record['author_sort'].capitalize()) for record in self.booksByAuthor ]
            books_by_current_author = 0
            current_author = authors[0]
            multiple_authors = False
            unique_authors = []
            for i, author in enumerate(authors):
                if author != current_author and i:
                    if author[0] == current_author[0]:
                        self.opts.log.warn("Warning: multiple entries for Author '%s' with differing Author Sort metadata:" % author[0])
                        self.opts.log.warn(" '%s' != '%s'" % (author[1], current_author[1]))

                    unique_authors.append((current_author[0], current_author[1].title(), books_by_current_author))
                    current_author = author
                    books_by_current_author = 1
                    continue
                if i == 0 and len(authors) == 1:
                    unique_authors.append((current_author[0], current_author[1].title(), books_by_current_author))
                    continue
                books_by_current_author += 1
            elif current_author == author or len(authors) > 1 or not multiple_authors:
                unique_authors.append((current_author[0], current_author[1].title(), books_by_current_author))

            if False and self.verbose:
                self.opts.log.info('\nfetchBooksByauthor(): %d unique authors' % len(unique_authors))
                for author in unique_authors:
                    self.opts.log.info((u' %-50s %-25s %2d' % (author[0][0:45], author[1][0:20], author[2])).encode('utf-8'))


            self.authors = unique_authors


        def fetchBookmarks(self):
            Device = Device
            import calibre.devices.usbms.device
            Bookmark = Bookmark
            import calibre.devices.kindle.driver
            MetaInformation = MetaInformation
            import calibre.ebooks.metadata
            MBP_FORMATS = [
                u'azw',
                u'mobi',
                u'prc',
                u'txt']
            mbp_formats = set(MBP_FORMATS)
            PDR_FORMATS = [
                u'pdf']
            pdr_formats = set(PDR_FORMATS)
            TAN_FORMATS = [
                u'tpz',
                u'azw1']
            tan_formats = set(TAN_FORMATS)

            class BookmarkDevice(Device):

                def initialize(self, save_template):
                    self._save_template = save_template
                    self.SUPPORTS_SUB_DIRS = True


                def save_template(self):
                    return self._save_template



            def resolve_bookmark_paths(storage, path_map):
                pop_list = []
                book_ext = { }
                for id in path_map:
                    file_fmts = set()
                    for fmt in path_map[id]['fmts']:
                        file_fmts.add(fmt)

                    bookmark_extension = None
                    if file_fmts.intersection(mbp_formats):
                        book_extension = list(file_fmts.intersection(mbp_formats))[0]
                        bookmark_extension = 'mbp'
                    elif file_fmts.intersection(tan_formats):
                        book_extension = list(file_fmts.intersection(tan_formats))[0]
                        bookmark_extension = 'tan'
                    elif file_fmts.intersection(pdr_formats):
                        book_extension = list(file_fmts.intersection(pdr_formats))[0]
                        bookmark_extension = 'pdr'

                    if bookmark_extension:
                        for vol in storage:
                            bkmk_path = path_map[id]['path'].replace(os.path.abspath('/<storage>'), vol)
                            bkmk_path = bkmk_path.replace('bookmark', bookmark_extension)
                            if os.path.exists(bkmk_path):
                                path_map[id] = bkmk_path
                                book_ext[id] = book_extension
                                break
                                continue

                    pop_list.append(id)

                for id in pop_list:
                    path_map.pop(id)

                return (path_map, book_ext)



        def generateHTMLDescriptions(self):
            self.updateProgressFullStep("'Descriptions'")
            for title_num, title in enumerate(self.booksByTitle):
                if False:
                    self.opts.log.info('%3s: %s - %s' % (title['id'], title['title'], title['author']))

                self.updateProgressMicroStep('Description %d of %d' % (title_num, len(self.booksByTitle)), float(title_num * 100 / len(self.booksByTitle)) / 100)
                soup = self.generateHTMLDescriptionHeader('%s' % title['title'])
                body = soup.find('body')
                btc = 0
                aTag = Tag(soup, 'a')
                aTag['name'] = 'book%d' % int(title['id'])
                body.insert(btc, aTag)
                btc += 1
                emTag = Tag(soup, 'em')
                if title['series']:
                    if self.opts.generate_series:
                        brTag = Tag(soup, 'br')
                        title_tokens = list(title['title'].partition(':'))
                        emTag.insert(0, escape(NavigableString(title_tokens[2].strip())))
                        emTag.insert(1, brTag)
                        smallTag = Tag(soup, 'small')
                        aTag = Tag(soup, 'a')
                        aTag['href'] = '%s.html#%s_series' % ('BySeries', re.sub('\\W', '', title['series']).lower())
                        aTag.insert(0, title_tokens[0])
                        smallTag.insert(0, aTag)
                        emTag.insert(2, smallTag)
                    else:
                        brTag = Tag(soup, 'br')
                        title_tokens = list(title['title'].partition(':'))
                        emTag.insert(0, escape(NavigableString(title_tokens[2].strip())))
                        emTag.insert(1, brTag)
                        smallTag = Tag(soup, 'small')
                        smallTag.insert(0, escape(NavigableString(title_tokens[0])))
                        emTag.insert(2, smallTag)
                else:
                    emTag.insert(0, NavigableString(escape(title['title'])))
                titleTag = body.find(attrs = {
                    'class': 'title' })
                titleTag.insert(0, emTag)
                authorTag = body.find(attrs = {
                    'class': 'author' })
                aTag = Tag(soup, 'a')
                aTag['href'] = '%s.html#%s' % ('ByAlphaAuthor', self.generateAuthorAnchor(title['author']))
                aTag.insert(0, title['author'])
                if title['read']:
                    authorTag.insert(0, NavigableString(self.READ_SYMBOL + 'by '))
                elif self.opts.connected_kindle and title['id'] in self.bookmarked_books:
                    authorTag.insert(0, NavigableString(self.READING_SYMBOL + ' by '))
                else:
                    authorTag.insert(0, NavigableString(self.NOT_READ_SYMBOL + 'by '))
                authorTag.insert(1, aTag)
                if 'tags' in title:
                    tagsTag = body.find(attrs = {
                        'class': 'tags' })
                    ttc = 0
                    fontTag = Tag(soup, 'font')
                    fontTag['style'] = 'color:white;font-size:large'
                    if self.opts.fmt == 'epub':
                        fontTag['style'] += ';opacity: 0.0'

                    fontTag.insert(0, NavigableString('by '))
                    tagsTag.insert(ttc, fontTag)
                    ttc += 1
                    for tag in title['tags']:
                        aTag = Tag(soup, 'a')
                        aTag['href'] = 'Genre_%s.html' % re.sub('\\W', '', tag.lower())
                        aTag.insert(0, escape(NavigableString(tag)))
                        emTag = Tag(soup, 'em')
                        emTag.insert(0, aTag)
                        if ttc < len(title['tags']):
                            emTag.insert(1, NavigableString(' · '))

                        tagsTag.insert(ttc, emTag)
                        ttc += 1


                imgTag = Tag(soup, 'img')
                if 'cover' in title:
                    imgTag['src'] = '../images/thumbnail_%d.jpg' % int(title['id'])
                else:
                    imgTag['src'] = '../images/thumbnail_default.jpg'
                imgTag['alt'] = 'cover'
                thumbnailTag = body.find(attrs = {
                    'class': 'thumbnail' })
                thumbnailTag.insert(0, imgTag)
                publisherTag = body.find(attrs = {
                    'class': 'publisher' })
                if 'publisher' in title:
                    publisherTag.insert(0, NavigableString(title['publisher'] + '<br/>'))
                else:
                    publisherTag.insert(0, NavigableString('<br/>'))
                pubdateTag = body.find(attrs = {
                    'class': 'date' })
                if title['date'] is not None:
                    pubdateTag.insert(0, NavigableString(title['date'] + '<br/>'))
                else:
                    pubdateTag.insert(0, NavigableString('<br/>'))
                stars = int(title['rating']) / 2
                star_string = self.FULL_RATING_SYMBOL * stars
                empty_stars = self.EMPTY_RATING_SYMBOL * (5 - stars)
                ratingTag = body.find(attrs = {
                    'class': 'rating' })
                ratingTag.insert(0, NavigableString('%s%s <br/>' % (star_string, empty_stars)))
                if 'notes' in title:
                    notesTag = body.find(attrs = {
                        'class': 'notes' })
                    notesTag.insert(0, NavigableString(title['notes'] + '<br/>'))
                else:
                    notes_labelTag = body.find(attrs = {
                        'class': 'notes_label' })
                    empty_labelTag = Tag(soup, 'td')
                    empty_labelTag.insert(0, NavigableString('<br/>'))
                    notes_labelTag.replaceWith(empty_labelTag)
                if 'description' in title and title['description'] > '':
                    blurbTag = body.find(attrs = {
                        'class': 'description' })
                    blurbTag.insert(0, NavigableString(title['description']))

                outfile = open('%s/book_%d.html' % (self.contentDir, int(title['id'])), 'w')
                outfile.write(soup.prettify())
                outfile.close()



        def generateHTMLByTitle(self):
            self.updateProgressFullStep("'Titles'")
            soup = self.generateHTMLEmptyHeader('Books By Alpha Title')
            body = soup.find('body')
            btc = 0
            aTag = Tag(soup, 'a')
            aTag['name'] = 'section_start'
            body.insert(btc, aTag)
            btc += 1
            aTag = Tag(soup, 'a')
            aTag['name'] = 'bytitle'
            body.insert(btc, aTag)
            btc += 1
            if not self._CatalogBuilder__generateForKindle:
                pTag = Tag(soup, 'p')
                pTag['class'] = 'title'
                aTag = Tag(soup, 'a')
                aTag['name'] = 'bytitle'
                pTag.insert(0, aTag)
                pTag.insert(1, NavigableString('Titles'))
                body.insert(btc, pTag)
                btc += 1

            divTag = Tag(soup, 'div')
            dtc = 0
            current_letter = ''
            if not self.useSeriesPrefixInTitlesSection:
                nspt = deepcopy(self.booksByTitle)
                for book in nspt:
                    if book['series']:
                        tokens = book['title'].partition(':')
                        book['title'] = '%s (%s)' % (tokens[2].strip(), tokens[0])
                        book['title_sort'] = self.generateSortTitle(book['title'])
                        continue

                nspt = sorted(nspt, key = (lambda x: (x['title_sort'].upper(), x['title_sort'].upper())))
                self.booksByTitle_noSeriesPrefix = nspt
                if False and self.verbose:
                    self.opts.log.info('no_series_prefix_titles: %d books' % len(nspt))
                    self.opts.log.info(' %-40s %-40s' % ('title', 'title_sort'))
                    for title in nspt:
                        self.opts.log.info((u' %-40s %-40s' % (title['title'][0:40], title['title_sort'][0:40])).encode('utf-8'))



            title_list = self.booksByTitle
            if not self.useSeriesPrefixInTitlesSection:
                title_list = self.booksByTitle_noSeriesPrefix

            for book in title_list:
                if self.letter_or_symbol(book['title_sort'][0]) != current_letter:
                    current_letter = self.letter_or_symbol(book['title_sort'][0])
                    pIndexTag = Tag(soup, 'p')
                    pIndexTag['class'] = 'letter_index'
                    aTag = Tag(soup, 'a')
                    aTag['name'] = '%s' % self.letter_or_symbol(book['title_sort'][0])
                    pIndexTag.insert(0, aTag)
                    pIndexTag.insert(1, NavigableString(self.letter_or_symbol(book['title_sort'][0])))
                    divTag.insert(dtc, pIndexTag)
                    dtc += 1

                pBookTag = Tag(soup, 'p')
                ptc = 0
                if book['read']:
                    pBookTag.insert(ptc, NavigableString(self.READ_SYMBOL))
                    pBookTag['class'] = 'read_book'
                    ptc += 1
                elif book['id'] in self.bookmarked_books:
                    pBookTag.insert(ptc, NavigableString(self.READING_SYMBOL))
                    pBookTag['class'] = 'read_book'
                    ptc += 1
                else:
                    pBookTag['class'] = 'unread_book'
                    pBookTag.insert(ptc, NavigableString(self.NOT_READ_SYMBOL))
                    ptc += 1
                aTag = Tag(soup, 'a')
                aTag['href'] = 'book_%d.html' % int(float(book['id']))
                aTag.insert(0, escape(book['title']))
                pBookTag.insert(ptc, aTag)
                ptc += 1
                pBookTag.insert(ptc, NavigableString(' · '))
                ptc += 1
                emTag = Tag(soup, 'em')
                aTag = Tag(soup, 'a')
                aTag['href'] = '%s.html#%s' % ('ByAlphaAuthor', self.generateAuthorAnchor(book['author']))
                aTag.insert(0, NavigableString(book['author']))
                emTag.insert(0, aTag)
                pBookTag.insert(ptc, emTag)
                ptc += 1
                divTag.insert(dtc, pBookTag)
                dtc += 1

            body.insert(btc, divTag)
            btc += 1
            outfile_spec = '%s/ByAlphaTitle.html' % self.contentDir
            outfile = open(outfile_spec, 'w')
            outfile.write(soup.prettify())
            outfile.close()
            self.htmlFileList.append('content/ByAlphaTitle.html')


        def generateHTMLByAuthor(self):
            self.updateProgressFullStep("'Authors'")
            friendly_name = 'Authors'
            soup = self.generateHTMLEmptyHeader(friendly_name)
            body = soup.find('body')
            btc = 0
            aTag = Tag(soup, 'a')
            aTag['name'] = 'section_start'
            body.insert(btc, aTag)
            btc += 1
            aTag = Tag(soup, 'a')
            anchor_name = friendly_name.lower()
            aTag['name'] = anchor_name.replace(' ', '')
            body.insert(btc, aTag)
            btc += 1
            divTag = Tag(soup, 'div')
            dtc = 0
            current_letter = ''
            current_author = ''
            current_series = None
            book_count = 0
            for book in self.booksByAuthor:
                book_count += 1
                if self.letter_or_symbol(book['author_sort'][0].upper()) != current_letter:
                    current_letter = self.letter_or_symbol(book['author_sort'][0].upper())
                    pIndexTag = Tag(soup, 'p')
                    pIndexTag['class'] = 'letter_index'
                    aTag = Tag(soup, 'a')
                    aTag['name'] = '%sauthors' % self.letter_or_symbol(current_letter)
                    pIndexTag.insert(0, aTag)
                    pIndexTag.insert(1, NavigableString(self.letter_or_symbol(book['author_sort'][0].upper())))
                    divTag.insert(dtc, pIndexTag)
                    dtc += 1

                if book['author'] != current_author:
                    current_author = book['author']
                    non_series_books = 0
                    current_series = None
                    pAuthorTag = Tag(soup, 'p')
                    pAuthorTag['class'] = 'author_index'
                    aTag = Tag(soup, 'a')
                    aTag['name'] = '%s' % self.generateAuthorAnchor(current_author)
                    aTag.insert(0, NavigableString(current_author))
                    pAuthorTag.insert(0, aTag)
                    divTag.insert(dtc, pAuthorTag)
                    dtc += 1

                if book['series'] and book['series'] != current_series:
                    current_series = book['series']
                    pSeriesTag = Tag(soup, 'p')
                    pSeriesTag['class'] = 'series'
                    if self.opts.generate_series:
                        aTag = Tag(soup, 'a')
                        aTag['href'] = '%s.html#%s_series' % ('BySeries', re.sub('\\W', '', book['series']).lower())
                        aTag.insert(0, book['series'])
                        pSeriesTag.insert(0, aTag)
                    else:
                        pSeriesTag.insert(0, NavigableString('%s' % book['series']))
                    divTag.insert(dtc, pSeriesTag)
                    dtc += 1

                if current_series and not book['series']:
                    current_series = None

                pBookTag = Tag(soup, 'p')
                ptc = 0
                if book['read']:
                    pBookTag.insert(ptc, NavigableString(self.READ_SYMBOL))
                    pBookTag['class'] = 'read_book'
                    ptc += 1
                elif book['id'] in self.bookmarked_books:
                    pBookTag.insert(ptc, NavigableString(self.READING_SYMBOL))
                    pBookTag['class'] = 'read_book'
                    ptc += 1
                else:
                    pBookTag['class'] = 'unread_book'
                    pBookTag.insert(ptc, NavigableString(self.NOT_READ_SYMBOL))
                    ptc += 1
                aTag = Tag(soup, 'a')
                aTag['href'] = 'book_%d.html' % int(float(book['id']))
                if current_series:
                    aTag.insert(0, '%s (%s)' % (escape(book['title'][len(book['series']) + 1:]), book['date'].split()[1]))
                else:
                    aTag.insert(0, '%s (%s)' % (escape(book['title']), book['date'].split()[1]))
                    non_series_books += 1
                pBookTag.insert(ptc, aTag)
                ptc += 1
                divTag.insert(dtc, pBookTag)
                dtc += 1

            if not self._CatalogBuilder__generateForKindle:
                pTag = Tag(soup, 'p')
                pTag['class'] = 'title'
                aTag = Tag(soup, 'a')
                anchor_name = friendly_name.lower()
                aTag['name'] = anchor_name.replace(' ', '')
                pTag.insert(0, aTag)
                pTag.insert(1, NavigableString('%s' % friendly_name))
                body.insert(btc, pTag)
                btc += 1

            body.insert(btc, divTag)
            outfile_spec = '%s/ByAlphaAuthor.html' % self.contentDir
            outfile = open(outfile_spec, 'w')
            outfile.write(soup.prettify())
            outfile.close()
            self.htmlFileList.append('content/ByAlphaAuthor.html')


        def generateHTMLByDateAdded(self):
            self.updateProgressFullStep("'Recently Added'")

            def add_books_to_HTML_by_month(this_months_list, dtc):
                if len(this_months_list):
                    this_months_list.sort(self.author_compare)
                    date_string = strftime(u'%B %Y', current_date.timetuple())
                    pIndexTag = Tag(soup, 'p')
                    pIndexTag['class'] = 'date_index'
                    aTag = Tag(soup, 'a')
                    aTag['name'] = 'bda_%s-%s' % (current_date.year, current_date.month)
                    pIndexTag.insert(0, aTag)
                    pIndexTag.insert(1, NavigableString(date_string))
                    divTag.insert(dtc, pIndexTag)
                    dtc += 1
                    current_author = None
                    current_series = None
                    for new_entry in this_months_list:
                        if new_entry['author'] != current_author:
                            current_author = new_entry['author']
                            non_series_books = 0
                            current_series = None
                            pAuthorTag = Tag(soup, 'p')
                            pAuthorTag['class'] = 'author_index'
                            aTag = Tag(soup, 'a')
                            aTag['name'] = '%s' % self.generateAuthorAnchor(current_author)
                            aTag.insert(0, NavigableString(current_author))
                            pAuthorTag.insert(0, aTag)
                            divTag.insert(dtc, pAuthorTag)
                            dtc += 1

                        if new_entry['series'] and new_entry['series'] != current_series:
                            current_series = new_entry['series']
                            pSeriesTag = Tag(soup, 'p')
                            pSeriesTag['class'] = 'series'
                            if self.opts.generate_series:
                                aTag = Tag(soup, 'a')
                                aTag['href'] = '%s.html#%s_series' % ('BySeries', re.sub('\\W', '', new_entry['series']).lower())
                                aTag.insert(0, new_entry['series'])
                                pSeriesTag.insert(0, aTag)
                            else:
                                pSeriesTag.insert(0, NavigableString('%s' % new_entry['series']))
                            divTag.insert(dtc, pSeriesTag)
                            dtc += 1

                        if current_series and not new_entry['series']:
                            current_series = None

                        pBookTag = Tag(soup, 'p')
                        ptc = 0
                        if new_entry['read']:
                            pBookTag.insert(ptc, NavigableString(self.READ_SYMBOL))
                            pBookTag['class'] = 'read_book'
                            ptc += 1
                        elif new_entry['id'] in self.bookmarked_books:
                            pBookTag.insert(ptc, NavigableString(self.READING_SYMBOL))
                            pBookTag['class'] = 'read_book'
                            ptc += 1
                        else:
                            pBookTag['class'] = 'unread_book'
                            pBookTag.insert(ptc, NavigableString(self.NOT_READ_SYMBOL))
                            ptc += 1
                        aTag = Tag(soup, 'a')
                        aTag['href'] = 'book_%d.html' % int(float(new_entry['id']))
                        if current_series:
                            aTag.insert(0, escape(new_entry['title'][len(new_entry['series']) + 1:]))
                        else:
                            aTag.insert(0, escape(new_entry['title']))
                            non_series_books += 1
                        pBookTag.insert(ptc, aTag)
                        ptc += 1
                        divTag.insert(dtc, pBookTag)
                        dtc += 1


                return dtc


            def add_books_to_HTML_by_date_range(date_range_list, date_range, dtc):
                if len(date_range_list):
                    pIndexTag = Tag(soup, 'p')
                    pIndexTag['class'] = 'date_index'
                    aTag = Tag(soup, 'a')
                    aTag['name'] = 'bda_%s' % date_range.replace(' ', '')
                    pIndexTag.insert(0, aTag)
                    pIndexTag.insert(1, NavigableString(date_range))
                    divTag.insert(dtc, pIndexTag)
                    dtc += 1
                    for new_entry in date_range_list:
                        pBookTag = Tag(soup, 'p')
                        ptc = 0
                        if new_entry['read']:
                            pBookTag.insert(ptc, NavigableString(self.READ_SYMBOL))
                            pBookTag['class'] = 'read_book'
                            ptc += 1
                        elif new_entry['id'] in self.bookmarked_books:
                            pBookTag.insert(ptc, NavigableString(self.READING_SYMBOL))
                            pBookTag['class'] = 'read_book'
                            ptc += 1
                        else:
                            pBookTag['class'] = 'unread_book'
                            pBookTag.insert(ptc, NavigableString(self.NOT_READ_SYMBOL))
                            ptc += 1
                        aTag = Tag(soup, 'a')
                        aTag['href'] = 'book_%d.html' % int(float(new_entry['id']))
                        aTag.insert(0, escape(new_entry['title']))
                        pBookTag.insert(ptc, aTag)
                        ptc += 1
                        pBookTag.insert(ptc, NavigableString(' · '))
                        ptc += 1
                        emTag = Tag(soup, 'em')
                        aTag = Tag(soup, 'a')
                        aTag['href'] = '%s.html#%s' % ('ByAlphaAuthor', self.generateAuthorAnchor(new_entry['author']))
                        aTag.insert(0, NavigableString(new_entry['author']))
                        emTag.insert(0, aTag)
                        pBookTag.insert(ptc, emTag)
                        ptc += 1
                        divTag.insert(dtc, pBookTag)
                        dtc += 1


                return dtc

            friendly_name = 'Recently Added'
            soup = self.generateHTMLEmptyHeader(friendly_name)
            body = soup.find('body')
            btc = 0
            aTag = Tag(soup, 'a')
            aTag['name'] = 'section_start'
            body.insert(btc, aTag)
            btc += 1
            aTag = Tag(soup, 'a')
            anchor_name = friendly_name.lower()
            aTag['name'] = anchor_name.replace(' ', '')
            body.insert(btc, aTag)
            btc += 1
            if not self._CatalogBuilder__generateForKindle:
                pTag = Tag(soup, 'p')
                pTag['class'] = 'title'
                aTag = Tag(soup, 'a')
                anchor_name = friendly_name.lower()
                aTag['name'] = anchor_name.replace(' ', '')
                pTag.insert(0, aTag)
                pTag.insert(1, NavigableString('%s' % friendly_name))
                body.insert(btc, pTag)
                btc += 1

            divTag = Tag(soup, 'div')
            dtc = 0
            if self.useSeriesPrefixInTitlesSection:
                self.booksByDateRange = sorted(self.booksByTitle, key = (lambda x: (x['timestamp'], x['timestamp'])), reverse = True)
            else:
                nspt = deepcopy(self.booksByTitle)
                for book in nspt:
                    if book['series']:
                        tokens = book['title'].partition(':')
                        book['title'] = '%s (%s)' % (tokens[2].strip(), tokens[0])
                        book['title_sort'] = self.generateSortTitle(book['title'])
                        continue

                self.booksByDateRange = sorted(nspt, key = (lambda x: (x['timestamp'], x['timestamp'])), reverse = True)
            date_range_list = []
            today_time = nowf().replace(hour = 23, minute = 59, second = 59)
            for i, date in enumerate(self.DATE_RANGE):
                date_range_limit = self.DATE_RANGE[i]
                if i:
                    date_range = '%d to %d days ago' % (self.DATE_RANGE[i - 1], self.DATE_RANGE[i])
                else:
                    date_range = 'Last %d days' % self.DATE_RANGE[i]
                for book in self.booksByDateRange:
                    book_time = book['timestamp']
                    delta = today_time - book_time
                    if delta.days <= date_range_limit:
                        date_range_list.append(book)
                        continue

                dtc = add_books_to_HTML_by_date_range(date_range_list, date_range, dtc)
                date_range_list = [
                    book]

            self.booksByMonth = sorted(self.booksByTitle, key = (lambda x: (x['timestamp'], x['timestamp'])), reverse = True)
            current_date = datetime.date.fromordinal(1)
            this_months_list = []
            for book in self.booksByMonth:
                if book['timestamp'].month != current_date.month or book['timestamp'].year != current_date.year:
                    dtc = add_books_to_HTML_by_month(this_months_list, dtc)
                    this_months_list = []
                    current_date = book['timestamp'].date()

                this_months_list.append(book)

            add_books_to_HTML_by_month(this_months_list, dtc)
            body.insert(btc, divTag)
            outfile_spec = '%s/ByDateAdded.html' % self.contentDir
            outfile = open(outfile_spec, 'w')
            outfile.write(soup.prettify())
            outfile.close()
            self.htmlFileList.append('content/ByDateAdded.html')


        def generateHTMLByDateRead(self):
            friendly_name = 'Recently Read'
            self.updateProgressFullStep("'%s'" % friendly_name)
            if not self.bookmarked_books:
                return None

            def add_books_to_HTML_by_day(todays_list, dtc):
                if len(todays_list):
                    date_string = strftime(u'%A, %B %d', current_date.timetuple())
                    pIndexTag = Tag(soup, 'p')
                    pIndexTag['class'] = 'date_index'
                    aTag = Tag(soup, 'a')
                    aTag['name'] = 'bdr_%s-%s-%s' % (current_date.year, current_date.month, current_date.day)
                    pIndexTag.insert(0, aTag)
                    pIndexTag.insert(1, NavigableString(date_string))
                    divTag.insert(dtc, pIndexTag)
                    dtc += 1
                    for new_entry in todays_list:
                        pBookTag = Tag(soup, 'p')
                        pBookTag['class'] = 'date_read'
                        ptc = 0
                        pBookTag.insert(ptc, NavigableString(new_entry['reading_progress']))
                        ptc += 1
                        aTag = Tag(soup, 'a')
                        aTag['href'] = 'book_%d.html' % int(float(new_entry['id']))
                        aTag.insert(0, escape(new_entry['title']))
                        pBookTag.insert(ptc, aTag)
                        ptc += 1
                        pBookTag.insert(ptc, NavigableString(' · '))
                        ptc += 1
                        emTag = Tag(soup, 'em')
                        aTag = Tag(soup, 'a')
                        aTag['href'] = '%s.html#%s' % ('ByAlphaAuthor', self.generateAuthorAnchor(new_entry['author']))
                        aTag.insert(0, NavigableString(new_entry['author']))
                        emTag.insert(0, aTag)
                        pBookTag.insert(ptc, emTag)
                        ptc += 1
                        divTag.insert(dtc, pBookTag)
                        dtc += 1


                return dtc


            def add_books_to_HTML_by_date_range(date_range_list, date_range, dtc):
                if len(date_range_list):
                    pIndexTag = Tag(soup, 'p')
                    pIndexTag['class'] = 'date_index'
                    aTag = Tag(soup, 'a')
                    aTag['name'] = 'bdr_%s' % date_range.replace(' ', '')
                    pIndexTag.insert(0, aTag)
                    pIndexTag.insert(1, NavigableString(date_range))
                    divTag.insert(dtc, pIndexTag)
                    dtc += 1
                    for new_entry in date_range_list:
                        pBookTag = Tag(soup, 'p')
                        pBookTag['class'] = 'date_read'
                        ptc = 0
                        dots = int((new_entry['percent_read'] + 5) / 10)
                        dot_string = self.READ_PROGRESS_SYMBOL * dots
                        empty_dots = self.UNREAD_PROGRESS_SYMBOL * (10 - dots)
                        pBookTag.insert(ptc, NavigableString('%s%s' % (dot_string, empty_dots)))
                        ptc += 1
                        aTag = Tag(soup, 'a')
                        aTag['href'] = 'book_%d.html' % int(float(new_entry['id']))
                        aTag.insert(0, escape(new_entry['title']))
                        pBookTag.insert(ptc, aTag)
                        ptc += 1
                        pBookTag.insert(ptc, NavigableString(' · '))
                        ptc += 1
                        emTag = Tag(soup, 'em')
                        aTag = Tag(soup, 'a')
                        aTag['href'] = '%s.html#%s' % ('ByAlphaAuthor', self.generateAuthorAnchor(new_entry['author']))
                        aTag.insert(0, NavigableString(new_entry['author']))
                        emTag.insert(0, aTag)
                        pBookTag.insert(ptc, emTag)
                        ptc += 1
                        divTag.insert(dtc, pBookTag)
                        dtc += 1


                return dtc

            soup = self.generateHTMLEmptyHeader(friendly_name)
            body = soup.find('body')
            btc = 0
            aTag = Tag(soup, 'a')
            aTag['name'] = 'section_start'
            body.insert(btc, aTag)
            btc += 1
            aTag = Tag(soup, 'a')
            anchor_name = friendly_name.lower()
            aTag['name'] = anchor_name.replace(' ', '')
            body.insert(btc, aTag)
            btc += 1
            divTag = Tag(soup, 'div')
            dtc = 0
            bookmarked_books = []
            for bm_book in self.bookmarked_books:
                book = self.bookmarked_books[bm_book]
                book[1]['bookmark_timestamp'] = book[0].timestamp

                try:
                    book[1]['percent_read'] = min(float(100 * book[0].last_read / book[0].book_length), 100)
                except:
                    (None, None, (None, None, None, self.bookmarked_books))
                    book[1]['percent_read'] = 0

                bookmarked_books.append(book[1])

            self.booksByDateRead = sorted(bookmarked_books, key = (lambda x: (x['bookmark_timestamp'], x['bookmark_timestamp'])), reverse = True)
            current_date = datetime.date.fromordinal(1)
            todays_list = []
            for book in self.booksByDateRead:
                bookmark_time = datetime.datetime.utcfromtimestamp(book['bookmark_timestamp'])
                if bookmark_time.day != current_date.day and bookmark_time.month != current_date.month or bookmark_time.year != current_date.year:
                    dtc = add_books_to_HTML_by_day(todays_list, dtc)
                    todays_list = []
                    current_date = datetime.datetime.utcfromtimestamp(book['bookmark_timestamp']).date()

                todays_list.append(book)

            add_books_to_HTML_by_day(todays_list, dtc)
            body.insert(btc, divTag)
            outfile_spec = '%s/ByDateRead.html' % self.contentDir
            outfile = open(outfile_spec, 'w')
            outfile.write(soup.prettify())
            outfile.close()
            self.htmlFileList.append('content/ByDateRead.html')


        def generateHTMLBySeries(self):
            self.updateProgressFullStep('Fetching series')
            self.opts.sort_by = 'series'
            empty_exclude_tags = None if len(self.opts.exclude_tags) else True
            search_phrase = 'series:true '
            if not empty_exclude_tags:
                exclude_tags = self.opts.exclude_tags.split(',')
                search_terms = []
                for tag in exclude_tags:
                    search_terms.append('tag:=%s' % tag)

                search_phrase += 'not (%s)' % ' or '.join(search_terms)

            if self.opts.ids:
                self.opts.search_text = search_phrase
            elif self.opts.search_text:
                self.opts.search_text += ' ' + search_phrase
            else:
                self.opts.search_text = search_phrase
            self.booksBySeries = self.plugin.search_sort_db(self.db, self.opts)
            if not self.booksBySeries:
                self.opts.generate_series = False
                self.opts.log(' no series found in selected books, cancelling series generation')
                return None
            friendly_name = 'Series'
            soup = self.generateHTMLEmptyHeader(friendly_name)
            body = soup.find('body')
            btc = 0
            aTag = Tag(soup, 'a')
            aTag['name'] = 'section_start'
            body.insert(btc, aTag)
            btc += 1
            aTag = Tag(soup, 'a')
            anchor_name = friendly_name.lower()
            aTag['name'] = anchor_name.replace(' ', '')
            body.insert(btc, aTag)
            btc += 1
            divTag = Tag(soup, 'div')
            dtc = 0
            current_letter = ''
            current_series = None
            series_count = 0
            for book in self.booksBySeries:
                sort_title = self.generateSortTitle(book['series'])
                if self.letter_or_symbol(sort_title[0].upper()) != current_letter:
                    current_letter = self.letter_or_symbol(sort_title[0].upper())
                    pIndexTag = Tag(soup, 'p')
                    pIndexTag['class'] = 'letter_index'
                    aTag = Tag(soup, 'a')
                    aTag['name'] = '%s_series' % self.letter_or_symbol(current_letter)
                    pIndexTag.insert(0, aTag)
                    pIndexTag.insert(1, NavigableString(self.letter_or_symbol(sort_title[0].upper())))
                    divTag.insert(dtc, pIndexTag)
                    dtc += 1

                if book['series'] != current_series:
                    series_count += 1
                    current_series = book['series']
                    pSeriesTag = Tag(soup, 'p')
                    pSeriesTag['class'] = 'series'
                    aTag = Tag(soup, 'a')
                    aTag['name'] = '%s_series' % re.sub('\\W', '', book['series']).lower()
                    pSeriesTag.insert(0, aTag)
                    pSeriesTag.insert(1, NavigableString('%s' % book['series']))
                    divTag.insert(dtc, pSeriesTag)
                    dtc += 1

                pBookTag = Tag(soup, 'p')
                ptc = 0
                for tag in book['tags']:
                    if tag == self.opts.read_tag:
                        book['read'] = True
                        break
                        continue
                else:
                    book['read'] = False
                if book['read']:
                    pBookTag.insert(ptc, NavigableString(self.READ_SYMBOL))
                    pBookTag['class'] = 'read_book'
                    ptc += 1
                elif book['id'] in self.bookmarked_books:
                    pBookTag.insert(ptc, NavigableString(self.READING_SYMBOL))
                    pBookTag['class'] = 'read_book'
                    ptc += 1
                else:
                    pBookTag['class'] = 'unread_book'
                    pBookTag.insert(ptc, NavigableString(self.NOT_READ_SYMBOL))
                    ptc += 1
                aTag = Tag(soup, 'a')
                aTag['href'] = 'book_%d.html' % int(float(book['id']))
                aTag.insert(0, '%d. %s (%s)' % (book['series_index'], escape(book['title']), strftime(u'%Y', book['pubdate'].timetuple())))
                pBookTag.insert(ptc, aTag)
                ptc += 1
                pBookTag.insert(ptc, NavigableString(' · '))
                ptc += 1
                aTag = Tag(soup, 'a')
                aTag['href'] = '%s.html#%s' % ('ByAlphaAuthor', self.generateAuthorAnchor(escape(' & '.join(book['authors']))))
                aTag.insert(0, NavigableString(' & '.join(book['authors'])))
                pBookTag.insert(ptc, aTag)
                ptc += 1
                divTag.insert(dtc, pBookTag)
                dtc += 1

            if not self._CatalogBuilder__generateForKindle:
                pTag = Tag(soup, 'p')
                pTag['class'] = 'title'
                aTag = Tag(soup, 'a')
                anchor_name = friendly_name.lower()
                aTag['name'] = anchor_name.replace(' ', '')
                pTag.insert(0, aTag)
                pTag.insert(1, NavigableString('%s' % friendly_name))
                body.insert(btc, pTag)
                btc += 1

            body.insert(btc, divTag)
            outfile_spec = '%s/BySeries.html' % self.contentDir
            outfile = open(outfile_spec, 'w')
            outfile.write(soup.prettify())
            outfile.close()
            self.htmlFileList.append('content/BySeries.html')


        def generateHTMLByTags(self):
            self.updateProgressFullStep("'Genres'")
            self.genre_tags_dict = self.filterDbTags(self.db.all_tags())
            genre_list = []
            for friendly_tag in sorted(self.genre_tags_dict):
                tag_list = { }
                for book in self.booksByAuthor:
                    if 'tags' in book and friendly_tag in book['tags']:
                        this_book = { }
                        this_book['author'] = book['author']
                        this_book['title'] = book['title']
                        this_book['author_sort'] = book['author_sort'].capitalize()
                        this_book['read'] = book['read']
                        this_book['id'] = book['id']
                        this_book['series'] = book['series']
                        normalized_tag = self.genre_tags_dict[friendly_tag]
                        genre_tag_list = [ key for genre in genre_list for key in genre ]
                        if normalized_tag in genre_tag_list:
                            for existing_genre in genre_list:
                                for key in existing_genre:
                                    new_book = None
                                    if key == normalized_tag:
                                        for book in existing_genre[key]:
                                            if book['title'] == this_book['title']:
                                                new_book = False
                                                break
                                                continue
                                            []
                                        else:
                                            new_book = True

                                    if new_book:
                                        existing_genre[key].append(this_book)
                                        continue


                        else:
                            tag_list[normalized_tag] = [
                                this_book]
                            genre_list.append(tag_list)
                    normalized_tag in genre_tag_list


            if self.opts.verbose:
                self.opts.log.info('     Genre summary: %d active genre tags used in generating catalog with %d titles' % (len(genre_list), len(self.booksByTitle)))
                for genre in genre_list:
                    for key in genre:
                        [](self.opts.log.info % ('      %s: %d %s', self.getFriendlyGenreTag(key), len(genre[key]) if len(genre[key]) > 1 else 'title'))



            master_genre_list = []
            for genre_tag_set in genre_list:
                for index, genre in enumerate(genre_tag_set):
                    authors = []
                    for book in genre_tag_set[genre]:
                        authors.append((book['author'], book['author_sort']))

                    books_by_current_author = 1
                    current_author = authors[0]
                    unique_authors = []
                    for i, author in enumerate(authors):
                        if author != current_author and i:
                            unique_authors.append((current_author[0], current_author[1], books_by_current_author))
                            current_author = author
                            books_by_current_author = 1
                            continue
                        if i == 0 and len(authors) == 1:
                            unique_authors.append((current_author[0], current_author[1], books_by_current_author))
                            continue
                        books_by_current_author += 1

                    titles_spanned = None(self.generateHTMLByGenre, genre if index == 0 else False, genre_tag_set[genre], '%s/Genre_%s.html' % (self.contentDir, genre))
                    tag_file = 'content/Genre_%s.html' % genre
                    master_genre_list.append({
                        'tag': genre,
                        'file': tag_file,
                        'authors': unique_authors,
                        'books': genre_tag_set[genre],
                        'titles_spanned': titles_spanned })


            if False and self.opts.verbose:
                for genre in master_genre_list:
                    print "genre['tag']: %s" % genre['tag']
                    for book in genre['books']:
                        print book['title']



            self.genres = master_genre_list


        def generateThumbnails(self):
            self.updateProgressFullStep("'Thumbnails'")
            thumbs = [
                'thumbnail_default.jpg']
            image_dir = '%s/images' % self.catalogPath
            for i, title in enumerate(self.booksByTitle):
                self.updateProgressMicroStep('Thumbnail %d of %d' % (i, len(self.booksByTitle)), i / float(len(self.booksByTitle)))
                if 'cover' in title and os.path.isfile(title['cover']):
                    thumbs.append('thumbnail_%d.jpg' % int(title['id']))
                    thumb_fp = '%s/thumbnail_%d.jpg' % (image_dir, int(title['id']))
                    thumb_file = 'thumbnail_%d.jpg' % int(title['id'])
                    if os.path.isfile(thumb_fp):
                        cover_timestamp = os.path.getmtime(title['cover'])
                        thumb_timestamp = os.path.getmtime(thumb_fp)
                        if thumb_timestamp < cover_timestamp:
                            self.generateThumbnail(title, image_dir, thumb_file)

                    else:
                        self.generateThumbnail(title, image_dir, thumb_file)
                os.path.isfile(thumb_fp)
                if False and self.verbose:
                    self.opts.log.warn(" using default cover for '%s'" % title['title'])

                thumb_fp = '%s/thumbnail_default.jpg' % image_dir
                cover = '%s/DefaultCover.png' % self.catalogPath
                if not os.path.exists(cover):
                    shutil.copyfile(I('book.png'), cover)

                if os.path.isfile(thumb_fp):
                    cover_timestamp = os.path.getmtime(cover)
                    thumb_timestamp = os.path.getmtime(thumb_fp)
                    if thumb_timestamp < cover_timestamp:
                        if False and self.verbose:
                            self.opts.log.warn('updating thumbnail_default for %s' % title['title'])

                        title['cover'] = cover
                        self.generateThumbnail(title, image_dir, 'thumbnail_default.jpg')

                thumb_timestamp < cover_timestamp
                if False and self.verbose:
                    self.opts.log.warn(' generating new thumbnail_default.jpg')

                title['cover'] = cover
                self.generateThumbnail(title, image_dir, 'thumbnail_default.jpg')

            self.thumbs = thumbs


        def generateOPF(self):
            self.updateProgressFullStep('Generating OPF')
            header = '\n                <?xml version="1.0" encoding="UTF-8"?>\n                <package xmlns="http://www.idpf.org/2007/opf" version="2.0" unique-identifier="calibre_id">\n                    <metadata xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns:opf="http://www.idpf.org/2007/opf" xmlns:calibre="http://calibre.kovidgoyal.net/2009/metadata" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">\n                        <dc:language>en-US</dc:language>\n                        <meta name="calibre:publication_type" content="periodical:default"/>\n                    </metadata>\n                    <manifest></manifest>\n                    <spine toc="ncx"></spine>\n                    <guide></guide>\n                </package>\n                '
            soup = BeautifulStoneSoup(header, selfClosingTags = [
                'item',
                'itemref',
                'reference'])
            metadata = soup.find('metadata')
            mtc = 0
            titleTag = Tag(soup, 'dc:title')
            titleTag.insert(0, self.title)
            metadata.insert(mtc, titleTag)
            mtc += 1
            creatorTag = Tag(soup, 'dc:creator')
            creatorTag.insert(0, self.creator)
            metadata.insert(mtc, creatorTag)
            mtc += 1
            manifest = soup.find('manifest')
            mtc = 0
            spine = soup.find('spine')
            stc = 0
            guide = soup.find('guide')
            itemTag = Tag(soup, 'item')
            itemTag['id'] = 'ncx'
            itemTag['href'] = '%s.ncx' % self.basename
            itemTag['media-type'] = 'application/x-dtbncx+xml'
            manifest.insert(mtc, itemTag)
            mtc += 1
            itemTag = Tag(soup, 'item')
            itemTag['id'] = 'stylesheet'
            itemTag['href'] = self.stylesheet
            itemTag['media-type'] = 'text/css'
            manifest.insert(mtc, itemTag)
            mtc += 1
            itemTag = Tag(soup, 'item')
            itemTag['id'] = 'mastheadimage-image'
            itemTag['href'] = 'images/mastheadImage.gif'
            itemTag['media-type'] = 'image/gif'
            manifest.insert(mtc, itemTag)
            mtc += 1
            for thumb in self.thumbs:
                itemTag = Tag(soup, 'item')
                itemTag['href'] = 'images/%s' % thumb
                end = thumb.find('.jpg')
                itemTag['id'] = '%s-image' % thumb[:end]
                itemTag['media-type'] = 'image/jpeg'
                manifest.insert(mtc, itemTag)
                mtc += 1

            sort_descriptions_by = None if self.opts.sort_descriptions_by_author else self.booksByTitle
            for file in self.htmlFileList:
                itemTag = Tag(soup, 'item')
                start = file.find('/') + 1
                end = file.find('.')
                itemTag['href'] = file
                itemTag['id'] = file[start:end].lower()
                itemTag['media-type'] = 'application/xhtml+xml'
                manifest.insert(mtc, itemTag)
                mtc += 1
                itemrefTag = Tag(soup, 'itemref')
                itemrefTag['idref'] = file[start:end].lower()
                spine.insert(stc, itemrefTag)
                stc += 1

            for genre in self.genres:
                if False:
                    self.opts.log.info('adding %s to manifest and spine' % genre['tag'])

                itemTag = Tag(soup, 'item')
                start = genre['file'].find('/') + 1
                end = genre['file'].find('.')
                itemTag['href'] = genre['file']
                itemTag['id'] = genre['file'][start:end].lower()
                itemTag['media-type'] = 'application/xhtml+xml'
                manifest.insert(mtc, itemTag)
                mtc += 1
                itemrefTag = Tag(soup, 'itemref')
                itemrefTag['idref'] = genre['file'][start:end].lower()
                spine.insert(stc, itemrefTag)
                stc += 1

            for book in sort_descriptions_by:
                itemTag = Tag(soup, 'item')
                itemTag['href'] = 'content/book_%d.html' % int(book['id'])
                itemTag['id'] = 'book%d' % int(book['id'])
                itemTag['media-type'] = 'application/xhtml+xml'
                manifest.insert(mtc, itemTag)
                mtc += 1
                itemrefTag = Tag(soup, 'itemref')
                itemrefTag['idref'] = 'book%d' % int(book['id'])
                spine.insert(stc, itemrefTag)
                stc += 1

            referenceTag = Tag(soup, 'reference')
            referenceTag['type'] = 'masthead'
            referenceTag['title'] = 'mastheadimage-image'
            referenceTag['href'] = 'images/mastheadImage.gif'
            guide.insert(0, referenceTag)
            outfile = open('%s/%s.opf' % (self.catalogPath, self.basename), 'w')
            outfile.write(soup.prettify())


        def generateNCXHeader(self):
            self.updateProgressFullStep('NCX header')
            header = '\n                <?xml version="1.0" encoding="utf-8"?>\n                <ncx xmlns="http://www.daisy.org/z3986/2005/ncx/" xmlns:calibre="http://calibre.kovidgoyal.net/2009/metadata" version="2005-1" xml:lang="en">\n                </ncx>\n            '
            soup = BeautifulStoneSoup(header, selfClosingTags = [
                'content',
                'calibre:meta-img'])
            ncx = soup.find('ncx')
            navMapTag = Tag(soup, 'navMap')
            navPointTag = Tag(soup, 'navPoint')
            navPointTag['class'] = 'periodical'
            navPointTag['id'] = 'title'
            navPointTag['playOrder'] = self.playOrder
            self.playOrder += 1
            navLabelTag = Tag(soup, 'navLabel')
            textTag = Tag(soup, 'text')
            textTag.insert(0, NavigableString(self.title))
            navLabelTag.insert(0, textTag)
            navPointTag.insert(0, navLabelTag)
            contentTag = Tag(soup, 'content')
            contentTag['src'] = 'content/ByAlphaAuthor.html'
            navPointTag.insert(1, contentTag)
            cmiTag = Tag(soup, 'calibre:meta-img')
            cmiTag['name'] = 'mastheadImage'
            cmiTag['src'] = 'images/mastheadImage.gif'
            navPointTag.insert(2, cmiTag)
            navMapTag.insert(0, navPointTag)
            ncx.insert(0, navMapTag)
            self.ncxSoup = soup


        def generateNCXDescriptions(self, tocTitle):
            self.updateProgressFullStep("NCX 'Descriptions'")
            ncx_soup = self.ncxSoup
            body = ncx_soup.find('navPoint')
            btc = len(body.contents)
            navPointTag = Tag(ncx_soup, 'navPoint')
            navPointTag['class'] = 'section'
            navPointTag['id'] = 'bytitle-ID'
            navPointTag['playOrder'] = self.playOrder
            self.playOrder += 1
            navLabelTag = Tag(ncx_soup, 'navLabel')
            textTag = Tag(ncx_soup, 'text')
            textTag.insert(0, NavigableString(tocTitle))
            navLabelTag.insert(0, textTag)
            nptc = 0
            navPointTag.insert(nptc, navLabelTag)
            nptc += 1
            contentTag = Tag(ncx_soup, 'content')
            contentTag['src'] = 'content/book_%d.html' % int(self.booksByTitle[0]['id'])
            navPointTag.insert(nptc, contentTag)
            nptc += 1
            sort_descriptions_by = self if self.opts.sort_descriptions_by_author else self.booksByTitle
            for book in sort_descriptions_by:
                navPointVolumeTag = Tag(ncx_soup, 'navPoint')
                navPointVolumeTag['class'] = 'article'
                navPointVolumeTag['id'] = 'book%dID' % int(book['id'])
                navPointVolumeTag['playOrder'] = self.playOrder
                self.playOrder += 1
                navLabelTag = Tag(ncx_soup, 'navLabel')
                textTag = Tag(ncx_soup, 'text')
                if book['series']:
                    tokens = list(book['title'].partition(':'))
                    if self.generateForKindle:
                        textTag.insert(0, NavigableString(self.formatNCXText('%s (%s)' % (tokens[2].strip(), tokens[0]), dest = 'title')))
                    else:
                        textTag.insert(0, NavigableString(self.formatNCXText('%s · %s (%s)' % (tokens[2].strip(), book['author'], tokens[0]), dest = 'title')))
                elif self.generateForKindle:
                    title_str = self.formatNCXText('%s' % book['title'], dest = 'title')
                    if self.opts.connected_kindle and book['id'] in self.bookmarked_books:
                        title_str += '*'

                    textTag.insert(0, NavigableString(title_str))
                else:
                    textTag.insert(0, NavigableString(self.formatNCXText('%s · %s' % (book['title'], book['author']), dest = 'title')))
                navLabelTag.insert(0, textTag)
                navPointVolumeTag.insert(0, navLabelTag)
                contentTag = Tag(ncx_soup, 'content')
                contentTag['src'] = 'content/book_%d.html#book%d' % (int(book['id']), int(book['id']))
                navPointVolumeTag.insert(1, contentTag)
                if self.generateForKindle:
                    cmTag = Tag(ncx_soup, 'calibre:meta')
                    cmTag['name'] = 'author'
                    navStr = '%s | %s' % (self.formatNCXText(book['author'], dest = 'author'), book['date'].split()[1])
                    if 'tags' in book and len(book['tags']):
                        navStr = self.formatNCXText(navStr + ' | ' + ' · '.join(sorted(book['tags'])), dest = 'author')

                    cmTag.insert(0, NavigableString(navStr))
                    navPointVolumeTag.insert(2, cmTag)
                    if book['short_description']:
                        cmTag = Tag(ncx_soup, 'calibre:meta')
                        cmTag['name'] = 'description'
                        cmTag.insert(0, NavigableString(self.formatNCXText(book['short_description'], dest = 'description')))
                        navPointVolumeTag.insert(3, cmTag)


                navPointTag.insert(nptc, navPointVolumeTag)
                nptc += 1

            body.insert(btc, navPointTag)
            btc += 1
            self.ncxSoup = ncx_soup


        def generateNCXBySeries(self, tocTitle):
            self.updateProgressFullStep("NCX 'Series'")

            def add_to_series_by_letter(current_series_list):
                current_series_list = ' • '.join(current_series_list)
                current_series_list = self.formatNCXText(current_series_list, dest = 'description')
                series_by_letter.append(current_series_list)

            soup = self.ncxSoup
            output = 'BySeries'
            body = soup.find('navPoint')
            btc = len(body.contents)
            navPointTag = Tag(soup, 'navPoint')
            navPointTag['class'] = 'section'
            navPointTag['id'] = 'byseries-ID'
            navPointTag['playOrder'] = self.playOrder
            self.playOrder += 1
            navLabelTag = Tag(soup, 'navLabel')
            textTag = Tag(soup, 'text')
            textTag.insert(0, NavigableString(tocTitle))
            navLabelTag.insert(0, textTag)
            nptc = 0
            navPointTag.insert(nptc, navLabelTag)
            nptc += 1
            contentTag = Tag(soup, 'content')
            contentTag['src'] = 'content/%s.html#section_start' % output
            navPointTag.insert(nptc, contentTag)
            nptc += 1
            series_by_letter = []
            title_list = self.booksBySeries
            current_letter = self.letter_or_symbol(title_list[0]['series'][0])
            title_letters = [
                current_letter]
            current_series_list = []
            current_series = ''
            for book in title_list:
                sort_title = self.generateSortTitle(book['series'])
                if self.letter_or_symbol(sort_title[0]) != current_letter:
                    add_to_series_by_letter(current_series_list)
                    current_letter = self.letter_or_symbol(sort_title[0])
                    title_letters.append(current_letter)
                    current_series = book['series']
                    current_series_list = [
                        book['series']]
                    continue
                self
                if len(current_series_list) < self.descriptionClip and book['series'] != current_series:
                    current_series = book['series']
                    current_series_list.append(book['series'])
                    continue
                (None, None)

            add_to_series_by_letter(current_series_list)
            for i, books in enumerate(series_by_letter):
                navPointByLetterTag = Tag(soup, 'navPoint')
                navPointByLetterTag['class'] = 'article'
                navPointByLetterTag['id'] = '%sSeries-ID' % title_letters[i].upper()
                navPointTag['playOrder'] = self.playOrder
                self.playOrder += 1
                navLabelTag = Tag(soup, 'navLabel')
                textTag = Tag(soup, 'text')
                self(textTag.insert, 0(NavigableString % u'Series beginning with %s' if len(title_letters[i]) > 1 else "'" + title_letters[i] + "'"))
                navLabelTag.insert(0, textTag)
                navPointByLetterTag.insert(0, navLabelTag)
                contentTag = Tag(soup, 'content')
                contentTag['src'] = 'content/%s.html#%s_series' % (output, title_letters[i])
                navPointByLetterTag.insert(1, contentTag)
                if self.generateForKindle:
                    cmTag = Tag(soup, 'calibre:meta')
                    cmTag['name'] = 'description'
                    cmTag.insert(0, NavigableString(self.formatNCXText(books, dest = 'description')))
                    navPointByLetterTag.insert(2, cmTag)

                navPointTag.insert(nptc, navPointByLetterTag)
                nptc += 1

            body.insert(btc, navPointTag)
            btc += 1
            self.ncxSoup = soup


        def generateNCXByTitle(self, tocTitle):
            self.updateProgressFullStep("NCX 'Titles'")

            def add_to_books_by_letter(current_book_list):
                current_book_list = ' • '.join(current_book_list)
                current_book_list = self.formatNCXText(current_book_list, dest = 'description')
                books_by_letter.append(current_book_list)

            soup = self.ncxSoup
            output = 'ByAlphaTitle'
            body = soup.find('navPoint')
            btc = len(body.contents)
            navPointTag = Tag(soup, 'navPoint')
            navPointTag['class'] = 'section'
            navPointTag['id'] = 'byalphatitle-ID'
            navPointTag['playOrder'] = self.playOrder
            self.playOrder += 1
            navLabelTag = Tag(soup, 'navLabel')
            textTag = Tag(soup, 'text')
            textTag.insert(0, NavigableString(tocTitle))
            navLabelTag.insert(0, textTag)
            nptc = 0
            navPointTag.insert(nptc, navLabelTag)
            nptc += 1
            contentTag = Tag(soup, 'content')
            contentTag['src'] = 'content/%s.html#section_start' % output
            navPointTag.insert(nptc, contentTag)
            nptc += 1
            books_by_letter = []
            current_letter = self.letter_or_symbol(title_list[0]['title_sort'][0])
            title_letters = [
                current_letter]
            current_book_list = []
            current_book = ''
            for book in title_list:
                if self.letter_or_symbol(book['title_sort'][0]) != current_letter:
                    add_to_books_by_letter(current_book_list)
                    current_letter = self.letter_or_symbol(book['title_sort'][0])
                    title_letters.append(current_letter)
                    current_book = book['title']
                    current_book_list = [
                        book['title']]
                    continue
                None if self.useSeriesPrefixInTitlesSection else (None, None)
                if len(current_book_list) < self.descriptionClip and book['title'] != current_book:
                    current_book = book['title']
                    current_book_list.append(book['title'])
                    continue

            add_to_books_by_letter(current_book_list)
            for i, books in enumerate(books_by_letter):
                navPointByLetterTag = Tag(soup, 'navPoint')
                navPointByLetterTag['class'] = 'article'
                navPointByLetterTag['id'] = '%sTitles-ID' % title_letters[i].upper()
                navPointTag['playOrder'] = self.playOrder
                self.playOrder += 1
                navLabelTag = Tag(soup, 'navLabel')
                textTag = Tag(soup, 'text')
                self(textTag.insert, 0(NavigableString % u'Titles beginning with %s' if len(title_letters[i]) > 1 else "'" + title_letters[i] + "'"))
                navLabelTag.insert(0, textTag)
                navPointByLetterTag.insert(0, navLabelTag)
                contentTag = Tag(soup, 'content')
                contentTag['src'] = 'content/%s.html#%s' % (output, title_letters[i])
                navPointByLetterTag.insert(1, contentTag)
                if self.generateForKindle:
                    cmTag = Tag(soup, 'calibre:meta')
                    cmTag['name'] = 'description'
                    cmTag.insert(0, NavigableString(self.formatNCXText(books, dest = 'description')))
                    navPointByLetterTag.insert(2, cmTag)

                navPointTag.insert(nptc, navPointByLetterTag)
                nptc += 1

            body.insert(btc, navPointTag)
            btc += 1
            self.ncxSoup = soup


        def generateNCXByAuthor(self, tocTitle):
            self.updateProgressFullStep("NCX 'Authors'")

            def add_to_author_list(current_author_list, current_letter):
                current_author_list = ' • '.join(current_author_list)
                current_author_list = self.formatNCXText(current_author_list, dest = 'description')
                master_author_list.append((current_author_list, current_letter))

            soup = self.ncxSoup
            HTML_file = 'content/ByAlphaAuthor.html'
            body = soup.find('navPoint')
            btc = len(body.contents)
            navPointTag = Tag(soup, 'navPoint')
            navPointTag['class'] = 'section'
            file_ID = '%s' % tocTitle.lower()
            file_ID = file_ID.replace(' ', '')
            navPointTag['id'] = '%s-ID' % file_ID
            navPointTag['playOrder'] = self.playOrder
            self.playOrder += 1
            navLabelTag = Tag(soup, 'navLabel')
            textTag = Tag(soup, 'text')
            textTag.insert(0, NavigableString('%s' % tocTitle))
            navLabelTag.insert(0, textTag)
            nptc = 0
            navPointTag.insert(nptc, navLabelTag)
            nptc += 1
            contentTag = Tag(soup, 'content')
            contentTag['src'] = '%s#section_start' % HTML_file
            navPointTag.insert(nptc, contentTag)
            nptc += 1
            master_author_list = []
            current_letter = self.letter_or_symbol(self.authors[0][1][0])
            current_author_list = []
            for author in self.authors:
                if self.letter_or_symbol(author[1][0]) != current_letter:
                    add_to_author_list(current_author_list, current_letter)
                    current_letter = self.letter_or_symbol(author[1][0])
                    current_author_list = [
                        author[0]]
                    continue
                self
                if len(current_author_list) < self.descriptionClip:
                    current_author_list.append(author[0])
                    continue
                (None, None)

            add_to_author_list(current_author_list, current_letter)
            for authors_by_letter in master_author_list:
                navPointByLetterTag = Tag(soup, 'navPoint')
                navPointByLetterTag['class'] = 'article'
                navPointByLetterTag['id'] = '%sauthors-ID' % authors_by_letter[1]
                navPointTag['playOrder'] = self.playOrder
                self.playOrder += 1
                navLabelTag = Tag(soup, 'navLabel')
                textTag = Tag(soup, 'text')
                textTag.insert(0, NavigableString("Authors beginning with '%s'" % authors_by_letter[1]))
                navLabelTag.insert(0, textTag)
                navPointByLetterTag.insert(0, navLabelTag)
                contentTag = Tag(soup, 'content')
                contentTag['src'] = '%s#%sauthors' % (HTML_file, authors_by_letter[1])
                navPointByLetterTag.insert(1, contentTag)
                if self.generateForKindle:
                    cmTag = Tag(soup, 'calibre:meta')
                    cmTag['name'] = 'description'
                    cmTag.insert(0, NavigableString(authors_by_letter[0]))
                    navPointByLetterTag.insert(2, cmTag)

                navPointTag.insert(nptc, navPointByLetterTag)
                nptc += 1

            body.insert(btc, navPointTag)
            btc += 1
            self.ncxSoup = soup


        def generateNCXByDateAdded(self, tocTitle):
            self.updateProgressFullStep("NCX 'Recently Added'")

            def add_to_master_month_list(current_titles_list):
                book_count = len(current_titles_list)
                current_titles_list = ' • '.join(current_titles_list)
                current_titles_list = self.formatNCXText(current_titles_list, dest = 'description')
                master_month_list.append((current_titles_list, current_date, book_count))


            def add_to_master_date_range_list(current_titles_list):
                book_count = len(current_titles_list)
                current_titles_list = ' • '.join(current_titles_list)
                current_titles_list = self.formatNCXText(current_titles_list, dest = 'description')
                master_date_range_list.append((current_titles_list, date_range, book_count))

            soup = self.ncxSoup
            HTML_file = 'content/ByDateAdded.html'
            body = soup.find('navPoint')
            btc = len(body.contents)
            navPointTag = Tag(soup, 'navPoint')
            navPointTag['class'] = 'section'
            file_ID = '%s' % tocTitle.lower()
            file_ID = file_ID.replace(' ', '')
            navPointTag['id'] = '%s-ID' % file_ID
            navPointTag['playOrder'] = self.playOrder
            self.playOrder += 1
            navLabelTag = Tag(soup, 'navLabel')
            textTag = Tag(soup, 'text')
            textTag.insert(0, NavigableString('%s' % tocTitle))
            navLabelTag.insert(0, textTag)
            nptc = 0
            navPointTag.insert(nptc, navLabelTag)
            nptc += 1
            contentTag = Tag(soup, 'content')
            contentTag['src'] = '%s#section_start' % HTML_file
            navPointTag.insert(nptc, contentTag)
            nptc += 1
            current_titles_list = []
            master_date_range_list = []
            today = datetime.datetime.now()
            today_time = datetime.datetime(today.year, today.month, today.day)
            for i, date in enumerate(self.DATE_RANGE):
                date_range_limit = self.DATE_RANGE[i]
                for book in self.booksByDateRange:
                    book_time = datetime.datetime(book['timestamp'].year, book['timestamp'].month, book['timestamp'].day)
                    if (today_time - book_time).days <= date_range_limit:
                        current_titles_list.append(book['title'])
                        continue
                    None if i else (None, None, (None, None, None))

                if current_titles_list:
                    add_to_master_date_range_list(current_titles_list)

                current_titles_list = [
                    book['title']]

            for books_by_date_range in master_date_range_list:
                navPointByDateRangeTag = Tag(soup, 'navPoint')
                navPointByDateRangeTag['class'] = 'article'
                navPointByDateRangeTag['id'] = '%s-ID' % books_by_date_range[1].replace(' ', '')
                navPointTag['playOrder'] = self.playOrder
                self.playOrder += 1
                navLabelTag = Tag(soup, 'navLabel')
                textTag = Tag(soup, 'text')
                textTag.insert(0, NavigableString(books_by_date_range[1]))
                navLabelTag.insert(0, textTag)
                navPointByDateRangeTag.insert(0, navLabelTag)
                contentTag = Tag(soup, 'content')
                contentTag['src'] = '%s#bda_%s' % (HTML_file, books_by_date_range[1].replace(' ', ''))
                navPointByDateRangeTag.insert(1, contentTag)
                navPointTag.insert(nptc, navPointByDateRangeTag)
                nptc += 1

            current_titles_list = []
            master_month_list = []
            current_date = self.booksByMonth[0]['timestamp']
            for book in self.booksByMonth:
                if book['timestamp'].month != current_date.month or book['timestamp'].year != current_date.year:
                    add_to_master_month_list(current_titles_list)
                    current_date = book['timestamp'].date()
                    current_titles_list = [
                        book['title']]
                    continue
                None if self.generateForKindle else self
                current_titles_list.append(book['title'])

            add_to_master_month_list(current_titles_list)
            for books_by_month in master_month_list:
                datestr = strftime(u'%B %Y', books_by_month[1].timetuple())
                navPointByMonthTag = Tag(soup, 'navPoint')
                navPointByMonthTag['class'] = 'article'
                navPointByMonthTag['id'] = 'bda_%s-%s-ID' % (books_by_month[1].year, books_by_month[1].month)
                navPointTag['playOrder'] = self.playOrder
                self.playOrder += 1
                navLabelTag = Tag(soup, 'navLabel')
                textTag = Tag(soup, 'text')
                textTag.insert(0, NavigableString(datestr))
                navLabelTag.insert(0, textTag)
                navPointByMonthTag.insert(0, navLabelTag)
                contentTag = Tag(soup, 'content')
                contentTag['src'] = '%s#bda_%s-%s' % (HTML_file, books_by_month[1].year, books_by_month[1].month)
                navPointByMonthTag.insert(1, contentTag)
                navPointTag.insert(nptc, navPointByMonthTag)
                nptc += 1

            body.insert(btc, navPointTag)
            btc += 1
            self.ncxSoup = soup


        def generateNCXByDateRead(self, tocTitle):
            self.updateProgressFullStep("NCX 'Recently Read'")
            if not self.booksByDateRead:
                return None

            def add_to_master_day_list(current_titles_list):
                book_count = len(current_titles_list)
                current_titles_list = ' • '.join(current_titles_list)
                current_titles_list = self.formatNCXText(current_titles_list, dest = 'description')
                master_day_list.append((current_titles_list, current_date, book_count))


            def add_to_master_date_range_list(current_titles_list):
                book_count = len(current_titles_list)
                current_titles_list = ' • '.join(current_titles_list)
                current_titles_list = self.formatNCXText(current_titles_list, dest = 'description')
                master_date_range_list.append((current_titles_list, date_range, book_count))

            soup = self.ncxSoup
            HTML_file = 'content/ByDateRead.html'
            body = soup.find('navPoint')
            btc = len(body.contents)
            navPointTag = Tag(soup, 'navPoint')
            navPointTag['class'] = 'section'
            file_ID = '%s' % tocTitle.lower()
            file_ID = file_ID.replace(' ', '')
            navPointTag['id'] = '%s-ID' % file_ID
            navPointTag['playOrder'] = self.playOrder
            self.playOrder += 1
            navLabelTag = Tag(soup, 'navLabel')
            textTag = Tag(soup, 'text')
            textTag.insert(0, NavigableString('%s' % tocTitle))
            navLabelTag.insert(0, textTag)
            nptc = 0
            navPointTag.insert(nptc, navLabelTag)
            nptc += 1
            contentTag = Tag(soup, 'content')
            contentTag['src'] = '%s#section_start' % HTML_file
            navPointTag.insert(nptc, contentTag)
            nptc += 1
            current_titles_list = []
            master_date_range_list = []
            today = datetime.datetime.now()
            today_time = datetime.datetime(today.year, today.month, today.day)
            for i, date in enumerate(self.DATE_RANGE):
                date_range_limit = self.DATE_RANGE[i]
                for book in self.booksByDateRead:
                    bookmark_time = datetime.datetime.utcfromtimestamp(book['bookmark_timestamp'])
                    if (today_time - bookmark_time).days <= date_range_limit:
                        current_titles_list.append(book['title'])
                        continue
                    None if i else (None, None, (None, None, self.booksByDateRead))

                if current_titles_list:
                    add_to_master_date_range_list(current_titles_list)

                current_titles_list = [
                    book['title']]

            current_titles_list = []
            master_day_list = []
            current_date = datetime.datetime.utcfromtimestamp(self.booksByDateRead[0]['bookmark_timestamp'])
            for book in self.booksByDateRead:
                bookmark_time = datetime.datetime.utcfromtimestamp(book['bookmark_timestamp'])
                if bookmark_time.day != current_date.day and bookmark_time.month != current_date.month or bookmark_time.year != current_date.year:
                    add_to_master_day_list(current_titles_list)
                    current_date = datetime.datetime.utcfromtimestamp(book['bookmark_timestamp']).date()
                    current_titles_list = [
                        book['title']]
                    continue
                current_titles_list.append(book['title'])

            add_to_master_day_list(current_titles_list)
            for books_by_day in master_day_list:
                datestr = strftime(u'%A, %B %d', books_by_day[1].timetuple())
                navPointByDayTag = Tag(soup, 'navPoint')
                navPointByDayTag['class'] = 'article'
                navPointByDayTag['id'] = 'bdr_%s-%s-%sID' % (books_by_day[1].year, books_by_day[1].month, books_by_day[1].day)
                navPointTag['playOrder'] = self.playOrder
                self.playOrder += 1
                navLabelTag = Tag(soup, 'navLabel')
                textTag = Tag(soup, 'text')
                textTag.insert(0, NavigableString(datestr))
                navLabelTag.insert(0, textTag)
                navPointByDayTag.insert(0, navLabelTag)
                contentTag = Tag(soup, 'content')
                contentTag['src'] = '%s#bdr_%s-%s-%s' % (HTML_file, books_by_day[1].year, books_by_day[1].month, books_by_day[1].day)
                navPointByDayTag.insert(1, contentTag)
                navPointTag.insert(nptc, navPointByDayTag)
                nptc += 1

            body.insert(btc, navPointTag)
            btc += 1
            self.ncxSoup = soup


        def generateNCXByGenre(self, tocTitle):
            self.updateProgressFullStep("NCX 'Genres'")
            if not len(self.genres):
                self.opts.log.warn(' No genres found in tags.\n No Genre section added to Catalog')
                return None
            ncx_soup = self.ncxSoup
            body = ncx_soup.find('navPoint')
            btc = len(body.contents)
            navPointTag = Tag(ncx_soup, 'navPoint')
            navPointTag['class'] = 'section'
            file_ID = '%s' % tocTitle.lower()
            file_ID = file_ID.replace(' ', '')
            navPointTag['id'] = '%s-ID' % file_ID
            navPointTag['playOrder'] = self.playOrder
            self.playOrder += 1
            navLabelTag = Tag(ncx_soup, 'navLabel')
            textTag = Tag(ncx_soup, 'text')
            textTag.insert(0, NavigableString('%s' % tocTitle))
            navLabelTag.insert(0, textTag)
            nptc = 0
            navPointTag.insert(nptc, navLabelTag)
            nptc += 1
            contentTag = Tag(ncx_soup, 'content')
            contentTag['src'] = 'content/Genre_%s.html#section_start' % self.genres[0]['tag']
            navPointTag.insert(nptc, contentTag)
            nptc += 1
            for genre in self.genres:
                navPointVolumeTag = Tag(ncx_soup, 'navPoint')
                navPointVolumeTag['class'] = 'article'
                navPointVolumeTag['id'] = 'genre-%s-ID' % genre['tag']
                navPointVolumeTag['playOrder'] = self.playOrder
                self.playOrder += 1
                navLabelTag = Tag(ncx_soup, 'navLabel')
                textTag = Tag(ncx_soup, 'text')
                normalized_tag = None
                for friendly_tag in self.genre_tags_dict:
                    if self.genre_tags_dict[friendly_tag] == genre['tag']:
                        normalized_tag = self.genre_tags_dict[friendly_tag]
                        break
                        continue
                    self

                textTag.insert(0, self.formatNCXText(NavigableString(friendly_tag), dest = 'description'))
                navLabelTag.insert(0, textTag)
                navPointVolumeTag.insert(0, navLabelTag)
                contentTag = Tag(ncx_soup, 'content')
                contentTag['src'] = 'content/Genre_%s.html#Genre_%s' % (normalized_tag, normalized_tag)
                navPointVolumeTag.insert(1, contentTag)
                if self.generateForKindle:
                    cmTag = Tag(ncx_soup, 'calibre:meta')
                    cmTag['name'] = 'author'
                    cmTag.insert(0, NavigableString(author_range))
                    navPointVolumeTag.insert(2, cmTag)
                    cmTag = Tag(ncx_soup, 'calibre:meta')
                    cmTag['name'] = 'description'
                    navPointVolumeTag.insert(3, cmTag)

                navPointTag.insert(nptc, navPointVolumeTag)
                nptc += 1

            body.insert(btc, navPointTag)
            btc += 1
            self.ncxSoup = ncx_soup


        def writeNCX(self):
            self.updateProgressFullStep('Saving NCX')
            outfile = open('%s/%s.ncx' % (self.catalogPath, self.basename), 'w')
            outfile.write(self.ncxSoup.prettify())


        def author_to_author_sort(self, author):
            tokens = author.split()
            tokens = tokens[-1:] + tokens[:-1]
            if len(tokens) > 1:
                tokens[0] += ','

            return ' '.join(tokens).capitalize()


        def author_compare(self, x, y):
            if x['author_sort'].capitalize() > y['author_sort'].capitalize():
                return 1
            if x['author_sort'].capitalize() < y['author_sort'].capitalize():
                return -1
            if x['series'] != y['series']:
                if not x['series']:
                    return -1
                if not y['series']:
                    return 1
                if x['title_sort'].lstrip() > y['title_sort'].lstrip():
                    return 1
                return -1
            x['series'] != y['series']
            if x['series'] == y['series']:
                if float(x['series_index']) > float(y['series_index']):
                    return 1
                if float(x['series_index']) < float(y['series_index']):
                    return -1
                return 0
            x['series'] == y['series']
            if x['series'] > y['series']:
                return 1
            return -1


        def calculateThumbnailSize(self):
            output_profiles = output_profiles
            import calibre.customize.ui
            for x in output_profiles():
                if x.short_name == self.opts.output_profile:
                    self.thumbWidth = int(x.dpi * 0.9)
                    self.thumbHeight = int(self.thumbWidth * 1.33)
                    if 'kindle' in x.short_name and self.opts.fmt == 'mobi':
                        self.thumbWidth = int(self.thumbWidth / 2)
                        self.thumbHeight = int(self.thumbHeight / 2)

                    break
                    continue

            if False and self.verbose:
                self.opts.log('     DPI = %d; thumbnail dimensions: %d x %d' % (x.dpi, self.thumbWidth, self.thumbHeight))



        def convertHTMLEntities(self, s):
            matches = re.findall('&#\\d+;', s)
            if len(matches) > 0:
                hits = set(matches)
                for hit in hits:
                    name = hit[2:-1]

                    try:
                        entnum = int(name)
                        s = s.replace(hit, unichr(entnum))
                    continue
                    except ValueError:
                        continue




            matches = re.findall('&\\w+;', s)
            hits = set(matches)
            amp = '&'
            if amp in hits:
                hits.remove(amp)

            for hit in hits:
                name = hit[1:-1]
                if htmlentitydefs.name2codepoint.has_key(name):
                    s = s.replace(hit, unichr(htmlentitydefs.name2codepoint[name]))
                    continue

            s = s.replace(amp, '&')
            return s


        def createDirectoryStructure(self):
            catalogPath = self.catalogPath
            self.cleanUp()
            if not os.path.isdir(catalogPath):
                os.makedirs(catalogPath)

            content_path = catalogPath + '/content'
            if not os.path.isdir(content_path):
                os.makedirs(content_path)

            images_path = catalogPath + '/images'
            if not os.path.isdir(images_path):
                os.makedirs(images_path)



        def filterDbTags(self, tags):
            normalized_tags = []
            friendly_tags = []
            for tag in tags:
                if tag[0] in self.markerTags:
                    continue

                if re.search(self.opts.exclude_genre, tag):
                    continue

                if tag == ' ':
                    continue

                normalized_tags.append(re.sub('\\W', '', tag).lower())
                friendly_tags.append(tag)

            genre_tags_dict = dict(zip(friendly_tags, normalized_tags))
            normalized_set = set(normalized_tags)
            for normalized in normalized_set:
                if normalized_tags.count(normalized) > 1:
                    self.opts.log.warn("      Warning: multiple tags resolving to genre '%s':" % normalized)
                    for key in genre_tags_dict:
                        if genre_tags_dict[key] == normalized:
                            self.opts.log.warn('       %s' % key)
                            continue


            return genre_tags_dict


        def formatNCXText(self, description, dest = None):
            massaged = unicode(BeautifulStoneSoup(description, convertEntities = BeautifulStoneSoup.HTML_ENTITIES))
            massaged = re.sub('&', '&', massaged)
            if massaged.strip() and dest:
                return self.generateShortDescription(massaged.strip(), dest = dest)
            return None


        def generateAuthorAnchor(self, author):
            return re.sub('\\W', '', author)


        def generateHTMLByGenre(self, genre, section_head, books, outfile):
            soup = self.generateHTMLGenreHeader(genre)
            body = soup.find('body')
            btc = 0
            if section_head:
                aTag = Tag(soup, 'a')
                aTag['name'] = 'section_start'
                body.insert(btc, aTag)
                btc += 1

            aTag = Tag(soup, 'a')
            aTag['name'] = 'Genre_%s' % genre
            body.insert(btc, aTag)
            btc += 1
            titleTag = body.find(attrs = {
                'class': 'title' })
            titleTag.insert(0, NavigableString('%s' % escape(self.getFriendlyGenreTag(genre))))
            divTag = body.find(attrs = {
                'class': 'authors' })
            dtc = 0
            current_author = ''
            current_series = None
            for book in books:
                if book['author'] != current_author:
                    current_author = book['author']
                    non_series_books = 0
                    current_series = None
                    pAuthorTag = Tag(soup, 'p')
                    pAuthorTag['class'] = 'author_index'
                    aTag = Tag(soup, 'a')
                    aTag['href'] = '%s.html#%s' % ('ByAlphaAuthor', self.generateAuthorAnchor(book['author']))
                    aTag.insert(0, book['author'])
                    pAuthorTag.insert(0, aTag)
                    divTag.insert(dtc, pAuthorTag)
                    dtc += 1

                if book['series'] and book['series'] != current_series:
                    current_series = book['series']
                    pSeriesTag = Tag(soup, 'p')
                    pSeriesTag['class'] = 'series'
                    if self.opts.generate_series:
                        aTag = Tag(soup, 'a')
                        aTag['href'] = '%s.html#%s_series' % ('BySeries', re.sub('\\W', '', book['series']).lower())
                        aTag.insert(0, book['series'])
                        pSeriesTag.insert(0, aTag)
                    else:
                        pSeriesTag.insert(0, NavigableString('%s' % book['series']))
                    divTag.insert(dtc, pSeriesTag)
                    dtc += 1

                if current_series and not book['series']:
                    current_series = None

                pBookTag = Tag(soup, 'p')
                ptc = 0
                if book['read']:
                    pBookTag.insert(ptc, NavigableString(self.READ_SYMBOL))
                    pBookTag['class'] = 'read_book'
                    ptc += 1
                elif book['id'] in self.bookmarked_books:
                    pBookTag.insert(ptc, NavigableString(self.READING_SYMBOL))
                    pBookTag['class'] = 'read_book'
                    ptc += 1
                else:
                    pBookTag['class'] = 'unread_book'
                    pBookTag.insert(ptc, NavigableString(self.NOT_READ_SYMBOL))
                    ptc += 1
                aTag = Tag(soup, 'a')
                aTag['href'] = 'book_%d.html' % int(float(book['id']))
                if current_series:
                    aTag.insert(0, escape(book['title'][len(book['series']) + 1:]))
                else:
                    aTag.insert(0, escape(book['title']))
                    non_series_books += 1
                pBookTag.insert(ptc, aTag)
                ptc += 1
                divTag.insert(dtc, pBookTag)
                dtc += 1

            outfile = open(outfile, 'w')
            outfile.write(soup.prettify())
            outfile.close()
            if len(books) > 1:
                titles_spanned = [
                    (books[0]['author'], books[0]['title']),
                    (books[-1]['author'], books[-1]['title'])]
            else:
                titles_spanned = [
                    (books[0]['author'], books[0]['title'])]
            return titles_spanned


        def generateHTMLDescriptionHeader(self, title):
            title_border = None if self.opts.fmt == 'epub' else '<hr class="description_divider"/>'
            header = '\n            <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.1//EN" "http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd">\n            <html xmlns="http://www.w3.org/1999/xhtml" xmlns:calibre="http://calibre.kovidgoyal.net/2009/metadata">\n            <head>\n            <meta http-equiv="Content-Type" content="text/html; charset=UTF-8" />\n                <link rel="stylesheet" type="text/css" href="stylesheet.css" media="screen" />\n            <title></title>\n            </head>\n            <body>\n            <p class="title"></p>\n            {0}\n            <p class="author"></p>\n            <!--p class="series"></p-->\n            <p class="tags"> </p>\n            <table width="100%" border="0">\n              <tr>\n                <td class="thumbnail" rowspan="7"></td>\n                <td> </td>\n                <td> </td>\n              </tr>\n              <tr>\n                <td> </td>\n                <td> </td>\n              </tr>\n              <tr>\n                <td>Publisher</td>\n                <td class="publisher"></td>\n              </tr>\n              <tr>\n                <td>Published</td>\n                <td class="date"></td>\n              </tr>\n              <tr>\n                <td>Rating</td>\n                <td class="rating"></td>\n              </tr>\n              <tr>\n                <td class="notes_label">Notes</td>\n                <td class="notes"></td>\n              </tr>\n              <tr>\n                <td> </td>\n                <td> </td>\n              </tr>\n            </table>\n            <hr class="description_divider" />\n            <div class="description"></div>\n            </body>\n            </html>\n            '.format(title_border)
            soup = BeautifulSoup(header, selfClosingTags = [
                'mbp:pagebreak'])
            titleTag = soup.find('title')
            titleTag.insert(0, NavigableString(escape(title)))
            return soup


        def generateHTMLEmptyHeader(self, title):
            header = '\n                <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.1//EN" "http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd">\n                <html xmlns="http://www.w3.org/1999/xhtml" xmlns:calibre="http://calibre.kovidgoyal.net/2009/metadata">\n                <head>\n                <meta http-equiv="Content-Type" content="text/html; charset=UTF-8" />\n                    <link rel="stylesheet" type="text/css" href="stylesheet.css" media="screen" />\n                <title></title>\n                </head>\n                <body>\n                </body>\n                </html>\n                '
            soup = BeautifulSoup(header)
            titleTag = soup.find('title')
            titleTag.insert(0, NavigableString(title))
            return soup


        def generateHTMLGenreHeader(self, title):
            header = '\n                <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.1//EN" "http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd">\n                <html xmlns="http://www.w3.org/1999/xhtml" xmlns:calibre="http://calibre.kovidgoyal.net/2009/metadata">\n                <head>\n                <meta http-equiv="Content-Type" content="text/html; charset=UTF-8" />\n                    <link rel="stylesheet" type="text/css" href="stylesheet.css" media="screen" />\n                <title></title>\n                </head>\n                <body>\n                    <p class="title"></p>\n                    <!--div class="hr"><blockquote><hr/></blockquote></div-->\n                    <div class="authors"></div>\n                </body>\n                </html>\n                '
            soup = BeautifulSoup(header)
            titleTag = soup.find('title')
            titleTag.insert(0, escape(NavigableString(title)))
            return soup


        def generateMastheadImage(self, out_path):
            load_defaults = load_defaults
            import calibre.ebooks.conversion.config
            fontconfig = fontconfig
            import calibre.utils.fonts
            font_path = default_font = P('fonts/liberation/LiberationSerif-Bold.ttf')
            recs = load_defaults('mobi_output')
            masthead_font_family = recs.get('masthead_font', 'Default')
            if masthead_font_family != 'Default':
                masthead_font = fontconfig.files_for_family(masthead_font_family)
                if 'normal' in masthead_font:
                    font_path = masthead_font['normal'][0]


            if not font_path or not os.access(font_path, os.R_OK):
                font_path = default_font

            MI_WIDTH = 600
            MI_HEIGHT = 60

            try:
                Image = Image
                ImageDraw = ImageDraw
                ImageFont = ImageFont
                import PIL
                (Image, ImageDraw, ImageFont)
            except ImportError:
                import Image
                import ImageDraw
                import ImageFont

            img = Image.new('RGB', (MI_WIDTH, MI_HEIGHT), 'white')
            draw = ImageDraw.Draw(img)

            try:
                font = ImageFont.truetype(font_path, 48)
            except:
                self.opts.log.error("     Failed to load user-specifed font '%s'" % font_path)
                font = ImageFont.truetype(default_font, 48)

            text = self.title.encode('utf-8')
            (width, height) = draw.textsize(text, font = font)
            left = max(int((MI_WIDTH - width) / 2), 0)
            top = max(int((MI_HEIGHT - height) / 2), 0)
            draw.text((left, top), text, fill = (0, 0, 0), font = font)
            img.save(open(out_path, 'wb'), 'GIF')


        def generateSeriesTitle(self, title):
            if float(title['series_index']) - int(title['series_index']):
                series_title = '%s %4.2f: %s' % (title['series'], title['series_index'], title['title'])
            else:
                series_title = '%s %d: %s' % (title['series'], title['series_index'], title['title'])
            return series_title


        def generateShortDescription(self, description, dest = None):

            def shortDescription(description, limit):
                short_description = ''
                words = description.split()
                for word in words:
                    short_description += word + ' '
                    if len(short_description) > limit:
                        short_description += '...'
                        return short_description


            if not description:
                return None
            if dest == 'title':
                return description
            if dest == 'author':
                if self.authorClip and len(description) < self.authorClip:
                    return description
                return shortDescription(description, self.authorClip)
            dest == 'author'
            if dest == 'description':
                if self.descriptionClip and len(description) < self.descriptionClip:
                    return description
                return shortDescription(description, self.descriptionClip)
            dest == 'description'
            print " returning description with unspecified destination '%s'" % description
            raise RuntimeError


        def generateSortTitle(self, title):
            title_sort = title_sort
            import calibre.ebooks.metadata
            title_words = title_sort(title).split()
            translated = []
            for i, word in enumerate(title_words):
                if i == 0:
                    if self.opts.numbers_as_text and re.match('[0-9]+', word[0]):
                        translated.append(EPUB_MOBI.NumberToText(word).text.capitalize())
                    elif re.match('[0-9]+', word[0]):
                        word = word.replace(',', '')
                        suffix = re.search('[\\D]', word)
                        if suffix:
                            word = '%10.0f%s' % (float(word[:suffix.start()]), word[suffix.start():])
                        else:
                            word = '%10.0f' % float(word)

                    if self.letter_or_symbol(word[0]) != word[0]:
                        if not word[0] > 'A':
                            if ord(word[0]) < ord(word[0]):
                                pass
                            elif ord(word[0]) < ord('A'):
                                translated.append('/')


                    translated.append(word.capitalize())
                    continue
                if re.search('[0-9]+', word[0]):
                    word = word.replace(',', '')
                    suffix = re.search('[\\D]', word)
                    if suffix:
                        word = '%10.0f%s' % (float(word[:suffix.start()]), word[suffix.start():])
                    else:
                        word = '%10.0f' % float(word)

                translated.append(word)

            return ' '.join(translated)


        def generateThumbnail(self, title, image_dir, thumb_file):
            Image = Image
            import calibre.utils.magick

            try:
                img = Image()
                img.open(title['cover'])
                img.thumbnail(self.thumbWidth, self.thumbHeight)
                img.save(os.path.join(image_dir, thumb_file))
            except:
                self.opts.log.error('generateThumbnail(): Error with %s' % title['title'])



        def getFriendlyGenreTag(self, genre):
            for friendly_tag in self.genre_tags_dict:
                if self.genre_tags_dict[friendly_tag] == genre:
                    return friendly_tag



        def getMarkerTags(self):
            markerTags = []
            markerTags.extend(self.opts.exclude_tags.split(','))
            markerTags.extend(self.opts.note_tag.split(','))
            markerTags.extend(self.opts.read_tag.split(','))
            return markerTags


        def letter_or_symbol(self, char):
            if not re.search('[a-zA-Z]', char):
                return 'Symbols'
            return char


        def markdownComments(self, comments):
            for lost_cr in re.finditer('([a-z])([\\.\\?!])([A-Z])', comments):
                comments = comments.replace(lost_cr.group(), '%s%s\n\n%s' % (lost_cr.group(1), lost_cr.group(2), lost_cr.group(3)))

            if not isinstance(comments, unicode):
                comments = comments.decode('utf-8', 'replace')

            soup = BeautifulSoup(comments)
            elems = soup.findAll('div')
            for elem in elems:
                elem.extract()

            comments = soup.renderContents(None)
            if re.search('\n\n', comments):
                soup = BeautifulSoup()
                split_ps = comments.split(u'\n\n')
                tsc = 0
                for p in split_ps:
                    pTag = Tag(soup, 'p')
                    pTag.insert(0, p)
                    soup.insert(tsc, pTag)
                    tsc += 1

                comments = soup.renderContents(None)

            comments = re.sub('[\r\n]', '<br />', comments)
            comments = re.sub('--', '—', comments)
            soup = BeautifulSoup(comments)
            result = BeautifulSoup()
            rtc = 0
            open_pTag = False
            all_tokens = list(soup.contents)
            for token in all_tokens:
                if type(token) is NavigableString:
                    if not open_pTag:
                        pTag = Tag(result, 'p')
                        open_pTag = True
                        ptc = 0

                    pTag.insert(ptc, prepare_string_for_xml(token))
                    ptc += 1
                    continue
                if token.name in ('br', 'b', 'i', 'em'):
                    if not open_pTag:
                        pTag = Tag(result, 'p')
                        open_pTag = True
                        ptc = 0

                    pTag.insert(ptc, token)
                    ptc += 1
                    continue
                if open_pTag:
                    result.insert(rtc, pTag)
                    rtc += 1
                    open_pTag = False
                    ptc = 0

                sub_tokens = list(token.contents)
                for sub_token in sub_tokens:
                    if type(sub_token) is NavigableString:
                        sub_token.replaceWith(prepare_string_for_xml(sub_token))
                        continue

                result.insert(rtc, token)
                rtc += 1

            if open_pTag:
                result.insert(rtc, pTag)

            paras = result.findAll('p')
            for p in paras:
                p['class'] = 'description'

            for elem in elems:
                result.insert(rtc, elem)
                rtc += 1

            return result.renderContents(encoding = None)


        def processSpecialTags(self, tags, this_title, opts):
            tag_list = []
            for tag in tags:
                tag = self.convertHTMLEntities(tag)
                if tag.startswith(opts.note_tag):
                    this_title['notes'] = tag[len(self.opts.note_tag):]
                    continue
                if tag == opts.read_tag:
                    this_title['read'] = True
                    continue
                if re.search(opts.exclude_genre, tag):
                    continue
                    continue
                tag_list.append(tag)

            return tag_list


        def updateProgressFullStep(self, description):
            self.currentStep += 1
            self.progressString = description
            self.progressInt = float((self.currentStep - 1) / self.totalSteps)
            self.reporter(self.progressInt, self.progressString)
            if self.opts.cli_environment:
                self.opts.log(u'%3.0f%% %s' % (self.progressInt * 100, self.progressString))



        def updateProgressMicroStep(self, description, micro_step_pct):
            step_range = 100 / self.totalSteps
            self.progressString = description
            coarse_progress = float((self.currentStep - 1) / self.totalSteps)
            fine_progress = float(micro_step_pct * step_range / 100)
            self.progressInt = coarse_progress + fine_progress
            self.reporter(self.progressInt, self.progressString)


        class NotImplementedError:

            def __init__(self, error):
                self.error = error


            def logerror(self):
                self.opts.log.info('%s not implemented' % self.error)




    def run(self, path_to_output, opts, db, notification = DummyReporter()):
        opts.log = log
        opts.fmt = self.fmt = path_to_output.rpartition('.')[2]
        opts.creator = '%s, %s %s, %s' % (strftime('%A'), strftime('%B'), strftime('%d').lstrip('0'), strftime('%Y'))
        opts.creator_sort_as = '%s %s' % ('calibre', strftime('%Y-%m-%d'))
        opts.connected_kindle = False
        op = opts.output_profile
        if op is None:
            op = 'default'

        if opts.connected_device['name'] and 'kindle' in opts.connected_device['name'].lower():
            opts.connected_kindle = True
            if opts.connected_device['serial'] and opts.connected_device['serial'][:4] in ('B004', 'B005'):
                op = 'kindle_dx'
            else:
                op = 'kindle'

        opts.descriptionClip = None if op.endswith('dx') or 'kindle' not in op else 100
        opts.authorClip = None if op.endswith('dx') or 'kindle' not in op else 60
        opts.output_profile = op
        opts.basename = 'Catalog'
        opts.cli_environment = not hasattr(opts, 'sync')
        opts.sort_descriptions_by_author = True
        build_log = []
        None(None % (build_log.append, u'%s(): Generating %s %sin %s environment', self.name, self.fmt if opts.output_profile else '' if opts.cli_environment else 'GUI'))
        if opts.exclude_genre.strip() == '':
            opts.exclude_genre = '\\[^.\\]'
            build_log.append(" converting empty exclude_genre to '\\[^.\\]'")

        if opts.connected_device['is_device_connected'] and opts.connected_device['kind'] == 'device':
            if opts.connected_device['serial']:
                build_log.append(u" connected_device: '%s' #%s%s " % (opts.connected_device['name'], opts.connected_device['serial'][0:4], 'x' * (len(opts.connected_device['serial']) - 4)))
                for storage in opts.connected_device['storage']:
                    if storage:
                        build_log.append(u'  mount point: %s' % storage)
                        continue

            else:
                build_log.append(u" connected_device: '%s'" % opts.connected_device['name'])

                try:
                    for storage in opts.connected_device['storage']:
                        if storage:
                            build_log.append(u'  mount point: %s' % storage)
                            continue
                build_log.append(u'  (no mount points)')

        else:
            build_log.append(u" connected_device: '%s'" % opts.connected_device['name'])
        opts_dict = vars(opts)
        if opts_dict['ids']:
            build_log.append(' book count: %d' % len(opts_dict['ids']))

        sections_list = [
            'Descriptions',
            'Authors']
        if opts.generate_titles:
            sections_list.append('Titles')

        if opts.generate_recently_added:
            sections_list.append('Recently Added')

        if not opts.exclude_genre.strip() == '.':
            sections_list.append('Genres')

        build_log.append(u' Sections: %s' % ', '.join(sections_list))
        keys = opts_dict.keys()
        keys.sort()
        build_log.append(' opts:')
        for key in keys:
            if key in ('catalog_title', 'authorClip', 'connected_kindle', 'descriptionClip', 'exclude_genre', 'exclude_tags', 'note_tag', 'numbers_as_text', 'read_tag', 'search_text', 'sort_by', 'sort_descriptions_by_author', 'sync'):
                build_log.append('  %s: %s' % (key, opts_dict[key]))
                continue

        if opts.verbose:
            log('\n'.join((lambda .0: for line in .0:
line)(build_log)))

        self.opts = opts
        catalog = self.CatalogBuilder(db, opts, self, report_progress = notification)
        if opts.verbose:
            log.info(' Begin catalog source generation')

        catalog.createDirectoryStructure()
        catalog.copyResources()
        catalog.calculateThumbnailSize()
        catalog_source_built = catalog.buildSources()
        if opts.verbose:
            if catalog_source_built:
                log.info(' Completed catalog source generation\n')
            else:
                log.warn(' No database hits with supplied criteria')

        if catalog_source_built:
            recommendations = []
            recommendations.append(('comments', '\n'.join((lambda .0: for line in .0:
line)(build_log)), OptionRecommendation.HIGH))
            dp = getattr(opts, 'debug_pipeline', None)
            if dp is not None:
                recommendations.append(('debug_pipeline', dp, OptionRecommendation.HIGH))

            if opts.fmt == 'mobi' and opts.output_profile and opts.output_profile.startswith('kindle'):
                recommendations.append(('output_profile', opts.output_profile, OptionRecommendation.HIGH))
                recommendations.append(('no_inline_toc', True, OptionRecommendation.HIGH))
                recommendations.append(('book_producer', opts.output_profile, OptionRecommendation.HIGH))

            Plumber = Plumber
            import calibre.ebooks.conversion.plumber
            plumber = Plumber(os.path.join(catalog.catalogPath, opts.basename + '.opf'), path_to_output, log, report_progress = notification, abort_after_input_dump = False)
            plumber.merge_ui_recommendations(recommendations)
            plumber.run()
            return 0
        return 1