home *** CD-ROM | disk | FTP | other *** search
/ Maximum CD 2010 November / maximum-cd-2010-11.iso / DiscContents / xbmc-9.11.exe / scripts / AppleMovieTrailers / resources / lib / trailers.py < prev    next >
Encoding:
Python Source  |  2009-11-03  |  37.1 KB  |  758 lines

  1. import sys
  2. import os
  3. import xbmc
  4. import xbmcgui
  5. import traceback
  6. import datetime
  7. import elementtree.ElementTree as ET
  8. import filecmp
  9. import re
  10.  
  11. import cacheurl
  12. import pil_util
  13. import database
  14.  
  15. fetcher = cacheurl.HTTP()
  16. BASE_CACHE_PATH = fetcher.cache_dir + os.sep
  17.  
  18. _ = sys.modules[ "__main__" ].__language__
  19. __scriptname__ = sys.modules[ "__main__" ].__scriptname__
  20. __version__ = sys.modules[ "__main__" ].__version__
  21. __svn_revision__ = sys.modules[ "__main__" ].__svn_revision__
  22.  
  23.  
  24. class Movie:
  25.     """
  26.         Exposes the following:
  27.         - idMovie (integer - movies id#)
  28.         - title (string)
  29.         - url (string - xml url)
  30.         - trailer_urls (string - list of movie urls)
  31.         - poster (string - path to poster)
  32.         - thumbnail (string - path to thumbnail)
  33.         - thumbnail_watched (string - path to watched thumbnail)
  34.         - plot (string - movie plot)
  35.         - runtime (string - movie runtime)
  36.         - rating (string - movie rating)
  37.         - rating_url (string - path to rating image file)
  38.         - release_date (text - date in theaters)
  39.         - watched (integer - number of times watched)
  40.         - watched_date (string - last watched date)
  41.         - favorite (integer - 1=favorite)
  42.         - saved (string - list of tuples with path to saved movies and player core)
  43.         - date_added (text - date the trailer was added to the database)
  44.         - cast (list - list of actors)
  45.         - studio (string - movies studio)
  46.         - genre (string - movies genres)
  47.     """
  48.     def __init__( self, *args, **kwargs ):
  49.         self.__dict__.update( kwargs )
  50.  
  51.  
  52. class Category:
  53.     """
  54.         Exposes the following:
  55.         - id (integer)
  56.         - title (string)
  57.         - updated (date - last date updated)
  58.         - count (integer - number of movies in category)
  59.     """
  60.     def __init__( self, *args, **kwargs ):
  61.         self.__dict__.update( kwargs )
  62.  
  63.  
  64. class Trailers:
  65.     """
  66.         Exposes the following:
  67.         - categories (sorted list of Category() object instances)
  68.         - trailers (sorted list of Movie() object instances)
  69.     """
  70.     def __init__( self ):
  71.         self.categories = []
  72.         self.base_url = "http://www.apple.com"
  73.         self.base_movie_url = "http://movies.apple.com"
  74.         self.base_xml = self.base_url + "/moviesxml/h/index.xml"
  75.         db = database.Database( language=_ )
  76.         self.complete = db.complete
  77.         self.query = database.Query()
  78.         newest_genre, last_updated = self.loadGenres()
  79.         if ( newest_genre ):
  80.             from utilities import Settings
  81.             settings = Settings().get_settings()
  82.             if ( settings[ "refresh_newest" ] ):
  83.                 self.refreshGenre( ( newest_genre, ), last_updated, settings[ "refresh_trailers" ] )
  84.  
  85.     def ns( self, text ):
  86.         base_ns = "{http://www.apple.com/itms/}"
  87.         result = list()
  88.         for each in text.split( "/" ):
  89.             result += [ base_ns + each ]
  90.         return "/".join( result )
  91.  
  92.     def refreshTrailerInfo( self, trailers ):
  93.         dialog = xbmcgui.DialogProgress()
  94.         def _progress_dialog( count=0 ):
  95.             if ( count is None ):
  96.                 dialog.create( _( 68 ) )
  97.             else:
  98.                 __line1__ =  _( 98 )
  99.                 if ( not count ):
  100.                     dialog.update( -1, _( 98 ) )
  101.                 elif ( count > 0 ):
  102.                     percent = int( count * ( float( 100 ) / len( self.movies ) ) )
  103.                     __line2__ = "%s: (%d of %d)" % ( _( 88 ), count, len( self.movies ), )
  104.                     __line3__ = title
  105.                     dialog.update( percent, __line1__, __line2__, __line3__ )
  106.                     if ( dialog.iscanceled() ): return False
  107.                     else: return True
  108.                 else:
  109.                     dialog.close()
  110.         try:
  111.             if ( len( trailers ) > 1 ):
  112.                 _progress_dialog( None )
  113.             records = database.Records()
  114.             for trailer in trailers:
  115.                 title = self.movies[ trailer ].title
  116.                 urls = []
  117.                 if ( len( trailers ) > 1 ):
  118.                     _progress_dialog( trailer + 1 )
  119.                 for url in self.movies[ trailer ].urls:
  120.                     urls += [ self.base_url + url ]
  121.                 self.removeXML( urls )
  122.                 ok = records.update( "movies", ( "date_added", ), ( "u%s" % ( self.movies[ trailer ].date_added, ), self.movies[ trailer ].idMovie, ), "idMovie", True )
  123.                 ok = records.delete( "actor_link_movie", ( "idMovie", ), ( self.movies[ trailer ].idMovie, ) )
  124.                 ok = records.delete( "studio_link_movie", ( "idMovie", ), ( self.movies[ trailer ].idMovie, ) )
  125.             records.close()
  126.             if ( len( trailers ) > 1 ):
  127.                 _progress_dialog( -99 )
  128.         except: traceback.print_exc()
  129.  
  130.     def refreshGenre( self, genres, last_updated=False, refresh_trailers=False ):
  131.         """
  132.             Updates the xml for each genre in genres from the site.
  133.         """
  134.         dialog = xbmcgui.DialogProgress()
  135.         def _progress_dialog( count=0 ):
  136.             if ( count is None ):
  137.                 dialog.create( _( 37 + refresh_trailers ) )
  138.             else:
  139.                 __line1__ =  "%s: %s - (%d of %d)" % ( _( 87 ), title, g_count + 1, len( genres ) )
  140.                 if ( count == -1 ):
  141.                     dialog.update( -1, __line1__, _( 62 ), _( 67 ) )
  142.                 elif ( not count ):
  143.                     dialog.update( -1, __line1__, _( 65 ), _( 67 ) )
  144.                 elif ( count > 0 ):
  145.                     percent = int( count * ( float( 100 ) / len( trailer_urls ) ) )
  146.                     __line2__ = "%s: (%d of %d)" % ( _( 88 ), count, len( trailer_urls ), )
  147.                     __line3__ = trailer_url[ 0 ]
  148.                     dialog.update( percent, __line1__, __line2__, __line3__ )
  149.                     if ( dialog.iscanceled() ): return False
  150.                     else: return True
  151.                 else:
  152.                     dialog.close()
  153.         
  154.         updated_date = datetime.date.today()
  155.         if ( last_updated and str( updated_date ) == str( last_updated ) ): return
  156.         _progress_dialog( None )
  157.         records = database.Records()
  158.         try:
  159.             local_fetcher = cacheurl.HTTP( os.path.join( BASE_CACHE_PATH, "backups" ) )
  160.             for g_count, genre in enumerate( genres ):
  161.                 title = self.categories[ genre ].title
  162.                 _progress_dialog( -1 )
  163.                 idGenre = self.categories[ genre ].id
  164.                 record = records.fetch( self.query[ "genre_urls_by_genre_id" ], ( idGenre, ) )
  165.                 urls = eval( record[ 0 ] )
  166.                 # fetch genre xml file and compare it to the current xml file
  167.                 for url in urls:
  168.                     original_filename = fetcher.make_cache_filename( url )
  169.                     filename = local_fetcher.urlretrieve( url )
  170.                     # if the files are different flag it
  171.                     new_trailers = False
  172.                     if ( filename is not None ):
  173.                         new_trailers = not filecmp.cmp( filename, original_filename )
  174.                     if ( new_trailers ):
  175.                         xbmc.executehttpapi( "FileCopy(%s,%s)" % ( filename, original_filename, ) )
  176.                         #shutil.copy( filename, original_filename )
  177.                     if ( filename is not None ):
  178.                         os.remove( filename )
  179.                 if ( new_trailers or refresh_trailers ):
  180.                     _progress_dialog()
  181.                     trailer_urls, genre_urls = self.loadGenreInfo( title, urls[ 0 ] )
  182.                     if ( trailer_urls ):
  183.                         idMovie_list = records.fetch( self.query[ "idMovie_by_genre_id" ], ( idGenre, ), all=True )
  184.                         #commit = 0
  185.                         for cnt, trailer_url in enumerate( trailer_urls ):
  186.                             if ( not _progress_dialog( cnt + 1 ) ): raise
  187.                             #commit += 1
  188.                             record = records.fetch( self.query[ "movie_exists" ], ( trailer_url[ 0 ], ) )
  189.                             if ( record is None ):
  190.                                 idMovie = records.add( "movies", ( trailer_url[ 0 ] , repr( [ trailer_url[ 1 ] ] ), ) )
  191.                                 success = records.add( "genre_link_movie", ( idGenre, idMovie, ) )
  192.                             else:
  193.                                 # remove the trailer urls if refresh_trailers is true
  194.                                 if ( refresh_trailers ):
  195.                                     url_list = []
  196.                                     for url2 in eval( record[ 1 ] ):
  197.                                         url_list += [ self.base_url + url2 ]
  198.                                     self.removeXML( url_list )
  199.                                     date_added = ( "", record[ 2 ], )[ record[ 2 ] is not None ]
  200.                                     ok = records.update( "movies", ( "date_added", ), ( "u%s" % ( record[ 2 ], ), record[ 0 ], ), "idMovie" )
  201.                                     ok = records.delete( "actor_link_movie", ( "idMovie", ), ( record[ 0 ], ) )
  202.                                     ok = records.delete( "studio_link_movie", ( "idMovie", ), ( record[ 0 ], ) )
  203.                                 try:
  204.                                     idMovie_list.remove( ( record[ 0 ], ) )
  205.                                 except:
  206.                                     success = records.add( "genre_link_movie", ( idGenre, record[ 0 ], ) )
  207.                             #if ( float( commit ) / 100 == int( commit / 100 ) ):
  208.                             #    success = records.commit()
  209.                             #    commit = 0
  210.                         for record in idMovie_list:
  211.                             success = records.delete( "genre_link_movie", ( "idGenre", "idMovie", ), ( idGenre, record[ 0 ], ) )
  212.                         success = records.update( "genres", ( "urls", "trailer_urls", "updated", ), ( repr( genre_urls ), repr( trailer_urls), updated_date, idGenre, ), "idGenre" )
  213.                 else:
  214.                     success = records.update( "genres", ( "updated", ), ( updated_date, idGenre, ), "idGenre" )
  215.                 success = records.commit()
  216.         except:
  217.             traceback.print_exc()
  218.         success = records.commit()
  219.         records.close()
  220.         _progress_dialog( -99 )
  221.  
  222.     def removeXML( self, urls ):
  223.         for url in urls:
  224.             try:
  225.                 filename = fetcher.make_cache_filename( url )
  226.                 filename = os.path.join( BASE_CACHE_PATH, filename )
  227.                 if os.path.isfile( filename ):
  228.                     os.remove( filename )
  229.             except:
  230.                 traceback.print_exc()
  231.                 
  232.     def loadGenres( self ):
  233.         """
  234.             Parses the main xml for genres
  235.         """
  236.         dialog = xbmcgui.DialogProgress()
  237.         def _progress_dialog( count=0, trailer_count=0 ):
  238.             if ( not count ):
  239.                 dialog.create( "%s   (%s)" % ( _( 66 ), _( 158 + ( not load_all ) ), ) )
  240.             elif ( count > 0 ):
  241.                 percent = int( count * ( float( 100 ) / len( genres ) ) )
  242.                 __line1__ = "%s: %s - (%d of %d)" % (_( 87 ), genre, count, len( genres ), )
  243.                 if ( trailer_count ):
  244.                     __line2__ = "%s: (%d of %d)" % ( _( 88 ), trailer_count, len( trailer_urls ), )
  245.                     __line3__ = url[ 0 ]
  246.                 else:
  247.                     __line2__ = ""
  248.                     __line3__ = ""
  249.                 dialog.update( percent, __line1__, __line2__, __line3__ )
  250.                 if ( dialog.iscanceled() ): return False
  251.                 else: return True
  252.             else:
  253.                 dialog.close()
  254.  
  255.         try:
  256.             records = database.Records()
  257.             genre_list = records.fetch( self.query[ "genre_table_list" ], all=True )
  258.             self.categories = []
  259.             if ( genre_list ):
  260.                 for cnt, genre in enumerate( genre_list ):
  261.                     self.categories += [ Category( id=genre[ 0 ], title=genre[ 1 ] ) ]
  262.                     if ( genre[ 1 ] == "Newest" ):
  263.                         newest_id = cnt
  264.                         last_updated = genre[ 2 ]
  265.                 records.close()
  266.                 return newest_id, last_updated
  267.             else:
  268.                 load_all = xbmcgui.Dialog().yesno( _( 44 ), "%s: %s" % ( _( 158 ), _( 40 ), ), "%s: %s" % ( _( 159 ), _( 41 ), ), _( 49 ), _( 159 ), _( 158 ) )
  269.  
  270.                 _progress_dialog()
  271.                 updated_date = datetime.date.today()
  272.                 source = fetcher.urlopen( self.base_xml )
  273.                 try:
  274.                     base_xml = ET.fromstring( source )
  275.                 except:
  276.                     source = self.cleanXML( source.decode( "utf-8", "replace" ).encode( "utf-8", "ignore" ) )
  277.                     base_xml = ET.fromstring( source )
  278.  
  279.                 view_matrix = {
  280.                     "view1": "Exclusives",
  281.                     "view2": "Newest",
  282.                     }
  283.                 elements = base_xml.getiterator( self.ns( "Include" ) )
  284.                 genre_id = 0
  285.                 genre_dict = dict()
  286.                 for each in elements:
  287.                     url = each.get( "url" )
  288.                     for view in view_matrix:
  289.                         if view in url:
  290.                             url = "/moviesxml/h/" + url
  291.                             genre_dict.update( { view_matrix[ view ]: url } )
  292.                 elements = base_xml.getiterator( self.ns( "GotoURL" ) )
  293.                 for each in elements:
  294.                     url = each.get( "url" )
  295.                     name = " ".join( url.split( "/" )[ -1 ].split( "_" )[ : -1 ] )
  296.                     genre_caps = list()
  297.                     # smart capitalization of the genre name
  298.                     for word in name.split():
  299.                         # only prevent capitalization of these words if they aren't the leading word in the genre name
  300.                         # ie, "the top rated" becomes "The Top Rated", but "action and adventure" becomes "Action and Adventure"
  301.                         cap = True
  302.                         if word != name[0] and ( word == "and" or word == "of" or word == "a" ):
  303.                             cap = False
  304.                         if cap:
  305.                             genre_caps += [ word.capitalize() ]
  306.                         else:
  307.                             genre_caps += [ word ]
  308.                     name = " ".join( genre_caps )
  309.                     if "/moviesxml/g" in url:
  310.                         genre_dict.update( { name: url } )
  311.                 genres = genre_dict.keys()
  312.                 genres.sort()
  313.                 for cnt, genre in enumerate( genres ):
  314.                     ok = _progress_dialog( cnt + 1, 0 )
  315.                     trailer_urls, genre_urls = self.loadGenreInfo( genre, genre_dict[genre] )
  316.                     if ( trailer_urls ):
  317.                         idGenre = records.add( "genres", ( genre, repr( genre_urls ), repr( trailer_urls), updated_date, ) )
  318.                         self.categories += [ Category( id=idGenre, title=genre ) ]
  319.                         for url_cnt, url in enumerate( trailer_urls ):
  320.                             ok = _progress_dialog( cnt + 1, url_cnt + 1 )
  321.                             record = records.fetch( self.query[ "movie_exists" ], ( url[ 0 ], ) )
  322.                             if ( record is None ):
  323.                                 idMovie = records.add( "movies", ( url[ 0 ] , repr( [ url[ 1 ] ] ), ) )
  324.                             else: idMovie = record[ 0 ]
  325.                             success = records.add( "genre_link_movie", ( idGenre, idMovie, ) )
  326.                         success = records.commit()
  327.                 records.close()
  328.                 _progress_dialog( -99 )
  329.                 if ( load_all ):
  330.                     updated = self.fullUpdate()
  331.         except:
  332.             records.close()
  333.             _progress_dialog( -99 )
  334.             traceback.print_exc()
  335.         return False, False
  336.         
  337.     def loadGenreInfo( self, genre, url ):
  338.         """
  339.             Follows all links from a genre page and fetches all trailer urls.
  340.             Returns two lists, trailer_urls (contains tuples of ( title, url ) 
  341.             for all trailers in genre and genre_urls (contains urls to all 
  342.             pages of genre)
  343.         """
  344.         try:
  345.             if url[ : 7 ] != "http://":
  346.                 url = self.base_url + url
  347.             is_special = genre in ( "Exclusives", "Newest", )
  348.             next_url = url
  349.             first_url = True
  350.             trailer_dict = dict()
  351.             genre_urls = list()
  352.             while next_url:
  353.                 try:
  354.                     source = fetcher.urlopen( next_url )
  355.                     if "<Document" not in source:
  356.                         source = "<Document>" + source + "</Document>"
  357.                     try:
  358.                         element = ET.fromstring( source )
  359.                     except:
  360.                         source = self.cleanXML( source.decode( "utf-8", "replace" ).encode( "utf-8", "ignore" ) )
  361.                         try:
  362.                             element = ET.fromstring( source )
  363.                         except:
  364.                             # if this failed, make sure there are no more
  365.                             id_number = re.findall( "_([0-9]*).xml", next_url )
  366.                             next_url = re.sub( "_([0-9]*).xml", "_%d.xml" % ( int( id_number[ 0 ] ) + 1, ), next_url )
  367.                             continue
  368.  
  369.                     lookup = "GotoURL"
  370.                     if not is_special:
  371.                         lookup = self.ns( lookup )
  372.                     elements = element.getiterator( lookup )
  373.                     # add next_url to the genre_urls list
  374.                     genre_urls.append( next_url )
  375.                     try:
  376.                         if first_url:
  377.                             next_url = elements[0].get( "url" )
  378.                             first_url = False
  379.                         else:
  380.                             next_url = elements[2].get( "url" )
  381.                         if next_url[0] != "/":
  382.                             next_url = "/".join( url.split( "/" )[ : -1 ] + [ next_url ] )
  383.                         else:
  384.                             next_url = None
  385.                     except:
  386.                         next_url = None
  387.                     if next_url is not None:
  388.                         if ( not is_special and "/moviesxml/g" not in next_url ) or ( is_special and "/moviesxml/h" not in next_url ):
  389.                             next_url = None
  390.  
  391.                     for element in elements:
  392.                         url2 = element.get( "url" )
  393.                         title = None
  394.                         if is_special:
  395.                             title = element.getiterator( "b" )[ 0 ].text.strip()#.encode( "ascii", "ignore" )
  396.                         if "index_1" in url2:
  397.                             continue
  398.                         if "/moviesxml/g" in url2:
  399.                             continue
  400.                         if url2[ 0 ] != "/" and not url2.startswith( "http://" ):
  401.                             continue
  402.                         if url2 in trailer_dict.keys():
  403.                             lookup = "b"
  404.                             if not is_special:
  405.                                 lookup = "B"
  406.                                 lookup = self.ns( lookup )
  407.                             try:
  408.                                 title = element.getiterator( lookup )[ 0 ].text.strip()#encode( "ascii", "ignore" )
  409.                                 trailer_dict[ url2 ] =  title.strip()
  410.                             except:
  411.                                 pass
  412.                             continue
  413.                         trailer_dict.update( { url2: title } )
  414.                 except:
  415.                     break
  416.  
  417.             reordered_dict = dict()
  418.             trailer_urls = []
  419.             for key in trailer_dict:
  420.                 reordered_dict.update( { trailer_dict[key]: key } )
  421.             keys = reordered_dict.keys()
  422.             keys.sort()
  423.             trailer_urls = []
  424.             for cnt, key in enumerate( keys ):
  425.                 try:
  426.                     trailer_urls.append( ( key, reordered_dict[key] ) )
  427.                 except:
  428.                     continue
  429.             return trailer_urls, genre_urls
  430.         except:
  431.             traceback.print_exc()
  432.             return [], []
  433.  
  434.     def fullUpdate( self ):
  435.         full, updated = self._get_movie_list( self.query[ "incomplete_movies" ], header="%s   (%s)" % ( _( 70 ), _( 158 ), ), full = True )
  436.         if ( full ): self.complete = self.updateRecord( "version", ( "complete", ), ( True, 1, ), "idVersion" )
  437.         return updated
  438.  
  439.     def getMovies( self, sql, params=None ):
  440.         self.movies = []
  441.         full, updated = self._get_movie_list( sql, params, _( 85 ), _( 67 ) )
  442.         return updated
  443.  
  444.     def _get_movie_list( self, sql, params=None, header="", line1="", full=False ):
  445.         dialog = xbmcgui.DialogProgress()
  446.         def _progress_dialog( count=0, commit=False ):
  447.             if ( not count ):
  448.                 dialog.create( header, line1 )
  449.             elif ( count > 0 ):
  450.                 __line1__ = "%s: (%d of %d)" % ( _( 88 ), count, len( movie_list ), )
  451.                 __line2__ = movie[ 1 ]
  452.                 __line3__ = [ "", "-----> %s <-----" % (_( 43 ), ) ][ commit ]
  453.                 percent = int( count * ( float( 100 ) / len( movie_list ) ) )
  454.                 dialog.update( percent, __line1__, __line2__, __line3__ )
  455.                 if ( dialog.iscanceled() ): return False
  456.                 else: return True
  457.             else:
  458.                 dialog.close()
  459.             
  460.         def _load_movie_info( movie ):
  461.             def _set_default_movie_info( movie ):
  462.                 self.idMovie = movie[ 0 ]
  463.                 self.title = movie[ 1 ]
  464.                 self.urls = eval( movie[ 2 ] )
  465.                 self.trailer_urls = []
  466.                 self.old_trailer_urls = []
  467.                 if ( movie[ 3 ] is not None ):
  468.                     self.old_trailer_urls = eval( movie[ 3 ] )
  469.                     self.old_trailer_urls.sort()
  470.                 self.poster = ""
  471.                 self.plot = ""
  472.                 self.rating = ""
  473.                 self.rating_url = ""
  474.                 self.release_date = ""
  475.                 self.runtime = ""
  476.                 if ( movie[ 10 ] ):
  477.                     self.times_watched = movie[ 10 ]
  478.                 else:
  479.                     self.times_watched = 0
  480.                 if ( movie[ 11 ] ):
  481.                     self.last_watched = movie[ 11 ]
  482.                 else:
  483.                     self.last_watched = ""
  484.                 if ( movie[ 12 ] ):
  485.                     self.favorite = movie[ 12 ]
  486.                 else:
  487.                     self.favorite = 0
  488.                 if ( movie[ 13 ] ):
  489.                     self.saved = eval( movie[ 13 ] )
  490.                 else:
  491.                     self.saved = []
  492.                 if ( movie[ 14 ] is not None ):
  493.                     self.date_added = movie[ 14 ].replace( "u", "" )
  494.                 else:
  495.                     self.date_added = str( datetime.date.today() )
  496.                 self.actors = []
  497.                 self.studio = ""
  498.  
  499.             try:
  500.                 _set_default_movie_info( movie )
  501.                 date_added = str( datetime.date.today() )
  502.                 
  503.                 # get the main index xml file
  504.                 for url in self.urls:
  505.                     if "index" in url:
  506.                         if ( not url.startswith( "http://" ) ):
  507.                             url = self.base_url + str( url )
  508.                         break
  509.                 # xml parsing. replace <b> and </b> for in theaters. TODO: remove if noticeably slower
  510.                 source = fetcher.urlopen( url ).replace( "<b>", "" ).replace( "</b>", "" )
  511.                 try:
  512.                     element = ET.fromstring( source )
  513.                 except:
  514.                     source = self.cleanXML( source.decode( "utf-8", "replace" ).encode( "utf-8", "ignore" ) )
  515.                     element = ET.fromstring( source )
  516.                 
  517.                 # -- poster & thumbnails --
  518.                 poster = element.getiterator( self.ns( "PictureView" ) )[ 1 ].get( "url" )
  519.                 if poster:
  520.                     # if it's not the full url add the base url
  521.                     if ( not poster.startswith( "http://" ) ):
  522.                         poster = self.base_movie_url + str( poster )
  523.                     # download the actual poster to the local filesystem (or get the cached filename)
  524.                     poster = fetcher.urlretrieve( poster )
  525.                     if poster:
  526.                         # make thumbnails
  527.                         success = pil_util.makeThumbnails( poster )
  528.                         self.poster = os.path.basename( poster )
  529.  
  530.                 # -- plot --
  531.                 plot = element.getiterator( self.ns( "SetFontStyle" ) )[ 2 ].text.strip()#encode( "ascii", "ignore" ).strip()
  532.                 if plot:
  533.                     # remove any linefeeds so we can wrap properly to the text control this is displayed in
  534.                     plot = plot.replace( "\r\n", " " )
  535.                     plot = plot.replace( "\r", " " )
  536.                     plot = plot.replace( "\n", " " )
  537.                     self.plot = plot
  538.                 
  539.                 # -- release date --
  540.                 release_date = element.getiterator( self.ns( "SetFontStyle" ) )[ 3 ].text.strip()
  541.                 if release_date and "In Theaters:" in release_date:
  542.                     self.release_date = release_date.split( ":" )[ 1 ].strip()
  543.  
  544.                 # -- actors --
  545.                 SetFontStyles = element.getiterator( self.ns( "SetFontStyle" ) )
  546.                 actors = list()
  547.                 for i in range( 5, 10 ):
  548.                     actor = SetFontStyles[ i ].text.replace( "(The voice of)", "" ).title().strip()
  549.                     if ( len( actor ) and not actor.startswith( "." ) and actor != "1:46" and not actor.startswith( "Posted:" ) and not actor.startswith( "Runtime:" ) and not actor == "Available Clips" and not actor == "Official Website" and not actor.startswith( "Trailer" ) ) :
  550.                         actors += [ ( actor, ) ]
  551.                         actor_id = records.fetch( self.query[ "actor_exists" ], ( actor, ) )
  552.                         if ( actor_id is None ): idActor = records.add( "actors", ( actor, ) )
  553.                         else: idActor = actor_id[ 0 ]
  554.                         records.add( "actor_link_movie", ( idActor, self.idMovie, ) )
  555.                 self.actors = actors
  556.                 self.actors.sort()
  557.  
  558.                 # -- runtime --
  559.                 try:
  560.                     runtime = element.getiterator( self.ns( "SetFontStyle" ) )[ 13 ].text
  561.                     if runtime and "Runtime" in runtime:
  562.                         runtime = runtime.replace( "Runtime", "" ).replace( ":", "" ).replace( ".", "" ).strip().rjust( 4, "0" )
  563.                         runtime = "%s:%s" % ( runtime[ : 2 ], runtime[ 2 : ], )
  564.                         self.runtime = runtime
  565.                 except:
  566.                     pass
  567.  
  568.                 # -- studio --
  569.                 studio = element.getiterator( self.ns( "PathElement" ) )[ 1 ].get( "displayName" ).strip()
  570.                 if studio:
  571.                     studio_id = records.fetch( self.query[ "studio_exists" ], ( studio, ) )
  572.                     if ( studio_id is None ): idStudio = records.add( "studios", ( studio, ) )
  573.                     else: idStudio = studio_id[ 0 ]
  574.                     records.add( "studio_link_movie", ( idStudio, self.idMovie, ) )
  575.                     self.studio = studio
  576.  
  577.                 # -- rating --
  578.                 temp_url = element.getiterator( self.ns( "PictureView" ) )[ 2 ].get( "url" )
  579.                 if temp_url:
  580.                     if "/mpaa" in temp_url:
  581.                         if ( not temp_url.startswith( "http://" ) ):
  582.                             if ( not temp_url.startswith( "/" ) ):
  583.                                 tmp_url = "/" + tmp_url
  584.                             temp_url = "http://images.apple.com" + temp_url
  585.                         rating_url = fetcher.urlretrieve( temp_url )
  586.                         if rating_url:
  587.                             self.rating_url = os.path.basename( rating_url )
  588.                             self.rating = os.path.split( temp_url )[ 1 ][ : -4 ].replace( "mpaa_", "" )
  589.  
  590.                 # TODO: maybe parse the index xml file(s) for other trailer xml files, keeping as a list of lists, so user can select
  591.                 # -- trailer urls --
  592.                 # get all url xml files
  593.                 for each in element.getiterator( self.ns( "GotoURL" ) ):
  594.                     temp_url = each.get( "url" )
  595.                     if not temp_url.endswith( ".xml" ): continue
  596.                     if "/moviesxml/g" in temp_url: continue
  597.                     if temp_url in self.urls: continue
  598.                     self.urls += [ temp_url ]
  599.                 all_urls = ()
  600.                 for xml_url in self.urls:
  601.                     new_xml_url = self.base_url + xml_url
  602.                     if new_xml_url != url:
  603.                         # xml parsing. replace <b> and </b> for in theaters. remove if noticeably slower
  604.                         source = fetcher.urlopen( new_xml_url ).replace( "<b>", "" ).replace( "</b>", "" )
  605.                         try:
  606.                             element = ET.fromstring( source )
  607.                         except:
  608.                             source = self.cleanXML( source.decode( "utf-8", "replace" ).encode( "utf-8", "ignore" ) )
  609.                             try:
  610.                                 element = ET.fromstring( source )
  611.                             except:
  612.                                 continue
  613.                     urls = ()
  614.                     for each in element.getiterator( self.ns( "string" ) ):
  615.                         text = each.text
  616.                         # invalid urls
  617.                         if text is None: continue
  618.                         if not text.endswith( ".mov" ): continue
  619.                         new_url = text.replace( "//", "/" ).replace( "/", "//", 1 )
  620.                         if new_url in all_urls:
  621.                             add_trailer = False
  622.                         else:
  623.                             add_trailer = True
  624.                             all_urls += ( new_url, )
  625.                         # add the trailer url to our list
  626.                         if add_trailer:
  627.                             urls += ( text.replace( "//", "/" ).replace( "/", "//", 1 ), )
  628.                     if len( urls ):
  629.                         self.trailer_urls += [ urls ]
  630.                 self.trailer_urls.sort()
  631.                 if ( self.trailer_urls == self.old_trailer_urls ):
  632.                     date_added = self.date_added
  633.             except:
  634.                 print "Trailer XML %s: %s is %s" % ( self.idMovie, repr( url ), ( "missing", "corrupt" )[ os.path.isfile( fetcher.make_cache_filename( url ) ) ] )
  635.  
  636.             info_list = ( self.idMovie, self.title, repr( self.urls ), repr( self.trailer_urls ), self.poster, self.plot, self.runtime,
  637.                             self.rating, self.rating_url, self.release_date, self.times_watched, self.last_watched, self.favorite,
  638.                             repr( self.saved ), date_added, self.actors, self.studio, )
  639.             success = records.update( "movies", ( 2, 15, ), ( info_list[ 2 : 15 ] ) + ( self.idMovie, ), "idMovie" )
  640.             return info_list
  641.  
  642.         def _get_actor_and_studio( movie ):
  643.             actor_list = records.fetch( self.query[ "actors_by_movie_id" ], ( movie[ 0 ], ), all=True )
  644.             if ( actor_list is not None ): movie += ( actor_list, )
  645.             else: movie += ( [], )
  646.             studio = records.fetch( self.query[ "studio_by_movie_id" ], ( movie[ 0 ], ) )
  647.             if ( studio is not None ): movie += ( studio[ 0 ], )
  648.             else: movie += ( "", )
  649.             return movie
  650.  
  651.         def _get_genres( movie ):
  652.             genre_list = records.fetch( self.query[ "genres_by_movie_id" ], ( movie[ 0 ], ), all=True )
  653.             if ( genre_list is not None ):
  654.                 movie += ( " / ".join( [ genre[ 0 ] for genre in genre_list ] ), )
  655.             else: movie += ( "", )
  656.             return movie
  657.  
  658.         try:
  659.             _progress_dialog()
  660.             records = database.Records()
  661.             movie_list = records.fetch( sql, params, all=True )
  662.             commit = info_missing = False
  663.             if ( movie_list ):
  664.                 dialog_ok = True
  665.                 for cnt, movie in enumerate( movie_list ):
  666.                     if ( movie[ 3 ] is None or movie[ 14 ] is None or movie[ 14 ].startswith( "u" ) ):
  667.                         movie = _load_movie_info( movie )
  668.                         info_missing = True
  669.                     else: movie = _get_actor_and_studio( movie )
  670.                     movie = _get_genres( movie )
  671.                     if ( info_missing ):
  672.                         if ( float( cnt + 1 ) / 100 == int( ( cnt + 1 ) / 100 ) or ( cnt + 1 ) == len( movie_list ) or not dialog_ok ):
  673.                             commit = True
  674.                         dialog_ok = _progress_dialog( cnt + 1, commit )
  675.                     if ( not full and movie is not None ):
  676.                         if ( movie[4] ): poster = os.path.join( BASE_CACHE_PATH, movie[4][0], movie[4] )
  677.                         else: poster = ""
  678.                         if ( movie[8] ): rating_url = os.path.join( BASE_CACHE_PATH, movie[8][0], movie[8] )
  679.                         else: rating_url = ""
  680.                         self.movies += [ 
  681.                             Movie(
  682.                                 idMovie = movie[ 0 ],
  683.                                 title = movie[1],
  684.                                 urls = eval( movie[2] ),
  685.                                 trailer_urls = eval( movie[3] ),
  686.                                 poster = poster,
  687.                                 thumbnail = "%s.png" % ( os.path.splitext( poster )[0], ),
  688.                                 thumbnail_watched = "%s-w.png" % ( os.path.splitext( poster )[0], ),
  689.                                 plot = movie[5],
  690.                                 runtime = movie[6],
  691.                                 rating = movie[7],
  692.                                 rating_url = rating_url,
  693.                                 release_date = movie[9],
  694.                                 watched = movie[10],
  695.                                 watched_date = movie[11],
  696.                                 favorite = movie[12],
  697.                                 saved = eval( movie[13] ),
  698.                                 date_added = movie[14],
  699.                                 cast = movie[15],
  700.                                 studio = movie[16],
  701.                                 genres = movie[17]
  702.                                 )
  703.                             ]
  704.  
  705.                     if ( commit or not dialog_ok ):
  706.                         success = records.commit()
  707.                         commit = False
  708.                         if ( not dialog_ok ):
  709.                             full = False
  710.                             break
  711.             elif ( not full ): self.movies = None
  712.         except: pass
  713.         records.close()
  714.         _progress_dialog( -99 )
  715.         return full, info_missing
  716.  
  717.     def getCategories( self, sql, params=None ):
  718.         try:
  719.             dialog = xbmcgui.DialogProgress()
  720.             dialog.create( _( 85 ) )
  721.             dialog.update( -1, _( 67 ) )
  722.             records = database.Records()
  723.             category_list = records.fetch( sql, params, all=True )
  724.             records.close()
  725.             if ( category_list is not None):
  726.                 self.categories = []
  727.                 for category in category_list:
  728.                     self.categories += [ Category( id=category[ 0 ], title=category[ 1 ], count=category[ 2 ], completed=category[ 3 ] >= category[ 2 ], ) ]
  729.             else: self.categories = None
  730.         except: traceback.print_exc()
  731.         dialog.close()
  732.  
  733.     def updateRecord( self, table, columns, values, key="title" ):
  734.         try:
  735.             records = database.Records()
  736.             success = records.update( table, columns, values, key, True )
  737.             records.close()
  738.         except:
  739.             traceback.print_exc()
  740.             success = False
  741.         return success
  742.  
  743.     def cleanXML( self, xml_source ):
  744.         xml_source = xml_source.replace( " & ", " & " )
  745.         xml_source = xml_source.replace( " ", " " )
  746.         xml_source = xml_source.replace( "í", "I" )
  747.         xml_source = re.sub( "(&)[^#]..[^;]", "&", xml_source )
  748.         items = re.findall( '="[^>]*>', xml_source )
  749.         if ( items ):
  750.             for item in items:
  751.                 items2 = re.findall( '=[^=]*', item )
  752.                 for it in items2:
  753.                     first = it.find( '"' )
  754.                     second = it.rfind( '"' )
  755.                     it2 = it[ : first + 1 ] + it[ first + 1 : second - 1 ].replace( '"', "'" ) + it[ second - 1 : ]
  756.                     if ( it != it2 ): xml_source = xml_source.replace( it, it2, 1 )
  757.         return xml_source
  758.