home *** CD-ROM | disk | FTP | other *** search
/ Mac Easy 2010 May / Mac Life Ubuntu.iso / casper / filesystem.squashfs / usr / share / pyshared / launchpadbugs / http_connection.py < prev    next >
Encoding:
Python Source  |  2009-03-12  |  15.5 KB  |  376 lines
  1. """
  2. TODO (thekorn20080709):
  3.     * auto fetching of edge cookie is not working
  4.        - no idea why an edge cookies is not created when accessing edge.launchpad.net
  5.     * cookie lifetime (auto extend?)
  6.     * adjust almost all testcases and connector
  7. """
  8.  
  9. import urllib2
  10. import multipartpost_handler
  11. import cookielib
  12. import cStringIO as StringIO
  13. import time
  14. import urlparse
  15. import cPickle
  16. import base64
  17. from tempfile import mkstemp
  18. try:
  19.     import sqlite3 as sqlite
  20. except ImportError:
  21.     try:
  22.         from pysqlite2 import dbapi2 as sqlite
  23.     except ImportError:
  24.         raise ImportError, "no module named sqlite3 or pysqlite2.dbapi2"
  25. import os
  26.  
  27. import exceptions
  28. import utils
  29. from lpconstants import BASEURL, HTTPCONNECTION
  30. from config import Config
  31.  
  32. class _result(object):
  33.     """ represents an object returned by HTTPConnection. """
  34.     def __init__(self, contenttype=None, text=None, url=None):
  35.         assert contenttype or text or url, "at least one argument needed"
  36.         self.contenttype = contenttype
  37.         self.text = text
  38.         self.url = url
  39.         
  40. class LPCookieProcessor(urllib2.HTTPCookieProcessor):
  41.     """ CookieProcessor with special functionality regarding launchpad.net
  42.         cookies.
  43.     """
  44.     
  45.     @staticmethod
  46.     def sqlite_to_txt(cookiefile ,domain):
  47.         match = '%%%s%%' % domain
  48.         con = sqlite.connect(cookiefile)
  49.         cur = con.cursor()
  50.         cur.execute("select host, path, isSecure, expiry, name, value from moz_cookies where host like ?", [match])
  51.         ftstr = ["FALSE","TRUE"]
  52.         (tmp, tmpname) = mkstemp()
  53.         os.write(tmp, "# HTTP Cookie File\n")
  54.         for item in cur.fetchall():
  55.             str = "%s\t%s\t%s\t%s\t%s\t%s\t%s\n" % ( item[0], \
  56.                    ftstr[item[0].startswith('.')], item[1], \
  57.                    ftstr[item[2]], item[3], item[4], item[5])
  58.             os.write(tmp, str)
  59.         os.close(tmp)
  60.         return tmpname
  61.         
  62.     def __nonzero__(self):
  63.         return ".launchpad.net" in [i.domain for i in self.cookiejar]
  64.         
  65.     def load_file(self, cookiefile):
  66.         if cookiefile:
  67.             cj = cookielib.MozillaCookieJar()
  68.             if cookiefile[-6:] == "sqlite":
  69.                 cookies = self.sqlite_to_txt(cookiefile, "launchpad.net")
  70.                 try:
  71.                     cj.load(cookies)
  72.                 except:
  73.                     os.unlink(cookies)
  74.                     raise IOError, "Invalid cookie file"
  75.                 os.unlink(cookies)
  76.             else:
  77.                 cj.load(cookiefile)
  78.             for cookie in cj:
  79.                 self.cookiejar.set_cookie(cookie)
  80.             return True
  81.         else:
  82.             raise IOError, "No cookie file given"
  83.         
  84.     def get_cookie(self, name, domain=None):
  85.         domain = domain or ".launchpad.net"
  86.         for i in self.cookiejar:
  87.             if i.name == name and i.domain == domain:
  88.                 return i
  89.         return False
  90.         
  91.     def change_inhibit_beta_redirect(self, mode):
  92.         """
  93.         cj.clear(".launchpad.net", "/", "inhibit_beta_redirect")
  94.  
  95.         name inhibit_beta_redirect
  96.         domain .launchpad.net
  97.         path /
  98.         Cookie(version=0, name='inhibit_beta_redirect', value='1', port=None, port_specified=False, domain='.launchpad.net', domain_specified=True, domain_initial_dot=True, path='/', path_specified=False, secure=True, expires=1202139678, discard=False, comment=None, comment_url=None, rest={}, rfc2109=False)
  99.         """
  100.         if mode in (HTTPCONNECTION.MODE.DEFAULT, HTTPCONNECTION.MODE.EDGE):
  101.             #delete redirecting cookie
  102.             try:
  103.                 self.cookiejar.clear(".launchpad.net", "/", "inhibit_beta_redirect")
  104.             except KeyError:
  105.                 pass
  106.         elif mode == HTTPCONNECTION.MODE.STABLE:
  107.             #enable redirection cookie
  108.             #redirection for 2 hours
  109.             c_expires = int(time.time()) + 2 * 60 * 60
  110.             c = self.get_cookie("inhibit_beta_redirect")
  111.             if not c:
  112.                 # create one
  113.                 try:
  114.                     c = cookielib.Cookie(version=0,
  115.                         name='inhibit_beta_redirect', value='1', port=None,
  116.                         port_specified=False, domain='.launchpad.net',
  117.                         domain_specified=True, domain_initial_dot=True,
  118.                         path='/', path_specified=False, secure=True,
  119.                         expires=c_expires, discard=False, comment=None,
  120.                         comment_url=None, rest={}, rfc2109=False)
  121.                 except TypeError:
  122.                     #missing 'rfc2109' argument in python 2.4
  123.                     c = cookielib.Cookie(version=0,
  124.                         name='inhibit_beta_redirect', value='1', port=None,
  125.                         port_specified=False, domain='.launchpad.net',
  126.                         domain_specified=True, domain_initial_dot=True,
  127.                         path='/', path_specified=False, secure=True,
  128.                         expires=c_expires, discard=False, comment=None,
  129.                         comment_url=None, rest={})                    
  130.                 self.cookiejar.set_cookie(c)
  131.             else:
  132.                 # check if expired
  133.                 # more time
  134.                 pass
  135.         return self.get_cookie("inhibit_beta_redirect")
  136.         
  137.     def save_cookie(self, filename):
  138.         cj = cookielib.MozillaCookieJar()
  139.         for cookie in self.cookiejar:
  140.             cj.set_cookie(cookie)
  141.         cj.save(filename)
  142.     
  143. def pickle_cookie_load(cookie_string):
  144.     """ load a cookie instance from the value of the config file """
  145.     try:
  146.         cookie_string = base64.decodestring(cookie_string)
  147.     except Exception:
  148.         pass
  149.     try:
  150.         c = cPickle.loads(cookie_string)
  151.         if isinstance(c, cookielib.Cookie):
  152.             return c
  153.         else:
  154.             return None
  155.     except Exception:
  156.         return None
  157.     
  158. def pickle_cookie_dump(cookie_obj):
  159.     """ dump a cookie object and encode the resulting string """
  160.     if not isinstance(cookie_obj, cookielib.Cookie):
  161.         if cookie_obj is None:
  162.             return ""
  163.         raise TypeError
  164.     return base64.encodestring(cPickle.dumps(cookie_obj))
  165.         
  166. class HTTPConnectionConfig(Config):
  167.     """ read the configuration of the HTTPConnection from a config file. """
  168.     
  169.     # define load and save handler for the cookie objects    
  170.     Config.MAPPING["cookies"] = {"lp": (pickle_cookie_load, pickle_cookie_dump),
  171.                                  "edge": (pickle_cookie_load, pickle_cookie_dump),
  172.                                  "staging": (pickle_cookie_load, pickle_cookie_dump),
  173.                                  "inhibit_beta_redirect": (pickle_cookie_load, pickle_cookie_dump)}
  174.  
  175. class HTTPConnection(object):
  176.     """ class to manage a https connection to launchpad.net. """
  177.     def __init__(self, cookiefile=None, attempts=5, content_types=None,
  178.                     config_file=None, config_ignore=None):
  179.         self.__config = HTTPConnectionConfig(config_file, config_ignore)
  180.         self.__username = None
  181.         version = "python-launchpad-bugs/%s (Python-urllib2/%s) (user: %s)" %(utils.find_version_number(show_nick=True),urllib2.__version__, self.user)
  182.         self.__cookiefile = cookiefile
  183.         self.__cookie_handler = LPCookieProcessor()
  184.         self.__opener = urllib2.build_opener(self.__cookie_handler)
  185.         self.__opener.addheaders = [('User-agent', version)]
  186.         self.__poster = urllib2.build_opener(self.__cookie_handler, multipartpost_handler.MultipartPostHandler)
  187.         self.__poster.addheaders = [('User-agent', version)]
  188.         self.__attempts = attempts
  189.         self.content_types = content_types or ["text/html","text/plain"]
  190.         self.__progress_hook = None
  191.         self.__mode = HTTPCONNECTION.MODE.DEFAULT
  192.         self.__cookies_to_dump = []
  193.         for key, value in self.__config["cookies"].iteritems():
  194.             if value is None:
  195.                 self.__cookies_to_dump.append(key)
  196.             else:
  197.                 self.__cookie_handler.cookiejar.set_cookie(value)
  198.         
  199.     def get_auth(self):
  200.         return self.__cookie_handler
  201.         
  202.     def set_auth(self, auth):
  203.         if isinstance(auth, str):
  204.             self.__cookiefile = auth
  205.             try:
  206.                 return self.__cookie_handler.load_file(self.__cookiefile)
  207.             except IOError, e:
  208.                 raise exceptions.PythonLaunchpadBugsIOError(str(e))
  209.         elif isinstance(auth, dict):
  210.             try:
  211.                 email = auth["email"]
  212.                 password = auth["password"]
  213.             except KeyError:
  214.                 raise ValueError, "The argument of .set_auth() needs to be either a path to a valid \
  215. mozilla cookie-file or a dict like {'email':<lp-email-address>,'password':<password>}, but it is %s" %auth
  216.             self._do_login(email, password)
  217.         else:
  218.             raise ValueError, "The argument of .set_auth() needs to be either a path to a valid \
  219. mozilla cookie-file or a dict like {'email':<lp-email-address>,'password':<password>}, but it is %s" %auth
  220.  
  221.     def _do_login(self, email, password, server=None):
  222.         if server is None:
  223.             server = (  "https://bugs.launchpad.net/+login",
  224.                         "https://bugs.edge.launchpad.net/+login",
  225.                         "https://bugs.staging.launchpad.net/+login",)
  226.         for url in server:
  227.             try:
  228.                 r = self.post(url, {"loginpage_email": email,
  229.                                     "loginpage_password": password,
  230.                                     "loginpage_submit_login":"Log In"})
  231.             except exceptions.LaunchpadLoginError:
  232.                 raise exceptions.LaunchpadLoginFailed(url)
  233.             except exceptions.LaunchpadError, e:
  234.                 raise exceptions.LaunchpadLoginError(url,
  235.                             "Connection failed with %s" %e)
  236.         
  237.     @property
  238.     def user(self):
  239.         return self.__username or self.__config["user"]["lplogin"] or "unknown"
  240.         
  241.     def set_username(self, user):
  242.         self.__username = user
  243.             
  244.     def needs_login(self, url=None):
  245.         """ checks if authentication is needed.
  246.         this considers connection-mode and cookies
  247.         """
  248.         url = self._change_url(url or "https://launchpad.net/people/+me")
  249.         url = self.__opener.open(url).geturl()
  250.         return url.endswith("+login")
  251.         
  252.     def get(self, url):
  253.         return self._safe_urlopen(url, None, False)
  254.         
  255.     def post(self, url, data):
  256.         return self._safe_urlopen(url, data, True)
  257.             
  258.     def set_mode(self, mode):
  259.         assert mode in (HTTPCONNECTION.MODE.DEFAULT,
  260.                         HTTPCONNECTION.MODE.EDGE,
  261.                         HTTPCONNECTION.MODE.STABLE,
  262.                         HTTPCONNECTION.MODE.STAGING), "Unknown mode"
  263.         self.__mode = mode
  264.         c = self.__cookie_handler.change_inhibit_beta_redirect(self.__mode)
  265.         self.__config["cookies"]["inhibit_beta_redirect"] = c or None
  266.         if "inhibit_beta_redirect" in self.__cookies_to_dump:
  267.             self.__cookies_to_dump.remove("inhibit_beta_redirect")
  268.         self.__config.save()
  269.             
  270.     def _change_url(self, url):
  271.         u = list(urlparse.urlsplit(url))
  272.         if self.__mode == HTTPCONNECTION.MODE.EDGE:
  273.             # return *.edge.launchpad.net/...
  274.             u[1] = u[1].replace("staging.","") #remove staging from netloc
  275.             u[1] = u[1].replace("edge.","") # remove edge from netloc
  276.             u[1] = u[1].replace("launchpad.net", "edge.launchpad.net")
  277.         elif self.__mode == HTTPCONNECTION.MODE.STAGING:
  278.             # return *.staging.launchpad.net/...
  279.             u[1] = u[1].replace("staging.","") #remove staging from netloc
  280.             u[1] = u[1].replace("edge.","") # remove edge from netloc
  281.             u[1] = u[1].replace("launchpad.net", "staging.launchpad.net")
  282.             u[1] = u[1].replace("launchpadlibrarian.net", "staging.launchpadlibrarian.net")
  283.  
  284.         elif self.__mode == HTTPCONNECTION.MODE.STABLE:
  285.             # return *.launchpad.net/...
  286.             u[1] = u[1].replace("staging.","") #remove staging from netloc
  287.             u[1] = u[1].replace("edge.","") # remove edge from netloc
  288.         return urlparse.urlunsplit(u)
  289.         
  290.         
  291.     def _safe_urlopen(self, url, data, post):
  292.         url = self._change_url(url)
  293.         count = 0
  294.         text = None
  295.         contenttype = None
  296.         geturl = None
  297.         sock = None
  298.         if post:
  299.             opener = self.__poster
  300.         else:
  301.             opener = self.__opener
  302.         while count < self.__attempts:
  303.             #print "count: %s, url: %s" %(count,url) #DEBUG
  304.             try:
  305.                 if url[:4] != 'http':
  306.                     url_old = url
  307.                     url = 'https://' + url
  308.                     print "wrong url <%s>, try <%s>" %(url_old, url)
  309.                 if data:
  310.                     sock = opener.open(url,data)
  311.                 else:
  312.                     sock = opener.open(url)
  313.                 contenttype = sock.info()["Content-type"]
  314.                 #~ print sock.geturl()
  315.                 if sock.geturl().endswith("+login"):
  316.                     raise exceptions.LaunchpadLoginError(url)
  317.                 for ct in self.content_types:
  318.                     if contenttype.startswith(ct):
  319.                         if self.__progress_hook is None:
  320.                             text = sock.read()
  321.                             geturl = sock.geturl()
  322.                             sock.close()
  323.                         else:
  324.                             tmp_text = StringIO.StringIO()
  325.                             i = 0
  326.                             counter = 0
  327.                             size = int(sock.info()['Content-Length'])
  328.                             while i < size:
  329.                                 tmp_text.write(sock.read(self.__block_size))
  330.                                 i += self.__block_size
  331.                                 counter += 1
  332.                                 self.__progress_hook(counter, self.__block_size, size)
  333.                             text = tmp_text.getvalue()
  334.                             geturl = sock.geturl()
  335.                             sock.close()
  336.                             tmp_text.close()
  337.                         x = self.__cookies_to_dump[:]
  338.                         for i in x:
  339.                             c = self.__cookie_handler.get_cookie(i)
  340.                             if c:
  341.                                 self.__config["cookies"][i] = c
  342.                                 self.__cookies_to_dump.remove(i)
  343.                         if not self.__cookies_to_dump == x:
  344.                             self.__config.save()
  345.                         return _result(contenttype=contenttype,text=text, url=geturl)
  346.                 sock.close()
  347.                 raise IOError, "unsupported contenttype (contenttype=%s, url=%s)" %(contenttype, url)
  348.             except urllib2.URLError, e:
  349.                 try:
  350.                     error = e.code
  351.                 except AttributeError:
  352.                     error = e.reason
  353.                 count += 1
  354.         raise exceptions.choose_LaunchpadError(error, url)
  355.         
  356.     def set_progress_hook(self, hook_func, blocksize=4096):
  357.         assert blocksize, "blocksize needs to be an integer greater than 0"
  358.         self.__block_size = blocksize
  359.         assert callable(hook_func), "hook_func needs to be callable with three arguments"
  360.         self.__progress_hook = hook_func
  361.         
  362.     def save_cookie(self, filename):
  363.         self.__cookie_handler.save_cookie(filename)
  364.         
  365.         
  366. if __name__ == '__main__':
  367.     c = HTTPConnection()
  368.     print repr(c.user), c.user
  369.     
  370.     def example_hook(counter, block_size, size):
  371.         print (counter, block_size, size)
  372.         
  373.     c.set_progress_hook(example_hook)
  374.     x = c.get("https://bugs.edge.launchpad.net/ubuntu/+source/linux/+bug/200500/")
  375.     print len(x.text)
  376.