home *** CD-ROM | disk | FTP | other *** search
# Source Generated with Decompyle++ # File: in.pyo (Python 2.6) from __future__ import with_statement import sys import traceback import re import struct import logging import random import StringIO import calendar import time import rfc822 import socks import socket import asynchat import urllib import urllib2 import urlparse import httplib import httplib2 import cookielib import simplejson import itertools from httplib import HTTPConnection from httplib import NotConnected import primitives.funcs as primitives import proxy_settings from Events import EventMixin from callbacks import callsback try: sentinel except NameError: class Sentinel(object): def __repr__(self): return '<Sentinel (%r backup) %#x>' % (__file__, id(self)) sentinel = Sentinel() log = logging.getLogger('util.net') default_chunksize = 4096 def get_ips_s(hostname = ''): if not hostname: pass return socket.gethostbyname_ex(socket.gethostname())[2] def get_ips(hostname = ''): return [ socket.inet_aton(ip) for ip in get_ips_s(hostname) ] myips = get_ips def myip(): return myips()[0] def ip_from_bytes(bytes): return socket.inet_ntoa(bytes) class FileChunker(object): def __init__(self, fileobj, chunksize = default_chunksize, close_when_done = False, progress_cb = (lambda bytes: pass), bytecounter = None): self.fileobj = fileobj self.chunksize = chunksize self.close_when_done = close_when_done if bytecounter is None: bytecounter = fileobj.tell self.total = bytecounter() self.progress_cb = progress_cb self.cancelled = False def more(self): try: data_read = self.fileobj.read(self.chunksize) except ValueError: try: self.fileobj.close() except: pass return '' sz = len(data_read) if sz == 0 and self.close_when_done: self.fileobj.close() self.total += sz self.progress_cb(self.total) return data_read def tofile(cls, sourcefile, outfile, progress_callback = (lambda : pass), bytecounter = None): gen = cls.tofile_gen(sourcefile, outfile, progress_callback, bytecounter) gen.next() try: gen.next() except StopIteration: pass tofile = classmethod(tofile) def tofile_gen(cls, sourcefile, outfile, progress_callback = (lambda : pass), bytecounter = None): fc = cls(sourcefile, close_when_done = True, bytecounter = bytecounter) yield fc chunk = fc.more() bytes_written = 0 write = outfile.write tell = outfile.tell more = fc.more while chunk and not (fc.cancelled): write(chunk) bytes_written += len(chunk) progress_callback(tell()) chunk = more() outfile.close() tofile_gen = classmethod(tofile_gen) class NoneFileChunker(FileChunker): def more(self): if not super(NoneFileChunker, self).more(): pass def httpjoin(base, path, keepquery = False): if path.startswith('http'): return path joined = urlparse.urljoin(base, path) if not keepquery: parsed = list(urlparse.urlparse(joined)) parsed[4] = '' return urlparse.urlunparse(parsed) return joined class UrlQuery(str): def form_encoder(cls): return WebFormData form_encoder = classmethod(form_encoder) def parse(cls, url, parse_query = True, utf8 = False): (scheme, netloc, path, params, query, fragment) = urlparse.urlparse(url) if parse_query: query = cls.form_encoder().parse(query, utf8 = utf8) return dict(scheme = scheme, netloc = netloc, path = path, params = params, query = query, fragment = fragment) parse = classmethod(parse) def unparse(cls, scheme = '', netloc = '', path = '', params = '', query = None, fragment = ''): if query is None: query = { } if isinstance(query, dict): query = cls.form_encoder()(query) return urlparse.urlunparse((scheme, netloc, path, params, query, fragment)) unparse = classmethod(unparse) def __new__(cls, link, d = { }, **kwargs): if not d or kwargs: return str.__new__(cls, link) if link.endswith('?'): link = link[:-1] if '?' in link: joiner = None if d or kwargs else '' else: joiner = '?' return str.__new__(cls, ''.join([ link, joiner, cls.form_encoder()(d = d, **kwargs)])) class WebFormData(str): def encoder(cls): return urllib.urlencode encoder = classmethod(encoder) def __new__(cls, d = { }, **kwargs): if d and kwargs: kwargs.update(d) elif not kwargs: pass kwargs = d base = cls.encoder()(kwargs) return str.__new__(cls, base) def parse(cls, s, utf8 = False): querymap = { } if not s: return querymap encoding = s if utf8 else 'url' for elem in s.split('&'): if '=' in elem: (key, value) = elem.split('=', 1) querymap[key] = value.decode(encoding) continue querymap[elem] = True return querymap parse = classmethod(parse) class WebFormObjectData(WebFormData): def encoder(cls): return urlencode_object encoder = classmethod(encoder) class UrlQueryObject(UrlQuery): def form_encoder(cls): return WebFormObjectData form_encoder = classmethod(form_encoder) def urlencode_object(query): return urllib.urlencode(param(query)) def param(a): s = list() def add(key, value): s.append((key, value)) for prefix in a: buildParams(prefix, a[prefix], add) return s def buildParams(prefix, obj, add): if isinstance(obj, dict) and obj: for k, v in obj.items(): buildParams(prefix + '[' + k + ']', v, add) elif isinstance(obj, list): for k, v in enumerate(obj): if not isinstance(v, (list, dict)): k = '' buildParams(prefix + '[' + str(k) + ']', v, add) else: add(prefix, obj) def int_to_ip(s, byteorder = '<'): return '.'.join((lambda .0: for c in .0: str(ord(c)))(struct.pack(byteorder + 'I', int(s)))) spacify_pattern = re.compile('( {2,})') def spacify_repl(m): l = len(m.group()) if l == 2: return ' ' return ' ' + ''.join([ ' '] * (l - 2)) + ' ' def spacify(s): return spacify_pattern.sub(spacify_repl, s) urlregex = re.compile("([A-Za-z][A-Za-z0-9+.-]{1,120}:[A-Za-z0-9/](([A-Za-z0-9$_.+!*,;/?:@&~=-])|%[A-Fa-f0-9]{2}){1,333}(#([a-zA-Z0-9][a-zA-Z0-9$_.+!*,;/?:@&~=%-']{0,1000}))?)[^\\. <]") TLDs = [ 'arpa', 'root', 'aero', 'asia', 'biz', 'com', 'coop', 'edu', 'gov', 'info', 'int', 'museum', 'name', 'net', 'org', 'pro', 'ac', 'ad', 'ae', 'af', 'ag', 'ai', 'al', 'am', 'an', 'ao', 'aq', 'ar', 'as', 'at', 'au', 'aw', 'ax', 'az', 'ba', 'bb', 'bd', 'be', 'bf', 'bg', 'bh', 'bi', 'bj', 'bm', 'bn', 'bo', 'br', 'bs', 'bt', 'bv', 'bw', 'by', 'bz', 'ca', 'cc', 'cd', 'cf', 'cg', 'ch', 'ci', 'ck', 'cl', 'cm', 'cn', 'co', 'cr', 'cu', 'cv', 'cx', 'cy', 'cz', 'de', 'dj', 'dk', 'dm', 'do', 'dz', 'ec', 'ee', 'eg', 'er', 'es', 'et', 'eu', 'fi', 'fj', 'fk', 'fm', 'fo', 'fr', 'ga', 'gb', 'gd', 'ge', 'gf', 'gg', 'gh', 'gi', 'gl', 'gm', 'gn', 'gp', 'gq', 'gr', 'gs', 'gt', 'gu', 'gw', 'gy', 'hk', 'hm', 'hn', 'hr', 'ht', 'hu', 'id', 'ie', 'il', 'im', 'in', 'io', 'iq', 'ir', 'is', 'it', 'je', 'jm', 'jo', 'jp', 'ke', 'kg', 'kh', 'ki', 'km', 'kn', 'kp', 'kr', 'kw', 'ky', 'kz', 'la', 'lb', 'lc', 'li', 'lk', 'lr', 'ls', 'lt', 'lu', 'lv', 'ly', 'ma', 'mc', 'md', 'me', 'mg', 'mh', 'mk', 'ml', 'mm', 'mn', 'mo', 'mp', 'mq', 'mr', 'ms', 'mt', 'mu', 'mv', 'mw', 'mx', 'my', 'mz', 'na', 'nc', 'ne', 'nf', 'ng', 'ni', 'nl', 'no', 'np', 'nr', 'nu', 'nz', 'om', 'pa', 'pe', 'pf', 'pg', 'ph', 'pk', 'pl', 'pm', 'pn', 'pr', 'ps', 'pt', 'pw', 'py', 'qa', 're', 'ro', 'rs', 'ru', 'rw', 'sa', 'sb', 'sc', 'sd', 'se', 'sg', 'sh', 'si', 'sj', 'sk', 'sl', 'sm', 'sn', 'so', 'sr', 'st', 'su', 'sv', 'sy', 'sz', 'tc', 'td', 'tf', 'tg', 'th', 'tj', 'tk', 'tl', 'tm', 'tn', 'to', 'tp', 'tr', 'tt', 'tv', 'tw', 'tz', 'ua', 'ug', 'uk', 'um', 'us', 'uy', 'uz', 'va', 'vc', 've', 'vg', 'vi', 'vn', 'vu', 'wf', 'ws', 'ye', 'yt', 'yu', 'za', 'zm', 'zw'] domains = '(?:%s)' % '|'.join(TLDs) email_regex_string = '(?:([a-zA-Z0-9_][a-zA-Z0-9_\\-\\.]*)(\\+[a-zA-Z0-9_\\-\\.]+)?@((?:[a-zA-Z0-9\\-_]+\\.?)*[a-zA-Z]{1,4}))' email_regex = re.compile(email_regex_string) email_wholestring_regex = re.compile('^' + email_regex_string + '$') is_email = primitives.funcs.ischeck((lambda s: bool(email_wholestring_regex.match(s)))) class EmailAddress(tuple): def __new__(cls, addr, default_domain = sentinel): try: (name, label, domain) = parse_email(addr) except: if default_domain is sentinel: raise default_domain is sentinel (name, label, domain) = parse_email(addr + '@' + default_domain) return tuple.__new__(cls, (name, label, domain)) def name(self): return self[0] name = property(name) def label(self): return self[1] label = property(label) def domain(self): return self[2] domain = property(domain) def __str__(self): if self.label: return '%s+%s@%s' % self return '%s@%s' % (self.name, self.domain) def __repr__(self): return '<EmailAddress %s>' % (self,) def parse_email(s): match = email_wholestring_regex.match(s) if match is None: raise ValueError('Not a valid email address: %r', s) match is None (user, lbl, dom) = match.groups() if lbl: lbl = lbl.strip('+') else: lbl = '' return (user, lbl, dom) protocols = 'ftp|https?|gopher|msnim|icq|telnet|nntp|aim|file|svn|svn+(?:\\w)+' single_letter_rule_tlds = frozenset(('net', 'com', 'org')) allowed_single_letter_domains = frozenset(('i.net', 'q.com', 'q.net', 'x.com', 'x.org', 'z.com')) linkify_url_pattern = re.compile('(?=[a-zA-Z0-9]) # Must start correctly\n ((?: # Match the leading part (proto://hostname, or just hostname)\n (?:(?P<protocol>%s) # protocol\n :// # ://\n (?: # Optional \'username:password@\'\n (?P<username>\\w+) # username\n (?::(?P<password>\\w+))? # optional :password\n @ # @\n )?)? #\n (?P<hostname> # hostname (sub.example.com). single-letter\n (?:[iqxz]|(?:[-\\w\\x7F-\\xFF]+)) # domains are not allowed, except those listed:\n (?:\\.[\\w\\x7F-\\xFF][-\\w\\x7F-\\xFF]*)*) # http://en.wikipedia.org/wiki/Single-letter_second-level_domains\n )? #\n (?::(?P<port>\\d+))? # Optional port number\n (?P<selector>\n (?: # Rest of the URL, optional\n /? # Start with \'/\'\n [^.!,?;:"<>\\[\\]{}\\s\\x7F-\\xFF]+ # Can\'t start with these\n (?: #\n [.!,?;:]+ # One or more of these\n [^.!,?;:"<>{}\\s\\x7F-\\xFF]+ # Can\'t finish with these\n #\'" # # or \' or "\n )*) #\n )?) #\n ' % protocols, re.VERBOSE) def isurl(text): m = linkify_url_pattern.match(text) if not m: return False protocol = m.group('protocol') host = m.group('hostname') if host is not None: if protocol is None: myTLDs = m if '.' in host else [ host] if len(myTLDs) < 2 or myTLDs[-1] not in TLDs: return False return True class LinkAccumulator(object): def __init__(self, s = None): self.links = [] self.spans = [] if s is not None: linkify_url_pattern.sub(self.repl, s) def repl(self, m): (url, protocol, after) = _url_from_match(m) if url is None: return '' href = url is None if protocol is None else url self.links.append(href) self.spans.append(m.span()) return '' def __iter__(self): return itertools.izip(self.links, self.spans) def find_links(text): return LinkAccumulator(text).links def _url_from_match(m): protocol = m.group('protocol') host = m.group('hostname') url = m.group() after = '' if url.endswith(')') and '(' not in url: url = url[:-1] after = ')' if host is not None: myTLDs = None if '.' in host else [ host] if protocol is None: if len(myTLDs) < 2 or myTLDs[-1] not in TLDs: return (None, None, None) if len(myTLDs) >= 2: second_level_domain = myTLDs[-2] top_level_domain = myTLDs[-1] if len(second_level_domain) == 1 and '.'.join((second_level_domain, top_level_domain)) not in allowed_single_letter_domains and top_level_domain in single_letter_rule_tlds: return (None, None, None) return (url, protocol, after) return (None, None, None) def _dolinkify(text): def repl(m): (url, protocol, after) = _url_from_match(m) if url is None: (i, j) = m.span() return text[i:j] href = url is None if protocol is None else url return '<a href="%s">%s</a>' % (href, url) + after text = linkify_url_pattern.sub(repl, text) return text def linkify(text): if isinstance(text, unicode): return _linkify(text.encode('utf-8')).decode('utf-8') return _linkify(text) def _linkify(text): if not re.search('<.*>', text): return _dolinkify(text) lines = [] prev_line = '' for line in re.split('(<.*?>)', text): if not re.match('<.*?>', line) and not prev_line.startswith('<a'): line = _dolinkify(line) prev_line = line lines.append(line) return ''.join(lines) class QueueableMixin(object): def __init__(self): object.__init__(self) self._on_queue = False def queue(self): if not self._on_queue: self._queue() self._on_queue = True def unqueue(self): if self._on_queue: self._unqueue() self._on_queue = False class ProducerQueuable(QueueableMixin): def __init__(self, sck): QueueableMixin.__init__(self) self.sck = sck def _queue(self): self.sck.push_with_producer(self) def _unqueue(self): try: self.sck.producer_fifo.remove(self) except ValueError: pass class RoundRobinProducer(ProducerQueuable): def __init__(self, sck): ProducerQueuable.__init__(self, sck) self.list = [] def add(self, prod): try: if not callable(prod.more): raise AssertionError('Producers must have a "more" method') callable(prod.more) except: traceback.print_exc() raise self.unqueue() self.list.append(prod) self.queue() def more(self): self._on_queue = True d = None l = self.list prod = None while not d and l: prod = l.pop(0) d = prod.more() if d: l.append(prod) else: self.unqueue() if self.list: self.queue() return d class PriorityProducer(ProducerQueuable): def __init__(self, sck): ProducerQueuable.__init__(self, sck) self.high = [] self.mid = [] self.low = [] def add(self, prod, pri = 'mid'): self.unqueue() getattr(self, pri).append(prod) self.queue() def more(self): self._on_queue = True d = None for l in (self.high, self.mid, self.low): if not l: continue while not d and l: prod = l.pop(0) d = prod.more() if d: l.insert(0, prod) break continue return d class HTTPConnProgress(HTTPConnection): def send_file_cb(self, fileobj, progress_cb, blocksize = default_chunksize, progressDelta = 0): if self.sock is None: if self.auto_open: self.connect() else: raise NotConnected() self.auto_open if self.debuglevel > 0: print 'sending contents of', fileobj try: read = fileobj.read sendall = self.sock.sendall chunk = read(blocksize) total = 0 while chunk: total += len(chunk) sendall(chunk) progress_cb(total - progressDelta) chunk = read(blocksize) except socket.error: v = None if v[0] == 32: self.close() raise class SocketEventMixin(EventMixin): events = EventMixin.events | set(('connected', 'connection_failed', 'socket_error', 'socket_closed')) def post_connect_error(self, e = None): self.event('socket_error') self.post_connect_disconnect() def post_connect_expt(self): self.event('socket_error') self.post_connect_disconnect() def post_connect_disconnect(self): self.close() self.event('socket_closed') def post_connect_close(self): self.close() self.event('socket_closed') def reassign(self): self.handle_expt = self.post_connect_expt self.handle_error = self.post_connect_error self.handle_close = self.post_connect_close self.do_disconnect = self.post_connect_disconnect def build_cookie(name, value, version = 0, domain = sentinel, port = sentinel, path = sentinel, secure = False, expires = None, discard = False, comment = None, comment_url = None, rest = { 'httponly': None }, rfc2109 = False): if domain is sentinel: domain = None domain_specified = False domain_initial_dot = False else: domain_specified = True domain_initial_dot = domain.startswith('.') if port is sentinel: port = None port_specified = False else: port_specified = True if path is sentinel: path = None path_specified = False else: path_specified = True return cookielib.Cookie(**locals()) def GetSocketType(): d = GetProxyInfo() if d: return socks.socksocket return socket.socket NONE = 'NONPROX' SYSDEFAULT = 'SYSPROX' CUSTOM = 'SETPROX' def GetProxyInfo(): ps = proxy_settings try: pd = ps.get_proxy_dict() except Exception: e = None print >>sys.stderr, 'No proxies because: %r' % e pd = { } get = pd.get proxytype = get('proxytype') port = get('port') try: port = int(port) except: port = None addr = get('addr') username = get('username') password = get('password') override = get('override') rdns = get('rdns', False) try: override = int(override) except: if override not in (SYSDEFAULT, CUSTOM, NONE): override = SYSDEFAULT if override: override = CUSTOM else: override = SYSDEFAULT if override == NONE: return { } if all((type, port, addr)): proxytype = getattr(socks, ('proxy_type_%s' % proxytype).upper(), None) return dict(addr = addr, port = port, username = username, password = password, proxytype = proxytype, rdns = rdns) return { } def GetProxyInfoHttp2(): i = GetProxyInfo() if not i: return None return httplib2.ProxyInfo(proxy_type = i['proxytype'], proxy_host = i['addr'], proxy_port = i['port'], proxy_user = i['username'], proxy_pass = i['password'], proxy_rdns = i.get('rdns', False)) def getproxies_digsby(): pinfo = GetProxyInfo() proxies = { } if pinfo.get('username', None) and pinfo.get('password', None): unpw = '%s:%s@' % (pinfo['username'], pinfo['password']) else: unpw = '' if pinfo.get('port', None): port = ':' + str(pinfo['port']) else: port = '' host = pinfo.get('addr', None) if not host: return proxies all = unpw + host + port proxies = urllib.OneProxy() proxies._proxyServer = all if pinfo['proxytype'] != socks.PROXY_TYPE_HTTP: proxy_url = host % 'socks%d://' if pinfo['proxytype'] == socks.PROXY_TYPE_SOCKS4 else 5 + all return dict(socks = proxy_url, http = proxy_url, https = proxy_url) proxies['https'] = 'http://' + all proxies['http'] = 'http://' + all proxies['ftp'] = 'http://' + all return proxies class SocksProxyHandler(urllib2.ProxyHandler): handler_order = 100 def proxy_open(self, req, type): try: req._proxied except AttributeError: proxyinfo = self.proxies.get(type, '') proxytype = urllib2._parse_proxy(proxyinfo)[0] if proxytype is None: req._proxied = False return urllib2.ProxyHandler.proxy_open(self, req, type) req._proxytype = proxytype req._proxied = True if proxytype == 'http' and type != 'https': return urllib2.ProxyHandler.proxy_open(self, req, type) return None except: type != 'https' return None def socks4_open(self, req): return self.socks_open(req, 4) def socks5_open(self, req): return self.socks_open(req, 5) def socks_open(self, req, sockstype): (orig_url_type, __, __, orighostport) = urllib2._parse_proxy(req.get_full_url()) req.set_proxy(orighostport, orig_url_type) endpoint = req.get_host() if ':' in endpoint: (host, port) = endpoint.rsplit(':', 1) port = int(port) else: host = endpoint port = 80 req._proxied = True return self.parent.open(req) try: import ssl except ImportError: pass class SocksHttpsOpener(urllib2.HTTPSHandler): handler_order = 101 def https_open(self, req): if getattr(req, '_proxied', False) and getattr(req, '_proxytype', None) is not None: return urllib2.HTTPSHandler.do_open(self, SocksHttpsConnection, req) return urllib2.HTTPSHandler.https_open(self, req) class SocksHttpsConnection(httplib.HTTPSConnection): _sockettype = socks.socksocket def connect(self): pd = urllib.getproxies().get('https', None) if pd is None: sockstype = '' else: (sockstype, user, password, hostport) = urllib2._parse_proxy(pd) (host, port) = hostport.rsplit(':', 1) port = int(port) sock = self._sockettype(socket.AF_INET, socket.SOCK_STREAM) sock.setproxy(proxytype = getattr(socks, 'PROXY_TYPE_%s' % sockstype.upper()), addr = host, port = port, rdns = True, username = user, password = password) sock.connect((self.host, self.port)) self.sock = ssl.wrap_socket(sock, self.key_file, self.cert_file) class SocksHttpOpener(urllib2.HTTPHandler): handler_order = 101 def http_open(self, req): proxytype = getattr(req, '_proxytype', None) if getattr(req, '_proxied', False) and proxytype not in ('http', None): return urllib2.HTTPHandler.do_open(self, SocksConnection, req) return urllib2.HTTPHandler.http_open(self, req) class SocksConnection(httplib.HTTPConnection): _sockettype = socks.socksocket def connect(self): pd = urllib.getproxies().get('http', None) if pd is None: sockstype = '' else: (sockstype, user, password, hostport) = urllib2._parse_proxy(pd) if 'socks' not in sockstype: return httplib.HTTPConnection.connect(self) (host, port) = hostport.rsplit(':', 1) port = int(port) for res in socket.getaddrinfo(self.host, self.port, 0, socket.SOCK_STREAM): (af, socktype, proto, canonname, sa) = res try: self.sock = self._sockettype(af, socktype, proto) self.sock.setproxy(proxytype = getattr(socks, 'PROXY_TYPE_%s' % sockstype.upper()), addr = host, port = port, rdns = False, username = user, password = password) self.sock.connect(sa) except socket.error: 'socks' not in sockstype msg = 'socks' not in sockstype if self.debuglevel > 0: print 'connect fail:', (self.host, self.port) if self.sock: self.sock.close() self.sock = None continue except: 'socks' not in sockstype if not self.sock: raise socket.error, msg self.sock class DigsbyHttpProxyPasswordManager(urllib2.HTTPPasswordMgr): def find_user_password(self, realm, uri): pi = GetProxyInfo() if not pi['username']: pass if not pi['password']: pass return (None, None) if not hasattr(urllib, '_getproxies'): urllib._getproxies = urllib.getproxies urllib.getproxies = getproxies_digsby urllib2.UnknownHandler.handler_order = sys.maxint urllib2.ProxyDigestAuthHandler.__bases__ = urllib2.ProxyDigestAuthHandler.__bases__[::-1] urllib2.ProxyBasicAuthHandler.handler_order = 499 httplib2.ProxyInfo.get_default_proxy = staticmethod(GetProxyInfoHttp2) def GetDefaultHandlers(): handlers = [ SocksProxyHandler, SocksHttpOpener] httpsopener = globals().get('SocksHttpsOpener', None) if httpsopener is not None: handlers.append(httpsopener) pwdmgr = DigsbyHttpProxyPasswordManager() for auth_handler_type in (urllib2.ProxyBasicAuthHandler, urllib2.ProxyDigestAuthHandler): handlers.append(auth_handler_type(pwdmgr)) return handlers def build_opener(*a, **k): if 'default_classes' not in k: k['default_classes'] = GetDefaultHandlers() + urllib2.default_opener_classes return urllib2.build_opener(*a, **k) opener = urllib2.build_opener(*GetDefaultHandlers()) urllib2.install_opener(opener) _hostprog = re.compile('^//([^/?]*)(.*)$') def splithost(url): match = _hostprog.match(url) if match: groups = match.group(1, 2) if groups[0] == '': return (groups[0], '/' + groups[1]) return groups match return (None, url) urllib.splithost = urllib2.splithost = splithost def _HTTPError__repr(self): if not hasattr(self, 'content'): try: self.content = self.read() self.close() except Exception: e = None self._error = e self.content = 'error reading body: %r' % e self.read = lambda : '' self._stringio = StringIO.StringIO(self.content) self.read = self._stringio.read etxt = self.content return '<HTTPError headers = %r, body = %r>' % (str(getattr(self, 'hdrs', { })), etxt) urllib2.HTTPError.__repr__ = _HTTPError__repr def httpok(_code): return getattr(_code, 'status', _code) // 100 == 2 class SimpleProducer(asynchat.simple_producer): def more(self): data = asynchat.simple_producer.more(self) if data == '': data = None return data class CallbackProducerMixin(object): def __init__(self): bases = self.__class__.__bases__ found_self = False self._siblingClass = None for base in bases: if base is CallbackProducerMixin: found_self = True continue if hasattr(base, 'more') and found_self: self._siblingClass = base break continue if self._siblingClass is None: raise AssertionError("This mix-in requires there is a sibling class with a 'more' method. Additionally, CallbackProducerMixin must be *before* that class in the inheritance list (for method resolution reasons).") self._siblingClass is None def set_callback(self, callback = None): self._callback = callback set_callback = callsback(set_callback) def more(self): if not hasattr(self, '_siblingClass'): result = None else: result = self._siblingClass.more(self) if result is None: if getattr(self, '_callback', None) is not None: self._callback.success() if getattr(self, '_callback', None) is not None: del self._callback return result class SimpleCallbackProducer(CallbackProducerMixin, SimpleProducer): def __init__(self, data): SimpleProducer.__init__(self, data) CallbackProducerMixin.__init__(self) def _fifo_remove(self, val): try: self.list.remove(val) except Exception: return False return True asynchat.fifo.remove = _fifo_remove def producer_cb(data, callback = None): prod = SimpleCallbackProducer(data) prod.set_callback(callback = callback) return prod producer_cb = callsback(producer_cb) class GeneratorProducer(object): def __init__(self, gen): self.gen = gen def more(self): if self.gen is None: return None try: return self.gen.next() except StopIteration: self.gen is None self.gen is None self.gen = None return None _short_domains = frozenset(('2big.at', '2me.tw', '3.ly', 'a.gd', 'a2n.eu', 'abbrr.com', 'adjix.com', 'arunaurl.com', 'beam.to', 'bit.ly', 'bitly.com', 'bkite.com', 'blip.fm', 'bloat.me', 'budurl.com', 'burnurl.com', 'canurl.com', 'chilp.it', 'cli.gs', 'decenturl.com', 'digg.com', 'digs.by', 'dn.vc', 'doiop.com', 'durl.us', 'dwarfurl.com', 'easyuri.com', 'easyurl.net', 'ff.im', 'fon.gs', 'fyiurl.com', 'ginx.com', 'goo.gl', 'go2.me', 'hex.io', 'hopurl.com', 'hurl.ws', 'icanhaz.com', 'idek.net', 'is.gd', 'ix.it', 'jijr.com', 'jmp2.net', 'knol.me', 'krz.ch', 'kurl.us', 'last.fm', 'lin.cr', 'lnk.in', 'makeitbrief.com', 'memurl.com', 'micurl.com', 'minu.ws', 'moourl.com', 'myturl.com', 'notlong.com', 'ow.ly', 'pic.im', 'pikchur.com', 'ping.fm', 'piurl.com', 'poprl.com', 'qurlyq.com', 'r.im', 'refurl.com', 'rubyurl.com', 'rurl.org', 'rurl.us', 's7y.us', 'sai.ly', 'sbt.sh', 'shorl.comshort.ie', 'short.to', 'shortna.me', 'shrinkify.com', 'shw.com', 'si9.org', 'skocz.pl', 'smalur.com', 'sn.im', 'snipr.com', 'snipurl.com', 'snurl.com', 'spedr.com', 'starturl.com', 'three.ly', 'timesurl.at', 'tiny.cc', 'tiny.pl', 'tinyarro.ws', 'tinylink.co.za', 'tinyuri.ca', 'tinyurl.com', 'tnij.org', 'tr.im', 'turo.us', 'twitclicks.com', 'twitpic.com', 'twt.fm', 'twurl.cc', 'twurl.nl', 'u.nu', 'ub0.cc', 'uris.jp', 'urlb.at', 'urlcut.com', 'urlenco.de', 'urlhawk.com', 'urltea.com', 'vieurl.com', 'w3t.org', 'x.se', 'xaddr.com', 'xr.com', 'xrl.us', 'yep.it', 'zi.ma', 'zombieurl.com', 'zz.gd')) def is_short_url(url, domains = _short_domains): parsed = urlparse.urlparse(url) if parsed.netloc in domains: return True return False def get_snurl(url): return get_short_url(url, 'snurl') def get_isgd(url): return get_short_url(url, 'isgd') def get_tinyurl(url): return get_short_url(url, 'tinyurl') class UrlShortenerException(Exception): pass from lrucache import LRU _short_url_cache = LRU(10) def cache_shortened_url(url, short_url): if short_url: _short_url_cache[short_url] = url class UrlShortener(object): endpoint = None def build_request_url(self, url): return UrlQuery(self.endpoint, d = self.get_args(url.encode('utf-8'))) def shorten(self, url): try: resp = urllib2.urlopen(self.build_request_url(url)) except urllib2.HTTPError: e = None resp = e short_url = self.process_response(resp) cache_shortened_url(url, short_url) return short_url def shorten_async(self, url, success, error = None): def async_success(req, resp): try: ret = self.process_response(resp) except Exception: e = None if error is not None: error(e) except: error is not None cache_shortened_url(url, ret) success(ret) def async_error(req = None, resp = (None,)): print req print resp if error is not None: error(None) import common.asynchttp as asynchttp asynchttp.httpopen(self.build_request_url(url), success = async_success, error = async_error) def get_args(self, url): raise NotImplementedError def process_response(self, resp): if resp.code != 200: body = resp.read() raise UrlShortenerException(body) resp.code != 200 ret = resp.read() return ret class isgd_shortener(UrlShortener): endpoint = 'http://is.gd/api.php' def get_args(self, url): return dict(longurl = url) class tinyurl_shortener(UrlShortener): endpoint = 'http://tinyurl.com/api-create.php' def get_args(self, url): return dict(url = url) class threely_shortener(UrlShortener): endpoint = 'http://3.ly/' def get_args(self, url): return dict(api = 'em5893833', u = url) def process_response(self, resp): ret = UrlShortener.process_response(self, resp) if not ret.startswith(self.endpoint): raise UrlShortenerException(ret) ret.startswith(self.endpoint) return ret class snipr_shortener(UrlShortener): endpoint = 'http://snipr.com/site/snip' def get_args(self, url): return dict(r = 'simple', link = url.encode('url')) def process_response(self, resp): ret = UrlShortener.process_response(self, resp) if not ret.startswith('http'): raise UrlShortenerException('bad url: %r' % ret, ret) ret.startswith('http') return ret class shortname_shortener(UrlShortener): endpoint = 'http://shortna.me/hash/' def get_args(self, url): return dict(snURL = url, api = 0) def process_response(self, resp): ret = UrlShortener.process_response(self, resp) import lxml.html as HTML doc = HTML.fromstring(ret) links = doc.findall('a') for link in links: href = link.attrib.get('href') if href is not None and href.startswith('http://shortna.me/') and href != 'http://shortna.me': return href raise UrlShortenerException('short link not found in %r' % ret, ret) class digsby_shortener(UrlShortener): endpoint = 'https://accounts.digsby.com/api/shorturl' def get_args(self, url): import common import hashlib username = common.profile.username.encode('utf8') password = hashlib.sha256(common.profile.password.encode('utf8')).digest() return { 'user': username, 'pass': password, 'link': url } def process_response(self, httpresp): ret = UrlShortener.process_response(self, httpresp) resp = simplejson.loads(ret) if resp['shorter']['status'] == 'error': raise UrlShortenerException(resp['shorter']['errormsg']) resp['shorter']['status'] == 'error' if resp['shorter']['status'] == 'ok': import common url = resp['shorter']['shortURL'] to_add = common.pref('urlshorteners.digsby.append_text', type = unicode, default = u'') return url + to_add class bitly_shortener(UrlShortener): login = 'digsby' api_key = 'R_1fdb0bb8ce9af01f9939c2ffdf391dc8' endpoint = 'http://api.bit.ly/shorten' def __init__(self, login = None, api_key = None): if login is not None: self.login = login if api_key is not None: self.api_key = api_key def get_args(self, url): return dict(longUrl = url, version = '2.0.1', login = self.login, apiKey = self.api_key) def process_response(self, resp): ret = UrlShortener.process_response(self, resp) try: info = simplejson.loads(ret) except Exception: raise UrlShortenerException('expected JSON') if info['errorCode'] == 0: return self.extract_shorturl(info) raise UrlShortenerException(info['errorMessage']) def extract_shorturl(self, info): return info['results'].values()[0]['shortUrl'] class digsby_bitly_shortener(bitly_shortener): def extract_shorturl(self, info): return 'http://digs.by/' + info['results'].values()[0]['userHash'] _shorteners = { 'isgd': isgd_shortener, 'tinyurl': tinyurl_shortener, 'tiny': tinyurl_shortener, 'threely': threely_shortener, '3ly': threely_shortener, 'shortname': shortname_shortener, 'digsby': digsby_bitly_shortener } def get_short_url(url, provider = None, choices = None): if choices is None: choices = list(_shorteners.keys()) choices = choices[:] random.shuffle(choices) if provider is not None: choices.append(provider) else: import common choices.append(common.pref('url_shortener.default', type = basestring, default = 'digsby')) e = None while choices: try: provider = choices.pop() shortener = _shorteners.get(provider) if shortener is None: raise Exception('UrlShortener provider %r not found', provider) shortener is None return shortener().shorten(url) continue except Exception: e = None log.error('error getting short URL from %r: %r', provider, e) shortener = None provider = None continue None<EXCEPTION MATCH>Exception if e is None: e = Exception('No shorteners found!') raise e def wget(url, data = None): closing = closing import contextlib import urllib2 try: web = _[1] return web.read() finally: pass def long_url_from_cache(shorturl): try: return _short_url_cache[shorturl] except KeyError: return None def unshorten_url(url, cb): longurl = long_url_from_cache(url) if url is not None: return cb(longurl) requrl = UrlQuery('http://untiny.me/api/1.0/extract', url = url, format = 'json') def success(req, resp): json = resp.read() unshortened_url = simplejson.loads(json)['org_url'] cb(unshortened_url) def error(req, resp): pass import common.asynchttp as asynchttp return asynchttp.httpopen(requrl, success = success, error = error) def timestamp_to_http_date(ts): return rfc822.formatdate(timeval = ts) def http_date_to_timestamp(date_str): if date_str is None: return None return calendar.timegm(rfc822.parsedate(date_str)) def user_agent(): return 'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US; rv:1.9.2.10) Gecko/20100914 Firefox/3.6.15' if __name__ == '__main__': print get_snurl('http://www.google.com') 
(.txt)