home *** CD-ROM | disk | FTP | other *** search
/ Maximum CD 2011 January / maximum-cd-2011-01.iso / DiscContents / calibre-0.7.26.msi / file_1465 (.txt) < prev    next >
Encoding:
Python Compiled Bytecode  |  2010-10-31  |  10.1 KB  |  356 lines
  1. # Source Generated with Decompyle++
  2. # File: in.pyc (Python 2.6)
  3.  
  4. from __future__ import with_statement
  5. __license__ = 'GPL v3'
  6. __copyright__ = '2009, Kovid Goyal <kovid@kovidgoyal.net>'
  7. __docformat__ = 'restructuredtext en'
  8. import sys
  9. import os
  10. import cPickle
  11. import time
  12. import tempfile
  13. from math import ceil
  14. from threading import Thread, RLock
  15. from Queue import Queue, Empty
  16. from multiprocessing.connection import Listener, arbitrary_address
  17. from collections import deque
  18. from binascii import hexlify
  19. from calibre.utils.ipc.launch import Worker
  20. from calibre.utils.ipc.worker import PARALLEL_FUNCS
  21. from calibre import detect_ncpus as cpu_count
  22. from calibre.constants import iswindows
  23. from calibre.ptempfile import base_dir
  24. _counter = 0
  25.  
  26. class ConnectedWorker(Thread):
  27.     
  28.     def __init__(self, worker, conn, rfile):
  29.         Thread.__init__(self)
  30.         self.daemon = True
  31.         self.conn = conn
  32.         self.worker = worker
  33.         self.notifications = Queue()
  34.         self._returncode = 'dummy'
  35.         self.killed = False
  36.         self.log_path = worker.log_path
  37.         self.rfile = rfile
  38.  
  39.     
  40.     def start_job(self, job):
  41.         notification = PARALLEL_FUNCS[job.name][-1] is not None
  42.         self.conn.send((job.name, job.args, job.kwargs, job.description))
  43.         if notification:
  44.             self.start()
  45.         else:
  46.             self.conn.close()
  47.         self.job = job
  48.  
  49.     
  50.     def run(self):
  51.         while True:
  52.             
  53.             try:
  54.                 x = self.conn.recv()
  55.                 self.notifications.put(x)
  56.             continue
  57.             except BaseException:
  58.                 break
  59.                 continue
  60.             
  61.  
  62.             None<EXCEPTION MATCH>BaseException
  63.         
  64.         try:
  65.             self.conn.close()
  66.         except BaseException:
  67.             pass
  68.  
  69.  
  70.     
  71.     def kill(self):
  72.         self.killed = True
  73.         
  74.         try:
  75.             self.worker.kill()
  76.         except BaseException:
  77.             pass
  78.  
  79.  
  80.     
  81.     def is_alive(self):
  82.         if not (self.killed):
  83.             pass
  84.         return self.worker.is_alive
  85.  
  86.     is_alive = property(is_alive)
  87.     
  88.     def returncode(self):
  89.         if self._returncode != 'dummy':
  90.             return self._returncode
  91.         r = self.worker.returncode
  92.         if self.killed and r is None:
  93.             self._returncode = 1
  94.             return 1
  95.         return r
  96.  
  97.     returncode = property(returncode)
  98.  
  99.  
  100. class CriticalError(Exception):
  101.     pass
  102.  
  103.  
  104. class Server(Thread):
  105.     
  106.     def __init__(self, notify_on_job_done = (lambda x: x), pool_size = None, limit = sys.maxint, enforce_cpu_limit = True):
  107.         global _counter
  108.         Thread.__init__(self)
  109.         self.daemon = True
  110.         self.id = _counter + 1
  111.         _counter += 1
  112.         if enforce_cpu_limit:
  113.             limit = min(limit, cpu_count())
  114.         
  115.         self.pool_size = None if pool_size is None else pool_size
  116.         self.notify_on_job_done = notify_on_job_done
  117.         self.auth_key = os.urandom(32)
  118.         self.address = None(arbitrary_address if iswindows else 'AF_UNIX')
  119.         if iswindows and self.address[1] == ':':
  120.             self.address = self.address[2:]
  121.         
  122.         self.listener = Listener(address = self.address, authkey = self.auth_key, backlog = 4)
  123.         self.add_jobs_queue = Queue()
  124.         self.changed_jobs_queue = Queue()
  125.         self.kill_queue = Queue()
  126.         self.waiting_jobs = []
  127.         self.pool = deque()
  128.         self.workers = deque()
  129.         self.launched_worker_count = 0
  130.         self._worker_launch_lock = RLock()
  131.         self.start()
  132.  
  133.     
  134.     def launch_worker(self, gui = False, redirect_output = None):
  135.         self._worker_launch_lock.__enter__()
  136.         
  137.         try:
  138.             self.launched_worker_count += 1
  139.             id = self.launched_worker_count
  140.         finally:
  141.             pass
  142.  
  143.         (fd, rfile) = tempfile.mkstemp(prefix = 'ipc_result_%d_%d_' % (self.id, id), dir = base_dir(), suffix = '.pickle')
  144.         os.close(fd)
  145.         env = {
  146.             'CALIBRE_WORKER_ADDRESS': hexlify(cPickle.dumps(self.listener.address, -1)),
  147.             'CALIBRE_WORKER_KEY': hexlify(self.auth_key),
  148.             'CALIBRE_WORKER_RESULT': hexlify(rfile) }
  149.         for i in range(2):
  150.             cw = self.do_launch(env, gui, redirect_output, rfile)
  151.             if isinstance(cw, ConnectedWorker):
  152.                 break
  153.                 continue
  154.             None if redirect_output is None else self._worker_launch_lock
  155.         
  156.         if isinstance(cw, basestring):
  157.             raise CriticalError('Failed to launch worker process:\n' + cw)
  158.         isinstance(cw, basestring)
  159.         return cw
  160.  
  161.     
  162.     def do_launch(self, env, gui, redirect_output, rfile):
  163.         w = Worker(env, gui = gui)
  164.         
  165.         try:
  166.             w(redirect_output = redirect_output)
  167.             conn = self.listener.accept()
  168.             if conn is None:
  169.                 raise Exception('Failed to launch worker process')
  170.             conn is None
  171.         except BaseException:
  172.             
  173.             try:
  174.                 w.kill()
  175.             except:
  176.                 pass
  177.  
  178.             import traceback
  179.             return traceback.format_exc()
  180.  
  181.         return ConnectedWorker(w, conn, rfile)
  182.  
  183.     
  184.     def add_job(self, job):
  185.         job.done2 = self.notify_on_job_done
  186.         self.add_jobs_queue.put(job)
  187.  
  188.     
  189.     def run_job(self, job, gui = True, redirect_output = False):
  190.         w = self.launch_worker(gui = gui, redirect_output = redirect_output)
  191.         w.start_job(job)
  192.  
  193.     
  194.     def run(self):
  195.         while True:
  196.             
  197.             try:
  198.                 job = self.add_jobs_queue.get(True, 0.2)
  199.                 if job is None:
  200.                     break
  201.                 
  202.                 self.waiting_jobs.insert(0, job)
  203.             except Empty:
  204.                 pass
  205.  
  206.             for worker in self.workers:
  207.                 while True:
  208.                     
  209.                     try:
  210.                         n = worker.notifications.get_nowait()
  211.                         worker.job.notifications.put(n)
  212.                         self.changed_jobs_queue.put(job)
  213.                     continue
  214.                     except Empty:
  215.                         break
  216.                         continue
  217.                     
  218.  
  219.                     None<EXCEPTION MATCH>Empty
  220.             
  221.             for w in self.workers:
  222.                 if not w.is_alive:
  223.                     continue
  224.                 _[1][w]
  225.             
  226.             if len(self.pool) + len(self.workers) < self.pool_size:
  227.                 
  228.                 try:
  229.                     self.pool.append(self.launch_worker())
  230.                 except Exception:
  231.                     [ job for worker in _[1] ]
  232.                     [ job for worker in _[1] ]
  233.                     []
  234.                 except:
  235.                     [ job for worker in _[1] ]<EXCEPTION MATCH>Exception
  236.                 
  237.  
  238.             [ job for worker in _[1] ]
  239.             sj = self.suitable_waiting_job()
  240.             while True:
  241.                 
  242.                 try:
  243.                     j = self.kill_queue.get_nowait()
  244.                     self._kill_job(j)
  245.                 continue
  246.                 except Empty:
  247.                     None if sj is not None else []
  248.                     None if sj is not None else []
  249.                     break
  250.                     continue
  251.                 
  252.  
  253.                 None if sj is not None else []<EXCEPTION MATCH>Empty
  254.             continue
  255.             None if sj is not None else []
  256.  
  257.     
  258.     def suitable_waiting_job(self):
  259.         available_workers = len(self.pool)
  260.         for worker in self.workers:
  261.             job = worker.job
  262.             if job.core_usage == -1:
  263.                 available_workers = 0
  264.             elif job.core_usage > 1:
  265.                 available_workers -= job.core_usage - 1
  266.             
  267.             if available_workers < 1:
  268.                 return None
  269.         
  270.         for i, job in enumerate(self.waiting_jobs):
  271.             if job.core_usage == -1:
  272.                 if available_workers >= self.pool_size:
  273.                     return i
  274.                 continue
  275.             available_workers >= self.pool_size
  276.             if job.core_usage <= available_workers:
  277.                 return i
  278.         
  279.  
  280.     
  281.     def kill_job(self, job):
  282.         self.kill_queue.put(job)
  283.  
  284.     
  285.     def killall(self):
  286.         for worker in self.workers:
  287.             self.kill_queue.put(worker.job)
  288.         
  289.  
  290.     
  291.     def _kill_job(self, job):
  292.         if job.start_time is None:
  293.             job.kill_on_start = True
  294.             return None
  295.         for worker in self.workers:
  296.             if job is worker.job:
  297.                 worker.kill()
  298.                 job.killed = True
  299.                 break
  300.                 continue
  301.             job.start_time is None
  302.         
  303.  
  304.     
  305.     def split(self, tasks):
  306.         ans = []
  307.         count = 0
  308.         pos = 0
  309.         delta = int(ceil(len(tasks) / float(self.pool_size)))
  310.         while count < len(tasks):
  311.             section = []
  312.             for t in tasks[pos:pos + delta]:
  313.                 section.append((count, t))
  314.                 count += 1
  315.             
  316.             ans.append(section)
  317.             pos += delta
  318.         return ans
  319.  
  320.     
  321.     def close(self):
  322.         
  323.         try:
  324.             self.add_jobs_queue.put(None)
  325.             self.listener.close()
  326.         except:
  327.             pass
  328.  
  329.         time.sleep(0.2)
  330.         for worker in self.workers:
  331.             
  332.             try:
  333.                 worker.kill()
  334.             continue
  335.             continue
  336.  
  337.         
  338.         for worker in self.pool:
  339.             
  340.             try:
  341.                 worker.kill()
  342.             continue
  343.             continue
  344.  
  345.         
  346.  
  347.     
  348.     def __enter__(self):
  349.         return self
  350.  
  351.     
  352.     def __exit__(self, *args):
  353.         self.close()
  354.  
  355.  
  356.