home *** CD-ROM | disk | FTP | other *** search
/ OS/2 Shareware BBS: Java / Java.zip / jmach08.zip / dmachine.java < prev    next >
Text File  |  2000-05-20  |  15KB  |  472 lines
  1. import java.io.*;
  2. import java.net.*;
  3. import java.util.*;
  4.  
  5. public class dmachine
  6. {
  7.  
  8.  public static final String VERSION="0.8";
  9.  public static final String NAME="Download Machine";
  10.  public static final String COPYRIGHT="Copyright (c) 1999-2000 Radim Kolar. Open Source Software; There is NO warranty.\n"+
  11.  "See the GNU General Public Licence version 2 or later for copying conditions.";
  12.  
  13.  // public static final String TMPDIR="tmp";
  14.  
  15.  public static InetAddress http_proxyserver;
  16.  public static int         http_proxyport;
  17.  public static boolean     http_proxydefined;
  18.  public static int       http_resume;
  19.  
  20.  public static InetAddress  ftp_proxyserver;
  21.  public static int          ftp_proxyport;
  22.  public static boolean      ftp_proxydefined;
  23.  public static int        ftp_resume;
  24.  
  25.  public static String queue_files[];
  26.  public static long queue_files_lastmod[];
  27.  
  28.  public static String download_dir;
  29.  public static String tmp_dir;
  30.  
  31.  public static boolean case_sensitive;
  32.  
  33.  public static short threads,retry,uretry;  
  34.  public static int timeout;
  35.  public static long qchecktime,reptime;
  36.  
  37.  public static String auto_prefix,auto_suffix;
  38.  
  39.  public static String log_fatal;
  40.  public static String log_ok;
  41.  
  42.  // system data
  43.  public static Hashtable files; /* for fast search by name */
  44.  public static Vector queue;
  45.  public static int qhead;
  46.  
  47.  public static ThreadGroup runners;
  48.   
  49.  public static final void main(String argv[]) throws IOException
  50.  {
  51.   // print copyright
  52.   System.out.println(NAME+" "+VERSION+"\n"+COPYRIGHT+"\n");
  53.  
  54.   // sysinit
  55.   files=new Hashtable();
  56.   queue=new Vector();
  57.   qhead=-1;
  58.   queue_files=new String[0];
  59.   queue_files_lastmod=new long[0];
  60.   auto_prefix=auto_suffix="";
  61.   runners=new ThreadGroup(NAME+"-workers");
  62.  
  63.   // cfginit
  64.   defaultinit();
  65.   configure("dmachine.cnf");
  66.   watchQueue();
  67.  }
  68.  
  69.  private static final void watchQueue()
  70.  {
  71.   boolean empty=false;
  72.   while(true)
  73.   {
  74.    // check queue files
  75.    for(int i=queue_files.length-1;i>=0;i--)
  76.    {
  77.     File q;
  78.     q=new File(queue_files[i]);
  79.     
  80.     if( !q.isFile() || !q.canRead() ) { queue_files_lastmod[i]=0;continue;}
  81.     if(q.lastModified()!=queue_files_lastmod[i])
  82.     {
  83.       parseQueue(q);
  84.       queue_files_lastmod[i]=q.lastModified();
  85.     }
  86.    }
  87.       
  88.    if(!startThreads())
  89.      { 
  90.        if(!empty) { System.out.println("[QUEUE] Download Machine is idle.");empty=true;}
  91.      }
  92.     else empty=false;
  93.      
  94.    try
  95.    {
  96.      Thread.sleep(qchecktime);
  97.    }
  98.    catch(InterruptedException intr)
  99.     {
  100.      System.out.println("[YUP!] GOT INTR, QueueWatch ended.");
  101.      break;
  102.     }
  103.   }
  104.  
  105.  } /* watchQueue */
  106.  private static final void defaultinit()
  107.  {
  108.   /* init to defaults */
  109.   ftp_proxydefined=http_proxydefined=false;
  110.   addQueueFile("queue");
  111.   download_dir="files";
  112.   tmp_dir="files"+File.separatorChar+"tmp";
  113.   case_sensitive=true;
  114.   threads=2; retry=12;uretry=3;
  115.   timeout=60*3*1000;
  116.   qchecktime=20L*1000L;
  117.   reptime=45L*1000L;
  118.  }
  119.  
  120.  private static final void configure(String cfgfile) throws IOException
  121.  {
  122.   DataInputStream dis=new DataInputStream(new BufferedInputStream(
  123.                         new FileInputStream(cfgfile) ) );
  124.   int lineno=0;                        
  125.   String line,token;
  126.   StringTokenizer st;                        
  127.   while ( (line = dis.readLine()) != null)
  128.   {
  129.           lineno++;
  130.           if(line.startsWith("#")) continue;
  131.       st=new StringTokenizer(line);
  132.           if(st.hasMoreTokens()==false) continue;
  133.           token=st.nextToken();
  134.           token=token.toLowerCase();
  135.           try
  136.           {
  137.            if(token.equals("http_proxy")) 
  138.            try
  139.             {
  140.                   http_proxyserver=InetAddress.getByName(st.nextToken());
  141.                   http_proxyport=Integer.valueOf(st.nextToken()).intValue();
  142.                   http_proxydefined=true;
  143.         }
  144.         catch (UnknownHostException hnf) 
  145.         {
  146.           System.err.println("[CONFIG_ERROR] "+cfgfile+":"+lineno+" http_proxy "+hnf.getMessage()+":Host not found.");
  147.         }
  148.         finally
  149.          {  continue; }
  150.           else                                      
  151.           if(token.equals("ftp_proxy")) 
  152.         try
  153.         {
  154.              ftp_proxyserver=InetAddress.getByName(st.nextToken());
  155.              ftp_proxyport=Integer.valueOf(st.nextToken()).intValue();
  156.              ftp_proxydefined=true;
  157.          }
  158.          catch (java.net.UnknownHostException hnf) {
  159.           System.err.println("[CONFIG_ERROR] "+cfgfile+":"+lineno+" ftp_proxy "+hnf.getMessage()+":Host not found.");
  160.          }
  161.          finally
  162.           { continue; }
  163.       else
  164.           if(token.equals("ftp_proxy_resume")) 
  165.         {
  166.              ftp_resume=decoderesumekw(st.nextToken());
  167.         }
  168.       else
  169.           if(token.equals("http_proxy_resume")) 
  170.         {
  171.              http_resume=decoderesumekw(st.nextToken());
  172.         }
  173.           else                                      
  174.           if(token.equals("queue_file")) 
  175.                                       { 
  176.                         addQueueFile(st.nextToken());
  177.                                         continue;
  178.                                       }
  179.       else                  
  180.           if(token.equals("auto_prefix")) 
  181.                                       { 
  182.                         auto_prefix=st.nextToken();
  183.                                         continue;
  184.                       }
  185.           else                        
  186.           if(token.equals("log_fatal")
  187.       || token.equals("fatal_log")
  188.       ) 
  189.                                       { 
  190.                         log_fatal=st.nextToken();
  191.                                         continue;
  192.                                       }
  193.           else
  194.           if(token.equals("log_ok")) 
  195.                                       { 
  196.                         log_ok=st.nextToken();
  197.                                         continue;
  198.                                       }
  199.           else
  200.           if(token.equals("auto_suffix")) 
  201.                                       { 
  202.                         auto_suffix=st.nextToken();
  203.                                         continue;
  204.                                       }
  205.           else                                      
  206.           if(token.equals("download_directory")) 
  207.                                       {  
  208.                                          download_dir=st.nextToken();
  209.                                          if(download_dir.endsWith(File.separator)) download_dir=download_dir.substring(0,download_dir.length()-1);
  210.                                          new File(download_dir).mkdirs();
  211.                                          continue;
  212.                                        }
  213.       else                                       
  214.           if(token.equals("temporary_directory")) 
  215.                                       {  
  216.                                          tmp_dir=st.nextToken();
  217.                                          if(tmp_dir.endsWith(File.separator)) tmp_dir=tmp_dir.substring(0,tmp_dir.length()-1);
  218.                                          new File(tmp_dir).mkdirs();
  219.                                          continue;
  220.                                       }
  221.                                        
  222.           else                                      
  223.           if(token.equals("case_sensitive_filenames")) {
  224.                                              char c=(char)Integer.valueOf(st.nextToken()).intValue();
  225.                                                            if(c==1) case_sensitive=true; else case_sensitive=false;
  226.                                                            continue;
  227.                                                       }
  228.                                                       
  229.           else                                      
  230.           if(token.equals("download_threads")) {           // Java 1.0.X doesn't have Short class, using Integer
  231.                                              threads=(short)Integer.valueOf(st.nextToken()).intValue();
  232.                                                            continue;
  233.                                                       }
  234.  
  235.           else                                      
  236.           if(token.equals("file_retry_count")) {           // Java 1.0.X doesn't have Short class, using Integer
  237.                                              retry=(short)Integer.valueOf(st.nextToken()).intValue();
  238.                                                            continue;
  239.                                                       }
  240.  
  241.           else                                      
  242.           if(token.equals("url_retry_count")) {           // Java 1.0.X doesn't have Short class, using Integer
  243.                                              uretry=(short)Integer.valueOf(st.nextToken()).intValue();
  244.                                                            continue;
  245.                                                       }
  246.                                                       
  247.           else                                      
  248.           if(token.equals("download_timeout")) {           // Java 1.0.X doesn't have Short class, using Integer
  249.                                              timeout=1000*Integer.valueOf(st.nextToken()).intValue();
  250.                                                            continue;
  251.                                                       }
  252.  
  253.           else                                      
  254.           if(token.equals("report_time")) {                // Java 1.0.X doesn't have Short class, using Integer
  255.                                              reptime=1000*Integer.valueOf(st.nextToken()).intValue();
  256.                                                            continue;
  257.                                                       }
  258.                                                       
  259.           else                                      
  260.           if(token.equals("queue_check_time")) {           // Java 1.0.X doesn't have Short class, using Integer
  261.                                              qchecktime=1000L*Integer.valueOf(st.nextToken()).intValue();
  262.                                                            continue;
  263.                                                       }
  264.                                                                                                                                                                                                         
  265.           else                                      
  266.           System.err.println("[CONFIG_ERROR] "+cfgfile+":"+lineno+" Unknown keyword: "+token);
  267.           
  268.           }
  269.           catch (NoSuchElementException nse)
  270.            { System.err.println("[CONFIG_ERROR] "+cfgfile+":"+lineno+" Missing arguent(s).");
  271.              continue;}                                     
  272.   }                      
  273.   dis.close();                        
  274.  
  275.  
  276.  }
  277.  
  278.  private final static void parseQueue(File q)
  279.  {
  280.   try
  281.   {
  282.   DataInputStream dis=new DataInputStream(new BufferedInputStream(
  283.                         new FileInputStream(q) ) );
  284.   String line;
  285.   int lineno=0;             
  286.   URL url;          
  287.   System.out.println("[QUEUE] Loading queue file: "+q);
  288.   while ( (line = dis.readLine()) != null)
  289.   {
  290.           lineno++;
  291.           if(line.startsWith("#")) continue;
  292.           try
  293.           {
  294.        StringTokenizer st=new StringTokenizer(line);
  295.            line=st.nextToken();
  296.        if(line.indexOf("://")==-1)
  297.        {
  298.          line=line.toLowerCase();
  299.          if(line.equals("referer"))
  300.          {
  301.            String f1,f2;
  302.            f1=st.nextToken();
  303.            f2=st.nextToken();
  304.                downloadfactory.addReferer(f1,f2);
  305.            continue;
  306.          }else if(line.equals("end")) break;
  307.             System.out.println("[QUEUE] Bad command '"+line+"' at line "+lineno);
  308.         continue;
  309.        }
  310.  
  311.           }
  312.           catch(NoSuchElementException kurva) {continue;}
  313.           try
  314.           {
  315.             String fn;
  316.             qfile qf;
  317.         fn=null;
  318.         if(line.indexOf('#')>-1)
  319.          {
  320.           int z=line.indexOf('#');
  321.           fn=line.substring(z+1);
  322.           line=line.substring(0,z);
  323.          }
  324.             url=new URL(line);
  325.         if(fn==null) fn=getFilename(url.getFile());
  326.             if(fn!=null && fn.length()!=0)
  327.              {
  328.            if(!case_sensitive) fn=fn.toLowerCase();
  329.              } else fn=line;
  330.              
  331.             qf=(qfile)files.get(fn);
  332.             if(qf==null)
  333.              {
  334.               System.out.println("[QUEUE] New file "+fn+" added. url="+line);
  335.               qf=new qfile(fn,line);
  336.               files.put(fn,qf);
  337.               queue.addElement(qf);
  338.              } else
  339.               qf.addURL(line);
  340.               
  341.             // System.out.println("URL="+url+" filename="+fn);
  342.           }
  343.           catch (MalformedURLException grr)
  344.            {
  345.             System.out.println("[QUEUE] Bad URL="+line+" at line "+lineno);continue;
  346.            }
  347.   }  
  348.   dis.close();  
  349.   }
  350.   catch (IOException grrr) {}
  351.   // startThreads();
  352.  } /* parse_queue */
  353.  
  354.  /* vraci true pokud je DM aktivni */
  355. private final static synchronized boolean startThreads()
  356. {
  357.  int ac=runners.activeCount();
  358.  boolean active=false;
  359.  if(ac>0) active=true;
  360.  if(ac>=threads) return true;
  361.  
  362.  boolean rotated=false;
  363.  int qs=queue.size();
  364.  if(qs==0) return active; // empty queue
  365.  while(true)
  366.  {
  367.   qhead++;
  368.   if(qhead>=qs)
  369.   {
  370.    if(rotated) return active;
  371.      else
  372.    {qhead=0;rotated=true;}
  373.   }
  374.    
  375.   qfile qf;
  376.   qf=(qfile)queue.elementAt(qhead);
  377.   if(!qf.needsDownload()) continue;
  378.   // start it up!
  379.   active=true;
  380.   Thread t;
  381.   t=new Thread(runners,qf);
  382.   // qf.downloader=t;
  383.   t.start();
  384.   Thread.yield();
  385.   ac++;
  386.   if(ac>=threads) return true;
  387.  }
  388. } 
  389.  
  390. public final static String getFilename(String fileurl)
  391. {
  392.    /* new code (from smart cache) */
  393.    
  394.    byte v[];
  395.    int j=fileurl.length();
  396.    
  397.    v=new byte[j];
  398.    fileurl.getBytes(0,j,v,0);
  399.    loop1:for(int zz=0;zz<j;zz++)
  400.     {
  401.       switch(v[zz])
  402.       {
  403.         case 0x3b: // ;
  404.         case 0x3a: // :
  405.         case 0x3d: // =
  406.         case 0x3f: // ?
  407.     case 0x7c: // |
  408.          //  case 0x23: // # - je jiz odstranen
  409.      
  410.            return null; // je to dotaz!
  411.       }
  412.     }
  413.       
  414.     /* novy kod - konec! */
  415.    if(fileurl.length()<2) return "";
  416.    j=fileurl.lastIndexOf('/',j);
  417.    int i=fileurl.indexOf('~');
  418.    if (i!=-1 && j<i) return "";
  419.     else 
  420.      return fileurl.substring(j+1); // soubor
  421. }
  422.  
  423. public synchronized static void log_fatal(String msg)
  424. {
  425.  System.out.println("[FATAL_ERR] "+msg);
  426.  if(log_fatal==null) return;
  427.  try
  428.  {
  429.   DataOutputStream dos=new DataOutputStream(new BufferedOutputStream(new FileOutputStream(log_fatal,true)));
  430.   dos.writeBytes(new Date().toString());
  431.   dos.writeBytes(" ");
  432.   dos.writeBytes(msg);
  433.   dos.writeBytes("\n");
  434.   dos.close();
  435. }
  436. catch (IOException i)
  437.  {
  438.   log_fatal=null;
  439.  }
  440. }
  441.  
  442. private final static void addQueueFile(String name)
  443. {
  444.  if(name==null || name.length()==0) return;
  445.  for(int i=queue_files.length-1;i>=0;i--)
  446.   if(queue_files[i].equals(name)) return;
  447.   
  448.  queue_files=util.addStringToArray(name,queue_files);
  449.  queue_files_lastmod=util.incLongArraySize(queue_files_lastmod);
  450. }
  451.  
  452. private final static int decoderesumekw(String kw)
  453. {
  454.  int res=downloadfactory.RESUME_UNKNOWN;
  455.  kw=kw.toLowerCase();
  456.  if(kw.equals("none")|| kw.equals("normal")|| kw.equals("standard"))
  457.   res=downloadfactory.RESUME_NONE;
  458.  else 
  459.  if(kw.equals("pragma")|| kw.equals("nocache"))
  460.   res=downloadfactory.RESUME_NOCACHE;
  461.  else
  462.  if(kw.equals("direct"))
  463.   res=downloadfactory.RESUME_DIRECT;
  464.  else
  465.    System.err.println("[CONFIG_ERROR] Unknown resume option "+kw);
  466.    
  467.  return res;
  468.  
  469. }
  470.  
  471. }
  472.