PC Online 1998 September/PCO_0998.ISO/browser/ns405lyc/netcast.z/ncjava10.jar/netscape/netcast/application/HTMLCrawler.class

Java Class File | 1998-03-09 | 15.2 KB | 1,005 lines

package netscape.netcast.application;

import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.PrintStream;
import java.io.RandomAccessFile;
import java.net.URL;
import java.util.Date;
import java.util.Enumeration;
import java.util.Hashtable;
import java.util.Vector;
import netscape.net.Cache;
import netscape.net.CacheRequest;
import netscape.security.PrivilegeManager;
import netscape.security.Target;

public class HTMLCrawler extends Thread {
   private Hashtable pagesParsed = new Hashtable();
   private Hashtable imagesCached = new Hashtable();
   private Hashtable resourcesCached = new Hashtable();
   private Hashtable linkedPagesList = new Hashtable();
   private Hashtable linkedImagesList = new Hashtable();
   private Hashtable linkedResourcesList = new Hashtable();
   private Hashtable numLinksAtDepth = new Hashtable();
   private Hashtable numImagesAtDepth = new Hashtable();
   private Hashtable numResourcesAtDepth = new Hashtable();
   private Hashtable robotExclusionList = new Hashtable();
   private URL site;
   private String siteName;
   private String localName;
   private int crawlerDepth;
   private String localFileName;
   private Cache cache;
   private UpdateParent parent;
   private boolean ignoreMaxUpdate;
   private int numLinks;
   private int numImages;
   private int numResources;
   private int totalContentLength;
   private int maxSize;
   private int dataUpdateStatus;
   boolean inSiteSpecified = false;
   boolean isStopUpdate = false;
   boolean isCacheFull = false;
   boolean isURLRemoved = false;
   boolean isDepthValid = true;
   boolean isSiteAllowed = true;
   boolean isSiteValid = true;
   private PrintStream psOut;
   private RandomAccessFile sourceStream;
   private RandomAccessFile rafOut;
   PrivilegeManager privilegeManager;
   Target target;
   // $FF: renamed from: t netscape.security.Target
   Target field_0;
   // $FF: renamed from: tr netscape.security.Target
   Target field_1;
   // $FF: renamed from: tw netscape.security.Target
   Target field_2;
   static final int BUF_SIZE = 1024;
   static byte[] junk = new byte[1024];
   private static final int NEW_LINK = 1;
   private static final int REPLACED_LINK = 2;
   private static final int OLD_LINK = 3;
   private static final int NEW_DATA = 1;
   private static final int OLD_DATA = 0;
   private static final int CACHE_FULL = -1;
   private static final int INVALID_DEPTH = -2;
   private static final int SITE_NOT_ALLOWED = -3;
   private static final int INVALID_URL = -4;

   public HTMLCrawler(String var1, Cache var2, UpdateParent var3, int var4, String var5, int var6, boolean var7, boolean var8) {
      this.siteName = var1;
      this.cache = var2;
      this.crawlerDepth = var4;
      this.parent = var3;
      this.inSiteSpecified = var7;
      this.localFileName = var5.substring(0, var5.length() - 3);
      this.maxSize = var6;
      this.ignoreMaxUpdate = var8;
   }

   public void HTMLCrawlerSite(String var1) {
      this.siteName = var1;
   }

   public void HTMLCrawlerCache(Cache var1) {
      this.cache = var1;
   }

   public void HTMLCrawlerDepth(int var1) {
      this.crawlerDepth = var1;
   }

   public void HTMLCrawlerParent(UpdateParent var1) {
      this.parent = var1;
   }

   public void HTMLCrawlerInSite(boolean var1) {
      this.inSiteSpecified = var1;
   }

   public void HTMLCrawlerLocalFileName(String var1) {
      this.localFileName = var1.substring(0, var1.length() - 3);
   }

   public void HTMLCrawlerMaxSize(int var1) {
      this.maxSize = var1;
   }

   public void setTotalContentLength(int var1) {
      this.totalContentLength = var1;
   }

   public int getTotalContentLength() {
      return this.totalContentLength;
   }

   public void run() {
      try {
         while(!SiteData.canICrawl(this.ignoreMaxUpdate || this.isStopUpdate)) {
            Thread.sleep(30000L);
         }

         this.parent.updateStarted();
         PrivilegeManager.enablePrivilege("Netcaster");
         this.site = new URL(this.siteName);
         this.localName = this.site.getFile();
         if (this.crawlerDepth < 1) {
            this.isDepthValid = false;
         } else {
            this.setTotalContentLength(0);
            if (this.getTotalContentLength() <= this.maxSize) {
               this.startCrawler(this.localName);
            }
         }
      } catch (Exception var6) {
         printException(var6, "HTMLCrawler.run");
      } finally {
         if (!this.isStopUpdate && (!this.pagesParsed.isEmpty() || !this.pagesParsed.isEmpty() || !this.pagesParsed.isEmpty())) {
            this.removeItems(this.cache);
         }

         if (this.isStopUpdate && this.isCacheFull && (!this.pagesParsed.isEmpty() || !this.pagesParsed.isEmpty() || !this.pagesParsed.isEmpty())) {
            this.removeItems(this.cache);
         }

         this.updateStatus();
         if (!this.pagesParsed.isEmpty() || !this.pagesParsed.isEmpty() || !this.pagesParsed.isEmpty()) {
            this.writeCachedItemListToFile();
         }

         this.clearTables();
         this.parent.updateComplete(this.dataUpdateStatus);
      }
   }

   private void startCrawler(String var1) {
      try {
         for(int var2 = 1; var2 <= this.crawlerDepth; ++var2) {
            if (var2 == 1) {
               URL var8 = new URL(this.site, var1);
               String var9 = var8.toString();
               this.processLink(var9, var2, this.numLinksAtDepth, this.numImagesAtDepth, this.numResourcesAtDepth, this.resourcesCached, this.robotExclusionList);
               this.addLinksToDepthTable(this.numLinksAtDepth, var2);
               this.addImagesToDepthTable(this.numImagesAtDepth, var2);
               this.addResourcesToDepthTable(this.numResourcesAtDepth, var2);
               this.numLinks = 0;
               this.numImages = 0;
               this.numResources = 0;
               this.numLinksAtDepth.clear();
               this.numImagesAtDepth.clear();
               this.numResourcesAtDepth.clear();
            } else {
               String var3 = this.composeLevelKey(var2 - 1);
               Hashtable var4 = (Hashtable)this.linkedPagesList.get(var3);
               if (var4 != null) {
                  Enumeration var5 = var4.elements();

                  while(var5.hasMoreElements()) {
                     String var6 = (String)var5.nextElement();
                     this.processLink(var6, var2, this.numLinksAtDepth, this.numImagesAtDepth, this.numResourcesAtDepth, this.resourcesCached, this.robotExclusionList);
                  }

                  this.addLinksToDepthTable(this.numLinksAtDepth, var2);
                  this.addImagesToDepthTable(this.numImagesAtDepth, var2);
                  this.addResourcesToDepthTable(this.numResourcesAtDepth, var2);
                  this.numLinks = 0;
                  this.numImages = 0;
                  this.numResources = 0;
                  this.numLinksAtDepth.clear();
                  this.numImagesAtDepth.clear();
                  this.numResourcesAtDepth.clear();
               }
            }
         }

         if (this.maxSize - this.getTotalContentLength() <= 2000) {
            this.isCacheFull = true;
            this.stopUpdate();
         }

         this.cacheImages(this.cache);
         if (this.maxSize - this.getTotalContentLength() <= 2000) {
            this.isCacheFull = true;
            this.stopUpdate();
         }

         this.cacheResources(this.cache);
      } catch (Exception var7) {
         printException(var7, "StartCrawler");
      }
   }

   private void processLink(String var1, int var2, Hashtable var3, Hashtable var4, Hashtable var5, Hashtable var6, Hashtable var7) {
      try {
         URL var8 = new URL(var1);
         if (this.maxSize - this.getTotalContentLength() <= 2000) {
            this.isCacheFull = true;
            this.stopUpdate();
         }

         Object var9 = this.pagesParsed.get(var1);
         if (var9 != null) {
            return;
         }

         if (this.inSiteSpecified && !this.inSite(var8)) {
            return;
         }

         String var10 = this.getSite(var8);
         Object var11 = var7.get(var10);
         if (var11 == null) {
            RobotExclusion var12 = new RobotExclusion(var10);
            var7.put(var10, var12);
         }

         RobotExclusion var17 = (RobotExclusion)var7.get(var10);
         if (var17 != null) {
            boolean var13 = var17.isRobotAllowed(var8, this.cache);
            if (!var13) {
               if (var2 == 1) {
                  this.isSiteAllowed = false;
               }

               return;
            }
         }

         PageScan var18 = new PageScan(this.site, var8, this.inSiteSpecified);
         var18.fetchInfo(this.cache, var6, this.maxSize, this);
         if (!this.isSiteValid && var2 == 1) {
            return;
         }

         if (!this.isURLRemoved) {
            if (this.isSiteValid) {
               this.pagesParsed.put(var1, var18);
            }
         } else {
            this.isURLRemoved = false;
         }

         this.isSiteValid = true;
         if (var18.getLinks() != null) {
            ++this.numLinks;
            String var14 = this.composePositionKey(this.numLinks);
            var3.put(var14, var18.getLinks());
         }

         if (var18.getImages() != null) {
            ++this.numImages;
            String var19 = this.composePositionKey(this.numImages);
            var4.put(var19, var18.getImages());
         }

         if (var18.getResources() != null) {
            ++this.numResources;
            String var20 = this.composePositionKey(this.numResources);
            var5.put(var20, var18.getResources());
         }

         if (var18.getFrames() != null) {
            Enumeration var21 = var18.getFrames().elements();

            while(var21.hasMoreElements()) {
               String var15 = (String)var21.nextElement();
               this.processLink(var15, var2, var3, var4, var5, var6, var7);
            }
         }

         if (var18.getLayers() != null) {
            Enumeration var22 = var18.getLayers().elements();

            while(var22.hasMoreElements()) {
               String var23 = (String)var22.nextElement();
               this.processLink(var23, var2, var3, var4, var5, var6, var7);
            }

            return;
         }
      } catch (Exception var16) {
         printException(var16, "processLink");
      }

   }

   public void setStopUpdate() {
      this.isStopUpdate = true;
      ((SiteData)this.parent).setStatus(0);
   }

   public void stopUpdate() {
      this.setStopUpdate();
      ((Thread)this).stop();
   }

   private String getSite(URL var1) {
      String var2 = var1.toString();
      String var3 = var1.getHost();
      int var4 = var1.getPort();
      String var5 = "";
      int var6 = 0;
      if (var4 != -1) {
         Integer var8 = new Integer(var4);
         String var9 = var8.toString();
         int var12 = var9.length();
         if (var12 > 0) {
            var6 = var2.indexOf(var9);
            var5 = var2.substring(0, var6 + var12);
         }

         return var5;
      } else {
         int var7 = var3.length();
         if (var7 > 0) {
            var6 = var2.indexOf(var3);
            var5 = var2.substring(0, var6 + var7);
         }

         return var5;
      }
   }

   private boolean inSite(URL var1) {
      String var2 = this.site.getHost();
      String var3 = var1.getHost();
      String var4 = "";
      String var5 = "";
      int var6 = var3.toLowerCase().indexOf(var2.toLowerCase());
      if (var6 >= 0) {
         return true;
      } else {
         var6 = var2.toLowerCase().indexOf(var3.toLowerCase());
         if (var6 >= 0) {
            return true;
         } else {
            int var7 = var2.indexOf(46);
            int var8 = var3.indexOf(46);
            if (var7 >= 0) {
               var4 = var2.substring(var7);
               if (var8 >= 0) {
                  var5 = var3.substring(var8);
                  return var4.equalsIgnoreCase(var5);
               } else {
                  return false;
               }
            } else {
               return false;
            }
         }
      }
   }

   private String composeLevelKey(int var1) {
      String var2 = "level" + var1;
      return var2;
   }

   private String composePositionKey(int var1) {
      String var2 = "pos" + var1;
      return var2;
   }

   private void addLinksToDepthTable(Hashtable var1, int var2) {
      try {
         Hashtable var3 = new Hashtable();
         Enumeration var4 = var1.elements();

         while(var4.hasMoreElements()) {
            Vector var5 = (Vector)var4.nextElement();
            Enumeration var6 = var5.elements();

            while(var6.hasMoreElements()) {
               String var7 = (String)var6.nextElement();
               boolean var8 = var3.containsKey(var7);
               if (!var8) {
                  var3.put(var7, var7);
               }
            }
         }

         String var11 = this.composeLevelKey(var2);
         this.linkedPagesList.put(var11, var3);
         Object var10 = null;
      } catch (Exception var9) {
         printException(var9, "addLinksToDepthTable");
      }
   }

   private void addImagesToDepthTable(Hashtable var1, int var2) {
      try {
         Hashtable var3 = new Hashtable();
         Enumeration var4 = var1.elements();

         while(var4.hasMoreElements()) {
            Vector var5 = (Vector)var4.nextElement();
            Enumeration var6 = var5.elements();

            while(var6.hasMoreElements()) {
               String var7 = (String)var6.nextElement();
               boolean var8 = var3.containsKey(var7);
               if (!var8) {
                  var3.put(var7, var7);
               }
            }
         }

         String var11 = this.composeLevelKey(var2);
         this.linkedImagesList.put(var11, var3);
         Object var10 = null;
      } catch (Exception var9) {
         printException(var9, "addImagesToDepthTable");
      }
   }

   private void addResourcesToDepthTable(Hashtable var1, int var2) {
      try {
         Hashtable var3 = new Hashtable();
         Enumeration var4 = var1.elements();

         while(var4.hasMoreElements()) {
            Vector var5 = (Vector)var4.nextElement();
            Enumeration var6 = var5.elements();

            while(var6.hasMoreElements()) {
               String var7 = (String)var6.nextElement();
               boolean var8 = var3.containsKey(var7);
               if (!var8) {
                  var3.put(var7, var7);
               }
            }
         }

         String var11 = this.composeLevelKey(var2);
         this.linkedResourcesList.put(var11, var3);
         Object var10 = null;
      } catch (Exception var9) {
         printException(var9, "addResourcesToDepthTable");
      }
   }

   private void cacheImages(Cache var1) {
      try {
         label59:
         for(int var2 = 1; var2 <= this.crawlerDepth; ++var2) {
            String var3 = this.composeLevelKey(var2);
            Hashtable var4 = (Hashtable)this.linkedImagesList.get(var3);
            if (var4 != null) {
               Enumeration var5 = var4.elements();

               while(true) {
                  String var6;
                  URL var7;
                  String var8;
                  byte var9;
                  while(true) {
                     while(true) {
                        if (!var5.hasMoreElements()) {
                           continue label59;
                        }

                        var6 = (String)var5.nextElement();
                        var7 = new URL(var6);
                        var8 = null;
                        var9 = 1;
                        Object var10 = this.imagesCached.get(var6);
                        if (var10 == null) {
                           String var11 = this.getSite(var7);
                           Object var12 = this.robotExclusionList.get(var11);
                           if (var12 == null) {
                              RobotExclusion var13 = new RobotExclusion(var11);
                              this.robotExclusionList.put(var11, var13);
                           }

                           RobotExclusion var20 = (RobotExclusion)this.robotExclusionList.get(var11);
                           if (var20 == null) {
                              break;
                           }

                           boolean var14 = var20.isRobotAllowed(var7, var1);
                           if (var14) {
                              break;
                           }
                        }
                     }

                     Object var21 = null;

                     try {
                        CacheRequest var15 = new CacheRequest(var6);
                        InputStream var22 = var1.getInputStream(var15);
                        var8 = var15.getLastModified();
                        int var16 = Integer.parseInt(var15.getContentLength());
                        int var17 = this.getTotalContentLength();
                        this.setTotalContentLength(var17 + var16);
                        var17 = this.getTotalContentLength();
                        if (var17 <= this.maxSize) {
                           this.readAllInputStream(var22);
                           this.closeInputStream(var22);
                           break;
                        }

                        this.setTotalContentLength(var17 - var16);
                        this.closeInputStream(var22);
                     } catch (Exception var18) {
                        printException(var18, "cacheImages.cacheRequest");
                        break;
                     }
                  }

                  LinkObject var23 = new LinkObject(var7, var8, var9);
                  this.imagesCached.put(var6, var23);
                  if (this.maxSize - this.getTotalContentLength() <= 2000) {
                     this.isCacheFull = true;
                     this.stopUpdate();
                  }
               }
            }
         }

      } catch (Exception var19) {
         printException(var19, "cacheImages");
      }
   }

   private void cacheResources(Cache var1) {
      try {
         label59:
         for(int var2 = 1; var2 <= this.crawlerDepth; ++var2) {
            String var3 = this.composeLevelKey(var2);
            Hashtable var4 = (Hashtable)this.linkedResourcesList.get(var3);
            if (var4 != null) {
               Enumeration var5 = var4.elements();

               while(true) {
                  String var6;
                  URL var7;
                  String var8;
                  byte var9;
                  while(true) {
                     while(true) {
                        if (!var5.hasMoreElements()) {
                           continue label59;
                        }

                        var6 = (String)var5.nextElement();
                        var7 = new URL(var6);
                        var8 = null;
                        var9 = 1;
                        Object var10 = this.resourcesCached.get(var6);
                        if (var10 == null) {
                           String var11 = this.getSite(var7);
                           Object var12 = this.robotExclusionList.get(var11);
                           if (var12 == null) {
                              RobotExclusion var13 = new RobotExclusion(var11);
                              this.robotExclusionList.put(var11, var13);
                           }

                           RobotExclusion var20 = (RobotExclusion)this.robotExclusionList.get(var11);
                           if (var20 == null) {
                              break;
                           }

                           boolean var14 = var20.isRobotAllowed(var7, var1);
                           if (var14) {
                              break;
                           }
                        }
                     }

                     Object var21 = null;

                     try {
                        CacheRequest var15 = new CacheRequest(var6);
                        InputStream var22 = var1.getInputStream(var15);
                        var8 = var15.getLastModified();
                        int var16 = Integer.parseInt(var15.getContentLength());
                        int var17 = this.getTotalContentLength();
                        this.setTotalContentLength(var17 + var16);
                        var17 = this.getTotalContentLength();
                        if (var17 <= this.maxSize) {
                           this.readAllInputStream(var22);
                           this.closeInputStream(var22);
                           break;
                        }

                        this.setTotalContentLength(var17 - var16);
                        this.closeInputStream(var22);
                     } catch (Exception var18) {
                        printException(var18, "cacheResources.cacheRequest");
                        break;
                     }
                  }

                  LinkObject var23 = new LinkObject(var7, var8, var9);
                  this.resourcesCached.put(var6, var23);
                  if (this.maxSize - this.getTotalContentLength() <= 2000) {
                     this.isCacheFull = true;
                     this.stopUpdate();
                  }
               }
            }
         }

      } catch (Exception var19) {
         printException(var19, "cacheResources");
      }
   }

   public void closeInputStream(InputStream var1) {
      try {
         if (var1 != null) {
            var1.close();
            return;
         }
      } catch (IOException var2) {
      }

   }

   public void readAllInputStream(InputStream var1) {
      try {
         while(var1.read(junk) >= 0) {
         }

      } catch (IOException var2) {
      }
   }

   private void print(String var1) {
      try {
         if (this.isStopUpdate && !this.isCacheFull) {
            this.rafOut.writeBytes(var1 + "\n");
         } else {
            this.psOut.println(var1);
         }
      } catch (IOException var2) {
      }
   }

   public static void printException(Exception var0, String var1) {
   }

   private void removeItems(Cache var1) {
      try {
         File var2 = new File(this.localFileName + ".dat");
         if (var2.exists() && var2.canRead()) {
            long var3 = 0L;
            boolean var5 = false;
            String var7 = "";
            int var9 = -1;
            this.sourceStream = new RandomAccessFile(var2, "r");
            this.sourceStream.seek(var3);

            String var6;
            while((var6 = this.sourceStream.readLine()) != null) {
               if (var6.startsWith("L>")) {
                  var9 = var6.lastIndexOf(47);
                  if (var9 + 1 >= 1) {
                     String var8 = var6.substring(var9 + 1);
                     var7 = var6.substring(2, var9);
                     var5 = this.searchPagesList(var7, var8);
                  }
               } else if (var6.startsWith("I>")) {
                  var9 = var6.lastIndexOf(47);
                  if (var9 + 1 >= 1) {
                     String var11 = var6.substring(var9 + 1);
                     var7 = var6.substring(2, var9);
                     var5 = this.searchImagesList(var7, var11);
                  }
               } else if (var6.startsWith("R>")) {
                  var9 = var6.lastIndexOf(47);
                  if (var9 + 1 >= 1) {
                     String var12 = var6.substring(var9 + 1);
                     var7 = var6.substring(2, var9);
                     var5 = this.searchResourcesList(var7, var12);
                  }
               }

               if (!var5) {
                  var1.remove(var7);
               }
            }

            if (this.sourceStream != null) {
               this.sourceStream.close();
               return;
            }
         }
      } catch (Exception var10) {
         printException(var10, "removeItems");
      }

   }

   private boolean searchPagesList(String var1, String var2) {
      try {
         boolean var3 = false;
         if (this.pagesParsed != null) {
            Object var4 = this.pagesParsed.get(var1);
            if (var4 != null) {
               String var5 = ((PageScan)var4).getLastModified();
               if (!var2.equals("null") && var5 != null) {
                  Date var6 = new Date(var2);
                  Date var7 = new Date(var5);
                  var3 = var7.after(var6);
               } else {
                  var3 = true;
               }

               if (var3) {
                  ((PageScan)var4).setLinkStatus(2);
               } else {
                  ((PageScan)var4).setLinkStatus(3);
               }

               return true;
            }
         }

         return false;
      } catch (Exception var8) {
         printException(var8, "searchPagesList");
         return false;
      }
   }

   private boolean searchImagesList(String var1, String var2) {
      try {
         boolean var3 = false;
         if (this.imagesCached != null) {
            Object var4 = this.imagesCached.get(var1);
            if (var4 != null) {
               String var5 = ((LinkObject)var4).getLastModified();
               if (!var2.equals("null") && var5 != null) {
                  Date var6 = new Date(var2);
                  Date var7 = new Date(var5);
                  var3 = var7.after(var6);
               } else {
                  var3 = true;
               }

               if (var3) {
                  ((LinkObject)var4).setLinkStatus(2);
               } else {
                  ((LinkObject)var4).setLinkStatus(3);
               }

               return true;
            }
         }

         return false;
      } catch (Exception var8) {
         printException(var8, "searchImagesList");
         return false;
      }
   }

   private boolean searchResourcesList(String var1, String var2) {
      try {
         boolean var3 = false;
         if (this.resourcesCached != null) {
            Object var4 = this.resourcesCached.get(var1);
            if (var4 != null) {
               String var5 = ((LinkObject)var4).getLastModified();
               if (!var2.equals("null") && var5 != null) {
                  Date var6 = new Date(var2);
                  Date var7 = new Date(var5);
                  var3 = var7.after(var6);
               } else {
                  var3 = true;
               }

               if (var3) {
                  ((LinkObject)var4).setLinkStatus(2);
               } else {
                  ((LinkObject)var4).setLinkStatus(3);
               }

               return true;
            }
         }

         return false;
      } catch (Exception var8) {
         printException(var8, "searchResourcesList");
         return false;
      }
   }

   private void updateStatus() {
      try {
         if (this.isCacheFull) {
            this.dataUpdateStatus = -1;
            return;
         }

         if (!this.isDepthValid) {
            this.dataUpdateStatus = -2;
            return;
         }

         if (!this.isSiteAllowed) {
            this.dataUpdateStatus = -3;
            return;
         }

         if (!this.isSiteValid) {
            this.dataUpdateStatus = -4;
            return;
         }

         if (this.pagesParsed != null) {
            Enumeration var1 = this.pagesParsed.keys();

            while(var1.hasMoreElements()) {
               String var2 = (String)var1.nextElement();
               PageScan var3 = (PageScan)this.pagesParsed.get(var2);
               int var4 = var3.getLinkStatus();
               if (var4 == 1 || var4 == 2) {
                  this.dataUpdateStatus = 1;
                  return;
               }
            }
         }

         if (this.imagesCached != null) {
            Enumeration var6 = this.imagesCached.keys();

            while(var6.hasMoreElements()) {
               String var8 = (String)var6.nextElement();
               LinkObject var10 = (LinkObject)this.imagesCached.get(var8);
               int var12 = var10.getLinkStatus();
               if (var12 == 1 || var12 == 2) {
                  this.dataUpdateStatus = 1;
                  return;
               }
            }
         }

         if (this.resourcesCached != null) {
            Enumeration var7 = this.resourcesCached.keys();

            while(var7.hasMoreElements()) {
               String var9 = (String)var7.nextElement();
               LinkObject var11 = (LinkObject)this.resourcesCached.get(var9);
               int var13 = var11.getLinkStatus();
               if (var13 == 1 || var13 == 2) {
                  this.dataUpdateStatus = 1;
                  return;
               }
            }

            return;
         }
      } catch (Exception var5) {
         printException(var5, "updateStatus");
      }

   }

   private void writeCachedItemListToFile() {
      try {
         File var3 = new File(this.localFileName + ".dat");
         long var4 = 0L;
         if (this.isStopUpdate && !this.isCacheFull) {
            this.rafOut = new RandomAccessFile(var3, "rw");
            var4 = this.rafOut.length();
            this.rafOut.seek(var4);
         } else {
            this.psOut = new PrintStream(new FileOutputStream(var3));
         }

         this.writeLinks();
         this.writeImages();
         this.writeResources();
      } catch (Exception var11) {
         printException(var11, "writeCachedItemListToFile");
      } finally {
         try {
            if (this.isStopUpdate && !this.isCacheFull) {
               if (this.rafOut != null) {
                  this.rafOut.close();
               }
            } else if (this.psOut != null) {
               this.psOut.close();
            }
         } catch (IOException var10) {
         }

      }

   }

   private void writeLinks() {
      try {
         if (this.pagesParsed != null) {
            Enumeration var1 = this.pagesParsed.keys();

            while(var1.hasMoreElements()) {
               String var2 = (String)var1.nextElement();
               PageScan var3 = (PageScan)this.pagesParsed.get(var2);
               String var4 = var3.getLastModified();
               this.print("L>" + var2 + "/" + var4);
            }

            return;
         }
      } catch (Exception var5) {
         printException(var5, "writeLinks");
      }

   }

   private void writeImages() {
      try {
         if (this.imagesCached != null) {
            Enumeration var1 = this.imagesCached.keys();

            while(var1.hasMoreElements()) {
               String var2 = (String)var1.nextElement();
               LinkObject var3 = (LinkObject)this.imagesCached.get(var2);
               String var4 = var3.getLastModified();
               this.print("I>" + var2 + "/" + var4);
            }

            return;
         }
      } catch (Exception var5) {
         printException(var5, "writeImages");
      }

   }

   private void writeResources() {
      try {
         if (this.resourcesCached != null) {
            Enumeration var1 = this.resourcesCached.keys();

            while(var1.hasMoreElements()) {
               String var2 = (String)var1.nextElement();
               LinkObject var3 = (LinkObject)this.resourcesCached.get(var2);
               String var4 = var3.getLastModified();
               this.print("R>" + var2 + "/" + var4);
            }

            return;
         }
      } catch (Exception var5) {
         printException(var5, "writeResources");
      }

   }

   private void clearTables() {
      try {
         this.pagesParsed.clear();
         this.pagesParsed = null;
         this.imagesCached.clear();
         this.imagesCached = null;
         this.resourcesCached.clear();
         this.resourcesCached = null;
         this.linkedPagesList.clear();
         this.linkedPagesList = null;
         this.linkedImagesList.clear();
         this.linkedImagesList = null;
         this.linkedResourcesList.clear();
         this.linkedResourcesList = null;
         this.numLinksAtDepth.clear();
         this.numLinksAtDepth = null;
         this.numImagesAtDepth.clear();
         this.numImagesAtDepth = null;
         this.numResourcesAtDepth.clear();
         this.numResourcesAtDepth = null;
      } catch (Exception var2) {
         printException(var2, "clearTables");
      }
   }
}