home *** CD-ROM | disk | FTP | other *** search
/ Popular Software (Premium Edition) / mycd.iso / INTERNET / NETSCAP4.06 / CP32E406.EXE / netcast.z / ncjava10.jar / netscape / netcast / application / PageScan.class (.txt) < prev    next >
Encoding:
Java Class File  |  1998-02-27  |  12.4 KB  |  832 lines
  1. package netscape.netcast.application;
  2.  
  3. import java.io.InputStream;
  4. import java.net.URL;
  5. import java.util.Hashtable;
  6. import java.util.Vector;
  7. import netscape.net.Cache;
  8. import netscape.net.CacheRequest;
  9. import netscape.palomar.htmllite.LiteParser;
  10. import netscape.palomar.sgml.SGMLAttribute;
  11. import netscape.palomar.sgml.SGMLElement;
  12. import netscape.palomar.sgml.SGMLTag;
  13. import netscape.palomar.sgml.SGMLTagImpl;
  14. import netscape.palomar.util.TreeEnumeration;
  15.  
  16. class PageScan {
  17.    private URL url;
  18.    private URL site;
  19.    private URL baseURL;
  20.    boolean baseURLExists;
  21.    boolean inSiteSpecified;
  22.    private Vector images = new Vector();
  23.    private Vector links = new Vector();
  24.    private Vector resources = new Vector();
  25.    private Vector frames = new Vector();
  26.    private Vector layers = new Vector();
  27.    private String lastModified;
  28.    private int linkStatus;
  29.    private static final int HREF = 1;
  30.    private static final int FRAME = 2;
  31.    private static final int LAYER = 3;
  32.  
  33.    public PageScan(URL var1, URL var2, boolean var3) {
  34.       this.url = var2;
  35.       this.site = var1;
  36.       this.baseURL = var2;
  37.       this.baseURLExists = false;
  38.       this.inSiteSpecified = var3;
  39.       this.lastModified = null;
  40.       this.linkStatus = 1;
  41.    }
  42.  
  43.    public Vector getLinks() {
  44.       return this.links;
  45.    }
  46.  
  47.    public Vector getImages() {
  48.       return this.images;
  49.    }
  50.  
  51.    public Vector getResources() {
  52.       return this.resources;
  53.    }
  54.  
  55.    public Vector getFrames() {
  56.       return this.frames;
  57.    }
  58.  
  59.    public Vector getLayers() {
  60.       return this.layers;
  61.    }
  62.  
  63.    public String getLastModified() {
  64.       return this.lastModified;
  65.    }
  66.  
  67.    public int getLinkStatus() {
  68.       return this.linkStatus;
  69.    }
  70.  
  71.    public void setLinkStatus(int var1) {
  72.       this.linkStatus = var1;
  73.    }
  74.  
  75.    private String getName() {
  76.       return this.url.toString();
  77.    }
  78.  
  79.    private static boolean inSite(URL var0, String var1) {
  80.       try {
  81.          String var2 = var0.getHost();
  82.          URL var3 = new URL(var1);
  83.          String var4 = var3.getHost();
  84.          String var5 = "";
  85.          String var6 = "";
  86.          int var7 = var4.toLowerCase().indexOf(var2.toLowerCase());
  87.          if (var7 >= 0) {
  88.             return true;
  89.          } else {
  90.             var7 = var2.toLowerCase().indexOf(var4.toLowerCase());
  91.             if (var7 >= 0) {
  92.                return true;
  93.             } else {
  94.                int var8 = var2.indexOf(46);
  95.                int var9 = var4.indexOf(46);
  96.                if (var8 >= 0) {
  97.                   var5 = var2.substring(var8);
  98.                   if (var9 >= 0) {
  99.                      var6 = var4.substring(var9);
  100.                      return var5.equalsIgnoreCase(var6);
  101.                   } else {
  102.                      return false;
  103.                   }
  104.                } else {
  105.                   return false;
  106.                }
  107.             }
  108.          }
  109.       } catch (Exception var10) {
  110.          HTMLCrawler.printException(var10, "PageScan.Insite");
  111.          return false;
  112.       }
  113.    }
  114.  
  115.    public void fetchInfo(Cache var1, Hashtable var2, int var3, HTMLCrawler var4) {
  116.       try {
  117.          String var6 = this.url.toString();
  118.          CacheRequest var7 = new CacheRequest(var6);
  119.          InputStream var5 = var1.getInputStream(var7);
  120.          String var8 = var7.getLocation();
  121.          if (var8 != null && !var8.equals(var6)) {
  122.             URL var9 = new URL(var8);
  123.             this.url = var9;
  124.             this.baseURL = this.url;
  125.          }
  126.  
  127.          this.lastModified = var7.getLastModified();
  128.  
  129.          try {
  130.             int var18 = Integer.parseInt(var7.getContentLength());
  131.             int var10 = var4.getTotalContentLength();
  132.             var4.setTotalContentLength(var10 + var18);
  133.             var10 = var4.getTotalContentLength();
  134.             if (var10 > var3) {
  135.                var4.setTotalContentLength(var10 - var18);
  136.                var4.isURLRemoved = true;
  137.                if (var5 != null) {
  138.                   var4.closeInputStream(var5);
  139.                }
  140.  
  141.                return;
  142.             }
  143.          } catch (Exception var16) {
  144.             HTMLCrawler.printException(var16, "PageScan.fetchInfo.cacheRequest");
  145.          }
  146.  
  147.          String var19 = var7.getContentType();
  148.          if (var5 != null) {
  149.             if (var19 == null) {
  150.                var4.closeInputStream(var5);
  151.             } else if (var19.indexOf("text/html") < 0) {
  152.                var4.closeInputStream(var5);
  153.             } else {
  154.                SGMLTag var21 = LiteParser.getParser().parseStream(var5);
  155.                var4.closeInputStream(var5);
  156.                if (var21 != null) {
  157.                   TreeEnumeration var11 = new TreeEnumeration(var21);
  158.                   if (var11 != null) {
  159.                      while(var11.hasMoreElements()) {
  160.                         SGMLElement var12 = (SGMLElement)var11.nextElement();
  161.                         if (var12 instanceof SGMLTag) {
  162.                            SGMLTagImpl var13 = (SGMLTagImpl)var12;
  163.  
  164.                            try {
  165.                               if ("A".equals(var13.getTagType())) {
  166.                                  this.processTagA(var13);
  167.                               } else if ("AREA".equals(var13.getTagType())) {
  168.                                  this.processTagAREA(var13);
  169.                               } else if ("BASE".equals(var13.getTagType())) {
  170.                                  this.processTagBASE(var13);
  171.                               } else if ("IMG".equals(var13.getTagType())) {
  172.                                  this.processTagIMG(var13);
  173.                               } else if ("FRAME".equals(var13.getTagType())) {
  174.                                  this.processTagFRAME(var13);
  175.                               } else if ("LAYER".equals(var13.getTagType())) {
  176.                                  this.processTagLAYER(var13);
  177.                               } else if ("SCRIPT".equals(var13.getTagType())) {
  178.                                  this.processTagSCRIPT(var13, var1, var2, var4, var3);
  179.                               } else if ("EMBED".equals(var13.getTagType())) {
  180.                                  this.processTagEMBED(var13);
  181.                               } else if ("APPLET".equals(var13.getTagType())) {
  182.                                  this.processTagAPPLET(var13);
  183.                               } else if ("META".equals(var13.getTagType())) {
  184.                                  this.processTagMETA(var13);
  185.                               } else if ("LINK".equals(var13.getTagType())) {
  186.                                  this.processTagLINK(var13, var1, var2, var4, var3);
  187.                               } else if ("STYLE".equals(var13.getTagType())) {
  188.                                  this.processTagSTYLE(var13, var1, var2, var4, var3);
  189.                               } else if ("BODY".equals(var13.getTagType())) {
  190.                                  this.processTagBODY(var13);
  191.                               }
  192.                            } catch (Exception var15) {
  193.                               HTMLCrawler.printException(var15, "Scanning the page");
  194.                            }
  195.                         }
  196.                      }
  197.  
  198.                   }
  199.                }
  200.             }
  201.          }
  202.       } catch (Exception var17) {
  203.          HTMLCrawler.printException(var17, "PageScan.fetchInfo");
  204.          var4.isSiteValid = false;
  205.       }
  206.    }
  207.  
  208.    private void insertImage(String var1) {
  209.       try {
  210.          if (!this.inSiteSpecified || inSite(this.site, var1)) {
  211.             this.images.addElement(var1);
  212.          }
  213.       } catch (Exception var3) {
  214.          HTMLCrawler.printException(var3, "PageScan.insertImage");
  215.       }
  216.    }
  217.  
  218.    private void insertResource(String var1) {
  219.       try {
  220.          if (!this.inSiteSpecified || inSite(this.site, var1)) {
  221.             this.resources.addElement(var1);
  222.          }
  223.       } catch (Exception var3) {
  224.          HTMLCrawler.printException(var3, "PageScan.insertResource");
  225.       }
  226.    }
  227.  
  228.    private void insertURL(String var1, int var2) {
  229.       try {
  230.          if (var2 == 1) {
  231.             this.links.addElement(var1);
  232.             return;
  233.          }
  234.  
  235.          if (var2 == 2) {
  236.             this.frames.addElement(var1);
  237.             return;
  238.          }
  239.  
  240.          if (var2 == 3) {
  241.             this.layers.addElement(var1);
  242.             return;
  243.          }
  244.       } catch (Exception var4) {
  245.          HTMLCrawler.printException(var4, "PageScan.insertURL");
  246.       }
  247.  
  248.    }
  249.  
  250.    private void processTagA(SGMLTagImpl var1) {
  251.       try {
  252.          SGMLAttribute var2 = var1.getAttribute("HREF");
  253.          if (var2 != null) {
  254.             String var3 = var2.getUnicode();
  255.             if (var3 != null) {
  256.                if (this.baseURLExists) {
  257.                   this.processAttributeURL(this.baseURL, var3, 1);
  258.                   return;
  259.                }
  260.  
  261.                this.processAttributeURL(this.url, var3, 1);
  262.                return;
  263.             }
  264.          }
  265.       } catch (Exception var4) {
  266.          HTMLCrawler.printException(var4, "PageScan.processTagA");
  267.       }
  268.  
  269.    }
  270.  
  271.    private void processTagAREA(SGMLTagImpl var1) {
  272.       try {
  273.          SGMLAttribute var2 = var1.getAttribute("HREF");
  274.          if (var2 != null) {
  275.             String var3 = var2.getUnicode();
  276.             if (var3 != null) {
  277.                if (this.baseURLExists) {
  278.                   this.processAttributeURL(this.baseURL, var3, 1);
  279.                   return;
  280.                }
  281.  
  282.                this.processAttributeURL(this.url, var3, 1);
  283.                return;
  284.             }
  285.          }
  286.       } catch (Exception var4) {
  287.          HTMLCrawler.printException(var4, "PageScan.processTagAREA");
  288.       }
  289.  
  290.    }
  291.  
  292.    private void processTagBASE(SGMLTagImpl var1) {
  293.       try {
  294.          SGMLAttribute var2 = var1.getAttribute("HREF");
  295.          if (var2 != null) {
  296.             String var3 = var2.getUnicode();
  297.             if (var3 != null) {
  298.                this.baseURL = new URL(var3);
  299.                this.baseURLExists = true;
  300.                return;
  301.             }
  302.          }
  303.       } catch (Exception var4) {
  304.          HTMLCrawler.printException(var4, "PageScan.processTagBASE");
  305.       }
  306.  
  307.    }
  308.  
  309.    private void processTagIMG(SGMLTagImpl var1) {
  310.       try {
  311.          SGMLAttribute var2 = var1.getAttribute("SRC");
  312.          if (var2 != null) {
  313.             String var3 = var2.getUnicode();
  314.             if (var3 != null) {
  315.                if (this.baseURLExists) {
  316.                   this.processAttributeInsertImage(this.baseURL, var3);
  317.                   return;
  318.                }
  319.  
  320.                this.processAttributeInsertImage(this.url, var3);
  321.                return;
  322.             }
  323.          }
  324.       } catch (Exception var4) {
  325.          HTMLCrawler.printException(var4, "PageScan.processTagIMG");
  326.       }
  327.  
  328.    }
  329.  
  330.    private void processTagFRAME(SGMLTagImpl var1) {
  331.       try {
  332.          SGMLAttribute var2 = var1.getAttribute("SRC");
  333.          if (var2 != null) {
  334.             String var3 = var2.getUnicode();
  335.             if (var3 != null) {
  336.                if (this.baseURLExists) {
  337.                   this.processAttributeURL(this.baseURL, var3, 2);
  338.                   return;
  339.                }
  340.  
  341.                this.processAttributeURL(this.url, var3, 2);
  342.                return;
  343.             }
  344.          }
  345.       } catch (Exception var4) {
  346.          HTMLCrawler.printException(var4, "PageScan.processTagFRAME");
  347.       }
  348.  
  349.    }
  350.  
  351.    private void processTagLAYER(SGMLTagImpl var1) {
  352.       try {
  353.          SGMLAttribute var2 = var1.getAttribute("SRC");
  354.          if (var2 != null) {
  355.             String var3 = var2.getUnicode();
  356.             if (var3 != null) {
  357.                if (this.baseURLExists) {
  358.                   this.processAttributeURL(this.baseURL, var3, 3);
  359.                } else {
  360.                   this.processAttributeURL(this.url, var3, 3);
  361.                }
  362.             }
  363.          }
  364.  
  365.          var2 = var1.getAttribute("BACKGROUND");
  366.          if (var2 != null) {
  367.             String var6 = var2.getUnicode();
  368.             if (var6 != null) {
  369.                if (this.baseURLExists) {
  370.                   this.processAttributeInsertImage(this.baseURL, var6);
  371.                   return;
  372.                }
  373.  
  374.                this.processAttributeInsertImage(this.url, var6);
  375.                return;
  376.             }
  377.          }
  378.       } catch (Exception var4) {
  379.          HTMLCrawler.printException(var4, "PageScan.processTagLAYER");
  380.       }
  381.  
  382.    }
  383.  
  384.    private void processTagSCRIPT(SGMLTagImpl var1, Cache var2, Hashtable var3, HTMLCrawler var4, int var5) {
  385.       try {
  386.          SGMLAttribute var6 = var1.getAttribute("ARCHIVE");
  387.          if (var6 != null) {
  388.             String var7 = var6.getUnicode();
  389.             if (var7 != null) {
  390.                if (this.baseURLExists) {
  391.                   this.processAttributeInsertResource(this.baseURL, var7);
  392.                } else {
  393.                   this.processAttributeInsertResource(this.url, var7);
  394.                }
  395.             }
  396.          }
  397.  
  398.          var6 = var1.getAttribute("SRC");
  399.          if (var6 != null) {
  400.             String var18 = var6.getUnicode();
  401.             if (var18 != null) {
  402.                Object var8 = null;
  403.                String var9 = "";
  404.                URL var19;
  405.                if (this.baseURLExists) {
  406.                   var19 = new URL(this.baseURL, var18);
  407.                   var9 = var19.toString();
  408.                } else {
  409.                   var19 = new URL(this.url, var18);
  410.                   var9 = var19.toString();
  411.                }
  412.  
  413.                CacheRequest var10 = new CacheRequest(var9);
  414.                var10.setMimeTypeHint("application/x-javascript");
  415.                InputStream var11 = var2.getInputStream(var10);
  416.                String var12 = var10.getLastModified();
  417.  
  418.                try {
  419.                   int var13 = Integer.parseInt(var10.getContentLength());
  420.                   int var14 = var4.getTotalContentLength();
  421.                   var4.setTotalContentLength(var14 + var13);
  422.                   var14 = var4.getTotalContentLength();
  423.                   if (var14 > var5) {
  424.                      var4.setTotalContentLength(var14 - var13);
  425.                      var4.isURLRemoved = true;
  426.                      var4.closeInputStream(var11);
  427.                      return;
  428.                   }
  429.  
  430.                   var4.readAllInputStream(var11);
  431.                   var4.closeInputStream(var11);
  432.                } catch (Exception var15) {
  433.                   HTMLCrawler.printException(var15, "PageScan.processTagScript.cacheRequest");
  434.                }
  435.  
  436.                LinkObject var21 = new LinkObject(var19, var12, this.linkStatus);
  437.                var3.put(var9, var21);
  438.                return;
  439.             }
  440.          }
  441.       } catch (Exception var16) {
  442.          HTMLCrawler.printException(var16, "PageScan.processTagSCRIPT");
  443.       }
  444.  
  445.    }
  446.  
  447.    private void processTagEMBED(SGMLTagImpl var1) {
  448.       try {
  449.          SGMLAttribute var2 = var1.getAttribute("SRC");
  450.          if (var2 != null) {
  451.             String var3 = var2.getUnicode();
  452.             if (var3 != null) {
  453.                if (this.baseURLExists) {
  454.                   this.processAttributeInsertResource(this.baseURL, var3);
  455.                   return;
  456.                }
  457.  
  458.                this.processAttributeInsertResource(this.url, var3);
  459.                return;
  460.             }
  461.          }
  462.       } catch (Exception var4) {
  463.          HTMLCrawler.printException(var4, "PageScan.processTagEMBED");
  464.       }
  465.  
  466.    }
  467.  
  468.    private void processTagAPPLET(SGMLTagImpl var1) {
  469.       try {
  470.          URL var2 = this.url;
  471.          boolean var3 = false;
  472.          SGMLAttribute var4 = var1.getAttribute("CODEBASE");
  473.          if (var4 != null) {
  474.             String var5 = var4.getUnicode();
  475.             if (var5 != null) {
  476.                var3 = true;
  477.                if (var5.indexOf("://") >= 0) {
  478.                   var2 = new URL(var5);
  479.                } else {
  480.                   String var6 = var5.concat("/");
  481.                   if (this.baseURLExists) {
  482.                      var2 = new URL(this.baseURL, var6);
  483.                   } else {
  484.                      var2 = new URL(this.url, var6);
  485.                   }
  486.                }
  487.             }
  488.          }
  489.  
  490.          var4 = var1.getAttribute("ARCHIVE");
  491.          if (var4 != null) {
  492.             String var10 = var4.getUnicode();
  493.             if (var10 != null) {
  494.                if (var3) {
  495.                   this.processAttributeInsertResource(var2, var10);
  496.                } else if (this.baseURLExists) {
  497.                   this.processAttributeInsertResource(this.baseURL, var10);
  498.                } else {
  499.                   this.processAttributeInsertResource(this.url, var10);
  500.                }
  501.             }
  502.          }
  503.  
  504.          var4 = var1.getAttribute("CODE");
  505.          if (var4 != null) {
  506.             String var11 = var4.getUnicode();
  507.             if (var11 != null) {
  508.                if (var3) {
  509.                   this.processAttributeInsertResource(var2, var11);
  510.                   return;
  511.                }
  512.  
  513.                if (this.baseURLExists) {
  514.                   this.processAttributeInsertResource(this.baseURL, var11);
  515.                   return;
  516.                }
  517.  
  518.                this.processAttributeInsertResource(this.url, var11);
  519.                return;
  520.             }
  521.          }
  522.       } catch (Exception var7) {
  523.          HTMLCrawler.printException(var7, "PageScan.processTagAPPLET");
  524.       }
  525.  
  526.    }
  527.  
  528.    private void processTagMETA(SGMLTagImpl var1) {
  529.       try {
  530.          SGMLAttribute var2 = var1.getAttribute("HTTP-EQUIV");
  531.          if (var2 != null) {
  532.             String var3 = var2.getUnicode();
  533.             if (var3 != null && "refresh".equalsIgnoreCase(var3)) {
  534.                SGMLAttribute var4 = var1.getAttribute("CONTENT");
  535.                if (var4 != null) {
  536.                   String var5 = var4.getUnicode();
  537.                   if (var5 != null) {
  538.                      String var6 = var5.toUpperCase();
  539.                      int var7 = var6.indexOf("URL");
  540.                      if (var7 >= 0) {
  541.                         int var8 = var6.indexOf("=");
  542.                         if (var8 > var7 + 2) {
  543.                            var6 = var5.substring(var8 + 1);
  544.                            String var9 = var6.trim();
  545.                            int var10 = var9.length();
  546.                            if (var10 > 1 && var9.startsWith("'") && var9.endsWith("'")) {
  547.                               var6 = var9.substring(1, var10 - 1);
  548.                               var9 = var6;
  549.                            }
  550.  
  551.                            this.insertURL(var9, 1);
  552.                            return;
  553.                         }
  554.                      }
  555.                   }
  556.                }
  557.             }
  558.          }
  559.       } catch (Exception var11) {
  560.          HTMLCrawler.printException(var11, "PageScan.processTagMETA");
  561.       }
  562.  
  563.    }
  564.  
  565.    private void processTagLINK(SGMLTagImpl var1, Cache var2, Hashtable var3, HTMLCrawler var4, int var5) {
  566.       try {
  567.          SGMLAttribute var6 = var1.getAttribute("REL");
  568.          if (var6 != null) {
  569.             String var7 = var6.getUnicode();
  570.             if (var7 != null && "stylesheet".equalsIgnoreCase(var7)) {
  571.                SGMLAttribute var8 = var1.getAttribute("HREF");
  572.                if (var8 != null) {
  573.                   var7 = var8.getUnicode();
  574.                   if (var7 != null) {
  575.                      SGMLAttribute var9 = var1.getAttribute("TYPE");
  576.                      if (var9 != null) {
  577.                         String var10 = var9.getUnicode();
  578.                         if (var10 != null && "text/javascript".equalsIgnoreCase(var10)) {
  579.                            Object var11 = null;
  580.                            String var12 = "";
  581.                            URL var21;
  582.                            if (this.baseURLExists) {
  583.                               var21 = new URL(this.baseURL, var7);
  584.                               var12 = var21.toString();
  585.                            } else {
  586.                               var21 = new URL(this.url, var7);
  587.                               var12 = var21.toString();
  588.                            }
  589.  
  590.                            CacheRequest var13 = new CacheRequest(var12);
  591.                            var13.setMimeTypeHint("text/javascript");
  592.                            InputStream var14 = var2.getInputStream(var13);
  593.                            String var15 = var13.getLastModified();
  594.  
  595.                            try {
  596.                               int var16 = Integer.parseInt(var13.getContentLength());
  597.                               int var17 = var4.getTotalContentLength();
  598.                               var4.setTotalContentLength(var17 + var16);
  599.                               var17 = var4.getTotalContentLength();
  600.                               if (var17 > var5) {
  601.                                  var4.setTotalContentLength(var17 - var16);
  602.                                  var4.isURLRemoved = true;
  603.                                  var4.closeInputStream(var14);
  604.                                  return;
  605.                               }
  606.  
  607.                               var4.readAllInputStream(var14);
  608.                               var4.closeInputStream(var14);
  609.                            } catch (Exception var18) {
  610.                               HTMLCrawler.printException(var18, "PageScan.processTagLink.cacheRequest");
  611.                               HTMLCrawler.printException(var18, "PageScan.processTagLink.cacheRequest");
  612.                            }
  613.  
  614.                            LinkObject var23 = new LinkObject(var21, var15, this.linkStatus);
  615.                            var3.put(var12, var23);
  616.                            return;
  617.                         }
  618.                      }
  619.  
  620.                      if (this.baseURLExists) {
  621.                         this.processAttributeInsertResource(this.baseURL, var7);
  622.                         return;
  623.                      }
  624.  
  625.                      this.processAttributeInsertResource(this.url, var7);
  626.                      return;
  627.                   }
  628.                }
  629.             }
  630.          }
  631.       } catch (Exception var19) {
  632.          HTMLCrawler.printException(var19, "PageScan.processTagLINK");
  633.       }
  634.  
  635.    }
  636.  
  637.    private void processTagSTYLE(SGMLTagImpl var1, Cache var2, Hashtable var3, HTMLCrawler var4, int var5) {
  638.       try {
  639.          SGMLAttribute var6 = var1.getAttribute("SRC");
  640.          if (var6 != null) {
  641.             String var7 = var6.getUnicode();
  642.             if (var7 != null) {
  643.                SGMLAttribute var8 = var1.getAttribute("TYPE");
  644.                if (var8 != null) {
  645.                   String var9 = var8.getUnicode();
  646.                   if (var9 != null && "text/javascript".equalsIgnoreCase(var9)) {
  647.                      Object var10 = null;
  648.                      String var11 = "";
  649.                      URL var19;
  650.                      if (this.baseURLExists) {
  651.                         var19 = new URL(this.baseURL, var7);
  652.                         var11 = var19.toString();
  653.                      } else {
  654.                         var19 = new URL(this.url, var7);
  655.                         var11 = var19.toString();
  656.                      }
  657.  
  658.                      CacheRequest var12 = new CacheRequest(var11);
  659.                      var12.setMimeTypeHint("text/javascript");
  660.                      InputStream var13 = var2.getInputStream(var12);
  661.                      String var14 = var12.getLastModified();
  662.  
  663.                      try {
  664.                         int var15 = Integer.parseInt(var12.getContentLength());
  665.                         int var16 = var4.getTotalContentLength();
  666.                         var4.setTotalContentLength(var16 + var15);
  667.                         var16 = var4.getTotalContentLength();
  668.                         if (var16 > var5) {
  669.                            var4.setTotalContentLength(var16 - var15);
  670.                            var4.isURLRemoved = true;
  671.                            var4.closeInputStream(var13);
  672.                            return;
  673.                         }
  674.  
  675.                         var4.readAllInputStream(var13);
  676.                         var4.closeInputStream(var13);
  677.                      } catch (Exception var17) {
  678.                         HTMLCrawler.printException(var17, "PageScan.processTagStyle.cacheRequest");
  679.                      }
  680.  
  681.                      LinkObject var21 = new LinkObject(var19, var14, this.linkStatus);
  682.                      var3.put(var11, var21);
  683.                      return;
  684.                   }
  685.                }
  686.  
  687.                if (this.baseURLExists) {
  688.                   this.processAttributeInsertResource(this.baseURL, var7);
  689.                   return;
  690.                }
  691.  
  692.                this.processAttributeInsertResource(this.url, var7);
  693.                return;
  694.             }
  695.          }
  696.       } catch (Exception var18) {
  697.          HTMLCrawler.printException(var18, "PageScan.processTagSTYLE");
  698.       }
  699.  
  700.    }
  701.  
  702.    private void processTagBODY(SGMLTagImpl var1) {
  703.       try {
  704.          SGMLAttribute var2 = var1.getAttribute("BACKGROUND");
  705.          if (var2 != null) {
  706.             String var3 = var2.getUnicode();
  707.             if (var3 != null) {
  708.                if (this.baseURLExists) {
  709.                   this.processAttributeInsertImage(this.baseURL, var3);
  710.                   return;
  711.                }
  712.  
  713.                this.processAttributeInsertImage(this.url, var3);
  714.                return;
  715.             }
  716.          }
  717.       } catch (Exception var4) {
  718.          HTMLCrawler.printException(var4, "PageScan.processTagBODY");
  719.       }
  720.  
  721.    }
  722.  
  723.    private void processAttributeURL(URL var1, String var2, int var3) {
  724.       try {
  725.          int var4 = var2.length();
  726.          if (var4 > 1 && var2.startsWith("'") && var2.endsWith("'")) {
  727.             String var5 = var2.substring(1, var4 - 1);
  728.             var2 = var5;
  729.          }
  730.  
  731.          String var24 = var2.toLowerCase();
  732.          int var6 = var24.indexOf("mailto");
  733.          if (var6 < 0) {
  734.             int var7 = var24.indexOf("mailbox");
  735.             if (var7 < 0) {
  736.                int var8 = var24.indexOf("news:");
  737.                if (var8 < 0) {
  738.                   int var9 = var24.indexOf("#");
  739.                   if (var9 != 0) {
  740.                      if (var9 > 0) {
  741.                         var2 = var2.substring(0, var9);
  742.                      }
  743.  
  744.                      URL var10 = new URL(var1, var2);
  745.                      String var11 = var10.toString();
  746.                      if (!var24.endsWith(".jpg") && !var24.endsWith(".jpeg") && !var24.endsWith(".gif") && !var24.endsWith(".xbm")) {
  747.                         if (var24.indexOf(".htm") < 0 && var24.indexOf(".html") < 0 && var24.indexOf(".shtml") < 0 && !var24.endsWith("/")) {
  748.                            String var12 = var10.getFile();
  749.                            String var13 = var12.toLowerCase();
  750.                            if (var12.length() == 1 && var12.endsWith("/")) {
  751.                               this.insertURL(var11, var3);
  752.                            } else if (var13.indexOf("cgi-bin") < 0 && var13.indexOf(".cgi") < 0) {
  753.                               int var14 = var12.lastIndexOf(47);
  754.                               if (var14 >= 0) {
  755.                                  String var15 = var12.substring(var14);
  756.                                  if (var15.length() > 1) {
  757.                                     if (var14 >= 2) {
  758.                                        String var16 = var12.substring(var14 - 2, var14);
  759.                                        if (var16.equals(":/")) {
  760.                                           this.insertURL(var11, var3);
  761.                                           return;
  762.                                        }
  763.                                     }
  764.  
  765.                                     int var25 = var15.lastIndexOf(46);
  766.                                     if (var25 >= 1) {
  767.                                        String var17 = var15.substring(var25);
  768.                                        int var18 = var17.length();
  769.                                        if (var18 > 1) {
  770.                                           boolean var19 = true;
  771.  
  772.                                           for(int var20 = 1; var20 < var18; ++var20) {
  773.                                              char var21 = var17.charAt(var20);
  774.                                              new Character(var21);
  775.                                              if (!Character.isLetterOrDigit(var21)) {
  776.                                                 var19 = false;
  777.                                                 break;
  778.                                              }
  779.                                           }
  780.  
  781.                                           if (!var19) {
  782.                                              this.insertURL(var11, var3);
  783.                                              return;
  784.                                           }
  785.                                        }
  786.  
  787.                                        this.insertResource(var11);
  788.                                        return;
  789.                                     }
  790.                                  }
  791.                               }
  792.  
  793.                               this.insertURL(var11, var3);
  794.                            } else {
  795.                               this.insertURL(var11, var3);
  796.                            }
  797.                         } else {
  798.                            this.insertURL(var11, var3);
  799.                         }
  800.                      } else {
  801.                         this.insertImage(var11);
  802.                      }
  803.                   }
  804.                }
  805.             }
  806.          }
  807.       } catch (Exception var23) {
  808.          HTMLCrawler.printException(var23, "PageScan.processAttributeURL");
  809.       }
  810.    }
  811.  
  812.    private void processAttributeInsertImage(URL var1, String var2) {
  813.       try {
  814.          URL var3 = new URL(var1, var2);
  815.          String var4 = var3.toString();
  816.          this.insertImage(var4);
  817.       } catch (Exception var5) {
  818.          HTMLCrawler.printException(var5, "PageScan.processAttributeInsertImage");
  819.       }
  820.    }
  821.  
  822.    private void processAttributeInsertResource(URL var1, String var2) {
  823.       try {
  824.          URL var3 = new URL(var1, var2);
  825.          String var4 = var3.toString();
  826.          this.insertResource(var4);
  827.       } catch (Exception var5) {
  828.          HTMLCrawler.printException(var5, "PageScan.processAttributeInsertResource");
  829.       }
  830.    }
  831. }
  832.