home *** CD-ROM | disk | FTP | other *** search
/ Enter 2006 February / enter-2006-02.iso / files / httrack-3.40.exe / {app} / src / htsparse.c < prev    next >
Encoding:
C/C++ Source or Header  |  2006-01-21  |  193.9 KB  |  4,554 lines
  1. /* ------------------------------------------------------------ */
  2. /*
  3. HTTrack Website Copier, Offline Browser for Windows and Unix
  4. Copyright (C) Xavier Roche and other contributors
  5.  
  6. This program is free software; you can redistribute it and/or
  7. modify it under the terms of the GNU General Public License
  8. as published by the Free Software Foundation; either version 2
  9. of the License, or any later version.
  10.  
  11. This program is distributed in the hope that it will be useful,
  12. but WITHOUT ANY WARRANTY; without even the implied warranty of
  13. MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14. GNU General Public License for more details.
  15.  
  16. You should have received a copy of the GNU General Public License
  17. along with this program; if not, write to the Free Software
  18. Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
  19.  
  20.  
  21. Important notes:
  22.  
  23. - We hereby ask people using this source NOT to use it in purpose of grabbing
  24. emails addresses, or collecting any other private information on persons.
  25. This would disgrace our work, and spoil the many hours we spent on it.
  26.  
  27.  
  28. Please visit our Website: http://www.httrack.com
  29. */
  30.  
  31.  
  32. /* ------------------------------------------------------------ */
  33. /* File: htsparse.c parser                                      */
  34. /*       html/javascript/css parser                             */
  35. /*       and other parser routines                              */
  36. /* Author: Xavier Roche                                         */
  37. /* ------------------------------------------------------------ */
  38.  
  39.  
  40. /* Internal engine bytecode */
  41. #define HTS_INTERNAL_BYTECODE
  42.  
  43. #ifndef  _WIN32_WCE
  44. #include <fcntl.h>
  45. #endif
  46. #include <ctype.h>
  47.  
  48. /* File defs */
  49. #include "htscore.h"
  50.  
  51. /* specific definitions */
  52. #include "htsbase.h"
  53. #include "htsnet.h"
  54. #include "htsbauth.h"
  55. #include "htsmd5.h"
  56. #include "htsindex.h"
  57.  
  58. /* external modules */
  59. #include "htsmodules.h"
  60.  
  61. // htswrap_add
  62. #include "htswrap.h"
  63.  
  64. // parser
  65. #include "htsparse.h"
  66. #include "htsback.h"
  67.  
  68. // specific defines
  69. #define urladr   (liens[ptr]->adr)
  70. #define urlfil   (liens[ptr]->fil)
  71. #define savename (liens[ptr]->sav)
  72. #define parenturladr   (liens[liens[ptr]->precedent]->adr)
  73. #define parenturlfil   (liens[liens[ptr]->precedent]->fil)
  74. #define parentsavename (liens[liens[ptr]->precedent]->sav)
  75. #define relativeurladr   ((!parent_relative)?urladr:parenturladr)
  76. #define relativeurlfil   ((!parent_relative)?urlfil:parenturlfil)
  77. #define relativesavename ((!parent_relative)?savename:parentsavename)
  78.  
  79. #define test_flush if (opt->flush) { if (opt->log) { fflush(opt->log); } if (opt->errlog) { fflush(opt->errlog);  } }
  80.  
  81. // does nothing
  82. #define XH_uninit do {} while(0)
  83.  
  84. // version optimisΘe, qui permet de ne pas toucher aux html non modifiΘs (update)
  85. #define REALLOC_SIZE 8192
  86. #define HT_ADD_CHK(A) if (((int) (A)+ht_len+1) >= ht_size) { \
  87.   ht_size=(A)+ht_len+REALLOC_SIZE; \
  88.   ht_buff=(char*) realloct(ht_buff,ht_size); \
  89.   if (ht_buff==NULL) { \
  90.   printf("PANIC! : Not enough memory [%d]\n",__LINE__); \
  91.   XH_uninit; \
  92.   abortLogFmt("not enough memory for current html document in HT_ADD_CHK : realloct(%d) failed" _ ht_size); \
  93.   exit(1); \
  94.   } \
  95. } \
  96.   ht_len+=A;
  97. #define HT_ADD_ADR \
  98.   if ((opt->getmode & 1) && (ptr>0)) { \
  99.   int i=((int) (adr - lastsaved)),j=ht_len; HT_ADD_CHK(i) \
  100.   memcpy(ht_buff+j, lastsaved, i); \
  101.   ht_buff[j+i]='\0'; \
  102.   lastsaved=adr; \
  103.   }
  104. #define HT_ADD(A) \
  105.   if ((opt->getmode & 1) && (ptr>0)) { \
  106.   int i_=strlen(A),j_=ht_len; \
  107.   if (i_) { \
  108.   HT_ADD_CHK(i_) \
  109.   memcpy(ht_buff+j_, A, i_); \
  110.   ht_buff[j_+i_]='\0'; \
  111.   } }
  112. #define HT_ADD_HTMLESCAPED(A) \
  113.   if ((opt->getmode & 1) && (ptr>0)) { \
  114.     int i_, j_; \
  115.     char BIGSTK tempo_[HTS_URLMAXSIZE*2]; \
  116.     escape_for_html_print(A, tempo_); \
  117.     i_=strlen(tempo_); \
  118.     j_=ht_len; \
  119.     if (i_) { \
  120.     HT_ADD_CHK(i_) \
  121.     memcpy(ht_buff+j_, tempo_, i_); \
  122.     ht_buff[j_+i_]='\0'; \
  123.   } }
  124. #define HT_ADD_HTMLESCAPED_FULL(A) \
  125.   if ((opt->getmode & 1) && (ptr>0)) { \
  126.     int i_, j_; \
  127.     char BIGSTK tempo_[HTS_URLMAXSIZE*2]; \
  128.     escape_for_html_print_full(A, tempo_); \
  129.     i_=strlen(tempo_); \
  130.     j_=ht_len; \
  131.     if (i_) { \
  132.     HT_ADD_CHK(i_) \
  133.     memcpy(ht_buff+j_, tempo_, i_); \
  134.     ht_buff[j_+i_]='\0'; \
  135.   } }
  136. #define HT_ADD_START \
  137.   int ht_size=(int)(r->size*5)/4+REALLOC_SIZE; \
  138.   int ht_len=0; \
  139.   char* ht_buff=NULL; \
  140.   if ((opt->getmode & 1) && (ptr>0)) { \
  141.   ht_buff=(char*) malloct(ht_size); \
  142.   if (ht_buff==NULL) { \
  143.   printf("PANIC! : Not enough memory [%d]\n",__LINE__); \
  144.   XH_uninit; \
  145.   abortLogFmt("not enough memory for current html document in HT_ADD_START : malloct(%d) failed" _ ht_size); \
  146.   exit(1); \
  147.   } \
  148.   ht_buff[0]='\0'; \
  149.   }
  150. #define HT_ADD_END { \
  151.   int ok=0;\
  152.   if (ht_buff) { \
  153.     char digest[32+2];\
  154.     INTsys fsize_old=fsize(fconv(savename));\
  155.     digest[0]='\0';\
  156.     domd5mem(ht_buff,ht_len,digest,1);\
  157.     if (fsize_old==ht_len) { \
  158.       int mlen = 0;\
  159.       char* mbuff;\
  160.       cache_readdata(cache,"//[HTML-MD5]//",savename,&mbuff,&mlen);\
  161.       if (mlen) \
  162.         mbuff[mlen]='\0';\
  163.       if ((mlen == 32) && (strcmp(((mbuff!=NULL)?mbuff:""),digest)==0)) {\
  164.         ok=1;\
  165.         if ( (opt->debug>1) && (opt->log!=NULL) ) {\
  166.           fspc(opt->log,"debug"); fprintf(opt->log,"File not re-written (md5): %s"LF,savename);\
  167.           test_flush;\
  168.         }\
  169.       } else {\
  170.         ok=0;\
  171.       } \
  172.     }\
  173.     if (!ok) { \
  174.       file_notify(urladr, urlfil, savename, 1, 1, r->notmodified); \
  175.       fp=filecreate(savename); \
  176.       if (fp) { \
  177.         if (ht_len>0) {\
  178.         if ((INTsys)fwrite(ht_buff,1,ht_len,fp) != ht_len) { \
  179.           int fcheck;\
  180.           if ((fcheck=check_fatal_io_errno())) {\
  181.             opt->state.exit_xh=-1;\
  182.           }\
  183.           if (opt->errlog) {   \
  184.             fspc(opt->errlog,"error"); fprintf(opt->errlog,"Unable to write HTML file %s: %s"LF, savename, strerror(errno));\
  185.             if (fcheck) {\
  186.               fspc(opt->errlog,"error");\
  187.               fprintf(opt->errlog,"* * Fatal write error, giving up"LF);\
  188.             }\
  189.             test_flush;\
  190.           }\
  191.         }\
  192.         }\
  193.         fclose(fp); fp=NULL; \
  194.         if (strnotempty(r->lastmodified)) \
  195.         set_filetime_rfc822(savename,r->lastmodified); \
  196.       } else {\
  197.         int fcheck;\
  198.         if ((fcheck=check_fatal_io_errno())) {\
  199.                   fspc(opt->log,"error"); fprintf(opt->log,"Mirror aborted: disk full or filesystem problems"LF); \
  200.                     test_flush; \
  201.           opt->state.exit_xh=-1;\
  202.         }\
  203.         if (opt->errlog) { \
  204.           fspc(opt->errlog,"error");\
  205.           fprintf(opt->errlog,"Unable to save file %s : %s"LF, savename, strerror(errno));\
  206.           if (fcheck) {\
  207.             fspc(opt->errlog,"error");\
  208.             fprintf(opt->errlog,"* * Fatal write error, giving up"LF);\
  209.           }\
  210.           test_flush;\
  211.         }\
  212.       }\
  213.     } else {\
  214.       file_notify(urladr, urlfil, savename, 0, 0, r->notmodified); \
  215.       filenote(savename,NULL); \
  216.     }\
  217.     if (cache->ndx)\
  218.       cache_writedata(cache->ndx,cache->dat,"//[HTML-MD5]//",savename,digest,(int)strlen(digest));\
  219.   } \
  220.   freet(ht_buff); ht_buff=NULL; \
  221. }
  222. #define HT_ADD_FOP 
  223.  
  224. // COPY IN HTSCORE.C
  225. #define HT_INDEX_END do { \
  226.   if (!makeindex_done) { \
  227.   if (makeindex_fp) { \
  228.   char BIGSTK tempo[1024]; \
  229.   if (makeindex_links == 1) { \
  230.   sprintf(tempo,"<meta HTTP-EQUIV=\"Refresh\" CONTENT=\"0; URL=%s\">"CRLF,makeindex_firstlink); \
  231.   } else \
  232.   tempo[0]='\0'; \
  233.   fprintf(makeindex_fp,template_footer, \
  234.   "<!-- Mirror and index made by HTTrack Website Copier/"HTTRACK_VERSION" "HTTRACK_AFF_AUTHORS" -->", \
  235.   tempo \
  236.   ); \
  237.   fflush(makeindex_fp); \
  238.   fclose(makeindex_fp);  /* α ne pas oublier sinon on passe une nuit blanche */  \
  239.   makeindex_fp=NULL; \
  240.   usercommand(opt,0,NULL,fconcat(opt->path_html,"index.html"),"primary","primary");  \
  241.   } \
  242.   } \
  243.   makeindex_done=1;    /* ok c'est fait */  \
  244. } while(0)
  245.  
  246. // Enregistrement d'un lien:
  247. // on calcule la taille nΘcessaire: taille des 3 chaεnes α stocker (taille forcΘe paire, plus 2 octets de sΘcuritΘ)
  248. // puis on vΘrifie qu'on a assez de marge dans le buffer - sinon on en rΘalloue un autre
  249. // enfin on Θcrit α l'adresse courante du buffer, qu'on incrΘmente. on dΘcrΘmente la taille dispo d'autant ensuite
  250. // codebase: si non nul et si .class stockee on le note pour chemin primaire pour classes
  251. // FA,FS: former_adr et former_fil, lien original
  252. #define liens_record_sav_len(A) 
  253.  
  254. // COPIE DE HTSCORE.C
  255.  
  256. #define liens_record(A,F,S,FA,FF) { \
  257.   int notecode=0; \
  258.   int lienurl_len=((sizeof(lien_url)+HTS_ALIGN-1)/HTS_ALIGN)*HTS_ALIGN,\
  259.   adr_len=strlen(A),\
  260.   fil_len=strlen(F),\
  261.   sav_len=strlen(S),\
  262.   cod_len=0,\
  263.   former_adr_len=strlen(FA),\
  264.   former_fil_len=strlen(FF); \
  265.   if (former_adr_len>0) {\
  266.     former_adr_len=(former_adr_len/HTS_ALIGN)*HTS_ALIGN+HTS_ALIGN*2; \
  267.     former_fil_len=(former_fil_len/HTS_ALIGN)*HTS_ALIGN+HTS_ALIGN*2; \
  268.   } else \
  269.     former_adr_len=former_fil_len=0;\
  270.   if (strlen(F)>6) if (strnotempty(codebase)) if (strfield(F+strlen(F)-6,".class")) {\
  271.     notecode=1; \
  272.     cod_len=strlen(codebase); \
  273.     cod_len=(cod_len/HTS_ALIGN)*HTS_ALIGN+HTS_ALIGN*2; \
  274.   } \
  275.   adr_len=(adr_len/HTS_ALIGN)*HTS_ALIGN+HTS_ALIGN*2; \
  276.   fil_len=(fil_len/HTS_ALIGN)*HTS_ALIGN+HTS_ALIGN*2; \
  277.   sav_len=(sav_len/HTS_ALIGN)*HTS_ALIGN+HTS_ALIGN*2; \
  278.   if ((int) lien_size < (int) (adr_len+fil_len+sav_len+cod_len+former_adr_len+former_fil_len+lienurl_len)) { \
  279.     lien_buffer=(char*) ((void*) calloct(add_tab_alloc,1)); \
  280.     lien_size=add_tab_alloc; \
  281.     if (lien_buffer!=NULL) { \
  282.     liens[lien_tot]=(lien_url*) (void*) lien_buffer; lien_buffer+=lienurl_len; lien_size-=lienurl_len; \
  283.     liens[lien_tot]->firstblock=1; \
  284.     } \
  285.   } else { \
  286.     liens[lien_tot]=(lien_url*) (void*) lien_buffer; lien_buffer+=lienurl_len; lien_size-=lienurl_len; \
  287.     liens[lien_tot]->firstblock=0; \
  288.   } \
  289.   if (liens[lien_tot]!=NULL) { \
  290.     liens[lien_tot]->adr=lien_buffer; lien_buffer+=adr_len; lien_size-=adr_len; \
  291.     liens[lien_tot]->fil=lien_buffer; lien_buffer+=fil_len; lien_size-=fil_len; \
  292.     liens[lien_tot]->sav=lien_buffer; lien_buffer+=sav_len; lien_size-=sav_len; \
  293.     liens[lien_tot]->cod=NULL; \
  294.     if (notecode) { \
  295.       liens[lien_tot]->cod=lien_buffer; \
  296.       lien_buffer+=cod_len; \
  297.       lien_size-=cod_len; \
  298.       strcpybuff(liens[lien_tot]->cod,codebase); \
  299.     } \
  300.     if (former_adr_len>0) {\
  301.       liens[lien_tot]->former_adr=lien_buffer; lien_buffer+=former_adr_len; lien_size-=former_adr_len; \
  302.       liens[lien_tot]->former_fil=lien_buffer; lien_buffer+=former_fil_len; lien_size-=former_fil_len; \
  303.       strcpybuff(liens[lien_tot]->former_adr,FA); \
  304.       strcpybuff(liens[lien_tot]->former_fil,FF); \
  305.     }\
  306.     strcpybuff(liens[lien_tot]->adr,A); \
  307.     strcpybuff(liens[lien_tot]->fil,F); \
  308.     strcpybuff(liens[lien_tot]->sav,S); \
  309.     liens_record_sav_len(liens[lien_tot]); \
  310.     hash_write(hashptr,lien_tot,opt->urlhack);  \
  311.   } \
  312. }
  313.  
  314. #define ENGINE_LOAD_CONTEXT() \
  315.   ENGINE_LOAD_CONTEXT_BASE(); \
  316.   /* */ \
  317.   htsblk* r = stre->r_; \
  318.   hash_struct* hash = stre->hash_; \
  319.   int lien_max = *stre->lien_max_; \
  320.   /* */ \
  321.   int error = * stre->error_; \
  322.   int store_errpage = * stre->store_errpage_; \
  323.   char* codebase = stre->codebase; \
  324.   char* base = stre->base; \
  325.   /* */ \
  326.   int makeindex_done = *stre->makeindex_done_; \
  327.   FILE* makeindex_fp = *stre->makeindex_fp_; \
  328.   int makeindex_links = *stre->makeindex_links_; \
  329.   char* makeindex_firstlink = stre->makeindex_firstlink_; \
  330.   /* */ \
  331.   char *template_header = stre->template_header_; \
  332.   char *template_body = stre->template_body_; \
  333.   char *template_footer = stre->template_footer_; \
  334.   /* */ \
  335.   LLint stat_fragment = *stre->stat_fragment_; \
  336.   TStamp makestat_time = stre->makestat_time; \
  337.   FILE* makestat_fp = stre->makestat_fp
  338.  
  339. #define ENGINE_SAVE_CONTEXT() \
  340.   ENGINE_SAVE_CONTEXT_BASE(); \
  341.   /* */ \
  342.   * stre->error_ = error; \
  343.   * stre->store_errpage_ = store_errpage; \
  344.   * stre->lien_max_ = lien_max; \
  345.   /* */ \
  346.   *stre->makeindex_done_ = makeindex_done; \
  347.   *stre->makeindex_fp_ = makeindex_fp; \
  348.   *stre->makeindex_links_ = makeindex_links; \
  349.   /* */ \
  350.   *stre->stat_fragment_ = stat_fragment
  351.  
  352. #define _FILTERS     (*opt->filters.filters)
  353. #define _FILTERS_PTR (opt->filters.filptr)
  354. #define _ROBOTS      ((robots_wizard*)opt->robotsptr)
  355.  
  356. /* Apply current *adr character for the script automate */
  357. #define AUTOMATE_LOOKUP_CURRENT_ADR() do { \
  358.   if (inscript) { \
  359.   int new_state_pos; \
  360.   new_state_pos=inscript_state[inscript_state_pos][(unsigned char)*adr]; \
  361.   if (new_state_pos < 0) { \
  362.   new_state_pos=inscript_state[inscript_state_pos][INSCRIPT_DEFAULT]; \
  363.   } \
  364.   assertf(new_state_pos >= 0); \
  365.   assertf(new_state_pos*sizeof(inscript_state[0]) < sizeof(inscript_state)); \
  366.   inscript_state_pos=new_state_pos; \
  367.   } \
  368. } while(0)  
  369.  
  370. /* Increment current pointer to 'steps' characters, modifying automate if necessary */
  371. #define INCREMENT_CURRENT_ADR(steps) do { \
  372.   int steps__ = (steps); \
  373.   while(steps__ > 0) { \
  374.   adr++; \
  375.   AUTOMATE_LOOKUP_CURRENT_ADR(); \
  376.   steps__ --; \
  377.   } \
  378. } while(0)
  379.  
  380.  
  381. /* Main parser */
  382. int htsparse(htsmoduleStruct* str, htsmoduleStructExtended* stre) {
  383.   /* Load engine variables */
  384.   ENGINE_LOAD_CONTEXT();
  385.  
  386. #if HTS_ANALYSTE
  387.   {
  388.     char* cAddr = r->adr;
  389.     int cSize = (int) r->size;
  390.     if ( (opt->debug>0) && (opt->log!=NULL) ) {
  391.       fspc(opt->log,"info"); fprintf(opt->log,"engine: preprocess-html: %s%s"LF, urladr, urlfil);
  392.     }
  393.     if (hts_htmlcheck_preprocess(&cAddr, &cSize, urladr, urlfil) == 1) {
  394.       r->adr = cAddr;
  395.       r->size = cSize;
  396.     }
  397.   }
  398.   if (hts_htmlcheck(r->adr,(int)r->size,urladr,urlfil)) {
  399. #endif          
  400.     FILE* fp=NULL;      // fichier Θcrit localement 
  401.     char* adr=r->adr;    // pointeur (on parcourt)
  402.     char* lastsaved;    // adresse du dernier octet sauvΘ + 1
  403.     if ( (opt->debug>1) && (opt->log!=NULL) ) {
  404.       fspc(opt->log,"debug"); fprintf(opt->log,"scanning file %s%s (%s).."LF, urladr, urlfil, savename); test_flush;
  405.     }
  406.  
  407.  
  408.     // Indexing!
  409. #if HTS_MAKE_KEYWORD_INDEX
  410.     if (opt->kindex) {
  411.       if (index_keyword(r->adr,r->size,r->contenttype,savename,opt->path_html)) {
  412.         if ( (opt->debug>1) && (opt->log!=NULL) ) {
  413.           fspc(opt->log,"debug"); fprintf(opt->log,"indexing file..done"LF); test_flush;
  414.         }
  415.       } else {
  416.         if ( (opt->debug>1) && (opt->log!=NULL) ) {
  417.           fspc(opt->log,"debug"); fprintf(opt->log,"indexing file..error!"LF); test_flush;
  418.         }
  419.       }
  420.     }
  421. #endif
  422.  
  423.     // Now, parsing
  424.     if ((opt->getmode & 1) && (ptr>0)) {  // rΘcupΘrer les html sur disque       
  425.       // crΘer le fichier html local
  426.       HT_ADD_FOP;   // Θcrire peu α peu le fichier
  427.     }
  428.  
  429.     if (!error) {
  430.       int detect_title=0;  // dΘtection  du title
  431.       int back_add_stats = opt->state.back_add_stats;
  432.       //
  433.       char* in_media=NULL; // in other media type (real media and so..)
  434.       int intag=0;         // on est dans un tag
  435.       int incomment=0;     // dans un <!--
  436.       int inscript=0;      // dans un scipt pour applets javascript)
  437.       signed char inscript_state[10][257];
  438.       typedef enum { 
  439.         INSCRIPT_START=0,
  440.         INSCRIPT_ANTISLASH,
  441.         INSCRIPT_INQUOTE,
  442.         INSCRIPT_INQUOTE2,
  443.         INSCRIPT_SLASH,
  444.         INSCRIPT_SLASHSLASH,
  445.         INSCRIPT_COMMENT,
  446.         INSCRIPT_COMMENT2,
  447.         INSCRIPT_ANTISLASH_IN_QUOTE,
  448.         INSCRIPT_ANTISLASH_IN_QUOTE2,
  449.         INSCRIPT_DEFAULT=256
  450.       } INSCRIPT;
  451.       INSCRIPT inscript_state_pos=INSCRIPT_START;
  452.       char* inscript_name=NULL; // script tag name
  453.       int inscript_tag=0;  // on est dans un <body onLoad="... terminΘ par >
  454.       char inscript_tag_lastc='\0';
  455.       // terminaison (" ou ') du "<body onLoad=.."
  456.       int inscriptgen=0;     // on est dans un code gΘnΘrant, ex aprΦs obj.write("..
  457.       //int inscript_check_comments=0, inscript_in_comments=0;    // javascript comments
  458.       char scriptgen_q='\0'; // caractΦre faisant office de guillemet (' ou ")
  459.       //int no_esc_utf=0;      // ne pas echapper chars > 127
  460.       int nofollow=0;        // ne pas scanner
  461.       //
  462.       int parseall_lastc='\0';     // dernier caractΦre parsΘ pour parseall
  463.       //int parseall_incomment=0;   // dans un /* */ (exemple: a = /* URL */ "img.gif";)
  464.       //
  465.       char* intag_start = adr;
  466.             char* intag_name = NULL;
  467.       char* intag_startattr=NULL;
  468.       int intag_start_valid=0;
  469.       int intag_ctype=0;
  470.       //
  471.       int   parent_relative=0;    // the parent is the base path (.js, .css..)
  472.       HT_ADD_START;    // dΘbuter
  473.  
  474.       /* Initialize script automate for comments, quotes.. */
  475.       memset(inscript_state, 0xff, sizeof(inscript_state));
  476.       inscript_state[INSCRIPT_START][INSCRIPT_DEFAULT]=INSCRIPT_START;     /* by default, stay in START */
  477.       inscript_state[INSCRIPT_START]['\\']=INSCRIPT_ANTISLASH;             /* #1: \ escapes the next character whatever it is */
  478.       inscript_state[INSCRIPT_ANTISLASH][INSCRIPT_DEFAULT]=INSCRIPT_START;
  479.       inscript_state[INSCRIPT_START]['\'']=INSCRIPT_INQUOTE;               /* #2: ' opens quote and only ' returns to 0 */
  480.       inscript_state[INSCRIPT_INQUOTE][INSCRIPT_DEFAULT]=INSCRIPT_INQUOTE;
  481.       inscript_state[INSCRIPT_INQUOTE]['\'']=INSCRIPT_START;
  482.       inscript_state[INSCRIPT_INQUOTE]['\\']=INSCRIPT_ANTISLASH_IN_QUOTE;
  483.       inscript_state[INSCRIPT_START]['\"']=INSCRIPT_INQUOTE2;              /* #3: " opens double-quote and only " returns to 0 */
  484.       inscript_state[INSCRIPT_INQUOTE2][INSCRIPT_DEFAULT]=INSCRIPT_INQUOTE2;
  485.       inscript_state[INSCRIPT_INQUOTE2]['\"']=INSCRIPT_START;
  486.       inscript_state[INSCRIPT_INQUOTE2]['\\']=INSCRIPT_ANTISLASH_IN_QUOTE2;
  487.       inscript_state[INSCRIPT_START]['/']=INSCRIPT_SLASH;                  /* #4: / state, default to #0 */
  488.       inscript_state[INSCRIPT_SLASH][INSCRIPT_DEFAULT]=INSCRIPT_START;
  489.       inscript_state[INSCRIPT_SLASH]['/']=INSCRIPT_SLASHSLASH;             /* #5: // with only LF to escape */
  490.       inscript_state[INSCRIPT_SLASHSLASH][INSCRIPT_DEFAULT]=INSCRIPT_SLASHSLASH;
  491.       inscript_state[INSCRIPT_SLASHSLASH]['\n']=INSCRIPT_START;
  492.       inscript_state[INSCRIPT_SLASH]['*']=INSCRIPT_COMMENT;                /* #6: / * with only * / to escape */
  493.       inscript_state[INSCRIPT_COMMENT][INSCRIPT_DEFAULT]=INSCRIPT_COMMENT;
  494.       inscript_state[INSCRIPT_COMMENT]['*']=INSCRIPT_COMMENT2;             /* #7: closing comments */
  495.       inscript_state[INSCRIPT_COMMENT2][INSCRIPT_DEFAULT]=INSCRIPT_COMMENT;
  496.       inscript_state[INSCRIPT_COMMENT2]['/']=INSCRIPT_START;
  497.       inscript_state[INSCRIPT_COMMENT2]['*']=INSCRIPT_COMMENT2;
  498.       inscript_state[INSCRIPT_ANTISLASH_IN_QUOTE][INSCRIPT_DEFAULT]=INSCRIPT_INQUOTE;    /* #8: escape in "" */
  499.       inscript_state[INSCRIPT_ANTISLASH_IN_QUOTE2][INSCRIPT_DEFAULT]=INSCRIPT_INQUOTE2;  /* #9: escape in '' */
  500.  
  501.  
  502.       /* statistics */
  503.       if ((opt->getmode & 1) && (ptr>0)) { 
  504.         /*
  505.         HTS_STAT.stat_files++;
  506.         HTS_STAT.stat_bytes+=r->size;
  507.         */
  508.       }
  509.  
  510.       /* Primary list or URLs */
  511.       if (ptr == 0) {
  512.         intag=1;
  513.         intag_start_valid=0;
  514.                 intag_name = NULL;
  515.       }
  516.       /* Check is the file is a .js file */
  517.       else if (
  518.         (compare_mime(r->contenttype, str->url_file, "application/x-javascript")!=0)
  519.         || (compare_mime(r->contenttype, str->url_file, "text/css")!=0)
  520.         ) {      /* JavaScript js file */
  521.           inscript=1;
  522.           if (opt->parsedebug) { HT_ADD("<@@ inscript @@>"); }
  523.           inscript_name="script";
  524.           intag=1;     // because aprΦs <script> on y est .. - pas utile
  525.           intag_start_valid=0;    // OUI car nous sommes dans du code, plus dans du "vrai" tag
  526.           if ((opt->debug>1) && (opt->log!=NULL)) {
  527.             fspc(opt->log,"debug"); fprintf(opt->log,"note: this file is a javascript file"LF); test_flush;
  528.           }
  529.           // for javascript only
  530.           if (compare_mime(r->contenttype, str->url_file, "application/x-javascript") != 0) {
  531.             // all links must be checked against parent, not this link
  532.             if (liens[ptr]->precedent != 0) {
  533.               parent_relative=1;
  534.             }
  535.           }
  536.         }
  537.         /* Or a real audio */
  538.       else if (compare_mime(r->contenttype, str->url_file, "audio/x-pn-realaudio")!=0) {      /* realaudio link file */
  539.         inscript=intag=0;
  540.         inscript_name="media";
  541.         intag_start_valid=0;
  542.         in_media="LNK";       // real media! -> links
  543.       } 
  544.       /* Or a m3u playlist */
  545.       else if (compare_mime(r->contenttype, str->url_file, "audio/x-mpegurl")!=0) {      /* mp3 link file */
  546.         inscript=intag=0;
  547.         inscript_name="media";
  548.         intag_start_valid=0;
  549.         in_media="LNK";       // m3u! -> links
  550.       } 
  551.       else if (compare_mime(r->contenttype, str->url_file, "application/x-authorware-map")!=0) {      /* macromedia aam file */
  552.         inscript=intag=0;
  553.         inscript_name="media";
  554.         intag_start_valid=0;
  555.         in_media="AAM";       // aam
  556.       } 
  557.  
  558.       // Detect UTF8 format
  559.       //if (is_unicode_utf8((unsigned char*) r->adr, (unsigned int) r->size) == 1) {
  560.       //  no_esc_utf=1;
  561.       //} else {
  562.       //  no_esc_utf=0;
  563.       //}
  564.  
  565.             // Hack to prevent any problems with ram files of other files
  566.       * ( r->adr + r->size ) = '\0';
  567.  
  568.       // ------------------------------------------------------------
  569.       // analyser ce qu'il y a en mΘmoire (fichier html)
  570.       // on scanne les balises
  571.       // ------------------------------------------------------------
  572. #if HTS_ANALYSTE
  573.       _hts_in_html_done=0;     // 0% scannΘs
  574.       _hts_cancel=0;           // pas de cancel
  575.       _hts_in_html_parsing=1;  // flag pour indiquer un parsing
  576. #endif
  577.       base[0]='\0';    // effacer base-href
  578.       lastsaved=adr;
  579.       do {
  580.         int p=0;
  581.         int valid_p=0;      // force to take p even if == 0
  582.         int ending_p='\0';  // ending quote?
  583.         int archivetag_p=0;  // avoid multiple-archives with commas
  584.         int  unquoted_script=0;
  585.         INSCRIPT inscript_state_pos_prev=inscript_state_pos;
  586.         error=0;
  587.  
  588.         /* Hack to avoid NULL char problems with C syntax */
  589.         /* Yes, some bogus HTML pages can embed null chars
  590.         and therefore can not be properly handled if this hack is not done
  591.         */
  592.         if ( ! (*adr) ) {
  593.           if ( ((int) (adr - r->adr)) < r->size)
  594.             *adr=' ';
  595.         }
  596.  
  597.  
  598.  
  599.         /*
  600.         index.html built here
  601.         */
  602.         // Construction index.html (sommaire)
  603.         // Avant de tester les a href,
  604.         // Ici on teste si l'on doit construire l'index vers le(s) site(s) miroir(s)
  605.         if (!makeindex_done) {  // autoriation d'Θcrire un index
  606.           if (!detect_title) {
  607.             if (opt->depth == liens[ptr]->depth) {    // on note toujours les premiers liens
  608.               if (!in_media) {
  609.                 if (opt->makeindex && (ptr>0)) {
  610.                   if (opt->getmode & 1) {  // autorisation d'Θcrire
  611.                     p=strfield(adr,"title");  
  612.                     if (p) {
  613.                       if (*(adr-1)=='/') p=0;    // /title
  614.                     } else {
  615.                       if (strfield(adr,"/html"))
  616.                         p=-1;                    // noter, mais sans titre
  617.                       else if (strfield(adr,"body"))
  618.                         p=-1;                    // noter, mais sans titre
  619.                       else if ( ((int) (adr - r->adr) ) >= (r->size-1) )
  620.                         p=-1;                    // noter, mais sans titre
  621.                       else if ( (int) (adr - r->adr) >= r->size - 2)   // we got to hurry
  622.                         p=-1; // xxc xxc xxc
  623.                     }
  624.                   } else
  625.                     p=0;
  626.  
  627.                   if (p) {    // ok center                            
  628.                     if (makeindex_fp==NULL) {
  629.                       file_notify("", "", fconcat(opt->path_html,"index.html"), 1, 1, 0);
  630.                       verif_backblue(opt,opt->path_html);    // gΘnΘrer gif
  631.                       makeindex_fp=filecreate(fconcat(opt->path_html,"index.html"));
  632.                       if (makeindex_fp!=NULL) {
  633.  
  634.                         // Header
  635.                         fprintf(makeindex_fp,template_header,
  636.                           "<!-- Mirror and index made by HTTrack Website Copier/"HTTRACK_VERSION" "HTTRACK_AFF_AUTHORS" -->"
  637.                           );
  638.  
  639.                       } else makeindex_done=-1;    // fait, erreur
  640.                     }
  641.  
  642.                     if (makeindex_fp!=NULL) {
  643.                       char BIGSTK tempo[HTS_URLMAXSIZE*2];
  644.                       char BIGSTK s[HTS_URLMAXSIZE*2];
  645.                       char* a=NULL;
  646.                       char* b=NULL;
  647.                       s[0]='\0';
  648.                       if (p>0) {
  649.                         a=strchr(adr,'>');
  650.                         if (a!=NULL) {
  651.                           a++;
  652.                           while(is_space(*a)) a++;    // sauter espaces & co
  653.                           b=strchr(a,'<');   // prochain tag
  654.                         }
  655.                       }
  656.                       if (lienrelatif(tempo,liens[ptr]->sav,concat(opt->path_html,"index.html"))==0) {
  657.                         detect_title=1;      // ok dΘtectΘ pour cette page!
  658.                         makeindex_links++;   // un de plus
  659.                         strcpybuff(makeindex_firstlink,tempo);
  660.                         //
  661.  
  662.                         /* Hack */
  663.                         if (opt->mimehtml) {
  664.                           strcpybuff(makeindex_firstlink, "cid:primary/primary");
  665.                         }
  666.  
  667.                         if ((b==a) || (a==NULL) || (b==NULL)) {    // pas de titre
  668.                           strcpybuff(s,tempo);
  669.                         } else if ((b-a)<256) {
  670.                           b--;
  671.                           while(is_space(*b)) b--;
  672.                           strncpy(s,a,b-a+1);
  673.                           *(s+(b-a)+1)='\0';
  674.                         }
  675.  
  676.                         // Body
  677.                         fprintf(makeindex_fp,template_body,
  678.                           tempo,
  679.                           s
  680.                           );
  681.  
  682.                       }
  683.                     }
  684.                   }
  685.                 }
  686.               }
  687.  
  688.             } else if (liens[ptr]->depth<opt->depth) {   // on a sautΘ level1+1 et level1
  689.               HT_INDEX_END;
  690.             }
  691.           } // if (opt->makeindex)
  692.         }
  693.         // FIN Construction index.html (sommaire)
  694.         /*
  695.         end -- index.html built here
  696.         */
  697.  
  698.  
  699.  
  700.         /* Parse */
  701.         if (
  702.           (*adr=='<')    /* No starting tag */
  703.           && (!inscript)    /* Not in (java)script */
  704.           && (!incomment)   /* Not in comment (<!--) */
  705.           && (!in_media)    /* Not in media */
  706.           ) { 
  707.             intag=1;
  708.                         intag_ctype=0;
  709.                         //parseall_incomment=0;
  710.                         //inquote=0;  // effacer quote
  711.                         intag_start = adr;
  712.                         for(intag_name = adr + 1 ; is_realspace(*intag_name) ; intag_name++ );
  713.                         intag_start_valid = 1;
  714.                         codebase[0]='\0';    // effacer Θventuel codebase
  715.  
  716.                         /* Meta ? */
  717.                         if (check_tag(intag_start, "meta")) {
  718.                             int pos;
  719.                             // <meta http-equiv="Content-Type" content="text/html; charset=UTF-8" />
  720.                             if ((pos = rech_tageq_all(adr, "http-equiv"))) {
  721.                                 const char* token = NULL;
  722.                                 int len = rech_endtoken(adr + pos, &token);
  723.                                 if (len > 0) {
  724.                                     if (strfield(token, "content-type")) {
  725.                                         intag_ctype=1;
  726.                                     }
  727.                                     else if (strfield(token, "refresh")) {
  728.                                         intag_ctype=2;
  729.                                     }
  730.                                 }
  731.                             }
  732.                         }
  733.  
  734.             if (opt->getmode & 1) {  // sauver html
  735.               p=strfield(adr,"</html");
  736.               if (p==0) p=strfield(adr,"<head>");
  737.               // if (p==0) p=strfield(adr,"<doctype");
  738.               if (p) {
  739.                 char* eol="\n";
  740.                 if (strchr(r->adr,'\r'))
  741.                   eol="\r\n";
  742.                 if (strnotempty(opt->footer)) {
  743.                   char BIGSTK tempo[1024+HTS_URLMAXSIZE*2];
  744.                   char gmttime[256];
  745.                   tempo[0]='\0';
  746.                   time_gmt_rfc822(gmttime);
  747.                   strcatbuff(tempo,eol);
  748.                   sprintf(tempo+strlen(tempo),opt->footer,jump_identification(urladr),urlfil,gmttime,HTTRACK_VERSIONID,"","","","","","","");
  749.                   strcatbuff(tempo,eol);
  750.                   //fwrite(tempo,1,strlen(tempo),fp);
  751.                   HT_ADD(tempo);
  752.                   if (r->charset[0]) {
  753.                     HT_ADD("<!-- Added by HTTrack --><meta http-equiv=\"content-type\" content=\"text/html;charset=");
  754.                     HT_ADD(r->charset);
  755.                     HT_ADD("\"><!-- /Added by HTTrack -->");
  756.                     HT_ADD(eol);
  757.                   }
  758.                 }
  759.               }
  760.             }        
  761.  
  762.             // Θliminer les <!-- (commentaires) : intag dΘvalidΘ
  763.             if (*(adr+1)=='!')
  764.               if (*(adr+2)=='-')
  765.                 if (*(adr+3)=='-') {
  766.                   intag=0;
  767.                   incomment=1;
  768.                   intag_start_valid=0;
  769.                 }
  770.  
  771.           }
  772.         else if (
  773.           (*adr=='>')                        /* ending tag */
  774.           && ( (!inscript && !in_media) || (inscript_tag) )  /* and in tag (or in script) */
  775.           ) {
  776.             if (inscript_tag) {
  777.               inscript_tag=inscript=0;
  778.               intag=0;
  779.               incomment=0;
  780.               intag_start_valid=0;
  781.                             intag_name = NULL;
  782.               if (opt->parsedebug) { HT_ADD("<@@ /inscript @@>"); }
  783.             } else if (!incomment) {
  784.               intag=0; //inquote=0;
  785.  
  786.               // entrΘe dans du javascript?
  787.               // on parse ICI car il se peut qu'on ait eu a parser les src=.. dedans
  788.               //if (!inscript) {  // sinon on est dans un obj.write("..
  789.               if ((intag_start_valid) && 
  790.                 (
  791.                 check_tag(intag_start,"script")
  792.                 ||
  793.                 check_tag(intag_start,"style")
  794.                 )
  795.                 ) {
  796.                   char* a=intag_start;    // <
  797.                   // ** while(is_realspace(*(--a)));
  798.                   if (*a=='<') {  // s√r que c'est un tag?
  799.                     if (check_tag(intag_start,"script"))
  800.                       inscript_name="script";
  801.                     else
  802.                       inscript_name="style";
  803.                     inscript=1;
  804.                     inscript_state_pos=INSCRIPT_START;
  805.                     intag=1;     // because aprΦs <script> on y est .. - pas utile
  806.                     intag_start_valid=0;    // OUI car nous sommes dans du code, plus dans du "vrai" tag
  807.                     if (opt->parsedebug) { HT_ADD("<@@ inscript @@>"); }
  808.                   }
  809.                 }
  810.             } else {                               /* end of comment? */
  811.               // vΘrifier fermeture correcte
  812.               if ( (*(adr-1)=='-') && (*(adr-2)=='-') ) {
  813.                 intag=0;
  814.                 incomment=0;
  815.                 intag_start_valid=0;
  816.                                 intag_name = NULL;
  817.               }
  818. #if GT_ENDS_COMMENT
  819.               /* wrong comment ending */
  820.               else {
  821.                 /* check if correct ending does not exists
  822.                 <!-- foo > example <!-- bar > is sometimes accepted by browsers
  823.                 when no --> is used somewhere else.. darn those browsers are dirty
  824.                 */
  825.                 if (!strstr(adr,"-->")) {
  826.                   intag=0;
  827.                   incomment=0;
  828.                   intag_start_valid=0;
  829.                                     intag_name = NULL;
  830.                 }
  831.               }
  832. #endif
  833.             }
  834.             //}
  835.           }
  836.           //else if (*adr==34) {
  837.           //  inquote=(inquote?0:1);
  838.           //}
  839.         else if (intag || inscript || in_media) {    // nous sommes dans un tag/commentaire, tester si on recoit un tag
  840.           int p_type=0;
  841.           int p_nocatch=0;
  842.           int p_searchMETAURL=0;  // chercher ..URL=<url>
  843.           int add_class=0;        // ajouter .class
  844.           int add_class_dots_to_patch=0;   // number of '.' in code="x.y.z<realname>"
  845.           char* p_flush=NULL;
  846.  
  847.  
  848.           // ------------------------------------------------------------
  849.           // parsing ΘvolΘ
  850.           // ------------------------------------------------------------
  851.           if (((isalpha((unsigned char)*adr)) || (*adr=='/') || (inscript) || (in_media) || (inscriptgen))) {  // sinon pas la peine de tester..
  852.  
  853.  
  854.             /* caractΦre de terminaison pour "miniparsing" javascript=.. ? 
  855.             (ex: <a href="javascript:()" action="foo"> ) */
  856.             if (inscript_tag) {
  857.               if (inscript_tag_lastc) {
  858.                 if (*adr == inscript_tag_lastc) {
  859.                   /* sortir */
  860.                   inscript_tag=inscript=0;
  861.                   incomment=0;
  862.                   if (opt->parsedebug) { HT_ADD("<@@ /inscript @@>"); }
  863.                 }
  864.               }
  865.             }
  866.  
  867.             /* automate */
  868.             AUTOMATE_LOOKUP_CURRENT_ADR();
  869.  
  870.  
  871.             // Note:
  872.             // Certaines pages ne respectent pas le html
  873.             // notamment les guillements ne sont pas fixΘs
  874.             // Nous sommes dans un tag, donc on peut faire un test plus
  875.             // large pour pouvoi prendre en compte ces particularitΘs
  876.  
  877.             // α vΘrifier: ACTION, CODEBASE, VRML
  878.  
  879.             if (in_media) {
  880.               if (strcmp(in_media,"LNK")==0) { // real media
  881.                 p=0;
  882.                 valid_p=1;
  883.               }
  884.               else if (strcmp(in_media, "AAM")==0) { // AAM
  885.                 if (is_space((unsigned char)adr[0]) && ! is_space((unsigned char)adr[1])) {
  886.                   char* a = adr + 1;
  887.                   int n = 0;
  888.                   int ok = 0;
  889.                   int dot = 0;
  890.                   while(n < HTS_URLMAXSIZE/2 && a[n] != '\0' &&
  891.                     ( ! is_space((unsigned char)a[n]) || ! ( ok = 1) )
  892.                     ) {
  893.                       if (a[n] == '.') {
  894.                         dot = n;
  895.                       }
  896.                       n++;
  897.                     }
  898.                     if (ok && dot > 0) {
  899.                       char BIGSTK tmp[HTS_URLMAXSIZE/2 + 2];
  900.                       tmp[0] = '\0';
  901.                       strncat(tmp, a + dot + 1, n - dot - 1);
  902.                       if (is_knowntype(tmp) || ishtml_ext(tmp) != -1) {
  903.                         adr++;
  904.                         p = 0;
  905.                         valid_p = 1;
  906.                         unquoted_script = 1;
  907.                       }
  908.                     }
  909.                 }
  910.               }
  911.             } else if (ptr>0) {        /* pas premiΦre page 0 (primary) */
  912.               p=0;  // saut pour le nom de fichier: adresse nom fichier=adr+p
  913.  
  914.               // ------------------------------
  915.               // dΘtection d'Θcriture JavaScript.
  916.               // osons les obj.write et les obj.href=.. ! osons!
  917.               // note: inscript==1 donc on sautera aprΦs les \"
  918.               if (inscript) {
  919.                 if (inscriptgen) {          // on est dΘja dans un objet gΘnΘrant..
  920.                   if (*adr==scriptgen_q) {  // fermeture des " ou '
  921.                     if (*(adr-1)!='\\') {   // non
  922.                       inscriptgen=0;        // ok parsing terminΘ
  923.                     }
  924.                   }
  925.                 } else {
  926.                   char* a=NULL;
  927.                   char check_this_fking_line=0;  // parsing code javascript..
  928.                   char must_be_terminated=0;     // caractΦre obligatoire de terminaison!
  929.                   int token_size;
  930.                   if (!(token_size=strfield(adr,".writeln"))) // dΘtection ...objet.write[ln]("code html")...
  931.                     token_size=strfield(adr,".write");
  932.                   if (token_size) {
  933.                     a=adr+token_size;
  934.                     while(is_realspace(*a)) a++; // sauter espaces
  935.                     if (*a=='(') {  // dΘbut parenthΦse
  936.                       check_this_fking_line=2;  // α parser!
  937.                       must_be_terminated=')';
  938.                       a++;  // sauter (
  939.                     }
  940.                   }
  941.                   // euhh ??? ???
  942.                   /* else if (strfield(adr,".href")) {  // dΘtection ...objet.href="...
  943.                   a=adr+5;
  944.                   while(is_realspace(*a)) a++; // sauter espaces
  945.                   if (*a=='=') {  // ohh un Θgal
  946.                   check_this_fking_line=1;  // α noter!
  947.                   must_be_terminated=';';   // et si t'as oubliΘ le ; tu sais pas coder
  948.                   a++;   // sauter =
  949.                   }
  950.  
  951.                   }*/
  952.  
  953.                   // on a un truc du genre instruction"code gΘnΘrΘ" dont on parse le code
  954.                   if (check_this_fking_line) {
  955.                     while(is_realspace(*a)) a++;
  956.                     if ((*a=='\'') || (*a=='"')) {  // dΘpart de '' ou ""
  957.                       char *b;
  958.                       scriptgen_q=*a;    // quote
  959.                       b=a+1;      // dΘpart de la chaεne
  960.                       // vΘrifier forme ("code") et pas ("code"+var), ingΘrable
  961.                       do {
  962.                         if (*a==scriptgen_q && *(a-1)!='\\')  // quote non slash
  963.                           break;            // sortie
  964.                         else if (*a==10 && *(a-1) != '\\'  /* LF and no continue (\) character */
  965.                           && ( *(a-1) != '\r' || *(a-2) != '\\' ) )  /* and not CRLF and no .. */
  966.                           break;
  967.                         else 
  968.                           a++;  // caractΦre suivant
  969.                       } while((a-b) < HTS_URLMAXSIZE / 2);
  970.                       if (*a==scriptgen_q) {  // fin du quote
  971.                         a++;
  972.                         while(is_realspace(*a)) a++;
  973.                         if (*a==must_be_terminated) {  // parenthΦse fermante: ("..")
  974.  
  975.                           // bon, on doit parser une ligne javascript
  976.                           // 1) si check.. ==1 alors c'est un nom de fichier direct, donc
  977.                           // on fixe p sur le saut nΘcessaire pour atteindre le nom du fichier
  978.                           // et le moteur se dΘbrouillera ensuite tout seul comme un grand
  979.                           // 2) si check==2 c'est un peu plus tordu car lα on gΘnΘre du
  980.                           // code html au sein de code javascript au sein de code html
  981.                           // dans ce cas on doit fixer un flag α un puis ensuite dans la boucle
  982.                           // on devra parser les instructions standard comme <a href etc
  983.                           // NOTE: le code javascript autogΘnΘrΘ n'est pas pris en compte!!
  984.                           // (et ne marche pas dans 50% des cas de toute facon!)
  985.                           if (check_this_fking_line==1) {
  986.                             p=(int) (b - adr);    // calculer saut!
  987.                           } else {
  988.                             inscriptgen=1;        // SCRIPTGEN actif
  989.                             adr=b;                // jump
  990.                           }
  991.  
  992.                           if ((opt->debug>1) && (opt->log!=NULL)) {
  993.                             char str[512];
  994.                             str[0]='\0';
  995.                             strncatbuff(str,b,minimum((int) (a - b + 1), 32));
  996.                             fspc(opt->log,"debug"); fprintf(opt->log,"active code (%s) detected in javascript: %s"LF,(check_this_fking_line==2)?"parse":"pickup",str); test_flush;
  997.                           }
  998.                         }
  999.  
  1000.                       }
  1001.  
  1002.                     }
  1003.  
  1004.  
  1005.                   }
  1006.                 }
  1007.               }
  1008.               // fin detection code gΘnΘrant javascript vers html
  1009.               // ------------------------------
  1010.  
  1011.  
  1012.               // analyse proprement dite, A HREF=.. etc..
  1013.               if (!p) {
  1014.                 // si dans un tag, et pas dans un script - sauf si on analyse un obj.write("..
  1015.                 if ((intag && (!inscript)) || inscriptgen) {
  1016.                   if ( (*(adr-1)=='<') || (is_space(*(adr-1))) ) {   // <tag < tag etc
  1017.                     // <A HREF=.. pour les liens HTML
  1018.                     p=rech_tageq(adr,"href");
  1019.                     if (p) {    // href.. tester si c'est une bas href!
  1020.                       if ((intag_start_valid) && check_tag(intag_start, "base")) {  // oui!
  1021.                         // ** note: base href et codebase ne font pas bon mΘnage..
  1022.                         p_type=2;    // c'est un chemin
  1023.                       }
  1024.                     }
  1025.  
  1026.                     /* Tags supplΘmentaires α vΘrifier (<img src=..> etc) */
  1027.                     if (p==0) {
  1028.                       int i=0;
  1029.                       while( (p==0) && (strnotempty(hts_detect[i])) ) {
  1030.                         p=rech_tageq(adr,hts_detect[i]);
  1031.                         if (p) {
  1032.                           /* This is a temporary hack to avoid archive=foo.jar,bar.jar .. */
  1033.                           if (strcmp(hts_detect[i], "archive") == 0) {
  1034.                             archivetag_p = 1;
  1035.                           }
  1036.                         }
  1037.                         i++;
  1038.                       }
  1039.                     }
  1040.  
  1041.                     /* Tags supplΘmentaires en dΘbut α vΘrifier (<object .. hotspot1=..> etc) */
  1042.                     if (p==0) {
  1043.                       int i=0;
  1044.                       while( (p==0) && (strnotempty(hts_detectbeg[i])) ) {
  1045.                         p=rech_tageqbegdigits(adr,hts_detectbeg[i]);
  1046.                         i++;
  1047.                       }
  1048.                     }
  1049.  
  1050.                     /* Tags supplΘmentaires α vΘrifier : URL=.. */
  1051.                     if (p==0) {
  1052.                       int i=0;
  1053.                       while( (p==0) && (strnotempty(hts_detectURL[i])) ) {
  1054.                         p=rech_tageq(adr,hts_detectURL[i]);
  1055.                         i++;
  1056.                       }
  1057.                       if (p) {
  1058.                         if (intag_ctype == 1) {
  1059.                           p = 0;
  1060. #if 0
  1061.                           //if ((pos=rech_tageq(adr, "content"))) {
  1062.                           char temp[256];
  1063.                           char* token = NULL;
  1064.                           int len = rech_endtoken(adr + pos, &token);
  1065.                           if (len > 0 && len < sizeof(temp) - 2) {
  1066.                             char* chpos;
  1067.                             temp[0] = '\0';
  1068.                             strncat(temp, token, len);
  1069.                             if ((chpos = strstr(temp, "charset"))
  1070.                               &&
  1071.                               (chpos = strchr(chpos, '='))
  1072.                               ) {
  1073.                                 chpos++;
  1074.                                 while(is_space(*chpos)) chpod++;
  1075.                                 chpos
  1076.                               }
  1077.                           }
  1078. #endif
  1079.                         }
  1080.                         // <META HTTP-EQUIV="Refresh" CONTENT="3;URL=http://www.example.com">
  1081.                         else if (intag_ctype == 2) {
  1082.                           p_searchMETAURL=1;
  1083.                         } else {
  1084.                           p = 0;            /* cancel */
  1085.                         }
  1086.                       }
  1087.  
  1088.  
  1089.                     }
  1090.  
  1091.                     /* Tags supplΘmentaires α vΘrifier, mais α ne pas capturer */
  1092.                     if (p==0) {
  1093.                       int i=0;
  1094.                       while( (p==0) && (strnotempty(hts_detectandleave[i])) ) {
  1095.                         p=rech_tageq(adr,hts_detectandleave[i]);
  1096.                         i++;
  1097.                       }
  1098.                       if (p)
  1099.                         p_nocatch=1;      /* ne pas rechercher */
  1100.                     }
  1101.  
  1102.                     /* EvΘnements */
  1103.                     if (p==0 && 
  1104.                       ! inscript          /* we don't want events inside document.write */
  1105.                       ) {
  1106.                         int i=0;
  1107.                         /* dΘtection onLoad etc */
  1108.                         while( (p==0) && (strnotempty(hts_detect_js[i])) ) {
  1109.                           p=rech_tageq(adr,hts_detect_js[i]);
  1110.                           i++;
  1111.                         }
  1112.                         /* non dΘtectΘ - dΘtecter Θgalement les onXxxxx= */
  1113.                         if (p==0) {
  1114.                           if ( (*adr=='o') && (*(adr+1)=='n') && isUpperLetter(*(adr+2)) ) {
  1115.                             p=0;
  1116.                             while(isalpha((unsigned char)adr[p]) && (p<64) ) p++;
  1117.                             if (p<64) {
  1118.                               while(is_space(adr[p])) p++;
  1119.                               if (adr[p]=='=')
  1120.                                 p++;
  1121.                               else p=0;
  1122.                             } else p=0;
  1123.                           }
  1124.                         }
  1125.                         /* OK, ΘvΘnement repΘrΘ */
  1126.                         if (p) {
  1127.                           inscript_tag_lastc=*(adr+p);     /* α attendre α la fin */
  1128.                           adr+=p+1;   /* saut */
  1129.                           /*
  1130.                           On est dΘsormais dans du code javascript
  1131.                           */
  1132.                           inscript_name="";
  1133.                           inscript=inscript_tag=1;
  1134.                           inscript_state_pos=INSCRIPT_START;
  1135.                           if (opt->parsedebug) { HT_ADD("<@@ inscript @@>"); }
  1136.                         }
  1137.                         p=0;        /* quoi qu'il arrive, ne rien dΘmarrer ici */
  1138.                       }
  1139.  
  1140.                       // <APPLET CODE=.. pour les applet java.. [CODEBASE (chemin..) α faire]
  1141.                       if (p==0) {
  1142.                         p=rech_tageq(adr,"code");
  1143.                         if (p) {
  1144.                           if ((intag_start_valid) && check_tag(intag_start,"applet")) {  // dans un <applet !
  1145.                             p_type=-1;  // juste le nom de fichier+dossier, Θcire avant codebase 
  1146.                             add_class=1;   // ajouter .class au besoin                         
  1147.  
  1148.                             // vΘrifier qu'il n'y a pas de codebase APRES
  1149.                             // sinon on swappe les deux.
  1150.                             // pas trΦs propre mais c'est ce qu'il y a de plus simple α faire!!
  1151.  
  1152.                             {
  1153.                               char *a;
  1154.                               a=adr;
  1155.                               while((*a) && (*a!='>') && (!rech_tageq(a,"codebase"))) a++;
  1156.                               if (rech_tageq(a,"codebase")) {  // banzai! codebase=
  1157.                                 char* b;
  1158.                                 b=strchr(a,'>');
  1159.                                 if (b) {
  1160.                                   if (((int) (b - adr)) < 1000) {    // au total < 1Ko
  1161.                                     char BIGSTK tempo[HTS_URLMAXSIZE*2];
  1162.                                     tempo[0]='\0';
  1163.                                     strncatbuff(tempo,a,(int) (b - a) );
  1164.                                     strcatbuff( tempo," ");
  1165.                                     strncatbuff(tempo,adr,(int) (a - adr - 1));
  1166.                                     // Θventuellement remplire par des espaces pour avoir juste la taille
  1167.                                     while((int) strlen(tempo)<((int) (b - adr)))
  1168.                                       strcatbuff(tempo," ");
  1169.                                     // pas d'erreur?
  1170.                                     if ((int) strlen(tempo) == ((int) (b - adr) )) {
  1171.                                       strncpy(adr,tempo,strlen(tempo));   // PAS d'octet nul α la fin!
  1172.                                       p=0;    // DEVALIDER!!
  1173.                                       p_type=0;
  1174.                                       add_class=0;
  1175.                                     }
  1176.                                   }
  1177.                                 }
  1178.                               }
  1179.                             }
  1180.  
  1181.                           }
  1182.                         }
  1183.                       }
  1184.  
  1185.                       // liens α patcher mais pas α charger (ex: codebase)
  1186.                       if (p==0) {  // note: si non chargΘ (ex: ignorer .class) patchΘ tout de mΩme
  1187.                         p=rech_tageq(adr,"codebase");
  1188.                         if (p) {
  1189.                           if ((intag_start_valid) && check_tag(intag_start,"applet")) {  // dans un <applet !
  1190.                             p_type=-2;
  1191.                           } else p=-1;   // ne plus chercher
  1192.                         }
  1193.                       }
  1194.  
  1195.  
  1196.                       // Meta tags pour robots
  1197.                       if (p==0) {
  1198.                         if (opt->robots) {
  1199.                           if ((intag_start_valid) && check_tag(intag_start,"meta")) {
  1200.                             if (rech_tageq(adr,"name")) {    // name=robots.txt
  1201.                               char tempo[1100];
  1202.                               char* a;
  1203.                               tempo[0]='\0';
  1204.                               a=strchr(adr,'>');
  1205. #if DEBUG_ROBOTS
  1206.                               printf("robots.txt meta tag detected\n");
  1207. #endif
  1208.                               if (a) {
  1209.                                 if (((int) (a - adr)) < 999 ) {
  1210.                                   strncatbuff(tempo,adr,(int) (a - adr));
  1211.                                   if (strstrcase(tempo,"content")) {
  1212.                                     if (strstrcase(tempo,"robots")) {
  1213.                                       if (strstrcase(tempo,"nofollow")) {
  1214. #if DEBUG_ROBOTS
  1215.                                         printf("robots.txt meta tag: nofollow in %s%s\n",urladr,urlfil);
  1216. #endif
  1217.                                         nofollow=1;       // NE PLUS suivre liens dans cette page
  1218.                                         if (opt->errlog) {
  1219.                                           fspc(opt->errlog,"warning"); fprintf(opt->errlog,"Link %s%s not scanned (follow robots meta tag)"LF,urladr,urlfil);
  1220.                                           test_flush;
  1221.                                         }
  1222.                                       }
  1223.                                     }
  1224.                                   }
  1225.                                 }
  1226.                               }
  1227.                             }
  1228.                           }
  1229.                         }
  1230.                       }
  1231.  
  1232.                                             // entrΘe dans une applet javascript
  1233.                       /*if (!inscript) {  // sinon on est dans un obj.write("..
  1234.                       if (p==0)
  1235.                       if (rech_sampletag(adr,"script"))
  1236.                       if (check_tag(intag_start,"script")) {
  1237.                       inscript=1;
  1238.                       }
  1239.                       }*/
  1240.  
  1241.                       // Ici on procΦde α une analyse du code javascript pour tenter de rΘcupΘrer
  1242.                       // certains fichiers Θvidents.
  1243.                       // C'est devenu obligatoire vu le nombre de pages qui intΦgrent
  1244.                       // des images rΘactives par exemple
  1245.                   }
  1246.                 } else if (inscript) {
  1247.  
  1248. #if 0
  1249.                   /* Check // javascript comments */
  1250.                   if (*adr == 10 || *adr == 13) {
  1251.                     inscript_check_comments = 1;
  1252.                     inscript_in_comments = 0;
  1253.                   }
  1254.                   else if (inscript_check_comments) {
  1255.                     if (!is_realspace(*adr)) {
  1256.                       inscript_check_comments = 0;
  1257.                       if (adr[0] == '/' && adr[1] == '/') {
  1258.                         inscript_in_comments = 1;
  1259.                       }
  1260.                     }
  1261.                   }
  1262. #endif
  1263.  
  1264.                   /* Parse */
  1265.                   assertf(inscript_name != NULL);
  1266.                   if (
  1267.                     *adr == '/' &&
  1268.                     (
  1269.                     (strfield(adr,"/script") && strfield(inscript_name, "script"))
  1270.                     ||
  1271.                     (strfield(adr,"/style")  && strfield(inscript_name, "style"))
  1272.                     )
  1273.                     ) {
  1274.                       char* a=adr;
  1275.                       //while(is_realspace(*(--a)));
  1276.                       while( is_realspace(*a) ) a--;
  1277.                       a--;
  1278.                       if (*a=='<') {  // s√r que c'est un tag?
  1279.                         inscript=0;
  1280.                         if (opt->parsedebug) { HT_ADD("<@@ /inscript @@>"); }
  1281.                       }
  1282.                     } else if (inscript_state_pos == INSCRIPT_START /*!inscript_in_comments*/) {
  1283.                       /*
  1284.                       Script Analyzing - different types supported:
  1285.                       foo="url"
  1286.                       foo("url") or foo(url)
  1287.                       foo "url"
  1288.                       */
  1289.                       int nc;
  1290.                       char  expected     = '=';          // caractΦre attendu aprΦs
  1291.                       char* expected_end = ";";
  1292.                       int can_avoid_quotes=0;
  1293.                       char quotes_replacement='\0';
  1294.                       int ensure_not_mime=0;
  1295.                       if (inscript_tag)
  1296.                         expected_end=";\"\'";            // voir a href="javascript:doc.location='foo'"
  1297.                       nc = strfield(adr,".src");  // nom.src="image";
  1298.                       if (!nc) nc = strfield(adr,".location");  // document.location="doc"
  1299.                       if (!nc) nc = strfield(adr,":location");  // javascript:location="doc"
  1300.                                             if (!nc) { // location="doc"
  1301.                                                 if ( ( nc = strfield(adr,"location") ) 
  1302.                                                     && !isspace(*(adr - 1))
  1303.                                                     )
  1304.                                                     nc = 0;
  1305.                                             }
  1306.                       if (!nc) nc = strfield(adr,".href");  // document.location="doc"
  1307.                       if (!nc) if ( (nc = strfield(adr,".open")) ) { // window.open("doc",..
  1308.                         expected='(';    // parenthΦse
  1309.                         expected_end="),";  // fin: virgule ou parenthΦse
  1310.                         ensure_not_mime=1;  //* ensure the url is not a mime type */
  1311.                       }
  1312.                       if (!nc) if ( (nc = strfield(adr,".replace")) ) { // window.replace("url")
  1313.                         expected='(';    // parenthΦse
  1314.                         expected_end=")";  // fin: parenthΦse
  1315.                       }
  1316.                       if (!nc) if ( (nc = strfield(adr,".link")) ) { // window.link("url")
  1317.                         expected='(';    // parenthΦse
  1318.                         expected_end=")";  // fin: parenthΦse
  1319.                       }
  1320.                       if (!nc) if ( (nc = strfield(adr,"url")) && (!isalnum(*(adr - 1))) 
  1321.                         && *(adr - 1) != '_'
  1322.                         ) { // url(url)
  1323.                           expected='(';    // parenthΦse
  1324.                           expected_end=")";  // fin: parenthΦse
  1325.                           can_avoid_quotes=1;
  1326.                           quotes_replacement=')';
  1327.                         }
  1328.                         if (!nc) if ( (nc = strfield(adr,"import")) ) { // import "url"
  1329.                           if (is_space(*(adr+nc))) {
  1330.                             expected=0;    // no char expected
  1331.                           } else
  1332.                             nc=0;
  1333.                         }
  1334.                         if (nc) {
  1335.                           char *a;
  1336.                           a=adr+nc;
  1337.                           while(is_realspace(*a)) a++;
  1338.                           if ((*a == expected) || (!expected)) {
  1339.                             if (expected)
  1340.                               a++;
  1341.                             while(is_realspace(*a)) a++;
  1342.                             if ((*a==34) || (*a=='\'') || (can_avoid_quotes)) {
  1343.                               char *b,*c;
  1344.                               int ndelim=1;
  1345.                               if ((*a==34) || (*a=='\''))
  1346.                                 a++;
  1347.                               else
  1348.                                 ndelim=0;
  1349.                               b=a;
  1350.                               if (ndelim) {
  1351.                                 while((*b!=34) && (*b!='\'') && (*b!='\0')) b++;
  1352.                               }
  1353.                               else {
  1354.                                 while((*b != quotes_replacement) && (*b!='\0')) b++;
  1355.                               }
  1356.                               c=b--; c+=ndelim;
  1357.                               while(*c==' ') c++;
  1358.                               if ((strchr(expected_end,*c)) || (*c=='\n') || (*c=='\r')) {
  1359.                                 c-=(ndelim+1);
  1360.                                 if ((int) (c - a + 1)) {
  1361.                                   if (ensure_not_mime) {
  1362.                                     int i = 0;
  1363.                                     while(a != NULL && hts_main_mime[i] != NULL && hts_main_mime[i][0] != '\0') {
  1364.                                       int p;
  1365.                                       if ((p=strfield(a, hts_main_mime[i])) && a[p] == '/') {
  1366.                                         a=NULL;
  1367.                                       }
  1368.                                       i++;
  1369.                                     }
  1370.                                   }
  1371.                                   if (a != NULL) {
  1372.                                     if ((opt->debug>1) && (opt->log!=NULL)) {
  1373.                                       char str[512];
  1374.                                       str[0]='\0';
  1375.                                       strncatbuff(str,a,minimum((int) (c - a + 1),32));
  1376.                                       fspc(opt->log,"debug"); fprintf(opt->log,"link detected in javascript: %s"LF,str); test_flush;
  1377.                                     }
  1378.                                     p=(int) (a - adr);    // p non nul: TRAITER CHAINE COMME FICHIER
  1379.                                     if (can_avoid_quotes) {
  1380.                                       ending_p=quotes_replacement;
  1381.                                     }
  1382.                                   }
  1383.                                 }
  1384.                               }
  1385.  
  1386.  
  1387.                             }
  1388.                           }
  1389.                         }
  1390.  
  1391.                     }
  1392.                 }
  1393.               }
  1394.  
  1395.             } else {      // ptr == 0
  1396.               //p=rech_tageq(adr,"primary");    // lien primaire, yeah
  1397.               p=0;          // No stupid tag anymore, raw link
  1398.               valid_p=1;    // Valid even if p==0
  1399.               while ((adr[p] == '\r') || (adr[p] == '\n'))
  1400.                 p++;
  1401.               //can_avoid_quotes=1;
  1402.               ending_p='\r';
  1403.             }       
  1404.  
  1405.           } else if (isspace((unsigned char)*adr)) {
  1406.             intag_startattr=adr+1;        // attribute in tag (for dirty parsing)
  1407.           }
  1408.  
  1409.  
  1410.           // ------------------------------------------------------------
  1411.           // dernier recours - parsing "sale" : dΘtection systΘmatique des .gif, etc.
  1412.           // risque: gΘnΘrer de faux fichiers parazites
  1413.           // fix: ne parse plus dans les commentaires
  1414.           // ------------------------------------------------------------
  1415.           if ( (opt->parseall) && (ptr>0) && (!in_media) /* && (!inscript_in_comments)*/ ) {   // option parsing "brut"
  1416.             //int incomment_justquit=0;
  1417.             if (!is_realspace(*adr)) {
  1418.               int noparse=0;
  1419.  
  1420.               // Gestion des /* */
  1421. #if 0
  1422.               if (inscript) {
  1423.                 if (parseall_incomment) {
  1424.                   if ((*adr=='/') && (*(adr-1)=='*'))
  1425.                     parseall_incomment=0;
  1426.                   incomment_justquit=1;       // ne pas noter dernier caractΦre
  1427.                 } else {
  1428.                   if ((*adr=='/') && (*(adr+1)=='*'))
  1429.                     parseall_incomment=1;
  1430.                 }
  1431.               } else
  1432.                 parseall_incomment=0;
  1433. #endif
  1434.               /* ensure automate state  0 (not in comments, quotes..) */
  1435.               if (inscript && ( 
  1436.                 inscript_state_pos != INSCRIPT_INQUOTE && inscript_state_pos != INSCRIPT_INQUOTE2
  1437.                 ) ) {
  1438.                   noparse=1;
  1439.                 }
  1440.  
  1441.                 /* vΘrifier que l'on est pas dans un <!-- --> pur */
  1442.                 if ( (!intag) && (incomment) && (!inscript))
  1443.                   noparse=1;        /* commentaire */
  1444.  
  1445.                 // recherche d'URLs
  1446.                 if (!noparse) {
  1447.                   //if ((!parseall_incomment) && (!noparse)) {
  1448.                   if (!p) {                   // non dΘja trouvΘ
  1449.                     if (adr != r->adr) {     // >1 caractΦre
  1450.                       // scanner les chaines
  1451.                       if ((*adr == '\"') || (*adr=='\'')) {         // "xx.gif" 'xx.gif'
  1452.                         if (strchr("=(,",parseall_lastc)) {    // exemple: a="img.gif.. (handles comments)
  1453.                           char *a=adr;
  1454.                           char stop=*adr;  // " ou '
  1455.                           int count=0;
  1456.  
  1457.                           // sauter caractΦres
  1458.                           a++;
  1459.                           // copier
  1460.                           while((*a) && (*a!='\'') && (*a!='\"') && (count<HTS_URLMAXSIZE)) { count++; a++; }
  1461.  
  1462.                           // ok chaine terminΘe par " ou '
  1463.                           if ((*a == stop) && (count<HTS_URLMAXSIZE) && (count>0)) {
  1464.                             char c;
  1465.                             char* aend;
  1466.                             //
  1467.                             aend=a;     // sauver dΘbut
  1468.                             a++;
  1469.                             while(is_taborspace(*a)) a++;
  1470.                             c=*a;
  1471.                             if (strchr("),;>/+\r\n",c)) {     // exemple: ..img.gif";
  1472.                               // le / est pour funct("img.gif" /* URL */);
  1473.                               char BIGSTK tempo[HTS_URLMAXSIZE*2];
  1474.                               char type[256];
  1475.                               int url_ok=0;      // url valide?
  1476.                               tempo[0]='\0'; type[0]='\0';
  1477.                               //
  1478.                               strncatbuff(tempo,adr+1,count);
  1479.                               //
  1480.                               if ((!strchr(tempo,' ')) || inscript) {   // espace dedans: mΘfiance! (sauf dans code javascript)
  1481.                                 int invalid_url=0;
  1482.  
  1483.                                 // escape                              
  1484.                                 unescape_amp(tempo);
  1485.  
  1486.                                 // Couper au # ou ? Θventuel
  1487.                                 {
  1488.                                   char* a=strchr(tempo,'#');
  1489.                                   if (a)
  1490.                                     *a='\0';
  1491.                                   a=strchr(tempo,'?');
  1492.                                   if (a)
  1493.                                     *a='\0';
  1494.                                 }
  1495.  
  1496.                                 // vΘrifier qu'il n'y a pas de caractΦres spΘciaux
  1497.                                 if (!strnotempty(tempo))
  1498.                                   invalid_url=1;
  1499.                                 else if (strchr(tempo,'*')
  1500.                                   || strchr(tempo,'<')
  1501.                                   || strchr(tempo,'>')
  1502.                                   || strchr(tempo,',')    /* list of files ? */
  1503.                                   || strchr(tempo,'\"')    /* potential parsing bug */
  1504.                                   || strchr(tempo,'\'')    /* potential parsing bug */
  1505.                                   )
  1506.                                   invalid_url=1;
  1507.                                 else if (tempo[0] == '.' && isalnum(tempo[1]))   // ".gif"
  1508.                                   invalid_url=1;
  1509.  
  1510.                                 /* non invalide? */
  1511.                                 if (!invalid_url) {
  1512.                                   // Un plus α la fin? Alors ne pas prendre sauf si extension ("/toto.html#"+tag)
  1513.                                   if (c!='+') {    // PAS de plus α la fin
  1514. #if 0
  1515.                                     char* a;
  1516. #endif
  1517.                                     // "Comparisons of scheme names MUST be case-insensitive" (RFC2616)                                  
  1518.                                     if (
  1519.                                       (strfield(tempo,"http:")) 
  1520.                                       || (strfield(tempo,"ftp:"))
  1521. #if HTS_USEOPENSSL
  1522.                                       || (
  1523.                                       SSL_is_available &&
  1524.                                       (strfield(tempo,"https:"))
  1525.                                       )
  1526. #endif
  1527. #if HTS_USEMMS
  1528.                                                                             || strfield(tempo,"mms:")
  1529. #endif
  1530.                                       )  // ok pas de problΦme
  1531.                                       url_ok=1;
  1532.                                     else if (tempo[strlen(tempo)-1]=='/') {        // un slash: ok..
  1533.                                       if (inscript)   // sinon si pas javascript, mΘfiance (rΘpertoire style base?)
  1534.                                         url_ok=1;
  1535.                                     } 
  1536. #if 0
  1537.                                     else if ((a=strchr(tempo,'/'))) {        // un slash: ok..
  1538.                                       if (inscript) {    // sinon si pas javascript, mΘfiance (style "text/css")
  1539.                                         if (strchr(a+1,'/'))     // un seul / : abandon (STYLE type='text/css')
  1540.                                           if (!strchr(tempo,' '))  // avoid spaces (too dangerous for comments)
  1541.                                             url_ok=1;
  1542.                                       }
  1543.                                     }
  1544. #endif
  1545.                                   }
  1546.                                   // Prendre si extension reconnue
  1547.                                   if (!url_ok) {
  1548.                                     get_httptype(type,tempo,0);
  1549.                                     if (strnotempty(type))     // type reconnu!
  1550.                                       url_ok=1;
  1551.                                     else if (is_dyntype(get_ext(tempo)))  // reconnu php,cgi,asp..
  1552.                                       url_ok=1;
  1553.                                     // MAIS pas les foobar@aol.com !!
  1554.                                     if (strchr(tempo,'@'))
  1555.                                       url_ok=0;
  1556.                                   }
  1557.                                   //
  1558.                                   // Ok, cela pourrait Ωtre une URL
  1559.                                   if (url_ok) {
  1560.  
  1561.                                     // Check if not fodbidden tag (id,name..)
  1562.                                     if (intag_start_valid) {
  1563.                                       if (intag_start)
  1564.                                         if (intag_startattr)
  1565.                                           if (intag)
  1566.                                             if (!inscript)
  1567.                                               if (!incomment) {
  1568.                                                 int i=0,nop=0;
  1569.                                                 while( (nop==0) && (strnotempty(hts_nodetect[i])) ) {
  1570.                                                   nop=rech_tageq(intag_startattr,hts_nodetect[i]);
  1571.                                                   i++;
  1572.                                                 }
  1573.                                                 // Forbidden tag
  1574.                                                 if (nop) {
  1575.                                                   url_ok=0;
  1576.                                                   if ((opt->debug>1) && (opt->log!=NULL)) {
  1577.                                                     fspc(opt->log,"debug"); fprintf(opt->log,"dirty parsing: bad tag avoided: %s"LF,hts_nodetect[i-1]); test_flush;
  1578.                                                   }
  1579.                                                 }
  1580.                                               }
  1581.                                     }
  1582.  
  1583.  
  1584.                                     // Accepter URL, on la traitera comme une URL normale!!
  1585.                                     if (url_ok) {
  1586.                                       valid_p = 1;
  1587.                                       p = 0;
  1588.                                     }
  1589.  
  1590.                                   }
  1591.                                 }
  1592.                               }
  1593.                             }
  1594.                           }
  1595.                         }
  1596.                       }
  1597.                     }
  1598.                   }  // p == 0               
  1599.  
  1600.                 } // not in comment
  1601.  
  1602.                 // plus dans un commentaire
  1603.                 if ( inscript_state_pos == INSCRIPT_START 
  1604.                   && inscript_state_pos_prev == INSCRIPT_START) {
  1605.                     parseall_lastc=*adr;             // caractΦre avant le prochain
  1606.                   }
  1607.  
  1608.  
  1609.             }  // if realspace
  1610.           }  // if parseall
  1611.  
  1612.  
  1613.           // ------------------------------------------------------------
  1614.           // p!=0 : on a repΘrΘ un Θventuel lien
  1615.           // ------------------------------------------------------------
  1616.           //
  1617.           if ((p>0) || (valid_p)) {    // on a repΘrΘ un lien
  1618.             //int lien_valide=0;
  1619.             char* eadr=NULL;          /* fin de l'URL */
  1620.             char* quote_adr=NULL;     /* adresse du ? dans l'adresse */
  1621.             int ok=1;
  1622.             char quote='\0';
  1623.             int quoteinscript=0;
  1624.             int  noquote=0;
  1625.                         char *tag_attr_start = adr;
  1626.  
  1627.             // si nofollow ou un stop a ΘtΘ dΘclenchΘ, rΘΘcrire tous les liens en externe
  1628.             if ((nofollow) || (opt->state.stop))
  1629.               p_nocatch=1;
  1630.  
  1631.             // Θcrire codebase avant, flusher avant code
  1632.             if ((p_type==-1) || (p_type==-2)) {
  1633.               if ((opt->getmode & 1) && (ptr>0)) {
  1634.                 HT_ADD_ADR;    // refresh
  1635.               }
  1636.               lastsaved=adr;    // dernier Θcrit+1
  1637.             }
  1638.  
  1639.             // sauter espaces
  1640.             // adr+=p;
  1641.             INCREMENT_CURRENT_ADR(p);
  1642.             while( ( is_space(*adr) || (
  1643.               inscriptgen 
  1644.               && adr[0] == '\\' 
  1645.               && is_space(adr[1])
  1646.               )
  1647.               )
  1648.               && quote == '\0'
  1649.               ) {
  1650.                 if (!quote)
  1651.                   if ((*adr=='\"') || (*adr=='\'')) {
  1652.                     quote=*adr;                     // on doit attendre cela α la fin
  1653.                     if (inscriptgen && *(adr - 1) == '\\') {
  1654.                       quoteinscript=1;  /* will wait for \" */
  1655.                     }
  1656.                   }
  1657.                   // puis quitter
  1658.                   // adr++;    // sauter les espaces, "" et cie
  1659.                   INCREMENT_CURRENT_ADR(1);
  1660.               }
  1661.  
  1662.               /* Stop at \n (LF) if primary links or link lists */
  1663.               if (ptr == 0 || (in_media && strcmp(in_media,"LNK")==0))
  1664.                 quote='\n';
  1665.               /* s'arrΩter que ce soit un ' ou un " : pour document.write('<img src="foo'+a); par exemple! */
  1666.               else if (inscript && ! unquoted_script)
  1667.                 noquote=1;
  1668.  
  1669.               // sauter Θventuel \" ou \' javascript
  1670.               if (inscript) {    // on est dans un obj.write("..
  1671.                 if (*adr=='\\') {
  1672.                   if ((*(adr+1)=='\'') || (*(adr+1)=='"')) {  // \" ou \'
  1673.                     // adr+=2;    // sauter
  1674.                     INCREMENT_CURRENT_ADR(2);
  1675.                   }
  1676.                 }
  1677.               }
  1678.  
  1679.               // sauter content="1;URL=http://..
  1680.               if (p_searchMETAURL) {
  1681.                 int l=0;
  1682.                 while(
  1683.                   (adr + l + 4 < r->adr + r->size)
  1684.                   && (!strfield(adr+l,"URL=")) 
  1685.                   && (l<128) ) l++;
  1686.                 if (!strfield(adr+l,"URL="))
  1687.                   ok=-1;
  1688.                 else
  1689.                   adr+=(l+4);
  1690.               }
  1691.  
  1692.               /* Θviter les javascript:document.location=.. : les parser, plut⌠t */
  1693.               if (ok!=-1) {
  1694.                 if (strfield(adr,"javascript:") 
  1695.                   && ! inscript       /* we don't want to parse 'javascript:' inside document.write inside scripts */
  1696.                   ) {
  1697.                     ok=-1;
  1698.                     /*
  1699.                     On est dΘsormais dans du code javascript
  1700.                     */
  1701.                     inscript_name="";
  1702.                     inscript_tag=inscript=1;
  1703.                     inscript_state_pos=INSCRIPT_START;
  1704.                     inscript_tag_lastc=quote;     /* α attendre α la fin */
  1705.                     if (opt->parsedebug) { HT_ADD("<@@ inscript @@>"); }
  1706.                   }
  1707.               }
  1708.  
  1709.               if (p_type==1) {
  1710.                 if (*adr=='#') {
  1711.                   adr++;           // sauter # pour usemap etc
  1712.                 }
  1713.               }
  1714.               eadr=adr;
  1715.  
  1716.               // ne pas flusher aprΦs code si on doit Θcrire le codebase avant!
  1717.               if ((p_type!=-1) && (p_type!=2) && (p_type!=-2)) {
  1718.                 if ((opt->getmode & 1) && (ptr>0)) {
  1719.                   HT_ADD_ADR;    // refresh
  1720.                 }
  1721.                 lastsaved=adr;    // dernier Θcrit+1
  1722.                 // aprΦs on Θcrira soit les donnΘes initiales,
  1723.                 // soir une URL/lien modifiΘ!
  1724.               } else if (p_type==-1) p_flush=adr;    // flusher jusqu'α adr ensuite
  1725.  
  1726.               if (ok!=-1) {    // continuer
  1727.                 // dΘcouper le lien
  1728.                 do {
  1729.                   if ((* (unsigned char*) eadr)<32) {   // caractΦre de contr⌠le (ou \0)
  1730.                     if (!is_space(*eadr))
  1731.                       ok=0; 
  1732.                   }
  1733.                   if ( ( ((int) (eadr - adr)) ) > HTS_URLMAXSIZE)  // ** trop long, >HTS_URLMAXSIZE caractΦres (on prΘvoit HTS_URLMAXSIZE autres pour path)
  1734.                     ok=-1;    // ne pas traiter ce lien
  1735.  
  1736.                   if (ok > 0) {
  1737.                     //if (*eadr!=' ') {  
  1738.                     if (is_space(*eadr)) {   // guillemets,CR, etc
  1739.                       if ( 
  1740.                         ( *eadr == quote && ( !quoteinscript || *(eadr -1) == '\\') )  // end quote
  1741.                         || ( noquote && (*eadr == '\"' || *eadr == '\'') )       // end at any quote
  1742.                         || (!noquote && quote == '\0' && is_realspace(*eadr) )   // unquoted href
  1743.                         )     // si pas d'attente de quote spΘciale ou si quote atteinte
  1744.                         ok=0; 
  1745.                     } else if (ending_p && (*eadr==ending_p))
  1746.                       ok=0;
  1747.                     else {
  1748.                       switch(*eadr) {
  1749.                     case '>': 
  1750.                       if (!quote) {
  1751.                         if (!inscript && !in_media) {
  1752.                           intag=0;    // PLUS dans un tag!
  1753.                           intag_start_valid=0;
  1754.                                                     intag_name = NULL;
  1755.                         }
  1756.                         ok=0;
  1757.                       }
  1758.                       break;
  1759.                       /*case '<':*/ 
  1760.                     case '#': 
  1761.                       if (*(eadr-1) != '&')       // (
  1762.                         ok=0; 
  1763.                       break;
  1764.                       // case '?': non!
  1765.                     case '\\': if (inscript) ok=0; break;     // \" ou \' point d'arrΩt
  1766.                     case '?': quote_adr=adr; break;           // noter position query
  1767.                       }
  1768.                     }
  1769.                     //}
  1770.                   } 
  1771.                   eadr++;
  1772.                 } while(ok==1);
  1773.  
  1774.                 // Empty link detected
  1775.                 if ( (((int) (eadr - adr))) <= 1) {       // link empty
  1776.                   ok=-1;        // No
  1777.                   if (*adr != '#') {        // Not empty+unique #
  1778.                     if ( (((int) (eadr - adr)) == 1)) {       // 1=link empty with delim (end_adr-start_adr)
  1779.                       if (quote) {
  1780.                         if ((opt->getmode & 1) && (ptr>0)) { 
  1781.                           HT_ADD("#");        // We add this for a <href="">
  1782.                         }
  1783.                       }
  1784.                     }
  1785.                   }
  1786.                 }
  1787.  
  1788.                 // This is a dirty and horrible hack to avoid parsing an Adobe GoLive bogus tag
  1789.                 if (strfield(adr, "(Empty Reference!)")) {
  1790.                   ok=-1;        // No
  1791.                 }
  1792.  
  1793.               }
  1794.  
  1795.               if (ok==0) {    // tester un lien
  1796.                 char BIGSTK lien[HTS_URLMAXSIZE*2];
  1797.                 int meme_adresse=0;      // 0 par dΘfaut pour primary
  1798.                 //char *copie_de_adr=adr;
  1799.                 //char* p;
  1800.  
  1801.                 // construire lien (dΘcoupage)
  1802.                 if ( (((int) (eadr -  adr))-1) < HTS_URLMAXSIZE  ) {    // pas trop long?
  1803.                   strncpy(lien,adr,((int) (eadr - adr))-1);
  1804.                   *(lien+  (((int) (eadr -  adr)))-1  )='\0';
  1805.                   //printf("link: %s\n",lien);          
  1806.                   // supprimer les espaces
  1807.                   while((lien[strlen(lien)-1]==' ') && (strnotempty(lien))) lien[strlen(lien)-1]='\0';
  1808.  
  1809.  
  1810.                 } else
  1811.                   lien[0]='\0';    // erreur
  1812.  
  1813.  
  1814.                 // ------------------------------------------------------
  1815.                 // Lien repΘrΘ et extrait
  1816.                 if (strnotempty(lien)>0) {           // construction du lien
  1817.                   char BIGSTK adr[HTS_URLMAXSIZE*2],fil[HTS_URLMAXSIZE*2];          // ATTENTION adr cache le "vrai" adr
  1818.                   int forbidden_url=-1;              // lien non interdit (mais non autorisΘ..)
  1819.                   int just_test_it=0;                // mode de test des liens
  1820.                   int set_prio_to=0;                 // pour capture de page isolΘe
  1821.                   int import_done=0;                 // lien importΘ (ne pas scanner ensuite *α priori*)
  1822.                   //
  1823.                   adr[0]='\0'; fil[0]='\0';
  1824.                   //
  1825.                   // 0: autorisΘ
  1826.                   // 1: interdit (patcher tout de mΩme adresse)
  1827.  
  1828.                   if ((opt->debug>1) && (opt->log!=NULL)) {
  1829.                     fspc(opt->log,"debug"); fprintf(opt->log,"link detected in html (tag): %s"LF,lien); test_flush;
  1830.                   }
  1831.  
  1832.                   // external check
  1833. #if HTS_ANALYSTE
  1834.                   if (!hts_htmlcheck_linkdetected(lien) || !hts_htmlcheck_linkdetected2(lien, intag_start)) {
  1835.                     error=1;    // erreur
  1836.                     if (opt->errlog) {
  1837.                       fspc(opt->errlog,"error"); fprintf(opt->errlog,"Link %s refused by external wrapper"LF,lien);
  1838.                       test_flush;
  1839.                     }
  1840.                   }
  1841. #endif
  1842.  
  1843. #if HTS_STRIP_DOUBLE_SLASH
  1844.                   // supprimer les // en / (sauf pour http://)
  1845.                   if (opt->urlhack) {
  1846.                     char *a,*p,*q;
  1847.                     int done=0;
  1848.                     a=strchr(lien,':');    // http://
  1849.                     if (a) {
  1850.                       a++;
  1851.                       while(*a=='/') a++;    // position aprΦs http://
  1852.                     } else {
  1853.                       a=lien;                // dΘbut
  1854.                       while(*a=='/') a++;    // position aprΦs http://
  1855.                     }
  1856.                     q=strchr(a,'?');     // ne pas traiter aprΦs '?'
  1857.                     if (!q)
  1858.                       q=a+strlen(a)-1;
  1859.                     while(( p=strstr(a,"//")) && (!done) ) {    // remplacer // par /
  1860.                       if ((int) p>(int) q) {   // aprΦs le ? (toto.cgi?param=1//2.3)
  1861.                         done=1;    // stopper
  1862.                       } else {
  1863.                         char BIGSTK tempo[HTS_URLMAXSIZE*2];
  1864.                         tempo[0]='\0';
  1865.                         strncatbuff(tempo,a,(int) p - (int) a);
  1866.                         strcatbuff (tempo,p+1);
  1867.                         strcpybuff(a,tempo);    // recopier
  1868.                       }
  1869.                     }
  1870.                   }
  1871. #endif
  1872.  
  1873.                   // purger espaces de dΘbut et fin, CR,LF rΘsiduels
  1874.                   // (IMG SRC="foo.<\n><\t>gif<\t>")
  1875.                   {
  1876.                     char* a = lien;
  1877.                     int llen;
  1878.  
  1879.                     // strip ending spaces
  1880.                     llen = ( *a != '\0' ) ? strlen(a) : 0;
  1881.                     while(llen > 0 && is_realspace(lien[llen - 1]) ) {
  1882.                       a[--llen]='\0';
  1883.                     } 
  1884.                     //  skip leading ones
  1885.                     while(is_realspace(*a)) a++;
  1886.                     // strip cr, lf, tab inside URL
  1887.                     llen = 0;
  1888.                     while(*a) {
  1889.                       if (*a != '\n' && *a != '\r' && *a != '\t') {
  1890.                         lien[llen++] = *a;
  1891.                       }
  1892.                       a++;
  1893.                     }
  1894.                     lien[llen] = '\0';
  1895.                   }
  1896.  
  1897.                   // commas are forbidden
  1898.                   if (archivetag_p) {
  1899.                     if (strchr(lien, ',')) {
  1900.                       error=1;    // erreur
  1901.                       if ((opt->debug>1) && (opt->log!=NULL)) {
  1902.                         fspc(opt->log,"debug"); fprintf(opt->log,"link rejected (multiple-archive) %s"LF,lien); test_flush;
  1903.                       }
  1904.                     }
  1905.                   }               
  1906.  
  1907.                   /* Unescape/escape %20 and other   */
  1908.                   {
  1909.                     char BIGSTK query[HTS_URLMAXSIZE*2];
  1910.                     char* a=strchr(lien,'?');
  1911.                     if (a) {
  1912.                       strcpybuff(query,a);
  1913.                       *a='\0';
  1914.                     } else
  1915.                       query[0]='\0';
  1916.                     // conversion & -> & et autres joyeusetΘs
  1917.                     unescape_amp(lien);
  1918.                     unescape_amp(query);
  1919.                     // dΘcoder l'inutile (%2E par exemple) et coder espaces
  1920.                     // XXXXXXXXXXXXXXXXX strcpybuff(lien,unescape_http(lien));
  1921.                     //strcpybuff(lien,unescape_http_unharm(lien, (no_esc_utf)?0:1));
  1922.                                         /* Never unescape high-chars (we don't know the encoding!!) */
  1923.                     strcpybuff(lien,unescape_http_unharm(lien, 1));   /* note: '%' is still escaped */
  1924.                     escape_remove_control(lien);
  1925.                     escape_spc_url(lien);
  1926.                     strcatbuff(lien,query);     /* restore */
  1927.                   }
  1928.  
  1929.                   // convertir les Θventuels \ en des / pour Θviter des problΦmes de reconnaissance!
  1930.                   {
  1931.                     char* a;
  1932.                     for(a = jump_identification(lien) ; *a != '\0' && *a != '?' ; a++) {
  1933.                       if (*a == '\\') {
  1934.                         *a = '/';
  1935.                       }
  1936.                     }
  1937.                   }
  1938.  
  1939.                   // supprimer le(s) ./
  1940.                   while ((lien[0]=='.') && (lien[1]=='/')) {
  1941.                     char BIGSTK tempo[HTS_URLMAXSIZE*2];
  1942.                     strcpybuff(tempo,lien+2);
  1943.                     strcpybuff(lien,tempo);
  1944.                   }
  1945.                   if (strnotempty(lien)==0)  // sauf si plus de nom de fichier
  1946.                     strcpybuff(lien,"./");
  1947.  
  1948.                   // vΘrifie les /~machin -> /~machin/
  1949.                   // supposition dangereuse?
  1950.                   // OUI!!
  1951. #if HTS_TILDE_SLASH
  1952.                   if (lien[strlen(lien)-1]!='/') {
  1953.                     char *a=lien+strlen(lien)-1;
  1954.                     // Θviter aussi index~1.html
  1955.                     while (((int) a>(int) lien) && (*a!='~') && (*a!='/') && (*a!='.')) a--;
  1956.                     if (*a=='~') {
  1957.                       strcatbuff(lien,"/");    // ajouter slash
  1958.                     }
  1959.                   }
  1960. #endif
  1961.  
  1962.                   // APPLET CODE="mixer.MixerApplet.class" --> APPLET CODE="mixer/MixerApplet.class"
  1963.                   // yes, this is dirty
  1964.                   // but I'm so lazzy..
  1965.                   // and besides the java "code" convention is really a pain in html code
  1966.                   if (p_type==-1) {
  1967.                     char* a=strrchr(lien,'.');
  1968.                     add_class_dots_to_patch=0;
  1969.                     if (a) {
  1970.                       char* b;
  1971.                       do {
  1972.                         b=strchr(lien,'.');
  1973.                         if ((b != a) && (b)) {
  1974.                           add_class_dots_to_patch++;
  1975.                           *b='/';
  1976.                         }
  1977.                       } while((b != a) && (b));
  1978.                     }
  1979.                   }
  1980.  
  1981.                   // Θliminer les Θventuels :80 (port par dΘfaut!)
  1982.                   if (link_has_authority(lien)) {
  1983.                     char * a;
  1984.                     a=strstr(lien,"//");    // "//" authority
  1985.                     if (a)
  1986.                       a+=2;
  1987.                     else
  1988.                       a=lien;
  1989.                     // while((*a) && (*a!='/') && (*a!=':')) a++;
  1990.                     a=jump_toport(a);
  1991.                     if (a) {  // port
  1992.                       int port=0;
  1993.                       int defport=80;
  1994.                       char* b=a+1;
  1995. #if HTS_USEOPENSSL
  1996.                       // FIXME
  1997.                       //if (strfield(adr, "https:")) {
  1998.                       //}
  1999. #endif
  2000.                       while(isdigit((unsigned char)*b)) { port*=10; port+=(int) (*b-'0'); b++; }
  2001.                       if (port==defport) {  // port 80, default - c'est dΘbile
  2002.                         char BIGSTK tempo[HTS_URLMAXSIZE*2];
  2003.                         tempo[0]='\0';
  2004.                         strncatbuff(tempo,lien,(int) (a - lien));
  2005.                         strcatbuff(tempo,a+3);  // sauter :80
  2006.                         strcpybuff(lien,tempo);
  2007.                       }
  2008.                     }
  2009.                   }
  2010.  
  2011.                   // filtrer les parazites (mailto & cie)
  2012.                   /*
  2013.                   if (strfield(lien,"mailto:")) {  // ne pas traiter
  2014.                   error=1;
  2015.                   } else if (strfield(lien,"news:")) {  // ne pas traiter
  2016.                   error=1;
  2017.                   }
  2018.                   */
  2019.  
  2020.                   // vΘrifier que l'on ne doit pas ajouter de .class
  2021.                   if (!error) {
  2022.                     if (add_class) {
  2023.                       char *a = lien+strlen(lien)-1;
  2024.                       while(( a > lien) && (*a!='/') && (*a!='.')) a--;
  2025.                       if (*a != '.')
  2026.                         strcatbuff(lien,".class");    // ajouter .class
  2027.                       else if (!strfield2(a,".class"))
  2028.                         strcatbuff(lien,".class");    // idem
  2029.                     }
  2030.                   }
  2031.  
  2032.                   // si c'est un chemin, alors vΘrifier (toto/toto.html -> http://www/toto/)
  2033.                   if (!error) {
  2034.                     if ((opt->debug>1) && (opt->log!=NULL)) {
  2035.                       fspc(opt->log,"debug"); fprintf(opt->log,"position link check %s"LF,lien); test_flush;
  2036.                     }
  2037.  
  2038.                     if ((p_type==2) || (p_type==-2)) {   // code ou codebase                        
  2039.                       // VΘrifier les codebase=applet (au lieu de applet/)
  2040.                       if (p_type==-2) {    // codebase
  2041.                         if (strnotempty(lien)) {
  2042.                           if (fil[strlen(lien)-1]!='/') {  // pas rΘpertoire
  2043.                             strcatbuff(lien,"/");
  2044.                           }
  2045.                         }
  2046.                       }
  2047.  
  2048.                       /* base has always authority */
  2049.                       if (p_type==2 && !link_has_authority(lien)) {
  2050.                         char BIGSTK tmp[HTS_URLMAXSIZE*2];
  2051.                         strcpybuff(tmp, "http://");
  2052.                         strcatbuff(tmp, lien);
  2053.                         strcpybuff(lien, tmp);
  2054.                       }
  2055.  
  2056.                       /* only one ending / (bug on some pages) */
  2057.                       if ((int)strlen(lien)>2) {
  2058.                         int len = (int) strlen(lien);
  2059.                         while(len > 1 && lien[len-1] == '/' && lien[len-2] == '/' )    /* double // (bug) */
  2060.                           lien[--len]='\0';
  2061.                       }
  2062.                       // copier nom host si besoin est
  2063.                       if (!link_has_authority(lien)) {  // pas de http://
  2064.                         char BIGSTK adr2[HTS_URLMAXSIZE*2],fil2[HTS_URLMAXSIZE*2];  // ** euh ident_url_relatif??
  2065.                         if (ident_url_relatif(lien,urladr,urlfil,adr2,fil2)<0) {                        
  2066.                           error=1;
  2067.                         } else {
  2068.                           strcpybuff(lien,"http://");
  2069.                           strcatbuff(lien,adr2);
  2070.                           if (*fil2!='/')
  2071.                             strcatbuff(lien,"/");
  2072.                           strcatbuff(lien,fil2);
  2073.                           {
  2074.                             char* a;
  2075.                             a=lien+strlen(lien)-1;
  2076.                             while((*a) && (*a!='/') && ( a> lien)) a--;
  2077.                             if (*a=='/') {
  2078.                               *(a+1)='\0';
  2079.                             }
  2080.                           }
  2081.                           //char BIGSTK tempo[HTS_URLMAXSIZE*2];
  2082.                           //strcpybuff(tempo,"http://");
  2083.                           //strcatbuff(tempo,urladr);    // host
  2084.                           //if (*lien!='/')
  2085.                           //  strcatbuff(tempo,"/");
  2086.                           //strcatbuff(tempo,lien);
  2087.                           //strcpybuff(lien,tempo);
  2088.                         }
  2089.                       }
  2090.  
  2091.                       if (!error) {  // pas d'erreur?
  2092.                         if (p_type==2) {   // code ET PAS codebase      
  2093.                           char* a=lien+strlen(lien)-1;
  2094.                           char* start_of_filename = jump_identification(lien);
  2095.                           if (start_of_filename != NULL 
  2096.                             && (start_of_filename = strchr(start_of_filename, '/')) != NULL)
  2097.                             start_of_filename++;
  2098.                           if (start_of_filename == NULL)
  2099.                             strcatbuff(lien, "/");
  2100.                           while( (a > lien) && (*a) && (*a!='/')) a--;
  2101.                           if (*a=='/') {     // ok on a repΘrΘ le dernier /
  2102.                             if (start_of_filename != NULL && a >= start_of_filename) {
  2103.                               *(a+1)='\0';   // couper
  2104.                             }
  2105.                           } else {
  2106.                             *lien='\0';    // Θliminer
  2107.                             error=1;   // erreur, ne pas poursuivre
  2108.                           }      
  2109.                         }
  2110.  
  2111.                         // stocker base ou codebase?
  2112.                         switch(p_type) {
  2113.                       case 2: { 
  2114.                         //if (*lien!='/') strcatbuff(base,"/");
  2115.                         strcpybuff(base,lien);
  2116.                               }
  2117.                               break;      // base
  2118.                       case -2: {
  2119.                         //if (*lien!='/') strcatbuff(codebase,"/");
  2120.                         strcpybuff(codebase,lien); 
  2121.                                }
  2122.                                break;  // base
  2123.                         }
  2124.  
  2125.                         if ((opt->debug>1) && (opt->log!=NULL)) {
  2126.                           fspc(opt->log,"debug"); fprintf(opt->log,"code/codebase link %s base %s"LF,lien,base); test_flush;
  2127.                         }
  2128.                         //printf("base code: %s - %s\n",lien,base);
  2129.                       }
  2130.  
  2131.                     } else {
  2132.                       char* _base;
  2133.                       if (p_type==-1)   // code (applet)
  2134.                         _base=codebase;
  2135.                       else
  2136.                         _base=base;
  2137.  
  2138.  
  2139.                       // ajouter chemin de base href..
  2140.                       if (strnotempty(_base)) {       // considΘrer base
  2141.                         if (!link_has_authority(lien)) {    // non absolue
  2142.                           if (*lien!='/') {           // non absolu sur le site (/)
  2143.                             if ( ((int) strlen(_base)+(int) strlen(lien))<HTS_URLMAXSIZE) {
  2144.                               // mailto: and co: do NOT add base
  2145.                               if (ident_url_relatif(lien,urladr,urlfil,adr,fil)>=0) {
  2146.                                 char BIGSTK tempo[HTS_URLMAXSIZE*2];
  2147.                                 // base est absolue
  2148.                                 strcpybuff(tempo,_base);
  2149.                                 strcatbuff(tempo,lien + ((*lien=='/')?1:0) );
  2150.                                 strcpybuff(lien,tempo);        // patcher en considΘrant base
  2151.                                 // ** vΘrifier que ../ fonctionne (ne doit pas arriver mais bon..)
  2152.  
  2153.                                 if ((opt->debug>1) && (opt->log!=NULL)) {
  2154.                                   fspc(opt->log,"debug"); fprintf(opt->log,"link modified with code/codebase %s"LF,lien); test_flush;
  2155.                                 }
  2156.                               }
  2157.                             } else {
  2158.                               error=1;    // erreur
  2159.                               if (opt->errlog) {
  2160.                                 fspc(opt->errlog,"error"); fprintf(opt->errlog,"Link %s too long with base href"LF,lien);
  2161.                                 test_flush;
  2162.                               }
  2163.                             }
  2164.                           } else {
  2165.                             char BIGSTK badr[HTS_URLMAXSIZE*2], bfil[HTS_URLMAXSIZE*2];
  2166.                             if (ident_url_absolute(_base, badr, bfil) >=0 ) {
  2167.                               if ( ((int) strlen(badr)+(int) strlen(lien)) < HTS_URLMAXSIZE) {
  2168.                                 char BIGSTK tempo[HTS_URLMAXSIZE*2];
  2169.                                 // base est absolue
  2170.                                 tempo[0] = '\0';
  2171.                                 if (!link_has_authority(badr)) {
  2172.                                   strcatbuff(tempo, "http://");
  2173.                                 }
  2174.                                 strcatbuff(tempo,badr);
  2175.                                 strcatbuff(tempo,lien);
  2176.                                 strcpybuff(lien,tempo);        // patcher en considΘrant base
  2177.  
  2178.                                 if ((opt->debug>1) && (opt->log!=NULL)) {
  2179.                                   fspc(opt->log,"debug"); fprintf(opt->log,"link modified with code/codebase %s"LF,lien); test_flush;
  2180.                                 }
  2181.                               } else {
  2182.                                 error=1;    // erreur
  2183.                                 if (opt->errlog) {
  2184.                                   fspc(opt->errlog,"error"); fprintf(opt->errlog,"Link %s too long with base href"LF,lien);
  2185.                                   test_flush;
  2186.                                 }
  2187.                               }
  2188.                             }
  2189.                           }
  2190.                         }
  2191.                       }
  2192.  
  2193.  
  2194.                     }
  2195.                   }
  2196.  
  2197.  
  2198.                   // transformer lien quelconque (http, relatif, etc) en une adresse
  2199.                   // et un chemin+fichier (adr,fil)
  2200.                   if (!error) {
  2201.                     int reponse;
  2202.                     if ((opt->debug>1) && (opt->log!=NULL)) {
  2203.                       fspc(opt->log,"debug"); fprintf(opt->log,"build relative link %s with %s%s"LF,lien,relativeurladr,relativeurlfil); test_flush;
  2204.                     }
  2205.                     if ((reponse=ident_url_relatif(lien,relativeurladr,relativeurlfil,adr,fil))<0) {                        
  2206.                       adr[0]='\0';    // erreur
  2207.                       if (reponse==-2) {
  2208.                         if (opt->errlog) {
  2209.                           fspc(opt->errlog,"warning"); fprintf(opt->errlog,"Link %s not caught (unknown protocol)"LF,lien);
  2210.                           test_flush;
  2211.                         }
  2212.                       } else {
  2213.                         if ((opt->debug>1) && (opt->errlog!=NULL)) {
  2214.                           fspc(opt->errlog,"debug"); fprintf(opt->errlog,"ident_url_relatif failed for %s with %s%s"LF,lien,relativeurladr,relativeurlfil); test_flush;
  2215.                         }
  2216.                       }
  2217.                     } else {
  2218.                       if ((opt->debug>1) && (opt->log!=NULL)) {
  2219.                         fspc(opt->log,"debug"); fprintf(opt->log,"built relative link %s with %s%s -> %s%s"LF,lien,relativeurladr,relativeurlfil,adr,fil); test_flush;
  2220.                       }
  2221.                     }
  2222.                   } else {
  2223.                     if ((opt->debug>1) && (opt->log!=NULL)) {
  2224.                       fspc(opt->log,"debug"); fprintf(opt->log,"link %s not build, error detected before"LF,lien); test_flush;
  2225.                     }
  2226.                     adr[0]='\0';
  2227.                   }
  2228.  
  2229. #if HTS_CHECK_STRANGEDIR
  2230.                   // !ATTENTION!
  2231.                   // Ici on teste les exotiques du genre www.truc.fr/machin (sans slash α la fin)
  2232.                   // je n'ai pas encore trouvΘ le moyen de faire la diffΘrence entre un rΘpertoire
  2233.                   // et un fichier en http A PRIORI : je fais donc un test
  2234.                   // En cas de moved xxx, on recalcule adr et fil, tout simplement
  2235.                   // DEFAUT: test effectuΘ plusieurs fois! α revoir!!!
  2236.                   if ((adr[0]!='\0') && (strcmp(adr,"file://") && (p_type!=2) && (p_type!=-2)) {
  2237.                     //## if ((adr[0]!='\0') && (adr[0]!=lOCAL_CHAR) && (p_type!=2) && (p_type!=-2)) {
  2238.                     if (fil[strlen(fil)-1]!='/') {  // pas rΘpertoire
  2239.                       if (ishtml(fil)==-2) {    // pas d'extension
  2240.                         char BIGSTK loc[HTS_URLMAXSIZE*2];  // Θventuelle nouvelle position
  2241.                         loc[0]='\0';
  2242.                         if ((opt->debug>1) && (opt->log!=NULL)) {
  2243.                           fspc(opt->log,"debug"); fprintf(opt->log,"link-check-directory: %s%s"LF,adr,fil);
  2244.                           test_flush;
  2245.                         }
  2246.  
  2247.                         // tester Θventuelle nouvelle position
  2248.                         switch (http_location(adr,fil,loc).statuscode) {
  2249.                       case 200: // ok au final
  2250.                         if (strnotempty(loc)) {  // a changΘ d'adresse
  2251.                           if (opt->errlog) {
  2252.                             fspc(opt->errlog,"warning"); fprintf(opt->errlog,"Link %s%s has moved to %s for %s%s"LF,adr,fil,loc,urladr,urlfil);
  2253.                             test_flush;
  2254.                           }
  2255.  
  2256.                           // recalculer adr et fil!
  2257.                           if (ident_url_absolute(loc,adr,fil)==-1) {
  2258.                             adr[0]='\0';  // cancel
  2259.                             if ((opt->debug>1) && (opt->log!=NULL)) {
  2260.                               fspc(opt->log,"debug"); fprintf(opt->log,"link-check-dir: %s%s"LF,adr,fil);
  2261.                               test_flush;
  2262.                             }
  2263.                           }
  2264.  
  2265.                         }
  2266.                         break;
  2267.                       case -2: case -3:  // timeout ou erreur grave
  2268.                         if (opt->errlog) {
  2269.                           fspc(opt->errlog,"warning"); fprintf(opt->errlog,"Connection too slow for testing link %s%s (from %s%s)"LF,adr,fil,urladr,urlfil);
  2270.                           test_flush;
  2271.                         }
  2272.  
  2273.                         break;
  2274.                         }
  2275.  
  2276.                       }
  2277.                     } 
  2278.                   }
  2279. #endif
  2280.  
  2281.                   // Le lien doit juste Ωtre rΘΘcrit, mais ne doit pas gΘnΘrer un lien
  2282.                   // exemple: <FORM ACTION="url_cgi">
  2283.                   if (p_nocatch) {
  2284.                     forbidden_url=1;    // interdire rΘcupΘration du lien
  2285.                     if ((opt->debug>1) && (opt->log!=NULL)) {
  2286.                       fspc(opt->log,"debug"); fprintf(opt->log,"link forced external at %s%s"LF,adr,fil);
  2287.                       test_flush;
  2288.                     }
  2289.                   }
  2290.  
  2291.                   // Tester si un lien doit Ωtre acceptΘ ou refusΘ (wizard)
  2292.                   // forbidden_url=1 : lien refusΘ
  2293.                   // forbidden_url=0 : lien acceptΘ
  2294.                   //if ((ptr>0) && (p_type!=2) && (p_type!=-2)) {    // tester autorisations?
  2295.                   if ((p_type!=2) && (p_type!=-2)) {    // tester autorisations?
  2296.                     if (!p_nocatch) {
  2297.                       if (adr[0]!='\0') {          
  2298.                         if ((opt->debug>1) && (opt->log!=NULL)) {
  2299.                           fspc(opt->log,"debug"); fprintf(opt->log,"wizard link test at %s%s.."LF,adr,fil);
  2300.                           test_flush;
  2301.                         }
  2302.                         forbidden_url=hts_acceptlink(opt,ptr,lien_tot,liens,
  2303.                           adr,fil,
  2304.                                                     intag_name ? intag_name : NULL, intag_name ? tag_attr_start : NULL,
  2305.                           &set_prio_to,
  2306.                           &just_test_it);
  2307.                         if ((opt->debug>1) && (opt->log!=NULL)) {
  2308.                           fspc(opt->log,"debug"); fprintf(opt->log,"result for wizard link test: %d"LF,forbidden_url);
  2309.                           test_flush;
  2310.                         }
  2311.                       }
  2312.                     }
  2313.                   }
  2314.  
  2315.                   // calculer meme_adresse
  2316.                   meme_adresse=strfield2(jump_identification(adr),jump_identification(urladr));
  2317.  
  2318.                   // DΘbut partie sauvegarde
  2319.  
  2320.                   // ici on forme le nom du fichier α sauver, et on patche l'URL
  2321.                   if (adr[0]!='\0') {
  2322.                     // savename: simplifier les ../ et autres joyeusetΘs
  2323.                     char BIGSTK save[HTS_URLMAXSIZE*2];
  2324.                     int r_sv=0;
  2325.                     // En cas de moved, adresse premiΦre
  2326.                     char BIGSTK former_adr[HTS_URLMAXSIZE*2];
  2327.                     char BIGSTK former_fil[HTS_URLMAXSIZE*2];
  2328.                     //
  2329.                     save[0]='\0'; former_adr[0]='\0'; former_fil[0]='\0';
  2330.                     //
  2331.  
  2332.                     // nom du chemin α sauver si on doit le calculer
  2333.                     // note: url_savename peut dΘcider de tester le lien si il le trouve
  2334.                     // suspect, et modifier alors adr et fil
  2335.                     // dans ce cas on aura une rΘfΘrence directe au lieu des traditionnels
  2336.                     // moved en cascade (impossible α reproduire α priori en local, lorsque des fichiers
  2337.                     // gif sont impliquΘs par exemple)
  2338.                     if ((p_type!=2) && (p_type!=-2)) {  // pas base href ou codebase
  2339.                       if (forbidden_url!=1) {
  2340.                         char BIGSTK last_adr[HTS_URLMAXSIZE*2];
  2341.                         last_adr[0]='\0';
  2342.                         //char last_fil[HTS_URLMAXSIZE*2]="";
  2343.                         strcpybuff(last_adr,adr);    // ancienne adresse
  2344.                         //strcpybuff(last_fil,fil);    // ancien chemin
  2345.                         r_sv=url_savename(adr,fil,save,former_adr,former_fil,liens[ptr]->adr,liens[ptr]->fil,opt,liens,lien_tot,sback,cache,hash,ptr,numero_passe,NULL);
  2346.                         if (strcmp(jump_identification(last_adr),jump_identification(adr)) != 0) {  // a changΘ
  2347.  
  2348.                           // 2e test si moved
  2349.  
  2350.                           // Tester si un lien doit Ωtre acceptΘ ou refusΘ (wizard)
  2351.                           // forbidden_url=1 : lien refusΘ
  2352.                           // forbidden_url=0 : lien acceptΘ
  2353.                           if ((ptr>0) && (p_type!=2) && (p_type!=-2)) {    // tester autorisations?
  2354.                             if (!p_nocatch) {
  2355.                               if (adr[0]!='\0') {          
  2356.                                 if ((opt->debug>1) && (opt->log!=NULL)) {
  2357.                                   fspc(opt->log,"debug"); fprintf(opt->log,"wizard moved link retest at %s%s.."LF,adr,fil);
  2358.                                   test_flush;
  2359.                                 }
  2360.                                 forbidden_url=hts_acceptlink(opt,ptr,lien_tot,liens,
  2361.                                   adr,fil,
  2362.                                   intag_name ? intag_name : NULL, intag_name ? tag_attr_start : NULL,
  2363.                                   &set_prio_to,
  2364.                                   &just_test_it);
  2365.                                 if ((opt->debug>1) && (opt->log!=NULL)) {
  2366.                                   fspc(opt->log,"debug"); fprintf(opt->log,"result for wizard moved link retest: %d"LF,forbidden_url);
  2367.                                   test_flush;
  2368.                                 }
  2369.                               }
  2370.                             }
  2371.                           }
  2372.  
  2373.                           //import_done=1;    // c'est un import!
  2374.                           meme_adresse=0;   // on a changΘ
  2375.                         }
  2376.                       } else {
  2377.                         strcpybuff(save,"");  // dummy
  2378.                       }
  2379.                     }
  2380.  
  2381.                     // resolve unresolved type
  2382.                     if (r_sv!=-1
  2383.                       && p_type != 2 && p_type != -2
  2384.                       && forbidden_url == 0
  2385.                       && IS_DELAYED_EXT(save)
  2386.                       ) 
  2387.                     {  // pas d'erreur, on continue
  2388.                       r_sv = hts_wait_delayed(str, adr, fil, save, former_adr, former_fil, &forbidden_url);
  2389.                     }
  2390.  
  2391.                     // record!
  2392.                     if (r_sv!=-1) {  // pas d'erreur, on continue
  2393.                       /* log */
  2394.                       if ((opt->debug>1) && (opt->log!=NULL)) {
  2395.                         fspc(opt->log,"debug");
  2396.                         if (forbidden_url!=1) {    // le lien va Ωtre chargΘ
  2397.                           if ((p_type==2) || (p_type==-2)) {  // base href ou codebase, pas un lien
  2398.                             fprintf(opt->log,"Code/Codebase: %s%s"LF,adr,fil);
  2399.                           } else if ((opt->getmode & 4)==0) {
  2400.                             fprintf(opt->log,"Record: %s%s -> %s"LF,adr,fil,save);
  2401.                           } else {
  2402.                             if (!ishtml(fil))
  2403.                               fprintf(opt->log,"Record after: %s%s -> %s"LF,adr,fil,save);
  2404.                             else
  2405.                               fprintf(opt->log,"Record: %s%s -> %s"LF,adr,fil,save);
  2406.                           } 
  2407.                         } else
  2408.                           fprintf(opt->log,"External: %s%s"LF,adr,fil);
  2409.                         test_flush;
  2410.                       }
  2411.                       /* FIN log */
  2412.  
  2413.                       // Θcrire lien
  2414.                       if ((p_type==2) || (p_type==-2)) {  // base href ou codebase, sauter
  2415.                         lastsaved=eadr-1+1;  // sauter "
  2416.                       }
  2417.                       /* */
  2418.                       else if (opt->urlmode==0) {    // URL absolue dans tous les cas
  2419.                         if ((opt->getmode & 1) && (ptr>0)) {    // ecrire les html
  2420.                           if (!link_has_authority(adr)) {
  2421.                             HT_ADD("http://");
  2422.                           } else {
  2423.                             char* aut = strstr(adr, "//");
  2424.                             if (aut) {
  2425.                               char tmp[256];
  2426.                               tmp[0]='\0';
  2427.                               strncatbuff(tmp, adr, (int) (aut - adr));   // scheme
  2428.                               HT_ADD(tmp);          // Protocol
  2429.                               HT_ADD("//");
  2430.                             }
  2431.                           }
  2432.  
  2433.                           if (!opt->passprivacy) {
  2434.                             HT_ADD_HTMLESCAPED(jump_protocol(adr));           // Password
  2435.                           } else {
  2436.                             HT_ADD_HTMLESCAPED(jump_identification(adr));     // No Password
  2437.                           }
  2438.                           if (*fil!='/')
  2439.                             HT_ADD("/");
  2440.                           HT_ADD_HTMLESCAPED(fil);
  2441.                         }
  2442.                         lastsaved=eadr-1;    // dernier Θcrit+1 (enfin euh apres on fait un ++ alors hein)
  2443.                         /* */
  2444.                       } else if (opt->urlmode >= 4) {    // ne rien faire dans tous les cas!
  2445.                         /* */
  2446.                         /* leave the link 'as is' */
  2447.                         /* Sinon, dΘpend de interne/externe */
  2448.                       } else if (forbidden_url==1) {    // le lien ne sera pas chargΘ, rΘfΘrence externe!
  2449.                         if ((opt->getmode & 1) && (ptr>0)) {
  2450.                           if (p_type!=-1) {     // pas que le nom de fichier (pas classe java)
  2451.                             if (!opt->external) {
  2452.                               if (!link_has_authority(adr)) {
  2453.                                 HT_ADD("http://");
  2454.                                 if (!opt->passprivacy) {
  2455.                                   HT_ADD_HTMLESCAPED(adr);     // Password
  2456.                                 } else {
  2457.                                   HT_ADD_HTMLESCAPED(jump_identification(adr));     // No Password
  2458.                                 }
  2459.                                 if (*fil!='/')
  2460.                                   HT_ADD("/");
  2461.                                 HT_ADD_HTMLESCAPED(fil);
  2462.                               } else {
  2463.                                 char* aut = strstr(adr, "//");
  2464.                                 if (aut) {
  2465.                                   char tmp[256];
  2466.                                   tmp[0]='\0';
  2467.                                   strncatbuff(tmp, adr, (int) (aut - adr));   // scheme
  2468.                                   HT_ADD(tmp);          // Protocol
  2469.                                   HT_ADD("//");
  2470.                                   if (!opt->passprivacy) {
  2471.                                     HT_ADD_HTMLESCAPED(jump_protocol(adr));          // Password
  2472.                                   } else {
  2473.                                     HT_ADD_HTMLESCAPED(jump_identification(adr));     // No Password
  2474.                                   }
  2475.                                   if (*fil!='/')
  2476.                                     HT_ADD("/");
  2477.                                   HT_ADD_HTMLESCAPED(fil);
  2478.                                 }
  2479.                               }
  2480.                               //
  2481.                             } else {    // fichier/page externe, mais on veut gΘnΘrer une erreur
  2482.                               //
  2483.                               int patch_it=0;
  2484.                               int add_url=0;
  2485.                               char* cat_name=NULL;
  2486.                               char* cat_data=NULL;
  2487.                               int cat_nb=0;
  2488.                               int cat_data_len=0;
  2489.  
  2490.                               // ajouter lien external
  2491.                               switch ( (link_has_authority(adr)) ? 1 : ( (fil[strlen(fil)-1]=='/')?1:(ishtml(fil))  ) ) {
  2492.                             case 1: case -2:       // html ou rΘpertoire
  2493.                               if (opt->getmode & 1) {  // sauver html
  2494.                                 patch_it=1;   // redirect
  2495.                                 add_url=1;    // avec link?
  2496.                                 cat_name="external.html";
  2497.                                 cat_nb=0;
  2498.                                 cat_data=HTS_DATA_UNKNOWN_HTML;
  2499.                                 cat_data_len=HTS_DATA_UNKNOWN_HTML_LEN;
  2500.                               }
  2501.                               break;
  2502.                             default:    // inconnu
  2503.                               // asp, cgi..
  2504.                               if ( (strfield2(fil+max(0,(int)strlen(fil)-4),".gif")) 
  2505.                                 || (strfield2(fil+max(0,(int)strlen(fil)-4),".jpg")) 
  2506.                                 || (strfield2(fil+max(0,(int)strlen(fil)-4),".xbm")) 
  2507.                                 /*|| (ishtml(fil)!=0)*/ ) {
  2508.                                 patch_it=1;   // redirect
  2509.                               add_url=1;    // avec link aussi
  2510.                               cat_name="external.gif";
  2511.                               cat_nb=1;
  2512.                               cat_data=HTS_DATA_UNKNOWN_GIF;
  2513.                               cat_data_len=HTS_DATA_UNKNOWN_GIF_LEN;
  2514.                                 } else /* if (is_dyntype(get_ext(fil))) */ {
  2515.                                   patch_it=1;   // redirect
  2516.                                   add_url=1;    // avec link?
  2517.                                   cat_name="external.html";
  2518.                                   cat_nb=0;
  2519.                                   cat_data=HTS_DATA_UNKNOWN_HTML;
  2520.                                   cat_data_len=HTS_DATA_UNKNOWN_HTML_LEN;
  2521.                                 }
  2522.                                 break;
  2523.                               }// html,gif
  2524.  
  2525.                               if (patch_it) {
  2526.                                 char BIGSTK save[HTS_URLMAXSIZE*2];
  2527.                                 char BIGSTK tempo[HTS_URLMAXSIZE*2];
  2528.                                 strcpybuff(save,opt->path_html);
  2529.                                 strcatbuff(save,cat_name);
  2530.                                 if (lienrelatif(tempo,save, relativesavename)==0) {
  2531.                                                                     /* Never escape high-chars (we don't know the encoding!!) */
  2532.                                   escape_uri_utf(tempo);     // escape with %xx
  2533.                                   //if (!no_esc_utf)
  2534.                                   //  escape_uri(tempo);     // escape with %xx
  2535.                                   //else
  2536.                                   //  escape_uri_utf(tempo);     // escape with %xx
  2537.                                   HT_ADD_HTMLESCAPED(tempo);    // page externe
  2538.                                   if (add_url) {
  2539.                                     HT_ADD("?link=");    // page externe
  2540.  
  2541.                                     // same as above
  2542.                                     if (!link_has_authority(adr)) {
  2543.                                       HT_ADD("http://");
  2544.                                       if (!opt->passprivacy) {
  2545.                                         HT_ADD_HTMLESCAPED(adr);     // Password
  2546.                                       } else {
  2547.                                         HT_ADD_HTMLESCAPED(jump_identification(adr));     // No Password
  2548.                                       }
  2549.                                       if (*fil!='/')
  2550.                                         HT_ADD("/");
  2551.                                       HT_ADD_HTMLESCAPED(fil);
  2552.                                     } else {
  2553.                                       char* aut = strstr(adr, "//");
  2554.                                       if (aut) {
  2555.                                         char tmp[256];
  2556.                                         tmp[0]='\0';
  2557.                                         strncatbuff(tmp, adr, (int) (aut - adr) + 2);   // scheme
  2558.                                         HT_ADD(tmp);
  2559.                                         if (!opt->passprivacy) {
  2560.                                           HT_ADD_HTMLESCAPED(jump_protocol(adr));          // Password
  2561.                                         } else {
  2562.                                           HT_ADD_HTMLESCAPED(jump_identification(adr));     // No Password
  2563.                                         }
  2564.                                         if (*fil!='/')
  2565.                                           HT_ADD("/");
  2566.                                         HT_ADD_HTMLESCAPED(fil);
  2567.                                       }
  2568.                                     }
  2569.                                     //
  2570.  
  2571.                                   }
  2572.                                 }
  2573.  
  2574.                                 // Θcrire fichier?
  2575.                                 if (verif_external(cat_nb,1)) {
  2576.                                   //if (!fexist(fconcat(opt->path_html,cat_name))) {
  2577.                                   FILE* fp = filecreate(fconcat(opt->path_html,cat_name));
  2578.                                   if (fp) {
  2579.                                     if (cat_data_len==0) {   // texte
  2580.                                       verif_backblue(opt,opt->path_html);
  2581.                                       fprintf(fp,"%s%s","<!-- Created by HTTrack Website Copier/"HTTRACK_VERSION" "HTTRACK_AFF_AUTHORS" -->"LF,cat_data);
  2582.                                     } else {                    // data
  2583.                                       fwrite(cat_data,cat_data_len,1,fp);
  2584.                                     }
  2585.                                     fclose(fp);
  2586.                                     usercommand(opt,0,NULL,fconcat(opt->path_html,cat_name),"","");
  2587.                                   }
  2588.                                 }
  2589.                               }  else {    // Θcrire normalement le nom de fichier
  2590.                                 HT_ADD("http://");
  2591.                                 if (!opt->passprivacy) {
  2592.                                   HT_ADD_HTMLESCAPED(adr);       // Password
  2593.                                 } else {
  2594.                                   HT_ADD_HTMLESCAPED(jump_identification(adr));       // No Password
  2595.                                 }
  2596.                                 if (*fil!='/')
  2597.                                   HT_ADD("/");
  2598.                                 HT_ADD_HTMLESCAPED(fil);
  2599.                               }// patcher?
  2600.                             }  // external
  2601.                           } else {  // que le nom de fichier (classe java)
  2602.                             // en gros recopie de plus bas: copier codebase et base
  2603.                             if (p_flush) {
  2604.                               char BIGSTK tempo[HTS_URLMAXSIZE*2];    // <-- ajoutΘ
  2605.                               char BIGSTK tempo_pat[HTS_URLMAXSIZE*2];
  2606.  
  2607.                               // Calculer chemin
  2608.                               tempo_pat[0]='\0';
  2609.                               strcpybuff(tempo,fil);  // <-- ajoutΘ
  2610.                               {
  2611.                                 char* a=strrchr(tempo,'/');
  2612.  
  2613.                                 // Example: we converted code="x.y.z.foo.class" into "x/y/z/foo.class"
  2614.                                 // we have to do the contrary now
  2615.                                 if (add_class_dots_to_patch>0) {
  2616.                                   while( (add_class_dots_to_patch>0) && (a) ) {
  2617.                                     *a='.';     // convert "false" java / into .
  2618.                                     add_class_dots_to_patch--;
  2619.                                     a=strrchr(tempo,'/');
  2620.                                   }
  2621.                                   // if add_class_dots_to_patch, this is because there is a problem!!
  2622.                                   if (add_class_dots_to_patch) {
  2623.                                     if (opt->errlog) {
  2624.                                       fspc(opt->errlog,"warning"); fprintf(opt->errlog,"Error: can not rewind java path %s, check html code"LF,tempo);
  2625.                                       test_flush;
  2626.                                     }
  2627.                                   }
  2628.                                 }
  2629.  
  2630.                                 // Cut path/filename
  2631.                                 if (a) {
  2632.                                   char BIGSTK tempo2[HTS_URLMAXSIZE*2];
  2633.                                   strcpybuff(tempo2,a+1);         // FICHIER
  2634.                                   strncatbuff(tempo_pat,tempo,(int) (a - tempo)+1);  // chemin
  2635.                                   strcpybuff(tempo,tempo2);                     // fichier
  2636.                                 }
  2637.                               }
  2638.  
  2639.                               // Θrire codebase="chemin"
  2640.                               if ((opt->getmode & 1) && (ptr>0)) {
  2641.                                 char BIGSTK tempo4[HTS_URLMAXSIZE*2];
  2642.                                 tempo4[0]='\0';
  2643.  
  2644.                                 if (strnotempty(tempo_pat)) {
  2645.                                   HT_ADD("codebase=\"http://");
  2646.                                   if (!opt->passprivacy) {
  2647.                                     HT_ADD_HTMLESCAPED(adr);  // Password
  2648.                                   } else {
  2649.                                     HT_ADD_HTMLESCAPED(jump_identification(adr));  // No Password
  2650.                                   }
  2651.                                   if (*tempo_pat!='/') HT_ADD("/");
  2652.                                   HT_ADD(tempo_pat);
  2653.                                   HT_ADD("\" ");
  2654.                                 }
  2655.  
  2656.                                 strncatbuff(tempo4,lastsaved,(int) (p_flush - lastsaved));
  2657.                                 HT_ADD(tempo4);    // refresh code="
  2658.                                 HT_ADD(tempo);
  2659.                               }
  2660.                             }
  2661.                           }
  2662.                         }
  2663.                         lastsaved=eadr-1;
  2664.                       }
  2665.                       /*
  2666.                       else if (opt->urlmode==1) {    // ABSOLU, c'est le cas le moins courant
  2667.                       //  NE FONCTIONNE PAS!!  (et est inutile)
  2668.                       if ((opt->getmode & 1) && (ptr>0)) {    // ecrire les html
  2669.                       // Θcrire le lien modifiΘ, absolu
  2670.                       HT_ADD("file:");
  2671.                       if (*save=='/')
  2672.                       HT_ADD(save+1)
  2673.                       else
  2674.                       HT_ADD(save)
  2675.                       }
  2676.                       lastsaved=eadr-1;    // dernier Θcrit+1 (enfin euh apres on fait un ++ alors hein)
  2677.                       }
  2678.                       */
  2679.                       else if (opt->mimehtml) {
  2680.                         char BIGSTK buff[HTS_URLMAXSIZE*3];
  2681.                         HT_ADD("cid:");
  2682.                         strcpybuff(buff, adr);
  2683.                         strcatbuff(buff, fil);
  2684.                         escape_in_url(buff);
  2685.                         { char* a = buff; while((a = strchr(a, '%'))) { *a = 'X'; a++; } }
  2686.                         HT_ADD_HTMLESCAPED(buff);
  2687.                         lastsaved=eadr-1;    // dernier Θcrit+1 (enfin euh apres on fait un ++ alors hein)
  2688.                       }
  2689.                       else if (opt->urlmode==3) {    // URI absolue /
  2690.                         if ((opt->getmode & 1) && (ptr>0)) {    // ecrire les html
  2691.                           HT_ADD_HTMLESCAPED(fil);
  2692.                         }
  2693.                         lastsaved=eadr-1;    // dernier Θcrit+1 (enfin euh apres on fait un ++ alors hein)
  2694.                       }
  2695.                       else if (opt->urlmode==2) {  // RELATIF
  2696.                         char BIGSTK tempo[HTS_URLMAXSIZE*2];
  2697.                         tempo[0]='\0';
  2698.                         // calculer le lien relatif
  2699.  
  2700.                         if (lienrelatif(tempo,save,relativesavename)==0) {
  2701.                           if (!in_media) {    // In media (such as real audio): don't patch
  2702.                                                         /* Never escape high-chars (we don't know the encoding!!) */
  2703.                                                         escape_uri_utf(tempo);
  2704.                             //if (!no_esc_utf)
  2705.                             //  escape_uri(tempo);     // escape with %xx
  2706.                             //else {
  2707.                             //  /* No escaping at all - remaining upper chars will be escaped below */
  2708.                             //  /* FIXME - Should be done in all local cases */
  2709.                             //  //x_escape_html(tempo);
  2710.                             //  //escape_uri_utf(tempo);     // FIXME - escape with %xx
  2711.                             //  //escape_uri(tempo);     // escape with %xx
  2712.                             //}
  2713.                           }
  2714.                           if ((opt->debug>1) && (opt->log!=NULL)) {
  2715.                             fspc(opt->log,"debug"); fprintf(opt->log,"relative link at %s build with %s and %s: %s"LF,adr,save,relativesavename,tempo);
  2716.                             test_flush;
  2717.                           }
  2718.  
  2719.                           // lien applet (code) - il faut placer un codebase avant
  2720.                           if (p_type==-1) {  // que le nom de fichier
  2721.  
  2722.                             if (p_flush) {
  2723.                               char BIGSTK tempo_pat[HTS_URLMAXSIZE*2];
  2724.                               tempo_pat[0]='\0';
  2725.                               {
  2726.                                 char* a=strrchr(tempo,'/');
  2727.  
  2728.                                 // Example: we converted code="x.y.z.foo.class" into "x/y/z/foo.class"
  2729.                                 // we have to do the contrary now
  2730.                                 if (add_class_dots_to_patch>0) {
  2731.                                   while( (add_class_dots_to_patch>0) && (a) ) {
  2732.                                     *a='.';     // convert "false" java / into .
  2733.                                     add_class_dots_to_patch--;
  2734.                                     a=strrchr(tempo,'/');
  2735.                                   }
  2736.                                   // if add_class_dots_to_patch, this is because there is a problem!!
  2737.                                   if (add_class_dots_to_patch) {
  2738.                                     if (opt->errlog) {
  2739.                                       fspc(opt->errlog,"warning"); fprintf(opt->errlog,"Error: can not rewind java path %s, check html code"LF,tempo);
  2740.                                       test_flush;
  2741.                                     }
  2742.                                   }
  2743.                                 }
  2744.  
  2745.                                 if (a) {
  2746.                                   char BIGSTK tempo2[HTS_URLMAXSIZE*2];
  2747.                                   strcpybuff(tempo2,a+1);
  2748.                                   strncatbuff(tempo_pat,tempo,(int) (a - tempo)+1);  // chemin
  2749.                                   strcpybuff(tempo,tempo2);                     // fichier
  2750.                                 }
  2751.                               }
  2752.  
  2753.                               // Θrire codebase="chemin"
  2754.                               if ((opt->getmode & 1) && (ptr>0)) {
  2755.                                 char BIGSTK tempo4[HTS_URLMAXSIZE*2];
  2756.                                 tempo4[0]='\0';
  2757.  
  2758.                                 if (strnotempty(tempo_pat)) {
  2759.                                   HT_ADD("codebase=\"");
  2760.                                   HT_ADD_HTMLESCAPED(tempo_pat);
  2761.                                   HT_ADD("\" ");
  2762.                                 }
  2763.  
  2764.                                 strncatbuff(tempo4,lastsaved,(int) (p_flush - lastsaved));
  2765.                                 HT_ADD(tempo4);    // refresh code="
  2766.                               }
  2767.                             }
  2768.                             //lastsaved=adr;    // dernier Θcrit+1
  2769.                           }                              
  2770.  
  2771.                           if ((opt->getmode & 1) && (ptr>0)) {
  2772.                             // Θcrire le lien modifiΘ, relatif
  2773.                             // Note: escape all chars, even >127 (no UTF)
  2774.                             HT_ADD_HTMLESCAPED_FULL(tempo);
  2775.  
  2776.                             // Add query-string, for informational purpose only
  2777.                             // Useless, because all parameters-pages are saved into different targets
  2778.                             if (opt->includequery) {
  2779.                               char* a=strchr(lien,'?');
  2780.                               if (a) {
  2781.                                 HT_ADD_HTMLESCAPED(a);
  2782.                               }
  2783.                             }
  2784.                           }
  2785.                           lastsaved=eadr-1;    // dernier Θcrit+1 (enfin euh apres on fait un ++ alors hein)
  2786.                         } else {
  2787.                           if (opt->errlog) {
  2788.                             fprintf(opt->errlog,"Error building relative link %s and %s"LF,save,relativesavename);
  2789.                             test_flush;
  2790.                           }
  2791.                         }
  2792.                       }  // sinon le lien sera Θcrit normalement
  2793.  
  2794.  
  2795. #if 0
  2796.                       if (fexist(save)) {    // le fichier existe..
  2797.                         adr[0]='\0';
  2798.                         //if ((opt->debug>0) && (opt->log!=NULL)) {
  2799.                         if (opt->errlog) {
  2800.                           fspc(opt->errlog,"warning"); fprintf(opt->errlog,"Link has already been written on disk, cancelled: %s"LF,save);
  2801.                           test_flush;
  2802.                         }
  2803.                       }
  2804. #endif                            
  2805.  
  2806.                       /* Security check */
  2807.                       if (strlen(save) >= HTS_URLMAXSIZE) {
  2808.                         adr[0]='\0';
  2809.                         if (opt->errlog) {
  2810.                           fspc(opt->errlog,"warning"); fprintf(opt->errlog,"Link is too long: %s"LF,save);
  2811.                           test_flush;
  2812.                         }
  2813.                       }
  2814.  
  2815.                       if ((adr[0]!='\0') && (p_type!=2) && (p_type!=-2) && (forbidden_url!=1) ) {  // si le fichier n'existe pas, ajouter α la liste                            
  2816.                         // n'y a-t-il pas trop de liens?
  2817.                         if (lien_tot+1 >= lien_max-4) {    // trop de liens!
  2818.                           printf("PANIC! : Too many URLs : >%d [%d]\n",lien_tot,__LINE__);
  2819.                           if (opt->errlog) {
  2820.                             fprintf(opt->errlog,LF"Too many URLs, giving up..(>%d)"LF,lien_max);
  2821.                             fprintf(opt->errlog,"To avoid that: use #L option for more links (example: -#L1000000)"LF);
  2822.                             test_flush;
  2823.                           }
  2824.                           if ((opt->getmode & 1) && (ptr>0)) { if (fp) { fclose(fp); fp=NULL; } }
  2825.                           XH_uninit;   // dΘsallocation mΘmoire & buffers
  2826.                           return -1;
  2827.  
  2828.                         } else {    // noter le lien sur la listes des liens α charger
  2829.                           int pass_fix,dejafait=0;
  2830.  
  2831.                           // Calculer la prioritΘ de ce lien
  2832.                           if ((opt->getmode & 4)==0) {    // traiter html aprΦs
  2833.                             pass_fix=0;
  2834.                           } else {    // vΘrifier que ce n'est pas un !html
  2835.                             if (!ishtml(fil))
  2836.                               pass_fix=1;        // prioritΘ infΘrieure (traiter aprΦs)
  2837.                             else
  2838.                               pass_fix=max(0,numero_passe);    // prioritΘ normale
  2839.                           }
  2840.  
  2841.                           /* If the file seems to be an html file, get depth-1 */
  2842.                           /*
  2843.                           if (strnotempty(save)) {
  2844.                           if (ishtml(save) == 1) {
  2845.                           // descore_prio = 2;
  2846.                           } else {
  2847.                           // descore_prio = 1;
  2848.                           }
  2849.                           }
  2850.                           */
  2851.  
  2852.                           // vΘrifier que le lien n'a pas dΘja ΘtΘ notΘ
  2853.                           // si c'est le cas, alors il faut s'assurer que la prioritΘ associΘe
  2854.                           // au fichier est la plus grande des deux prioritΘs
  2855.                           //
  2856.                           // On part de la fin et on essaye de se presser (Θconomise temps machine)
  2857.                           {
  2858.                             int i=hash_read(hash,save,"",0,opt->urlhack);      // lecture type 0 (sav)
  2859.                             if (i>=0) {
  2860.                               if ((opt->debug>1) && (opt->log!=NULL)) {
  2861.                                 if (
  2862.                                   strcmp(adr, liens[i]->adr) != 0 
  2863.                                   || strcmp(fil, liens[i]->fil) != 0
  2864.                                   ) {
  2865.                                     fspc(opt->log,"debug"); fprintf(opt->log,"merging similar links %s%s and %s%s"LF,adr,fil,liens[i]->adr,liens[i]->fil);
  2866.                                     test_flush;
  2867.                                   }
  2868.                               }
  2869.                               liens[i]->depth=maximum(liens[i]->depth,liens[ptr]->depth - 1);
  2870.                               dejafait=1;
  2871.                             }
  2872.                           }
  2873.  
  2874.                           // le lien n'a jamais ΘtΘ crΘΘ.
  2875.                           // cette fois ci, on le crΘe!
  2876.                           if (!dejafait) {                                
  2877.                             //
  2878.                             // >>>> CREER LE LIEN <<<<
  2879.                             //
  2880.                             // enregistrer lien α charger
  2881.                             //liens[lien_tot]->adr[0]=liens[lien_tot]->fil[0]=liens[lien_tot]->sav[0]='\0';
  2882.                             // mΩme adresse: l'objet pΦre est l'objet pΦre de l'actuel
  2883.  
  2884.                                                         // DEBUT ROBOTS.TXT AJOUT
  2885.                                                         if (!just_test_it) {
  2886.                                                             if ((!strfield(adr,"ftp://"))         // non ftp
  2887.                                                                 && (!strfield(adr,"file://")) 
  2888. #if HTS_USEMMS
  2889.                                                                 && (!strfield(adr,"mms://")) 
  2890. #endif
  2891.                                                                 ) 
  2892.                                                             {    // non file
  2893.                                                                 if (opt->robots) {    // rΘcupΘrer robots
  2894.                                                                     if (ishtml(fil)!=0) {                       // pas la peine pour des fichiers isolΘs
  2895.                                                                         if (checkrobots(_ROBOTS,adr,"") != -1) {    // robots.txt ?
  2896.                                                                             checkrobots_set(_ROBOTS ,adr,"");          // ajouter entrΘe vide
  2897.                                                                             if (checkrobots(_ROBOTS,adr,"") == -1) {    // robots.txt ?
  2898.                                                                                 // enregistrer robots.txt (MACRO)
  2899.                                                                                 liens_record(adr,"/robots.txt","","","");
  2900.                                                                                 if (liens[lien_tot]==NULL) {  // erreur, pas de place rΘservΘe
  2901.                                                                                     printf("PANIC! : Not enough memory [%d]\n",__LINE__);
  2902.                                                                                     if (opt->errlog) { 
  2903.                                                                                         fprintf(opt->errlog,"Not enough memory, can not re-allocate %d bytes"LF,(int)((add_tab_alloc+1)*sizeof(lien_url)));
  2904.                                                                                         test_flush;
  2905.                                                                                     }
  2906.                                                                                     if ((opt->getmode & 1) && (ptr>0)) { if (fp) { fclose(fp); fp=NULL; } }
  2907.                                                                                     XH_uninit;    // dΘsallocation mΘmoire & buffers
  2908.                                                                                     return -1;
  2909.                                                                                 }  
  2910.                                                                                 liens[lien_tot]->testmode=0;          // pas mode test
  2911.                                                                                 liens[lien_tot]->link_import=0;       // pas mode import     
  2912.                                                                                 liens[lien_tot]->premier=lien_tot;
  2913.                                                                                 liens[lien_tot]->precedent=ptr;
  2914.                                                                                 liens[lien_tot]->depth=0;
  2915.                                                                                 liens[lien_tot]->pass2=max(0,numero_passe);
  2916.                                                                                 liens[lien_tot]->retry=0;
  2917.                                                                                 lien_tot++;  // UN LIEN DE PLUS
  2918. #if DEBUG_ROBOTS
  2919.                                                                                 printf("robots.txt: added file robots.txt for %s\n",adr);
  2920. #endif
  2921.                                                                                 if ((opt->debug>1) && (opt->log!=NULL)) {
  2922.                                                                                     fspc(opt->log,"debug"); fprintf(opt->log,"robots.txt added at %s"LF,adr);
  2923.                                                                                     test_flush;
  2924.                                                                                 }
  2925.                                                                             } else {
  2926.                                                                                 if (opt->errlog) {   
  2927.                                                                                     fprintf(opt->errlog,"Unexpected robots.txt error at %d"LF,__LINE__);
  2928.                                                                                     test_flush;
  2929.                                                                                 }
  2930.                                                                             }
  2931.                                                                         }
  2932.                                                                     }
  2933.                                                                 }
  2934.                                                             }
  2935.                                                         }
  2936.                                                         // FIN ROBOTS.TXT AJOUT
  2937.  
  2938.                                                         // enregistrer (MACRO)
  2939.                                                         liens_record(adr,fil,save,former_adr,former_fil);
  2940.                                                         if (liens[lien_tot]==NULL) {  // erreur, pas de place rΘservΘe
  2941.                                                             printf("PANIC! : Not enough memory [%d]\n",__LINE__);
  2942.                                                             if (opt->errlog) { 
  2943.                                                                 fprintf(opt->errlog,"Not enough memory, can not re-allocate %d bytes"LF,(int)((add_tab_alloc+1)*sizeof(lien_url)));
  2944.                                                                 test_flush;
  2945.                                                             }
  2946.                               if ((opt->getmode & 1) && (ptr>0)) { if (fp) { fclose(fp); fp=NULL; } }
  2947.                               XH_uninit;    // dΘsallocation mΘmoire & buffers
  2948.                               return -1;
  2949.                             }  
  2950.  
  2951.                             // mode test?
  2952.                             if (!just_test_it)
  2953.                               liens[lien_tot]->testmode=0;          // pas mode test
  2954.                             else
  2955.                               liens[lien_tot]->testmode=1;          // mode test
  2956.                             if (!import_done)
  2957.                               liens[lien_tot]->link_import=0;       // pas mode import
  2958.                             else
  2959.                               liens[lien_tot]->link_import=1;       // mode import
  2960.                             // Θcrire autres paramΦtres de la structure-lien
  2961.                             if ((meme_adresse) && (!import_done) && (liens[ptr]->premier != 0))
  2962.                               liens[lien_tot]->premier=liens[ptr]->premier;
  2963.                             else    // sinon l'objet pΦre est le prΘcΘdent lui mΩme
  2964.                               liens[lien_tot]->premier=lien_tot;
  2965.                             // liens[lien_tot]->premier=ptr;
  2966.  
  2967.                             liens[lien_tot]->precedent=ptr;
  2968.                             // noter la prioritΘ
  2969.                             if (!set_prio_to)
  2970.                               liens[lien_tot]->depth=liens[ptr]->depth - 1;
  2971.                             else
  2972.                               liens[lien_tot]->depth=max(0,min(liens[ptr]->depth-1,set_prio_to-1));         // PRIORITE NULLE (catch page)
  2973.                             // noter pass
  2974.                             liens[lien_tot]->pass2=pass_fix;
  2975.                             liens[lien_tot]->retry=opt->retry;
  2976.  
  2977.                             //strcpybuff(liens[lien_tot]->adr,adr);
  2978.                             //strcpybuff(liens[lien_tot]->fil,fil);
  2979.                             //strcpybuff(liens[lien_tot]->sav,save); 
  2980.                             if ((opt->debug>1) && (opt->log!=NULL)) {
  2981.                               if (!just_test_it) {
  2982.                                 fspc(opt->log,"debug"); fprintf(opt->log,"OK, NOTE: %s%s -> %s"LF,liens[lien_tot]->adr,liens[lien_tot]->fil,liens[lien_tot]->sav);
  2983.                               } else {
  2984.                                 fspc(opt->log,"debug"); fprintf(opt->log,"OK, TEST: %s%s"LF,liens[lien_tot]->adr,liens[lien_tot]->fil);
  2985.                               }
  2986.                               test_flush;
  2987.                             }
  2988.  
  2989.                             lien_tot++;  // UN LIEN DE PLUS
  2990.                           } else { // if !dejafait
  2991.                             if ((opt->debug>1) && (opt->log!=NULL)) {
  2992.                               fspc(opt->log,"debug"); fprintf(opt->log,"link has already been recorded, cancelled: %s"LF,save);
  2993.                               test_flush;
  2994.                             }
  2995.  
  2996.                           }
  2997.  
  2998.  
  2999.                         }   // si pas trop de liens
  3000.                       }   // si adr[0]!='\0'
  3001.  
  3002.  
  3003.                     }  // if adr[0]!='\0' 
  3004.  
  3005.                   }  // if adr[0]!='\0'
  3006.  
  3007.                 }    // if strlen(lien)>0
  3008.  
  3009.               }   // if ok==0      
  3010.  
  3011.               assertf(eadr - adr >= 0);       // Should not go back
  3012.               if (eadr > adr) {
  3013.                 INCREMENT_CURRENT_ADR(eadr - 1 - adr);
  3014.               }
  3015.               // adr=eadr-1;  // ** sauter
  3016.  
  3017.               /* We skipped bytes and skip the " : reset state */
  3018.               /*if (inscript) {
  3019.               inscript_state_pos = INSCRIPT_START;
  3020.               }*/
  3021.  
  3022.           }  // if (p) 
  3023.  
  3024.         }  // si '<' ou '>'
  3025.  
  3026.         // plus loin
  3027.         adr++;      // automate will be checked next loop
  3028.  
  3029.  
  3030.         /* Otimization: if we are scanning in HTML data (not in tag or script), 
  3031.         then jump to the next starting tag */
  3032.         if (ptr>0) {
  3033.           if ( (!intag)         /* Not in tag */
  3034.             && (!inscript)      /* Not in (java)script */
  3035.             && (!in_media)      /* Not in media */
  3036.             && (!incomment)     /* Not in comment (<!--) */
  3037.             && (!inscript_tag)  /* Not in tag with script inside */
  3038.             ) 
  3039.           {
  3040.             /* Not at the end */
  3041.             if (( ((int) (adr - r->adr)) ) < r->size) {
  3042.               /* Not on a starting tag yet */
  3043.               if (*adr != '<') {
  3044.                 /* strchr does not well behave with null chrs.. */
  3045.                 /* char* adr_next = strchr(adr,'<'); */
  3046.                 char* adr_next = adr;
  3047.                 while(*adr_next != '<' && (adr_next - r->adr) < r->size ) {
  3048.                   adr_next++;
  3049.                 }
  3050.                 /* Jump to near end (index hack) */
  3051.                 if (!adr_next || *adr_next != '<') {
  3052.                   if (
  3053.                     ( (int)(adr - r->adr) < (r->size - 4)) 
  3054.                     &&
  3055.                     (r->size > 4)
  3056.                     ) {
  3057.                       adr = r->adr + r->size - 2;
  3058.                     }
  3059.                 } else {
  3060.                   adr = adr_next;
  3061.                 }
  3062.               }
  3063.             }
  3064.           }
  3065.         }
  3066.  
  3067.         // ----------
  3068.         // Θcrire peu α peu
  3069.         if ((opt->getmode & 1) && (ptr>0)) HT_ADD_ADR;
  3070.         lastsaved=adr;    // dernier Θcrit+1
  3071.         // ----------
  3072.  
  3073.         // Checks
  3074.         if (back_add_stats != opt->state.back_add_stats) {
  3075.           back_add_stats = opt->state.back_add_stats;
  3076.  
  3077.           // Check max time
  3078.           if (!back_checkmirror(opt)) {
  3079.             adr = r->adr + r->size;
  3080.           }
  3081.         }
  3082.  
  3083.         // pour les stats du shell si parsing trop long
  3084. #if HTS_ANALYSTE
  3085.         if (r->size)
  3086.           _hts_in_html_done=(100 * ((int) (adr - r->adr)) ) / (int)(r->size);
  3087.         if (_hts_in_html_poll) {
  3088.           _hts_in_html_poll=0;
  3089.           // temps α attendre, et remplir autant que l'on peut le cache (backing)
  3090.           back_wait(sback,opt,cache,HTS_STAT.stat_timestart);        
  3091.           back_fillmax(sback,opt,cache,liens,ptr,numero_passe,lien_tot);
  3092.  
  3093.           // Transfer rate
  3094.           engine_stats();
  3095.  
  3096.           // Refresh various stats
  3097.           HTS_STAT.stat_nsocket=back_nsoc(sback);
  3098.           HTS_STAT.stat_errors=fspc(NULL,"error");
  3099.           HTS_STAT.stat_warnings=fspc(NULL,"warning");
  3100.           HTS_STAT.stat_infos=fspc(NULL,"info");
  3101.           HTS_STAT.nbk=backlinks_done(sback,liens,lien_tot,ptr);
  3102.           HTS_STAT.nb=back_transfered(HTS_STAT.stat_bytes,sback);
  3103.  
  3104.           if (!hts_htmlcheck_loop(sback->lnk, sback->count, 0,ptr,lien_tot,(int) (time_local()-HTS_STAT.stat_timestart),&HTS_STAT)) {
  3105.             if (opt->errlog) {
  3106.               fspc(opt->errlog,"info"); fprintf(opt->errlog,"Exit requested by shell or user"LF);
  3107.               test_flush;
  3108.             } 
  3109.             *stre->exit_xh_=1;  // exit requested
  3110.             XH_uninit;
  3111.             return -1;
  3112.             //adr = r->adr + r->size;  // exit
  3113.           } else if (_hts_cancel==1) {
  3114.             // adr = r->adr + r->size;  // exit
  3115.             nofollow=1;               // moins violent
  3116.             _hts_cancel=0;
  3117.           }
  3118.         }
  3119.  
  3120.         // refresh the backing system each 2 seconds
  3121.         if (engine_stats()) {
  3122.           back_wait(sback,opt,cache,HTS_STAT.stat_timestart);        
  3123.           back_fillmax(sback,opt,cache,liens,ptr,numero_passe,lien_tot);
  3124.         }
  3125. #endif
  3126.       } while(( ((int) (adr - r->adr)) ) < r->size);
  3127. #if HTS_ANALYSTE
  3128.       _hts_in_html_parsing=0;  // flag
  3129.       _hts_cancel=0;           // pas de cancel
  3130. #endif
  3131.       if ((opt->getmode & 1) && (ptr>0)) {
  3132.         {
  3133.           char* cAddr = ht_buff;
  3134.           int cSize = ht_len;
  3135.           if ( (opt->debug>0) && (opt->log!=NULL) ) {
  3136.             fspc(opt->log,"info"); fprintf(opt->log,"engine: postprocess-html: %s%s"LF, urladr, urlfil);
  3137.           }
  3138.           if (hts_htmlcheck_postprocess(&cAddr, &cSize, urladr, urlfil) == 1) {
  3139.             ht_buff = cAddr;
  3140.             ht_len = cSize;
  3141.           }
  3142.         }
  3143.  
  3144.         /* Flush and save to disk */
  3145.         HT_ADD_END;    // achever
  3146.       }
  3147.       //
  3148.       //
  3149.       //
  3150.     }  // if !error
  3151.  
  3152.  
  3153.     if (opt->getmode & 1) { if (fp) { fclose(fp); fp=NULL; } }
  3154.     // sauver fichier
  3155.     //structcheck(savename);
  3156.     //filesave(opt,r->adr,r->size,savename);
  3157.  
  3158. #if HTS_ANALYSTE
  3159.   }  // analyse OK
  3160. #endif
  3161.  
  3162.   /* Apply changes */
  3163.   ENGINE_SAVE_CONTEXT();
  3164.  
  3165.   return 0;
  3166. }
  3167.  
  3168.  
  3169.  
  3170.  
  3171. /*
  3172. Check 301, 302, .. statuscodes (moved)
  3173. */
  3174. int hts_mirror_check_moved(htsmoduleStruct* str, htsmoduleStructExtended* stre) {
  3175.   /* Load engine variables */
  3176.   ENGINE_LOAD_CONTEXT();  
  3177.  
  3178.   // DEBUT rattrapage des 301,302,307..
  3179.   // ------------------------------------------------------------
  3180.   if (!error) {
  3181.     ////////{
  3182.     // on a chargΘ un fichier en plus
  3183.     // if (!error) stat_loaded+=r.size;
  3184.  
  3185.     // ------------------------------------------------------------
  3186.     // Rattrapage des 301,302,307 (moved) et 412,416 - les 304 le sont dans le backing 
  3187.     // ------------------------------------------------------------
  3188.     if (HTTP_IS_REDIRECT(r->statuscode)) {          
  3189.         //if (r->adr!=NULL) {   // adr==null si fichier direct. [catch: davename normalement si cgi]
  3190.         //int i=0;
  3191.         char *rn=NULL;
  3192.         // char* p;
  3193.  
  3194.         if ( (opt->debug>0) && (opt->errlog!=NULL) ) {
  3195.           //if (opt->errlog) {
  3196.           fspc(opt->errlog,"warning"); fprintf(opt->errlog,"%s for %s%s"LF,r->msg,urladr,urlfil);
  3197.           test_flush;
  3198.         }
  3199.  
  3200.  
  3201.         {
  3202.           char BIGSTK mov_url[HTS_URLMAXSIZE*2],mov_adr[HTS_URLMAXSIZE*2],mov_fil[HTS_URLMAXSIZE*2];
  3203.           int get_it=0;         // ne pas prendre le fichier α la mΩme adresse par dΘfaut
  3204.           int reponse=0;
  3205.           mov_url[0]='\0'; mov_adr[0]='\0'; mov_fil[0]='\0';
  3206.           //
  3207.  
  3208.           strcpybuff(mov_url,r->location);
  3209.  
  3210.           // url qque -> adresse+fichier
  3211.           if ((reponse=ident_url_relatif(mov_url,urladr,urlfil,mov_adr,mov_fil))>=0) {                        
  3212.             int set_prio_to=0;    // pas de priotitΘ fixΘd par wizard
  3213.  
  3214.             // check whether URLHack is harmless or not
  3215.             if (opt->urlhack) {
  3216.               char BIGSTK n_adr[HTS_URLMAXSIZE*2], n_fil[HTS_URLMAXSIZE*2];
  3217.               char BIGSTK pn_adr[HTS_URLMAXSIZE*2], pn_fil[HTS_URLMAXSIZE*2];
  3218.               n_adr[0] = n_fil[0] = '\0';
  3219.               (void) adr_normalized(mov_adr, n_adr);
  3220.               (void) fil_normalized(mov_fil, n_fil);
  3221.               (void) adr_normalized(urladr, pn_adr);
  3222.               (void) fil_normalized(urlfil, pn_fil);
  3223.               if (strcasecmp(n_adr, pn_adr) == 0 && strcasecmp(n_fil, pn_fil) == 0) {
  3224.                 if (opt->errlog) {
  3225.                   fspc(opt->errlog,"warning"); fprintf(opt->errlog,"Redirected link is identical because of 'URL Hack' option: %s%s and %s%s"LF, urladr, urlfil, mov_adr, mov_fil);
  3226.                   test_flush;
  3227.                 }
  3228.               }
  3229.             }
  3230.  
  3231.             //if (ident_url_absolute(mov_url,mov_adr,mov_fil)!=-1) {    // ok URL reconnue
  3232.             // c'est (en gros) la mΩme URL..
  3233.             // si c'est un problΦme de casse dans le host c'est que le serveur est buggΘ
  3234.             // ("RFC says.." : host name IS case insensitive)
  3235.             if ((strfield2(mov_adr,urladr)!=0) && (strfield2(mov_fil,urlfil)!=0)) {  // identique α casse prΦs
  3236.               // on tourne en rond
  3237.               if (strcmp(mov_fil,urlfil)==0) {
  3238.                 error=1;
  3239.                 get_it=-1;        // ne rien faire
  3240.                 if (opt->errlog) {
  3241.                   fspc(opt->errlog,"warning"); fprintf(opt->errlog,"Can not bear crazy server (%s) for %s%s"LF,r->msg,urladr,urlfil);
  3242.                   test_flush;
  3243.                 }
  3244.               } else {    // mauvaise casse, effacer entrΘe dans la pile et rejouer une fois
  3245.                 get_it=1;
  3246.               }
  3247.             } else {        // adresse diffΘrente
  3248.               if (ishtml(mov_url)==0) {   // pas mΩme adresse MAIS c'est un fichier non html (pas de page moved possible)
  3249.                 // -> on prend α cette adresse, le lien sera enregistrΘ avec lien_record() (hash)
  3250.                 if ((opt->debug>1) && (opt->log!=NULL)) {
  3251.                   fspc(opt->log,"debug"); fprintf(opt->log,"wizard link test for moved file at %s%s.."LF,mov_adr,mov_fil);
  3252.                   test_flush;
  3253.                 }
  3254.                 // acceptΘ?
  3255.                 if (hts_acceptlink(opt,ptr,lien_tot,liens,
  3256.                   mov_adr,mov_fil,
  3257.                   NULL, NULL,
  3258.                   &set_prio_to,
  3259.                   NULL) != 1) {                /* nouvelle adresse non refusΘe ? */
  3260.                     get_it=1;
  3261.                     if ((opt->debug>1) && (opt->log!=NULL)) {
  3262.                       fspc(opt->log,"debug"); fprintf(opt->log,"moved link accepted: %s%s"LF,mov_adr,mov_fil);
  3263.                       test_flush;
  3264.                     }
  3265.                   }
  3266.               } /* sinon traitΘ normalement */
  3267.             }
  3268.  
  3269.             //if ((strfield2(mov_adr,urladr)!=0) && (strfield2(mov_fil,urlfil)!=0)) {  // identique α casse prΦs
  3270.             if (get_it==1) {
  3271.               // court-circuiter le reste du traitement
  3272.               // et reculer pour mieux sauter
  3273.               if (opt->errlog) {
  3274.                 fspc(opt->errlog,"warning"); fprintf(opt->errlog,"Warning moved treated for %s%s (real one is %s%s)"LF,urladr,urlfil,mov_adr,mov_fil);
  3275.                 test_flush;
  3276.               }          
  3277.               // canceller lien actuel
  3278.               error=1;
  3279.               strcpybuff(liens[ptr]->adr,"!");  // caractΦre bidon (invalide hash)
  3280.               // noter NOUVEAU lien
  3281.               //xxc xxc
  3282.               //  set_prio_to=0+1;  // protection if the moved URL is an html page!!
  3283.               //xxc xxc
  3284.               {
  3285.                 char BIGSTK mov_sav[HTS_URLMAXSIZE*2];
  3286.                 // calculer lien et Θventuellement modifier addresse/fichier
  3287.                 if (url_savename(mov_adr,mov_fil,mov_sav,NULL,NULL,liens[liens[ptr]->precedent]->adr,liens[liens[ptr]->precedent]->fil,opt,liens,lien_tot,sback,cache,hash,ptr,numero_passe,NULL)!=-1) { 
  3288.                   if (hash_read(hash,mov_sav,"",0,0)<0) {      // n'existe pas dΘja
  3289.                     // enregistrer lien (MACRO) avec SAV IDENTIQUE
  3290.                     liens_record(mov_adr,mov_fil,liens[ptr]->sav,"","");
  3291.                     //liens_record(mov_adr,mov_fil,mov_sav,"","");
  3292.                     if (liens[lien_tot]!=NULL) {    // OK, pas d'erreur
  3293.                       // mode test?
  3294.                       liens[lien_tot]->testmode=liens[ptr]->testmode;
  3295.                       liens[lien_tot]->link_import=0;       // mode normal
  3296.                       if (!set_prio_to)
  3297.                         liens[lien_tot]->depth=liens[ptr]->depth;
  3298.                       else
  3299.                         liens[lien_tot]->depth=max(0,min(set_prio_to-1,liens[ptr]->depth));       // PRIORITE NULLE (catch page)
  3300.                       liens[lien_tot]->pass2=max(liens[ptr]->pass2,numero_passe);
  3301.                       liens[lien_tot]->retry=liens[ptr]->retry;
  3302.                       liens[lien_tot]->premier=liens[ptr]->premier;
  3303.                       liens[lien_tot]->precedent=liens[ptr]->precedent;
  3304.                       lien_tot++;
  3305.                     } else {  // oups erreur, plus de mΘmoire!!
  3306.                       printf("PANIC! : Not enough memory [%d]\n",__LINE__);
  3307.                       if (opt->errlog) {
  3308.                         fprintf(opt->errlog,"Not enough memory, can not re-allocate %d bytes"LF,(int)((add_tab_alloc+1)*sizeof(lien_url)));
  3309.                         test_flush;
  3310.                       }
  3311.                       //if (opt->getmode & 1) { if (fp) { fclose(fp); fp=NULL; } }
  3312.                       XH_uninit;    // dΘsallocation mΘmoire & buffers
  3313.                       return 0;
  3314.                     }
  3315.                   } else {
  3316.                     if ( (opt->debug>0) && (opt->errlog!=NULL) ) {
  3317.                       fspc(opt->errlog,"warning"); fprintf(opt->errlog,"moving %s to an existing file %s"LF,liens[ptr]->fil,urlfil);
  3318.                       test_flush;
  3319.                     }
  3320.                   }
  3321.  
  3322.                 }
  3323.               }
  3324.  
  3325.               //printf("-> %s %s %s\n",liens[lien_tot-1]->adr,liens[lien_tot-1]->fil,liens[lien_tot-1]->sav);
  3326.  
  3327.               // note mΘtaphysique: il se peut qu'il y ait un index.html et un INDEX.HTML
  3328.               // sous DOS ca marche pas trΦs bien... mais comme je suis gΘnial url_savename()
  3329.               // est α mΩme de rΘgler ce problΦme
  3330.             }
  3331.           } // ident_url_xx
  3332.  
  3333.           if (get_it==0) {    // adresse vraiment diffΘrente et potentiellement en html (pas de possibilitΘ de bouger la page tel quel α cause des <img src..> et cie)
  3334.             rn=(char*) calloct(8192,1);
  3335.             if (rn!=NULL) {
  3336.               if (opt->errlog) {
  3337.                 fspc(opt->errlog,"warning"); fprintf(opt->errlog,"File has moved from %s%s to %s"LF,urladr,urlfil,mov_url);
  3338.                 test_flush;
  3339.               }
  3340.               if (!opt->mimehtml) {
  3341.                 escape_uri(mov_url);
  3342.               } else {
  3343.                 char BIGSTK buff[HTS_URLMAXSIZE*3];
  3344.                 strcpybuff(buff, mov_adr);
  3345.                 strcatbuff(buff, mov_fil);
  3346.                 escape_in_url(buff);
  3347.                 { char* a = buff; while((a = strchr(a, '%'))) { *a = 'X'; a++; } }
  3348.                 strcpybuff(mov_url, "cid:");
  3349.                 strcatbuff(mov_url, buff);
  3350.               }
  3351.               // On prΘpare une page qui sautera immΘdiatement sur la bonne URL
  3352.               // Le scanner re-changera, ensuite, cette URL, pour la mirrorer!
  3353.               strcpybuff(rn,"<HTML>"CRLF);
  3354.               strcatbuff(rn,"<!-- Created by HTTrack Website Copier/"HTTRACK_VERSION" "HTTRACK_AFF_AUTHORS" -->"CRLF);
  3355.               strcatbuff(rn,"<HEAD>"CRLF"<TITLE>Page has moved</TITLE>"CRLF"</HEAD>"CRLF"<BODY>"CRLF);
  3356.               strcatbuff(rn,"<META HTTP-EQUIV=\"Refresh\" CONTENT=\"0; URL=");
  3357.               strcatbuff(rn,mov_url);    // URL
  3358.               strcatbuff(rn,"\">"CRLF);
  3359.               strcatbuff(rn,"<A HREF=\"");
  3360.               strcatbuff(rn,mov_url);
  3361.               strcatbuff(rn,"\">");
  3362.               strcatbuff(rn,"<B>Click here...</B></A>"CRLF);
  3363.               strcatbuff(rn,"</BODY>"CRLF);
  3364.               strcatbuff(rn,"<!-- Created by HTTrack Website Copier/"HTTRACK_VERSION" "HTTRACK_AFF_AUTHORS" -->"CRLF);
  3365.               strcatbuff(rn,"</HTML>"CRLF);
  3366.  
  3367.               // changer la page
  3368.               if (r->adr) { 
  3369.                 freet(r->adr); 
  3370.                 r->adr=NULL; 
  3371.               }
  3372.               r->adr=rn;
  3373.               r->size=strlen(r->adr);
  3374.               strcpybuff(r->contenttype, "text/html");
  3375.             }
  3376.           }  // get_it==0
  3377.  
  3378.         }     // bloc
  3379.         // erreur HTTP (ex: 404, not found)
  3380.       } else if (
  3381.         (r->statuscode==412)
  3382.         || (r->statuscode==416)
  3383.         ) {    // Precondition Failed, c'est α dire pour nous redemander TOUT le fichier
  3384.           if (fexist(liens[ptr]->sav)) {
  3385.             remove(liens[ptr]->sav);    // Eliminer
  3386.             if (!fexist(liens[ptr]->sav)) {  // Bien ΘliminΘ? (sinon on boucle..)
  3387. #if HDEBUG
  3388.               printf("Partial content NOT up-to-date, reget all file for %s\n",liens[ptr]->sav);
  3389. #endif
  3390.               if ( (opt->debug>1) && (opt->errlog!=NULL) ) {
  3391.                 //if (opt->errlog) {
  3392.                 fspc(opt->errlog,"debug"); fprintf(opt->errlog,"Partial file reget (%s) for %s%s"LF,r->msg,urladr,urlfil);
  3393.                 test_flush;
  3394.               }
  3395.               // enregistrer le MEME lien (MACRO)
  3396.               liens_record(liens[ptr]->adr,liens[ptr]->fil,liens[ptr]->sav,"","");
  3397.               if (liens[lien_tot]!=NULL) {    // OK, pas d'erreur
  3398.                 liens[lien_tot]->testmode=liens[ptr]->testmode;          // mode test?
  3399.                 liens[lien_tot]->link_import=0;       // pas mode import
  3400.                 liens[lien_tot]->depth=liens[ptr]->depth;
  3401.                 liens[lien_tot]->pass2=max(liens[ptr]->pass2,numero_passe);
  3402.                 liens[lien_tot]->retry=liens[ptr]->retry;
  3403.                 liens[lien_tot]->premier=liens[ptr]->premier;
  3404.                 liens[lien_tot]->precedent=ptr;
  3405.                 lien_tot++;
  3406.                 //
  3407.                 // canceller lien actuel
  3408.                 error=1;
  3409.                 strcpybuff(liens[ptr]->adr,"!");  // caractΦre bidon (invalide hash)
  3410.                 //
  3411.               } else {  // oups erreur, plus de mΘmoire!!
  3412.                 printf("PANIC! : Not enough memory [%d]\n",__LINE__);
  3413.                 if (opt->errlog) {
  3414.                   fprintf(opt->errlog,"Not enough memory, can not re-allocate %d bytes"LF,(int)((add_tab_alloc+1)*sizeof(lien_url)));
  3415.                   test_flush;
  3416.                 }
  3417.                 //if (opt->getmode & 1) { if (fp) { fclose(fp); fp=NULL; } }
  3418.                 XH_uninit;    // dΘsallocation mΘmoire & buffers
  3419.                 return 0;
  3420.               } 
  3421.             } else {
  3422.               if (opt->errlog!=NULL) {
  3423.                 fspc(opt->errlog,"error"); fprintf(opt->errlog,"Can not remove old file %s"LF,urlfil);
  3424.                 test_flush;
  3425.               }
  3426.             }
  3427.           } else {
  3428.             if (opt->errlog!=NULL) {
  3429.               fspc(opt->errlog,"warning"); fprintf(opt->errlog,"Unexpected 412/416 error (%s) for %s%s"LF,r->msg,urladr,urlfil);
  3430.               test_flush;
  3431.             }
  3432.           }
  3433.         } else if (r->statuscode!=200) {
  3434.           int can_retry=0;
  3435.  
  3436.           // cas o∙ l'on peut reessayer
  3437.           switch(r->statuscode) {
  3438.             //case -1: can_retry=1; break;
  3439.           case STATUSCODE_TIMEOUT:
  3440.             if (opt->hostcontrol) {    // timeout et retry ΘpuisΘs
  3441.               if ((opt->hostcontrol & 1) && (liens[ptr]->retry<=0)) {
  3442.                 if ((opt->debug>1) && (opt->log!=NULL)) {
  3443.                   fspc(opt->log,"debug"); fprintf(opt->log,"Link banned: %s%s"LF,urladr,urlfil); test_flush;
  3444.                 }
  3445.                 host_ban(opt,liens,ptr,lien_tot,sback,jump_identification(urladr));
  3446.                 if ((opt->debug>1) && (opt->log!=NULL)) {
  3447.                   fspc(opt->log,"debug"); fprintf(opt->log,"Info: previous log - link banned: %s%s"LF,urladr,urlfil); test_flush;
  3448.                 }
  3449.               } else can_retry=1;
  3450.             } else can_retry=1;
  3451.             break;
  3452.           case STATUSCODE_SLOW:
  3453.             if ((opt->hostcontrol) && (liens[ptr]->retry<=0)) {    // too slow
  3454.               if (opt->hostcontrol & 2) {
  3455.                 if ((opt->debug>1) && (opt->log!=NULL)) {
  3456.                   fspc(opt->log,"debug"); fprintf(opt->log,"Link banned: %s%s"LF,urladr,urlfil); test_flush;
  3457.                 }
  3458.                 host_ban(opt,liens,ptr,lien_tot,sback,jump_identification(urladr));
  3459.                 if ((opt->debug>1) && (opt->log!=NULL)) {
  3460.                   fspc(opt->log,"debug"); fprintf(opt->log,"Info: previous log - link banned: %s%s"LF,urladr,urlfil); test_flush;
  3461.                 }
  3462.               } else can_retry=1;
  3463.             } else can_retry=1;
  3464.             break;
  3465.           case STATUSCODE_CONNERROR:            // connect closed
  3466.             can_retry=1;
  3467.             break;
  3468.           case STATUSCODE_NON_FATAL:            // other (non fatal) error
  3469.             can_retry=1;
  3470.             break;
  3471.           case STATUSCODE_SSL_HANDSHAKE:            // bad SSL handskake
  3472.             can_retry=1;
  3473.             break;
  3474.           case 408: case 409: case 500: case 502: case 504: 
  3475.             can_retry=1;
  3476.             break;
  3477.           }
  3478.  
  3479.           if ( strcmp(liens[ptr]->fil,"/primary") != 0 ) {  // no primary (internal page 0)
  3480.             if ((liens[ptr]->retry<=0) || (!can_retry) ) {  // retry ΘpuisΘs (ou retry impossible)
  3481.               if (opt->errlog) {
  3482.                 if ((opt->retry>0) && (can_retry)){
  3483.                   fspc(opt->errlog,"error"); 
  3484.                   fprintf(opt->errlog,"\"%s\" (%d) after %d retries at link %s%s (from %s%s)"LF,r->msg,r->statuscode,opt->retry,urladr,urlfil,liens[liens[ptr]->precedent]->adr,liens[liens[ptr]->precedent]->fil);
  3485.                 } else {
  3486.                   if (r->statuscode==STATUSCODE_TEST_OK) {    // test OK
  3487.                     if ((opt->debug>0) && (opt->errlog!=NULL)) {
  3488.                       fspc(opt->errlog,"info"); 
  3489.                       fprintf(opt->errlog,"Test OK at link %s%s (from %s%s)"LF,urladr,urlfil,liens[liens[ptr]->precedent]->adr,liens[liens[ptr]->precedent]->fil);
  3490.                     }
  3491.                   } else {
  3492.                     if (strcmp(urlfil,"/robots.txt")) {       // ne pas afficher d'infos sur robots.txt par dΘfaut
  3493.                       fspc(opt->errlog,"error"); 
  3494.                       fprintf(opt->errlog,"\"%s\" (%d) at link %s%s (from %s%s)"LF,r->msg,r->statuscode,urladr,urlfil,liens[liens[ptr]->precedent]->adr,liens[liens[ptr]->precedent]->fil);
  3495.                     } else {
  3496.                       if (opt->debug>1) {
  3497.                         fspc(opt->errlog,"info"); fprintf(opt->errlog,"No robots.txt rules at %s"LF,urladr);
  3498.                         test_flush;
  3499.                       }
  3500.                     }
  3501.                   }
  3502.                 }
  3503.                 test_flush;
  3504.               }
  3505.  
  3506.               // NO error in trop level
  3507.               // due to the "no connection -> previous restored" hack
  3508.               // This prevent the engine from wiping all data if the website has been deleted (or moved)
  3509.               // since last time (which is quite annoying)
  3510.               if (liens[ptr]->precedent != 0) {
  3511.                 // ici on teste si on doit enregistrer la page tout de mΩme
  3512.                 if (opt->errpage) {
  3513.                   store_errpage=1;
  3514.                 }
  3515.               } else {
  3516.                 if (strcmp(urlfil,"/robots.txt") != 0) {
  3517.                   /*
  3518.                   This is an error caused by a link entered by the user
  3519.                   That is, link(s) entered by user are invalid (404, 500, connect error, proxy error->.)
  3520.                   If all links entered are invalid, the session failed and we will attempt to restore
  3521.                   the previous one
  3522.                   Example: Try to update a website which has been deleted remotely: this may delete
  3523.                   the website locally, which is really not desired (especially if the website disappeared!)
  3524.                   With this hack, the engine won't wipe local files (how clever)
  3525.                   */
  3526.                   HTS_STAT.stat_errors_front++;
  3527.                 }
  3528.               }
  3529.  
  3530.             } else {    // retry!!
  3531.               if (opt->debug>0 && opt->errlog != NULL) {  // on fera un alert si le retry Θchoue               
  3532.                 fspc(opt->errlog,"warning"); fprintf(opt->errlog,"Retry after error %d (%s) at link %s%s (from %s%s)"LF,r->statuscode,r->msg,urladr,urlfil,liens[liens[ptr]->precedent]->adr,liens[liens[ptr]->precedent]->fil);
  3533.                 test_flush;
  3534.               }
  3535.               // redemander fichier
  3536.               liens_record(urladr,urlfil,savename,"","");
  3537.               if (liens[lien_tot]!=NULL) {    // OK, pas d'erreur
  3538.                 liens[lien_tot]->testmode=liens[ptr]->testmode;          // mode test?
  3539.                 liens[lien_tot]->link_import=0;       // pas mode import
  3540.                 liens[lien_tot]->depth=liens[ptr]->depth;
  3541.                 liens[lien_tot]->pass2=max(liens[ptr]->pass2,numero_passe);
  3542.                 liens[lien_tot]->retry=liens[ptr]->retry-1;    // moins 1 retry!
  3543.                 liens[lien_tot]->premier=liens[ptr]->premier;
  3544.                 liens[lien_tot]->precedent=liens[ptr]->precedent;
  3545.                 lien_tot++;
  3546.               } else {  // oups erreur, plus de mΘmoire!!
  3547.                 printf("PANIC! : Not enough memory [%d]\n",__LINE__);
  3548.                 if (opt->errlog) {
  3549.                   fspc(opt->errlog,"panic"); 
  3550.                   fprintf(opt->errlog,"Not enough memory, can not re-allocate %d bytes"LF,(int)((add_tab_alloc+1)*sizeof(lien_url)));
  3551.                   test_flush;
  3552.                 }
  3553.                 //if (opt->getmode & 1) { if (fp) { fclose(fp); fp=NULL; } }
  3554.                 XH_uninit;    // dΘsallocation mΘmoire & buffers
  3555.                 return 0;
  3556.               } 
  3557.             }
  3558.           } else {
  3559.             if (opt->errlog) {
  3560.               if (opt->debug>1) {
  3561.                 fspc(opt->errlog,"info"); 
  3562.                 fprintf(opt->errlog,"Info: no robots.txt at %s%s"LF,urladr,urlfil);
  3563.               }
  3564.             }
  3565.           }
  3566.           if (!store_errpage) {
  3567.             if (r->adr) {     // dΘsalloc
  3568.               freet(r->adr); 
  3569.               r->adr=NULL; 
  3570.             }
  3571.             error=1;  // erreur!
  3572.           }
  3573.         }
  3574.         // FIN rattrapage des 301,302,307..
  3575.         // ------------------------------------------------------------
  3576.  
  3577.   }  // if !error
  3578.  
  3579.  
  3580.   /* Apply changes */
  3581.   ENGINE_SAVE_CONTEXT();
  3582.  
  3583.   return 0;
  3584.  
  3585.  
  3586. }
  3587.  
  3588.  
  3589.  
  3590. /*
  3591. Wait for next file and
  3592. check 301, 302, .. statuscodes (moved)
  3593. */
  3594. int hts_mirror_wait_for_next_file(htsmoduleStruct* str, htsmoduleStructExtended* stre) {
  3595.   /* Load engine variables */
  3596.   ENGINE_LOAD_CONTEXT();
  3597.   /* */
  3598.   int b;
  3599.   int n;
  3600.  
  3601. #if BDEBUG==1
  3602.   printf("\nBack test..\n");
  3603. #endif
  3604.  
  3605.   // pause/lock files
  3606.   {
  3607.     int do_pause=0;
  3608.  
  3609.     // user pause lockfile : create hts-paused.lock --> HTTrack will be paused
  3610.     if (fexist(fconcat(opt->path_log,"hts-stop.lock"))) {
  3611.       // remove lockfile
  3612.       remove(fconcat(opt->path_log,"hts-stop.lock"));
  3613.       if (!fexist(fconcat(opt->path_log,"hts-stop.lock"))) {
  3614.         do_pause=1;
  3615.       }
  3616.     }
  3617.  
  3618.     // after receving N bytes, pause
  3619.     if (opt->fragment>0) {
  3620.       if ((HTS_STAT.stat_bytes-stat_fragment) > opt->fragment) {
  3621.         do_pause=1;
  3622.       }
  3623.     }
  3624.  
  3625.     // pause?
  3626.     if (do_pause) {
  3627.       if ( (opt->debug>0) && (opt->log!=NULL) ) {
  3628.         fspc(opt->log,"info"); fprintf(opt->log,"engine: pause requested.."LF);
  3629.       }
  3630.       while (back_nsoc(sback)>0) {                  // attendre fin des transferts
  3631.         back_wait(sback,opt,cache,HTS_STAT.stat_timestart);
  3632.         Sleep(200);
  3633. #if HTS_ANALYSTE
  3634.         {
  3635.           back_wait(sback,opt,cache,HTS_STAT.stat_timestart);
  3636.  
  3637.           // Transfer rate
  3638.           engine_stats();
  3639.  
  3640.           // Refresh various stats
  3641.           HTS_STAT.stat_nsocket=back_nsoc(sback);
  3642.           HTS_STAT.stat_errors=fspc(NULL,"error");
  3643.           HTS_STAT.stat_warnings=fspc(NULL,"warning");
  3644.           HTS_STAT.stat_infos=fspc(NULL,"info");
  3645.           HTS_STAT.nbk=backlinks_done(sback,liens,lien_tot,ptr);
  3646.           HTS_STAT.nb=back_transfered(HTS_STAT.stat_bytes,sback);
  3647.  
  3648.           b=0;
  3649.           if (!hts_htmlcheck_loop(sback->lnk, sback->count, b,ptr,lien_tot,(int) (time_local()-HTS_STAT.stat_timestart),&HTS_STAT)
  3650.             || !back_checkmirror(opt)) {
  3651.               if (opt->errlog) {
  3652.                 fspc(opt->errlog,"info"); fprintf(opt->errlog,"Exit requested by shell or user"LF);
  3653.                 test_flush;
  3654.               }
  3655.               *stre->exit_xh_=1;  // exit requested
  3656.               XH_uninit;
  3657.               return 0;
  3658.             }
  3659.         }
  3660. #endif
  3661.       }
  3662.       // On dΘsalloue le buffer d'enregistrement des chemins crΘΘe, au cas o∙ pendant la pause
  3663.       // l'utilisateur ferait un rm -r aprΦs avoir effectuΘ un tar
  3664.       // structcheck_init(1);
  3665.       {
  3666.         FILE* fp = fopen(fconcat(opt->path_log,"hts-paused.lock"),"wb");
  3667.         if (fp) {
  3668.           fspc(fp,"info");  // dater
  3669.           fprintf(fp,"Pause"LF"HTTrack is paused after retreiving "LLintP" bytes"LF"Delete this file to continue the mirror->.."LF""LF"",(LLint)HTS_STAT.stat_bytes);
  3670.           fclose(fp);
  3671.         }
  3672.       }
  3673.       stat_fragment=HTS_STAT.stat_bytes;
  3674.       /* Info for wrappers */
  3675.       if ( (opt->debug>0) && (opt->log!=NULL) ) {
  3676.         fspc(opt->log,"info"); fprintf(opt->log,"engine: pause: %s"LF,fconcat(opt->path_log,"hts-paused.lock"));
  3677.       }
  3678. #if HTS_ANALYSTE
  3679.       hts_htmlcheck_pause(fconcat(opt->path_log,"hts-paused.lock"));
  3680. #else
  3681.       while (fexist(fconcat(opt->path_log,"hts-paused.lock"))) {
  3682.         //back_wait(sback,opt,cache,HTS_STAT.stat_timestart);   inutile!! (plus de sockets actives)
  3683.         Sleep(1000);
  3684.       }
  3685. #endif
  3686.     }
  3687.     //
  3688.   }
  3689.   // end of pause/lock files
  3690.  
  3691. #if HTS_ANALYSTE
  3692.   // changement dans les prΘfΘrences
  3693.   /*
  3694.   if (_hts_setopt) {
  3695.   copy_htsopt(_hts_setopt,opt);    // copier au besoin
  3696.   _hts_setopt=NULL;                 // effacer callback
  3697.   }
  3698.   */
  3699.   if (_hts_addurl) {
  3700.     char BIGSTK add_adr[HTS_URLMAXSIZE*2];
  3701.     char BIGSTK add_fil[HTS_URLMAXSIZE*2];
  3702.     while(*_hts_addurl) {
  3703.       char BIGSTK add_url[HTS_URLMAXSIZE*2];
  3704.       add_adr[0]=add_fil[0]=add_url[0]='\0';
  3705.       if (!link_has_authority(*_hts_addurl))
  3706.         strcpybuff(add_url,"http://");          // ajouter http://
  3707.       strcatbuff(add_url,*_hts_addurl);
  3708.       if (ident_url_absolute(add_url,add_adr,add_fil)>=0) {
  3709.         // ----Ajout----
  3710.         // noter NOUVEAU lien
  3711.         char BIGSTK add_sav[HTS_URLMAXSIZE*2];
  3712.         // calculer lien et Θventuellement modifier addresse/fichier
  3713.         if (url_savename(add_adr,add_fil,add_sav,NULL,NULL,NULL,NULL,opt,liens,lien_tot,sback,cache,hash,ptr,numero_passe,NULL)!=-1) { 
  3714.           if (hash_read(hash,add_sav,"",0,0)<0) {      // n'existe pas dΘja
  3715.             // enregistrer lien (MACRO)
  3716.             liens_record(add_adr,add_fil,add_sav,"","");
  3717.             if (liens[lien_tot]!=NULL) {    // OK, pas d'erreur
  3718.               liens[lien_tot]->testmode=0;          // mode test?
  3719.               liens[lien_tot]->link_import=0;       // mode normal
  3720.               liens[lien_tot]->depth=opt->depth;
  3721.               liens[lien_tot]->pass2=max(0,numero_passe);
  3722.               liens[lien_tot]->retry=opt->retry;
  3723.               liens[lien_tot]->premier=lien_tot;
  3724.               liens[lien_tot]->precedent=lien_tot;
  3725.               lien_tot++;
  3726.               //
  3727.               if ((opt->debug>0) && (opt->log!=NULL)) {
  3728.                 fspc(opt->log,"info"); fprintf(opt->log,"Link added by user: %s%s"LF,add_adr,add_fil); test_flush;
  3729.               }
  3730.               //
  3731.             } else {  // oups erreur, plus de mΘmoire!!
  3732.               printf("PANIC! : Not enough memory [%d]\n",__LINE__);
  3733.               if (opt->errlog) {
  3734.                 fprintf(opt->errlog,"Not enough memory, can not re-allocate %d bytes"LF,(int)((add_tab_alloc+1)*sizeof(lien_url)));
  3735.                 test_flush;
  3736.               }
  3737.               //if (opt->getmode & 1) { if (fp) { fclose(fp); fp=NULL; } }
  3738.               XH_uninit;    // dΘsallocation mΘmoire & buffers
  3739.               return 0;
  3740.             }
  3741.           } else {
  3742.             if ( (opt->debug>0) && (opt->errlog!=NULL) ) {
  3743.               fspc(opt->errlog,"warning"); fprintf(opt->errlog,"Existing link %s%s not added after user request"LF,add_adr,add_fil);
  3744.               test_flush;
  3745.             }
  3746.           }
  3747.  
  3748.         }
  3749.       } else {
  3750.         if (opt->errlog) {
  3751.           fspc(opt->errlog,"error");
  3752.           fprintf(opt->errlog,"Error during URL decoding for %s"LF,add_url);
  3753.           test_flush;
  3754.         }
  3755.       }
  3756.       // ----Fin Ajout----
  3757.       _hts_addurl++;                  // suivante
  3758.     }
  3759.     _hts_addurl=NULL;           // libΘrer _hts_addurl
  3760.   }
  3761.   // si une pause a ΘtΘ demandΘe
  3762.   if (_hts_setpause || back_pluggable_sockets_strict(sback, opt) <= 0) {
  3763.     // index du lien actuel
  3764.     int b=back_index(sback,urladr,urlfil,savename);
  3765.     int prev = _hts_in_html_parsing;
  3766.     if (b<0) b=0;    // forcer pour les stats
  3767.     while(_hts_setpause || back_pluggable_sockets_strict(sback, opt) <= 0) {    // on fait la pause..
  3768.       _hts_in_html_parsing = 6;
  3769.       back_wait(sback,opt,cache,HTS_STAT.stat_timestart);
  3770.  
  3771.       // Transfer rate
  3772.       engine_stats();
  3773.  
  3774.       // Refresh various stats
  3775.       HTS_STAT.stat_nsocket=back_nsoc(sback);
  3776.       HTS_STAT.stat_errors=fspc(NULL,"error");
  3777.       HTS_STAT.stat_warnings=fspc(NULL,"warning");
  3778.       HTS_STAT.stat_infos=fspc(NULL,"info");
  3779.       HTS_STAT.nbk=backlinks_done(sback,liens,lien_tot,ptr);
  3780.       HTS_STAT.nb=back_transfered(HTS_STAT.stat_bytes,sback);
  3781.  
  3782.       if (!hts_htmlcheck_loop(sback->lnk, sback->count, b,ptr,lien_tot,(int) (time_local()-HTS_STAT.stat_timestart),&HTS_STAT)) {
  3783.         if (opt->errlog) {
  3784.           fspc(opt->errlog,"info"); fprintf(opt->errlog,"Exit requested by shell or user"LF);
  3785.           test_flush;
  3786.         }
  3787.         *stre->exit_xh_=1;  // exit requested
  3788.         XH_uninit;
  3789.         return 0;
  3790.       }
  3791.       Sleep(100);  // pause
  3792.     }
  3793.     _hts_in_html_parsing = prev;
  3794.   }
  3795. #endif
  3796.  
  3797.   // si le fichier n'est pas en backing, le mettre..
  3798.   if (!back_exist(sback,urladr,urlfil,savename)) {
  3799. #if BDEBUG==1
  3800.     printf("crash backing: %s%s\n",liens[ptr]->adr,liens[ptr]->fil);
  3801. #endif
  3802.     if (back_add(sback,opt,cache,urladr,urlfil,savename,liens[liens[ptr]->precedent]->adr,liens[liens[ptr]->precedent]->fil,liens[ptr]->testmode,&liens[ptr]->pass2)==-1) {
  3803.       printf("PANIC! : Crash adding error, unexpected error found.. [%d]\n",__LINE__);
  3804. #if BDEBUG==1
  3805.       printf("error while crash adding\n");
  3806. #endif
  3807.       if (opt->errlog) {
  3808.         fspc(opt->errlog,"error"); fprintf(opt->errlog,"Unexpected backing error for %s%s"LF,urladr,urlfil);
  3809.         test_flush;
  3810.       } 
  3811.  
  3812.     }
  3813.   }
  3814.  
  3815. #if BDEBUG==1
  3816.   printf("test number of socks\n");
  3817. #endif
  3818.  
  3819.   // ajouter autant de socket qu'on peut ajouter
  3820.   n=opt->maxsoc-back_nsoc(sback);
  3821. #if BDEBUG==1
  3822.   printf("%d sockets available for backing\n",n);
  3823. #endif
  3824.  
  3825. #if HTS_ANALYSTE
  3826.   if ((n>0) && (!_hts_setpause)) {   // si sockets libre et pas en pause, ajouter
  3827. #else
  3828.   if (n>0) {                         // si sockets libre
  3829. #endif
  3830.     // remplir autant que l'on peut le cache (backing)
  3831.     back_fillmax(sback,opt,cache,liens,ptr,numero_passe,lien_tot);
  3832.   }
  3833.  
  3834.   // index du lien actuel
  3835.   /*
  3836.   b=back_index(sback,urladr,urlfil,savename);
  3837.  
  3838.   if (b>=0) 
  3839.   */
  3840.   {
  3841.     // ------------------------------------------------------------
  3842.     // attendre que le fichier actuel soit prΩt - BOUCLE D'ATTENTE
  3843.     do {
  3844.  
  3845.       // index du lien actuel
  3846.       b=back_index(sback,urladr,urlfil,savename);
  3847. #if BDEBUG==1
  3848.       printf("back index %d, waiting\n",b);
  3849. #endif
  3850.       // Continue to the loop if link still present
  3851.       if (b<0)
  3852.         break;
  3853.  
  3854.       // Receive data
  3855.       if (back[b].status>0)
  3856.         back_wait(sback,opt,cache,HTS_STAT.stat_timestart);
  3857.  
  3858.       // Continue to the loop if link still present
  3859.       b=back_index(sback,urladr,urlfil,savename);
  3860.       if (b<0)
  3861.         break;
  3862.  
  3863.       // Stop the mirror
  3864.       if (!back_checkmirror(opt)) {
  3865.         *stre->exit_xh_=1;  // exit requested
  3866.         XH_uninit;
  3867.         return 0;
  3868.       }
  3869.  
  3870.       // And fill the backing stack
  3871.       if (back[b].status>0)
  3872.         back_fillmax(sback,opt,cache,liens,ptr,numero_passe,lien_tot);
  3873.  
  3874.       // Continue to the loop if link still present
  3875.       b=back_index(sback,urladr,urlfil,savename);
  3876.       if (b<0)
  3877.         break;
  3878.  
  3879.       // autres occupations de HTTrack: statistiques, boucle d'attente, etc.
  3880.       if ((opt->makestat) || (opt->maketrack)) {
  3881.         TStamp l=time_local();
  3882.         if ((int) (l-makestat_time) >= 60) {   
  3883.           if (makestat_fp != NULL) {
  3884.             fspc(makestat_fp,"info");
  3885.             fprintf(makestat_fp,"Rate= %d (/"LLintP") \11NewLinks= %d (/%d)"LF,(int) ((HTS_STAT.HTS_TOTAL_RECV-*stre->makestat_total_)/(l-makestat_time)), (LLint)HTS_STAT.HTS_TOTAL_RECV,(int) lien_tot-*stre->makestat_lnk_,(int) lien_tot);
  3886.             fflush(makestat_fp);
  3887.             *stre->makestat_total_=HTS_STAT.HTS_TOTAL_RECV;
  3888.             *stre->makestat_lnk_=lien_tot;
  3889.           }
  3890.           if (stre->maketrack_fp != NULL) {
  3891.             int i;
  3892.             fspc(stre->maketrack_fp,"info"); fprintf(stre->maketrack_fp,LF);
  3893.             for(i=0;i<back_max;i++) {
  3894.               back_info(sback,i,3,stre->maketrack_fp);
  3895.             }
  3896.             fprintf(stre->maketrack_fp,LF);
  3897.             fflush(stre->maketrack_fp);
  3898.  
  3899.           }
  3900.           makestat_time=l;
  3901.         }
  3902.       }
  3903. #if HTS_ANALYSTE
  3904.       {
  3905.         int i;
  3906.         {
  3907.           char* s=hts_cancel_file("");
  3908.           if (strnotempty(s)) {    // fichier α canceller
  3909.             for(i=0;i<back_max;i++) {
  3910.               if ((back[i].status>0)) {
  3911.                 if (strcmp(back[i].url_sav,s)==0) {  // ok trouvΘ
  3912.                   if (back[i].status != 1000) {
  3913. #if HTS_DEBUG_CLOSESOCK
  3914.                     DEBUG_W("user cancel: deletehttp\n");
  3915. #endif
  3916.                     if (back[i].r.soc!=INVALID_SOCKET) deletehttp(&back[i].r);
  3917.                     back[i].r.soc=INVALID_SOCKET;
  3918.                     back[i].r.statuscode=STATUSCODE_INVALID;
  3919.                     strcpybuff(back[i].r.msg,"Cancelled by User");
  3920.                     back[i].status=0;  // terminΘ
  3921.                     back_set_finished(sback, i);
  3922.                   } else    // cancel ftp.. flag α 1
  3923.                     back[i].stop_ftp = 1;
  3924.                 }
  3925.               }
  3926.             }
  3927.             s[0]='\0';
  3928.           }
  3929.         }
  3930.  
  3931.         // Transfer rate
  3932.         engine_stats();
  3933.  
  3934.         // Refresh various stats
  3935.         HTS_STAT.stat_nsocket=back_nsoc(sback);
  3936.         HTS_STAT.stat_errors=fspc(NULL,"error");
  3937.         HTS_STAT.stat_warnings=fspc(NULL,"warning");
  3938.         HTS_STAT.stat_infos=fspc(NULL,"info");
  3939.         HTS_STAT.nbk=backlinks_done(sback,liens,lien_tot,ptr);
  3940.         HTS_STAT.nb=back_transfered(HTS_STAT.stat_bytes,sback);
  3941.  
  3942.         if (!hts_htmlcheck_loop(sback->lnk, sback->count, b,ptr,lien_tot,(int) (time_local()-HTS_STAT.stat_timestart),&HTS_STAT)) {
  3943.           if (opt->errlog) {
  3944.             fspc(opt->errlog,"info"); fprintf(opt->errlog,"Exit requested by shell or user"LF);
  3945.             test_flush;
  3946.           } 
  3947.           *stre->exit_xh_=1;  // exit requested
  3948.           XH_uninit;
  3949.           return 0;
  3950.         }
  3951.       }
  3952.  
  3953. #endif
  3954. #if HTS_POLL
  3955.       if ((opt->shell) || (opt->keyboard) || (opt->verbosedisplay) || (!opt->quiet)) {
  3956.         TStamp tl;
  3957.         *stre->info_shell_=1;
  3958.  
  3959.         /* Toggle with ENTER */
  3960.         if (!opt->quiet) {
  3961.           if (check_stdin()) {
  3962.             char com[256];
  3963.             linput(stdin,com,200);
  3964.             if (opt->verbosedisplay==2)
  3965.               opt->verbosedisplay=1;
  3966.             else
  3967.               opt->verbosedisplay=2;
  3968.             /* Info for wrappers */
  3969.             if ( (opt->debug>0) && (opt->log!=NULL) ) {
  3970.               fspc(opt->log,"info"); fprintf(opt->log,"engine: change-options"LF);
  3971.             }
  3972. #if HTS_ANALYSTE
  3973.             hts_htmlcheck_chopt(opt);
  3974. #endif
  3975.           }
  3976.         }
  3977.  
  3978.         tl=time_local();
  3979.  
  3980.         // gΘnΘrer un message d'infos sur l'Θtat actuel
  3981.         if (opt->shell) {    // si shell
  3982.           if ((tl-*stre->last_info_shell_)>0) {    // toute les 1 sec
  3983.             FILE* fp=stdout;
  3984.             int a=0;
  3985.             *stre->last_info_shell_=tl;
  3986.             if (fexist(fconcat(opt->path_log,"hts-autopsy"))) {  // dΘbuggage: teste si le robot est vivant
  3987.               // (oui je sais un robot vivant.. mais bon.. il a le droit de vivre lui aussi)
  3988.               // (libΘrons les robots esclaves de l'internet!)
  3989.               remove(fconcat(opt->path_log,"hts-autopsy"));
  3990.               fp=fopen(fconcat(opt->path_log,"hts-isalive"),"wb");
  3991.               a=1;
  3992.             }
  3993.             if ((*stre->info_shell_) || a) {
  3994.               int i,j;
  3995.  
  3996.               fprintf(fp,"TIME %d"LF,(int) (tl-HTS_STAT.stat_timestart));
  3997.               fprintf(fp,"TOTAL %d"LF,(int) HTS_STAT.stat_bytes);
  3998.               fprintf(fp,"RATE %d"LF,(int) (HTS_STAT.HTS_TOTAL_RECV/(tl-HTS_STAT.stat_timestart)));
  3999.               fprintf(fp,"SOCKET %d"LF,back_nsoc(sback));
  4000.               fprintf(fp,"LINK %d"LF,lien_tot);
  4001.               {
  4002.                 LLint mem=0;
  4003.                 for(i=0;i<back_max;i++)
  4004.                   if (back[i].r.adr!=NULL)
  4005.                     mem+=back[i].r.size;
  4006.                 fprintf(fp,"INMEM "LLintP""LF,(LLint)mem);
  4007.               }
  4008.               for(j=0;j<2;j++) {  // passes pour ready et wait
  4009.                 for(i=0;i<back_max;i++) {
  4010.                   back_info(sback,i,j+1,stdout);    // maketrack_fp a la place de stdout ?? // **
  4011.                 }
  4012.               }
  4013.               fprintf(fp,LF);
  4014.               if (a)
  4015.                 fclose(fp);
  4016.               io_flush;
  4017.             }
  4018.           }
  4019.         }  // si shell
  4020.  
  4021.       }  // si shell ou keyboard (option)
  4022.       //
  4023. #endif
  4024.     } while((b>=0) && (back[max(b,0)].status>0));
  4025.  
  4026.  
  4027.     // If link not found on the stack, it's because it has already been downloaded
  4028.     // in background
  4029.     // Then, skip it and go to the next one
  4030.     if (b<0) {
  4031.       if ((opt->debug>1) && (opt->log!=NULL)) {
  4032.         fspc(opt->log,"debug"); fprintf(opt->log,"link #%d is ready, no more on the stack, skipping: %s%s.."LF,ptr,urladr,urlfil);
  4033.         test_flush;
  4034.       }
  4035.  
  4036.       // prochain lien
  4037.       // ptr++;
  4038.  
  4039.       return 2; // goto jump_if_done;
  4040.  
  4041.     }
  4042. #if 0
  4043.     /* FIXME - finalized HAS NO MORE THIS MEANING */
  4044.     /* link put in cache by the backing system for memory spare - reclaim */
  4045.     else if (back[b].finalized) {
  4046.       assertf(back[b].r.adr == NULL);
  4047.       /* read file in cache */
  4048.       back[b].r = cache_read_ro(opt,cache,back[b].url_adr,back[b].url_fil,back[b].url_sav, back[b].location_buffer);
  4049.       /* ensure correct location buffer set */
  4050.       back[b].r.location=back[b].location_buffer;
  4051.       if (back[b].r.statuscode == STATUSCODE_INVALID) {
  4052.         if (opt->errlog) {
  4053.           fspc(opt->errlog,"error"); fprintf(opt->errlog,"Unexpected error: %s%s not found anymore in cache"LF,back[b].url_adr,back[b].url_fil);
  4054.           test_flush;
  4055.         }
  4056.       } else {
  4057.         if ( (opt->debug>1) && (opt->log!=NULL) ) {
  4058.           fspc(opt->log,"debug"); fprintf(opt->log,"reclaim file %s%s (%d)"LF,back[b].url_adr,back[b].url_fil,back[b].r.statuscode); test_flush;
  4059.         }
  4060.       }
  4061.     }
  4062. #endif
  4063.  
  4064. #if HTS_ANALYSTE==2
  4065. #else
  4066.     //if (!opt->quiet) {  // petite animation
  4067.     if (!opt->verbosedisplay) {
  4068.       if (!opt->quiet) {
  4069.         static int roll=0;  /* static: ok */
  4070.         roll=(roll+1)%4;
  4071.         printf("%c\x0d",("/-\\|")[roll]);
  4072.         fflush(stdout);
  4073.       }
  4074.     } else if (opt->verbosedisplay==1) {
  4075.       if (b >= 0) {
  4076.         if (back[b].r.statuscode==200)
  4077.           printf("%d/%d: %s%s ("LLintP" bytes) - OK\33[K\r",ptr,lien_tot,back[b].url_adr,back[b].url_fil,(LLint)back[b].r.size);
  4078.         else
  4079.           printf("%d/%d: %s%s ("LLintP" bytes) - %d\33[K\r",ptr,lien_tot,back[b].url_adr,back[b].url_fil,(LLint)back[b].r.size,back[b].r.statuscode);
  4080.       } else {
  4081.         fspc(opt->errlog,"error"); fprintf(opt->errlog,"Link disappeared");
  4082.       }
  4083.       fflush(stdout);
  4084.     }
  4085.     //}
  4086. #endif
  4087.     // ------------------------------------------------------------
  4088.     // VΘrificateur d'intΘgritΘ
  4089. #if DEBUG_CHECKINT
  4090.     _CHECKINT(&back[b],"Retour de back_wait, aprΦs le while")
  4091.     {
  4092.       int i;
  4093.       for(i=0;i<back_max;i++) {
  4094.         char si[256];
  4095.         sprintf(si,"Test global aprΦs back_wait, index %d",i);
  4096.         _CHECKINT(&back[i],si)
  4097.       }
  4098.     }
  4099. #endif
  4100.  
  4101.     // copier structure rΘponse htsblk
  4102.     if (b >= 0) {
  4103.       memcpy(r, &(back[b].r), sizeof(htsblk));
  4104.       r->location=stre->loc_;    // ne PAS copier location!! adresse, pas de buffer
  4105.       if (back[b].r.location) 
  4106.         strcpybuff(r->location,back[b].r.location);
  4107.       back[b].r.adr=NULL;    // ne pas faire de desalloc ensuite
  4108.  
  4109.       // libΘrer emplacement backing
  4110.       back_maydelete(opt,cache,sback,b);
  4111.     }
  4112.  
  4113.     // progression
  4114. #if 0
  4115.     if (opt->aff_progress) {
  4116.       TStamp tl=time_local();
  4117.       if ((tl-HTS_STAT.stat_timestart)>0) {
  4118.         char s[32];
  4119.         int i=0;
  4120.         lastime=tl;
  4121.         _CLRSCR; _GOTOXY("1","1");
  4122.         printf("Rate=%d B/sec\n",(int) (HTS_STAT.HTS_TOTAL_RECV/(tl-HTS_STAT.stat_timestart)));
  4123.         while(i<minimum(back_max,99)) {  // **
  4124.           if (back[i].status>=0) {  // loading..
  4125.             s[0]='\0';
  4126.             if (strlen(back[i].url_fil)>16)
  4127.               strcatbuff(s,back[i].url_fil+strlen(back[i].url_fil)-16);       
  4128.             else
  4129.               strncatbuff(s,back[i].url_fil,16);
  4130.             printf("%s : ",s);
  4131.  
  4132.             printf("[");
  4133.             if (back[i].r.totalsize>0) {
  4134.               int p;
  4135.               int j;
  4136.               p=(int)((back[i].r.size*10)/back[i].r.totalsize);
  4137.               p=minimum(10,p);
  4138.               for(j=0;j<p;j++) printf("*");
  4139.               for(j=0;j<(10-p);j++) printf("-");
  4140.             } else { 
  4141.               printf(LLintP,(LLint)back[i].r.size);                      
  4142.             }
  4143.             printf("]");
  4144.  
  4145.             //} else if (back[i].status==0) {
  4146.             //  strcpybuff(s,"ENDED");
  4147.           } 
  4148.           printf("\n");
  4149.           i++;
  4150.         }
  4151.         io_flush;
  4152.       }
  4153.     }
  4154. #endif
  4155.  
  4156.     // dΘbug graphique
  4157. #if BDEBUG==2
  4158.     {
  4159.       char s[12];
  4160.       int i=0;
  4161.       _GOTOXY(1,1);
  4162.       printf("Rate=%d B/sec\n",(int) (HTS_STAT.HTS_TOTAL_RECV/(time_local()-HTS_STAT.stat_timestart)));
  4163.       while(i<minimum(back_max,160)) {
  4164.         if (back[i].status>0) {
  4165.           sprintf(s,"%d",back[i].r.size);
  4166.         } else if (back[i].status==0) {
  4167.           strcpybuff(s,"ENDED");
  4168.         } else 
  4169.           strcpybuff(s,"   -   ");
  4170.         while(strlen(s)<8) strcatbuff(s," ");
  4171.         printf("%s",s); io_flush;
  4172.         i++;
  4173.       }
  4174.     }
  4175. #endif
  4176.  
  4177.  
  4178. #if BDEBUG==1
  4179.     printf("statuscode=%d with %s / msg=%s\n",r->statuscode,r->contenttype,r->msg);
  4180. #endif
  4181.  
  4182.   }
  4183.   /*else {
  4184.   #if BDEBUG==1
  4185.   printf("back index error\n");
  4186.   #endif
  4187.   }
  4188.   */
  4189.  
  4190.   ENGINE_SAVE_CONTEXT();
  4191.   return 0;
  4192. }
  4193.  
  4194. /* Wait for delayed types */
  4195. int hts_wait_delayed(htsmoduleStruct* str, 
  4196.                      char* adr, char* fil, char* save, 
  4197.                      char* former_adr, char* former_fil, 
  4198.                      int* forbidden_url) {
  4199.   ENGINE_LOAD_CONTEXT_BASE();
  4200.   hash_struct* hash = hashptr;
  4201.  
  4202.   int r_sv=0;
  4203.  
  4204.   // resolve unresolved type
  4205.   if (opt->savename_delayed != 0
  4206.     && *forbidden_url == 0 
  4207.     && IS_DELAYED_EXT(save) 
  4208.     && !opt->state.stop
  4209.     )
  4210.   {
  4211.     int loops=0;
  4212.     int continue_loop = 1;
  4213.     if ((opt->debug>1) && (opt->log!=NULL)) {
  4214.       fspc(opt->log,"debug"); fprintf(opt->log,"Waiting for type to be known: %s%s"LF, adr, fil);
  4215.       test_flush;
  4216.     }
  4217.  
  4218.     /* Follow while type is unknown and redirects occurs */
  4219.     while(IS_DELAYED_EXT(save) && continue_loop && loops++ < 7) {
  4220.       continue_loop = 0;
  4221.  
  4222.       /*
  4223.       Wait for an available slot 
  4224.       */
  4225.       WAIT_FOR_AVAILABLE_SOCKET();
  4226.  
  4227.       /* We can lookup directly in the cache to speedup this mess */
  4228.       if (opt->delayed_cached) {
  4229.                 lien_back back;
  4230.                 memset(&back, 0, sizeof(back));
  4231.                 back.r = cache_read(opt, cache, adr, fil, NULL, NULL);              // test uniquement
  4232.         if (back.r.statuscode == 200 && strnotempty(back.r.contenttype)) {      // cache found, and aswer is 'OK'
  4233.           if ((opt->debug>1) && (opt->log!=NULL)) {
  4234.             fspc(opt->log,"debug"); fprintf(opt->log,"Direct type lookup in cache (-%%D1): %s"LF, back.r.contenttype);
  4235.             test_flush;
  4236.           }
  4237.  
  4238.           /* Recompute filename with MIME type */
  4239.           save[0] = '\0';
  4240.           r_sv=url_savename(adr,fil,save,former_adr,former_fil,liens[ptr]->adr,liens[ptr]->fil,opt,liens,lien_tot,sback,cache,hash,ptr,numero_passe,&back);
  4241.  
  4242.           /* Recompute authorization with MIME type */
  4243.           {
  4244.             int new_forbidden_url = hts_acceptmime(opt, ptr, lien_tot, liens, adr,fil, back.r.contenttype);
  4245.             if (new_forbidden_url != -1) {
  4246.               if ((opt->debug>1) && (opt->log!=NULL)) {
  4247.                 fspc(opt->log,"debug"); fprintf(opt->log,"result for wizard mime test: %s"LF,new_forbidden_url);
  4248.                 test_flush;
  4249.               }
  4250.               if (new_forbidden_url == 1) {
  4251.                 *forbidden_url = new_forbidden_url;
  4252.                 if ((opt->debug>1) && (opt->log!=NULL)) {
  4253.                   fspc(opt->log,"debug"); fprintf(opt->log,"link forbidden because of MIME types restrictions: %s%s"LF, adr, fil);
  4254.                   test_flush;
  4255.                 }
  4256.                 break;        // exit loop
  4257.               }
  4258.             }
  4259.           }
  4260.  
  4261.           /* And exit loop */
  4262.           break;
  4263.         }
  4264.       }
  4265.  
  4266.       /* Add in backing (back_index() will respond correctly) */
  4267.       if (back_add_if_not_exists(sback,opt,cache,adr,fil,save,NULL,NULL,0,NULL) != -1) {
  4268.         int b;
  4269.         b=back_index(sback,adr,fil,save); 
  4270.         if (b<0) {
  4271.           printf("PANIC! : Crash adding error, unexpected error found.. [%d]\n",__LINE__);
  4272.           XH_uninit;    // dΘsallocation mΘmoire & buffers
  4273.           return -1;
  4274.         }
  4275.  
  4276.         /* Cache read failed because file does not exists (bad delayed name!)
  4277.         Just re-add with the correct name, as we know the MIME now!
  4278.         */
  4279.         if (back[b].r.statuscode == STATUSCODE_INVALID && back[b].r.adr == NULL) {
  4280.                     lien_back delayed_back;
  4281.           //char BIGSTK delayed_ctype[128];
  4282.           // delayed_ctype[0] = '\0';
  4283.           // strncatbuff(delayed_ctype, back[b].r.contenttype, sizeof(delayed_ctype) - 1);    // copier content-type
  4284.                     back_copy_static(&back[b], &delayed_back);
  4285.  
  4286.           /* Delete entry */
  4287.           back_delete(opt,cache,sback,b);       // cancel
  4288.           b = -1;
  4289.  
  4290.           /* Recompute filename with MIME type */
  4291.           save[0] = '\0';
  4292.           r_sv=url_savename(adr,fil,save,former_adr,former_fil,liens[ptr]->adr,liens[ptr]->fil,opt,liens,lien_tot,sback,cache,hash,ptr,numero_passe,&delayed_back);
  4293.  
  4294.           /* Recompute authorization with MIME type */
  4295.           {
  4296.             int new_forbidden_url = hts_acceptmime(opt, ptr, lien_tot, liens, adr,fil, delayed_back.r.contenttype);
  4297.             if (new_forbidden_url != -1) {
  4298.               if ((opt->debug>1) && (opt->log!=NULL)) {
  4299.                 fspc(opt->log,"debug"); fprintf(opt->log,"result for wizard mime test: %d"LF,*forbidden_url);
  4300.                 test_flush;
  4301.               }
  4302.               if (new_forbidden_url == 1) {
  4303.                 *forbidden_url = new_forbidden_url;
  4304.                 if ((opt->debug>1) && (opt->log!=NULL)) {
  4305.                   fspc(opt->log,"debug"); fprintf(opt->log,"link forbidden because of MIME types restrictions: %s%s"LF, adr, fil);
  4306.                   test_flush;
  4307.                 }
  4308.                 break;        // exit loop
  4309.               }
  4310.             }
  4311.           }
  4312.  
  4313.           /* Re-Add wiht correct type */
  4314.           if (back_add_if_not_exists(sback,opt,cache,adr,fil,save,NULL,NULL,0,NULL) != -1) {
  4315.             b=back_index(sback,adr,fil,save); 
  4316.           }
  4317.           if (b<0) {
  4318.             printf("PANIC! : Crash adding error, unexpected error found.. [%d]\n",__LINE__);
  4319.             XH_uninit;    // dΘsallocation mΘmoire & buffers
  4320.             return -1;
  4321.           }
  4322.           if ((opt->debug>1) && (opt->log!=NULL)) {
  4323.             fspc(opt->log,"debug"); fprintf(opt->log,"Type immediately loaded from cache: %s"LF, delayed_back.r.contenttype);
  4324.             test_flush;
  4325.           }
  4326.         }
  4327.  
  4328.         /* Wait for headers to be received */
  4329.         do {
  4330.           if (b < 0)
  4331.             break;
  4332.  
  4333.           // temps α attendre, et remplir autant que l'on peut le cache (backing)
  4334.           if (back[b].status>0) {
  4335.             back_wait(sback,opt,cache,0);
  4336.           }
  4337.           if (ptr>=0) {
  4338.             back_fillmax(sback,opt,cache,liens,ptr,numero_passe,lien_tot);
  4339.           }
  4340.  
  4341.           // on est obligΘ d'appeler le shell pour le refresh..
  4342. #if HTS_ANALYSTE
  4343.           {
  4344.  
  4345.             // Transfer rate
  4346.             engine_stats();
  4347.  
  4348.             // Refresh various stats
  4349.             HTS_STAT.stat_nsocket=back_nsoc(sback);
  4350.             HTS_STAT.stat_errors=fspc(NULL,"error");
  4351.             HTS_STAT.stat_warnings=fspc(NULL,"warning");
  4352.             HTS_STAT.stat_infos=fspc(NULL,"info");
  4353.             HTS_STAT.nbk=backlinks_done(sback,liens,lien_tot,ptr);
  4354.             HTS_STAT.nb=back_transfered(HTS_STAT.stat_bytes,sback);
  4355.  
  4356.             if (!hts_htmlcheck_loop(sback->lnk, sback->count, b,ptr,lien_tot,(int) (time_local()-HTS_STAT.stat_timestart),&HTS_STAT)) {
  4357.               return -1;
  4358.             } else if (_hts_cancel || !back_checkmirror(opt)) {    // cancel 2 ou 1 (cancel parsing)
  4359.               back_delete(opt,cache,sback,b);       // cancel test
  4360.               break;
  4361.             }
  4362.           }
  4363. #endif
  4364.         } while(
  4365.           /* dns/connect/request */ 
  4366.                     ( back[b].status >= 99 && back[b].status <= 101 )
  4367.           ||
  4368.           /* For redirects, wait for request to be terminated */
  4369.           ( HTTP_IS_REDIRECT(back[b].r.statuscode) && back[b].status > 0 )
  4370.           ||
  4371.           /* Same for errors */
  4372.           ( HTTP_IS_ERROR(back[b].r.statuscode) && back[b].status > 0 )
  4373.           );
  4374.         /* ready (chunked) or ready (regular download) or ready (completed) */
  4375.  
  4376.         // Note: filename NOT in hashtable yet - liens_record will do it, with the correct ext!
  4377.         if (b >= 0) {
  4378.                     lien_back delayed_back;
  4379.           //char BIGSTK delayed_ctype[128];
  4380.           //delayed_ctype[0] = '\0';
  4381.           //strncatbuff(delayed_ctype, back[b].r.contenttype, sizeof(delayed_ctype) - 1);    // copier content-type
  4382.                     back_copy_static(&back[b], &delayed_back);
  4383.  
  4384.           /* Error */
  4385.           if (HTTP_IS_ERROR(back[b].r.statuscode))
  4386.           {
  4387.             /* 'no error page' selected or file discarded by size rules! */
  4388.             if (!opt->errpage || ( back[b].r.statuscode == STATUSCODE_TOO_BIG ) ) {
  4389.               /* Note: the cache 'cached_tests' system will remember this error, and we'll only issue ONE request */
  4390.               *forbidden_url = 1;          /* Forbidden! */
  4391.               if (opt->log != NULL) {
  4392.                 if (back[b].r.statuscode == STATUSCODE_TOO_BIG) {
  4393.                   fspc(opt->log,"error"); fprintf(opt->log,"link not taken because of its size (%d bytes) at %s%s"LF,(int)back[b].r.totalsize,adr,fil);
  4394.                 } else {
  4395.                   fspc(opt->log,"error"); fprintf(opt->log,"link not taken because of error (%d '%s') at %s%s"LF,back[b].r.statuscode,back[b].r.msg,adr,fil);
  4396.                 }
  4397.                 test_flush;
  4398.               }
  4399.               break;
  4400.             }
  4401.           }
  4402.           /* Moved! */
  4403.           else if (HTTP_IS_REDIRECT(back[b].r.statuscode))
  4404.           {
  4405.             char BIGSTK mov_url[HTS_URLMAXSIZE*2];
  4406.             mov_url[0] = '\0';
  4407.             strcpybuff(mov_url, back[b].r.location);    // copier URL
  4408.  
  4409.             /* Remove (temporarily created) file if it was created */
  4410.             unlink(fconv(back[b].url_sav));
  4411.  
  4412.             /* Remove slot! */
  4413.             if (back[b].status == 0) {
  4414.               back_maydelete(opt, cache, sback, b);
  4415.             } else {    /* should not happend */
  4416.               back_delete(opt, cache, sback, b);
  4417.             }
  4418.             b = -1;
  4419.  
  4420.             /* Handle redirect */
  4421.             if ((int) strnotempty(mov_url)) {    // location existe!
  4422.               char BIGSTK mov_adr[HTS_URLMAXSIZE*2],mov_fil[HTS_URLMAXSIZE*2];
  4423.               mov_adr[0]=mov_fil[0]='\0';
  4424.               //
  4425.               if (ident_url_relatif(mov_url,adr,fil,mov_adr,mov_fil)>=0) {                        
  4426.                 if ((opt->debug>1) && (opt->log!=NULL)) {
  4427.                   fspc(opt->log,"debug"); fprintf(opt->log,"Redirect while resolving type: %s%s -> %s%s"LF, adr, fil, mov_adr, mov_fil);
  4428.                   test_flush;
  4429.                 }
  4430.                 // si non bouclage sur soi mΩme, ou si test avec GET non testΘ
  4431.                 if (strcmp(mov_adr,adr) != 0 || strcmp(mov_fil,fil) != 0) {
  4432.  
  4433.                   // recopier former_adr/fil?
  4434.                   if ((former_adr) && (former_fil)) {
  4435.                     if (strnotempty(former_adr)==0) {    // Pas dΘja notΘ
  4436.                       strcpybuff(former_adr,adr);
  4437.                       strcpybuff(former_fil,fil);
  4438.                     }
  4439.                   }
  4440.  
  4441.                   // check explicit forbidden - don't follow 3xx in this case
  4442.                   {
  4443.                     int set_prio_to=0;
  4444.                     robots_wizard* robots = (robots_wizard*) opt->robotsptr;
  4445.                     if (hts_acceptlink(opt,ptr,lien_tot,liens,
  4446.                       mov_adr,mov_fil,
  4447.                       NULL, NULL,
  4448.                       &set_prio_to,
  4449.                       NULL) == 1) 
  4450.                     {  /* forbidden */
  4451.                       /* Note: the cache 'cached_tests' system will remember this error, and we'll only issue ONE request */
  4452.                       *forbidden_url = 1;          /* Forbidden! */
  4453.                       if ((opt->debug>1) && (opt->log!=NULL)) {
  4454.                         fspc(opt->log,"debug"); fprintf(opt->log,"link forbidden because of redirect beyond the mirror scope at %s%s"LF,adr,fil);
  4455.                         test_flush;
  4456.                       }
  4457.                       strcpybuff(adr,mov_adr);
  4458.                       strcpybuff(fil,mov_fil);
  4459.                       mov_url[0]='\0';
  4460.                       break;
  4461.                     }
  4462.                   }
  4463.  
  4464.                   // ftp: stop!
  4465.                   if (strfield(mov_url,"ftp://")
  4466. #if HTS_USEMMS
  4467.                                         || strfield(mov_url,"mms://")
  4468. #endif
  4469.                                         ) {
  4470.                     strcpybuff(adr,mov_adr);
  4471.                     strcpybuff(fil,mov_fil);
  4472.                     break;
  4473.                   }
  4474.  
  4475.                   /* ok, continue */
  4476.                   strcpybuff(adr,mov_adr);
  4477.                   strcpybuff(fil,mov_fil);
  4478.                   continue_loop = 1;
  4479.                 } else {
  4480.                   if ( opt->errlog!=NULL ) {
  4481.                     fspc(opt->errlog,"warning"); fprintf(opt->errlog,"Unable to test %s%s (loop to same filename)"LF,adr,fil);
  4482.                     test_flush;
  4483.                   }
  4484.                 }  // loop to same location
  4485.               }  // ident_url_relatif()
  4486.             }  // location
  4487.           }  // redirect
  4488.           if ((opt->debug>1) && (opt->log!=NULL)) {
  4489.                         fspc(opt->log,"debug"); fprintf(opt->log,"Final type for %s%s: '%s'"LF, adr, fil, delayed_back.r.contenttype);
  4490.             test_flush;
  4491.           }
  4492.  
  4493.           /* Recompute filename with MIME type */
  4494.           save[0] = '\0';
  4495.           r_sv=url_savename(adr,fil,save,former_adr,former_fil,liens[ptr]->adr,liens[ptr]->fil,opt,liens,lien_tot,sback,cache,hash,ptr,numero_passe,&delayed_back);
  4496.  
  4497.           /* Recompute authorization with MIME type */
  4498.           {
  4499.                         int new_forbidden_url = hts_acceptmime(opt, ptr, lien_tot, liens, adr,fil, delayed_back.r.contenttype);
  4500.             if (new_forbidden_url != -1) {
  4501.               if ((opt->debug>1) && (opt->log!=NULL)) {
  4502.                 fspc(opt->log,"debug"); fprintf(opt->log,"result for wizard mime test: %d"LF,forbidden_url);
  4503.                 test_flush;
  4504.               }
  4505.               if (new_forbidden_url == 1) {
  4506.                 *forbidden_url = new_forbidden_url;
  4507.                 if ((opt->debug>1) && (opt->log!=NULL)) {
  4508.                   fspc(opt->log,"debug"); fprintf(opt->log,"link forbidden because of MIME types restrictions: %s%s"LF, adr, fil);
  4509.                   test_flush;
  4510.                 }
  4511.                 break;        // exit loop
  4512.               }
  4513.             }
  4514.           }
  4515.  
  4516.           /* Still have a back reference */
  4517.           if (b >= 0) {
  4518.             /* Finalize now as we have the type */
  4519.                         if (back[b].status == 0) {
  4520.                             if (!back[b].finalized) {
  4521.                                 back_finalize(opt,cache,sback,b);
  4522.                             }
  4523.                         }
  4524.             /* Patch destination filename for direct-to-disk mode */
  4525.             strcpybuff(back[b].url_sav, save);
  4526.           }
  4527.  
  4528.         }  // b >= 0
  4529.       } else {
  4530.         printf("PANIC! : Crash adding error, unexpected error found.. [%d]\n",__LINE__);
  4531.         XH_uninit;    // dΘsallocation mΘmoire & buffers
  4532.         return -1;
  4533.       }
  4534.  
  4535.     } // while(IS_DELAYED_EXT(save))
  4536.  
  4537.     // error
  4538.     if (*forbidden_url != 1
  4539.       && IS_DELAYED_EXT(save)) {
  4540.       *forbidden_url = 1;
  4541.       if (opt->log!=NULL) {
  4542.         fspc(opt->log,"warning"); fprintf(opt->log,"link is probably looping, type unknown, aborting: %s%s"LF, adr, fil);
  4543.         test_flush;
  4544.       }
  4545.     }
  4546.  
  4547.   }  // delayed type check ?
  4548.  
  4549.   ENGINE_SAVE_CONTEXT_BASE();
  4550.  
  4551.   return 0;
  4552. }
  4553.  
  4554.