OS/2 Shareware BBS: 35 Internet/35-Internet.zip/srev13g.zip/POSTFCHK.RXX

Text File | 1999-06-27 | 44KB | 1,496 lines

/* Recording and postfilter module for SRE-http */

/* Some comments on the "record_all_file" cache algorithim:
   To speed up performance, SRE-http caches all writes
   to the record_all_file.  This saves disk i/o (and repetitive
   parsing of the entries).

   There are a few points worth considering in regards to this caching:
      1) More memory is required to store the cache.  One
         can limit the size of the cache by changing the record_cache_lines
         variable, with smaller values yielding smaller caches.
         For example, the default value of 1000 means "1000 entries".
         At an average size of 50 bytes, that's around
         5000 bytes total storage.

      2) If you have a lot of files being requested, with each having
         their own entry  (that is, few if any wildcards), you may fill up
         this cache. When this happens, SRE-http will write the
         current cache to an archive file (with name .1 to .999),
         and reset all values. "Reset" means:
                a) Set "counter" values on wildcards to 0
                b) Remove non-wildcard entries
        Of course, this new file could also fill up quickly, yielding
        a multiplicity of archive files.
        Should this happen, you should probably increase the size of the cache.

      3) As a safety measure, the cache will be written to the record_all_file
         every 5 minutes. When doing this, the current record_all_file
         is given the name .bak (overwriting a preexisting .bak file).

      4) If you should change the record_all_file by hand (you add
         a wildcard entry), SRE-http will detect the change and
         re-read the file into the cache.  This will ERASE
         all "recorded in cache but not written to file" entries --
         that is, everything in the (up to) last 5 minutes.
         If this minor inaccuracy (assuming that adding entries to
         the Record_all_file is a rare event) can be avoided by:
                1) Turning off caching (set RECORD_CACHE_LINES=0)
                2) Setting a very fast update rate (set UPDATE_RATE=0.2,
                   see below for details)
                3) Issue a !WRITE_CACHE request just prior to editing
                   the record_all_file (though this any changes that
                   occur whilst editing the file will be missed).
         Only method #1 is guaranteed to work, but it also is the slowest.
         Then again, absolute accuracy in this type of general audit
         file is usually not a priority (you should use SENDFILE
         for greater precision)

     5) When writing the cache, SRE-http will
           a)put ALL comments at the top
           b)write the list of "wildcard entries" (if any)
           c)write the "non-wildcard entries" (if any)
        Thus, placements of comments in between entries will be
        lost (sorry, if that's really a problem contact us
        and we'll consider keeping track of placement so as
        to preserve order on writes).

        In fact, the most frequent "non-wildcard" entries will
        be written first!

      6) If active, the HIT_CACHE will be examined before
         recording an entry.

*/

/* -----------------------------------------*/
/* Some user configurable parameters -- you have to restart goserve
for them to take effect*/

/* enter frequency, in minutes, for writing record_all cache to file (0.2 < x < 60)
Note: to suppress caching, set record_cache_lines=0 in srefiltr.80
*/
update_rate=5


/* enter lazy_write time for common log cache, in seconds */
lazy_clog=22

/* Note: as of 1.3b.1098b; log_toobig is no longer used */

/* starting verbosity level */
verbose=1


/* ---- Do NOT change code below here ---------------------------*/

parse upper arg   usequeue , USESEM,basesem,max_semwait, stuff,serverport,logfile_maxsize
parse var stuff record_cache_lines workdir
workdir=strip(workdir,'t','\')

if logfile_maxsize='' then logfile_maxsize=0
if datatype(logfile_maxsize)<>'NUM' then logfile_maxsize=0
if logfile_maxsize<0 then logfile_maxsize=0
ttlog=logfile_maxsize
if ttlog=0 then ttlog='unlimited'

logfile_maxsize=logfile_maxsize*1000

/*
foo=pmprintf_sref(' SRE-http post filter thread: queue='||usequeue,,serverport)
foo=pmprintf_sref(' SRE-http post filter thread: semaphore='||usesem,,serverport)
foo=pmprintf_sref(' SRE-http post filter thread: thread id='||mytid,,serverport)
*/

mytid=dostid()
foo=pmprintf_sref(' SRE-http Postfilter: thread and queue= '||mytid', 'usequeue,,serverport)

myqueue=usequeue||'_X'
mysem=usesem||'_X'
a=rxqueue('d',myqueue)
a=rxqueue('c',myqueue)
if a<>myqueue then do
  bb=rxqueue('d',a)
  a=rxqueue('s',myqueue)
  do queued()
       pull
  end /* do */
end

a=eventsem_close(mysem)
A=EVENTSEM_CREATE(mySEM)
IF A<>0 then DO
    foo=pmprintf_sref('SRE-http Postfilter ERROR: Error creating own semaphore: 'A' 'mysem,,serverport)
    return 0
end


if  usequeue="" | USESEM="" then do
   foo=pmprintf_sref('SRE-http Postfilter ERROR: initialization ERROR: '||accessfile||','||usequeue,,serverport)
   exit
end

signal on error name iserror
signal on syntax name iserror

comments.filestamp=0
if datatype(update_rate)<>'NUM' then update_rate=5
update_rate=max(min(update_rate,60),0.2)
update_rate=update_rate/(24*60)

if datatype(lazy_clog)<>'NUM' then lazy_clog=30
if lazy_clog>1000 then lazy_clog=1000   /* in case some on messes up */

if datatype(log_toobig)<>'NUM' then log_toobig=1000

/* get sreflogs.ini from cfgsdir (or from workdir, if not found in cfgsdir) */

cfgsdir=value('SRE_CFGS_DIR',,'os2environment')
logini=stream(cfgsdir||'\SREFLOGS.INI','c','query exists')
if logini='' then do
      logini=stream(workdir||'\SREFLOGS.INI','c','query exists')
end

sched_file=cfgsdir'\SCHEDULE.CFG'

got_log=1
if logini="" then got_log=0   /* no .ini file -- no log writing */
if got_log=1 then
  foo=pmprintf_sref(' SRE-http Postfilter: using 'logini'; w/ max file size='ttlog'K',,serverport)
else
  foo=pmprintf_sref(' SRE-http Postfilter:  Warning, no common-log configuration file=SREFLOGS.INI',,serverport)

foo=GET_sreflogs()
blog.0=0; clog.0=0 ; rlog.0=0 ; alllog.0=0
lasttime=time('s') ; THISTIME=LASTTIME
do_write_cache=0
last_hour_checked=-1

/* loop forevvefeerrrererere ...... */
bakme:
 thistime=time('s')
/* write common log, and perhaps update the .ini file */
 eekit=10000
 if datatype(thistime)='NUM' & datatype('LASTTIME')='NUM' then eekit=thistime-lasttime
 if ( abs(eekit)>lazy_clog  | do_wRITE_CACHE=1) & got_log=1 then do
       do_write_cache=0
       foo=write_clog()
       foo=write_blog()
       foo=write_rlog()
       foo=write_alllog()
       if (write_logs<>0) & ( (verbose>2 & (blog.0+clog.0+rlog.0)>0) |  (abbrev(upper(seloriginal),'!WRITE_CACHE')=1) ) then
          foo=pmprintf_sref(' Writing common-log entries: 'clog.0','blog.0','rlog.0','alllog.0,,serverport)
       ostamp=sysfiletree(logini,'FOO','F');OSTAMP=FOO.1
       sstamp=sysfiletree(sched_file,'FOO','F');sstamp=foo.1
       if  ostamp<>stuff.!stamp | sstamp<>stuff.!sstamp then do
         if verbose>1 then foo=pmprintf_sref(' SRE-http Postfilter: updating 'logini,,serverport)
         foo=GET_sreflogs()
       end

       lasttime=thistime
       blog.0=0; clog.0=0 ; rlog.0=0 ; alllog.0=0
 end

 foo=do_scheduler()   /* take this opportunity to run scheduled events */
 signal on error name iserror
 signal on syntax name iserror

 a=rxqueue('s',usequeue)
 goo=queued()
 if goo=0 then do
    WOW=EVENTSEM_WAIT(USESEM,max_semwait)

    if wow=640 then do
        signal bakme
    end
    IF WOW<>0 THEN do         /* FATAL ERROR */
        EXIT
    end
 end
 wow=EVENTSEM_RESET(usesem)
 if queued()<1 then signal bakme
 parse pull isit0
 isit0=translate(isit0,' ','000d0a09'x)

 if isit0=" " then signal bakme

 if abbrev(strip(translate(isit0)),'*DIE*')=1 then do
     foo=pmprintf_sref('SRE-http Postfilter: terminating ',,serverport)
     exit
 end
 if isit0="" then do
      foo=pmprintf_sref('SRE-http Postfilter: bad message ',,serverport)
     signal bakme
end

parse var isit0 record_option ',' record_all_file ',' post_filter ',' postfilter_name ',' serverport ',' ,
                 post_filter_message ',' source0 ',' request0a ',' ,
               sel0 ',' tempfile ','  servername ',' host_nickname ',' used_file ',' who  ',' ,
               enmadd ',' thereferer ',' req_num ',' REcord2 ',' write_logs ',' clientname0 ',' user ','  browser ',' verbose ',' ,
               privset ',' privset_secret',' meterdo

parse var record2 record_status record_bytes
write_logs=strip(write_logs) ; clientname0=upper(strip(clientname0))
if got_Log=0 then write_logs=0
browser=strip(browser) ; user=strip(user)
if user=' ' then user='-'
verbose=strip(verbose)
req_num=strip(req_num) /* the request number */

record_option=strip(record_option) ; post_filter=strip(post_filter)
if post_filter=0 then
   postfilter_name=' '
else
   postfilter_name=strip(postfilter_name)

record_all_file=strip(record_all_file)
serverport=strip(serverport) ; servername=strip(servername)
host_Nickname=strip(Host_nickname) ; used_file=strip(used_file)
parse var sel0 doorig sel0
seloriginal=packur(strip(sel0))

who=strip(who)
request0=strip(request0a)
parse upper var request0 method .
post_filter_message=packur(post_filter_message)
enmadd=strip(enmadd) ;
thereferer=strip(thereferer)

parse var seloriginal doit '?' .

if abbrev(upper(seloriginal),'!WRITE_CACHE')=1 then do_write_cache=1

if record_option<>"NO" then do
     select
        when record_option="YES" then
                parse var seloriginal doit '?' .
        when record_option='YES_ALL' then
                doit=seloriginal
        otherwise                       /* file */
              doit=used_File
     end

     doit=strip(strip(doit),'l','/')
     if host_Nickname<>"" & record_option<>'FILE' then
               doit=host_Nickname'//'doit
      njust=' '
      if do_write_cache<>1 then   njust=look_hit_cache('P',doit,who,enmadd)

      if njust=' ' then do      /* not in cache, so record */
         zfoo=0                   /* do we use a recordall file cache? */
         if record_cache_lines>0 then do
            zfoo=do_cache_record(record_all_file,doit,seloriginal)
         end
         if zfoo=0 then do
             foo=sref_lookup_count(record_all_file,doit,'ADD','OK',2)
         end
         foo=add_hit_cache("P",doit,who,enmadd,zfoo)
      end               /* not just hit */
 end                   /* record option */


if strip(meterdo)<>'0' then do          /* add to hit-meter tree */
  call add_hitmeter
end


/* log this one (uses stuff. */
 iik=is_logok(seloriginal,doorig,clientname0,record_status,write_logs,user,method)
 if iik=1 then do               /* write to the common log file */
      d1=space(strip(date('n')));
      parse var d1 d1a d1b d1c
      if d1a<10 then d1a='0'||d1a
      d1=d1a||'/'||d1b||'/'||d1c
      t1=time('n')
      d1t1=d1||':'||t1
      agmt=gmtoffset() ; if datatype(agmt)<>'NUM' then agmt=0

      if datatype(agmt)='NUM' then do
         agmt=agmt/36
         if agmt=0 then do
            agmt='+0000'
         end
         else do
           if abs(agmt)<1000 then do
               if agmt>0 then
                   agmt='0'||agmt
               else
                   agmt='-0'||abs(agmt)
           end
         end
      end
      d1t1_0=d1t1
      d1t1=d1t1||' '||agmt
      parse var request0 mkme a2a a3
      thereq=strip(a2a)
      if stuff.!nooptions=1 then parse var thereq thereq '?' .
      mkme=mkme||' '||thereq
      if stuff.!nohttp=0 then mkme=mkme||' '||a3
      d1t1='['||d1t1||']'
      d1t1_0='['||d1t1_0||']'
/* this is the output to the common-log file */
      ii=clog.0+1
      clogq=clientname0||' - '||user||' '||d1t1||' "'||mkme||'" '||strip(record2)

      clog.ii=clogq
      clog.ii.!file=which_logs(serverport,host_nickname,'COMMONLOG')
      clog.0=ii

/* the browser log */
      ii=blog.0+1
      blog.ii.!file=which_logs(serverport,host_nickname,'BROWSERLOG')
      if stuff.!apache_mode=1 then
         blogq=d1t1_0||' '||browser
      else
         blogq=d1t1||' '||clientname0||' '||browser
      blog.ii=blogq
      blog.0=ii

/* the referer log */
      if stuff.!force_referer=1 & thereferer=' ' then thereferer=' - '
      if thereferer<>" " then do
          parse var thereferer . '//' refip '/' .
          iik=is_logok(-1,,refip)
          if iik=1 then do
             ii=rlog.0+1
             if stuff.!apache_mode=1 then
                rlogq=d1t1_0||' '||thereferer||' -> '||thereq
             else
                rlogq=d1t1||' "'||thereq||'" '||thereferer
             rlog.ii.!file=which_logs(serverport,host_nickname,'REFERERLOG')
             rlog.ii=rlogq
             rlog.0=ii
          end
      end

/* the combined log */
/* eg: (all on one line) :
lion.statslab.cam.ac.uk - - [18/Jan/1996:12:04:23 +0000]
"GET /~sret1/analog/ HTTP/1.0" 200 578
"http://www.statslab.cam.ac.uk/~sret1/"
"Mozilla/2.0 (X11; I; HP-UX A.09.05 9000/735)"
*/
     ii=alllog.0+1
     alllog.ii=clogq||' "'||thereferer||'" "'||browser||'"'
     alllog.ii.!file=which_logs(serverport,host_nickname,'COMBINEDLOG')
     alllog.0=ii
     if verbose>0 then  foo=pmprintf_sref(clogq,1,serverport)  /* do NOT write to srehttp.log */

  end

 tookie=postfilter_name

 do mm=1 to words(tookie)
   postfilter_name=word(tookie,mm)
   if pos('.',postfilter_name)=0 then
        postfilter_name=postfilter_name||'.'||serverport

   signal on syntax name bad2
   signal on error name bad2
   yow='foo3='||postfilter_name||'(post_filter_message,source0,request0a,sel0,tempfile,servername,HOST_NICKNAME,used_file,thereferer,req_num,record2,privset,privset_secret)'
   interpret yow

   signal off syntax;  signal off error

 end

 signal bakme

bad2:
  signal off syntax; signal off error
  foo=pmprintf_sref(" SRE-http Postfilter: Problem in post_filter routine " postfilter_name " ("seloriginal,,serverport)
  signal bakme


iserror:
signal off error ; signal off syntax
foo=pmprintf_sref(' error in postfilter : resetting 'sigl' (rc='rc,,serverport)
a=rxqueue('d',usequeue)
a=rxqueue('c',usequeue)
a=eventsem_close(usesem)
a=eventsem_create(usesem)
foo=pmprintf_sref(' done resetting postfilter ',,serverport)
signal on error name iserror
signal on syntax name iserror
signal bakme


/* ========================================================== */
/* --- handle "cached recordall_file -----*/
do_cache_record:procedure expose record_cache_lines wilds. comments. simples. ,
                update_rate verbose serverport

parse arg record_all_file,doit,doit0

numeric digits 15

/* no cache -- then get one */
if comments.filestamp=0 then do
    nrecs=open_record_cache(record_all_file)
   if verbose>0 then foo=pmprintf_sref(" First read of RECORD_ALL_FILE: " nrecs,,serverport)
    if nrecs=0 then return 0
end
else do
    nrecs=comments.0+simples.0+wilds.0
end  /* Do */

/* cache too big? */
if nrecs>record_cache_lines then do
    nrecs=reset_cache(record_all_file,record_cache_lines)
end

/* add current hit,  or re-read file  */
foo=update_cache(doit)

/* do a timed updated to disk */
d1=date('b')
t1=time('s')/(24*60*60)
nowtime=d1+t1
yeeks=comments.istime+update_rate
if (yeeks<nowtime) | strip(upper(doit0))="!WRITE_CACHE" then do
   foo=save_cache(record_all_file)
end


return 1


/* ---------------- read recordall file into cache */
open_record_cache:procedure expose wilds. comments. simples. verbose serverport
parse arg record_all_file

aa=sref_fileread(record_all_file,thelines,,'E')
if aa<0 then return 0                /* error, use default method */

drop wilds. comments. simples.

nsimples=0 ; ncomments=0 ; nwilds=0
simples.0=0 ; comments.0=0 ; wilds.0=0
do mm=1 to thelines.0
   aline=strip(thelines.mm)
   if abbrev(aline,';')=1 | aline=' '  then do
        if aline=' ' then iterate
        ncomments=ncomments+1
        comments.ncomments=aline
        comments.0=ncomments
   end
   else do
       parse  var aline aname act amess
       if pos('*',aname)>0 then do
           aname=strip(aname)
           aname=translate(aname,'/','\')
           aname=strip(aname,'l','/')
           if act="" | datatype(act)<>'NUM' then act=0
           nwilds=nwilds+1
           wilds.nwilds.qname=aname ; wilds.nwilds.qct=act
           wilds.nwilds.qmess=amess
           wilds.0=nwilds
       end
       else do
            nsimples=nsimples+1
            if act="" | datatype(act)<>'NUM' then act=0
            simples.nsimples.qname=aname; simples.nsimples.qct=act
            simples.nsimples.qmess=amess
            simples.0=nsimples
       end
   end
end

/* get time stamp */
d1=date('b')
t1=time('s')/(24*60*60)
comments.istime=d1+t1

/* get file stamp */
oo=sysfiletree(record_all_file,'yow')
comments.filestamp=strip(upper(yow.1))


return nwilds+ncomments+nsimples


/* ------------------- Write cache---------------*/
write_cache:procedure expose comments. wilds. simples. verbose serverport
parse arg rfile,addtimestamp


nj=0
if addtimestamp=1 then do
   nj=1
   alines.nj="; File reset on: "||time('N')||' '||date('N')
end  /* Do */

do mm=1 to comments.0
  nj=nj+1
  alines.nj=comments.mm
end
do mm=1 to wilds.0
  nj=nj+1
  alines.nj=wilds.mm.qname||' '||wilds.mm.qct||' '||wilds.mm.qmess
end


/* sort simples in descending order, by frequency */
do iy=1 to simples.0
   foobar.iy=simples.iy.qct'         'iy
end
if simples.0>0 then wow=arraysort(foobar,1,simples.0,1,8,'D','N')

do mm=1 to simples.0
    parse var foobar.mm act indx ; indx=strip(indx)
    nj=nj+1
    alines.nj=simples.indx.qname||' '||simples.indx.qct||' '||simples.indx.qmess
end

alines.0=nj



wow=sref_filewrite(rfile,alines)

/* get time stamp */
d1=date('b')
t1=time('s')/(24*60*60)
comments.istime=d1+t1

/* get file stamp */
oo=sysfiletree(rfile,'yow')
comments.filestamp=strip(upper(yow.1))


if wow=0 then return 0

return nj


/* ------------- Reset the cache: --------------------- */
/*     Save current cache to a backup recordall file, delete current
     cache file, and rewrite minimal (just wildcards and comments)
     set
*/

reset_cache:procedure expose comments. wilds. simples. verbose serverport
parse arg rfile,maxlines

/* but update if external changes occured  (might obviate need to shrink cache) */
isold=cache_unsync(rfile,comments.filestamp)
if isold=1 then do
     nrecs=open_record_cache(rfile)
     if verbose>0 then foo=pmprintf_sref(' External change, reread RECORD_ALL_FILE into cache ',,serverport)
     return nrecs
end

rfile2=rfile
/* rename recordallfile to record.nnn (nnn=01 .. 999) */
foofoo=lastpos('.',rfile)
if foofoo>0 then
   rfile2=delstr(rfile,foofoo)

useme=' '
do mm=1 to 999
   if stream(rfile2||'.'||mm,'c','query exists')=' ' then do
       useme=rfile2||'.'||mm
       leave
   end
end
if useme=' ' then useme=dostempname(rfile2||'.???') /* should never happen ..*/
foo=dosrename(rfile,useme)
if foo=0 then return 0
if verbose>0 then foo=pmprintf_sref('SRE-http Postfilter: RECORD_ALL_FILE too big; reset (old file renamed to '||useme,,serverport)

if wilds.0+comments.0> maxlines then do
   if verbose>0 then foo=pmprintf_sref("SRE-http Postfilter: Warning: truncating comments in (RECORD_ALL_FILE) " rfile,,serverport)
   comments.0=trunc(maxlines/2)
end

/* reset counts */
simples.0=0
do mm=1 to wilds.0
   wilds.mm.qct=0
end

nrec=write_cache(rfile,1)
if verbose>0 then foo=pmprintf_sref(" SRE-http Postfilter: Write RECORD_ALL_FILE cache: " nrec,,serverport)

return wilds.0





/* --------------------- update cache ---------*/
/* match an entry to existing cache -- augment the match
or add a new entry */

update_cache:procedure expose wilds. simples. verbose serverport

parse arg doit

/* fix up doit */
doit=translate(strip(doit),'/','\')
doit=strip(doit,'l','/')

nsimples=simples.0
do mm=1 to nsimples
   if upper(simples.mm.qname)=upper(doit) then do
        simples.mm.qct=simples.mm.qct+1
        simples.mm.qmess=date('o')
        return 1
   end
end


nwilds=wilds.0
asterat=0 ; afterstar=0 ; gotit=0
do mm=1 to nwilds
    ares=sref_wildcard(upper(doit),upper(wilds.mm.qname),0)
    parse var ares astat "," aurl ;  astat=strip(astat)
    if astat=0 then iterate   /* no match */

   foo1=pos('*',wilds.mm.qname)
   if foo1 >= asterat then do
       tmp1=length(wilds.mm.qname)-foo1
       if foo1>asterat | tmp1>afteraster then  do
          asterat=foo1 ; afteraster=tmp1
          gotit=mm
       end              /* either condition */
    end
end

if gotit>0 then do
    wilds.gotit.qct=wilds.gotit.qct+1
    wilds.gotit.qmess=date('O')
    return 1
end


nsimples=nsimples+1 ;simples.0=nsimples
simples.nsimples.qname=doit
simples.nsimples.qct=1
simples.nsimples.qmess=date('o')


return 1


/* ------------- save the cache: --------------------- */
/*     Save current cache to recordall file, rename current
      file to .bak
*/

save_cache:procedure expose comments. wilds. simples. verbose serverport
parse arg rfile


/* but update if external changes occured  */
isold=cache_unsync(rfile,comments.filestamp)
if isold=1 then do
     nrecs=open_record_cache(rfile)
     if verbose>0 then foo=pmprintf_sref('SRE-http Postfilter: RECORD_ALL_FILE changed, did not write cache (reread RECORD_ALL_FILE instead ',,serverport)
     return nrecs
end



rfile2=rfile
/* rename recordallfile to record.nnn (nnn=01 .. 999) */
foofoo=lastpos('.',rfile)
if foofoo>0 then
   rfile2=delstr(rfile,foofoo)
useme=rfile2||'.bak'
a=sysfiledelete(useme)
a=dosrename(rfile,useme)
if verbose>1 then  foo=pmprintf_sref('SRE-http Postfilter: Writing current RECORD_ALL_FILE cache ',,serverport)

nrec=write_cache(rfile,0)

return nrec


/* --- return 1 if filestamp has changed */
cache_unsync:procedure expose verbose serverport
parse arg record_all_file,thestamp

/* get file stamp */

oo=sysfiletree(record_all_file,'yow')
afilestamp=strip(upper(yow.1))

if afilestamp=thestamp then return 0
return 1


/* -------- get julian date/time of file ----*/
file_jul_date:procedure expose serverport
parse arg afile

how0=dosfileinfo(afile,'W')
parse var how0 how hh ':' mm ':' ss
nsec=((hh*60*60) + (mm*60) + ss)/(24*60*60)
if right(strip(how),2)<80 then do  /* y2k correction */
   parse var how a1 '/' a2 '/' a3
   how=2000+a3
   how=how||a1||a2
   ndays=dateconv(how,'S','B')
end
else do
  ndays=dateconv(how,'U','B')
end
return ndays+nsec


/***************************/
/* check for log suppression condition */
is_logok:procedure expose stuff. serverport
parse upper arg sel1,doorig,clientname0,record_status,write_logs,user,method


if sel1=-1 then do
    aa=stuff.!noreferer
    if aa=' ' then return 1
end
else do
    if write_logs=0 then return 0
 /* check status */
    aa=stuff.!nocodes
   if wordpos(strip(record_status),aa)>0 then return 0
   if method<>'' & wordpos(strip(method),aa)>0 then return 0
/* check clientname */

    aa=stuff.!nouser
/* is it a direct match to the "user" */
    if wordpos(upper(user),aa)>0 then return 0
end

/* nope, so now check ip addresses  (possibly noreferer) */
c1=translate(clientname0,' ','.')
do ll=1 to words(aa)
   aa1=upper(strip(word(aa,ll)))  /* for each entry in the nouser list */
   if clientname0=aa1 then return 0       /* exact match */
   if user=aa1 then return 0       /* exact match */

   if pos('*',aa1)>0 then do     /* possible wild card match */
        aa1=translate(aa1,' ','.')
        if words(aa1) <> words(c1) then iterate /* can't be a match */
        die=1
        do ll2=1 to words(c1)
             c1q=strip(word(c1,ll2)) ; aa1q=strip(word(aa1,ll2))
             if aa1q='*' then iterate
             if c1q=aa1q then iterate
             die=0 ; leave              /* if here, segment did not match */
        end /* do */
        if die=1 then return 0  /* all segments matched */
   end  /* Do wildcard  */
end /* do */

if sel=-1 then return 1         /* noreferer did NOT match */

/* check urls */
theurl=strip(sel1)
if doorig=1 then theurl=' '
ido=stuff.!nourl.0
starat=0 ; afterstar=0

do mm=1 to ido
    aurl0=stuff.!nourl.mm
    ares=sref_wildcard(theurl,aurl0,0)
    parse var ares astat  .
    astat=strip(astat)
    if astat=0 then iterate   /* no match */
    return 0  /* match, so it's dead */
end
return 1



/***************************/
/* read sreflogs.ini file, return stuff in stuff. */
get_sreflogs:procedure expose stuff. workdir verbose serverport logini sched_file
parse arg afl
/* possible varialbes */
  varlist="COMMONLOG BROWSERLOG REFERERLOG COMBINEDLOG NOOPTIONS FORCE_REFERER APACHE_MODE NOHTTP NOCODES NOUSER NOREFERER NOURL SCHED"
  stemlist="COMMONLOG BROWSERLOG REFERERLOG COMBINEDLOG NOURL SCHED "

/* default values */
 stuff.!nooptions=0 ; stuff.!nohttp=0
 stuff.!nourl.0=0 ; stuff.!noreferer=' '
 stuff.!nocodes=' ' ; stuff.!nouser=' '
 stuff.!commonlog=workdir||'\COMMON.LOG'
 stuff.!browserlog=workdir||'\BROWSER.LOG'
 stuff.!refererlog=workdir||'\REFEFER.LOG'
 stuff.!combinedlog=workdir||'\COMBINED.LOG'

 stuff.!list='COMMONLOG BROWSERLOG REFERERLOG COMBINEDLOG'
 stuff.!stamp=' '
 stuff.!sstamp=' '
 stuff.!sched.0=0

/* look in sreflogs.ini for info */
afl=logini
foo=sref_fileread(afl,'logsl',,'E')
if foo<0 then do
   if verbose >2 then foo=pmprintf_sref(' SRE-http Postfilter: ' foo ' problem in sreflogs.ini file: 'afl,,serverport)
   return 1
end
if logsl.0=0 then signal jumpsched       /* use the defaults */

stuff.!stamp=sysfiletree(afl,'FOO','f')
STUFF.!STAMP=FOO.1


do mm=1 to logsl.0
   aline=strip(upper(logsl.mm))
   if aline="" | abbrev(aline,';')=1 then iterate
   parse upper var aline avar '=' aval
   aval=strip(translate(aval,' ','"'||"'"))
   avar0=strip(avar)
   avar=translate(avar0,' ','.')
   avar1=strip(word(avar,1))
   if words(avar)=1 then do      /* check the 1 word opts */
      if wordpos(avar1,varlist)=0 then iterate  /* not a valid variable */
      if wordpos(avar1,'COMMONLOG BROWSERLOG REFERERLOG COMBINEDLOG')>0 then do /* check validity */
          aval=chk_file(workdir,aval)
      end

      exx='!'||avar1
      stuff.exx=aval

      if avar1<>'SCHED' & avar1<>'NOURL' then iterate
   end
/* else, one of the stem boys */

  if wordpos(avar1,stemlist)=0 then iterate  /* not valid stem var */
  select
       when avar1="NOURL" then do
          inq=stuff.!nourl.0+1
          if pos('?',aval)>0 then do
               parse var aval a1 '?' a2
               a1=strip(translate(a1,'/','\'),'l','/')
               aval=a1||'?'||a2
          end
          else do
             aval=strip(translate(aval,'/','\'),'l','/')
          end
          stuff.!nourl.inq=aval
          stuff.!nourl.0=inq
        end
       when avar1="SCHED" then do       /* deprecated -- use SCHEDULE.CFG instead */
          if aval<>0 then do
             inq=stuff.!sched.0+1
             stuff.!sched.inq=aval
             stuff.!sched.0=inq
          end
        end

        otherwise  do      /* check for allowables */
            useme=chk_file(workdir,aval)
            if useme=0 then iterate
            exx=avar1||'.!'||word(avar,2)
            if words(avar)>2 then exx=exx||'.!'||word(avar,3)
             exx2='!'||exx
            stuff.exx2=useme
            stuff.!list=stuff.!list||' '||exx
        end             /*otherwise */
  end                /* select */

end          /* logsl */

jumpsched:
aa=sysfiletree(sched_file,'FOO','F')
stuff.!sstamp=foo.1

/* now get schedule.cfg; add to stuff.Sched */
foo=sref_fileread(sched_file,'logsl',,'E')
do mm=1 to logsl.0

   aline=strip(upper(logsl.mm))
   if aline="" | abbrev(aline,';')=1 then iterate
   inq=stuff.!sched.0+1
   stuff.!sched.inq=aline
   stuff.!sched.0=inq
end

if verbose >2 then foo=pmprintf_sref(' SRE-http Postfilter: Log file list:loglist 'stuff.!list,,serverport)
return 1

/************/
/* check for valid file name, and return it. Return 0 if no good */
chk_file:procedure expose serverport
parse arg workdir, aval

  fn1=translate(strip(upper(aval)),'\','/')  /* the filename */
  if fn1=0 then return 0

  fn1=strip(translate(fn1,' ',"'"||'"'))
  if abbrev(fn1,'\')=1 | pos(':',fn1)>0 then
       fuse=fn1
   else
       fuse=workdir||'\'||fn1
   foo=strip(filespec('d',fuse)||filespec('p',fuse),'t','\')
   if right(foo,1)=':' then foo=foo||'\'

   if dosisdir(foo)=0 then do
       call pmprintf_sref('  SRE-http Postfilter ERROR: bad log file: 'aval)
       return 0  /* no such dir--error, so ignore */
   end

   return fuse



/***************/
/* write to the log files */
write_clog:procedure expose clog. serverport logfile_maxsize thistime
parse arg ttime,ltime

if clog.0=0 then return 0

/* are they all to the same file? */
bname=clog.1.!file
idid=1
do mm=2 to clog.0
      if clog.mm.!file<>bname then leave
      idid=mm
end
if idid=clog.0  then do  /* all same name, dump the bunch */

   if bname=0 then return 0 /* all suppressed */
    do nn=1 to clog.0
       tt.nn=clog.nn
    end
    foo2=check_too_big(bname,logfile_maxsize)
    tt.0=clog.0
    foo=sref_filewrite(bname,tt,'A')

/*    cfoo=jeepers(bname,lbig) */
end
else do         /* several files (multi host system */
  tt.0=1
  do nn=1 to clog.0
    if clog.nn.!file=0 then iterate
    tt.1=clog.nn
    foo=check_too_big(clog.nn.!file,logfile_maxsize)
    foo=sref_filewrite(clog.nn.!file,tt,'A')
/*    foo=jeepers(clog.nn.!file,lbig) */
 end
end

return 1


/***********/
/* is logfile too big? if so, rename it */
check_too_big:procedure expose serverport thistime
parse arg fname,mxsize
if mxsize=0 then return 0               /* no max size specified*/
if right(thistime,1)<9 then return 0     /* only check on every 10th request */
foo=stream(fname,'c','query size')
if foo='' then return 0         /* no such file means 0 length */
if foo<mxsize then return 0     /* small enough */

yow=lastpos('.',fname)
if yow=0 then do
    f2name=fname||'.???'
end
else do
   f2name=left(fname,yow)||'???'
end
f3name=dostempname(f2name)
if f3name='' then do
   call pmprintf(' SRE-http Postfilter: Can not find new name for 'fname)
   return 0
end
foo=dosrename(fname,f3name)
if foo=0 then do
   call pmprintf(' SRE-http Postfilter: Can not find rename  'fname)
   return 0
end

foo=pmprintf_sref(' **** SRE-http Postfilter: renamed large logfile: ',,serverport)
foo=pmprintf_sref('     'fname' -->  to --> 'f3name,,serverport)

return 1


/********/
/*****************
 is log file too big (check every 4th time?  -- DEPRECATED

jeepers:procedure expose  serverport
parse arg bname0,toob

if toob=0 then return 1
foo1=random(1,5)
if foo1<5 then return 0   /* every 5th write */

foo=sysfiletree(bname0,'boo','F')

if boo.0>0 then do
  ih=strip(word(boo.1,3))
  if ih>(1000*toob) then do
      foo=pmprintf_sref(' ',,serverport)
      foo=pmprintf_sref(' ******             WARNING                  ****** ',,serverport)
      foo=pmprintf_sref(' SRE-http Postfilter: Your log file ('bname0') is becoming large ('ih,,serverport)
      foo=pmprintf_sref(' ******             WARNING                  ****** ',,serverport)
      foo=pmprintf_sref(' ',,serverport)

  end
end
return 0

****************************************************/



/***************/
/* this is sort of a stupid algorithim, but it's probably adequate
(given rexx's problems with passing stem variables */

/* write to the common log files */
write_blog:procedure expose blog.  serverport logfile_maxsize thistime

if blog.0=0 then return 0

/* are they all to the same file? */
bname=blog.1.!file
idid=1
do mm=2 to blog.0
      if blog.mm.!file<>bname then leave
      idid=mm
end
if idid=blog.0  then do  /* all same name, dump the bunch */
   if bname=0 then return 0 /* all suppressed */
    do nn=1 to blog.0
       tt.nn=blog.nn
    end
    tt.0=blog.0
    foo=check_too_big(bname,logfile_maxsize)
    foo=sref_filewrite(bname,tt,'A')
end
else do         /* several files (multi host system */
  tt.0=1
  do nn=1 to blog.0
    if blog.nn.!file=0 then iterate
    tt.1=blog.nn
    foo=check_too_big(blog.nn.!file,logfile_maxsize)
    foo=sref_filewrite(blog.nn.!file,tt,'A')
 end
end
return 1


/***************/
/* write to the refeerer log files */
write_rlog:procedure expose rlog.  serverport logfile_maxsize thistime

if rlog.0=0 then return 0

/* are they all to the same file? */
bname=rlog.1.!file
idid=1
do mm=2 to rlog.0
      if rlog.mm.!file<>bname then leave
      idid=mm
end
if idid=rlog.0  then do  /* all same name, dump the bunch */
   if bname=0 then return 0 /* all suppressed */
    do nn=1 to rlog.0
       tt.nn=rlog.nn
    end
    tt.0=rlog.0
    foo=check_too_big(bname,logfile_maxsize)
    foo=sref_filewrite(bname,tt,'A')
end
else do         /* several files (multi host system */
  tt.0=1
  do nn=1 to rlog.0
    if rlog.nn.!file=0 then iterate
    tt.1=rlog.nn
    foo=check_too_big(rlog.nn.!file,logfile_maxsize)
    foo=sref_filewrite(rlog.nn.!file,tt,'A')
 end
end
return 1


/***************/
/* write to the common log files */
write_alllog:procedure expose alllog.  serverport logfile_maxsize thistime

if alllog.0=0 then return 0

/* are they all to the same file? */
bname=alllog.1.!file
idid=1
do mm=2 to alllog.0
      if alllog.mm.!file<>bname then leave
      idid=mm
end
if idid=alllog.0  then do  /* all same name, dump the bunch */
   if bname=0 then return 0 /* all suppressed */
    do nn=1 to alllog.0
       tt.nn=alllog.nn
    end
    tt.0=alllog.0
    foo=check_too_big(bname,logfile_maxsize)
    foo=sref_filewrite(bname,tt,'A')
end
else do         /* several files (multi host system */
  tt.0=1
  do nn=1 to alllog.0
    if alllog.nn.!file=0 then iterate
    tt.1=alllog.nn
    foo=check_too_big(alllog.nn.!file,logfile_maxsize)
    foo=sref_filewrite(alllog.nn.!file,tt,'A')
 end
end
return 1



/************/
/* which log file to use */
which_logs:procedure expose stuff.  serverport
parse arg sport,hname,dalog
if hname<>' ' then do
  alist=dalog||'.!'||hname||'.!'||sport
  alist=alist||' '||dalog||'.!'||hname||' '||dalog
end
else do
    alist=dalog||'.!'||sport||' '||dalog
end
alist=upper(alist)
do li=1 to words(alist)
   a1=strip(word(alist,li))
   if wordpos(a1,stuff.!list)>0 then do
       axe='!'||a1
       return stuff.axe
    end
end
return 0


/*************/
/* The scheduler */

do_scheduler:procedure expose stuff. last_hour_checked verbose  serverport

if verbose>3 & stuff.!sched.0>0 then foo=pmprintf_sref('SRE-http Postfilter: Checking ' stuff.!sched.0' SCHED. entries',,serverport)

if stuff.!sched.0=0  | datatype(stuff.!sched.0)<>'NUM'then return 0   /* no scheduled programs to run */

atime=time('n')
parse var atime ahr ':' amin ':' asec
if ahr=last_hour_checked then return 0
adate=date('n')
parse var adate mday .
aweekday=upper(date('w'))

/* else, check scheduled programs */
do ii=1 to stuff.!sched.0
  parse upper var stuff.!sched.ii  afreq aprog stuff ; aprog=strip(aprog); afreq=strip(afreq)
  doit=0
  select
     when abbrev(afreq,'HOUR')=1 then   doit=1
     when abbrev(afreq,'DA')=1 & ahr=0 then doit=1
     when abbrev(afreq,'WE')=1 & aweekday="SUNDAY" then doit=1
     when abbrev(afreq,'MO')=1 & mday=1 then doit=1
     otherwise doit=0
   end
   signal off error  ; signal off syntax
   signal on error name hoy1
   signal on syntax name  hoy1
   if doit =1 then do
        astat=rexxrun('f',aprog,,stuff)
        if verbose>1 & astat<>0 then foo=pmprintf_sref('SRE-http Postfilter: REXXLIB REXXRUN error code = ' astat ' on scheduled event: 'aprog,,serverport)
   end
end

last_hour_checked=ahr

signal off error ; signal off syntax
return 0

hoy1:
foo=pmprintf_sref('SRE-http Postfilter ERROR: Error: could not execute scheduled program: 'aprog,,serverport)
signal off error ; signal off syntax
return 0


/******************************/
/* Add entry to the hit cache*/

add_hit_cache:procedure expose myqueue basesem mysem mytid  serverport
parse arg  thetype, theurl0, who , enmadd,stuff
theurl0=upper(theurl0) ; thetype=upper(thetype)

nogi=digits()

lenc=value(enmadd||'HIT_CACHE_LEN',,'OS2ENVIRONMENT')

if lenc=0 then return ' '               /* suppressed */
transaction=mytid

numeric digits 11
ttc=value(enmadd||'HIT_CACHE_DURATION',,'OS2ENVIRONMENT')
ttc=ttc/(60*24)
d1=date('b')
t1=time('m')/(24*60)
nowtime=d1+t1
endtime=nowtime+ttc
theurl0=translate(theurl0,'/','\')
moo=lenc||' '||thetype||' '||theurl0||' '||who||' '||endtime||' '||stuff

if lenc='FILE' then do          /* add to cache file */
  itis=file_addhitc(moo,nowtime)
  numeric digits nogi
  return 0
end

/* or use VARSTORE thread  */
goober=enmadd||'VARSTORE'
a=rxqueue('s',goober)
    queue  transaction ' ' host_nickname ',' myqueue ',' mysem ',' 'PUT_HIT,'||moo
dothread=basesem||'VARSTORE'
a=eventsem_post(dothread)  /* no return expected */
return 0


/* --------------- */
/* add to the  .stem  file (slower, but perhaps better for huge numbers of hits */
file_addhitc:procedure expose enmadd   serverport
parse arg  moo,nowtime

arf=serverport

adir=strip(value(enmadd||'TEMPDATA_DIR',,'os2environment'),'t','\')||'\'

cfile=adir||'_HITCACH.'||arf
if stream(cfile,'c','query exists')=' ' then do
   foo=pmprintf_sref("SRE-http Postfilter: Creating hit-cache file " cfile,,serverport)
   eek.1=";The SRE-http hit cache file-- do NOT edit ! "
   foo=sref_filewrite(cfile,'eek','R',1,1)
   if foo=0 then do
        foo=pmprintf_sref( " SRE-http Postfilter ERROR: Error creating hit-cache file: " cfile,,serverport)
        return
   end
end

/* wait for it to be unlocked */
isnow=time('r')
do until twait>30
  twait=time('e')
  aa=sref_fileread(cfile,'yy',,'E')
  if aa>=0 then leave
  ddt=syssleep(0.2)
end
if aa<0 then do
   foo=pmprintf_sref("SRE-http Postfilter: Hit cache file could not be opened for augmentation: " cfile,,serverport)
   return ' '
end

/* now lock it! We'll clean it up, and add current hit to beginning */
aa=sref_open_read(cfile,10,'write')
if aa<0 then do
   foo=pmprintf_sref("SRE-http Postfilter: Hit cache file could not be opened for augmentation: " cfile,,serverport)
   return ' '
end


newl.1=moo
i1=1
/* remove expired entries */
do iu=1 to yy.0
   aline=yy.iu
   if aline=' ' then iterate
   if abbrev(strip(aline),';')=1 then do
       i1=i1+1
       newl.i1=aline
       iterate
    end
    parse var aline atype aurl aip  aendtime .
    aendtime=strip(aendtime)
    if aendtime >= nowtime then do
        i1=i1+1
        newl.i1=aline
    end
    else do
    end
    if i1>10000 then leave   /* prevent too big */
end
newl.0=i1
aa=stream(cfile,'c','close')
foo=sref_filewrite(cfile,'newl','R',i1,1)
if foo=0 | foo<=0 then
  foo=pmprintf_sref( "SRE-http Postfilter: Could not augment&update hit-cache file ",,serverport)
return ' '


/*****************************/
/* see if url is in the  "hit cache"  */
look_hit_cache:procedure expose basesem mysem myqueue mytid  serverport
parse upper arg thetype, theurl, who , enmadd

theurl=translate(theurl,'/','\')

lenc=value(enmadd||'HIT_CACHE_LEN',,'OS2ENVIRONMENT')
if lenc=0 then return ' '               /* suppresed */
transaction=mytid
nogi=digits()

numeric digits 12
d1=date('b')
t1=time('m')/(24*60)
nowtime=d1+t1

if lenc='FILE' then do
  itis=file_lenhitc(thetype,theurl,who,enmadd,nowtime)
   numeric digits nogi
  return itis
end


/* here to read from VARSTORE thread */
/* or use VARSTORE thread  */
goober=enmadd||'VARSTORE'
moo=thetype||' '||who||' '||theurl
a=eventsem_reset(mysem)
a=rxqueue('s',goober)
    queue  transaction ' ' host_nickname ',' myqueue ',' mysem ',' 'GET_HIT, '||moo
dothread=basesem||'VARSTORE'
a=eventsem_post(dothread)

a=eventsem_wait(mysem) /* wait for answer */

if a<>0 then do
   foo=pmprintf_sref(' A Fatal Semaphore failure in POSTF-Look_hit_cache: 'a,,serverport)
   return ' '
end
a=rxqueue('s',myqueue)

parse pull aline

PARSE VAR ALINE idnum ',' aline

idnum=strip(translate(idnum,' ','000d0a'x));TRANSACTION=STRIP(TRANSACTION)
if idnum<>transaction then  do /*wierd error: got someone else's message, give up */
      say ' Read odd id from VARSTORE queue :' transaction ',' idnum
      return ' '
end
return strip(aline)


/* --------------- */
/* read from cachelist from an .idx file,, slower but perhaps better for huge number of hits */

file_lenhitc:procedure  expose  serverport
parse arg thetype,theurl,who,enmadd,nowtime

arf=serverport
adir=strip(value(enmadd||'TEMPDATA_DIR',,'os2environment'),'t','\')||'\'
theurl=strip(translate(theurl,'/','\'))

cfile=adir||'_HITCACH.'||arf
if stream(cfile,'c','query exists')=' ' then do
   say "  No hit cache file " cfile
   return ' '
end

/* wait for it to be unlocked */
isnow=time('r')
do until twait>30
  twait=time('e')
  aa=sref_fileread(cfile,'yy')
  if aa>=0 then leave
  ddt=syssleep(0.2)
end
if aa<0 then do
   say " Hit cache file could not be opened: " cfile
   return ' '
end

/* see if a match exists */
do iu=1 to aa
   aline=yy.iu
   if aline=' ' then iterate
   if abbrev(strip(aline),';')=1 then iterate
    parse var aline atype aurl aip  aendtime stuff
    atype=strip(atype) ; aurl=strip(aurl) ; aip=strip(aip)
    aendtime=strip(aendtime)
   if theurl<>aurl | aip<>who | atype<>thetype then iterate
   if aendtime >= nowtime then do
        return stuff
   end
end
return ' '              /* no match */





/**************/
/* add to hitmeter file */
add_hitmeter:

mfile=value('SREF_HITMETER_FILE',,'os2environment')
/* now lock it! */
aa=sref_open_read(mfile,10,'BOTH')
if aa<0 then do
      foo=pmprintf_sref("SRE-http Postfilter: Hit-meterfile could not be opened for augmentation: " mfile,,serverport)
      return 0
end
iaa=stream(mfile,'c','query size')
if iaa=0 | iaa='' then do
   foo=pmprintf_sref("SRE-http Postfilter: unable to read hit-meterfile: " mfile,,serverport)
   return 0
end

parse var meterdo n3a '/' n4a
if datatype(n3a)<>'NUM' then n3a=0
if datatype(n4a)<>'NUM' then n4a=0

mstuff=strip(charin(mfile,1,iaa),'t','1a'x)

lookfor=host_nickname'//'||translate(seloriginal)
lookfor=left(lookfor,min(80,length(lookfor)))
iat=pos(' 'lookfor,mstuff)
A0=RIGHT(0,7); A1=RIGHT(1,7)
if iat=0 then do        /* add to end */
  IF RECORD_STATUS=304 THEN
     mstuff=mstuff||'0d0a'x||a0||a1||right(n3a,7)||right(n4a,7)||'  '||lookfor
  ELSE
     mstuff=mstuff||'0d0a'x||a1||a0||right(n3a,7)||right(n4a,7)||'  '||lookfor

  foo=charout(mfile,mstuff,1)
  foo=stream(mfile,'c','close')
  return 0
end

/* else, augment entry  -- iat2 will be start of this record */
iat2=lastpos('0d0a'x,mstuff,iat)
if iat2=0 then
   iat2=1
else
   iat2=iat2+2

iat3=pos('0d0a'x,mstuff,iat2)           /* to end of record */
if iat3=0 then
  iat3=length(mstuff)
else
  iat3=iat3-1
rlen=1+iat3-iat2
aline=substr(mstuff,iat2,rlen)
parse VAR aline n1 n2 n3 n4 itis
IF RECORD_STATUS<>304 THEN
   n1=n1+1
ELSE
   n2=n2+1
n3=n3a+n3
n4=n4a+n4
aline=right(n1,7)||right(n2,7)||right(n3,7)||right(n4,7)||'  '||lookfor
mstuff=overlay(aline,mstuff,iat2)
foo=charout(mfile,mstuff,1)
foo=stream(mfile,'c','close')
return 0