OS/2 Spezial/SPEZIAL2_97.zip/SPEZIAL2_97.iso/ANWEND/ONLINE/SREFV12J/POSTFCHK.RXX

Text File | 1997-09-19 | 39KB | 1,334 lines

/* Recording and postfilter module for sre-filter */

/* Some comments on the "record_all_file" cache algorithim:
   To speed up performance, SRE-Filter caches all writes
   to the record_all_file.  This saves disk i/o (and repetitive
   parsing of the entries).

   There are a few points worth considering in regards to this caching:
      1) More memory is required to store the cache.  One
         can limit the size of the cache by changing the record_cache_lines
         variable, with smaller values yielding smaller caches.
         For example, the default value of 1000 means "1000 entries".
         At an average size of 50 bytes, that's around
         5000 bytes total storage.

      2) If you have a lot of files being requested, with each having
         their own entry  (that is, few if any wildcards), you may fill up
         this cache. When this happens, SRE-Filter will write the
         current cache to an archive file (with name .1 to .999),
         and reset all values. "Reset" means:
                a) Set "counter" values on wildcards to 0
                b) Remove non-wildcard entries
        Of course, this new file could also fill up quickly, yielding
        a multiplicity of archive files.
        Should this happen, you should probably increase the size of the cache.

      3) As a safety measure, the cache will be written to the record_all_file
         every 5 minutes. When doing this, the current record_all_file
         is given the name .bak (overwriting a preexisting .bak file).

      4) If you should change the record_all_file by hand (you add
         a wildcard entry), SRE-Filter will detect the change and
         re-read the file into the cache.  This will ERASE
         all "recorded in cache but not written to file" entries --
         that is, everything in the (up to) last 5 minutes.
         If this minor inaccuracy (assuming that adding entries to
         the Record_all_file is a rare event) can be avoided by:
                1) Turning off caching (set RECORD_CACHE_LINES=0)
                2) Setting a very fast update rate (set UPDATE_RATE=0.2,
                   see below for details)
                3) Issue a !WRITE_CACHE request just prior to editing
                   the record_all_file (though this any changes that
                   occur whilst editing the file will be missed).
         Only method #1 is guaranteed to work, but it also is the slowest.
         Then again, absolute accuracy in this type of general audit
         file is usually not a priority (you should use SENDFILE
         for greater precision)

     5) When writing the cache, SRE-Filter will
           a)put ALL comments at the top
           b)write the list of "wildcard entries" (if any)
           c)write the "non-wildcard entries" (if any)
        Thus, placements of comments in between entries will be
        lost (sorry, if that's really a problem contact us
        and we'll consider keeping track of placement so as
        to preserve order on writes).

        In fact, the most frequent "non-wildcard" entries will
        be written first!

      6) If active, the HIT_CACHE will be examined before
         recording an entry.

*/

/* -----------------------------------------*/
/* Some user configurable parameters -- you have to restart goserve
for them to take effect*/

/* enter frequency, in minutes, for writing record_all cache to file (0.2 < x < 60)
Note: to suppress caching, set record_cache_lines=0 in srefiltr.80
*/
update_rate=5


/* enter lazy_write time for common log cache, in seconds */
lazy_clog=22


/* warn (via pmprintf) when log file gets over this size (in kbytes) */
/* to suppress this check, set log_toobig=0 (not checking saves a little bit of time)*/
/* hint: consider use of RENLOGS.CMD and the SCHED. option */
log_toobig=1000

/* starting verbosity level */
verbose=1


/* ---- Do NOT change code below here ---------------------------*/

parse upper arg   usequeue , USESEM,basesem,max_semwait, stuff
parse var stuff record_cache_lines workdir
workdir=strip(workdir,'t','\')

call pmprintf_sref(' SRE-FILTER postfilter thread: queue='||usequeue)
call pmprintf_sref(' SRE-FILTER postfilter thread: semaphore='||usesem)



mytid=dostid()
call pmprintf_sref(' SRE-FILTER postfilter thread: thread id='||mytid)

myqueue=usequeue||'_X'
mysem=usesem||'_X'
a=rxqueue('d',myqueue)
a=rxqueue('c',myqueue)
if a<>myqueue then do
  bb=rxqueue('d','a')
  a=rxqueue('s',myqueue)
  do queued()
       pull
  end /* do */
end

a=eventsem_close(mysem)
A=EVENTSEM_CREATE(mySEM)
IF A<>0 then DO
    call pmprintf_sref('SRE-Filter post filter thread:: Error creating own semaphore: 'A' 'mysem)
    return 0
end


if  usequeue="" | USESEM="" then do
   call pmprintf_sref('SRE-FILTER postfilter thread: initialization ERROR: '||accessfile||','||usequeue)
   exit
end

signal on error name iserror
signal on syntax name iserror

comments.filestamp=0
if datatype(update_rate)<>'NUM' then update_rate=5
update_rate=max(min(update_rate,60),0.2)
update_rate=update_rate/(24*60)

if datatype(lazy_clog)<>'NUM' then lazy_clog=30
if lazy_clog>1000 then lazy_clog=1000   /* in case some on messes up */

if datatype(log_toobig)<>'NUM' then log_toobig=1000

/* get sreflogs.ini from workdir */
afl=workdir||'\SREFLOGS.INI'
logini=stream(afl,'c','query exists')
got_log=1
if logini="" then got_log=0   /* no .ini file -- no log writing */
if got_log=1 then
  call pmprintf_sref(' SRE-FILTER postfilter thread: common-log configuration file='logini)
else
  call pmprintf_sref(' SRE-FILTER postfilter thread:  ERROR, no common-log configuration file='afl)

foo=GET_sreflogs()
blog.0=0; clog.0=0 ; rlog.0=0 ; alllog.0=0
lasttime=time('s') ; THISTIME=LASTTIME
do_write_cache=0
last_hour_checked=-1

/* loop forevvefeerrrererere ...... */
bakme:

 thistime=time('s')
/* write common log, and perhaps update the .ini file */
 eekit=10000
 if datatype(thistime)='NUM' & datatype('LASTTIME')='NUM' then eekit=thistime-lasttime
 if ( abs(eekit)>lazy_clog  | do_wRITE_CACHE=1) & got_log=1 then do
       do_write_cache=0
       foo=write_clog(thistime,lasttime,log_toobig)
       foo=write_blog()
       foo=write_rlog()
       foo=write_alllog()
       if (write_logs<>0) & ( (verbose>2 & clog.0>0) | verbose>3 |  (abbrev(upper(seloriginal),'!WRITE_CACHE')=1) ) then
          call pmprintf_sref(' Writing common-log entries: 'clog.0','blog.0','rlog.0','alllog.0)
       ostamp=sysfiletree(logini,'FOO','F');OSTAMP=FOO.1
       if  ostamp<>stuff.!stamp then do
         if verbose>1 then call pmprintf_sref(' SRE-FILTER postfilter thread: updating 'logini)
         foo=GET_sreflogs()
       end

       lasttime=thistime
       blog.0=0; clog.0=0 ; rlog.0=0 ; alllog.0=0
 end

 foo=do_scheduler()   /* take this opportunity to run scheduled events */
 signal on error name iserror
 signal on syntax name iserror

 a=rxqueue('s',usequeue)
 goo=queued()
 if goo=0 then do
    WOW=EVENTSEM_WAIT(USESEM,max_semwait)

    if wow=640 then do
        signal bakme
    end
    IF WOW<>0 THEN do         /* FATAL ERROR */
        EXIT
    end
 end
 wow=EVENTSEM_RESET(usesem)
 if queued()<1 then signal bakme
 parse pull isit0
 isit0=translate(isit0,' ','000d0a09'x)

 if isit0=" " then signal bakme

 if abbrev(strip(translate(isit0)),'*DIE*')=1 then do
     call pmprintf_sref('SRE-FILTER postfilter thread: terminating ')
     exit
 end
 if isit0="" then do
      call pmprintf_sref('SRE-FILTER postfilter thread: bad message ')
     signal bakme
end

parse var isit0 record_option ',' record_all_file ',' post_filter ',' postfilter_name ',' serverport ',' ,
                 post_filter_message ',' source0 ',' request0a ',' ,
               sel0 ',' tempfile ','  servername ',' host_nickname ',' used_file ',' who  ',' ,
               enmadd ',' thereferer ',' tcache ',' REcord2 ',' write_logs ',' clientname0 ',' user ','  browser ',' verbose

parse var record2 record_status record_bytes
write_logs=strip(write_logs) ; clientname0=upper(strip(clientname0))
if got_Log=0 then write_logs=0
browser=strip(browser) ; user=strip(user)
if user=' ' then user='-'
verbose=strip(verbose)

 record_option=strip(record_option) ; post_filter=strip(post_filter)
 postfilter_name=strip(postfilter_name) ; record_all_file=strip(record_all_file)
serverport=strip(serverport) ; servername=strip(servername)
host_Nickname=strip(Host_nickname) ; used_file=strip(used_file)
parse var sel0 doorig sel0
seloriginal=packur(strip(sel0))

who=strip(who)
request0=strip(request0a)
post_filter_message=packur(post_filter_message)
enmadd=strip(enmadd) ; tcache=strip(tcache)
thereferer=strip(thereferer)

parse var seloriginal doit '?' .

if abbrev(upper(seloriginal),'!WRITE_CACHE')=1 then do_write_cache=1


 if record_option<>"NO" then do
     select
        when record_option="YES" then
                parse var seloriginal doit '?' .
        when record_option='YES_ALL' then
                doit=seloriginal
        otherwise                       /* file */
              doit=used_File
     end

     doit=strip(strip(doit),'l','/')
     if host_Nickname<>"" & record_option<>'FILE' then
               doit=host_Nickname'//'doit
      njust=' '
      if do_write_cache<>1 then   njust=look_hit_cache('P',doit,who,enmadd)

      if njust=' ' then do      /* not in cache, so record */
         zfoo=0                   /* do we use a recordall file cache? */
         if record_cache_lines>0 then do
            zfoo=do_cache_record(record_all_file,doit,seloriginal)
         end
         if zfoo=0 then do
             foo=sref_lookup_count(record_all_file,doit,'ADD','OK',2)
         end
         foo=add_hit_cache("P",doit,who,enmadd,zfoo)
      end               /* not just hit */
  end                   /* record option */

/* log this one (uses stuff. */
 iik=is_logok(seloriginal,doorig,clientname0,record_status,write_logs,user)
 if iik=1 then do               /* write to the common log file */
      d1=space(strip(date('n')));
      parse var d1 d1a d1b d1c
      if d1a<10 then d1a='0'||d1a
      d1=d1a||'/'||d1b||'/'||d1c
      t1=time('n')
      d1t1=d1||':'||t1
      agmt=gmtoffset() ; if datatype(agmt)<>'NUM' then agmt=0

      if datatype(agmt)='NUM' then do
         agmt=agmt/36
         if abs(agmt)<1000 then do
             if agmt>0 then
                 agmt='0'||agmt
             else
                 agmt='-0'||abs(agmt)
         end
      end
      d1t1_0=d1t1
      d1t1=d1t1||' '||agmt
      parse var request0 mkme a2a a3
      thereq=strip(a2a)
      if stuff.!nooptions=1 then parse var thereq thereq '?' .
      mkme=mkme||' '||thereq
      if stuff.!nohttp=0 then mkme=mkme||' '||a3
      d1t1='['||d1t1||']'
      d1t1_0='['||d1t1_0||']'
/* this is the output to the common-log file */
      ii=clog.0+1
      clogq=clientname0||' - '||user||' '||d1t1||' "'||mkme||'" '||strip(record2)

      clog.ii=clogq
      clog.ii.!file=which_logs(serverport,host_nickname,'COMMONLOG')
      clog.0=ii

/* the browser log */
      ii=blog.0+1
      blog.ii.!file=which_logs(serverport,host_nickname,'BROWSERLOG')
      if stuff.!apache_mode=1 then
         blogq=d1t1_0||' '||browser
      else
         blogq=d1t1||' '||clientname0||' '||browser
      blog.ii=blogq
      blog.0=ii

/* the referer log */
      if stuff.!force_referer=1 & thereferer=' ' then thereferer=' - '
      if thereferer<>" " then do
          parse var thereferer . '//' refip '/' .
          iik=is_logok(-1,,refip)
          if iik=1 then do
             ii=rlog.0+1
             if stuff.!apache_mode=1 then
                rlogq=d1t1_0||' '||thereferer||' -> '||thereq
             else
                rlogq=d1t1||' "'||thereq||'" '||thereferer
             rlog.ii.!file=which_logs(serverport,host_nickname,'REFERERLOG')
             rlog.ii=rlogq
             rlog.0=ii
          end
      end

/* the combined log */
/* eg: (all on one line) :
lion.statslab.cam.ac.uk - - [18/Jan/1996:12:04:23 +0000]
"GET /~sret1/analog/ HTTP/1.0" 200 578
"http://www.statslab.cam.ac.uk/~sret1/"
"Mozilla/2.0 (X11; I; HP-UX A.09.05 9000/735)"
*/
     ii=alllog.0+1
     alllog.ii=clogq||' "'||thereferer||'" "'||browser||'"'
     alllog.ii.!file=which_logs(serverport,host_nickname,'COMBINEDLOG')
     alllog.0=ii
     if verbose>0 then  foo=pmprintf_sref(clogq,1)  /* do NOT write to srefiltr.log */

  end

 IF POST_FILTER="NO" | post_filter=0 THEN do
    if tcache<>' ' then foo=sysfiledelete(tcache)
    signal bakme
  end

 tookie=postfilter_name
 do mm=1 to words(tookie)
   postfilter_name=word(tookie,mm)
   if pos('.',postfilter_name)=0 then
        postfilter_name=postfilter_name||'.'||serverport

   signal on syntax name bad2
   signal on error name bad2
    yow='foo3='||postfilter_name||'(post_filter_message,source0,request0a,sel0,tempfile,servername,HOST_NICKNAME,used_file,thereferer,tcache,record2)'

   interpret yow

   signal off syntax;  signal off error
 end

 if tcache<>' ' then foo=sysfiledelete(tcache)
 signal bakme

bad2:
  signal off syntax; signal off error
  call pmprintf_sref(" SRE-Filter Postfilter Thread: Problem in post_filter routine " postfilter_name " ("seloriginal)
  if tcache<>' ' then foo=sysfiledelete(tcache)
  signal bakme


iserror:
signal off error ; signal off syntax
call pmprintf_sref(' error in postfilter : resetting 'sigl' (rc='rc)
a=rxqueue('d',usequeue)
a=rxqueue('c',usequeue)
a=eventsem_close(usesem)
a=eventsem_create(usesem)
call pmprintf_sref(' done resetting postfilter ')
signal on error name iserror
signal on syntax name iserror
signal bakme


/* ========================================================== */
/* --- handle "cached recordall_file -----*/
do_cache_record:procedure expose record_cache_lines wilds. comments. simples. ,
                update_rate verbose

parse arg record_all_file,doit,doit0

numeric digits 15

/* no cache -- then get one */
if comments.filestamp=0 then do
    nrecs=open_record_cache(record_all_file)
   if verbose>0 then call pmprintf_sref(" First read of RECORD_ALL_FILE: " nrecs)
    if nrecs=0 then return 0
end
else do
    nrecs=comments.0+simples.0+wilds.0
end  /* Do */

/* cache too big? */
if nrecs>record_cache_lines then do
    nrecs=reset_cache(record_all_file,record_cache_lines)
end

/* add current hit,  or re-read file  */
foo=update_cache(doit)

/* do a timed updated to disk */
d1=date('b')
t1=time('s')/(24*60*60)
nowtime=d1+t1
yeeks=comments.istime+update_rate
if (yeeks<nowtime) | strip(upper(doit0))="!WRITE_CACHE" then do
   foo=save_cache(record_all_file)
end


return 1


/* ---------------- read recordall file into cache */
open_record_cache:procedure expose wilds. comments. simples. verbose
parse arg record_all_file

aa=sref_fileread(record_all_file,thelines,,'E')
if aa<0 then return 0                /* error, use default method */

drop wilds. comments. simples.

nsimples=0 ; ncomments=0 ; nwilds=0
simples.0=0 ; comments.0=0 ; wilds.0=0
do mm=1 to thelines.0
   aline=strip(thelines.mm)
   if abbrev(aline,';')=1 | aline=' '  then do
        if aline=' ' then iterate
        ncomments=ncomments+1
        comments.ncomments=aline
        comments.0=ncomments
   end
   else do
       parse  var aline aname act amess
       if pos('*',aname)>0 then do
           aname=strip(aname)
           aname=translate(aname,'/','\')
           aname=strip(aname,'l','/')
           if act="" | datatype(act)<>'NUM' then act=0
           nwilds=nwilds+1
           wilds.nwilds.qname=aname ; wilds.nwilds.qct=act
           wilds.nwilds.qmess=amess
           wilds.0=nwilds
       end
       else do
            nsimples=nsimples+1
            if act="" | datatype(act)<>'NUM' then act=0
            simples.nsimples.qname=aname; simples.nsimples.qct=act
            simples.nsimples.qmess=amess
            simples.0=nsimples
       end
   end
end

/* get time stamp */
d1=date('b')
t1=time('s')/(24*60*60)
comments.istime=d1+t1

/* get file stamp */
oo=sysfiletree(record_all_file,'yow')
comments.filestamp=strip(upper(yow.1))


return nwilds+ncomments+nsimples


/* ------------------- Write cache---------------*/
write_cache:procedure expose comments. wilds. simples. verbose
parse arg rfile,addtimestamp


nj=0
if addtimestamp=1 then do
   nj=1
   alines.nj="; File reset on: "||time('N')||' '||date('N')
end  /* Do */

do mm=1 to comments.0
  nj=nj+1
  alines.nj=comments.mm
end
do mm=1 to wilds.0
  nj=nj+1
  alines.nj=wilds.mm.qname||' '||wilds.mm.qct||' '||wilds.mm.qmess
end


/* sort simples in descending order, by frequency */
do iy=1 to simples.0
   foobar.iy=simples.iy.qct'         'iy
end
if simples.0>0 then wow=arraysort(foobar,1,simples.0,1,8,'D','N')

do mm=1 to simples.0
    parse var foobar.mm act indx ; indx=strip(indx)
    nj=nj+1
    alines.nj=simples.indx.qname||' '||simples.indx.qct||' '||simples.indx.qmess
end

alines.0=nj



wow=filewrite(rfile,alines)

/* get time stamp */
d1=date('b')
t1=time('s')/(24*60*60)
comments.istime=d1+t1

/* get file stamp */
oo=sysfiletree(rfile,'yow')
comments.filestamp=strip(upper(yow.1))


if wow=0 then return 0

return nj


/* ------------- Reset the cache: --------------------- */
/*     Save current cache to a backup recordall file, delete current
     cache file, and rewrite minimal (just wildcards and comments)
     set
*/

reset_cache:procedure expose comments. wilds. simples. verbose
parse arg rfile,maxlines

/* but update if external changes occured  (might obviate need to shrink cache) */
isold=cache_unsync(rfile,comments.filestamp)
if isold=1 then do
     nrecs=open_record_cache(rfile)
     if verbose>0 then call pmprintf_sref(' External change, reread RECORD_ALL_FILE into cache ')
     return nrecs
end

rfile2=rfile
/* rename recordallfile to record.nnn (nnn=01 .. 999) */
foofoo=lastpos('.',rfile)
if foofoo>0 then
   rfile2=delstr(rfile,foofoo)

useme=' '
do mm=1 to 999
   if stream(rfile2||'.'||mm,'c','query exists')=' ' then do
       useme=rfile2||'.'||mm
       leave
   end
end
if useme=' ' then useme=dostempname(rfile2||'.???') /* should never happen ..*/
foo=dosrename(rfile,useme)
if foo=0 then return 0
if verbose>0 then call pmprintf_sref('RECORD_ALL_FILE too big; reset (old file renamed to '||useme)

if wilds.0+comments.0> maxlines then do
   if verbose>0 then call pmprintf_sref(" Warning: truncating comments in (RECORD_ALL_FILE) " rfile)
   comments.0=trunc(maxlines/2)
end

/* reset counts */
simples.0=0
do mm=1 to wilds.0
   wilds.mm.qct=0
end

nrec=write_cache(rfile,1)
if verbose>0 then call pmprintf_sref(" Write RECORD_ALL_FILE cache: " nrec)

return wilds.0





/* --------------------- update cache ---------*/
/* match an entry to existing cache -- augment the match
or add a new entry */

update_cache:procedure expose wilds. simples. verbose

parse arg doit

/* fix up doit */
doit=translate(strip(doit),'/','\')
doit=strip(doit,'l','/')

nsimples=simples.0
do mm=1 to nsimples
   if upper(simples.mm.qname)=upper(doit) then do
        simples.mm.qct=simples.mm.qct+1
        simples.mm.qmess=date('o')
        return 1
   end
end


nwilds=wilds.0
asterat=0 ; afterstar=0 ; gotit=0
do mm=1 to nwilds
    ares=sref_wildcard(upper(doit),upper(wilds.mm.qname),0)
    parse var ares astat "," aurl ;  astat=strip(astat)
    if astat=0 then iterate   /* no match */

   foo1=pos('*',wilds.mm.qname)
   if foo1 >= asterat then do
       tmp1=length(wilds.mm.qname)-foo1
       if foo1>asterat | tmp1>afteraster then  do
          asterat=foo1 ; afteraster=tmp1
          gotit=mm
       end              /* either condition */
    end
end

if gotit>0 then do
    wilds.gotit.qct=wilds.gotit.qct+1
    wilds.gotit.qmess=date('O')
    return 1
end


nsimples=nsimples+1 ;simples.0=nsimples
simples.nsimples.qname=doit
simples.nsimples.qct=1
simples.nsimples.qmess=date('o')


return 1


/* ------------- save the cache: --------------------- */
/*     Save current cache to recordall file, rename current
      file to .bak
*/

save_cache:procedure expose comments. wilds. simples. verbose
parse arg rfile


/* but update if external changes occured  */
isold=cache_unsync(rfile,comments.filestamp)
if isold=1 then do
     nrecs=open_record_cache(rfile)
     if verbose>0 then call pmprintf_sref(' RECORD_ALL_FILE changed, did not write cache (reread RECORD_ALL_FILE instead ')
     return nrecs
end



rfile2=rfile
/* rename recordallfile to record.nnn (nnn=01 .. 999) */
foofoo=lastpos('.',rfile)
if foofoo>0 then
   rfile2=delstr(rfile,foofoo)
useme=rfile2||'.bak'
a=sysfiledelete(useme)
a=dosrename(rfile,useme)
if verbose>1 then  call pmprintf_sref(' Writing current RECORD_ALL_FILE cache ')

nrec=write_cache(rfile,0)

return nrec


/* --- return 1 if filestamp has changed */
cache_unsync:procedure expose verbose
parse arg record_all_file,thestamp

/* get file stamp */

oo=sysfiletree(record_all_file,'yow')
afilestamp=strip(upper(yow.1))

if afilestamp=thestamp then return 0
return 1


/* -------- get julian date/time of file ----*/
file_jul_date:procedure
parse arg afile

how0=dosfileinfo(afile,'W')
parse var how0 how hh ':' mm ':' ss
nsec=((hh*60*60) + (mm*60) + ss)/(24*60*60)
ndays=dateconv(how,'U','B')
return ndays+nsec


/***************************/
/* check for log suppression condition */
is_logok:procedure expose stuff.
parse upper arg sel1,doorig,clientname0,record_status,write_logs,user


if sel1=-1 then do
    aa=stuff.!noreferer
    if aa=' ' then return 1
end
else do
    if write_logs=0 then return 0
 /* check status */
    aa=stuff.!nocodes
   if wordpos(strip(record_status),aa)>0 then return 0

/* check clientname */

    aa=stuff.!nouser
/* is it a direct match to the "user" */
    if wordpos(upper(user),aa)>0 then return 0
end

/* nope, so now check ip addresses  (possibly noreferer) */
c1=translate(clientname0,' ','.')
do ll=1 to words(aa)
   aa1=upper(strip(word(aa,ll)))  /* for each entry in the nouser list */
   if clientname0=aa1 then return 0       /* exact match */
   if user=aa1 then return 0       /* exact match */

   if pos('*',aa1)>0 then do     /* possible wild card match */
        aa1=translate(aa1,' ','.')
        if words(aa1) <> words(c1) then iterate /* can't be a match */
        die=1
        do ll2=1 to words(c1)
             c1q=strip(word(c1,ll2)) ; aa1q=strip(word(aa1,ll2))
             if aa1q='*' then iterate
             if c1q=aa1q then iterate
             die=0 ; leave              /* if here, segment did not match */
        end /* do */
        if die=1 then return 0  /* all segments matched */
   end  /* Do wildcard  */
end /* do */

if sel=-1 then return 1         /* noreferer did NOT match */

/* check urls */
theurl=strip(sel1)
if doorig=1 then theurl=' '
ido=stuff.!nourl.0
starat=0 ; afterstar=0

do mm=1 to ido
    aurl0=stuff.!nourl.mm
    ares=sref_wildcard(theurl,aurl0,0)
    parse var ares astat  .
    astat=strip(astat)
    if astat=0 then iterate   /* no match */
    return 0  /* match, so it's dead */
end
return 1



/***************************/
/* read sreflogs.ini file, return stuff in stuff. */
get_sreflogs:procedure expose stuff. workdir verbose
parse arg afl
/* possible varialbes */
  varlist="COMMONLOG BROWSERLOG REFERERLOG COMBINEDLOG NOOPTIONS FORCE_REFERER APACHE_MODE NOHTTP NOCODES NOUSER NOREFERER NOURL "
  stemlist="COMMONLOG BROWSERLOG REFERERLOG COMBINEDLOG NOURL SCHED "

/* default values */
 stuff.!nooptions=0 ; stuff.!nohttp=0
 stuff.!nourl.0=0 ; stuff.!noreferer=' '
 stuff.!nocodes=' ' ; stuff.!nouser=' '
 stuff.!commonlog=workdir||'\COMMON.LOG'
 stuff.!browserlog=workdir||'\BROWSER.LOG'
 stuff.!refererlog=workdir||'\REFEFER.LOG'
 stuff.!combinedlog=workdir||'\COMBINED.LOG'

 stuff.!list='COMMONLOG BROWSERLOG REFERERLOG COMBINEDLOG'
 stuff.!stamp=' '
 stuff.!sched.0=0

/* look in sreflogs.ini for info */
afl=workdir||'\SREFLOGS.INI'
foo=sref_fileread(afl,logsl,,'E')
if foo<0 then do
   if verbose >2 then call pmprintf_sref(' SRE-Filter Post-Filter Thread: ' foo ' problem in sreflogs.ini file: 'afl)
   return 1
end
if logsl.0=0 then return 1       /* use the defaults */


stuff.!stamp=sysfiletree(afl,'FOO','f')
STUFF.!STAMP=FOO.1


do mm=1 to logsl.0
   aline=strip(upper(logsl.mm))
   if aline="" | abbrev(aline,';')=1 then iterate
   parse upper var aline avar '=' aval
   aval=strip(translate(aval,' ','"'||"'"))
   avar0=strip(avar)
   avar=translate(avar0,' ','.')
   avar1=strip(word(avar,1))
   if words(avar)=1 then do      /* check the 1 word opts */
      if wordpos(avar1,varlist)=0 then iterate  /* not a valid variable */
      if wordpos(avar1,'COMMONLOG BROWSERLOG REFERERLOG COMBINEDLOG')>0 then do /* check validity */
          aval=chk_file(workdir,aval)
      end

      exx='!'||avar1
      stuff.exx=aval

      iterate
   end
/* else, one of the stem boys */

  if wordpos(avar1,stemlist)=0 then iterate  /* not valid stem var */
  select
       when avar1="NOURL" then do
          inq=stuff.!nourl.0+1
          if pos('?',aval)>0 then do
               parse var aval a1 '?' a2
               a1=strip(translate(a1,'/','\'),'l','/')
               aval=a1||'?'||a2
          end
          else do
             aval=strip(translate(aval,'/','\'),'l','/')
          end
          stuff.!nourl.inq=aval
          stuff.!nourl.0=inq
        end
       when avar1="SCHED" then do
          if aval<>0 then do
             inq=stuff.!sched.0+1
             stuff.!sched.inq=aval
             stuff.!sched.0=inq
          end
        end

        otherwise  do      /* check for allowables */
            useme=chk_file(workdir,aval)
            if useme=0 then iterate
            exx=avar1||'.!'||word(avar,2)
            if words(avar)>2 then exx=exx||'.!'||word(avar,3)
             exx2='!'||exx
            stuff.exx2=useme
            stuff.!list=stuff.!list||' '||exx
        end             /*otherwise */
  end                /* select */

end          /* logsl */

if verbose >2 then call pmprintf_sref(' SRE-Filter Post-Filter Thread: Log file list:loglist 'stuff.!list)
return 1

/************/
/* check for valid file name, and return it. Return 0 if no good */
chk_file:procedure
parse arg workdir, aval

  fn1=translate(strip(upper(aval)),'\','/')  /* the filename */
  if fn1=0 then return 0

  fn1=strip(translate(fn1,' ',"'"||'"'))
  if abbrev(fn1,'\')=1 | pos(':',fn1)>0 then
       fuse=fn1
   else
       fuse=workdir||'\'||fn1
   foo=strip(filespec('d',fuse)||filespec('p',fuse),'t','\')
   if right(foo,1)=':' then foo=foo||'\'

   if dosisdir(foo)=0 then return 0  /* no such dir--error, so ignore */

   return fuse



/***************/
/* write to the log files */
write_clog:procedure expose clog.
parse arg ttime,ltime,lbig

if clog.0=0 then return 0

/* are they all to the same file? */
bname=clog.1.!file
idid=1
do mm=2 to clog.0
      if clog.mm.!file<>bname then leave
      idid=mm
end
if idid=clog.0  then do  /* all same name, dump the bunch */

   if bname=0 then return 0 /* all suppressed */
    do nn=1 to clog.0
       tt.nn=clog.nn
    end
    tt.0=clog.0
    foo=filewrite(bname,tt,'A')
    cfoo=jeepers(bname,lbig)
end
else do         /* several files (multi host system */
  tt.0=1
  do nn=1 to clog.0
    if clog.nn.!file=0 then iterate
    tt.1=clog.nn
    foo=filewrite(clog.nn.!file,tt,'A')
    foo=jeepers(clog.nn.!file,lbig)
 end
end

return 1

/********/
/* is log file too big (check every 4th time? */
jeepers:procedure
parse arg bname0,toob

if toob=0 then return 1
foo1=random(1,5)
if foo1<5 then return 0   /* every 5th write */

foo=sysfiletree(bname0,'boo','F')

if boo.0>0 then do
  ih=strip(word(boo.1,3))
  if ih>(1000*toob) then do
      call pmprintf_sref(' ')
      call pmprintf_sref(' ******             WARNING                  ****** ')
      call pmprintf_sref(' Your log file ('bname0') is becoming large ('ih)
      call pmprintf_sref(' ******             WARNING                  ****** ')
      call pmprintf_sref(' ')

  end
end
return 0





/***************/
/* this is sort of a stupid algorithim, but it's probably adequate
(given rexx's problems with passing stem variables */

/* write to the common log files */
write_blog:procedure expose blog.

if blog.0=0 then return 0

/* are they all to the same file? */
bname=blog.1.!file
idid=1
do mm=2 to blog.0
      if blog.mm.!file<>bname then leave
      idid=mm
end
if idid=blog.0  then do  /* all same name, dump the bunch */
   if bname=0 then return 0 /* all suppressed */
    do nn=1 to blog.0
       tt.nn=blog.nn
    end
    tt.0=blog.0
    foo=filewrite(bname,tt,'A')
end
else do         /* several files (multi host system */
  tt.0=1
  do nn=1 to blog.0
    if blog.nn.!file=0 then iterate
    tt.1=blog.nn
    foo=filewrite(blog.nn.!file,tt,'A')
 end
end
return 1


/***************/
/* write to the refeerer log files */
write_rlog:procedure expose rlog.

if rlog.0=0 then return 0

/* are they all to the same file? */
bname=rlog.1.!file
idid=1
do mm=2 to rlog.0
      if rlog.mm.!file<>bname then leave
      idid=mm
end
if idid=rlog.0  then do  /* all same name, dump the bunch */
   if bname=0 then return 0 /* all suppressed */
    do nn=1 to rlog.0
       tt.nn=rlog.nn
    end
    tt.0=rlog.0
    foo=filewrite(bname,tt,'A')
end
else do         /* several files (multi host system */
  tt.0=1
  do nn=1 to rlog.0
    if rlog.nn.!file=0 then iterate
    tt.1=rlog.nn
    foo=filewrite(rlog.nn.!file,tt,'A')
 end
end
return 1


/***************/
/* write to the common log files */
write_alllog:procedure expose alllog.

if alllog.0=0 then return 0

/* are they all to the same file? */
bname=alllog.1.!file
idid=1
do mm=2 to alllog.0
      if alllog.mm.!file<>bname then leave
      idid=mm
end
if idid=alllog.0  then do  /* all same name, dump the bunch */
   if bname=0 then return 0 /* all suppressed */
    do nn=1 to alllog.0
       tt.nn=alllog.nn
    end
    tt.0=alllog.0
    foo=filewrite(bname,tt,'A')
end
else do         /* several files (multi host system */
  tt.0=1
  do nn=1 to alllog.0
    if alllog.nn.!file=0 then iterate
    tt.1=alllog.nn
    foo=filewrite(alllog.nn.!file,tt,'A')
 end
end
return 1



/************/
/* which log file to use */
which_logs:procedure expose stuff.
parse arg sport,hname,dalog
if hname<>' ' then do
  alist=dalog||'.!'||hname||'.!'||sport
  alist=alist||' '||dalog||'.!'||hname||' '||dalog
end
else do
    alist=dalog||'.!'||sport||' '||dalog
end
alist=upper(alist)
do li=1 to words(alist)
   a1=strip(word(alist,li))
   if wordpos(a1,stuff.!list)>0 then do
       axe='!'||a1
       return stuff.axe
    end
end
return 0


/*************/
/* The scheduler */

do_scheduler:procedure expose stuff. last_hour_checked verbose

if verbose>3 & stuff.!sched.0>0 then call pmprintf_sref(' Checking ' stuff.!sched.0' SCHED. entries')

if stuff.!sched.0=0  | datatype(stuff.!sched.0)<>'NUM'then return 0   /* no scheduled programs to run */

atime=time('n')
parse var atime ahr ':' amin ':' asec
if ahr=last_hour_checked then return 0
adate=date('n')
parse var adate mday .
aweekday=upper(date('w'))

/* else, check scheduled programs */
do ii=1 to stuff.!sched.0
  parse upper var stuff.!sched.ii  afreq aprog stuff ; aprog=strip(aprog); afreq=strip(afreq)
  doit=0
  select
     when abbrev(afreq,'HOUR')=1 then   doit=1
     when abbrev(afreq,'DA')=1 & ahr=0 then doit=1
     when abbrev(afreq,'WE')=1 & aweekday="SUNDAY" then doit=1
     when abbrev(afreq,'MO')=1 & mday=1 then doit=1
     otherwise doit=0
   end
   signal off error  ; signal off syntax
   signal on error name hoy1
   signal on syntax name  hoy1
   if doit =1 then do
        astat=rexxrun('f',aprog,,stuff)
        if verbose>1 & astat<>0 then call pmprintf_sref(' REXXLIB REXXRUN error code = ' astat ' on scheduled event: 'aprog)
   end
end

last_hour_checked=ahr

signal off error ; signal off syntax
return 0

hoy1:
call pmprintf_sref(' Error: could not execute scheduled program: 'aprog)
signal off error ; signal off syntax
return 0


/******************************/
/* Add entry to the hit cache*/

add_hit_cache:procedure expose myqueue basesem mysem mytid
parse arg  thetype, theurl0, who , enmadd,stuff
theurl0=upper(theurl0) ; thetype=upper(thetype)

nogi=digits()

lenc=value(enmadd||'HIT_CACHE_LEN',,'OS2ENVIRONMENT')

if lenc=0 then return ' '               /* suppressed */
transaction=mytid

numeric digits 11
ttc=value(enmadd||'HIT_CACHE_DURATION',,'OS2ENVIRONMENT')
ttc=ttc/(60*24)
d1=date('b')
t1=time('m')/(24*60)
nowtime=d1+t1
endtime=nowtime+ttc
theurl0=translate(theurl0,'/','\')
moo=lenc||' '||thetype||' '||theurl0||' '||who||' '||endtime||' '||stuff

if lenc='FILE' then do          /* add to cache file */
  itis=file_addhitc(moo,nowtime)
  numeric digits nogi
  return 0
end

/* or use VARSTORE thread  */
goober=enmadd||'VARSTORE'
a=rxqueue('s',goober)
    queue  transaction ' ' host_nickname ',' myqueue ',' mysem ',' 'PUT_HIT, '||moo
dothread=basesem||'VARSTORE'
a=eventsem_post(dothread)  /* no return expected */
return 0


/* --------------- */
/* add to the  .stem  file (slower, but perhaps better for huge numbers of hits */
file_addhitc:procedure expose enmadd
parse arg  moo,nowtime

arf=extract('serverport')

adir=strip(value(enmadd||'TEMPDATA_DIR',,'os2environment'),'t','\')||'\'

cfile=adir||'_HITCACH.'||arf
if stream(cfile,'c','query exists')=' ' then do
   call pmprintf_sref(" Creating hit-cache file " cfile)
   eek.1=";The sre-filter hit cache file-- do NOT edit ! "
   foo=filewrite(cfile,'eek','R',1,1)
   if foo=0 then do
        call pmprintf_sref( " Error creating hit-cache file: " cfile)
        return
   end
end

/* wait for it to be unlocked */
isnow=time('r')
do until twait>30
  twait=time('e')
  aa=sref_fileread(cfile,'yy',,'E')
  if aa>=0 then leave
  ddt=syssleep(0.2)
end
if aa<0 then do
   call pmprintf_sref(" Hit cache file could not be opened for augmentation: " cfile)
   return ' '
end

/* now lock it! We'll clean it up, and add current hit to beginning */
aa=sref_open_read(cfile,10,'write')
if aa<0 then do
   call pmprintf_sref(" Hit cache file could not be opened for augmentation: " cfile)
   return ' '
end


newl.1=moo
i1=1
/* remove expired entries */
do iu=1 to yy.0
   aline=yy.iu
   if aline=' ' then iterate
   if abbrev(strip(aline),';')=1 then do
       i1=i1+1
       newl.i1=aline
       iterate
    end
    parse var aline atype aurl aip  aendtime .
    aendtime=strip(aendtime)
    if aendtime >= nowtime then do
        i1=i1+1
        newl.i1=aline
    end
    else do
    end
    if i1>10000 then leave   /* prevent too big */
end
newl.0=i1
aa=stream(cfile,'c','close')
foo=filewrite(cfile,'newl','R',i1,1)
if foo=0 | foo<=0 then
  call pmprintf_sref( " Could not augment&update hit-cache file ")
return ' '


/*****************************/
/* see if url is in the  "hit cache"  */
look_hit_cache:procedure expose basesem mysem myqueue mytid
parse upper arg thetype, theurl, who , enmadd

theurl=translate(theurl,'/','\')

lenc=value(enmadd||'HIT_CACHE_LEN',,'OS2ENVIRONMENT')
if lenc=0 then return ' '               /* suppresed */
transaction=mytid
nogi=digits()

numeric digits 12
d1=date('b')
t1=time('m')/(24*60)
nowtime=d1+t1

if lenc='FILE' then do
  itis=file_lenhitc(thetype,theurl,who,enmadd,nowtime)
   numeric digits nogi
  return itis
end


/* here to read from VARSTORE thread */
/* or use VARSTORE thread  */
goober=enmadd||'VARSTORE'
moo=thetype||' '||who||' '||theurl
a=eventsem_reset(mysem)
a=rxqueue('s',goober)
    queue  transaction ' ' host_nickname ',' myqueue ',' mysem ',' 'GET_HIT, '||moo
dothread=basesem||'VARSTORE'
a=eventsem_post(dothread)

a=eventsem_wait(mysem) /* wait for answer */

if a<>0 then do
   call pmprintf_sref(' A Fatal Semaphore failure in POSTF-Look_hit_cache: 'a)
   return ' '
end
a=rxqueue('s',myqueue)

parse pull aline

PARSE VAR ALINE idnum ',' aline

idnum=strip(translate(idnum,' ','000d0a'x));TRANSACTION=STRIP(TRANSACTION)
if idnum<>transaction then  do /*wierd error: got someone else's message, give up */
      say ' Read odd id from VARSTORE queue :' transaction ',' idnum
      return ' '
end
return strip(aline)


/* --------------- */
/* read from cachelist from an .idx file,, slower but perhaps better for huge number of hits */

file_lenhitc:procedure
parse arg thetype,theurl,who,enmadd,nowtime

arf=extract('serverport')
adir=strip(value(enmadd||'TEMPDATA_DIR',,'os2environment'),'t','\')||'\'
theurl=strip(translate(theurl,'/','\'))

cfile=adir||'_HITCACH.'||arf
if stream(cfile,'c','query exists')=' ' then do
   say "  No hit cache file " cfile
   return ' '
end

/* wait for it to be unlocked */
isnow=time('r')
do until twait>30
  twait=time('e')
  aa=sref_fileread(cfile,'yy')
  if aa>=0 then leave
  ddt=syssleep(0.2)
end
if aa<0 then do
   say " Hit cache file could not be opened: " cfile
   return ' '
end

/* see if a match exists */
do iu=1 to aa
   aline=yy.iu
   if aline=' ' then iterate
   if abbrev(strip(aline),';')=1 then iterate
    parse var aline atype aurl aip  aendtime stuff
    atype=strip(atype) ; aurl=strip(aurl) ; aip=strip(aip)
    aendtime=strip(aendtime)
   if theurl<>aurl | aip<>who | atype<>thetype then iterate
   if aendtime >= nowtime then do
        return stuff
   end
end
return ' '              /* no match */