home *** CD-ROM | disk | FTP | other *** search
/ OS/2 Shareware BBS: 35 Internet / 35-Internet.zip / rxiutl16.zip / ngfetch.cmd < prev    next >
OS/2 REXX Batch file  |  2000-01-11  |  18KB  |  608 lines
  1. /* Fetch list of newsgroups */
  2. call RxFuncAdd 'SysLoadFuncs','RexxUtil','SysLoadFuncs'
  3. call SysLoadFuncs
  4. call RxFuncAdd 'SockLoadFuncs','RxSock','SockLoadFuncs'
  5. signal on SYNTAX
  6. nosock=1
  7. call SockLoadFuncs 'q'
  8. nosock=0
  9. signal off SYNTAX
  10.  
  11. .local['RXIVERSION']=0.16
  12. parse version rxversion
  13. if word(rxversion,1)\='OBJREXX' then do
  14.     say ''
  15.     say 'NGFETCH.CMD requires Object REXX to function.'
  16.     say ''
  17.     say 'You are running: '||rxversion
  18.     say ''
  19.     say 'README.TXT contains more information.'
  20.     exit 4
  21. end
  22.  
  23. do until stream(rfile,'c','query exists')=''
  24.     tmp1=random(0,65535)
  25.     tmp2=random(0,65535)
  26.     rfile=tmp1~d2x~right(4,'0')||tmp2~d2x~right(4,'0')||'.tmp'
  27. end
  28.  
  29. '@ver > '||rfile
  30. call SysFileSearch 'Windows 95',rfile,'chk1.'
  31. call SysFileSearch 'Win95',rfile,'chk2.'
  32.  
  33. if chk1.0>0 | chk2.0>0 then win95=.true
  34. else win95=.false
  35.  
  36. signal on HALT
  37.  
  38. parse arg argstring
  39.  
  40. .local['BLOCKSIZE']=10240
  41. if \win95 then .local['STINTERVAL']=50
  42. else .local['STINTERVAL']=100
  43. .local['MAXRETRIES']=1000
  44. .local['WRITERAW']=.false
  45. .local['SUPERQUIET']=.false
  46. quietmode=.false
  47. hostport=''
  48. sort=.false
  49.  
  50. do i=1 to argstring~words
  51.     badvar=.false
  52.     currarg=argstring~word(i)~strip
  53.     select
  54.         when (currarg~left(1)='/' | currarg~left(1)='-') then do
  55.             select
  56.                 when currarg~length=2 then do
  57.                     select
  58.                         when currarg~right(1)~translate='Q' then quietmode=.true
  59.                         when currarg~right(1)~translate='W' then .local['WRITERAW']=.true
  60.                         when currarg~right(1)~translate='S' then sort=.true
  61.                         otherwise do
  62.                             call qasay ''
  63.                             call qasay 'Invalid switch, '||currarg||'.'
  64.                             call usage
  65.                             exit 5
  66.                         end
  67.                     end
  68.                 end
  69.                 when currarg~length=3 & currarg~right(2)~translate='QQ' then do
  70.                     .local['SUPERQUIET']=.true
  71.                     quietmode=.true
  72.                 end
  73.                 when currarg~left(2)~right(1)~translate='B' then do
  74.                     .local['BLOCKSIZE']=currarg~right(currarg~length-2)
  75.                     if .blocksize~datatype('W')=0 then badvar=.true
  76.                     else if (.blocksize<512 | .blocksize>65535) then badvar=.true
  77.                     if badvar then do
  78.                         say ''
  79.                         say 'Invalid blocksize, '||.blocksize||'.'
  80.                         exit 5
  81.                     end
  82.                 end
  83.                 when currarg~left(2)~right(1)~translate='T' then do
  84.                     .local['STINTERVAL']=currarg~right(currarg~length-2)
  85.                     if .blocksize~datatype('W')=0 then badvar=.true
  86.                     else do
  87.                         if win95 then do
  88.                             if (.stinterval<1 | .stinterval>5) then badvar=.true
  89.                             else .local['STINTERVAL']=.stinterval*100
  90.                         end
  91.                         else if (.stinterval<1 | .stinterval>500) then badvar=.true
  92.                     end
  93.                     if badvar then do
  94.                         say ''
  95.                         say 'Invalid status time interval, '||.stinterval||'.'
  96.                         exit 5
  97.                     end
  98.                 end
  99.                 when currarg~left(2)~right(1)~translate='M' then do
  100.                     .local['MAXRETRIES']=currarg~right(currarg~length-2)
  101.                     if .maxretries~datatype('W')=0 then badvar=.true
  102.                     else if (.maxretries<0 | .maxretries>9999999999) then badvar=.true
  103.                     if badvar then do
  104.                         say ''
  105.                         say 'Invalid maximum retries, '||.maxretries||'.'
  106.                         exit 5
  107.                     end
  108.                 end
  109.                 otherwise do
  110.                     call qasay ''
  111.                     call qasay 'Invalid parameter ('||currarg||').'
  112.                     call usage 
  113.                     exit 5
  114.                 end
  115.             end
  116.         end
  117.         otherwise do
  118.             hostport=currarg
  119.         end
  120.     end
  121. end
  122.  
  123. if .writeraw & sort then do
  124.     call qasay ''
  125.     call qasay 'Raw writing and sorting are mutually exclusive.'
  126.     call usage
  127.     exit 5
  128. end
  129.  
  130. if hostport='' then do
  131.     call qasay ''
  132.     call qasay 'No host specified.'
  133.     call usage
  134.     exit 5
  135. end
  136.  
  137. .local['CRLF']='0d0a'x
  138. if \.superquiet then call SysCurState 'off'
  139. .local['CON']=.estream~new('stdout')
  140. listend='0d0a2e0d0a'x
  141. lgroupm='2e0d0a'x
  142. exitcode=0
  143. .local['XFS']='XFER_FINISHED'
  144. .local['STFS']='STATUS_THREAD_EXIT'
  145.  
  146. outfile=.stream~new('newsrc')
  147.  
  148. parse var hostport host ':' port
  149. if port='' then port=119
  150.  
  151. sendstring='LIST'||.crlf
  152.  
  153. csock=.socket~new
  154.  
  155. if \.superquiet then call SysCls
  156. .con~~charout(.crlf~copies(5))~flush
  157. .con~~charout('Connecting to host '||host||' on port '||port||'... ')~flush
  158. check=csock~connect(host||':'||port)
  159. if check='HALT' then signal HALT
  160. if check=-1 then do
  161.     .con~~charout('Failed!'||.crlf||.crlf)~flush
  162.     csock~shutdown('both')
  163.     csock~close
  164.     exit 1
  165. end
  166. else .con~~charout('Done'||.crlf)~flush
  167.  
  168. total=0
  169. data=''
  170. groups=0
  171. glist=.queue~new
  172. check=csock~receive(.blocksize)
  173. if check='HALT' then signal HALT
  174. total=total+check[1]~length
  175. if check[1]~word(1)='400' then do
  176.     call qasay ''
  177.     call qasay 'News server disconnected (probably exceeded per host connection limit).'
  178.     call qasay ''
  179.     csock~shutdown('both')
  180.     csock~close
  181.     exit 2
  182. end
  183. else do
  184.     if outfile~open('write replace')\='READY:' then do
  185.         call qasay ''
  186.         call qasay 'Unable to open '||outfile~qualify||' for writing.'
  187.         csock~close
  188.         exit 4
  189.     end
  190.     if .writeraw then outfile~charout(check[1])
  191. end
  192.  
  193. .con~~charout('Requesting group list... ')~flush
  194. check=csock~send(sendstring)
  195. if check='HALT' then signal HALT
  196. if check=-1 then do
  197.     .con~~charout('Failed!'||.crlf||.crlf)~flush
  198.     csock~shutdown('both')
  199.     csock~close
  200.     exit 1
  201. end
  202. else .con~~charout('Done'||.crlf)~flush
  203.  
  204. .con~~charout('Receiving group list... '||.crlf||.crlf)~flush
  205. call time 'r'
  206.  
  207. .local['SQ']=.queue~new
  208. .local['TQ']=.queue~new
  209. .local['GQ']=.queue~new
  210. scs=.scrstatus~new
  211. .sq~queue(total)
  212. .gq~queue(groups)
  213. retries=0
  214.  
  215. if \quietmode then scs~template
  216. if \quietmode then scs~begin
  217.  
  218. headgone=.false
  219. do until check[1]~right(5)=listend
  220.     check=csock~receive(.blocksize)
  221.     if check='HALT' then signal HALT
  222.     rvar=check[2]
  223.     if rvar<0 then do
  224.         if retries>.maxretries then do
  225.             call qasay ''
  226.             call qasay ''
  227.             call qasay 'Maximum retries reached, transfer failed.'
  228.             csock~close
  229.             exit 2
  230.         end
  231.         else retries=retries+1
  232.     end
  233.     total=total+check[1]~length
  234.     if \.superquiet then .sq~queue(total)
  235.     data=data||check[1]
  236.     if .writeraw then do
  237.         outfile~charout(data)
  238.         data=''
  239.     end
  240.     else do
  241.         if \headgone then if data~word(1)='215' then do
  242.             parse var data '215' . (.crlf) data
  243.             headgone=.true
  244.         end
  245.         do while data~pos(.crlf)>0
  246.             parse var data group ' ' (.crlf) data
  247.             if sort then glist~queue(group)
  248.             else outfile~lineout(group)
  249.             groups=groups+1
  250.             .gq~queue(groups)
  251.             if data~left(3)=lgroupm then leave
  252.         end        
  253.     end
  254. end 
  255. ct=time('e')
  256. if \quietmode then .sq~queue(total)
  257. if \quietmode then .gq~queue(groups)
  258. if \quietmode then .sq~queue(.xfs)
  259. if \quietmode then do
  260.     stcheck=''
  261.     do until stcheck=.stfs
  262.         stcheck=.tq~pull
  263.     end
  264. end
  265. rate=total%ct
  266. if \sort then outfile~close 
  267. if \quietmode then do
  268.     call qasay ''
  269.     call qasay ''
  270. end
  271. call qasay codel(total)||' bytes transferred in '||codel(ct~format(,2))||' seconds ('||,
  272.     codel(rate)||' bytes/sec.)'
  273. call qasay ''
  274. if \.writeraw | sort then do
  275.     call qasay codel(groups)||' groups received.'
  276.     call qasay ''
  277. end
  278.  
  279. csock~shutdown('both')
  280. csock~close
  281.  
  282. if sort then do
  283.     .con~~charout('Sorting groups... ')~flush
  284.     .local['QSARRAY']=glist~makearray
  285.     call QuickSort 1, glist~items
  286.     .con~~charout('Done'||.crlf)~flush
  287.     .con~~charout('Writing groups... ')~flush
  288.     do i=1 to .qsarray~items
  289.         outfile~lineout(.qsarray[i])
  290.     end
  291.     outfile~close
  292.     .con~~charout('Done'||.crlf)~flush
  293. end    
  294.  
  295. exit 0
  296.  
  297. HALT:
  298.  
  299. if datatype(grow,'NUM')=1 then call SysCurPos grow,70
  300. call qasay ''
  301. call qasay ''
  302. call qasay 'Aborting!'
  303. if csock~objectname='a socket' then do
  304.     csock~shutdown('both')
  305.     csock~close
  306. end
  307. if \quietmode then .sq~queue(.xfs)
  308. exit 3
  309.  
  310. SYNTAX:
  311.  
  312. if nosock=1 then do
  313.     call qasay ''
  314.     call qasay 'Failed to load REXX Socket functions.'
  315.     call qasay ''
  316.     call qasay 'Program cannot continue.'
  317. end
  318. exit 4
  319.  
  320. /* Socket class - work in progress **************************************/
  321. /*                                                                      */
  322. ::class socket
  323.  
  324. ::method init
  325. expose portnum
  326. use arg type,proto
  327. if symbol('type')='LIT' then type='SOCK_STREAM'
  328. if symbol('proto')='LIT' then proto=0
  329. if type~datatype('num')=1 then portnum=type
  330. else portnum=SockSocket('AF_INET',type,proto)
  331. return
  332.  
  333. ::method sock
  334. expose portnum
  335. return portnum
  336.  
  337. ::method connect
  338. use arg servname
  339. signal on HALT
  340. parse var servname hostname ':' host.!port
  341. parse var hostname o1 '.' o2 '.' o3 '.' o4
  342. if o1~datatype='NUM' & o2~datatype='NUM' & o3~datatype='NUM' &,
  343.    o4~datatype='NUM' then do
  344.     if o1~datatype('w')=1 & o2~datatype('w')=1 & o3~datatype('w')=1 &,
  345.        o4~datatype('w')=1 then do
  346.         if (o1>=0 & o1<=255) & (o2>=0 & o2<=255) & (o3>=0 & o3<=255) &,
  347.            (o4>=0 & o4<=255) then do
  348.             stype='IP'
  349.             host.!addr=hostname
  350.         end
  351.     end
  352. end
  353. else stype='NAME'            
  354. if stype='NAME' then call SockGetHostByName hostname,'host.!'
  355. host.!family='AF_INET'
  356. rc=SockConnect(self~sock,'host.!')
  357. return rc
  358. HALT:
  359. return 'HALT'
  360.  
  361. ::method send
  362. use arg sendstring
  363. signal on HALT
  364. rc=SockSend(self~sock,sendstring)
  365. return rc
  366. HALT:
  367. return 'HALT'
  368.  
  369. ::method receive
  370. use arg length
  371. signal on HALT
  372. if length='LENGTH' then length=1024
  373. check=SockRecv(self~sock,'sockdata',length)
  374. return .array~new~~put(sockdata,1)~~put(check,2)
  375. HALT:
  376. return 'HALT'
  377.  
  378. ::method shutdown
  379. use arg how
  380. select
  381.     when how~translate='FROM' then how=0
  382.     when how~translate='TO' then how=1
  383.     otherwise how=2
  384. end
  385. call SockShutDown self~sock,how
  386. return
  387.  
  388. ::method close
  389. call SockClose self~sock
  390. return
  391. /*                                                                      */
  392. /************************************************************************/
  393.  
  394. /* Screen status object *************************************************/
  395. /*                                                                      */
  396. ::class scrstatus
  397.  
  398. ::method init
  399. expose row srow col
  400. return
  401.  
  402. ::method template
  403. expose row srow col
  404. parse value SysCurPos() with row col
  405. srow=row+2
  406. call SysCurPos row,0
  407. .con~~charout('Bytes received: ')~flush
  408. call SysCurPos row,32
  409. .con~~charout('Bytes/second: ')~~charout(.crlf||.crlf)~flush
  410. call SysCurPos srow,0
  411. if \.writeraw then do
  412.     call SysCurPos srow,5
  413.     .con~~charout('Groups received: ')~flush
  414. end     
  415. return
  416.  
  417. ::method begin
  418. expose srow
  419. reply
  420. lastt=0.01
  421. currt=0.01
  422. tdiff=0.01
  423. lastb=-1
  424. lastg=0
  425. currb=0
  426. ctotal=0
  427. gtotal=0
  428. stdone=.false
  429. do until stdone
  430.     do while .sq~items>0
  431.         ctotal=.sq~pull
  432.         if ctotal=.xfs then do
  433.             stdone=.true
  434.             ctotal=lntotal
  435.         end
  436.         else lntotal=ctotal           
  437.     end
  438.     do while .gq~items>0
  439.         gtotal=.gq~pull
  440.     end
  441.     if time('e')<0.1 then iterate
  442.     if ctotal>lastb then self~stored(ctotal)
  443.     currt=time('e')
  444.     if currt=0 then currt=0.01
  445.     currb=ctotal-lastb
  446.     tdiff=currt-lastt
  447.     if tdiff=0 then tdiff=0.01
  448.     crate=currb%(tdiff)
  449.     self~rate(crate)
  450.     lastt=currt
  451.     lastb=ctotal
  452.     if gtotal>lastg then self~groups(gtotal)
  453.     lastg=gtotal
  454.     call SysSleep (.stinterval/100)
  455. end
  456. call SysCurPos srow,0
  457. .tq~queue(.stfs)
  458. return
  459.     
  460. ::method stored
  461. expose row srow col
  462. use arg bstored
  463. call SysCurPos row,16
  464. .con~~charout(codel(bstored)~left(12))~flush
  465. return
  466.  
  467. ::method rate
  468. expose row srow col
  469. use arg rval
  470. call SysCurPos row,46
  471. .con~~charout(codel(rval)~left(12))~flush
  472. return
  473.  
  474. ::method groups
  475. expose row srow col
  476. use arg gval
  477. call SysCurPos srow,23
  478. .con~~charout(codel(gval)~left(8))~flush
  479. return
  480. /*                                                                      */
  481. /************************************************************************/
  482.  
  483. /* Comma deliminator routine ********************************************/
  484. /*                                                                      */
  485. ::routine codel
  486. use arg RNum
  487. if RNum~length<3 then return RNum
  488. if RNum~pos('.')>0 then do
  489.     parse var RNum RNum '.' Fract
  490.     Fract='.'||Fract
  491. end
  492. else Fract=''
  493. RNum=RNum~reverse
  494. FNum=''
  495. do while RNum~length>3
  496.     parse var RNum TriDig +3 RNum
  497.     FNum=FNum||TriDig||','
  498. end
  499. FNum=FNum||RNum
  500. return FNum~reverse||Fract
  501. /*                                                                      */
  502. /************************************************************************/
  503.  
  504. /** QuickSort implementation for Object REXX ****************************/
  505. /*                                                                      */
  506. /*  This QuickSort implementation assumes that the array to be sorted   */
  507. /*  is a local environment object called .qsarray, since passing the    */
  508. /*  array around and keeping track of changes would be a pain.          */
  509. /*                                                                      */
  510. /*  This particular implementation uses randomization to avoid the      */
  511. /*  nasty consequences to trying to sort an array that's already        */
  512. /*  sorted, or has many identical values.                               */
  513. /*                                                                      */
  514. ::routine QuickSort
  515. use arg first,last
  516. if first<last then do
  517.     pivot_index=RandomizedPartition(first,last)
  518.     call QuickSort first,pivot_index-1
  519.     call QuickSort pivot_index+1,last
  520. end
  521.  
  522. ::routine RandomizedPartition
  523. use arg first,last
  524. i=random(first,last)
  525. temp=.qsarray[first]
  526. .qsarray[first]=.qsarray[i]
  527. .qsarray[i]=temp
  528. pivot_index=Partition(first,last)
  529. return pivot_index
  530.  
  531. ::routine Partition
  532. use arg first,last
  533. pivot=.qsarray[first]
  534. lastS1=first
  535. first_unknown=first+1
  536. do while first_unknown<=last
  537.     if .qsarray[first_unknown]<pivot then do
  538.         lastS1=lastS1+1
  539.         temp=.qsarray[first_unknown]
  540.         .qsarray[first_unknown]=.qsarray[lastS1]
  541.         .qsarray[lastS1]=temp
  542.     end
  543.     first_unknown=first_unknown+1
  544. end
  545. temp=.qsarray[first]
  546. .qsarray[first]=.qsarray[lastS1]
  547. .qsarray[lastS1]=temp
  548. pivot_index=lastS1
  549. return pivot_index
  550. /*                                                                      */
  551. /************************************************************************/
  552.  
  553. /* Program usage ********************************************************/
  554. /*                                                                      */
  555. ::routine usage
  556. call qasay ''
  557. call qasay 'News group list fetcher v'||.rxiversion
  558. call qasay ''
  559. call qasay 'Usage: ngfetch.cmd <host[:port]> [options]'
  560. call qasay ''
  561. call qasay 'host - hostname of news server'
  562. call qasay 'port - port of news server (default is 119)'
  563. call qasay ''
  564. call qasay 'Options:'
  565. call qasay ''
  566. call qasay '/w - write all received info without parsing'
  567. call qasay '/q - operate in quiet mode'
  568. call qasay '/qq - operate in super quiet mode'
  569. call qasay '/s - sort the groups (rather slow)'
  570. call qasay '/b# - transfer block size (512-65535); default=10240'
  571. if \win95 then call qasay '/t# - status timing interval in 1/100th seconds (1-100); default=50'
  572. else call qasay '/t# - status timing interval in seconds (1-5); default=1' 
  573. call qasay '/m# - maximum retries on error (0-9999999999); default=1000'
  574. return
  575. /*                                                                      */
  576. /************************************************************************/
  577.  
  578. /* Enhanced stream object ***********************************************/
  579. /*                                                                      */
  580. ::class estream subclass stream
  581.  
  582. ::method charout
  583. use arg string,start
  584. if .superquiet then return 0
  585. else do
  586.     if symbol('START')='VAR' then return self~charout:super(string,start)
  587.     else return self~charout:super(string)
  588. end
  589.  
  590. ::method lineout
  591. use arg string,line
  592. if .superquiet then return 0
  593. else do
  594.     if symbol('LINE')='VAR' then return self~lineout:super(string,line)
  595.     else return self~lineout:super(string)
  596. end
  597. /*                                                                      */
  598. /************************************************************************/
  599.  
  600. /* Quiet aware say ******************************************************/
  601. /*                                                                      */
  602. ::routine qasay
  603. use arg string
  604. if .superquiet then return
  605. else say string
  606. /*                                                                      */
  607. /************************************************************************/
  608.