Relay-Version: version B 2.10 5/3/83; site utzoo.UUCP Path: utzoo!mnetor!uunet!nuchat!steve From: steve@nuchat.UUCP (Steve Nuchia) Newsgroups: news.admin Subject: Re: --- Multiple UUXQTs causing thrashing --- Message-ID: <420@nuchat.UUCP> Date: Mon, 26-Oct-87 19:31:17 EST Article-I.D.: nuchat.420 Posted: Mon Oct 26 19:31:17 1987 Date-Received: Wed, 28-Oct-87 06:13:19 EST References: <173@zap.UUCP> <768@tolerant.UUCP> <7539@e.ms.uky.edu> Organization: Public Access - Houston, Tx Lines: 357 Keywords: Unix, compress, LCK.XQT, thrashing Summary: here's my home-brew solution On my system, one of those silly intel-based boxes, the load from a few compress -d's and rnews's would bring the thing to its knees. Also, when I ran out of disk space rnews would happily dump stuff into the bit bucket. To solve thes problems I hacked together the following little program. I know there is a SPOOLNEWS option, but I haven't tried it, since I would still have to wrap a disk-space-checker around it. This program is run as a daemon. It nices itself and always sleeps at least a minute between bouts of work to let things rest. I have uuxqt executing a shell script that copies the incoming news to a hold directory, where this daemon finds it and tries to despool it. I also hacked sendbatch to send only one batch per invokation, and the "sendsome" script just calls sendbatch for each neighbor with the proper magic options. The daemon will run sendsome in any pass that can't do any de-spooling, either because there are no new batches or because of space problems. It should fire off an expire when it runs out of room, but I've got bigger plans and until then I'll just do it by hand (killing the daemon and restarting it afterwards). It suspends outgoing batching if there is not enough room (a different threshold) on the uucp spool device. (my systems has a separate partition for the news spool). I also started calling uunet recently, and wanted to implement a "delayed sendme" feature. I noticed that the "ihave" processing in inews was incredibly slow, and cooked up a solution, also included below. Basically I hacked control.c to stick incoming ihave messages into a hold file, named after the machine that sent the ihave. The "sendme" program, when run, loads the entire history database into memory in a reasonably efficient form and then reads the ihave spool file. It rejects articles it has seen in the history file or already in the spool and writes the novel ones to either "order" or "toonew". One sets the "toonew" threshold in the hope that a cheaper neighbor will supply the article "soon" so you won't have to order it from the expensive neighbor. You then replace the old spool file with "toonew", limiting its growth. ----this is the "rnews" that you let UUXQT find---- umask 0222 cat - >> /usr/spool/news/incoming/in.$$ ls -l /usr/spool/news/incoming/in.$$ >> /usr/lib/news/log ----- ----this is the news de-spooler daemon----- /* * newsd.c - daemon for unbundling news and doing other * essential stuff, like running expire. Obviates certain * cron processing and allows the slow components of news * to run niced without playing uucp games. * * Steve Nuchia * 27 Sept 1987 */ #include #include #include #include #include #define INCOMING "/usr/spool/news/incoming" #define RNEWS "/usr/lbin/rnews" #define IHAVE "/files/news/.ihave" #define OUTGOING "/usr/spool/batch" #define SENDSOME "/usr/lib/news/sendsome" #define NEWSDIR "/files/news" #define SPOOLDIR "/usr/spool" main() { int incoming; struct direct art; char aname[DIRSIZ+1], fullname[80], best[80], command[160]; long btime; int i; struct stat sbuf; struct ustat usbuf; nice ( 20 ); if ( (incoming = open ( INCOMING, 0 )) < 0 ) perror ( INCOMING ); while ( 1 ) { /* see how the space situation looks */ stat ( NEWSDIR, &sbuf ); ustat ( sbuf.st_dev, &usbuf ); if ( usbuf.f_tfree > 1000 && usbuf.f_tinode > 500 ) { /* look around in INCOMING */ sleep ( 60 ); lseek ( incoming, 2L * sizeof(struct direct), 0 ); best[0] = 0; while ( read ( incoming, &art, sizeof(struct direct) ) > 0 ) { if ( ! art.d_ino ) continue; for ( i = 0; i < DIRSIZ; i++ ) aname[i] = art.d_name[i]; aname[i] = 0; sprintf ( fullname, "%s/%s", INCOMING, aname ); stat ( fullname, &sbuf ); if ( ! best[0] || btime > sbuf.st_mtime ) { btime = sbuf.st_mtime; strcpy ( best, fullname ); } } /* if there is anything, take care of oldest */ if ( best[0] ) { sprintf ( command, "%s < %s", RNEWS, best ); if ( ! system ( command ) ) unlink ( best ); continue; } } else { printf ( "space problem in NEWSDIR\n" ); fflush ( stdout ); sleep ( 600 ); } /* otherwise we are free to do housekeeping */ stat ( SPOOLDIR, &sbuf ); ustat ( sbuf.st_dev, &usbuf ); if ( usbuf.f_tfree > 1000 && usbuf.f_tinode > 500 ) { /* for now, just fire sendbatch.all and sleep */ system ( SENDSOME ); } else { printf ( "space problem in SPOOLDIR\n" ); fflush ( stdout ); sleep ( 600 ); } sleep ( 60 ); } } ---- ----this is the ihave line processor---- #define reg register #include main ( argc, argv ) char *argv[]; { int fn, i; char line[256]; char art[128]; FILE *from, *hf, *toonew, *order; int am, ad; int m, d; d = atoi ( argv[2] ); m = atoi ( argv[3] ); from = fopen ( argv[1], "r" ); toonew = fopen ( "toonew", "w" ); order = fopen ( "order", "w" ); fprintf ( order, "Newsgroups: to.%s.ctl\n", argv[1] ); fprintf ( order, "Subject: sendme nuchat\n" ); fprintf ( order, "Control: sendme nuchat\n\n" ); for ( fn = 0; fn < 10; fn++ ) { sprintf ( line, "/usr/lib/news/history.d/%d", fn ); hf = fopen ( line, "r" ); while ( fgets ( line, 256, hf ) ) { if ( sscanf ( line, "<%[^>\n ]>", art ) == 1 ) save ( art ); } fclose ( hf ); } while ( fgets ( line, 256, from ) ) { if ( sscanf ( line, "<%[^>\n ]> %d/%d", art, &am, &ad ) == 3 ) { if ( save ( art ) ) { if ( am < m || ad <= d ) fprintf ( order, "<%s>\n", art ); else fprintf ( toonew, "%s", line ); } } else { fprintf ( stderr, "can't grok '%s'\n", line ); } } report(); } typedef struct _saved saved; struct _saved { saved *sp; char ss[4]; }; extern char *malloc(), *ssave(); char *my_alloc ( n ) { static char *chunk; static int size; if ( size < n ) chunk = malloc ( size = 32000 ); size -= n; chunk += n; return ( chunk - n ); } #define NHASH (16000/sizeof(char *)) typedef struct _smsg smsg; struct _smsg { smsg *nx; char *seq; char *site; }; smsg *sites[NHASH]; save ( name ) char *name; { char buf[128]; reg smsg *p; reg int i = 0; char *site, *seq; while ( *name && *name != '@' ) buf[i++] = *name++; buf[i] = 0; if ( *name ) name++; site = ssave ( name ); seq = ssave ( buf ); i = ((long)site >> 16) ^ ((long)seq >> 16) ^ (long)site ^ (long)seq; i &= 0x7fff; i %= NHASH; for ( p = sites[i]; p; p = p->nx ) if ( p->seq == seq && p->site == site ) return ( 0 ); if ( !(p = (smsg *) my_alloc ( sizeof(smsg) )) ) abort(); p->nx = sites[i]; p->seq = seq; p->site = site; sites[i] = p; return ( 1 ); } static saved *savedp[NHASH]; report() { int i, count; reg saved *p; reg long strs = 0, total = 0; for ( i = 0; i < NHASH; i++ ) { count = 0; for ( p = savedp[i]; p; p = p->sp ) { strs++; count++; } total += count*((long)count/2); } printf ( "strs = %ld, NHASH = %d, total = %ld\n", strs, NHASH, total ); } char *ssave ( s ) reg char *s; { reg saved *t; int h; reg char *hs; if ( !s || !*s ) return ( (char *) 0 ); for ( hs = s, h = 0; *hs; h = ((h << 1) ^ (h >> 3) ^ *hs++) & 0x7fff ); h %= NHASH; for ( t = savedp[h]; t; t = t->sp ) if ( t->ss[0] == *s && !strcmp ( t->ss, s ) ) return ( t->ss ); t = (saved *) my_alloc ( sizeof(saved) + strlen(s) - 3 ); if ( !t ) abort(); t->sp = savedp[h]; strcpy ( t->ss, s ); savedp[h] = t; return ( t->ss ); } ---- ----this is the replacement body for ihave in control.c---- #define SPOOLIHAVE "/usr/spool/news/ihave/%s" #ifdef SPOOLIHAVE c_ihave(argc, argv) register char ** argv; { long t; char tstamp[40], lineout[200]; int i, ihv_file; struct tm *tm; (void) time(&t); tm = localtime(&t); #ifdef USG sprintf(tstamp,"%2.2d/%2.2d/%d %2.2d:%2.2d", #else /* !USG */ sprintf(tstamp,"%02d/%02d/%d %02d:%02d\tcancelled", #endif /* !USG */ tm->tm_mon+1, tm->tm_mday, tm->tm_year, tm->tm_hour, tm->tm_min); if (argc < 2) error("ihave: Too few arguments."); if (strncmp(FULLSYSNAME, argv[argc - 1], SNLN) == 0) return 0; sprintf ( lineout, SPOOLIHAVE, argv[argc - 1] ); if ( (ihv_file = open ( lineout, 1 )) < 0 ) ihv_file = creat ( lineout, 0600 ); lseek ( ihv_file, 0L, 2 ); if (argc > 2) { for (i = 1; i < (argc - 1); ++i) { sprintf ( lineout, "%s %s\n", argv[i], tstamp ); write ( ihv_file, lineout, strlen(lineout) ); } } else { char myid[256]; while ( fgets(myid, sizeof myid, infp) ) { for ( i = 0; myid[i] && myid[i] != ' ' && myid[i] != '\n'; i++ ); myid[i] = '\0'; sprintf ( lineout, "%s %s\n", myid, tstamp ); write ( ihv_file, lineout, strlen(lineout) ); } } close ( ihv_file ); return 0; } #else #endif ---- -- Steve Nuchia | [...] but the machine would probably be allowed no mercy. uunet!nuchat!steve | In other words then, if a machine is expected to be (713) 334 6720 | infallible, it cannot be intelligent. - Alan Turing, 1947