Xref: utzoo news.software.b:1507 news.software.nntp:54
Path: utzoo!attcan!uunet!husc6!ukma!david
From: david@ms.uky.edu (David Herron -- One of the vertebrae)
Newsgroups: news.software.b,news.software.nntp
Subject: SPOOLNEWS & nntp
Message-ID: <10080@e.ms.uky.edu>
Date: 27 Jul 88 20:34:05 GMT
Organization: U of Ky, Math. Sciences, Lexington KY
Lines: 405

It's amazing what you'll do when you're bored.  One afternoon I moved the
SPOOLNEWS code from inews.c into nntpd to see what sort of effect it would
have on the system load, it "seems" to help quite a bit.  [it's rather hard
to test you see]

What we had noticed here was that we'd often have 3 or 4 nntpd's running
at a time sending us news.  When nntp receives an article it starts up
an rnews to handle the reception.  While that's nice and clean and elegant,
that rnews in reality executes very little code, which raises the overhead
percentage -- that is, it always costs "x" amount of resources to start up
a process and the whole process execution costs "y" and since very little
is done (when you have SPOOLNEWS defined) to insert the article into the
system "y" is very close to "x".

Moving the SPOOLNEWS stuff into nntpd avoids "x" -- or rather having many
"x"'s going on all at once.  Because eventually, when the "rnews -U" is
run, all those "x"'s will happen anyway but now they'll be happening
serially rather than in parallel.  (parallel would be ok if this were
running over on our Sequent, but it's not over there yet ... instead
we're on a uVaxII with 13 megs of memory.)

The patch includes a context diff for server/spawn.c and also a line to
add to common/conf.h, and is derived from unadulterated nntp v1.5 sources.


As an added bonus you get to see one of the silliest programs I've ever
written.  I'm almost embarassed to post this, especially since I'm going
to be looking for a job later this year :-).  Anyway.  The program is
probably the MOST stupid way of unbatching a batch in existance.  It
looks for "#! rnews" lines and assumes that it's the beginning of a new
article.  It also strips blanks out of otherwise empty lines, strips away
empty lines before headers and otherwise cleans up the batch file.  All
this so that I can have a news feed from an IBM mainframe at Penn State.

The reason that it's being posted is that it also has had the SPOOLNEWS
code put into it.



#! /bin/sh
: This is a shell archive, meaning:
: 1. Remove everything above the '#! /bin/sh' line.
: 2. Save the resulting text in a file.
: 3. Execute the file with /bin/sh '(not csh)' to create the files:
:	'conf.DIFF'
:	'spawn.DIFF'
:	'split.batch.c'
: This archive created: 'Wed Jul 27 16:03:27 1988'
: By:	'David Herron -- One of the vertebrae ()'
export PATH; PATH=/bin:$PATH
echo shar: extracting "'conf.DIFF'" '(61 characters)'
if test -f 'conf.DIFF'
then
	echo shar: will not over-write existing file "'conf.DIFF'"
else
sed 's/^X//'  >'conf.DIFF' <<'SHAR_EOF'
X#define SPOOLNEWS	/* Emulate the SPOOLNEWS code from news */
SHAR_EOF
if test 61 -ne "`wc -c < 'conf.DIFF'`"
then
	echo shar: error transmitting "'conf.DIFF'" '(should have been 61 characters)'
fi
fi # end of overwriting check
echo shar: extracting "'spawn.DIFF'" '(3343 characters)'
if test -f 'spawn.DIFF'
then
	echo shar: will not over-write existing file "'spawn.DIFF'"
else
sed 's/^X//'  >'spawn.DIFF' <<'SHAR_EOF'
X*** spawn.c.orif	Tue May 31 10:39:10 1988
X--- spawn.c	Thu Jul 14 22:02:37 1988
X***************
X*** 3,9 ****
X  #endif
X  
X  #include "common.h"
X! 
X  #include <signal.h>
X  
X  #ifdef XFER_TIMEOUT
X--- 3,10 ----
X  #endif
X  
X  #include "common.h"
X! #include <stdio.h>
X! #include <time.h>
X  #include <signal.h>
X  
X  #ifdef XFER_TIMEOUT
X***************
X*** 11,16 ****
X--- 12,33 ----
X  static int	old_xfer_lines;
X  #endif
X  
X+ 
X+ char *
X+ errmsg(code)
X+ int code;
X+ {
X+ 	extern int sys_nerr;
X+ 	extern char *sys_errlist[];
X+ 	static char ebuf[6+5+1];
X+ 
X+ 	if (code > sys_nerr) {
X+ 		(void) sprintf(ebuf, "Error %d", code);
X+ 		return ebuf;
X+ 	} else
X+ 		return sys_errlist[code];
X+ }
X+ 
X  static char	tempfile[256];
X  
X  /*
X***************
X*** 59,66 ****
X--- 76,88 ----
X  #endif not USG
X  	register FILE	*fp;
X  
X+ #ifdef SPOOLNEWS
X+ 	(void) sprintf(tempfile, "%s/.spXXXXXX", spooldir);
X+ 	(void) mktemp(tempfile);
X+ #else
X  	(void) strcpy(tempfile, "/tmp/rpostXXXXXX");
X  	(void) mktemp(tempfile);
X+ #endif /* SPOOLNEWS */
X  
X  	fp = fopen(tempfile, "w");
X  	if (fp == NULL) {
X***************
X*** 122,127 ****
X--- 144,209 ----
X  	(void) chown(tempfile, uid_poster, gid_poster);
X  #endif
X  
X+ 	/*
X+ 	 * Ok, now we have the article in "tempfile".  We
X+ 	 * should be able to fork off, close fd's 0 to 31 (or
X+ 	 * whatever), open "tempfile" for input, thus making
X+ 	 * it stdin, and then execl the inews.  We think.
X+ 	 */
X+ 
X+ #ifdef SPOOLNEWS
X+ 
X+ 	{
X+ 	register struct tm *tp;
X+ 	time_t t;
X+ #define BUFLEN 512
X+ 	char buf[BUFLEN];
X+ 	extern struct tm *gmtime();
X+ 	int randno = getpid();
X+ 
X+ 	(void) time(&t);
X+ 	tp = gmtime(&t);
X+ retry:
X+ 	/* This file name "has to" be unique  (right?) */
X+ 	(void) sprintf(buf, "%s/.rnews/%02d%02d%02d%02d%02d%x",
X+ 		spooldir,
X+ 		tp->tm_year, tp->tm_mon+1, tp->tm_mday,
X+ 		tp->tm_hour, tp->tm_min, randno);
X+ 
X+ 	if (link(tempfile, buf) < 0) {
X+ 		char dbuf[BUFLEN];
X+ 		if (errno == EEXIST) {
X+ 			randno++;
X+ 			goto retry;
X+ 		}
X+ 		sprintf(dbuf, "%s/.rnews", spooldir);
X+ #define N_UMASK 022 /* from localize.ukma */
X+ 		if (mkdir(dbuf, 0777&~N_UMASK) < 0) {
X+ 			sprintf(errbuf, "%s dospool: Cannot mkdir %s: %s",
X+ 				hostname, dbuf, errmsg(errno));
X+ # ifdef LOG
X+ 			syslog(LOG_ERR, "%s", errbuf);
X+ 			/* xerror("Cannot mkdir %s: %s", dbuf, errmsg(errno)); */
X+ #endif
X+ 			return(-1);
X+ 		}
X+ 		if (link(tempfile, buf) < 0) {
X+ 			sprintf(errbuf,  "%s dospool: Cannot link(%s,%s): %s",
X+ 				hostname, tempfile, buf, errmsg(errno));
X+ # ifdef LOG
X+ 			syslog(LOG_ERR, "%s", errbuf);
X+ 			/* xerror("Cannot link(%s,%s): %s", tempfile, buf,
X+ 			 *	errmsg(errno)); */
X+ #endif
X+ 			return(-1);
X+ 		}
X+ 	}
X+ 	(void) unlink(tempfile);
X+ 	return(1);
X+ 	}
X+ 
X+ #else /* SPOOLNEWS */
X+ 
X  	/* Set up a pipe so we can see errors from rnews */
X  
X  	if (pipe(fds) < 0) {
X***************
X*** 132,144 ****
X  		return (-1);
X  	}
X  
X- 	/*
X- 	 * Ok, now we have the article in "tempfile".  We
X- 	 * should be able to fork off, close fd's 0 to 31 (or
X- 	 * whatever), open "tempfile" for input, thus making
X- 	 * it stdin, and then execl the inews.  We think.
X- 	 */
X- 
X  	pid = vfork();
X  	if (pid == 0) {		/* We're in child */
X  #ifdef POSTER
X--- 214,219 ----
X***************
X*** 225,230 ****
X--- 300,306 ----
X  			
X  		return (exit_status ? -1 : 1);
X  	}
X+ #endif /* SPOOLNEWS */
X  }
X  
X  #ifdef XFER_TIMEOUT
SHAR_EOF
if test 3343 -ne "`wc -c < 'spawn.DIFF'`"
then
	echo shar: error transmitting "'spawn.DIFF'" '(should have been 3343 characters)'
fi
fi # end of overwriting check
echo shar: extracting "'split.batch.c'" '(3105 characters)'
if test -f 'split.batch.c'
then
	echo shar: will not over-write existing file "'split.batch.c'"
else
sed 's/^X//'  >'split.batch.c' <<'SHAR_EOF'
X
X#include <stdio.h>
X#include <time.h>
X#include <sys/types.h>
X
X#define SPOOLNEWS
X
X#ifdef SPOOLNEWS
X
X#define spooldir "/net/spool.news"
X
Xextern struct tm *gmtime();
Xstruct tm *curtime2;
Xtime_t curtime1;
X
Xextern char *mktemp();
Xchar tempfile[256], outfile[256];
Xint tfd;
X
X#endif
X
Xmain(argc, argv)
Xint argc;
Xchar **argv;
X{
X	FILE *outf, *fopen();
X	register int bol, opn, c, boa;
X	int spccount;
X	char tmp[100], *command;
X	int len, randno, getpid();
X
X	/* puts("beginning");
X	 * fflush(stdout);
X	 * puts(argv[0]);
X	 * fflush(stdout);
X	 * puts(argv[1]);
X	 * fflush(stdout); */
X	if (argc != 2) {
X		fprintf(stderr, "Usage: split.batch command <file\n");
X		exit(1);
X	}
X	command = argv[1];
X	randno = getpid();
X	bol = (1==1);
X	boa = (1==1);
X	opn = (1==0);
X	while((c=getchar()) != EOF) {
X		if (bol && c == ' ') {
X			spccount = 1;
X			while ((c=getchar()) == ' ')
X				spccount++;
X			if (c == EOF) {
X				for (; spccount >= 1; spccount--)
X					fputc(' ', outf);
X				goto out;
X			}
X			if (c == '\n' && boa) {
X				bol = (1==1);
X				continue;
X			}
X			if (c != '\n')
X				for (; spccount >= 1; spccount--)
X					fputc(' ', outf);
X			/*
X			 * This falls into the part at the end of the while
X			 * loop.
X			 */
X		}
X		else if (boa && bol && c == '\n')
X			continue;
X		else if (bol && c == '#') {
X			bol = (1==0);
X			c = getchar();
X			if (c != '!')
X				fprintf(outf, "#%c", c);
X			else if ((c=getchar()) != ' ') 
X				fprintf(outf, "#!%c", c);
X			else if ((c=getchar()) != 'r')
X				fprintf(outf, "#! %c", c);
X			else if ((c=getchar()) != 'n')
X				fprintf(outf, "#! r%c", c);
X			else if ((c=getchar()) != 'e')
X				fprintf(outf, "#! rn%c", c);
X			else if ((c=getchar()) != 'w')
X				fprintf(outf, "#! rne%c", c);
X			else if ((c=getchar()) != 's')
X				fprintf(outf, "#! rnew%c", c);
X			else if ((c=getchar()) != ' ')
X				fprintf(outf, "#! rnews%c", c);
X			else {
X				gets(tmp);
X				/* puts(tmp); */
X				if (opn) {
X#ifndef SPOOLNEWS
X					pclose(outf);
X#else
X					fclose(outf);
X#endif
X					opn = (1==0);
X				}
X				/*
X				 * len = atoi(tmp);
X				 * sprintf(tmp, "%s.%d.%d", base, pid++, len);
X				 * printf("%s\n", tmp);
X				 */
X#ifndef SPOOLNEWS
X				outf = popen(command, "w");
X				if (outf == NULL) {
X					fprintf(stderr, 
X				    "Gosh darn! Can't open |%s!\n", command);
X					exit(1);
X				}
X#else
X				do {
X					(void) sprintf(tempfile, "%s/.spXXXXXX", spooldir);
X					(void) mktemp(tempfile);
X				} while ((tfd = creat(tempfile, 0644) < -1));
X				(void) close(tfd);
X				(void) time(&curtime1);
X				curtime2 = gmtime(&curtime1);
Xretry:
X			(void) sprintf(outfile, "%s/.rnews/%02d%02d%02d%02d%02d%x",
X					spooldir,
X					curtime2->tm_year, curtime2->tm_mon+1,
X					curtime2->tm_mday, curtime2->tm_hour,
X					curtime2->tm_min, randno);
X				if (link(tempfile, outfile) < 0) {
X					randno++;
X					goto retry;
X				}
X				unlink(tempfile);
X				outf = fopen(outfile, "w");
X#endif
X				opn = (1==1);
X				boa = (1==1);
X				bol = (1==1);
X				continue;
X			}
X		}
X		boa = (1==0);
X		if (opn)
X			fputc(c, outf);
X		/* putchar(c); */
X		if (c == '\n')
X			bol = (1==1);
X		else
X			bol = (1==0);
X	}
Xout:
X	if (opn) {
X		fflush(outf);
X		fclose(outf);
X	}
X	exit(0);
X}
SHAR_EOF
if test 3105 -ne "`wc -c < 'split.batch.c'`"
then
	echo shar: error transmitting "'split.batch.c'" '(should have been 3105 characters)'
fi
fi # end of overwriting check
:	End of shell archive
exit 0
-- 
<---- David Herron -- The E-Mail guy                         <david@ms.uky.edu>
<---- ska: David le casse\*'      {rutgers,uunet}!ukma!david, david@UKMA.BITNET
<----
<---- Looking forward to a particularly blatant, talkative and period bikini ...