token.c: (Matching .h file)


/* token.c - Routines for basic token scanning.
 * © Copyright 1999 by John Halleck
 * All rights reserved.
 */
/* Version of September 2nd, 1999 */

#include <ctype.h>
 /* Character test routines */

#include <stdio.h>
 /* value of EOF on this system */

#include "errors.h"
 /* We return package standard error codes. */
 /* All of the routines can return:
  * ERRnotfound - We didn't find an appropriate token.
  * ERRendoffile - We incountered end of file before a token.
  * ERRsize  - Buffer must have non-negative size.
  *            Zero sized buffer just do a scan.
  * ERRnil   - Buffer nil.
  * ERRendoffile
  *
  * The routines scan to the end of the token.
  * They fill to the end of the buffer.
  * If the token is larger than the buffer, they still
  * continue to end of token, but then return
  * ERRoverflow 
  */

#include "token.h"
 /* What we tell the world */

/* Usually these routines take a buffer for the result,
 * and an integer size for the buffer.
 * They also give a starting location.
 * They fill the buffer, and return the number of characters 
 * used.
 */

/* Have we been through the initialization routines? */
static int initialized = 0;

/* Current character */
int thischaracter = EOF;
/* the character after that */
int nextcharacter      = EOF;
int validnextcharacter = 0; /* Have we prefetched the next character? */

/* What do we call for characters? */
static error (*thegetroutine)(int *place) = 0;


/* Have we overflow the buffer? */
static int overflow       = 0;
static char *givenbuffer  = 0;  /* buffer we are dealing with */
static int  sizeremaining = 0;  /* How much is left */
static int *userssize     = 0;  /* Where does the user want the size put? */

/* Housekeeping. */

static error stuffchar (char given) {
  if (overflow) return NoError;
  if (sizeremaining) {
     *givenbuffer++ = thischaracter; sizeremaining--;
     (*userssize)++;
  }
  else overflow = 1;
  return NoError;
}

/* -------------------------------------------------------------------------- */

/* Let's get all our error checking and initialization together. */
static error verify (int bufsize, char *buffer, int *size){
 if (!initialized)          return ERRuninitialized;
 if (bufsize<1)             return ERRsize;
 if (!buffer)               return ERRnil;
 if (!size)                 return ERRnil;
 if (thischaracter == EOF)  return ERRendoffile;

 givenbuffer = buffer;
 *buffer = 0; /* Make traditional C programmers happy */
              /* (I.E. make a bailout look like an empty string) */
 sizeremaining = bufsize;
 userssize  = size;
 *userssize = 0;
 overflow = 0; /* Not yet, anyway */

 return NoError;
}


static error done () {
  if (overflow)    return ERRoverflow; 
  if (!userssize)  return ERRcorrupt;
  if (!*userssize) return ERRnotfound; /* oops */
  if (sizeremaining) *givenbuffer = 0; /* Just to keep  C folk happy.
                                        * IF we have room, give it a terminating
                                        * zero byte.  Since we were given the
                                        * entrie buffer, this is safe.
                                        * But it is not part of the count,
                                        * since that is the actual size of
                                        * the token
                                        */
  return NoError;
}

/* -------------------------------------------------------------------------- */

/* Initialize package */
error tokinitialize(error (*getroutine)(int *place)) {
  error problem;
  if (initialized)  return ERRreinitialized;
  if (!getroutine)  return ERRnil;
  thegetroutine = getroutine;
  thischaracter = EOF;
  nextcharacter = EOF;
  if ((problem = (* getroutine)(&thischaracter))) return problem;
  initialized = 1;
  return NoError;
}

/* Close and cleanup package */
error tokfinalize () {
  if (!initialized) return ERRuninitialized;
  initialized = 0;
  thegetroutine = 0;
  thischaracter = EOF;
  return NoError;
}

/* -------------------------------------------------------------------------- */
/*              Scanning routines                                             *//* -------------------------------------------------------------------------- */

error nextchar() {
 if (!initialized)         return ERRuninitialized;
 if (thischaracter == EOF) return ERRendoffile;
 if (validnextcharacter) {
    thischaracter = nextcharacter;
    nextcharacter = EOF;
    validnextcharacter = 0;
 } else if (thischaracter != EOF) return (*thegetroutine)(&thischaracter);
 return NoError;
}

error prefetch () {
  error problem;
  if (!initialized)         return ERRuninitialized;
  if (thischaracter == EOF) return ERRendoffile;
  nextcharacter = EOF;
  validnextcharacter = 0;
  if ((problem =  (*thegetroutine)(&nextcharacter))) return problem;
  validnextcharacter = 1;
  return NoError;
}

error skipwhite() {
 error problem;
 if (!initialized)             return ERRuninitialized;
 if (thischaracter == EOF)     return ERRendoffile;
 if (!isspace(thischaracter))  return ERRnotfound;
 while (thischaracter != EOF && isspace(thischaracter))
         if ((problem = nextchar())) return problem;
 return NoError;
}

error skipnonwhite() {
 error problem;
 if (!initialized)           return ERRuninitialized;
 if (thischaracter == EOF)   return ERRendoffile;
 if (isspace(thischaracter)) return ERRnotfound;
 while (thischaracter != EOF && !isspace(thischaracter))
        if ((problem = nextchar())) return problem;
 return NoError;
}

error skipeol() {
 error problem;
 if (!initialized)          return ERRuninitialized;
 if (thischaracter == EOF)  return ERRendoffile;
 while (thischaracter != EOF && thischaracter != '\n')
         if ((problem = nextchar())) return problem;
 if (thischaracter == EOF)  return ERRendoffile;
 if (thischaracter == '\n') return nextchar();
 return NoError;
}

error skiptoeol() {
 error problem;
 if (!initialized)          return ERRuninitialized;
 if (thischaracter == EOF)  return ERRendoffile;
 while (thischaracter != EOF && thischaracter != '\n')
         if ((problem = nextchar())) return problem;
 /* Since this is a skip to, and not through, we'll accept an end
  * of file as ending the line.
  */
 return NoError;
}

error skipnoneolwhite() {
 error problem;
 if (!initialized)            return ERRuninitialized;
 if (thischaracter == EOF)    return ERRendoffile;
 if (!isspace(thischaracter)) return ERRnotfound;
 while (thischaracter != EOF && isspace(thischaracter)
                             && thischaracter != '\n')
        if ((problem = nextchar())) return problem;
 return NoError;
}

/* Is the current character in the list? */
static error isone (char *list) {
  if (thischaracter == EOF)                   return ERRendoffile;
  if (!list)                                  return ERRnil;
  if (!*list)                                 return ERRempty;
  while (*list) if (*list++ == thischaracter) return NoError;
                                              return ERRnotfound;
}

error skipto (char *list) {
  error problem = NoError;
  int found;
  if (!initialized)         return ERRuninitialized;
  if (!list)                return ERRnil;
  if (!*list)               return ERRempty;
  if (thischaracter == EOF) return ERRendoffile;

  found = 0;
  while (!found && thischaracter != EOF) {
     if (!( problem = isone (list))) return NoError;
     if (problem != ERRnotfound)     return problem;
     if ((problem = nextchar()))     return problem;
  }
 return ERRnotfound;
}

/* -------------------------------------------------------------------------- */
/* ========================================================================== */
/*                  Actual token scans                                        */
/* -------------------------------------------------------------------------- */


static error rawnonwhitetok () {
 error problem;

 while (thischaracter != EOF && !isspace(thischaracter)) {
   if ((problem = stuffchar(thischaracter))) return problem; 
   if ((problem = nextchar()))               return problem;
 } 
 return NoError;
}

error nonwhitetok (int bufsize, char *buffer, int *size) {
  error problem;
  if ((problem = verify(bufsize, buffer, size)))         return problem;
  if (isspace(thischaracter))                            return ERRnotfound;
  if ((problem = rawnonwhitetok()))                      return problem;
  return done();
}



static error rawpuncttok () {
 error problem;

 if ( thischaracter == EOF
   || !isprint(thischaracter)
   || isalnum(thischaracter)
   || isspace(thischaracter)
    ) return ERRnotfound;
 else {
    if ((problem = stuffchar(thischaracter))) return problem;
    if ((problem = nextchar()))               return problem;
    return NoError;
 }
}

error puncttok    (int bufsize, char *buffer, int *size) {
 error problem;
 if ((problem = verify(bufsize, buffer, size)))         return problem;
 if ((problem = rawpuncttok()))                         return problem;
 return done();
}


static error rawinttok () {
 error problem;
 if (!isdigit(thischaracter))                return ERRnotfound;
 while (thischaracter != EOF && isdigit(thischaracter)) {
   if ((problem = stuffchar(thischaracter))) return problem;
   if ((problem = nextchar()))               return problem;
 }
 return NoError;
}

error inttok      (int bufsize, char *buffer, int *size) {
  error problem;
  if ((problem = verify(bufsize, buffer, size)))         return problem;
  if ((problem = rawinttok()))                           return problem;
  return done();
}


static error rawsinttok () {
  error problem;
  if (thischaracter == EOF)                                 return ERRendoffile;
  if (isdigit(thischaracter))                               return rawinttok();
  if ((problem = prefetch()))                               return problem;
  if (nextcharacter == EOF)                                 return ERRnotfound;
  if (!isdigit(nextcharacter))                              return ERRnotfound;
  else {
     if (thischaracter == '-') {
      if ((problem = stuffchar(thischaracter)))             return problem;
      if ((problem = nextchar()))                           return problem;
     } else if (thischaracter == '+') {
      if ((problem = nextchar()))                           return problem;
     } else                                                 return ERRnotfound;
  }
  if ((problem = rawinttok()))                              return problem;
  return done();
}

error sinttok  (int bufsize, char *buffer, int *size) {
  error problem;
  if ((problem = verify(bufsize, buffer, size)))            return problem;
  return rawsinttok();
}

error rawalphatok () {
 error problem;
 if (!isalpha(thischaracter))                return ERRnotfound;
 while (thischaracter != EOF && isalpha(thischaracter)) {
   if ((problem = stuffchar(thischaracter))) return problem;
   if ((problem = nextchar()))               return problem;
 }
 return NoError;
}

error alphatok      (int bufsize, char *buffer, int *size) {
  error problem;
  if ((problem = verify(bufsize, buffer, size)))         return problem;
  if ((problem = rawalphatok()))                         return problem;
  return done();
}


error rawalphanumtok () {
 error problem;
 if (!isalnum(thischaracter))                return ERRnotfound;
 while (thischaracter != EOF && isalnum(thischaracter)) {
   if ((problem = stuffchar(thischaracter))) return problem;
   if ((problem = nextchar()))               return problem;
 }
 return NoError;
}

error alphanumtok      (int bufsize, char *buffer, int *size) {
  error problem;
  if ((problem = verify(bufsize, buffer, size)))         return problem;
  if ((problem = rawalphanumtok()))                      return problem;
  return done();
}



static error rawflttok () {
 error problem;
 
 if (!isdigit(thischaracter))  return ERRnotfound;
 else if ((problem = rawinttok ())) return problem;

 if (thischaracter != '.') return done();
 if ((problem = stuffchar('.'))) return problem;
 if ((problem = nextchar()))     return problem;

 if (thischaracter != EOF && isdigit(thischaracter))
    if ((problem = rawinttok()))
       if (problem != ERRnotfound) return problem;

 return NoError;
}

error flttok      (int bufsize, char *buffer, int *size) {
  error problem;
  if ((problem = verify(bufsize, buffer, size)))         return problem;
  if ((problem = rawflttok()))                           return problem;
  return done();
}

static error rawsflttok () {
 error problem;

 if ((problem = rawsinttok ())) return problem;

 if (thischaracter != '.') return done();
 if ((problem = stuffchar('.'))) return problem;
 if ((problem = nextchar()))     return problem;

 if (thischaracter != EOF && isdigit(thischaracter))
    if ((problem = rawinttok()))
       if (problem != ERRnotfound) return problem;

 return NoError;
}

error sflttok (int bufsize, char *buffer, int *size) {
  error problem;
  if ((problem = verify(bufsize, buffer, size)))         return problem;
  if ((problem = rawsflttok()))                          return problem;
  return done();
}


static error rawvarnamtok () {
  error problem;

  if (!isalpha(thischaracter))           return ERRnotfound;
  else if ((problem = rawalphanumtok())) return problem;
  else                                   return NoError;
}

error varnamtok   (int bufsize, char *buffer, int *size) {
  error problem;
  if ((problem = verify(bufsize, buffer, size)))            return problem;
  if ((problem = rawvarnamtok()))                           return problem;
  return done();
}

static error rawtokfrom (char *list) {
  error problem;

  if ((problem = isone (list)))  return problem;

  while (thischaracter != EOF) {
   if ((problem = stuffchar(thischaracter))) return problem;
   if ((problem = nextchar()))               return problem;
   if ((problem = isone(list))) {
      if (problem == ERRnotfound) return NoError;
      else return problem;
   }
  }
  return NoError;
}

error tokfrom (int bufsize, char *buffer, int *size, char *list) {
  error problem;
  if (!initialized)                              return ERRuninitialized;
  if (!list)                                     return ERRnil;
  if (!*list)                                    return ERRempty;
  if ((problem = verify(bufsize, buffer, size))) return problem;
  if ((problem = rawtokfrom(list)))              return problem;
  return done();
}


Go to ...


This page is http://www.cc.utah.edu/~nahaj/cave/survey/code/c/token.c.html
© Copyright 2000 by John Halleck, All Rights Reserved.
This snapshot was last modified on August 23rd, 2000
And the underlying file was last modified on May 11th, 2000