examples/pullscan/pullscan.rl

   1 #include <stdio.h>
   2 #include <stdlib.h>
   3 #include <string.h>
   4
   5 #define BUFSIZE 4096
   6
   7 typedef struct _Scanner {
   8         /* Scanner state. */
   9     int cs;
  10     int act;
  11     int have;
  12     int curline;
  13     char *tokstart;
  14     char *tokend;
  15     char *p;
  16     char *pe;
  17         FILE *file;
  18         int done;
  19
  20         /* Token data */
  21         char *data;
  22         int len;
  23     int value;
  24
  25         char buf[BUFSIZE];
  26 } Scanner;
  27
  28
  29 void scan_init( Scanner *s, FILE *file )
  30 {
  31         memset (s, '\0', sizeof(Scanner));
  32         s->curline = 1;
  33         s->file = file;
  34 }
  35
  36 #define TK_NO_TOKEN (-1)
  37 #define TK_ERR 128
  38 #define TK_EOF 129
  39 #define TK_Identifier 130
  40 #define TK_Number 131
  41
  42
  43 %%{
  44         machine Scanner;
  45         write data;
  46 }%%
  47
  48 #define ret_tok( _tok ) token = _tok; s->data = s->tokstart
  49
  50 int scan( Scanner *s )
  51 {
  52         char *p = s->p;
  53         char *pe = s->pe;
  54         int token = TK_NO_TOKEN;
  55         int space, readlen;
  56
  57         while ( 1 ) {
  58                 if ( p == pe ) {
  59                         printf("scanner: need more data\n");
  60
  61                         if ( s->tokstart == 0 )
  62                                 s->have = 0;
  63                         else {
  64                                 /* There is data that needs to be shifted over. */
  65                                 printf("scanner: buffer broken mid token\n");
  66                                 s->have = pe - s->tokstart;
  67                                 memmove( s->buf, s->tokstart, s->have );
  68                                 s->tokend -= (s->tokstart-s->buf);
  69                                 s->tokstart = s->buf;
  70                         }
  71
  72                         p = s->buf + s->have;
  73                         space = BUFSIZE - s->have;
  74
  75                         if ( space == 0 ) {
  76                                 /* We filled up the buffer trying to scan a token. */
  77                                 printf("scanner: out of buffer space\n");
  78                                 return TK_ERR;
  79                         }
  80
  81                         if ( s->done ) {
  82                                 printf("scanner: end of file\n");
  83                                 p[0] = 0;
  84                                 readlen = 1;
  85                         }
  86                         else {
  87                                 readlen = fread( p, 1, space, s->file );
  88                                 if ( readlen < space )
  89                                         s->done = 1;
  90                         }
  91
  92                         pe = p + readlen;
  93                 }
  94
  95                 %%{
  96                         machine Scanner;
  97                         access s->;
  98
  99                         main := |*
 100
 101                         # Identifiers
 102                         ( [a-zA-Z_] [a-zA-Z0-9_]* ) =>
 103                                 { ret_tok( TK_Identifier ); fbreak; };
 104
 105                         # Whitespace
 106                         [ \t\n];
 107
 108                         # Number
 109                         digit+ =>
 110                                 { ret_tok( TK_Number ); fbreak; };
 111
 112                         # EOF
 113                         0 =>
 114                                 { ret_tok( TK_EOF ); fbreak; };
 115
 116                         # Anything else
 117                         any =>
 118                                 { ret_tok( *p ); fbreak; };
 119
 120                         *|;
 121
 122                         write exec;
 123                 }%%
 124
 125                 if ( s->cs == Scanner_error )
 126                         return TK_ERR;
 127
 128                 if ( token != TK_NO_TOKEN ) {
 129                         /* Save p and pe. fbreak does not advance p. */
 130                         s->p = p + 1;
 131                         s->pe = pe;
 132                         s->len = s->p - s->data;
 133                         return token;
 134                 }
 135         }
 136 }
 137
 138
 139 int main (int argc, char** argv)
 140 {
 141         Scanner ss;
 142         int tok;
 143
 144         scan_init(&ss, stdin);
 145
 146         while ( 1 ) {
 147                 tok = scan (&ss);
 148                 if ( tok == TK_EOF ) {
 149                         printf ("parser: EOF\n");
 150                         break;
 151                 }
 152                 else if ( tok == TK_ERR ) {
 153                         printf ("parser: ERR\n");
 154                         break;
 155                 }
 156                 else {
 157                         printf ("parser: %d \"", tok);
 158                         fwrite ( ss.data, 1, ss.len, stdout );
 159                         printf ("\"\n" );
 160                 }
 161         }
 162
 163         return 0;
 164 }
 165
 166