examples/pullscan.rl

   1 #include <stdio.h>
   2 #include <stdlib.h>
   3 #include <string.h>
   4
   5 #define BUFSIZE 4096
   6
   7 typedef struct _Scanner {
   8         /* Scanner state. */
   9     int cs;
  10     int act;
  11     int have;
  12     int curline;
  13     char *ts;
  14     char *te;
  15     char *p;
  16     char *pe;
  17     char *eof;
  18         FILE *file;
  19         int done;
  20
  21         /* Token data */
  22         char *data;
  23         int len;
  24     int value;
  25
  26         char buf[BUFSIZE];
  27 } Scanner;
  28
  29
  30 %%{
  31         machine Scanner;
  32         write data;
  33 }%%
  34
  35 void scan_init( Scanner *s, FILE *file )
  36 {
  37         memset (s, '\0', sizeof(Scanner));
  38         s->curline = 1;
  39         s->file = file;
  40         s->eof = 0;
  41         %% write init;
  42 }
  43
  44 #define TK_NO_TOKEN (-1)
  45 #define TK_ERR 128
  46 #define TK_EOF 129
  47 #define TK_Identifier 130
  48 #define TK_Number 131
  49 #define TK_String 132
  50
  51 #define ret_tok( _tok ) token = _tok; s->data = s->ts
  52
  53 int scan( Scanner *s )
  54 {
  55         int token = TK_NO_TOKEN;
  56         int space, readlen;
  57
  58         while ( 1 ) {
  59                 if ( s->p == s->pe ) {
  60                         printf("scanner: need more data\n");
  61
  62                         if ( s->ts == 0 )
  63                                 s->have = 0;
  64                         else {
  65                                 /* There is data that needs to be shifted over. */
  66                                 printf("scanner: buffer broken mid token\n");
  67                                 s->have = s->pe - s->ts;
  68                                 memmove( s->buf, s->ts, s->have );
  69                                 s->te -= (s->ts-s->buf);
  70                                 s->ts = s->buf;
  71                         }
  72
  73                         s->p = s->buf + s->have;
  74                         space = BUFSIZE - s->have;
  75
  76                         if ( space == 0 ) {
  77                                 /* We filled up the buffer trying to scan a token. */
  78                                 printf("scanner: out of buffer space\n");
  79                                 return TK_ERR;
  80                         }
  81
  82                         if ( s->done ) {
  83                                 printf("scanner: end of file\n");
  84                                 s->p[0] = 0;
  85                                 readlen = 1;
  86                         }
  87                         else {
  88                                 readlen = fread( s->p, 1, space, s->file );
  89                                 if ( readlen < space )
  90                                         s->done = 1;
  91                         }
  92
  93                         s->pe = s->p + readlen;
  94                 }
  95
  96                 %%{
  97                         machine Scanner;
  98                         access s->;
  99                         variable p s->p;
 100                         variable pe s->pe;
 101                         variable eof s->eof;
 102
 103                         main := |*
 104
 105                         # Identifiers
 106                         ( [a-zA-Z_] [a-zA-Z0-9_]* ) =>
 107                                 { ret_tok( TK_Identifier ); fbreak; };
 108
 109                         # Whitespace
 110                         [ \t\n];
 111
 112                         '"' ( [^\\"] | '\\' any ) * '"' =>
 113                                 { ret_tok( TK_String ); fbreak; };
 114
 115                         # Number
 116                         digit+ =>
 117                                 { ret_tok( TK_Number ); fbreak; };
 118
 119                         # EOF
 120                         0 =>
 121                                 { ret_tok( TK_EOF ); fbreak; };
 122
 123                         # Anything else
 124                         any =>
 125                                 { ret_tok( *s->p ); fbreak; };
 126
 127                         *|;
 128
 129                         write exec;
 130                 }%%
 131
 132                 if ( s->cs == Scanner_error )
 133                         return TK_ERR;
 134
 135                 if ( token != TK_NO_TOKEN ) {
 136                         s->len = s->p - s->data;
 137                         return token;
 138                 }
 139         }
 140 }
 141
 142
 143 int main (int argc, char** argv)
 144 {
 145         Scanner ss;
 146         int tok;
 147
 148         scan_init(&ss, stdin);
 149
 150         while ( 1 ) {
 151                 tok = scan (&ss);
 152                 if ( tok == TK_EOF ) {
 153                         printf ("parser: EOF\n");
 154                         break;
 155                 }
 156                 else if ( tok == TK_ERR ) {
 157                         printf ("parser: ERR\n");
 158                         break;
 159                 }
 160                 else {
 161                         printf ("parser: %d \"", tok);
 162                         fwrite ( ss.data, 1, ss.len, stdout );
 163                         printf ("\"\n" );
 164                 }
 165         }
 166
 167         return 0;
 168 }
 169
 170