From: Sven Verdoolaege <sven@cs.kuleuven.be>
Date: Thu, 25 Dec 2008 13:43:40 +0000 (+0100)
Subject: isl_stream.c: extracted from isl_input_omega.c
X-Git-Tag: isl-0.01~326
X-Git-Url: http://review.tizen.org/git/?a=commitdiff_plain;h=445da2d3cc5d14abd2d3f1fc71c2498507f61b2c;p=platform%2Fupstream%2Fisl.git

isl_stream.c: extracted from isl_input_omega.c
---

diff --git a/Makefile.am b/Makefile.am
index 9523777..9147c11 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -66,6 +66,8 @@ libisl_la_SOURCES = \
 	isl_output.c \
 	isl_sample.h \
 	isl_sample.c \
+	isl_stream.c \
+	isl_stream.h \
 	isl_seq.c \
 	isl_vec.c \
 	isl_vec.h
diff --git a/isl_input_omega.c b/isl_input_omega.c
index 6b59647..3b3ba04 100644
--- a/isl_input_omega.c
+++ b/isl_input_omega.c
@@ -2,340 +2,10 @@
 #include <string.h>
 #include <strings.h>
 #include <isl_seq.h>
+#include "isl_stream.h"
 #include "isl_map_private.h"
 #include "isl_input_omega.h"
 
-enum token_type { TOKEN_UNKNOWN = 256, TOKEN_VALUE, TOKEN_IDENT, TOKEN_GE,
-		  TOKEN_LE, TOKEN_TO, TOKEN_AND, TOKEN_EXISTS };
-
-struct token {
-	enum token_type  type;
-
-	unsigned int on_new_line : 1;
-	int line;
-	int col;
-
-	union {
-		isl_int	v;
-		char	*s;
-	} u;
-};
-
-static struct token *token_new(struct isl_ctx *ctx,
-	int line, int col, unsigned on_new_line)
-{
-	struct token *tok = isl_alloc_type(ctx, struct token);
-	if (!tok)
-		return NULL;
-	tok->line = line;
-	tok->col = col;
-	tok->on_new_line = on_new_line;
-	return tok;
-}
-
-void token_free(struct token *tok)
-{
-	if (!tok)
-		return;
-	if (tok->type == TOKEN_VALUE)
-		isl_int_clear(tok->u.v);
-	else if (tok->type == TOKEN_IDENT)
-		free(tok->u.s);
-	free(tok);
-}
-
-struct stream {
-	struct isl_ctx	*ctx;
-	FILE        	*file;
-	const char  	*str;
-	int	    	line;
-	int	    	col;
-	int	    	eof;
-
-	char	    	*buffer;
-	size_t	    	size;
-	size_t	    	len;
-	int	    	c;
-
-	struct token	*tokens[5];
-	int	    	n_token;
-};
-
-void stream_error(struct stream *s, struct token *tok, char *msg)
-{
-	int line = tok ? tok->line : s->line;
-	int col = tok ? tok->col : s->col;
-	fprintf(stderr, "syntax error (%d, %d): %s\n", line, col, msg);
-	if (tok) {
-		if (tok->type < 256)
-			fprintf(stderr, "got '%c'\n", tok->type);
-		else
-			fprintf(stderr, "got token type %d\n", tok->type);
-	}
-}
-
-static void stream_free(struct stream *s);
-
-static struct stream* stream_new(struct isl_ctx *ctx)
-{
-	int i;
-	struct stream *s = isl_alloc_type(ctx, struct stream);
-	if (!s)
-		return NULL;
-	s->ctx = ctx;
-	s->size = 256;
-	s->file = NULL;
-	s->str = NULL;
-	s->buffer = isl_alloc_array(ctx, char, s->size);
-	if (!s->buffer)
-		goto error;
-	s->len = 0;
-	s->line = 1;
-	s->col = 0;
-	s->eof = 0;
-	s->c = -1;
-	for (i = 0; i < 5; ++i)
-		s->tokens[i] = NULL;
-	s->n_token = 0;
-	return s;
-error:
-	stream_free(s);
-	return NULL;
-}
-
-static struct stream* stream_new_file(struct isl_ctx *ctx, FILE *file)
-{
-	struct stream *s = stream_new(ctx);
-	if (!s)
-		return NULL;
-	s->file = file;
-	return s;
-}
-
-static struct stream* stream_new_str(struct isl_ctx *ctx, const char *str)
-{
-    struct stream *s = stream_new(ctx);
-    s->str = str;
-    return s;
-}
-
-static int stream_getc(struct stream *s)
-{
-	int c;
-	if (s->eof)
-		return -1;
-	if (s->file)
-		c = fgetc(s->file);
-	else {
-		c = *s->str++;
-		if (c == '\0')
-			c = -1;
-	}
-	if (c == -1)
-		s->eof = 1;
-	if (!s->eof) {
-		if (s->c == '\n') {
-			s->line++;
-			s->col = 0;
-		} else
-			s->col++;
-	}
-	s->c = c;
-	return c;
-}
-
-static void stream_ungetc(struct stream *s, int c)
-{
-	if (s->file)
-		ungetc(c, s->file);
-	else
-		--s->str;
-	s->c = -1;
-}
-
-static int stream_push_char(struct stream *s, int c)
-{
-	if (s->len >= s->size) {
-		s->size = (3*s->size)/2;
-		s->buffer = isl_realloc_array(ctx, s->buffer, char, s->size);
-		if (!s->buffer)
-			return -1;
-	}
-	s->buffer[s->len++] = c;
-	return 0;
-}
-
-static void stream_push_token(struct stream *s, struct token *tok)
-{
-	isl_assert(s->ctx, s->n_token < 5, return);
-	s->tokens[s->n_token++] = tok;
-}
-
-static struct token *stream_next_token(struct stream *s)
-{
-	int c;
-	struct token *tok = NULL;
-	int line, col;
-	int old_line = s->line;
-
-	if (s->n_token)
-		return s->tokens[--s->n_token];
-
-	s->len = 0;
-
-	/* skip spaces */
-	while ((c = stream_getc(s)) != -1 && isspace(c))
-		/* nothing */
-		;
-
-	line = s->line;
-	col = s->col;
-
-	if (c == -1)
-		return NULL;
-	if (c == '(' ||
-	    c == ')' ||
-	    c == '+' ||
-	    c == '/' ||
-	    c == '*' ||
-	    c == '^' ||
-	    c == '=' ||
-	    c == ',' ||
-	    c == ':' ||
-	    c == '[' ||
-	    c == ']' ||
-	    c == '{' ||
-	    c == '}') {
-		tok = token_new(s->ctx, line, col, old_line != line);
-		if (!tok)
-			return NULL;
-		tok->type = (enum token_type)c;
-		return tok;
-	}
-	if (c == '-') {
-		int c;
-		if ((c = stream_getc(s)) == '>') {
-			tok = token_new(s->ctx, line, col, old_line != line);
-			if (!tok)
-				return NULL;
-			tok->type = TOKEN_TO;
-			return tok;
-		}
-		if (c != -1)
-			stream_ungetc(s, c);
-	}
-	if (c == '-' || isdigit(c)) {
-		tok = token_new(s->ctx, line, col, old_line != line);
-		if (!tok)
-			return NULL;
-		tok->type = TOKEN_VALUE;
-		isl_int_init(tok->u.v);
-		if (stream_push_char(s, c))
-			goto error;
-		while ((c = stream_getc(s)) != -1 && isdigit(c))
-			if (stream_push_char(s, c))
-				goto error;
-		if (c != -1)
-			stream_ungetc(s, c);
-		if (s->len == 1 && s->buffer[0] == '-')
-			isl_int_set_si(tok->u.v, -1);
-		else {
-			stream_push_char(s, '\0');
-			isl_int_read(tok->u.v, s->buffer);
-		}
-		return tok;
-	}
-	if (isalpha(c)) {
-		tok = token_new(s->ctx, line, col, old_line != line);
-		if (!tok)
-			return NULL;
-		stream_push_char(s, c);
-		while ((c = stream_getc(s)) != -1 && isalnum(c))
-			stream_push_char(s, c);
-		if (c != -1)
-			stream_ungetc(s, c);
-		stream_push_char(s, '\0');
-		if (!strcasecmp(s->buffer, "exists"))
-			tok->type = TOKEN_EXISTS;
-		else {
-			tok->type = TOKEN_IDENT;
-			tok->u.s = strdup(s->buffer);
-		}
-		return tok;
-	}
-	if (c == '>') {
-		int c;
-		if ((c = stream_getc(s)) == '=') {
-			tok = token_new(s->ctx, line, col, old_line != line);
-			if (!tok)
-				return NULL;
-			tok->type = TOKEN_GE;
-			return tok;
-		}
-		if (c != -1)
-			stream_ungetc(s, c);
-	}
-	if (c == '<') {
-		int c;
-		if ((c = stream_getc(s)) == '=') {
-			tok = token_new(s->ctx, line, col, old_line != line);
-			if (!tok)
-				return NULL;
-			tok->type = TOKEN_LE;
-			return tok;
-		}
-		if (c != -1)
-			stream_ungetc(s, c);
-	}
-	if (c == '&') {
-		tok = token_new(s->ctx, line, col, old_line != line);
-		if (!tok)
-			return NULL;
-		tok->type = TOKEN_AND;
-		if ((c = stream_getc(s)) != '&' && c != -1)
-			stream_ungetc(s, c);
-		return tok;
-	}
-
-	tok = token_new(s->ctx, line, col, old_line != line);
-	if (!tok)
-		return NULL;
-	tok->type = TOKEN_UNKNOWN;
-	return tok;
-error:
-	token_free(tok);
-	return NULL;
-}
-
-static int stream_eat(struct stream *s, int type)
-{
-	struct token *tok;
-
-	tok = stream_next_token(s);
-	if (!tok)
-		return -1;
-	if (tok->type == type) {
-		token_free(tok);
-		return 0;
-	}
-	stream_error(s, tok, "expecting other token");
-	stream_push_token(s, tok);
-	return -1;
-}
-
-static void stream_free(struct stream *s)
-{
-	if (!s)
-		return;
-	free(s->buffer);
-	if (s->n_token != 0) {
-		struct token *tok = stream_next_token(s);
-		stream_error(s, tok, "unexpected token");
-		token_free(tok);
-	}
-	free(s);
-}
-
 struct variable {
 	char    	    	*name;
 	int	     		 pos;
@@ -418,88 +88,88 @@ static int vars_pos(struct vars *v, const char *s, int len)
 	return pos;
 }
 
-static struct vars *read_var_list(struct stream *s, struct vars *v)
+static struct vars *read_var_list(struct isl_stream *s, struct vars *v)
 {
-	struct token *tok;
+	struct isl_token *tok;
 
-	while ((tok = stream_next_token(s)) != NULL) {
+	while ((tok = isl_stream_next_token(s)) != NULL) {
 		int p;
 		int n = v->n;
 
-		if (tok->type != TOKEN_IDENT)
+		if (tok->type != ISL_TOKEN_IDENT)
 			break;
 
 		p = vars_pos(v, tok->u.s, -1);
 		if (p < 0)
 			goto error;
 		if (p < n) {
-			stream_error(s, tok, "expecting unique identifier");
+			isl_stream_error(s, tok, "expecting unique identifier");
 			goto error;
 		}
-		token_free(tok);
-		tok = stream_next_token(s);
+		isl_token_free(tok);
+		tok = isl_stream_next_token(s);
 		if (!tok || tok->type != ',')
 			break;
 
-		token_free(tok);
+		isl_token_free(tok);
 	}
 	if (tok)
-		stream_push_token(s, tok);
+		isl_stream_push_token(s, tok);
 
 	return v;
 error:
-	token_free(tok);
+	isl_token_free(tok);
 	vars_free(v);
 	return NULL;
 }
 
-static struct vars *read_tuple(struct stream *s, struct vars *v)
+static struct vars *read_tuple(struct isl_stream *s, struct vars *v)
 {
-	struct token *tok;
+	struct isl_token *tok;
 
-	tok = stream_next_token(s);
+	tok = isl_stream_next_token(s);
 	if (!tok || tok->type != '[') {
-		stream_error(s, tok, "expecting '['");
+		isl_stream_error(s, tok, "expecting '['");
 		goto error;
 	}
-	token_free(tok);
+	isl_token_free(tok);
 	v = read_var_list(s, v);
-	tok = stream_next_token(s);
+	tok = isl_stream_next_token(s);
 	if (!tok || tok->type != ']') {
-		stream_error(s, tok, "expecting ']'");
+		isl_stream_error(s, tok, "expecting ']'");
 		goto error;
 	}
-	token_free(tok);
+	isl_token_free(tok);
 
 	return v;
 error:
 	if (tok)
-		token_free(tok);
+		isl_token_free(tok);
 	vars_free(v);
 	return NULL;
 }
 
-static struct isl_basic_map *add_constraints(struct stream *s,
+static struct isl_basic_map *add_constraints(struct isl_stream *s,
 	struct vars **v, struct isl_basic_map *bmap);
 
-static struct isl_basic_map *add_exists(struct stream *s,
+static struct isl_basic_map *add_exists(struct isl_stream *s,
 	struct vars **v, struct isl_basic_map *bmap)
 {
-	struct token *tok;
+	struct isl_token *tok;
 	int n = (*v)->n;
 	int extra;
 	int seen_paren = 0;
 	int i;
 	unsigned total;
 
-	tok = stream_next_token(s);
+	tok = isl_stream_next_token(s);
 	if (!tok)
 		goto error;
 	if (tok->type == '(') {
 		seen_paren = 1;
-		token_free(tok);
+		isl_token_free(tok);
 	} else
-		stream_push_token(s, tok);
+		isl_stream_push_token(s, tok);
 	*v = read_var_list(s, *v);
 	if (!*v)
 		goto error;
@@ -515,10 +185,10 @@ static struct isl_basic_map *add_exists(struct stream *s,
 	}
 	if (!bmap)
 		return NULL;
-	if (stream_eat(s, ':'))
+	if (isl_stream_eat(s, ':'))
 		goto error;
 	bmap = add_constraints(s, v, bmap);
-	if (seen_paren && stream_eat(s, ')'))
+	if (seen_paren && isl_stream_eat(s, ')'))
 		goto error;
 	return bmap;
 error:
@@ -526,7 +196,7 @@ error:
 	return NULL;
 }
 
-static struct isl_basic_map *add_constraint(struct stream *s,
+static struct isl_basic_map *add_constraint(struct isl_stream *s,
 	struct vars **v, struct isl_basic_map *bmap)
 {
 	unsigned total = isl_basic_map_total_dim(bmap);
@@ -534,16 +204,16 @@ static struct isl_basic_map *add_constraint(struct stream *s,
 	int sign = 1;
 	int equality = 0;
 	int op = 0;
-	struct token *tok = NULL;
+	struct isl_token *tok = NULL;
 
-	tok = stream_next_token(s);
+	tok = isl_stream_next_token(s);
 	if (!tok)
 		goto error;
-	if (tok->type == TOKEN_EXISTS) {
-		token_free(tok);
+	if (tok->type == ISL_TOKEN_EXISTS) {
+		isl_token_free(tok);
 		return add_exists(s, v, bmap);
 	}
-	stream_push_token(s, tok);
+	isl_stream_push_token(s, tok);
 
 	bmap = isl_basic_map_extend_constraints(bmap, 0, 1);
 	k = isl_basic_map_alloc_inequality(bmap);
@@ -552,18 +222,18 @@ static struct isl_basic_map *add_constraint(struct stream *s,
 	isl_seq_clr(bmap->ineq[k], 1+total);
 
 	for (;;) {
-		tok = stream_next_token(s);
+		tok = isl_stream_next_token(s);
 		if (!tok) {
-			stream_error(s, NULL, "unexpected EOF");
+			isl_stream_error(s, NULL, "unexpected EOF");
 			goto error;
 		}
-		if (tok->type == TOKEN_IDENT) {
+		if (tok->type == ISL_TOKEN_IDENT) {
 			int n = (*v)->n;
 			int pos = vars_pos(*v, tok->u.s, -1);
 			if (pos < 0)
 				goto error;
 			if (pos >= n) {
-				stream_error(s, tok, "unknown identifier");
+				isl_stream_error(s, tok, "unknown identifier");
 				goto error;
 			}
 			if (sign > 0)
@@ -572,51 +242,51 @@ static struct isl_basic_map *add_constraint(struct stream *s,
 			else
 				isl_int_sub_ui(bmap->ineq[k][1+pos],
 						bmap->ineq[k][1+pos], 1);
-		} else if (tok->type == TOKEN_VALUE) {
-			struct token *tok2;
+		} else if (tok->type == ISL_TOKEN_VALUE) {
+			struct isl_token *tok2;
 			int n = (*v)->n;
 			int pos = -1;
-			tok2 = stream_next_token(s);
-			if (tok2 && tok2->type == TOKEN_IDENT) {
+			tok2 = isl_stream_next_token(s);
+			if (tok2 && tok2->type == ISL_TOKEN_IDENT) {
 				pos = vars_pos(*v, tok2->u.s, -1);
 				if (pos < 0)
 					goto error;
 				if (pos >= n) {
-					stream_error(s, tok2,
+					isl_stream_error(s, tok2,
 						"unknown identifier");
-					token_free(tok2);
+					isl_token_free(tok2);
 					goto error;
 				}
-				token_free(tok2);
+				isl_token_free(tok2);
 			} else if (tok2)
-				stream_push_token(s, tok2);
+				isl_stream_push_token(s, tok2);
 			if (sign < 0)
 				isl_int_neg(tok->u.v, tok->u.v);
 			isl_int_add(bmap->ineq[k][1+pos],
 					bmap->ineq[k][1+pos], tok->u.v);
-		} else if (tok->type == TOKEN_LE) {
+		} else if (tok->type == ISL_TOKEN_LE) {
 			op = 1;
 			isl_seq_neg(bmap->ineq[k], bmap->ineq[k], 1+total);
-		} else if (tok->type == TOKEN_GE) {
+		} else if (tok->type == ISL_TOKEN_GE) {
 			op = 1;
 			sign = -1;
 		} else if (tok->type == '=') {
 			if (op) {
-				stream_error(s, tok, "too many operators");
+				isl_stream_error(s, tok, "too many operators");
 				goto error;
 			}
 			op = 1;
 			equality = 1;
 			sign = -1;
 		} else {
-			stream_push_token(s, tok);
+			isl_stream_push_token(s, tok);
 			break;
 		}
-		token_free(tok);
+		isl_token_free(tok);
 	}
 	tok = NULL;
 	if (!op) {
-		stream_error(s, tok, "missing operator");
+		isl_stream_error(s, tok, "missing operator");
 		goto error;
 	}
 	if (equality)
@@ -624,88 +294,88 @@ static struct isl_basic_map *add_constraint(struct stream *s,
 	return bmap;
 error:
 	if (tok)
-		token_free(tok);
+		isl_token_free(tok);
 	isl_basic_map_free(bmap);
 	return NULL;
 }
 
-static struct isl_basic_map *add_constraints(struct stream *s,
+static struct isl_basic_map *add_constraints(struct isl_stream *s,
 	struct vars **v, struct isl_basic_map *bmap)
 {
-	struct token *tok;
+	struct isl_token *tok;
 
 	for (;;) {
 		bmap = add_constraint(s, v, bmap);
 		if (!bmap)
 			return NULL;
-		tok = stream_next_token(s);
+		tok = isl_stream_next_token(s);
 		if (!tok) {
-			stream_error(s, NULL, "unexpected EOF");
+			isl_stream_error(s, NULL, "unexpected EOF");
 			goto error;
 		}
-		if (tok->type != TOKEN_AND)
+		if (tok->type != ISL_TOKEN_AND)
 			break;
-		token_free(tok);
+		isl_token_free(tok);
 	}
-	stream_push_token(s, tok);
+	isl_stream_push_token(s, tok);
 
 	return bmap;
 error:
 	if (tok)
-		token_free(tok);
+		isl_token_free(tok);
 	isl_basic_map_free(bmap);
 	return NULL;
 }
 
-static struct isl_basic_map *basic_map_read(struct stream *s)
+static struct isl_basic_map *basic_map_read(struct isl_stream *s)
 {
 	struct isl_basic_map *bmap = NULL;
-	struct token *tok;
+	struct isl_token *tok;
 	struct vars *v = NULL;
 	int n1;
 	int n2;
 
-	tok = stream_next_token(s);
+	tok = isl_stream_next_token(s);
 	if (!tok || tok->type != '{') {
-		stream_error(s, tok, "expecting '{'");
+		isl_stream_error(s, tok, "expecting '{'");
 		if (tok)
-			stream_push_token(s, tok);
+			isl_stream_push_token(s, tok);
 		goto error;
 	}
-	token_free(tok);
+	isl_token_free(tok);
 	v = vars_new(s->ctx);
 	v = read_tuple(s, v);
 	if (!v)
 		return NULL;
 	n1 = v->n;
-	tok = stream_next_token(s);
-	if (tok && tok->type == TOKEN_TO) {
-		token_free(tok);
+	tok = isl_stream_next_token(s);
+	if (tok && tok->type == ISL_TOKEN_TO) {
+		isl_token_free(tok);
 		v = read_tuple(s, v);
 		if (!v)
 			return NULL;
 		n2 = v->n - n1;
 	} else {
 		if (tok)
-			stream_push_token(s, tok);
+			isl_stream_push_token(s, tok);
 		n2 = n1;
 		n1 = 0;
 	}
 	bmap = isl_basic_map_alloc(s->ctx, 0, n1, n2, 0, 0,0);
 	if (!bmap)
 		goto error;
-	tok = stream_next_token(s);
+	tok = isl_stream_next_token(s);
 	if (tok && tok->type == ':') {
-		token_free(tok);
+		isl_token_free(tok);
 		bmap = add_constraints(s, &v, bmap);
-		tok = stream_next_token(s);
+		tok = isl_stream_next_token(s);
 	}
 	if (tok && tok->type == '}') {
-		token_free(tok);
+		isl_token_free(tok);
 	} else {
-		stream_error(s, tok, "unexpected token");
+		isl_stream_error(s, tok, "unexpected isl_token");
 		if (tok)
-			token_free(tok);
+			isl_token_free(tok);
 		goto error;
 	}
 	vars_free(v);
@@ -722,11 +392,11 @@ struct isl_basic_map *isl_basic_map_read_from_file_omega(
 		struct isl_ctx *ctx, FILE *input)
 {
 	struct isl_basic_map *bmap;
-	struct stream *s = stream_new_file(ctx, input);
+	struct isl_stream *s = isl_stream_new_file(ctx, input);
 	if (!s)
 		return NULL;
 	bmap = basic_map_read(s);
-	stream_free(s);
+	isl_stream_free(s);
 	return bmap;
 }
 
@@ -748,11 +418,11 @@ struct isl_basic_map *isl_basic_map_read_from_str_omega(
 		struct isl_ctx *ctx, const char *str)
 {
 	struct isl_basic_map *bmap;
-	struct stream *s = stream_new_str(ctx, str);
+	struct isl_stream *s = isl_stream_new_str(ctx, str);
 	if (!s)
 		return NULL;
 	bmap = basic_map_read(s);
-	stream_free(s);
+	isl_stream_free(s);
 	return bmap;
 }
 
diff --git a/isl_stream.c b/isl_stream.c
new file mode 100644
index 0000000..478eb4e
--- /dev/null
+++ b/isl_stream.c
@@ -0,0 +1,301 @@
+#include <ctype.h>
+#include <string.h>
+#include <strings.h>
+#include <isl_ctx.h>
+#include "isl_stream.h"
+
+static struct isl_token *isl_token_new(struct isl_ctx *ctx,
+	int line, int col, unsigned on_new_line)
+{
+	struct isl_token *tok = isl_alloc_type(ctx, struct isl_token);
+	if (!tok)
+		return NULL;
+	tok->line = line;
+	tok->col = col;
+	tok->on_new_line = on_new_line;
+	return tok;
+}
+
+void isl_token_free(struct isl_token *tok)
+{
+	if (!tok)
+		return;
+	if (tok->type == ISL_TOKEN_VALUE)
+		isl_int_clear(tok->u.v);
+	else if (tok->type == ISL_TOKEN_IDENT)
+		free(tok->u.s);
+	free(tok);
+}
+
+void isl_stream_error(struct isl_stream *s, struct isl_token *tok, char *msg)
+{
+	int line = tok ? tok->line : s->line;
+	int col = tok ? tok->col : s->col;
+	fprintf(stderr, "syntax error (%d, %d): %s\n", line, col, msg);
+	if (tok) {
+		if (tok->type < 256)
+			fprintf(stderr, "got '%c'\n", tok->type);
+		else
+			fprintf(stderr, "got token type %d\n", tok->type);
+	}
+}
+
+static struct isl_stream* isl_stream_new(struct isl_ctx *ctx)
+{
+	int i;
+	struct isl_stream *s = isl_alloc_type(ctx, struct isl_stream);
+	if (!s)
+		return NULL;
+	s->ctx = ctx;
+	s->size = 256;
+	s->file = NULL;
+	s->str = NULL;
+	s->buffer = isl_alloc_array(ctx, char, s->size);
+	if (!s->buffer)
+		goto error;
+	s->len = 0;
+	s->line = 1;
+	s->col = 0;
+	s->eof = 0;
+	s->c = -1;
+	for (i = 0; i < 5; ++i)
+		s->tokens[i] = NULL;
+	s->n_token = 0;
+	return s;
+error:
+	isl_stream_free(s);
+	return NULL;
+}
+
+struct isl_stream* isl_stream_new_file(struct isl_ctx *ctx, FILE *file)
+{
+	struct isl_stream *s = isl_stream_new(ctx);
+	if (!s)
+		return NULL;
+	s->file = file;
+	return s;
+}
+
+struct isl_stream* isl_stream_new_str(struct isl_ctx *ctx, const char *str)
+{
+    struct isl_stream *s = isl_stream_new(ctx);
+    s->str = str;
+    return s;
+}
+
+static int isl_stream_getc(struct isl_stream *s)
+{
+	int c;
+	if (s->eof)
+		return -1;
+	if (s->file)
+		c = fgetc(s->file);
+	else {
+		c = *s->str++;
+		if (c == '\0')
+			c = -1;
+	}
+	if (c == -1)
+		s->eof = 1;
+	if (!s->eof) {
+		if (s->c == '\n') {
+			s->line++;
+			s->col = 0;
+		} else
+			s->col++;
+	}
+	s->c = c;
+	return c;
+}
+
+static void isl_stream_ungetc(struct isl_stream *s, int c)
+{
+	if (s->file)
+		ungetc(c, s->file);
+	else
+		--s->str;
+	s->c = -1;
+}
+
+static int isl_stream_push_char(struct isl_stream *s, int c)
+{
+	if (s->len >= s->size) {
+		s->size = (3*s->size)/2;
+		s->buffer = isl_realloc_array(ctx, s->buffer, char, s->size);
+		if (!s->buffer)
+			return -1;
+	}
+	s->buffer[s->len++] = c;
+	return 0;
+}
+
+void isl_stream_push_token(struct isl_stream *s, struct isl_token *tok)
+{
+	isl_assert(s->ctx, s->n_token < 5, return);
+	s->tokens[s->n_token++] = tok;
+}
+
+struct isl_token *isl_stream_next_token(struct isl_stream *s)
+{
+	int c;
+	struct isl_token *tok = NULL;
+	int line, col;
+	int old_line = s->line;
+
+	if (s->n_token)
+		return s->tokens[--s->n_token];
+
+	s->len = 0;
+
+	/* skip spaces */
+	while ((c = isl_stream_getc(s)) != -1 && isspace(c))
+		/* nothing */
+		;
+
+	line = s->line;
+	col = s->col;
+
+	if (c == -1)
+		return NULL;
+	if (c == '(' ||
+	    c == ')' ||
+	    c == '+' ||
+	    c == '/' ||
+	    c == '*' ||
+	    c == '^' ||
+	    c == '=' ||
+	    c == ',' ||
+	    c == ':' ||
+	    c == '[' ||
+	    c == ']' ||
+	    c == '{' ||
+	    c == '}') {
+		tok = isl_token_new(s->ctx, line, col, old_line != line);
+		if (!tok)
+			return NULL;
+		tok->type = (enum isl_token_type)c;
+		return tok;
+	}
+	if (c == '-') {
+		int c;
+		if ((c = isl_stream_getc(s)) == '>') {
+			tok = isl_token_new(s->ctx, line, col, old_line != line);
+			if (!tok)
+				return NULL;
+			tok->type = ISL_TOKEN_TO;
+			return tok;
+		}
+		if (c != -1)
+			isl_stream_ungetc(s, c);
+	}
+	if (c == '-' || isdigit(c)) {
+		tok = isl_token_new(s->ctx, line, col, old_line != line);
+		if (!tok)
+			return NULL;
+		tok->type = ISL_TOKEN_VALUE;
+		isl_int_init(tok->u.v);
+		if (isl_stream_push_char(s, c))
+			goto error;
+		while ((c = isl_stream_getc(s)) != -1 && isdigit(c))
+			if (isl_stream_push_char(s, c))
+				goto error;
+		if (c != -1)
+			isl_stream_ungetc(s, c);
+		if (s->len == 1 && s->buffer[0] == '-')
+			isl_int_set_si(tok->u.v, -1);
+		else {
+			isl_stream_push_char(s, '\0');
+			isl_int_read(tok->u.v, s->buffer);
+		}
+		return tok;
+	}
+	if (isalpha(c)) {
+		tok = isl_token_new(s->ctx, line, col, old_line != line);
+		if (!tok)
+			return NULL;
+		isl_stream_push_char(s, c);
+		while ((c = isl_stream_getc(s)) != -1 && isalnum(c))
+			isl_stream_push_char(s, c);
+		if (c != -1)
+			isl_stream_ungetc(s, c);
+		isl_stream_push_char(s, '\0');
+		if (!strcasecmp(s->buffer, "exists"))
+			tok->type = ISL_TOKEN_EXISTS;
+		else {
+			tok->type = ISL_TOKEN_IDENT;
+			tok->u.s = strdup(s->buffer);
+		}
+		return tok;
+	}
+	if (c == '>') {
+		int c;
+		if ((c = isl_stream_getc(s)) == '=') {
+			tok = isl_token_new(s->ctx, line, col, old_line != line);
+			if (!tok)
+				return NULL;
+			tok->type = ISL_TOKEN_GE;
+			return tok;
+		}
+		if (c != -1)
+			isl_stream_ungetc(s, c);
+	}
+	if (c == '<') {
+		int c;
+		if ((c = isl_stream_getc(s)) == '=') {
+			tok = isl_token_new(s->ctx, line, col, old_line != line);
+			if (!tok)
+				return NULL;
+			tok->type = ISL_TOKEN_LE;
+			return tok;
+		}
+		if (c != -1)
+			isl_stream_ungetc(s, c);
+	}
+	if (c == '&') {
+		tok = isl_token_new(s->ctx, line, col, old_line != line);
+		if (!tok)
+			return NULL;
+		tok->type = ISL_TOKEN_AND;
+		if ((c = isl_stream_getc(s)) != '&' && c != -1)
+			isl_stream_ungetc(s, c);
+		return tok;
+	}
+
+	tok = isl_token_new(s->ctx, line, col, old_line != line);
+	if (!tok)
+		return NULL;
+	tok->type = ISL_TOKEN_UNKNOWN;
+	return tok;
+error:
+	isl_token_free(tok);
+	return NULL;
+}
+
+int isl_stream_eat(struct isl_stream *s, int type)
+{
+	struct isl_token *tok;
+
+	tok = isl_stream_next_token(s);
+	if (!tok)
+		return -1;
+	if (tok->type == type) {
+		isl_token_free(tok);
+		return 0;
+	}
+	isl_stream_error(s, tok, "expecting other token");
+	isl_stream_push_token(s, tok);
+	return -1;
+}
+
+void isl_stream_free(struct isl_stream *s)
+{
+	if (!s)
+		return;
+	free(s->buffer);
+	if (s->n_token != 0) {
+		struct isl_token *tok = isl_stream_next_token(s);
+		isl_stream_error(s, tok, "unexpected token");
+		isl_token_free(tok);
+	}
+	free(s);
+}
diff --git a/isl_stream.h b/isl_stream.h
new file mode 100644
index 0000000..3d1b6fc
--- /dev/null
+++ b/isl_stream.h
@@ -0,0 +1,57 @@
+#ifndef ISL_STREAM_H
+#define ISL_STREAM_H
+
+#include <stdio.h>
+
+#if defined(__cplusplus)
+extern "C" {
+#endif
+
+enum isl_token_type { ISL_TOKEN_UNKNOWN = 256, ISL_TOKEN_VALUE,
+			ISL_TOKEN_IDENT, ISL_TOKEN_GE,
+			ISL_TOKEN_LE, ISL_TOKEN_TO, ISL_TOKEN_AND,
+			ISL_TOKEN_EXISTS };
+
+struct isl_token {
+	enum isl_token_type  type;
+
+	unsigned int on_new_line : 1;
+	int line;
+	int col;
+
+	union {
+		isl_int	v;
+		char	*s;
+	} u;
+};
+
+struct isl_stream {
+	struct isl_ctx	*ctx;
+	FILE        	*file;
+	const char  	*str;
+	int	    	line;
+	int	    	col;
+	int	    	eof;
+
+	char	    	*buffer;
+	size_t	    	size;
+	size_t	    	len;
+	int	    	c;
+
+	struct isl_token	*tokens[5];
+	int	    	n_token;
+};
+
+struct isl_stream* isl_stream_new_file(struct isl_ctx *ctx, FILE *file);
+struct isl_stream* isl_stream_new_str(struct isl_ctx *ctx, const char *str);
+void isl_stream_free(struct isl_stream *s);
+
+struct isl_token *isl_stream_next_token(struct isl_stream *s);
+void isl_stream_push_token(struct isl_stream *s, struct isl_token *tok);
+int isl_stream_eat(struct isl_stream *s, int type);
+
+#if defined(__cplusplus)
+}
+#endif
+
+#endif