2 /* Parser-tokenizer link implementation */
4 #include "pgenheaders.h"
17 static node *parsetok(struct tok_state *, grammar *, int, perrdetail *, int *);
18 static void initerr(perrdetail *err_ret, const char* filename);
20 /* Parse input coming from a string. Return error code, print some errors. */
22 PyParser_ParseString(const char *s, grammar *g, int start, perrdetail *err_ret)
24 return PyParser_ParseStringFlagsFilename(s, NULL, g, start, err_ret, 0);
28 PyParser_ParseStringFlags(const char *s, grammar *g, int start,
29 perrdetail *err_ret, int flags)
31 return PyParser_ParseStringFlagsFilename(s, NULL,
32 g, start, err_ret, flags);
36 PyParser_ParseStringFlagsFilename(const char *s, const char *filename,
37 grammar *g, int start,
38 perrdetail *err_ret, int flags)
41 return PyParser_ParseStringFlagsFilenameEx(s, filename, g, start,
46 PyParser_ParseStringFlagsFilenameEx(const char *s, const char *filename,
47 grammar *g, int start,
48 perrdetail *err_ret, int *flags)
50 struct tok_state *tok;
52 initerr(err_ret, filename);
54 if ((tok = PyTokenizer_FromString(s, start == file_input)) == NULL) {
55 err_ret->error = PyErr_Occurred() ? E_DECODE : E_NOMEM;
59 tok->filename = filename ? filename : "<string>";
60 if (Py_TabcheckFlag || Py_VerboseFlag) {
61 tok->altwarning = (tok->filename != NULL);
62 if (Py_TabcheckFlag >= 2)
66 return parsetok(tok, g, start, err_ret, flags);
69 /* Parse input coming from a file. Return error code, print some errors. */
72 PyParser_ParseFile(FILE *fp, const char *filename, grammar *g, int start,
73 char *ps1, char *ps2, perrdetail *err_ret)
75 return PyParser_ParseFileFlags(fp, filename, g, start, ps1, ps2,
80 PyParser_ParseFileFlags(FILE *fp, const char *filename, grammar *g, int start,
81 char *ps1, char *ps2, perrdetail *err_ret, int flags)
84 return PyParser_ParseFileFlagsEx(fp, filename, g, start, ps1, ps2, err_ret, &iflags);
88 PyParser_ParseFileFlagsEx(FILE *fp, const char *filename, grammar *g, int start,
89 char *ps1, char *ps2, perrdetail *err_ret, int *flags)
91 struct tok_state *tok;
93 initerr(err_ret, filename);
95 if ((tok = PyTokenizer_FromFile(fp, ps1, ps2)) == NULL) {
96 err_ret->error = E_NOMEM;
99 tok->filename = filename;
100 if (Py_TabcheckFlag || Py_VerboseFlag) {
101 tok->altwarning = (filename != NULL);
102 if (Py_TabcheckFlag >= 2)
106 return parsetok(tok, g, start, err_ret, flags);
110 static char with_msg[] =
111 "%s:%d: Warning: 'with' will become a reserved keyword in Python 2.6\n";
113 static char as_msg[] =
114 "%s:%d: Warning: 'as' will become a reserved keyword in Python 2.6\n";
117 warn(const char *msg, const char *filename, int lineno)
119 if (filename == NULL)
120 filename = "<string>";
121 PySys_WriteStderr(msg, filename, lineno);
125 /* Parse input coming from the given tokenizer structure.
126 Return error code. */
129 parsetok(struct tok_state *tok, grammar *g, int start, perrdetail *err_ret,
134 int started = 0, handling_import = 0, handling_with = 0;
136 if ((ps = PyParser_New(g, start)) == NULL) {
137 fprintf(stderr, "no mem for new parser\n");
138 err_ret->error = E_NOMEM;
139 PyTokenizer_Free(tok);
142 #ifdef PY_PARSER_REQUIRES_FUTURE_KEYWORD
143 if (*flags & PyPARSE_PRINT_IS_FUNCTION) {
144 ps->p_flags |= CO_FUTURE_PRINT_FUNCTION;
146 if (*flags & PyPARSE_UNICODE_LITERALS) {
147 ps->p_flags |= CO_FUTURE_UNICODE_LITERALS;
159 type = PyTokenizer_Get(tok, &a, &b);
160 if (type == ERRORTOKEN) {
161 err_ret->error = tok->done;
164 if (type == ENDMARKER && started) {
165 type = NEWLINE; /* Add an extra newline */
166 handling_with = handling_import = 0;
168 /* Add the right number of dedent tokens,
169 except if a certain flag is given --
170 codeop.py uses this. */
172 !(*flags & PyPARSE_DONT_IMPLY_DEDENT))
174 tok->pendin = -tok->indent;
180 len = b - a; /* XXX this may compute NULL - NULL */
181 str = (char *) PyObject_MALLOC(len + 1);
183 fprintf(stderr, "no mem for next token\n");
184 err_ret->error = E_NOMEM;
188 strncpy(str, a, len);
191 #ifdef PY_PARSER_REQUIRES_FUTURE_KEYWORD
193 if (a >= tok->line_start)
194 col_offset = a - tok->line_start;
198 if ((err_ret->error =
199 PyParser_AddToken(ps, (int)type, str, tok->lineno, col_offset,
200 &(err_ret->expected))) != E_OK) {
201 if (err_ret->error != E_DONE) {
203 err_ret->token = type;
209 if (err_ret->error == E_DONE) {
216 #ifdef PY_PARSER_REQUIRES_FUTURE_KEYWORD
217 *flags = ps->p_flags;
222 if (tok->lineno <= 1 && tok->done == E_EOF)
223 err_ret->error = E_EOF;
224 err_ret->lineno = tok->lineno;
225 if (tok->buf != NULL) {
228 assert(tok->cur - tok->buf < INT_MAX);
229 err_ret->offset = (int)(tok->cur - tok->buf);
230 len = tok->inp - tok->buf;
231 #ifdef Py_USING_UNICODE
232 text = PyTokenizer_RestoreEncoding(tok, len, &err_ret->offset);
236 text = (char *) PyObject_MALLOC(len + 1);
239 strncpy(text, tok->buf, len);
243 err_ret->text = text;
245 } else if (tok->encoding != NULL) {
246 /* 'nodes->n_str' uses PyObject_*, while 'tok->encoding' was
247 * allocated using PyMem_
249 node* r = PyNode_New(encoding_decl);
251 r->n_str = PyObject_MALLOC(strlen(tok->encoding)+1);
252 if (!r || !r->n_str) {
253 err_ret->error = E_NOMEM;
259 strcpy(r->n_str, tok->encoding);
260 PyMem_FREE(tok->encoding);
261 tok->encoding = NULL;
268 PyTokenizer_Free(tok);
274 initerr(perrdetail *err_ret, const char *filename)
276 err_ret->error = E_OK;
277 err_ret->filename = filename;
280 err_ret->text = NULL;
282 err_ret->expected = -1;