- Delete the old "Tokenizer" code.
- Write separate tests for the lexer distinct from the parser.
- Switch the parser to use the new code.
- New lexer error output has file:line numbers so e.g. Emacs can
jump your editor to the syntax error.
- The EvalEnv ($-interpolation) code is now part of the lexer as well.
objs += cxx('browse', order_only=built('browse_py.h'))
n.newline()
-n.comment('the depfile parser is generated using re2c.')
+n.comment('the depfile parser and ninja lexers are generated using re2c.')
n.rule('re2c',
command='re2c -b -i --no-generation-date -o $out $in',
description='RE2C $out')
-# Generate the .cc file in the source directory so we can check it in.
+# Generate the .cc files in the source directory so we can check them in.
n.build(src('depfile_parser.cc'), 're2c', src('depfile_parser.in.cc'))
+n.build(src('lexer.cc'), 're2c', src('lexer.in.cc'))
n.newline()
n.comment('Core source files all build into ninja library.')
'eval_env',
'graph',
'graphviz',
+ 'lexer',
'parsers',
'state',
'util']:
'depfile_parser_test',
'disk_interface_test',
'edit_distance_test',
- 'eval_env_test',
'graph_test',
+ 'lexer_test',
'parsers_test',
'state_test',
'subprocess_test',
bindings_[key] = val;
}
-bool EvalString::Parse(const string& input, string* err, size_t* err_index) {
- unparsed_ = input;
-
- string::size_type start, end;
- start = 0;
- do {
- end = input.find('$', start);
- if (end == string::npos) {
- end = input.size();
- break;
- }
- if (end > start)
- parsed_.push_back(make_pair(input.substr(start, end - start), RAW));
- start = end + 1;
- if (start < input.size() && input[start] == '{') {
- ++start;
- for (end = start + 1; end < input.size(); ++end) {
- if (input[end] == '}')
- break;
- }
- if (end >= input.size()) {
- *err = "expected closing curly after ${";
- if (err_index)
- *err_index = end;
- return false;
- }
- parsed_.push_back(make_pair(input.substr(start, end - start), SPECIAL));
- ++end;
- } else if (start < input.size() && input[start] == '$') {
- parsed_.push_back(make_pair("$", RAW));
- end = start + 1;
- } else if (start < input.size() && input[start] == ' ') {
- parsed_.push_back(make_pair(" ", RAW));
- end = start + 1;
- } else {
- for (end = start; end < input.size(); ++end) {
- char c = input[end];
- if (!(('a' <= c && c <= 'z') || ('A' <= c && c <= 'Z') ||
- ('0' <= c && c <= '9') || c == '_')) {
- break;
- }
- }
- if (end == start) {
- *err = "expected variable after $";
- if (err_index)
- *err_index = start;
- return false;
- }
- parsed_.push_back(make_pair(input.substr(start, end - start), SPECIAL));
- }
- start = end;
- } while (end < input.size());
- if (end > start)
- parsed_.push_back(make_pair(input.substr(start, end - start), RAW));
-
- return true;
-}
-
string EvalString::Evaluate(Env* env) const {
string result;
for (TokenList::const_iterator i = parsed_.begin(); i != parsed_.end(); ++i) {
}
return result;
}
+
+void EvalString::Add(TokenType type, StringPiece text) {
+ // Add it to the end of an existing RAW token if possible.
+ if (type == RAW && !parsed_.empty() && parsed_.back().second == RAW) {
+ parsed_.back().first.append(text.str_, text.len_);
+ } else {
+ parsed_.push_back(make_pair(text.AsString(), type));
+ }
+}
+
+string EvalString::Serialize() const {
+ string result;
+ for (TokenList::const_iterator i = parsed_.begin();
+ i != parsed_.end(); ++i) {
+ result.append("[");
+ if (i->second == SPECIAL)
+ result.append("$");
+ result.append(i->first);
+ result.append("]");
+ }
+ return result;
+}
#include <vector>
using namespace std;
+#include "string_piece.h"
+
/// An interface for a scope for variable (e.g. "$foo") lookups.
struct Env {
virtual ~Env() {}
/// A tokenized string that contains variable references.
/// Can be evaluated relative to an Env.
struct EvalString {
- bool Parse(const string& input, string* err, size_t* err_index=NULL);
string Evaluate(Env* env) const;
- const string& unparsed() const { return unparsed_; }
- bool empty() const { return unparsed_.empty(); }
+ void Clear() { parsed_.clear(); }
+ bool empty() const { return parsed_.empty(); }
- string unparsed_;
enum TokenType { RAW, SPECIAL };
+ void Add(TokenType type, StringPiece text);
+
+ /// Construct a human-readable representation of the parsed state,
+ /// for use in tests.
+ string Serialize() const;
+
typedef vector<pair<string, TokenType> > TokenList;
TokenList parsed_;
};
+++ /dev/null
-// Copyright 2011 Google Inc. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-#include <gtest/gtest.h>
-
-#include <map>
-#include <string>
-
-#include "eval_env.h"
-
-namespace {
-
-struct TestEnv : public Env {
- virtual string LookupVariable(const string& var) {
- return vars[var];
- }
- map<string, string> vars;
-};
-
-TEST(EvalString, PlainText) {
- EvalString str;
- string err;
- EXPECT_TRUE(str.Parse("plain text", &err));
- EXPECT_EQ("", err);
- EXPECT_EQ("plain text", str.Evaluate(NULL));
-}
-
-TEST(EvalString, OneVariable) {
- EvalString str;
- string err;
- EXPECT_TRUE(str.Parse("hi $var", &err));
- EXPECT_EQ("", err);
- EXPECT_EQ("hi $var", str.unparsed());
- TestEnv env;
- EXPECT_EQ("hi ", str.Evaluate(&env));
- env.vars["var"] = "there";
- EXPECT_EQ("hi there", str.Evaluate(&env));
-}
-
-TEST(EvalString, OneVariableUpperCase) {
- EvalString str;
- string err;
- EXPECT_TRUE(str.Parse("hi $VaR", &err));
- EXPECT_EQ("", err);
- EXPECT_EQ("hi $VaR", str.unparsed());
- TestEnv env;
- EXPECT_EQ("hi ", str.Evaluate(&env));
- env.vars["VaR"] = "there";
- EXPECT_EQ("hi there", str.Evaluate(&env));
-}
-
-TEST(EvalString, Error) {
- EvalString str;
- string err;
- size_t err_index;
- EXPECT_FALSE(str.Parse("bad $", &err, &err_index));
- EXPECT_EQ("expected variable after $", err);
- EXPECT_EQ(5u, err_index);
-}
-TEST(EvalString, CurlyError) {
- EvalString str;
- string err;
- size_t err_index;
- EXPECT_FALSE(str.Parse("bad ${bar", &err, &err_index));
- EXPECT_EQ("expected closing curly after ${", err);
- EXPECT_EQ(9u, err_index);
-}
-
-TEST(EvalString, Curlies) {
- EvalString str;
- string err;
- EXPECT_TRUE(str.Parse("foo ${var}baz", &err));
- EXPECT_EQ("", err);
- TestEnv env;
- EXPECT_EQ("foo baz", str.Evaluate(&env));
- env.vars["var"] = "barbar";
- EXPECT_EQ("foo barbarbaz", str.Evaluate(&env));
-}
-
-TEST(EvalString, Dollars) {
- EvalString str;
- string err;
- EXPECT_TRUE(str.Parse("foo$$bar$bar", &err));
- ASSERT_EQ("", err);
- TestEnv env;
- env.vars["bar"] = "baz";
- EXPECT_EQ("foo$barbaz", str.Evaluate(&env));
-}
-
-} // namespace
const EvalString& description() const { return description_; }
const EvalString& depfile() const { return depfile_; }
- private:
+ // TODO: private:
+
// Allow the parsers to reach into this object and fill out its fields.
friend class ManifestParser;
- friend class ParserTest;
string name_;
--- /dev/null
+/* Generated by re2c 0.13.5 */
+// Copyright 2011 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "lexer.h"
+
+#include <stdio.h>
+
+#include "eval_env.h"
+
+bool Lexer::Error(const string& message, string* err) {
+ // Compute line/column.
+ int line = 1;
+ const char* context = input_.str_;
+ for (const char* p = input_.str_; p < last_token_; ++p) {
+ if (*p == '\n') {
+ ++line;
+ context = p + 1;
+ }
+ }
+ int col = last_token_ ? last_token_ - context : 0;
+
+ char buf[1024];
+ snprintf(buf, sizeof(buf), "%s:%d: ", filename_.AsString().c_str(), line);
+ *err = buf;
+ *err += message + "\n";
+
+ // Add some context to the message.
+ const int kTruncateColumn = 72;
+ if (col > 0 && col < kTruncateColumn) {
+ int len;
+ bool truncated = true;
+ for (len = 0; len < kTruncateColumn; ++len) {
+ if (context[len] == 0 || context[len] == '\n') {
+ truncated = false;
+ break;
+ }
+ }
+ *err += string(context, len);
+ if (truncated)
+ *err += "...";
+ *err += "\n";
+ *err += string(col, ' ');
+ *err += "^ near here\n";
+ }
+
+ return false;
+}
+
+Lexer::Lexer(const char* input) {
+ Start("input", input);
+}
+
+void Lexer::Start(StringPiece filename, StringPiece input) {
+ filename_ = filename;
+ input_ = input;
+ ofs_ = input_.str_;
+ last_token_ = NULL;
+}
+
+const char* Lexer::TokenName(Token t) {
+ switch (t) {
+ case ERROR: return "lexing error";
+ case BUILD: return "'build'";
+ case COLON: return "':'";
+ case DEFAULT: return "'default'";
+ case EQUALS: return "'='";
+ case IDENT: return "identifier";
+ case INCLUDE: return "'include'";
+ case INDENT: return "indent";
+ case NEWLINE: return "newline";
+ case PIPE2: return "'||'";
+ case PIPE: return "'|'";
+ case RULE: return "'rule'";
+ case SUBNINJA: return "'subninja'";
+ case TEOF: return "eof";
+ }
+ return NULL; // not reached
+}
+
+void Lexer::UnreadToken() {
+ ofs_ = last_token_;
+}
+
+Lexer::Token Lexer::ReadToken() {
+ const char* p = ofs_;
+ const char* q;
+ const char* start;
+ Lexer::Token token;
+ for (;;) {
+ start = p;
+
+{
+ char yych;
+ static const unsigned char yybm[] = {
+ 0, 128, 128, 128, 128, 128, 128, 128,
+ 128, 128, 0, 128, 128, 128, 128, 128,
+ 128, 128, 128, 128, 128, 128, 128, 128,
+ 128, 128, 128, 128, 128, 128, 128, 128,
+ 192, 128, 128, 128, 128, 128, 128, 128,
+ 128, 128, 128, 128, 128, 128, 160, 128,
+ 160, 160, 160, 160, 160, 160, 160, 160,
+ 160, 160, 128, 128, 128, 128, 128, 128,
+ 128, 160, 160, 160, 160, 160, 160, 160,
+ 160, 160, 160, 160, 160, 160, 160, 160,
+ 160, 160, 160, 160, 160, 160, 160, 160,
+ 160, 160, 160, 128, 128, 128, 128, 160,
+ 128, 160, 160, 160, 160, 160, 160, 160,
+ 160, 160, 160, 160, 160, 160, 160, 160,
+ 160, 160, 160, 160, 160, 160, 160, 160,
+ 160, 160, 160, 128, 128, 128, 128, 128,
+ 128, 128, 128, 128, 128, 128, 128, 128,
+ 128, 128, 128, 128, 128, 128, 128, 128,
+ 128, 128, 128, 128, 128, 128, 128, 128,
+ 128, 128, 128, 128, 128, 128, 128, 128,
+ 128, 128, 128, 128, 128, 128, 128, 128,
+ 128, 128, 128, 128, 128, 128, 128, 128,
+ 128, 128, 128, 128, 128, 128, 128, 128,
+ 128, 128, 128, 128, 128, 128, 128, 128,
+ 128, 128, 128, 128, 128, 128, 128, 128,
+ 128, 128, 128, 128, 128, 128, 128, 128,
+ 128, 128, 128, 128, 128, 128, 128, 128,
+ 128, 128, 128, 128, 128, 128, 128, 128,
+ 128, 128, 128, 128, 128, 128, 128, 128,
+ 128, 128, 128, 128, 128, 128, 128, 128,
+ 128, 128, 128, 128, 128, 128, 128, 128,
+ 128, 128, 128, 128, 128, 128, 128, 128,
+ };
+
+ yych = *p;
+ if (yych <= 'Z') {
+ if (yych <= '-') {
+ if (yych <= 0x1F) {
+ if (yych <= 0x00) goto yy21;
+ if (yych == '\n') goto yy4;
+ goto yy23;
+ } else {
+ if (yych <= ' ') goto yy6;
+ if (yych != '#') goto yy23;
+ }
+ } else {
+ if (yych <= ':') {
+ if (yych == '/') goto yy23;
+ if (yych <= '9') goto yy20;
+ goto yy14;
+ } else {
+ if (yych == '=') goto yy12;
+ if (yych <= '@') goto yy23;
+ goto yy20;
+ }
+ }
+ } else {
+ if (yych <= 'h') {
+ if (yych <= 'a') {
+ if (yych == '_') goto yy20;
+ if (yych <= '`') goto yy23;
+ goto yy20;
+ } else {
+ if (yych <= 'b') goto yy8;
+ if (yych == 'd') goto yy11;
+ goto yy20;
+ }
+ } else {
+ if (yych <= 's') {
+ if (yych <= 'i') goto yy18;
+ if (yych <= 'q') goto yy20;
+ if (yych <= 'r') goto yy10;
+ goto yy19;
+ } else {
+ if (yych <= 'z') goto yy20;
+ if (yych == '|') goto yy16;
+ goto yy23;
+ }
+ }
+ }
+ yych = *(q = ++p);
+ if (yych >= 0x01) goto yy62;
+yy3:
+ { token = ERROR; break; }
+yy4:
+ ++p;
+ { token = NEWLINE; break; }
+yy6:
+ ++p;
+ yych = *p;
+ goto yy60;
+yy7:
+ { token = INDENT; break; }
+yy8:
+ ++p;
+ if ((yych = *p) == 'u') goto yy54;
+ goto yy25;
+yy9:
+ { token = IDENT; break; }
+yy10:
+ yych = *++p;
+ if (yych == 'u') goto yy50;
+ goto yy25;
+yy11:
+ yych = *++p;
+ if (yych == 'e') goto yy43;
+ goto yy25;
+yy12:
+ ++p;
+ { token = EQUALS; break; }
+yy14:
+ ++p;
+ { token = COLON; break; }
+yy16:
+ ++p;
+ if ((yych = *p) == '|') goto yy41;
+ { token = PIPE; break; }
+yy18:
+ yych = *++p;
+ if (yych == 'n') goto yy34;
+ goto yy25;
+yy19:
+ yych = *++p;
+ if (yych == 'u') goto yy26;
+ goto yy25;
+yy20:
+ yych = *++p;
+ goto yy25;
+yy21:
+ ++p;
+ { token = TEOF; break; }
+yy23:
+ yych = *++p;
+ goto yy3;
+yy24:
+ ++p;
+ yych = *p;
+yy25:
+ if (yybm[0+yych] & 32) {
+ goto yy24;
+ }
+ goto yy9;
+yy26:
+ yych = *++p;
+ if (yych != 'b') goto yy25;
+ yych = *++p;
+ if (yych != 'n') goto yy25;
+ yych = *++p;
+ if (yych != 'i') goto yy25;
+ yych = *++p;
+ if (yych != 'n') goto yy25;
+ yych = *++p;
+ if (yych != 'j') goto yy25;
+ yych = *++p;
+ if (yych != 'a') goto yy25;
+ ++p;
+ if (yybm[0+(yych = *p)] & 32) {
+ goto yy24;
+ }
+ { token = SUBNINJA; break; }
+yy34:
+ yych = *++p;
+ if (yych != 'c') goto yy25;
+ yych = *++p;
+ if (yych != 'l') goto yy25;
+ yych = *++p;
+ if (yych != 'u') goto yy25;
+ yych = *++p;
+ if (yych != 'd') goto yy25;
+ yych = *++p;
+ if (yych != 'e') goto yy25;
+ ++p;
+ if (yybm[0+(yych = *p)] & 32) {
+ goto yy24;
+ }
+ { token = INCLUDE; break; }
+yy41:
+ ++p;
+ { token = PIPE2; break; }
+yy43:
+ yych = *++p;
+ if (yych != 'f') goto yy25;
+ yych = *++p;
+ if (yych != 'a') goto yy25;
+ yych = *++p;
+ if (yych != 'u') goto yy25;
+ yych = *++p;
+ if (yych != 'l') goto yy25;
+ yych = *++p;
+ if (yych != 't') goto yy25;
+ ++p;
+ if (yybm[0+(yych = *p)] & 32) {
+ goto yy24;
+ }
+ { token = DEFAULT; break; }
+yy50:
+ yych = *++p;
+ if (yych != 'l') goto yy25;
+ yych = *++p;
+ if (yych != 'e') goto yy25;
+ ++p;
+ if (yybm[0+(yych = *p)] & 32) {
+ goto yy24;
+ }
+ { token = RULE; break; }
+yy54:
+ yych = *++p;
+ if (yych != 'i') goto yy25;
+ yych = *++p;
+ if (yych != 'l') goto yy25;
+ yych = *++p;
+ if (yych != 'd') goto yy25;
+ ++p;
+ if (yybm[0+(yych = *p)] & 32) {
+ goto yy24;
+ }
+ { token = BUILD; break; }
+yy59:
+ ++p;
+ yych = *p;
+yy60:
+ if (yybm[0+yych] & 64) {
+ goto yy59;
+ }
+ goto yy7;
+yy61:
+ ++p;
+ yych = *p;
+yy62:
+ if (yybm[0+yych] & 128) {
+ goto yy61;
+ }
+ if (yych >= 0x01) goto yy64;
+ p = q;
+ goto yy3;
+yy64:
+ ++p;
+ { continue; }
+}
+
+ }
+
+ last_token_ = start;
+ ofs_ = p;
+ if (token != NEWLINE && token != TEOF)
+ EatWhitespace();
+ return token;
+}
+
+bool Lexer::PeekToken(Token token) {
+ Token t = ReadToken();
+ if (t == token)
+ return true;
+ UnreadToken();
+ return false;
+}
+
+void Lexer::EatWhitespace() {
+ const char* p = ofs_;
+ for (;;) {
+ ofs_ = p;
+
+{
+ char yych;
+ static const unsigned char yybm[] = {
+ 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0,
+ 128, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0,
+ };
+ yych = *p;
+ if (yych <= ' ') {
+ if (yych <= 0x00) goto yy72;
+ if (yych <= 0x1F) goto yy74;
+ } else {
+ if (yych == '$') goto yy70;
+ goto yy74;
+ }
+ ++p;
+ yych = *p;
+ goto yy78;
+yy69:
+ { continue; }
+yy70:
+ ++p;
+ if ((yych = *p) == '\n') goto yy75;
+yy71:
+ { break; }
+yy72:
+ ++p;
+ { break; }
+yy74:
+ yych = *++p;
+ goto yy71;
+yy75:
+ ++p;
+ { continue; }
+yy77:
+ ++p;
+ yych = *p;
+yy78:
+ if (yybm[0+yych] & 128) {
+ goto yy77;
+ }
+ goto yy69;
+}
+
+ }
+}
+
+bool Lexer::ReadIdent(string* out) {
+ const char* p = ofs_;
+ for (;;) {
+ const char* start = p;
+
+{
+ char yych;
+ static const unsigned char yybm[] = {
+ 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 128, 0,
+ 128, 128, 128, 128, 128, 128, 128, 128,
+ 128, 128, 0, 0, 0, 0, 0, 0,
+ 0, 128, 128, 128, 128, 128, 128, 128,
+ 128, 128, 128, 128, 128, 128, 128, 128,
+ 128, 128, 128, 128, 128, 128, 128, 128,
+ 128, 128, 128, 0, 0, 0, 0, 128,
+ 0, 128, 128, 128, 128, 128, 128, 128,
+ 128, 128, 128, 128, 128, 128, 128, 128,
+ 128, 128, 128, 128, 128, 128, 128, 128,
+ 128, 128, 128, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0,
+ };
+ yych = *p;
+ if (yych <= '@') {
+ if (yych <= '.') {
+ if (yych <= '-') goto yy83;
+ } else {
+ if (yych <= '/') goto yy83;
+ if (yych >= ':') goto yy83;
+ }
+ } else {
+ if (yych <= '_') {
+ if (yych <= 'Z') goto yy81;
+ if (yych <= '^') goto yy83;
+ } else {
+ if (yych <= '`') goto yy83;
+ if (yych >= '{') goto yy83;
+ }
+ }
+yy81:
+ ++p;
+ yych = *p;
+ goto yy86;
+yy82:
+ {
+ out->assign(start, p - start);
+ break;
+ }
+yy83:
+ ++p;
+ { return false; }
+yy85:
+ ++p;
+ yych = *p;
+yy86:
+ if (yybm[0+yych] & 128) {
+ goto yy85;
+ }
+ goto yy82;
+}
+
+ }
+ ofs_ = p;
+ EatWhitespace();
+ return true;
+}
+
+bool Lexer::ReadEvalString(EvalString* eval, bool path, string* err) {
+ const char* p = ofs_;
+ const char* q;
+ const char* start;
+ for (;;) {
+ start = p;
+
+{
+ char yych;
+ static const unsigned char yybm[] = {
+ 0, 128, 128, 128, 128, 128, 128, 128,
+ 128, 128, 0, 128, 128, 128, 128, 128,
+ 128, 128, 128, 128, 128, 128, 128, 128,
+ 128, 128, 128, 128, 128, 128, 128, 128,
+ 16, 128, 128, 128, 0, 128, 128, 128,
+ 128, 128, 128, 128, 128, 128, 160, 128,
+ 224, 224, 224, 224, 224, 224, 224, 224,
+ 224, 224, 0, 128, 128, 128, 128, 128,
+ 128, 224, 224, 224, 224, 224, 224, 224,
+ 224, 224, 224, 224, 224, 224, 224, 224,
+ 224, 224, 224, 224, 224, 224, 224, 224,
+ 224, 224, 224, 128, 128, 128, 128, 224,
+ 128, 224, 224, 224, 224, 224, 224, 224,
+ 224, 224, 224, 224, 224, 224, 224, 224,
+ 224, 224, 224, 224, 224, 224, 224, 224,
+ 224, 224, 224, 128, 0, 128, 128, 128,
+ 128, 128, 128, 128, 128, 128, 128, 128,
+ 128, 128, 128, 128, 128, 128, 128, 128,
+ 128, 128, 128, 128, 128, 128, 128, 128,
+ 128, 128, 128, 128, 128, 128, 128, 128,
+ 128, 128, 128, 128, 128, 128, 128, 128,
+ 128, 128, 128, 128, 128, 128, 128, 128,
+ 128, 128, 128, 128, 128, 128, 128, 128,
+ 128, 128, 128, 128, 128, 128, 128, 128,
+ 128, 128, 128, 128, 128, 128, 128, 128,
+ 128, 128, 128, 128, 128, 128, 128, 128,
+ 128, 128, 128, 128, 128, 128, 128, 128,
+ 128, 128, 128, 128, 128, 128, 128, 128,
+ 128, 128, 128, 128, 128, 128, 128, 128,
+ 128, 128, 128, 128, 128, 128, 128, 128,
+ 128, 128, 128, 128, 128, 128, 128, 128,
+ 128, 128, 128, 128, 128, 128, 128, 128,
+ };
+ yych = *p;
+ if (yych <= '#') {
+ if (yych <= '\n') {
+ if (yych <= 0x00) goto yy95;
+ if (yych >= '\n') goto yy91;
+ } else {
+ if (yych == ' ') goto yy91;
+ }
+ } else {
+ if (yych <= ':') {
+ if (yych <= '$') goto yy93;
+ if (yych >= ':') goto yy91;
+ } else {
+ if (yych == '|') goto yy91;
+ }
+ }
+ ++p;
+ yych = *p;
+ goto yy117;
+yy90:
+ {
+ eval->Add(EvalString::RAW, StringPiece(start, p - start));
+ continue;
+ }
+yy91:
+ ++p;
+ {
+ if (path) {
+ p = start;
+ break;
+ } else {
+ if (*start == '\n')
+ break;
+ eval->Add(EvalString::RAW, StringPiece(start, 1));
+ continue;
+ }
+ }
+yy93:
+ ++p;
+ if ((yych = *p) <= '9') {
+ if (yych <= ' ') {
+ if (yych == '\n') goto yy106;
+ if (yych <= 0x1F) goto yy97;
+ goto yy99;
+ } else {
+ if (yych == '$') goto yy101;
+ if (yych <= '/') goto yy97;
+ goto yy103;
+ }
+ } else {
+ if (yych <= '_') {
+ if (yych <= '@') goto yy97;
+ if (yych <= 'Z') goto yy103;
+ if (yych <= '^') goto yy97;
+ goto yy103;
+ } else {
+ if (yych <= '`') goto yy97;
+ if (yych <= 'z') goto yy103;
+ if (yych <= '{') goto yy105;
+ goto yy97;
+ }
+ }
+ {
+ last_token_ = start;
+ return Error("lexing error", err);
+ }
+yy95:
+ ++p;
+ {
+ last_token_ = start;
+ return Error("unexpected EOF", err);
+ }
+yy97:
+ ++p;
+yy98:
+ {
+ last_token_ = start;
+ return Error("bad $-escape (literal $ must be written as $$)", err);
+ }
+yy99:
+ ++p;
+ {
+ eval->Add(EvalString::RAW, StringPiece(" ", 1));
+ continue;
+ }
+yy101:
+ ++p;
+ {
+ eval->Add(EvalString::RAW, StringPiece("$", 1));
+ continue;
+ }
+yy103:
+ ++p;
+ yych = *p;
+ goto yy115;
+yy104:
+ {
+ eval->Add(EvalString::SPECIAL, StringPiece(start + 1, p - start - 1));
+ continue;
+ }
+yy105:
+ yych = *(q = ++p);
+ if (yybm[0+yych] & 32) {
+ goto yy109;
+ }
+ goto yy98;
+yy106:
+ ++p;
+ yych = *p;
+ if (yybm[0+yych] & 16) {
+ goto yy106;
+ }
+ {
+ continue;
+ }
+yy109:
+ ++p;
+ yych = *p;
+ if (yybm[0+yych] & 32) {
+ goto yy109;
+ }
+ if (yych == '}') goto yy112;
+ p = q;
+ goto yy98;
+yy112:
+ ++p;
+ {
+ eval->Add(EvalString::SPECIAL, StringPiece(start + 2, p - start - 3));
+ continue;
+ }
+yy114:
+ ++p;
+ yych = *p;
+yy115:
+ if (yybm[0+yych] & 64) {
+ goto yy114;
+ }
+ goto yy104;
+yy116:
+ ++p;
+ yych = *p;
+yy117:
+ if (yybm[0+yych] & 128) {
+ goto yy116;
+ }
+ goto yy90;
+}
+
+ }
+ last_token_ = start;
+ ofs_ = p;
+ if (path)
+ EatWhitespace();
+ // Non-path strings end in newlines, so there's no whitespace to eat.
+ return true;
+}
--- /dev/null
+// Copyright 2011 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "string_piece.h"
+
+struct EvalString;
+
+struct Lexer {
+ Lexer() {}
+ /// Helper ctor useful for tests.
+ explicit Lexer(const char* input);
+
+ enum Token {
+ ERROR,
+ BUILD,
+ COLON,
+ DEFAULT,
+ EQUALS,
+ IDENT,
+ INCLUDE,
+ INDENT,
+ NEWLINE,
+ PIPE,
+ PIPE2,
+ RULE,
+ SUBNINJA,
+ TEOF,
+ };
+
+ /// Return a human-readable form of a token, used in error messages.
+ static const char* TokenName(Token t);
+
+ /// Start parsing some input.
+ void Start(StringPiece filename, StringPiece input);
+
+ /// Read a Token from the Token enum.
+ Token ReadToken();
+
+ /// Rewind to the last read Token.
+ void UnreadToken();
+
+ /// If the next token is \a token, read it and return true.
+ bool PeekToken(Token token);
+
+ /// Read a simple identifier (a rule or variable name).
+ /// Returns false if a name can't be read.
+ bool ReadIdent(string* out);
+
+ /// Read a path (complete with $escapes).
+ /// Returns false only on error, returned path may be empty if a delimiter
+ /// (space, newline) is hit.
+ bool ReadPath(EvalString* path, string* err) {
+ return ReadEvalString(path, true, err);
+ }
+
+ /// Read the value side of a var = value line (complete with $escapes).
+ /// Returns false only on error.
+ bool ReadVarValue(EvalString* value, string* err) {
+ return ReadEvalString(value, false, err);
+ }
+
+ /// Construct an error message with context.
+ bool Error(const string& message, string* err);
+
+private:
+ /// Skip past whitespace (called after each read token/ident/etc.).
+ void EatWhitespace();
+
+ /// Read a $-escaped string.
+ bool ReadEvalString(EvalString* eval, bool path, string* err);
+
+ StringPiece filename_;
+ StringPiece input_;
+ const char* ofs_;
+ const char* last_token_;
+};
+
--- /dev/null
+// Copyright 2011 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "lexer.h"
+
+#include "eval_env.h"
+
+bool Lexer::Error(const string& message, string* err) {
+ // Compute line/column.
+ int line = 1;
+ const char* context = input_.str_;
+ for (const char* p = input_.str_; p < last_token_; ++p) {
+ if (*p == '\n') {
+ ++line;
+ context = p + 1;
+ }
+ }
+ int col = last_token_ ? last_token_ - context : 0;
+
+ char buf[1024];
+ snprintf(buf, sizeof(buf), "%s:%d: ", filename_.AsString().c_str(), line);
+ *err = buf;
+ *err += message + "\n";
+
+ // Add some context to the message.
+ const int kTruncateColumn = 72;
+ if (col > 0 && col < kTruncateColumn) {
+ int len;
+ bool truncated = true;
+ for (len = 0; len < kTruncateColumn; ++len) {
+ if (context[len] == 0 || context[len] == '\n') {
+ truncated = false;
+ break;
+ }
+ }
+ *err += string(context, len);
+ if (truncated)
+ *err += "...";
+ *err += "\n";
+ *err += string(col, ' ');
+ *err += "^ near here\n";
+ }
+
+ return false;
+}
+
+Lexer::Lexer(const char* input) {
+ Start("input", input);
+}
+
+void Lexer::Start(StringPiece filename, StringPiece input) {
+ filename_ = filename;
+ input_ = input;
+ ofs_ = input_.str_;
+ last_token_ = NULL;
+}
+
+const char* Lexer::TokenName(Token t) {
+ switch (t) {
+ case ERROR: return "lexing error";
+ case BUILD: return "'build'";
+ case COLON: return "':'";
+ case DEFAULT: return "'default'";
+ case EQUALS: return "'='";
+ case IDENT: return "identifier";
+ case INCLUDE: return "'include'";
+ case INDENT: return "indent";
+ case NEWLINE: return "newline";
+ case PIPE2: return "'||'";
+ case PIPE: return "'|'";
+ case RULE: return "'rule'";
+ case SUBNINJA: return "'subninja'";
+ case TEOF: return "eof";
+ }
+ return NULL; // not reached
+}
+
+void Lexer::UnreadToken() {
+ ofs_ = last_token_;
+}
+
+Lexer::Token Lexer::ReadToken() {
+ const char* p = ofs_;
+ const char* q;
+ const char* start;
+ Lexer::Token token;
+ for (;;) {
+ start = p;
+ /*!re2c
+ re2c:define:YYCTYPE = "char";
+ re2c:define:YYCURSOR = p;
+ re2c:define:YYMARKER = q;
+ re2c:yyfill:enable = 0;
+
+ nul = "\000";
+ simple_varname = [a-zA-Z0-9_]+;
+ varname = [a-zA-Z0-9_.]+;
+
+ "#"[^\000\n]*"\n" { continue; }
+ [\n] { token = NEWLINE; break; }
+ [ ]+ { token = INDENT; break; }
+ "build" { token = BUILD; break; }
+ "rule" { token = RULE; break; }
+ "default" { token = DEFAULT; break; }
+ "=" { token = EQUALS; break; }
+ ":" { token = COLON; break; }
+ "||" { token = PIPE2; break; }
+ "|" { token = PIPE; break; }
+ "include" { token = INCLUDE; break; }
+ "subninja" { token = SUBNINJA; break; }
+ varname { token = IDENT; break; }
+ nul { token = TEOF; break; }
+ [^] { token = ERROR; break; }
+ */
+ }
+
+ last_token_ = start;
+ ofs_ = p;
+ if (token != NEWLINE && token != TEOF)
+ EatWhitespace();
+ return token;
+}
+
+bool Lexer::PeekToken(Token token) {
+ Token t = ReadToken();
+ if (t == token)
+ return true;
+ UnreadToken();
+ return false;
+}
+
+void Lexer::EatWhitespace() {
+ const char* p = ofs_;
+ for (;;) {
+ ofs_ = p;
+ /*!re2c
+ [ ]+ { continue; }
+ "$\n" { continue; }
+ nul { break; }
+ [^] { break; }
+ */
+ }
+}
+
+bool Lexer::ReadIdent(string* out) {
+ const char* p = ofs_;
+ for (;;) {
+ const char* start = p;
+ /*!re2c
+ varname {
+ out->assign(start, p - start);
+ break;
+ }
+ [^] { return false; }
+ */
+ }
+ ofs_ = p;
+ EatWhitespace();
+ return true;
+}
+
+bool Lexer::ReadEvalString(EvalString* eval, bool path, string* err) {
+ const char* p = ofs_;
+ const char* q;
+ const char* start;
+ for (;;) {
+ start = p;
+ /*!re2c
+ [^$ :\n|\000]+ {
+ eval->Add(EvalString::RAW, StringPiece(start, p - start));
+ continue;
+ }
+ [ :|\n] {
+ if (path) {
+ p = start;
+ break;
+ } else {
+ if (*start == '\n')
+ break;
+ eval->Add(EvalString::RAW, StringPiece(start, 1));
+ continue;
+ }
+ }
+ "$$" {
+ eval->Add(EvalString::RAW, StringPiece("$", 1));
+ continue;
+ }
+ "$ " {
+ eval->Add(EvalString::RAW, StringPiece(" ", 1));
+ continue;
+ }
+ "$\n"[ ]* {
+ continue;
+ }
+ "${"varname"}" {
+ eval->Add(EvalString::SPECIAL, StringPiece(start + 2, p - start - 3));
+ continue;
+ }
+ "$"simple_varname {
+ eval->Add(EvalString::SPECIAL, StringPiece(start + 1, p - start - 1));
+ continue;
+ }
+ "$". {
+ last_token_ = start;
+ return Error("bad $-escape (literal $ must be written as $$)", err);
+ }
+ nul {
+ last_token_ = start;
+ return Error("unexpected EOF", err);
+ }
+ [^] {
+ last_token_ = start;
+ return Error("lexing error", err);
+ }
+ */
+ }
+ last_token_ = start;
+ ofs_ = p;
+ if (path)
+ EatWhitespace();
+ // Non-path strings end in newlines, so there's no whitespace to eat.
+ return true;
+}
--- /dev/null
+// Copyright 2011 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "lexer.h"
+
+#include <gtest/gtest.h>
+
+#include "eval_env.h"
+
+TEST(Lexer, ReadVarValue) {
+ Lexer lexer("plain text $var $VaR ${x}\n");
+ EvalString eval;
+ string err;
+ EXPECT_TRUE(lexer.ReadVarValue(&eval, &err));
+ EXPECT_EQ("", err);
+ EXPECT_EQ("[plain text ][$var][ ][$VaR][ ][$x]",
+ eval.Serialize());
+}
+
+TEST(Lexer, ReadEvalStringEscapes) {
+ Lexer lexer("$ $$ab $\ncde\n");
+ EvalString eval;
+ string err;
+ EXPECT_TRUE(lexer.ReadVarValue(&eval, &err));
+ EXPECT_EQ("", err);
+ EXPECT_EQ("[ $ab cde]",
+ eval.Serialize());
+}
+
+TEST(Lexer, ReadIdent) {
+ Lexer lexer("foo baR baz_123 blah.dots");
+ string ident;
+ EXPECT_TRUE(lexer.ReadIdent(&ident));
+ EXPECT_EQ("foo", ident);
+ EXPECT_TRUE(lexer.ReadIdent(&ident));
+ EXPECT_EQ("baR", ident);
+ EXPECT_TRUE(lexer.ReadIdent(&ident));
+ EXPECT_EQ("baz_123", ident);
+}
+
+TEST(Lexer, ReadIdentCurlies) {
+ // Verify that ReadIdent includes dots in the name,
+ // but in an expansion $bar.dots stops at the dot.
+ Lexer lexer("foo.dots $bar.dots ${bar.dots}\n");
+ string ident;
+ EXPECT_TRUE(lexer.ReadIdent(&ident));
+ EXPECT_EQ("foo.dots", ident);
+
+ EvalString eval;
+ string err;
+ EXPECT_TRUE(lexer.ReadVarValue(&eval, &err));
+ EXPECT_EQ("", err);
+ EXPECT_EQ("[$bar][.dots ][$bar.dots]",
+ eval.Serialize());
+}
+
+TEST(Lexer, Error) {
+ Lexer lexer("foo$\nbad $");
+ EvalString eval;
+ string err;
+ ASSERT_FALSE(lexer.ReadVarValue(&eval, &err));
+ EXPECT_EQ("input:2: bad $-escape (literal $ must be written as $$)\n"
+ "bad $\n"
+ " ^ near here\n"
+ , err);
+}
+
+TEST(Lexer, CommentEOF) {
+ // Verify we don't run off the end of the string when the EOF is
+ // mid-comment.
+ Lexer lexer("# foo");
+ Lexer::Token token = lexer.ReadToken();
+ EXPECT_EQ(Lexer::ERROR, token);
+}
int CmdRules(State* state, int argc, char* /* argv */[]) {
for (map<string, const Rule*>::iterator i = state->rules_.begin();
i != state->rules_.end(); ++i) {
- if (i->second->description().unparsed_.empty()) {
+ if (i->second->description().empty()) {
printf("%s\n", i->first.c_str());
} else {
printf("%s: %s\n",
i->first.c_str(),
- i->second->description().unparsed_.c_str());
+ // XXX I changed it such that we don't have an easy way
+ // to get the source text anymore, so this output is
+ // unsatisfactory. How useful is this command, anyway?
+ i->second->description().Serialize().c_str());
}
}
return 0;
ManifestParser parser(&state, &file_reader);
string err;
if (!parser.Load(input_file, &err)) {
- Error("loading '%s': %s", input_file, err.c_str());
+ // The pattern in Ninja for errors is to return a one-line string,
+ // but parse errors are special in that they are multiline with
+ // context. Just report it verbatim.
+ fprintf(stderr, "%s", err.c_str());
return 1;
}
#include "state.h"
#include "util.h"
-string Token::AsString() const {
- switch (type_) {
- case IDENT: return "'" + string(pos_, end_ - pos_) + "'";
- case UNKNOWN: return "unknown '" + string(pos_, end_ - pos_) + "'";
- case NEWLINE: return "newline";
- case EQUALS: return "'='";
- case COLON: return "':'";
- case PIPE: return "'|'";
- case PIPE2: return "'||'";
- case TEOF: return "eof";
- case INDENT: return "indenting in";
- case OUTDENT: return "indenting out";
- case NONE: break;
- }
- assert(false);
- return "";
-}
-
-bool Tokenizer::ErrorAt(const char* pos, const string& message, string* err) {
- // Re-scan the input, counting newlines so that we can compute the
- // correct position.
- int line = 1;
- const char* line_start = start_;
- for (const char* p = start_; p < pos; ++p) {
- if (*p == '\n') {
- ++line;
- line_start = p + 1;
- }
- }
- int col = pos - line_start + 1;
-
- char buf[1024];
- snprintf(buf, sizeof(buf),
- "line %d, col %d: %s", line, col, message.c_str());
- err->assign(buf);
- return false;
-}
-
-void Tokenizer::Start(const char* start, const char* end) {
- cur_line_ = cur_ = start_ = start;
- end_ = end;
-}
-
-bool Tokenizer::ErrorExpected(const string& expected, string* err) {
- return Error("expected " + expected + ", got " + token_.AsString(), err);
-}
-
-void Tokenizer::SkipWhitespace(bool newline) {
- if (token_.type_ == Token::NEWLINE && newline)
- Newline(NULL);
-
- while (cur_ < end_) {
- if (*cur_ == ' ') {
- ++cur_;
- } else if (newline && *cur_ == '\n') {
- Newline(NULL);
- } else if (*cur_ == '$' && cur_ + 1 < end_ && cur_[1] == '\n') {
- ++cur_; ++cur_;
- } else if (*cur_ == '#' && cur_ == cur_line_) {
- while (cur_ < end_ && *cur_ != '\n')
- ++cur_;
- if (cur_ < end_ && *cur_ == '\n')
- ++cur_;
- cur_line_ = cur_;
- } else {
- break;
- }
- }
-}
-
-bool Tokenizer::Newline(string* err) {
- if (!ExpectToken(Token::NEWLINE, err))
- return false;
-
- return true;
-}
-
-/// Return true if |c| is part of an identifier.
-static bool IsIdentChar(char c) {
- // This function shows up hot on profiles. Instead of the natural
- // 'if' statement, use a table as generated by this Python script:
- // import string
- // cs = set()
- // for c in string.ascii_letters + string.digits + r'+,-./\_$':
- // cs.add(ord(c))
- // for i in range(128):
- // if i in cs:
- // print '1,',
- // else:
- // print '0,',
- // if i % 16 == 15:
- // print
- static const bool kIdents[] = {
- 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1,
- 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0,
- 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
- 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 0, 0, 1,
- 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
- 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0,
- };
- return kIdents[(int)c];
-}
-
-bool Tokenizer::ExpectToken(Token::Type expected, string* err) {
- PeekToken();
- if (token_.type_ != expected)
- return ErrorExpected(Token(expected).AsString(), err);
- ConsumeToken();
- return true;
-}
-
-bool Tokenizer::ExpectIdent(const char* expected, string* err) {
- PeekToken();
- if (token_.type_ != Token::IDENT ||
- strncmp(token_.pos_, expected, token_.end_ - token_.pos_) != 0) {
- return ErrorExpected(string("'") + expected + "'", err);
- }
- ConsumeToken();
- return true;
-}
-
-bool Tokenizer::ReadIdent(StringPiece* out) {
- PeekToken();
- if (token_.type_ != Token::IDENT)
- return false;
- out->str_ = token_.pos_;
- out->len_ = token_.end_ - token_.pos_;
- ConsumeToken();
- return true;
-}
-
-bool Tokenizer::ReadIdent(string* out) {
- StringPiece token;
- if (!ReadIdent(&token))
- return false;
- out->assign(token.str_, token.len_);
- return true;
-}
-
-bool Tokenizer::ReadToNewline(string *text, string* err, size_t max_length) {
- // XXX token_.clear();
- while (cur_ < end_ && *cur_ != '\n') {
- if (*cur_ == '$') {
- // Might be a line continuation; peek ahead to check.
- if (cur_ + 1 >= end_)
- return Error("unexpected eof", err);
- if (*(cur_ + 1) == '\n') {
- // Let SkipWhitespace handle the continuation logic.
- SkipWhitespace();
- continue;
- }
-
- // Otherwise, just treat it like a normal character.
- text->push_back(*cur_);
- ++cur_;
- } else {
- text->push_back(*cur_);
- ++cur_;
- }
- if (text->size() >= max_length) {
- token_.pos_ = cur_;
- return false;
- }
- }
- return Newline(err);
-}
-
-Token::Type Tokenizer::PeekToken() {
- if (token_.type_ != Token::NONE)
- return token_.type_;
-
- token_.pos_ = cur_;
- if (cur_indent_ == -1) {
- cur_indent_ = cur_ - cur_line_;
- if (cur_indent_ != last_indent_) {
- if (cur_indent_ > last_indent_) {
- token_.type_ = Token::INDENT;
- } else if (cur_indent_ < last_indent_) {
- token_.type_ = Token::OUTDENT;
- }
- last_indent_ = cur_indent_;
- return token_.type_;
- }
- }
-
- if (cur_ >= end_) {
- token_.type_ = Token::TEOF;
- return token_.type_;
- }
-
- if (IsIdentChar(*cur_)) {
- while (cur_ < end_ && IsIdentChar(*cur_)) {
- if (*cur_ == '$' && cur_ + 1 < end_ && cur_[1] == ' ') {
- ++cur_;
- }
- ++cur_;
- }
- token_.end_ = cur_;
- token_.type_ = Token::IDENT;
- } else if (*cur_ == ':') {
- token_.type_ = Token::COLON;
- ++cur_;
- } else if (*cur_ == '=') {
- token_.type_ = Token::EQUALS;
- ++cur_;
- } else if (*cur_ == '|') {
- if (cur_ + 1 < end_ && cur_[1] == '|') {
- token_.type_ = Token::PIPE2;
- cur_ += 2;
- } else {
- token_.type_ = Token::PIPE;
- ++cur_;
- }
- } else if (*cur_ == '\n') {
- token_.type_ = Token::NEWLINE;
- ++cur_;
- cur_line_ = cur_;
- cur_indent_ = -1;
- }
-
- SkipWhitespace();
-
- if (token_.type_ == Token::NONE) {
- token_.type_ = Token::UNKNOWN;
- token_.end_ = cur_ + 1;
- }
-
- return token_.type_;
-}
-
-void Tokenizer::ConsumeToken() {
- token_.Clear();
-}
-
ManifestParser::ManifestParser(State* state, FileReader* file_reader)
: state_(state), file_reader_(file_reader) {
env_ = &state->bindings_;
string contents;
if (!file_reader_->ReadFile(filename, &contents, err))
return false;
- return Parse(contents, err);
+ contents.resize(contents.size() + 10);
+ return Parse(filename, contents, err);
}
-bool ManifestParser::Parse(const string& input, string* err) {
- tokenizer_.Start(input.data(), input.data() + input.size());
-
- tokenizer_.SkipWhitespace(true);
-
- while (tokenizer_.token().type_ != Token::TEOF) {
- switch (tokenizer_.PeekToken()) {
- case Token::IDENT: {
- const Token& token = tokenizer_.token();
- int len = token.end_ - token.pos_;
- if (len == 4 && memcmp(token.pos_, "rule", 4) == 0) {
- if (!ParseRule(err))
- return false;
- } else if (len == 5 && memcmp(token.pos_, "build", 5) == 0) {
- if (!ParseEdge(err))
- return false;
- } else if (len == 7 && memcmp(token.pos_, "default", 7) == 0) {
- if (!ParseDefaults(err))
- return false;
- } else if ((len == 7 && memcmp(token.pos_, "include", 7) == 0) ||
- (len == 8 && memcmp(token.pos_, "subninja", 8) == 0)) {
- if (!ParseFileInclude(err))
- return false;
- } else {
- string name, value;
- if (!ParseLet(&name, &value, err))
- return false;
- env_->AddBinding(name, value);
- }
- break;
- }
- case Token::TEOF:
- continue;
- default:
- return tokenizer_.Error("unhandled " + tokenizer_.token().AsString(), err);
+bool ManifestParser::Parse(const string& filename, const string& input,
+ string* err) {
+ lexer_.Start(filename, input);
+
+ for (;;) {
+ Lexer::Token token = lexer_.ReadToken();
+ switch (token) {
+ case Lexer::BUILD:
+ if (!ParseEdge(err))
+ return false;
+ break;
+ case Lexer::RULE:
+ if (!ParseRule(err))
+ return false;
+ break;
+ case Lexer::DEFAULT:
+ if (!ParseDefault(err))
+ return false;
+ break;
+ case Lexer::IDENT: {
+ lexer_.UnreadToken();
+ string name;
+ EvalString value;
+ if (!ParseLet(&name, &value, err))
+ return false;
+ env_->AddBinding(name, value.Evaluate(env_));
+ break;
+ }
+ case Lexer::INCLUDE:
+ if (!ParseFileInclude(false, err))
+ return false;
+ break;
+ case Lexer::SUBNINJA:
+ if (!ParseFileInclude(true, err))
+ return false;
+ break;
+ case Lexer::ERROR:
+ return lexer_.Error("lexing error", err);
+ case Lexer::TEOF:
+ return true;
+ case Lexer::NEWLINE:
+ break;
+ default:
+ return lexer_.Error(string("unexpected") + Lexer::TokenName(token),
+ err);
}
- tokenizer_.SkipWhitespace(true);
}
-
- return true;
+ return false; // not reached
}
bool ManifestParser::ParseRule(string* err) {
- if (!tokenizer_.ExpectIdent("rule", err))
- return false;
string name;
- if (!tokenizer_.ReadIdent(&name))
- return tokenizer_.ErrorExpected("rule name", err);
- if (!tokenizer_.Newline(err))
+ if (!lexer_.ReadIdent(&name))
+ return lexer_.Error("expected rule name", err);
+
+ if (!ExpectToken(Lexer::NEWLINE, err))
return false;
if (state_->LookupRule(name) != NULL) {
Rule* rule = new Rule(name); // XXX scoped_ptr
- if (tokenizer_.PeekToken() == Token::INDENT) {
- tokenizer_.ConsumeToken();
-
- while (tokenizer_.PeekToken() != Token::OUTDENT) {
- const char* let_loc = tokenizer_.token_.pos_;
-
- string key;
- if (!ParseLetKey(&key, err))
- return false;
+ while (lexer_.PeekToken(Lexer::INDENT)) {
+ string key;
+ EvalString value;
+ if (!ParseLet(&key, &value, err))
+ return false;
- EvalString* eval_target = NULL;
- if (key == "command") {
- eval_target = &rule->command_;
- } else if (key == "depfile") {
- eval_target = &rule->depfile_;
- } else if (key == "description") {
- eval_target = &rule->description_;
- } else if (key == "generator") {
- rule->generator_ = true;
- string dummy;
- if (!tokenizer_.ReadToNewline(&dummy, err))
- return false;
- continue;
- } else if (key == "restat") {
- rule->restat_ = true;
- string dummy;
- if (!tokenizer_.ReadToNewline(&dummy, err))
- return false;
- continue;
- } else {
- // Die on other keyvals for now; revisit if we want to add a
- // scope here.
- return tokenizer_.ErrorAt(let_loc, "unexpected variable '" + key + "'",
- err);
- }
-
- if (!ParseLetValue(eval_target, err))
- return false;
+ if (key == "command") {
+ rule->command_ = value;
+ } else if (key == "depfile") {
+ rule->depfile_ = value;
+ } else if (key == "description") {
+ rule->description_ = value;
+ } else if (key == "generator") {
+ rule->generator_ = true;
+ } else if (key == "restat") {
+ rule->restat_ = true;
+ } else {
+ // Die on other keyvals for now; revisit if we want to add a
+ // scope here.
+ return lexer_.Error("unexpected variable '" + key + "'", err);
}
- tokenizer_.ConsumeToken();
}
- if (rule->command_.unparsed().empty())
- return tokenizer_.Error("expected 'command =' line", err);
+ if (rule->command_.empty())
+ return lexer_.Error("expected 'command =' line", err);
state_->AddRule(rule);
return true;
}
-bool ManifestParser::ParseLet(string* key, string* value, string* err) {
- if (!ParseLetKey(key, err))
+bool ManifestParser::ParseLet(string* key, EvalString* value, string* err) {
+ if (!lexer_.ReadIdent(key))
return false;
-
- EvalString eval;
- if (!ParseLetValue(&eval, err))
+ if (!ExpectToken(Lexer::EQUALS, err))
return false;
-
- *value = eval.Evaluate(env_);
-
- return true;
-}
-
-bool ManifestParser::ParseLetKey(string* key, string* err) {
- if (!tokenizer_.ReadIdent(key))
- return tokenizer_.ErrorExpected("variable name", err);
- if (!tokenizer_.ExpectToken(Token::EQUALS, err))
- return false;
- return true;
-}
-
-bool ManifestParser::ParseLetValue(EvalString* eval, string* err) {
- // Backup the tokenizer state prior to consuming the line, for reporting
- // the source location in case of a parse error later.
- Tokenizer tokenizer_backup = tokenizer_;
-
- // XXX should we tokenize here? it means we'll need to understand
- // command syntax, though...
- string value;
- if (!tokenizer_.ReadToNewline(&value, err))
+ if (!lexer_.ReadVarValue(value, err))
return false;
-
- string eval_err;
- size_t err_index;
- if (!eval->Parse(value, &eval_err, &err_index)) {
- value.clear();
- // Advance the saved tokenizer state up to the error index to report the
- // error at the correct source location.
- tokenizer_backup.ReadToNewline(&value, err, err_index);
- return tokenizer_backup.Error(eval_err, err);
- }
-
return true;
}
-bool ManifestParser::ParseDefaults(string* err) {
- if (!tokenizer_.ExpectIdent("default", err))
+bool ManifestParser::ParseDefault(string* err) {
+ EvalString eval;
+ if (!lexer_.ReadPath(&eval, err))
return false;
-
- string target;
- if (!tokenizer_.ReadIdent(&target))
- return tokenizer_.ErrorExpected("target name", err);
+ if (eval.empty())
+ return lexer_.Error("expected target name", err);
do {
- EvalString eval;
- string eval_err;
- if (!eval.Parse(target, &eval_err))
- return tokenizer_.Error(eval_err, err);
string path = eval.Evaluate(env_);
- if (!CanonicalizePath(&path, &eval_err))
- return tokenizer_.Error(eval_err, err);
- if (!state_->AddDefault(path, &eval_err))
- return tokenizer_.Error(eval_err, err);
- } while (tokenizer_.ReadIdent(&target));
+ string path_err;
+ if (!CanonicalizePath(&path, &path_err))
+ return lexer_.Error(path_err, err);
+ if (!state_->AddDefault(path, &path_err))
+ return lexer_.Error(path_err, err);
+
+ eval.Clear();
+ if (!lexer_.ReadPath(&eval, err))
+ return false;
+ } while (!eval.empty());
- if (!tokenizer_.Newline(err))
+ if (!ExpectToken(Lexer::NEWLINE, err))
return false;
return true;
}
bool ManifestParser::ParseEdge(string* err) {
- vector<string> ins, outs;
+ vector<EvalString> ins, outs;
- if (!tokenizer_.ExpectIdent("build", err))
- return false;
+ {
+ EvalString out;
+ if (!lexer_.ReadPath(&out, err))
+ return false;
+ if (out.empty())
+ return lexer_.Error("expected path", err);
- for (;;) {
- if (tokenizer_.PeekToken() == Token::COLON) {
- tokenizer_.ConsumeToken();
- break;
- }
+ do {
+ outs.push_back(out);
- string out;
- if (!tokenizer_.ReadIdent(&out))
- return tokenizer_.ErrorExpected("output file list", err);
- outs.push_back(out);
+ out.Clear();
+ if (!lexer_.ReadPath(&out, err))
+ return false;
+ } while (!out.empty());
}
- // XXX check outs not empty
+
+ if (!ExpectToken(Lexer::COLON, err))
+ return false;
string rule_name;
- if (!tokenizer_.ReadIdent(&rule_name))
- return tokenizer_.ErrorExpected("build command name", err);
+ if (!lexer_.ReadIdent(&rule_name))
+ return lexer_.Error("expected build command name", err);
const Rule* rule = state_->LookupRule(rule_name);
if (!rule)
- return tokenizer_.Error("unknown build rule '" + rule_name + "'", err);
+ return lexer_.Error("unknown build rule '" + rule_name + "'", err);
for (;;) {
- string in;
- if (!tokenizer_.ReadIdent(&in))
+ // XXX should we require one path here?
+ EvalString in;
+ if (!lexer_.ReadPath(&in, err))
+ return false;
+ if (in.empty())
break;
ins.push_back(in);
}
// Add all order-only deps, counting how many as we go.
int implicit = 0;
- if (tokenizer_.PeekToken() == Token::PIPE) {
- tokenizer_.ConsumeToken();
+ if (lexer_.PeekToken(Lexer::PIPE)) {
for (;;) {
- string in;
- if (!tokenizer_.ReadIdent(&in))
+ EvalString in;
+ if (!lexer_.ReadPath(&in, err))
+ return err;
+ if (in.empty())
break;
ins.push_back(in);
++implicit;
// Add all order-only deps, counting how many as we go.
int order_only = 0;
- if (tokenizer_.PeekToken() == Token::PIPE2) {
- tokenizer_.ConsumeToken();
+ if (lexer_.PeekToken(Lexer::PIPE2)) {
for (;;) {
- string in;
- if (!tokenizer_.ReadIdent(&in))
+ EvalString in;
+ if (!lexer_.ReadPath(&in, err))
+ return false;
+ if (in.empty())
break;
ins.push_back(in);
++order_only;
}
}
- if (!tokenizer_.Newline(err))
+ if (!ExpectToken(Lexer::NEWLINE, err))
return false;
// Default to using outer env.
BindingEnv* env = env_;
- // But use a nested env if there are variables in scope.
- if (tokenizer_.PeekToken() == Token::INDENT) {
- tokenizer_.ConsumeToken();
-
+ // But create and fill a nested env if there are variables in scope.
+ if (lexer_.PeekToken(Lexer::INDENT)) {
// XXX scoped_ptr to handle error case.
env = new BindingEnv;
env->parent_ = env_;
- while (tokenizer_.PeekToken() != Token::OUTDENT) {
- string key, val;
+ do {
+ string key;
+ EvalString val;
if (!ParseLet(&key, &val, err))
return false;
- env->AddBinding(key, val);
- }
- tokenizer_.ConsumeToken();
- }
-
- // Evaluate all variables in paths.
- // XXX: fast path skip the eval parse if there's no $ in the path?
- vector<string>* paths[2] = { &ins, &outs };
- for (int p = 0; p < 2; ++p) {
- for (vector<string>::iterator i = paths[p]->begin();
- i != paths[p]->end(); ++i) {
- EvalString eval;
- string eval_err;
- if (!eval.Parse(*i, &eval_err))
- return tokenizer_.Error(eval_err, err);
- string path = eval.Evaluate(env);
- if (!CanonicalizePath(&path, &eval_err))
- return tokenizer_.Error(eval_err, err);
- *i = path;
- }
+ env->AddBinding(key, val.Evaluate(env_));
+ } while (lexer_.PeekToken(Lexer::INDENT));
}
Edge* edge = state_->AddEdge(rule);
edge->env_ = env;
- for (vector<string>::iterator i = ins.begin(); i != ins.end(); ++i)
- state_->AddIn(edge, *i);
- for (vector<string>::iterator i = outs.begin(); i != outs.end(); ++i)
- state_->AddOut(edge, *i);
+ for (vector<EvalString>::iterator i = ins.begin(); i != ins.end(); ++i) {
+ string path = i->Evaluate(env);
+ string path_err;
+ if (!CanonicalizePath(&path, &path_err))
+ return lexer_.Error(path_err, err);
+ state_->AddIn(edge, path);
+ }
+ for (vector<EvalString>::iterator i = outs.begin(); i != outs.end(); ++i) {
+ string path = i->Evaluate(env);
+ string path_err;
+ if (!CanonicalizePath(&path, &path_err))
+ return lexer_.Error(path_err, err);
+ state_->AddOut(edge, path);
+ }
edge->implicit_deps_ = implicit;
edge->order_only_deps_ = order_only;
return true;
}
-bool ManifestParser::ParseFileInclude(string* err) {
- string type;
- tokenizer_.ReadIdent(&type);
-
- string path;
- if (!tokenizer_.ReadIdent(&path))
- return tokenizer_.ErrorExpected("path to ninja file", err);
+bool ManifestParser::ParseFileInclude(bool new_scope, string* err) {
+ // XXX this should use ReadPath!
+ EvalString eval;
+ if (!lexer_.ReadPath(&eval, err))
+ return false;
+ string path = eval.Evaluate(env_);
string contents;
string read_err;
if (!file_reader_->ReadFile(path, &contents, &read_err))
- return tokenizer_.Error("loading " + path + ": " + read_err, err);
+ return lexer_.Error("loading '" + path + "': " + read_err, err);
ManifestParser subparser(state_, file_reader_);
- if (type == "subninja") {
- // subninja: Construct a new scope for the new parser.
+ if (new_scope) {
subparser.env_ = new BindingEnv;
subparser.env_->parent_ = env_;
} else {
- // include: Reuse the current scope.
subparser.env_ = env_;
}
- string sub_err;
- if (!subparser.Parse(contents, &sub_err))
- return tokenizer_.Error("in '" + path + "': " + sub_err, err);
+ if (!subparser.Parse(path, contents, err))
+ return false;
- if (!tokenizer_.Newline(err))
+ if (!ExpectToken(Lexer::NEWLINE, err))
return false;
return true;
}
+
+bool ManifestParser::ExpectToken(Lexer::Token expected, string* err) {
+ Lexer::Token token = lexer_.ReadToken();
+ if (token != expected) {
+ string message = string("expected ") + Lexer::TokenName(expected);
+ message += string(", got ") + Lexer::TokenName(token);
+ return lexer_.Error(message, err);
+ }
+ return true;
+}
using namespace std;
+#include "lexer.h"
#include "string_piece.h"
struct BindingEnv;
-
-/// A single parsed token in an input stream.
-struct Token {
- enum Type {
- NONE,
- UNKNOWN,
- IDENT,
- NEWLINE,
- EQUALS,
- COLON,
- PIPE,
- PIPE2,
- INDENT,
- OUTDENT,
- TEOF
- };
- explicit Token(Type type) : type_(type) {}
-
- void Clear() { type_ = NONE; }
- string AsString() const;
-
- Type type_;
- const char* pos_;
- const char* end_;
-};
-
-/// Processes an input stream into Tokens.
-struct Tokenizer {
- Tokenizer() :
- token_(Token::NONE),
- last_indent_(0), cur_indent_(-1) {}
-
- void Start(const char* start, const char* end);
- /// Report an error at a particular location.
- bool ErrorAt(const char* pos, const string& message, string* err);
- /// Report an error with a location pointing at the current token.
- bool Error(const string& message, string* err) {
- return ErrorAt(token_.pos_, message, err);
- }
- /// Call Error() with "expected foo, got bar".
- bool ErrorExpected(const string& expected, string* err);
-
- const Token& token() const { return token_; }
-
- void SkipWhitespace(bool newline=false);
- bool Newline(string* err);
- bool ExpectToken(Token::Type expected, string* err);
- bool ExpectIdent(const char* expected, string* err);
- bool ReadIdent(StringPiece* out);
- bool ReadIdent(string* out);
- bool ReadToNewline(string* text, string* err,
- size_t max_length=std::numeric_limits<size_t>::max());
-
- Token::Type PeekToken();
- void ConsumeToken();
-
- const char* start_; /// Start of the input.
- const char* cur_; /// Current position within the input.
- const char* end_; /// End of the input.
-
- const char* cur_line_; /// Start of current line.
- Token token_;
- int last_indent_, cur_indent_;
-};
-
struct EvalString;
struct State;
ManifestParser(State* state, FileReader* file_reader);
+ /// Load and parse a file.
bool Load(const string& filename, string* err);
- bool Parse(const string& input, string* err);
+ /// Parse a text string of input. Used by tests.
+ bool ParseTest(const string& input, string* err) {
+ return Parse("input", input, err);
+ }
+
+private:
+ /// Parse a file, given its contents as a string.
+ bool Parse(const string& filename, const string& input, string* err);
+
+ /// Parse various statement types.
bool ParseRule(string* err);
- /// Parse a key=val statement, expanding $vars in the value with the
- /// current env.
- bool ParseLet(string* key, string* val, string* err);
+ bool ParseLet(string* key, EvalString* val, string* err);
bool ParseEdge(string* err);
- bool ParseDefaults(string* err);
+ bool ParseDefault(string* err);
/// Parse either a 'subninja' or 'include' line.
- bool ParseFileInclude(string* err);
-
+ bool ParseFileInclude(bool new_scope, string* err);
- /// Parse the "key=" half of a key=val statement.
- bool ParseLetKey(string* key, string* err);
- /// Parse the val half of a key=val statement, writing and parsing
- /// output into an EvalString (ready for expansion).
- bool ParseLetValue(EvalString* eval, string* err);
+ /// If the next token is not \a expected, produce an error string
+ /// saying "expectd foo, got bar".
+ bool ExpectToken(Lexer::Token expected, string* err);
State* state_;
BindingEnv* env_;
FileReader* file_reader_;
- Tokenizer tokenizer_;
+ Lexer lexer_;
};
#endif // NINJA_PARSERS_H_
void AssertParse(const char* input) {
ManifestParser parser(&state, this);
string err;
- ASSERT_TRUE(parser.Parse(input, &err)) << err;
+ ASSERT_TRUE(parser.ParseTest(input, &err)) << err;
ASSERT_EQ("", err);
}
ASSERT_EQ(3u, state.rules_.size());
const Rule* rule = state.rules_.begin()->second;
EXPECT_EQ("cat", rule->name());
- EXPECT_EQ("cat $in > $out", rule->command().unparsed());
+ EXPECT_EQ("[cat ][$in][ > ][$out]", rule->command().Serialize());
}
TEST_F(ParserTest, Variables) {
ASSERT_EQ(2u, state.rules_.size());
const Rule* rule = state.rules_.begin()->second;
EXPECT_EQ("link", rule->name());
- EXPECT_EQ("foo bar baz", rule->command().unparsed());
+ EXPECT_EQ("[foo bar baz]", rule->command().Serialize());
}
TEST_F(ParserTest, Backslash) {
TEST_F(ParserTest, EscapeSpaces) {
ASSERT_NO_FATAL_FAILURE(AssertParse(
-"rule has$ spaces\n"
+"rule spaces\n"
" command = something\n"
-"build foo$ bar: has$ spaces $$one two$$$ three\n"
+"build foo$ bar: spaces $$one two$$$ three\n"
));
EXPECT_TRUE(state.LookupNode("foo bar"));
EXPECT_EQ(state.edges_[0]->outputs_[0]->path(), "foo bar");
{
ManifestParser parser(NULL, NULL);
string err;
- EXPECT_FALSE(parser.Parse("foobar", &err));
- EXPECT_EQ("line 1, col 7: expected '=', got eof", err);
+ EXPECT_FALSE(parser.ParseTest("foobar", &err));
+ EXPECT_EQ("input:1: expected '=', got eof\n"
+ "foobar\n"
+ " ^ near here\n"
+ , err);
}
{
ManifestParser parser(NULL, NULL);
string err;
- EXPECT_FALSE(parser.Parse("x 3", &err));
- EXPECT_EQ("line 1, col 3: expected '=', got '3'", err);
+ EXPECT_FALSE(parser.ParseTest("x 3", &err));
+ EXPECT_EQ("input:1: expected '=', got identifier\n"
+ "x 3\n"
+ " ^ near here\n"
+ , err);
}
{
ManifestParser parser(NULL, NULL);
string err;
- EXPECT_FALSE(parser.Parse("x = 3", &err));
- EXPECT_EQ("line 1, col 6: expected newline, got eof", err);
+ EXPECT_FALSE(parser.ParseTest("x = 3", &err));
+ EXPECT_EQ("input:1: unexpected EOF\n"
+ "x = 3\n"
+ " ^ near here\n"
+ , err);
}
{
State state;
ManifestParser parser(&state, NULL);
string err;
- EXPECT_FALSE(parser.Parse("x = 3\ny 2", &err));
- EXPECT_EQ("line 2, col 3: expected '=', got '2'", err);
+ EXPECT_FALSE(parser.ParseTest("x = 3\ny 2", &err));
+ EXPECT_EQ("input:2: expected '=', got identifier\n"
+ "y 2\n"
+ " ^ near here\n"
+ , err);
}
{
State state;
ManifestParser parser(&state, NULL);
string err;
- EXPECT_FALSE(parser.Parse("x = $", &err));
- EXPECT_EQ("line 1, col 3: unexpected eof", err);
+ EXPECT_FALSE(parser.ParseTest("x = $", &err));
+ EXPECT_EQ("input:1: bad $-escape (literal $ must be written as $$)\n"
+ "x = $\n"
+ " ^ near here\n"
+ , err);
}
{
State state;
ManifestParser parser(&state, NULL);
string err;
- EXPECT_FALSE(parser.Parse("x = $\n $[\n", &err));
- EXPECT_EQ("line 2, col 3: expected variable after $", err);
+ EXPECT_FALSE(parser.ParseTest("x = $\n $[\n", &err));
+ EXPECT_EQ("input:2: bad $-escape (literal $ must be written as $$)\n"
+ " $[\n"
+ " ^ near here\n"
+ , err);
}
{
State state;
ManifestParser parser(&state, NULL);
string err;
- EXPECT_FALSE(parser.Parse("x = a$\n b$\n $\n", &err));
- EXPECT_EQ("line 4, col 1: expected newline, got eof", err);
+ EXPECT_FALSE(parser.ParseTest("x = a$\n b$\n $\n", &err));
+ EXPECT_EQ("input:4: unexpected EOF\n"
+ , err);
}
{
State state;
ManifestParser parser(&state, NULL);
string err;
- EXPECT_FALSE(parser.Parse("build x: y z\n", &err));
- EXPECT_EQ("line 1, col 10: unknown build rule 'y'", err);
+ EXPECT_FALSE(parser.ParseTest("build x: y z\n", &err));
+ EXPECT_EQ("input:1: unknown build rule 'y'\n"
+ "build x: y z\n"
+ " ^ near here\n"
+ , err);
}
{
State state;
ManifestParser parser(&state, NULL);
string err;
- EXPECT_FALSE(parser.Parse("build x:: y z\n", &err));
- EXPECT_EQ("line 1, col 9: expected build command name, got ':'", err);
+ EXPECT_FALSE(parser.ParseTest("build x:: y z\n", &err));
+ EXPECT_EQ("input:1: expected build command name\n"
+ "build x:: y z\n"
+ " ^ near here\n"
+ , err);
}
{
State state;
ManifestParser parser(&state, NULL);
string err;
- EXPECT_FALSE(parser.Parse("rule cat\n command = cat ok\n"
- "build x: cat $\n :\n",
- &err));
- EXPECT_EQ("line 4, col 2: expected newline, got ':'", err);
+ EXPECT_FALSE(parser.ParseTest("rule cat\n command = cat ok\n"
+ "build x: cat $\n :\n",
+ &err));
+ EXPECT_EQ("input:4: expected newline, got ':'\n"
+ " :\n"
+ " ^ near here\n"
+ , err);
}
{
State state;
ManifestParser parser(&state, NULL);
string err;
- EXPECT_FALSE(parser.Parse("rule cat\n",
- &err));
- EXPECT_EQ("line 2, col 1: expected 'command =' line", err);
+ EXPECT_FALSE(parser.ParseTest("rule cat\n",
+ &err));
+ EXPECT_EQ("input:2: expected 'command =' line\n", err);
}
{
State state;
ManifestParser parser(&state, NULL);
string err;
- EXPECT_FALSE(parser.Parse("rule cat\n command = ${fafsd\n foo = bar\n",
- &err));
- EXPECT_EQ("line 2, col 20: expected closing curly after ${", err);
+ EXPECT_FALSE(parser.ParseTest("rule cat\n"
+ " command = ${fafsd\n"
+ "foo = bar\n",
+ &err));
+ EXPECT_EQ("input:2: bad $-escape (literal $ must be written as $$)\n"
+ " command = ${fafsd\n"
+ " ^ near here\n"
+ , err);
}
State state;
ManifestParser parser(&state, NULL);
string err;
- EXPECT_FALSE(parser.Parse("rule cat\n command = cat\nbuild $: cat foo\n",
- &err));
- // XXX EXPECT_EQ("line 3, col 7: expected variable after $", err);
- EXPECT_EQ("line 4, col 1: expected variable after $", err);
+ EXPECT_FALSE(parser.ParseTest("rule cat\n"
+ " command = cat\nbuild $: cat foo\n",
+ &err));
+ EXPECT_EQ("input:3: bad $-escape (literal $ must be written as $$)\n"
+ "build $: cat foo\n"
+ " ^ near here\n"
+ , err);
}
{
State state;
ManifestParser parser(&state, NULL);
string err;
- EXPECT_FALSE(parser.Parse("rule %foo\n",
- &err));
- EXPECT_EQ("line 1, col 6: expected rule name, got unknown '%'", err);
+ EXPECT_FALSE(parser.ParseTest("rule %foo\n",
+ &err));
+ EXPECT_EQ("input:1: expected rule name\n", err);
}
{
State state;
ManifestParser parser(&state, NULL);
string err;
- EXPECT_FALSE(parser.Parse("rule cc\n command = foo\n othervar = bar\n",
- &err));
- EXPECT_EQ("line 3, col 3: unexpected variable 'othervar'", err);
+ EXPECT_FALSE(parser.ParseTest("rule cc\n"
+ " command = foo\n"
+ " othervar = bar\n",
+ &err));
+ EXPECT_EQ("input:3: unexpected variable 'othervar'\n"
+ " othervar = bar\n"
+ " ^ near here\n"
+ , err);
}
{
State state;
ManifestParser parser(&state, NULL);
string err;
- EXPECT_FALSE(parser.Parse("rule cc\n command = foo\n"
- "build $: cc bar.cc\n",
- &err));
- EXPECT_EQ("line 4, col 1: expected variable after $", err);
+ EXPECT_FALSE(parser.ParseTest("rule cc\n command = foo\n"
+ "build $: cc bar.cc\n",
+ &err));
+ EXPECT_EQ("input:3: bad $-escape (literal $ must be written as $$)\n"
+ "build $: cc bar.cc\n"
+ " ^ near here\n"
+ , err);
}
{
State state;
ManifestParser parser(&state, NULL);
string err;
- EXPECT_FALSE(parser.Parse("default\n",
- &err));
- EXPECT_EQ("line 1, col 8: expected target name, got newline", err);
+ EXPECT_FALSE(parser.ParseTest("default\n",
+ &err));
+ EXPECT_EQ("input:1: expected target name\n"
+ "default\n"
+ " ^ near here\n"
+ , err);
}
{
State state;
ManifestParser parser(&state, NULL);
string err;
- EXPECT_FALSE(parser.Parse("default nonexistent\n",
- &err));
- EXPECT_EQ("line 1, col 9: unknown target 'nonexistent'", err);
+ EXPECT_FALSE(parser.ParseTest("default nonexistent\n",
+ &err));
+ EXPECT_EQ("input:1: unknown target 'nonexistent'\n"
+ "default nonexistent\n"
+ " ^ near here\n"
+ , err);
}
{
State state;
ManifestParser parser(&state, NULL);
string err;
- EXPECT_FALSE(parser.Parse("rule r\n command = r\n"
- "build b: r\n"
- "default b:\n",
- &err));
- EXPECT_EQ("line 4, col 10: expected newline, got ':'", err);
+ EXPECT_FALSE(parser.ParseTest("rule r\n command = r\n"
+ "build b: r\n"
+ "default b:\n",
+ &err));
+ EXPECT_EQ("input:4: expected newline, got ':'\n"
+ "default b:\n"
+ " ^ near here\n"
+ , err);
}
{
State state;
ManifestParser parser(&state, NULL);
string err;
- EXPECT_FALSE(parser.Parse("default $a\n", &err));
- EXPECT_EQ("line 1, col 9: empty path", err);
+ EXPECT_FALSE(parser.ParseTest("default $a\n", &err));
+ EXPECT_EQ("input:1: empty path\n"
+ "default $a\n"
+ " ^ near here\n"
+ , err);
}
{
State state;
ManifestParser parser(&state, NULL);
string err;
- EXPECT_FALSE(parser.Parse("rule r\n"
- " command = r\n"
- "build $a: r $c\n", &err));
+ EXPECT_FALSE(parser.ParseTest("rule r\n"
+ " command = r\n"
+ "build $a: r $c\n", &err));
// XXX the line number is wrong; we should evaluate paths in ParseEdge
// as we see them, not after we've read them all!
- EXPECT_EQ("line 4, col 1: empty path", err);
+ EXPECT_EQ("input:4: empty path\n", err);
}
}
State state;
ManifestParser parser(&state, NULL);
string err;
- EXPECT_TRUE(parser.Parse("rule cc\n command = foo\n depfile = bar\n"
- "build a.o b.o: cc c.cc\n",
- &err));
+ EXPECT_TRUE(parser.ParseTest("rule cc\n command = foo\n depfile = bar\n"
+ "build a.o b.o: cc c.cc\n",
+ &err));
EXPECT_EQ("", err);
}
TEST_F(ParserTest, MissingSubNinja) {
ManifestParser parser(&state, this);
string err;
- EXPECT_FALSE(parser.Parse("subninja foo.ninja\n", &err));
- EXPECT_EQ("line 1, col 10: loading foo.ninja: No such file or directory",
- err);
+ EXPECT_FALSE(parser.ParseTest("subninja foo.ninja\n", &err));
+ EXPECT_EQ("input:1: loading 'foo.ninja': No such file or directory\n"
+ "subninja foo.ninja\n"
+ " ^ near here\n"
+ , err);
}
TEST_F(ParserTest, Include) {
TEST_F(ParserTest, Implicit) {
ASSERT_NO_FATAL_FAILURE(AssertParse(
-"rule cat\n command = cat $in > $out\n"
+"rule cat\n"
+" command = cat $in > $out\n"
"build foo: cat bar | baz\n"));
Edge* edge = state.LookupNode("foo")->in_edge();
TEST(State, Basic) {
State state;
+
Rule* rule = new Rule("cat");
- string err;
- EXPECT_TRUE(rule->command().Parse("cat $in > $out", &err));
- ASSERT_EQ("", err);
+ rule->command_.Add(EvalString::RAW, "cat ");
+ rule->command_.Add(EvalString::SPECIAL, "in");
+ rule->command_.Add(EvalString::RAW, " > ");
+ rule->command_.Add(EvalString::SPECIAL, "out");
state.AddRule(rule);
+
Edge* edge = state.AddEdge(rule);
state.AddIn(edge, "in1");
state.AddIn(edge, "in2");
void AssertParse(State* state, const char* input) {
ManifestParser parser(state, NULL);
string err;
- ASSERT_TRUE(parser.Parse(input, &err)) << err;
+ ASSERT_TRUE(parser.ParseTest(input, &err)) << err;
ASSERT_EQ("", err);
}
projects / platform / upstream / ninja.git / commitdiff