1 // Copyright 2011 The Go Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style
3 // license that can be found in the LICENSE file.
14 // item represents a token or text string returned from the scanner.
20 func (i item) String() string {
22 case i.typ == itemEOF:
24 case i.typ == itemError:
26 case i.typ > itemKeyword:
27 return fmt.Sprintf("<%s>", i.val)
29 return fmt.Sprintf("%.10q...", i.val)
31 return fmt.Sprintf("%q", i.val)
34 // itemType identifies the type of lex items.
38 itemError itemType = iota // error occurred; value is text of error
39 itemBool // boolean constant
40 itemChar // printable ASCII character; grab bag for comma etc.
41 itemCharConstant // character constant
42 itemComplex // complex constant (1+2i); imaginary is just a number
43 itemColonEquals // colon-equals (':=') introducing a declaration
45 itemField // alphanumeric identifier, starting with '.', possibly chained ('.x.y')
46 itemIdentifier // alphanumeric identifier
47 itemLeftDelim // left action delimiter
48 itemNumber // simple number, including imaginary
49 itemPipe // pipe symbol
50 itemRawString // raw quoted string (includes quotes)
51 itemRightDelim // right action delimiter
52 itemString // quoted string (includes quotes)
53 itemText // plain text
54 itemVariable // variable starting with '$', such as '$' or '$1' or '$hello'.
55 // Keywords appear after all the rest.
56 itemKeyword // used only to delimit the keywords
57 itemDot // the cursor, spelled '.'.
58 itemDefine // define keyword
59 itemElse // else keyword
60 itemEnd // end keyword
62 itemRange // range keyword
63 itemTemplate // template keyword
64 itemWith // with keyword
67 // Make the types prettyprint.
68 var itemName = map[itemType]string{
72 itemCharConstant: "charconst",
73 itemComplex: "complex",
74 itemColonEquals: ":=",
77 itemIdentifier: "identifier",
78 itemLeftDelim: "left delim",
81 itemRawString: "raw string",
82 itemRightDelim: "right delim",
84 itemVariable: "variable",
92 itemTemplate: "template",
96 func (i itemType) String() string {
99 return fmt.Sprintf("item%d", int(i))
104 var key = map[string]itemType{
106 "define": itemDefine,
111 "template": itemTemplate,
117 // stateFn represents the state of the scanner as a function that returns the next state.
118 type stateFn func(*lexer) stateFn
120 // lexer holds the state of the scanner.
122 name string // the name of the input; used only for error reports.
123 input string // the string being scanned.
124 leftDelim string // start of action.
125 rightDelim string // end of action.
126 state stateFn // the next lexing function to enter.
127 pos int // current position in the input.
128 start int // start position of this item.
129 width int // width of last rune read from input.
130 items chan item // channel of scanned items.
133 // next returns the next rune in the input.
134 func (l *lexer) next() (r rune) {
135 if l.pos >= len(l.input) {
139 r, l.width = utf8.DecodeRuneInString(l.input[l.pos:])
144 // peek returns but does not consume the next rune in the input.
145 func (l *lexer) peek() rune {
151 // backup steps back one rune. Can only be called once per call of next.
152 func (l *lexer) backup() {
156 // emit passes an item back to the client.
157 func (l *lexer) emit(t itemType) {
158 l.items <- item{t, l.input[l.start:l.pos]}
162 // ignore skips over the pending input before this point.
163 func (l *lexer) ignore() {
167 // accept consumes the next rune if it's from the valid set.
168 func (l *lexer) accept(valid string) bool {
169 if strings.IndexRune(valid, l.next()) >= 0 {
176 // acceptRun consumes a run of runes from the valid set.
177 func (l *lexer) acceptRun(valid string) {
178 for strings.IndexRune(valid, l.next()) >= 0 {
183 // lineNumber reports which line we're on. Doing it this way
184 // means we don't have to worry about peek double counting.
185 func (l *lexer) lineNumber() int {
186 return 1 + strings.Count(l.input[:l.pos], "\n")
189 // error returns an error token and terminates the scan by passing
190 // back a nil pointer that will be the next state, terminating l.nextItem.
191 func (l *lexer) errorf(format string, args ...interface{}) stateFn {
192 l.items <- item{itemError, fmt.Sprintf(format, args...)}
196 // nextItem returns the next item from the input.
197 func (l *lexer) nextItem() item {
200 case item := <-l.items:
209 // lex creates a new scanner for the input string.
210 func lex(name, input, left, right string) *lexer {
223 items: make(chan item, 2), // Two items of buffering is sufficient for all state functions
237 // lexText scans until an opening action delimiter, "{{".
238 func lexText(l *lexer) stateFn {
240 if strings.HasPrefix(l.input[l.pos:], l.leftDelim) {
250 // Correctly reached EOF.
258 // lexLeftDelim scans the left delimiter, which is known to be present.
259 func lexLeftDelim(l *lexer) stateFn {
260 if strings.HasPrefix(l.input[l.pos:], l.leftDelim+leftComment) {
263 l.pos += len(l.leftDelim)
264 l.emit(itemLeftDelim)
265 return lexInsideAction
268 // lexComment scans a comment. The left comment marker is known to be present.
269 func lexComment(l *lexer) stateFn {
270 i := strings.Index(l.input[l.pos:], rightComment+l.rightDelim)
272 return l.errorf("unclosed comment")
274 l.pos += i + len(rightComment) + len(l.rightDelim)
279 // lexRightDelim scans the right delimiter, which is known to be present.
280 func lexRightDelim(l *lexer) stateFn {
281 l.pos += len(l.rightDelim)
282 l.emit(itemRightDelim)
286 // lexInsideAction scans the elements inside action delimiters.
287 func lexInsideAction(l *lexer) stateFn {
288 // Either number, quoted string, or identifier.
289 // Spaces separate and are ignored.
290 // Pipe symbols separate and are emitted.
291 if strings.HasPrefix(l.input[l.pos:], l.rightDelim) {
294 switch r := l.next(); {
295 case r == eof || r == '\n':
296 return l.errorf("unclosed action")
301 return l.errorf("expected :=")
303 l.emit(itemColonEquals)
315 // special look-ahead for ".field" so we don't break l.backup().
316 if l.pos < len(l.input) {
318 if r < '0' || '9' < r {
319 return lexIdentifier // itemDot comes from the keyword table.
322 fallthrough // '.' can start a number.
323 case r == '+' || r == '-' || ('0' <= r && r <= '9'):
326 case isAlphaNumeric(r):
329 case r <= unicode.MaxASCII && unicode.IsPrint(r):
331 return lexInsideAction
333 return l.errorf("unrecognized character in action: %#U", r)
335 return lexInsideAction
338 // lexIdentifier scans an alphanumeric or field.
339 func lexIdentifier(l *lexer) stateFn {
342 switch r := l.next(); {
343 case isAlphaNumeric(r):
345 case r == '.' && (l.input[l.start] == '.' || l.input[l.start] == '$'):
346 // field chaining; absorb into one token.
349 word := l.input[l.start:l.pos]
350 if !l.atTerminator() {
351 return l.errorf("unexpected character %+U", r)
354 case key[word] > itemKeyword:
360 case word == "true", word == "false":
363 l.emit(itemIdentifier)
368 return lexInsideAction
371 // atTerminator reports whether the input is at valid termination character to
372 // appear after an identifier. Mostly to catch cases like "$x+2" not being
373 // acceptable without a space, in case we decide one day to implement
375 func (l *lexer) atTerminator() bool {
381 case eof, ',', '|', ':':
384 // Does r start the delimiter? This can be ambiguous (with delim=="//", $x/2 will
385 // succeed but should fail) but only in extremely rare cases caused by willfully
386 // bad choice of delimiter.
387 if rd, _ := utf8.DecodeRuneInString(l.rightDelim); rd == r {
393 // lexChar scans a character constant. The initial quote is already
394 // scanned. Syntax checking is done by the parse.
395 func lexChar(l *lexer) stateFn {
400 if r := l.next(); r != eof && r != '\n' {
405 return l.errorf("unterminated character constant")
410 l.emit(itemCharConstant)
411 return lexInsideAction
414 // lexNumber scans a number: decimal, octal, hex, float, or imaginary. This
415 // isn't a perfect number scanner - for instance it accepts "." and "0x0.2"
416 // and "089" - but when it's wrong the input is invalid and the parser (via
417 // strconv) will notice.
418 func lexNumber(l *lexer) stateFn {
420 return l.errorf("bad number syntax: %q", l.input[l.start:l.pos])
422 if sign := l.peek(); sign == '+' || sign == '-' {
423 // Complex: 1+2i. No spaces, must end in 'i'.
424 if !l.scanNumber() || l.input[l.pos-1] != 'i' {
425 return l.errorf("bad number syntax: %q", l.input[l.start:l.pos])
431 return lexInsideAction
434 func (l *lexer) scanNumber() bool {
435 // Optional leading sign.
438 digits := "0123456789"
439 if l.accept("0") && l.accept("xX") {
440 digits = "0123456789abcdefABCDEF"
448 l.acceptRun("0123456789")
452 // Next thing mustn't be alphanumeric.
453 if isAlphaNumeric(l.peek()) {
460 // lexQuote scans a quoted string.
461 func lexQuote(l *lexer) stateFn {
466 if r := l.next(); r != eof && r != '\n' {
471 return l.errorf("unterminated quoted string")
477 return lexInsideAction
480 // lexRawQuote scans a raw quoted string.
481 func lexRawQuote(l *lexer) stateFn {
486 return l.errorf("unterminated raw quoted string")
491 l.emit(itemRawString)
492 return lexInsideAction
495 // isSpace reports whether r is a space character.
496 func isSpace(r rune) bool {
498 case ' ', '\t', '\n', '\r':
504 // isAlphaNumeric reports whether r is an alphabetic, digit, or underscore.
505 func isAlphaNumeric(r rune) bool {
506 return r == '_' || unicode.IsLetter(r) || unicode.IsDigit(r)