if (S.empty())
return Ret;
- // Quoted token
+ // Quoted token.
if (S.startswith("\"")) {
size_t E = S.find("\"", 1);
if (E == StringRef::npos) {
continue;
}
- // Unquoted token
+ // Unquoted token. This is more relaxed than tokens in C-like language,
+ // so that you can write "file-name.cpp" as one bare token, for example.
size_t Pos = S.find_first_not_of(
"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"
"0123456789_.$/\\~=+[]*?-:!<>");
+
// A character that cannot start a word (which is usually a
// punctuation) forms a single character token.
if (Pos == 0)