return UnicodeWhitespaceChars.contains(Codepoint);
}
-static bool isAllowedIDChar(uint32_t C, const LangOptions &LangOpts) {
+static llvm::SmallString<5> codepointAsHexString(uint32_t C) {
+ llvm::SmallString<5> CharBuf;
+ llvm::raw_svector_ostream CharOS(CharBuf);
+ llvm::write_hex(CharOS, C, llvm::HexPrintStyle::Upper, 4);
+ return CharBuf;
+}
+
+// To mitigate https://github.com/llvm/llvm-project/issues/54732,
+// we allow "Mathematical Notation Characters" in identifiers.
+// This is a proposed profile that extends the XID_Start/XID_continue
+// with mathematical symbols, superscipts and subscripts digits
+// found in some production software.
+// https://www.unicode.org/L2/L2022/22230-math-profile.pdf
+static bool isMathematicalExtensionID(uint32_t C, const LangOptions &LangOpts,
+ bool IsStart, bool &IsExtension) {
+ static const llvm::sys::UnicodeCharSet MathStartChars(
+ MathematicalNotationProfileIDStartRanges);
+ static const llvm::sys::UnicodeCharSet MathContinueChars(
+ MathematicalNotationProfileIDContinueRanges);
+ if (MathStartChars.contains(C) ||
+ (!IsStart && MathContinueChars.contains(C))) {
+ IsExtension = true;
+ return true;
+ }
+ return false;
+}
+
+static bool isAllowedIDChar(uint32_t C, const LangOptions &LangOpts,
+ bool &IsExtension) {
if (LangOpts.AsmPreprocessor) {
return false;
} else if (LangOpts.DollarIdents && '$' == C) {
// '_' doesn't have the XID_Continue property but is allowed in C and C++.
static const llvm::sys::UnicodeCharSet XIDStartChars(XIDStartRanges);
static const llvm::sys::UnicodeCharSet XIDContinueChars(XIDContinueRanges);
- return C == '_' || XIDStartChars.contains(C) ||
- XIDContinueChars.contains(C);
+ if (C == '_' || XIDStartChars.contains(C) || XIDContinueChars.contains(C))
+ return true;
+ return isMathematicalExtensionID(C, LangOpts, /*IsStart=*/false,
+ IsExtension);
} else if (LangOpts.C11) {
static const llvm::sys::UnicodeCharSet C11AllowedIDChars(
C11AllowedIDCharRanges);
}
}
-static bool isAllowedInitiallyIDChar(uint32_t C, const LangOptions &LangOpts) {
+static bool isAllowedInitiallyIDChar(uint32_t C, const LangOptions &LangOpts,
+ bool &IsExtension) {
assert(C > 0x7F && "isAllowedInitiallyIDChar called with an ASCII codepoint");
+ IsExtension = false;
if (LangOpts.AsmPreprocessor) {
return false;
}
if (LangOpts.CPlusPlus || LangOpts.C2x) {
static const llvm::sys::UnicodeCharSet XIDStartChars(XIDStartRanges);
- return XIDStartChars.contains(C);
+ if (XIDStartChars.contains(C))
+ return true;
+ return isMathematicalExtensionID(C, LangOpts, /*IsStart=*/true,
+ IsExtension);
}
- if (!isAllowedIDChar(C, LangOpts))
+ if (!isAllowedIDChar(C, LangOpts, IsExtension))
return false;
if (LangOpts.C11) {
static const llvm::sys::UnicodeCharSet C11DisallowedInitialIDChars(
return !C99DisallowedInitialIDChars.contains(C);
}
+static void diagnoseExtensionInIdentifier(DiagnosticsEngine &Diags, uint32_t C,
+ CharSourceRange Range) {
+
+ static const llvm::sys::UnicodeCharSet MathStartChars(
+ MathematicalNotationProfileIDStartRanges);
+ static const llvm::sys::UnicodeCharSet MathContinueChars(
+ MathematicalNotationProfileIDContinueRanges);
+
+ assert((MathStartChars.contains(C) || MathContinueChars.contains(C)) &&
+ "Unexpected mathematical notation codepoint");
+ Diags.Report(Range.getBegin(), diag::ext_mathematical_notation)
+ << codepointAsHexString(C) << Range;
+}
+
static inline CharSourceRange makeCharRange(Lexer &L, const char *Begin,
const char *End) {
return CharSourceRange::getCharRange(L.getSourceLocation(Begin),
std::lower_bound(std::begin(SortedHomoglyphs),
std::end(SortedHomoglyphs) - 1, HomoglyphPair{C, '\0'});
if (Homoglyph->Character == C) {
- llvm::SmallString<5> CharBuf;
- {
- llvm::raw_svector_ostream CharOS(CharBuf);
- llvm::write_hex(CharOS, C, llvm::HexPrintStyle::Upper, 4);
- }
if (Homoglyph->LooksLike) {
const char LooksLikeStr[] = {Homoglyph->LooksLike, 0};
Diags.Report(Range.getBegin(), diag::warn_utf8_symbol_homoglyph)
- << Range << CharBuf << LooksLikeStr;
+ << Range << codepointAsHexString(C) << LooksLikeStr;
} else {
Diags.Report(Range.getBegin(), diag::warn_utf8_symbol_zero_width)
- << Range << CharBuf;
+ << Range << codepointAsHexString(C);
}
}
}
if (isASCII(CodePoint))
return;
- bool IsIDStart = isAllowedInitiallyIDChar(CodePoint, LangOpts);
- bool IsIDContinue = IsIDStart || isAllowedIDChar(CodePoint, LangOpts);
+ bool IsExtension;
+ bool IsIDStart = isAllowedInitiallyIDChar(CodePoint, LangOpts, IsExtension);
+ bool IsIDContinue =
+ IsIDStart || isAllowedIDChar(CodePoint, LangOpts, IsExtension);
if ((IsFirst && IsIDStart) || (!IsFirst && IsIDContinue))
return;
bool InvalidOnlyAtStart = IsFirst && !IsIDStart && IsIDContinue;
- llvm::SmallString<5> CharBuf;
- llvm::raw_svector_ostream CharOS(CharBuf);
- llvm::write_hex(CharOS, CodePoint, llvm::HexPrintStyle::Upper, 4);
-
if (!IsFirst || InvalidOnlyAtStart) {
Diags.Report(Range.getBegin(), diag::err_character_not_allowed_identifier)
- << Range << CharBuf << int(InvalidOnlyAtStart)
+ << Range << codepointAsHexString(CodePoint) << int(InvalidOnlyAtStart)
<< FixItHint::CreateRemoval(Range);
} else {
Diags.Report(Range.getBegin(), diag::err_character_not_allowed)
- << Range << CharBuf << FixItHint::CreateRemoval(Range);
+ << Range << codepointAsHexString(CodePoint)
+ << FixItHint::CreateRemoval(Range);
}
}
if (CodePoint == 0) {
return false;
}
-
- if (!isAllowedIDChar(CodePoint, LangOpts)) {
+ bool IsExtension = false;
+ if (!isAllowedIDChar(CodePoint, LangOpts, IsExtension)) {
if (isASCII(CodePoint) || isUnicodeWhitespace(CodePoint))
return false;
if (!isLexingRawMode() && !ParsingPreprocessorDirective &&
// We got a unicode codepoint that is neither a space nor a
// a valid identifier part.
// Carry on as if the codepoint was valid for recovery purposes.
- } else if (!isLexingRawMode())
+ } else if (!isLexingRawMode()) {
+ if (IsExtension)
+ diagnoseExtensionInIdentifier(PP->getDiagnostics(), CodePoint,
+ makeCharRange(*this, CurPtr, UCNPtr));
+
maybeDiagnoseIDCharCompat(PP->getDiagnostics(), CodePoint,
makeCharRange(*this, CurPtr, UCNPtr),
/*IsFirst=*/false);
+ }
Result.setFlag(Token::HasUCN);
if ((UCNPtr - CurPtr == 6 && CurPtr[1] == 'u') ||
if (Result != llvm::conversionOK)
return false;
- if (!isAllowedIDChar(static_cast<uint32_t>(CodePoint), LangOpts)) {
+ bool IsExtension = false;
+ if (!isAllowedIDChar(static_cast<uint32_t>(CodePoint), LangOpts,
+ IsExtension)) {
if (isASCII(CodePoint) || isUnicodeWhitespace(CodePoint))
return false;
// a valid identifier part. Carry on as if the codepoint was
// valid for recovery purposes.
} else if (!isLexingRawMode()) {
+ if (IsExtension)
+ diagnoseExtensionInIdentifier(PP->getDiagnostics(), CodePoint,
+ makeCharRange(*this, CurPtr, UnicodePtr));
maybeDiagnoseIDCharCompat(PP->getDiagnostics(), CodePoint,
makeCharRange(*this, CurPtr, UnicodePtr),
/*IsFirst=*/false);
bool Lexer::LexUnicodeIdentifierStart(Token &Result, uint32_t C,
const char *CurPtr) {
- if (isAllowedInitiallyIDChar(C, LangOpts)) {
+ bool IsExtension = false;
+ if (isAllowedInitiallyIDChar(C, LangOpts, IsExtension)) {
if (!isLexingRawMode() && !ParsingPreprocessorDirective &&
!PP->isPreprocessedOutput()) {
+ if (IsExtension)
+ diagnoseExtensionInIdentifier(PP->getDiagnostics(), C,
+ makeCharRange(*this, BufferPtr, CurPtr));
maybeDiagnoseIDCharCompat(PP->getDiagnostics(), C,
makeCharRange(*this, BufferPtr, CurPtr),
/*IsFirst=*/true);
if (!isLexingRawMode() && !ParsingPreprocessorDirective &&
!PP->isPreprocessedOutput() && !isASCII(*BufferPtr) &&
- !isAllowedInitiallyIDChar(C, LangOpts) && !isUnicodeWhitespace(C)) {
+ !isUnicodeWhitespace(C)) {
// Non-ASCII characters tend to creep into source code unintentionally.
// Instead of letting the parser complain about the unknown token,
// just drop the character.
{0x1E4EC, 0x1E4F9}, {0x1E8D0, 0x1E8D6}, {0x1E944, 0x1E94A},
{0x1E950, 0x1E959}, {0x1FBF0, 0x1FBF9}, {0xE0100, 0xE01EF}};
+// Clang supports the "Mathematical notation profile" as an extension,
+// as described in https://www.unicode.org/L2/L2022/22230-math-profile.pdf
+// Math_Start
+static const llvm::sys::UnicodeCharRange
+ MathematicalNotationProfileIDStartRanges[] = {
+ {0x02202, 0x02202}, // โ
+ {0x02207, 0x02207}, // โ
+ {0x0221E, 0x0221E}, // โ
+ {0x1D6C1, 0x1D6C1}, // ๐
+ {0x1D6DB, 0x1D6DB}, // ๐
+ {0x1D6FB, 0x1D6FB}, // ๐ป
+ {0x1D715, 0x1D715}, // ๐
+ {0x1D735, 0x1D735}, // ๐ต
+ {0x1D74F, 0x1D74F}, // ๐
+ {0x1D76F, 0x1D76F}, // ๐ฏ
+ {0x1D789, 0x1D789}, // ๐
+ {0x1D7A9, 0x1D7A9}, // ๐ฉ
+ {0x1D7C3, 0x1D7C3}, // ๐
+};
+
+// Math_Continue
+static const llvm::sys::UnicodeCharRange
+ MathematicalNotationProfileIDContinueRanges[] = {
+ {0x000B2, 0x000B3}, // ยฒ-ยณ
+ {0x000B9, 0x000B9}, // ยน
+ {0x02070, 0x02070}, // โฐ
+ {0x02074, 0x0207E}, // โด-โพ
+ {0x02080, 0x0208E}, // โ-โ
+};
+
// C11 D.1, C++11 [charname.allowed]
static const llvm::sys::UnicodeCharRange C11AllowedIDCharRanges[] = {
// 1