clang
6.0.0
|
#include "clang/Lex/Lexer.h"
#include "UnicodeCharSets.h"
#include "clang/Basic/CharInfo.h"
#include "clang/Basic/IdentifierTable.h"
#include "clang/Basic/LangOptions.h"
#include "clang/Basic/SourceLocation.h"
#include "clang/Basic/SourceManager.h"
#include "clang/Basic/TokenKinds.h"
#include "clang/Lex/LexDiagnostic.h"
#include "clang/Lex/LiteralSupport.h"
#include "clang/Lex/MultipleIncludeOpt.h"
#include "clang/Lex/Preprocessor.h"
#include "clang/Lex/PreprocessorOptions.h"
#include "clang/Lex/Token.h"
#include "clang/Basic/Diagnostic.h"
#include "clang/Basic/LLVM.h"
#include "llvm/ADT/None.h"
#include "llvm/ADT/Optional.h"
#include "llvm/ADT/StringExtras.h"
#include "llvm/ADT/StringSwitch.h"
#include "llvm/ADT/StringRef.h"
#include "llvm/Support/Compiler.h"
#include "llvm/Support/ConvertUTF.h"
#include "llvm/Support/MathExtras.h"
#include "llvm/Support/MemoryBuffer.h"
#include "llvm/Support/NativeFormatting.h"
#include "llvm/Support/UnicodeCharRanges.h"
#include <algorithm>
#include <cassert>
#include <cstddef>
#include <cstdint>
#include <cstring>
#include <string>
#include <tuple>
#include <utility>
Go to the source code of this file.
Enumerations | |
enum | PreambleDirectiveKind |
Functions | |
template<typename T > | |
static void | StringifyImpl (T &Str, char Quote) |
static size_t | getSpellingSlow (const Token &Tok, const char *BufPtr, const LangOptions &LangOpts, char *Spelling) |
Slow case of getSpelling. More... | |
static const char * | findBeginningOfLine (StringRef Buffer, unsigned Offset) |
Returns the pointer that points to the beginning of line that contains the given offset, or null if the offset if invalid. More... | |
static SourceLocation | getBeginningOfFileToken (SourceLocation Loc, const SourceManager &SM, const LangOptions &LangOpts) |
static CharSourceRange | makeRangeFromFileLocs (CharSourceRange Range, const SourceManager &SM, const LangOptions &LangOpts) |
static LLVM_ATTRIBUTE_NOINLINE SourceLocation | GetMappedTokenLoc (Preprocessor &PP, SourceLocation FileLoc, unsigned CharNo, unsigned TokLen) |
GetMappedTokenLoc - If lexing out of a 'mapped buffer', where we pretend the lexer buffer was all expanded at a single point, perform the mapping. More... | |
static char | GetTrigraphCharForLetter (char Letter) |
GetTrigraphCharForLetter - Given a character that occurs after a ?? pair, return the decoded trigraph letter it corresponds to, or '\0' if nothing. More... | |
static char | DecodeTrigraphChar (const char *CP, Lexer *L) |
DecodeTrigraphChar - If the specified character is a legal trigraph when prefixed with ??, emit a trigraph warning. More... | |
static bool | isAllowedIDChar (uint32_t C, const LangOptions &LangOpts) |
static bool | isAllowedInitiallyIDChar (uint32_t C, const LangOptions &LangOpts) |
static CharSourceRange | makeCharRange (Lexer &L, const char *Begin, const char *End) |
static void | maybeDiagnoseIDCharCompat (DiagnosticsEngine &Diags, uint32_t C, CharSourceRange Range, bool IsFirst) |
static void | maybeDiagnoseUTF8Homoglyph (DiagnosticsEngine &Diags, uint32_t C, CharSourceRange Range) |
After encountering UTF-8 character C and interpreting it as an identifier character, check whether it's a homoglyph for a common non-identifier source character that is unlikely to be an intentional identifier character and warn if so. More... | |
static bool | isEndOfBlockCommentWithEscapedNewLine (const char *CurPtr, Lexer *L) |
isBlockCommentEndOfEscapedNewLine - Return true if the specified newline character (either \n or \r) is part of an escaped newline sequence. More... | |
static const char * | FindConflictEnd (const char *CurPtr, const char *BufferEnd, ConflictMarkerKind CMK) |
Find the end of a version control conflict marker. More... | |
static const char * | findPlaceholderEnd (const char *CurPtr, const char *BufferEnd) |
|
static |
DecodeTrigraphChar - If the specified character is a legal trigraph when prefixed with ??, emit a trigraph warning.
If trigraphs are enabled, return the result character. Finally, emit a warning about trigraph use whether trigraphs are enabled or not.
Definition at line 1171 of file Lexer.cpp.
References clang::Lexer::Diag(), clang::Lexer::getLangOpts(), GetTrigraphCharForLetter(), clang::PreprocessorLexer::isLexingRawMode(), clang::isWhitespace(), and P.
Referenced by clang::Lexer::findLocationAfterToken().
|
static |
|
static |
Find the end of a version control conflict marker.
Definition at line 2745 of file Lexer.cpp.
References clang::CMK_None, clang::CMK_Normal, clang::CMK_Perforce, clang::Lexer::Diag(), End, and clang::PreprocessorLexer::isLexingRawMode().
|
static |
Definition at line 2849 of file Lexer.cpp.
References clang::frontend::After, clang::C, clang::Token::clearFlag(), clang::frontend::CPlusPlus, clang::frontend::CPlusPlus2a, clang::FixItHint::CreateInsertion(), clang::FixItHint::CreateRemoval(), clang::FixItHint::CreateReplacement(), clang::Lexer::Diag(), End, clang::Preprocessor::getCodeCompletionLoc(), clang::Preprocessor::getDiagnostics(), clang::Lexer::getLangOpts(), clang::SourceLocation::getLocWithOffset(), clang::Preprocessor::getPreprocessorOpts(), clang::Lexer::getSourceLocation(), clang::Token::hasLeadingEmptyMacro(), clang::Token::hasLeadingSpace(), clang::Token::HasUCN, clang::Lexer::inKeepCommentMode(), isAllowedIDChar(), isAllowedInitiallyIDChar(), clang::isASCII(), clang::Token::isAtStartOfLine(), clang::Preprocessor::isCodeCompletionEnabled(), clang::Token::IsEditorPlaceholder, clang::isHorizontalWhitespace(), clang::Lexer::isKeepWhitespaceMode(), clang::PreprocessorLexer::isLexingRawMode(), clang::Preprocessor::isPreprocessedOutput(), clang::Token::LeadingEmptyMacro, clang::Token::LeadingSpace, clang::PreprocessorOptions::LexEditorPlaceholders, clang::PreprocessorLexer::LexingRawMode, clang::Preprocessor::LookUpIdentifierInfo(), makeCharRange(), maybeDiagnoseIDCharCompat(), clang::PreprocessorLexer::MIOpt, clang::Token::NeedsCleaning, clang::PreprocessorLexer::ParsingFilename, clang::PreprocessorLexer::ParsingPreprocessorDirective, clang::MultipleIncludeOpt::ReadToken(), clang::Lexer::resetExtendedTokenMode(), clang::Token::setFlag(), clang::Token::setIdentifierInfo(), clang::Token::setRawIdentifierData(), clang::Token::StartOfLine, clang::Token::startToken(), and UnicodeWhitespaceCharRanges.
|
static |
Definition at line 498 of file Lexer.cpp.
References clang::SourceManager::getDecomposedLoc(), and clang::SourceLocation::isFileID().
Referenced by clang::Lexer::GetBeginningOfToken().
|
static |
GetMappedTokenLoc - If lexing out of a 'mapped buffer', where we pretend the lexer buffer was all expanded at a single point, perform the mapping.
This is currently only used for _Pragma implementation, so it is the slow path of the hot getSourceLocation method. Do not allow it to be inlined.
Definition at line 1098 of file Lexer.cpp.
References clang::SourceLocation::getLocWithOffset(), clang::Preprocessor::getSourceManager(), clang::SourceLocation::isMacroID(), and SM.
Referenced by clang::Lexer::getSourceLocation().
|
static |
Slow case of getSpelling.
Extract the characters comprising the spelling of this token from the provided input buffer.
Definition at line 262 of file Lexer.cpp.
References clang::Token::needsCleaning().
Referenced by clang::Lexer::getSpelling().
|
static |
GetTrigraphCharForLetter - Given a character that occurs after a ?? pair, return the decoded trigraph letter it corresponds to, or '\0' if nothing.
Definition at line 1152 of file Lexer.cpp.
Referenced by DecodeTrigraphChar(), and clang::Lexer::findLocationAfterToken().
|
static |
Definition at line 1428 of file Lexer.cpp.
References C11AllowedIDCharRanges, C99AllowedIDCharRanges, and CXX03AllowedIDCharRanges.
Referenced by findPlaceholderEnd(), isAllowedInitiallyIDChar(), and maybeDiagnoseUTF8Homoglyph().
|
static |
Definition at line 1446 of file Lexer.cpp.
References C11DisallowedInitialIDCharRanges, C99DisallowedInitialIDCharRanges, and isAllowedIDChar().
Referenced by findPlaceholderEnd().
isBlockCommentEndOfEscapedNewLine - Return true if the specified newline character (either \n or \r) is part of an escaped newline sequence.
Issue a diagnostic if so. We know that the newline is inside of a block comment.
Definition at line 2340 of file Lexer.cpp.
References _mm_cmpeq_epi8(), _mm_movemask_epi8(), _mm_set1_epi8(), clang::C, clang::Preprocessor::CodeCompleteNaturalLanguage(), clang::Lexer::Diag(), clang::Preprocessor::getCodeCompletionFileLoc(), clang::Lexer::getLangOpts(), clang::Lexer::getSourceLocation(), clang::Preprocessor::HandleComment(), clang::Lexer::inKeepCommentMode(), clang::isHorizontalWhitespace(), clang::Lexer::isKeepWhitespaceMode(), clang::PreprocessorLexer::isLexingRawMode(), clang::Token::LeadingSpace, clang::Token::setFlag(), and vec_any_eq().
|
inlinestatic |
Definition at line 1463 of file Lexer.cpp.
References clang::CharSourceRange::getCharRange(), and clang::Lexer::getSourceLocation().
Referenced by findPlaceholderEnd(), and maybeDiagnoseUTF8Homoglyph().
|
static |
Definition at line 844 of file Lexer.cpp.
References Begin, End, clang::PreprocessorLexer::FID, clang::CharSourceRange::getBegin(), clang::CharSourceRange::getCharRange(), clang::SourceManager::getDecomposedLoc(), clang::CharSourceRange::getEnd(), clang::Lexer::getLocForEndOfToken(), clang::SourceLocation::isFileID(), clang::SourceManager::isInFileID(), clang::FileID::isInvalid(), clang::SourceLocation::isInvalid(), and clang::CharSourceRange::isTokenRange().
Referenced by clang::Lexer::makeFileCharRange().
|
static |
Definition at line 1469 of file Lexer.cpp.
References C99AllowedIDCharRanges, C99DisallowedInitialIDCharRanges, CXX03AllowedIDCharRanges, clang::CharSourceRange::getBegin(), clang::DiagnosticsEngine::isIgnored(), and clang::DiagnosticsEngine::Report().
Referenced by findPlaceholderEnd(), and maybeDiagnoseUTF8Homoglyph().
|
static |
After encountering UTF-8 character C and interpreting it as an identifier character, check whether it's a homoglyph for a common non-identifier source character that is unlikely to be an intentional identifier character and warn if so.
Definition at line 1508 of file Lexer.cpp.
References clang::C, clang::frontend::CPlusPlus14, clang::frontend::CPlusPlus17, clang::Lexer::Diag(), clang::CharSourceRange::getBegin(), clang::Lexer::getCharAndSizeNoWarn(), clang::Preprocessor::getDiagnostics(), clang::Lexer::getLangOpts(), clang::IdentifierInfo::getObjCKeywordID(), clang::IdentifierInfo::getPPKeywordID(), clang::IdentifierInfo::getTokenID(), clang::Preprocessor::HandleIdentifier(), clang::Token::HasUCN, isAllowedIDChar(), clang::isASCII(), clang::IdentifierInfo::isHandleIdentifierCase(), clang::isIdentifierBody(), clang::PreprocessorLexer::isLexingRawMode(), clang::isPreprocessingNumberBody(), clang::PreprocessorLexer::LexingRawMode, clang::Preprocessor::LookUpIdentifierInfo(), makeCharRange(), maybeDiagnoseIDCharCompat(), clang::operator<(), clang::DiagnosticsEngine::Report(), clang::Result, clang::Token::setFlag(), clang::Token::setKind(), and clang::Token::setRawIdentifierData().
|
static |
Definition at line 222 of file Lexer.cpp.
Referenced by clang::Lexer::Stringify().