clang 8.0.0

Enumerations | Functions
Lexer.cpp File Reference
`#include "clang/Lex/Lexer.h"`
`#include "UnicodeCharSets.h"`
`#include "clang/Basic/CharInfo.h"`
`#include "clang/Basic/IdentifierTable.h"`
`#include "clang/Basic/LangOptions.h"`
`#include "clang/Basic/SourceLocation.h"`
`#include "clang/Basic/SourceManager.h"`
`#include "clang/Basic/TokenKinds.h"`
`#include "clang/Lex/LexDiagnostic.h"`
`#include "clang/Lex/LiteralSupport.h"`
`#include "clang/Lex/MultipleIncludeOpt.h"`
`#include "clang/Lex/Preprocessor.h"`
`#include "clang/Lex/PreprocessorOptions.h"`
`#include "clang/Lex/Token.h"`
`#include "clang/Basic/Diagnostic.h"`
`#include "clang/Basic/LLVM.h"`
`#include "llvm/ADT/None.h"`
`#include "llvm/ADT/Optional.h"`
`#include "llvm/ADT/StringExtras.h"`
`#include "llvm/ADT/StringSwitch.h"`
`#include "llvm/ADT/StringRef.h"`
`#include "llvm/Support/Compiler.h"`
`#include "llvm/Support/ConvertUTF.h"`
`#include "llvm/Support/MathExtras.h"`
`#include "llvm/Support/MemoryBuffer.h"`
`#include "llvm/Support/NativeFormatting.h"`
`#include "llvm/Support/UnicodeCharRanges.h"`
`#include <algorithm>`
`#include <cassert>`
`#include <cstddef>`
`#include <cstdint>`
`#include <cstring>`
`#include <string>`
`#include <tuple>`
`#include <utility>`
Include dependency graph for Lexer.cpp:

Go to the source code of this file.

## Enumerations
enum | PreambleDirectiveKind

## Functions
template<typename T >
static void | StringifyImpl (T &Str, char Quote)
static size_t | getSpellingSlow (const Token &Tok, const char *BufPtr, const LangOptions &LangOpts, char *Spelling)
| Slow case of getSpelling. More...
static const char * | findBeginningOfLine (StringRef Buffer, unsigned Offset)
| Returns the pointer that points to the beginning of line that contains the given offset, or null if the offset if invalid. More...
static SourceLocation | getBeginningOfFileToken (SourceLocation Loc, const SourceManager &SM, const LangOptions &LangOpts)
static CharSourceRange | makeRangeFromFileLocs (CharSourceRange Range, const SourceManager &SM, const LangOptions &LangOpts)
static LLVM_ATTRIBUTE_NOINLINE SourceLocation | GetMappedTokenLoc (Preprocessor &PP, SourceLocation FileLoc, unsigned CharNo, unsigned TokLen)
| GetMappedTokenLoc - If lexing out of a 'mapped buffer', where we pretend the lexer buffer was all expanded at a single point, perform the mapping. More...
static char | GetTrigraphCharForLetter (char Letter)
| GetTrigraphCharForLetter - Given a character that occurs after a ?? pair, return the decoded trigraph letter it corresponds to, or '\0' if nothing. More...
static char | DecodeTrigraphChar (const char *CP, Lexer *L)
| DecodeTrigraphChar - If the specified character is a legal trigraph when prefixed with ??, emit a trigraph warning. More...
static bool | isAllowedIDChar (uint32_t C, const LangOptions &LangOpts)
static bool | isAllowedInitiallyIDChar (uint32_t C, const LangOptions &LangOpts)
static CharSourceRange | makeCharRange (Lexer &L, const char *Begin, const char *End)
static void | maybeDiagnoseIDCharCompat (DiagnosticsEngine &Diags, uint32_t C, CharSourceRange Range, bool IsFirst)
static void | maybeDiagnoseUTF8Homoglyph (DiagnosticsEngine &Diags, uint32_t C, CharSourceRange Range)
| After encountering UTF-8 character C and interpreting it as an identifier character, check whether it's a homoglyph for a common non-identifier source character that is unlikely to be an intentional identifier character and warn if so. More...
static bool | isEndOfBlockCommentWithEscapedNewLine (const char *CurPtr, Lexer *L)
| isBlockCommentEndOfEscapedNewLine - Return true if the specified newline character (either \n or \r) is part of an escaped newline sequence. More...
static const char * | FindConflictEnd (const char *CurPtr, const char *BufferEnd, ConflictMarkerKind CMK)
| Find the end of a version control conflict marker. More...
static const char * | findPlaceholderEnd (const char *CurPtr, const char *BufferEnd)

## Enumeration Type Documentation

## ◆ PreambleDirectiveKind

enum PreambleDirectiveKind

Definition at line 567 of file Lexer.cpp.

## Function Documentation

## ◆ DecodeTrigraphChar()

static char DecodeTrigraphChar | ( | const char * | CP,
|  | Lexer * | L
| ) |  |
| static

DecodeTrigraphChar - If the specified character is a legal trigraph when prefixed with ??, emit a trigraph warning.

If trigraphs are enabled, return the result character. Finally, emit a warning about trigraph use whether trigraphs are enabled or not.

Definition at line 1167 of file Lexer.cpp.

References clang::Lexer::Diag(), clang::Lexer::getLangOpts(), GetTrigraphCharForLetter(), clang::PreprocessorLexer::isLexingRawMode(), clang::isWhitespace(), and P.

Referenced by clang::Lexer::findLocationAfterToken().

## ◆ findBeginningOfLine()

static const char* findBeginningOfLine | ( | StringRef | Buffer,
|  | unsigned | Offset
| ) |  |
| static

Returns the pointer that points to the beginning of line that contains the given offset, or null if the offset if invalid.

Definition at line 481 of file Lexer.cpp.

References clang::Lexer::isNewLineEscaped(), clang::isVerticalWhitespace(), and Offset.

Referenced by getBeginningOfFileToken(), and clang::Lexer::getIndentationForLine().

## ◆ FindConflictEnd()

static const char* FindConflictEnd | ( | const char * | CurPtr,
|  | const char * | BufferEnd,
|  | ConflictMarkerKind | CMK
| ) |  |
| static

Find the end of a version control conflict marker.

Definition at line 2811 of file Lexer.cpp.

References clang::CMK_None, clang::CMK_Normal, clang::CMK_Perforce, clang::Lexer::Diag(), End, and clang::PreprocessorLexer::isLexingRawMode().

## ◆ findPlaceholderEnd()

static const char* findPlaceholderEnd | ( | const char * | CurPtr,
|  | const char * | BufferEnd
| ) |  |
| static

Definition at line 2915 of file Lexer.cpp.

References clang::frontend::After, clang::C, clang::Token::clearFlag(), clang::frontend::CPlusPlus, clang::frontend::CPlusPlus2a, clang::FixItHint::CreateInsertion(), clang::FixItHint::CreateRemoval(), clang::FixItHint::CreateReplacement(), clang::Lexer::Diag(), End, clang::Preprocessor::getCodeCompletionLoc(), clang::Preprocessor::getDiagnostics(), clang::Lexer::getLangOpts(), clang::SourceLocation::getLocWithOffset(), clang::Preprocessor::getPreprocessorOpts(), clang::Lexer::getSourceLocation(), clang::Token::hasLeadingEmptyMacro(), clang::Token::hasLeadingSpace(), clang::Token::HasUCN, clang::Lexer::inKeepCommentMode(), isAllowedIDChar(), isAllowedInitiallyIDChar(), clang::isASCII(), clang::Token::isAtStartOfLine(), clang::Preprocessor::isCodeCompletionEnabled(), clang::Token::IsEditorPlaceholder, clang::isHorizontalWhitespace(), clang::Lexer::isKeepWhitespaceMode(), clang::PreprocessorLexer::isLexingRawMode(), clang::Preprocessor::isPreprocessedOutput(), clang::Token::LeadingEmptyMacro, clang::Token::LeadingSpace, clang::PreprocessorOptions::LexEditorPlaceholders, clang::PreprocessorLexer::LexingRawMode, clang::Preprocessor::LookUpIdentifierInfo(), makeCharRange(), maybeDiagnoseIDCharCompat(), maybeDiagnoseUTF8Homoglyph(), clang::PreprocessorLexer::MIOpt, clang::Token::NeedsCleaning, clang::PreprocessorLexer::ParsingFilename, clang::PreprocessorLexer::ParsingPreprocessorDirective, clang::MultipleIncludeOpt::ReadToken(), clang::Lexer::resetExtendedTokenMode(), clang::Token::setFlag(), clang::Token::setIdentifierInfo(), clang::Token::setRawIdentifierData(), clang::Token::StartOfLine, clang::Token::startToken(), and UnicodeWhitespaceCharRanges.

## ◆ getBeginningOfFileToken()

static SourceLocation getBeginningOfFileToken | ( | SourceLocation | Loc,
|  | const SourceManager & | SM,
|  | const LangOptions & | LangOpts
| ) |  |
| static

Definition at line 498 of file Lexer.cpp.

References clang::comments::tok::eof, findBeginningOfLine(), clang::SourceManager::getBufferData(), clang::SourceManager::getDecomposedLoc(), clang::Token::getKind(), clang::Token::getLength(), clang::Token::getLocation(), clang::SourceLocation::getLocWithOffset(), clang::SourceLocation::isFileID(), and clang::Lexer::SetCommentRetentionState().

Referenced by clang::Lexer::GetBeginningOfToken().

## ◆ GetMappedTokenLoc()

static SourceLocation GetMappedTokenLoc | ( | Preprocessor & | PP,
|  | SourceLocation | FileLoc,
|  | unsigned | CharNo,
|  | unsigned | TokLen
| ) |  |
| static

GetMappedTokenLoc - If lexing out of a 'mapped buffer', where we pretend the lexer buffer was all expanded at a single point, perform the mapping.

This is currently only used for _Pragma implementation, so it is the slow path of the hot getSourceLocation method. Do not allow it to be inlined.

Definition at line 1095 of file Lexer.cpp.

References clang::CharSourceRange::getBegin(), clang::CharSourceRange::getEnd(), clang::SourceLocation::getLocWithOffset(), clang::Preprocessor::getSourceManager(), clang::SourceLocation::isMacroID(), and SM.

Referenced by clang::Lexer::getIndentationForLine(), and clang::Lexer::getSourceLocation().

## ◆ getSpellingSlow()

static size_t getSpellingSlow | ( | const Token & | Tok,
|  | const char * | BufPtr,
|  | const LangOptions & | LangOpts,
|  | char * | Spelling
| ) |  |
| static

Slow case of getSpelling.

Extract the characters comprising the spelling of this token from the provided input buffer.

Definition at line 262 of file Lexer.cpp.

References clang::Lexer::getCharAndSizeNoWarn(), clang::Token::getKind(), clang::Token::getLength(), clang::tok::isStringLiteral(), memcpy(), and clang::Token::needsCleaning().

Referenced by clang::Lexer::getSpelling().

## ◆ GetTrigraphCharForLetter()

static char GetTrigraphCharForLetter | ( | char | Letter | ) |
| static

GetTrigraphCharForLetter - Given a character that occurs after a ?? pair, return the decoded trigraph letter it corresponds to, or '\0' if nothing.

Definition at line 1148 of file Lexer.cpp.

Referenced by DecodeTrigraphChar(), and clang::Lexer::findLocationAfterToken().

## ◆ isAllowedIDChar()

static bool isAllowedIDChar | ( | uint32_t | C,
|  | const LangOptions & | LangOpts
| ) |  |
| static

Definition at line 1424 of file Lexer.cpp.

References C11AllowedIDCharRanges, C99AllowedIDCharRanges, and CXX03AllowedIDCharRanges.

Referenced by findPlaceholderEnd(), isAllowedInitiallyIDChar(), and maybeDiagnoseUTF8Homoglyph().

## ◆ isAllowedInitiallyIDChar()

static bool isAllowedInitiallyIDChar | ( | uint32_t | C,
|  | const LangOptions & | LangOpts
| ) |  |
| static

Definition at line 1442 of file Lexer.cpp.

References C11DisallowedInitialIDCharRanges, C99DisallowedInitialIDCharRanges, and isAllowedIDChar().

Referenced by findPlaceholderEnd().

## ◆ isEndOfBlockCommentWithEscapedNewLine()

static bool isEndOfBlockCommentWithEscapedNewLine | ( | const char * | CurPtr,
|  | Lexer * | L
| ) |  |
| static

isBlockCommentEndOfEscapedNewLine - Return true if the specified newline character (either \n or \r) is part of an escaped newline sequence.

Issue a diagnostic if so. We know that the newline is inside of a block comment.

Definition at line 2406 of file Lexer.cpp.

References _mm_cmpeq_epi8(), _mm_movemask_epi8(), _mm_set1_epi8(), clang::C, clang::Preprocessor::CodeCompleteNaturalLanguage(), clang::Lexer::Diag(), clang::Preprocessor::getCodeCompletionFileLoc(), clang::Lexer::getLangOpts(), clang::Lexer::getSourceLocation(), clang::Preprocessor::HandleComment(), clang::Lexer::inKeepCommentMode(), clang::isHorizontalWhitespace(), clang::Lexer::isKeepWhitespaceMode(), clang::PreprocessorLexer::isLexingRawMode(), clang::Token::LeadingSpace, clang::Token::setFlag(), and vec_any_eq().

## ◆ makeCharRange()

static CharSourceRange makeCharRange | ( | Lexer & | L,
|  | const char * | Begin,
|  | const char * | End
| ) |  |
| inline static

Definition at line 1459 of file Lexer.cpp.

References clang::CharSourceRange::getCharRange(), and clang::Lexer::getSourceLocation().

Referenced by findPlaceholderEnd(), and maybeDiagnoseUTF8Homoglyph().

## ◆ makeRangeFromFileLocs()

static CharSourceRange makeRangeFromFileLocs | ( | CharSourceRange | Range,
|  | const SourceManager & | SM,
|  | const LangOptions & | LangOpts
| ) |  |
| static

Definition at line 841 of file Lexer.cpp.

References Begin, End, clang::PreprocessorLexer::FID, clang::CharSourceRange::getBegin(), clang::CharSourceRange::getCharRange(), clang::SourceManager::getDecomposedLoc(), clang::CharSourceRange::getEnd(), clang::Lexer::getLocForEndOfToken(), clang::SourceLocation::isFileID(), clang::SourceManager::isInFileID(), clang::FileID::isInvalid(), clang::SourceLocation::isInvalid(), and clang::CharSourceRange::isTokenRange().

Referenced by clang::Lexer::makeFileCharRange().

## ◆ maybeDiagnoseIDCharCompat()

static void maybeDiagnoseIDCharCompat | ( | DiagnosticsEngine & | Diags,
|  | uint32_t | C,
|  | CharSourceRange | Range,
|  | bool | IsFirst
| ) |  |
| static

Definition at line 1465 of file Lexer.cpp.

References C99AllowedIDCharRanges, C99DisallowedInitialIDCharRanges, CXX03AllowedIDCharRanges, clang::CharSourceRange::getBegin(), clang::DiagnosticsEngine::isIgnored(), and clang::DiagnosticsEngine::Report().

Referenced by findPlaceholderEnd(), and maybeDiagnoseUTF8Homoglyph().

## ◆ maybeDiagnoseUTF8Homoglyph()

static void maybeDiagnoseUTF8Homoglyph | ( | DiagnosticsEngine & | Diags,
|  | uint32_t | C,
|  | CharSourceRange | Range
| ) |  |
| static

After encountering UTF-8 character C and interpreting it as an identifier character, check whether it's a homoglyph for a common non-identifier source character that is unlikely to be an intentional identifier character and warn if so.

Definition at line 1504 of file Lexer.cpp.

References clang::C, clang::frontend::CPlusPlus14, clang::frontend::CPlusPlus17, clang::Lexer::Diag(), clang::CharSourceRange::getBegin(), clang::Lexer::getCharAndSizeNoWarn(), clang::Preprocessor::getDiagnostics(), clang::Lexer::getLangOpts(), clang::Preprocessor::HandleIdentifier(), clang::Token::HasUCN, isAllowedIDChar(), clang::isASCII(), clang::IdentifierInfo::isHandleIdentifierCase(), clang::isIdentifierBody(), clang::PreprocessorLexer::isLexingRawMode(), clang::isPreprocessingNumberBody(), clang::PreprocessorLexer::LexingRawMode, clang::Preprocessor::LookUpIdentifierInfo(), makeCharRange(), maybeDiagnoseIDCharCompat(), clang::operator<(), clang::DiagnosticsEngine::Report(), clang::Result, clang::Token::setFlag(), clang::Token::setKind(), and clang::Token::setRawIdentifierData().

Referenced by findPlaceholderEnd().

## ◆ StringifyImpl()

template<typename T >

static void StringifyImpl | ( | T & | Str,
|  | char | Quote
| ) |  |
| static

Definition at line 222 of file Lexer.cpp.

Referenced by clang::Lexer::Stringify().