23 #include "llvm/Support/Path.h" 24 using namespace clang;
34 explicit ActionCommentHandler(
Sema &S) : S(S) { }
45 if (!Ident__except && (getLangOpts().MicrosoftExt || getLangOpts().Borland))
46 Ident__except = PP.getIdentifierInfo(
"__except");
52 : PP(pp), Actions(actions), Diags(PP.getDiagnostics()),
53 GreaterThanIsOperator(
true), ColonIsSacred(
false),
54 InMessageExpression(
false), TemplateParameterDepth(0),
55 ParsingInObjCContainer(
false) {
59 Actions.CurScope =
nullptr;
61 CurParsedObjCImpl =
nullptr;
65 initializePragmaHandlers();
67 CommentSemaHandler.reset(
new ActionCommentHandler(actions));
74 return Diags.
Report(Loc, DiagID);
103 switch (ExpectedTok) {
105 return Tok.
is(tok::colon) || Tok.
is(tok::comma);
106 default:
return false;
110 bool Parser::ExpectAndConsume(
tok::TokenKind ExpectedTok,
unsigned DiagID,
112 if (Tok.
is(ExpectedTok) || Tok.
is(tok::code_completion)) {
124 if (DiagID == diag::err_expected)
126 else if (DiagID == diag::err_expected_after)
127 DB << Msg << ExpectedTok;
138 const char *Spelling =
nullptr;
146 if (DiagID == diag::err_expected)
148 else if (DiagID == diag::err_expected_after)
149 DB << Msg << ExpectedTok;
156 bool Parser::ExpectAndConsumeSemi(
unsigned DiagID) {
160 if (Tok.
is(tok::code_completion)) {
161 handleUnexpectedCodeCompletionToken();
165 if ((Tok.
is(tok::r_paren) || Tok.
is(tok::r_square)) &&
167 Diag(Tok, diag::err_extraneous_token_before_semi)
175 return ExpectAndConsume(tok::semi, DiagID);
178 void Parser::ConsumeExtraSemi(ExtraSemiKind
Kind,
unsigned TST) {
179 if (!Tok.
is(tok::semi))
return;
181 bool HadMultipleSemis =
false;
187 HadMultipleSemis =
true;
196 Diag(StartLoc, diag::warn_cxx98_compat_top_level_semi)
199 Diag(StartLoc, diag::ext_extra_semi_cxx11)
204 if (Kind != AfterMemberFunctionDefinition || HadMultipleSemis)
205 Diag(StartLoc, diag::ext_extra_semi)
211 Diag(StartLoc, diag::warn_extra_semi_after_mem_fn_def)
215 bool Parser::expectIdentifier() {
216 if (Tok.
is(tok::identifier))
220 Diag(Tok, diag::err_expected_token_instead_of_objcxx_keyword)
226 Diag(Tok, diag::err_expected) << tok::identifier;
235 return (static_cast<unsigned>(L) & static_cast<unsigned>(R)) != 0;
249 bool isFirstTokenSkipped =
true;
252 for (
unsigned i = 0, NumToks = Toks.size(); i != NumToks; ++i) {
253 if (Tok.
is(Toks[i])) {
266 if (Toks.size() == 1 && Toks[0] ==
tok::eof &&
279 case tok::annot_pragma_openmp:
280 case tok::annot_pragma_openmp_end:
282 case tok::annot_module_begin:
283 case tok::annot_module_end:
284 case tok::annot_module_include:
290 case tok::code_completion:
292 handleUnexpectedCodeCompletionToken();
326 if (ParenCount && !isFirstTokenSkipped)
331 if (BracketCount && !isFirstTokenSkipped)
336 if (BraceCount && !isFirstTokenSkipped)
350 isFirstTokenSkipped =
false;
360 if (NumCachedScopes) {
361 Scope *N = ScopeCache[--NumCachedScopes];
363 Actions.CurScope = N;
378 Actions.CurScope = OldScope->
getParent();
380 if (NumCachedScopes == ScopeCacheSize)
383 ScopeCache[NumCachedScopes++] = OldScope;
388 Parser::ParseScopeFlags::ParseScopeFlags(
Parser *Self,
unsigned ScopeFlags,
390 : CurScope(ManageFlags ? Self->
getCurScope() :
nullptr) {
392 OldFlags = CurScope->getFlags();
393 CurScope->setFlags(ScopeFlags);
399 Parser::ParseScopeFlags::~ParseScopeFlags() {
401 CurScope->setFlags(OldFlags);
412 Actions.CurScope =
nullptr;
415 for (
unsigned i = 0, e = NumCachedScopes; i != e; ++i)
416 delete ScopeCache[i];
418 resetPragmaHandlers();
433 assert(TemplateIds.empty() &&
"Still alive TemplateIdAnnotations around?");
440 assert(
getCurScope() ==
nullptr &&
"A scope is already active?");
455 ObjCTypeQuals[objc_null_unspecified]
459 Ident_instancetype =
nullptr;
460 Ident_final =
nullptr;
461 Ident_sealed =
nullptr;
462 Ident_override =
nullptr;
463 Ident_GNU_final =
nullptr;
467 Ident_vector =
nullptr;
468 Ident_bool =
nullptr;
469 Ident_pixel =
nullptr;
477 Ident_introduced =
nullptr;
478 Ident_deprecated =
nullptr;
479 Ident_obsoleted =
nullptr;
480 Ident_unavailable =
nullptr;
481 Ident_strict =
nullptr;
482 Ident_replacement =
nullptr;
484 Ident_language = Ident_defined_in = Ident_generated_declaration =
nullptr;
486 Ident__except =
nullptr;
488 Ident__exception_code = Ident__exception_info =
nullptr;
489 Ident__abnormal_termination = Ident___exception_code =
nullptr;
490 Ident___exception_info = Ident___abnormal_termination =
nullptr;
491 Ident_GetExceptionCode = Ident_GetExceptionInfo =
nullptr;
492 Ident_AbnormalTermination =
nullptr;
505 PP.
SetPoisonReason(Ident__exception_code,diag::err_seh___except_block);
506 PP.
SetPoisonReason(Ident___exception_code,diag::err_seh___except_block);
507 PP.
SetPoisonReason(Ident_GetExceptionCode,diag::err_seh___except_block);
508 PP.
SetPoisonReason(Ident__exception_info,diag::err_seh___except_filter);
509 PP.
SetPoisonReason(Ident___exception_info,diag::err_seh___except_filter);
510 PP.
SetPoisonReason(Ident_GetExceptionInfo,diag::err_seh___except_filter);
511 PP.
SetPoisonReason(Ident__abnormal_termination,diag::err_seh___finally_block);
512 PP.
SetPoisonReason(Ident___abnormal_termination,diag::err_seh___finally_block);
513 PP.
SetPoisonReason(Ident_AbnormalTermination,diag::err_seh___finally_block);
522 void Parser::LateTemplateParserCleanupCallback(
void *
P) {
539 Diag(diag::ext_empty_translation_unit);
541 return NoTopLevelDecls;
556 case tok::annot_pragma_unused:
557 HandlePragmaUnused();
565 Result = ParseModuleDecl();
568 case tok::annot_module_include:
570 reinterpret_cast<Module *
>(
572 ConsumeAnnotationToken();
575 case tok::annot_module_begin:
578 ConsumeAnnotationToken();
581 case tok::annot_module_end:
584 ConsumeAnnotationToken();
587 case tok::annot_pragma_attribute:
588 HandlePragmaAttribute();
596 LateTemplateParserCleanupCallback :
nullptr,
607 ParsedAttributesWithRange attrs(AttrFactory);
608 MaybeParseCXX11Attributes(attrs);
610 Result = ParseExternalDeclaration(attrs);
641 Parser::ParseExternalDeclaration(ParsedAttributesWithRange &attrs,
651 Decl *SingleDecl =
nullptr;
653 case tok::annot_pragma_vis:
654 HandlePragmaVisibility();
656 case tok::annot_pragma_pack:
659 case tok::annot_pragma_msstruct:
660 HandlePragmaMSStruct();
662 case tok::annot_pragma_align:
665 case tok::annot_pragma_weak:
668 case tok::annot_pragma_weakalias:
669 HandlePragmaWeakAlias();
671 case tok::annot_pragma_redefine_extname:
672 HandlePragmaRedefineExtname();
674 case tok::annot_pragma_fp_contract:
675 HandlePragmaFPContract();
677 case tok::annot_pragma_fenv_access:
678 HandlePragmaFEnvAccess();
680 case tok::annot_pragma_fp:
683 case tok::annot_pragma_opencl_extension:
684 HandlePragmaOpenCLExtension();
686 case tok::annot_pragma_openmp: {
688 return ParseOpenMPDeclarativeDirectiveWithExtDecl(AS, attrs);
690 case tok::annot_pragma_ms_pointers_to_members:
691 HandlePragmaMSPointersToMembers();
693 case tok::annot_pragma_ms_vtordisp:
694 HandlePragmaMSVtorDisp();
696 case tok::annot_pragma_ms_pragma:
697 HandlePragmaMSPragma();
699 case tok::annot_pragma_dump:
706 ConsumeExtraSemi(OutsideFunction);
709 Diag(Tok, diag::err_extraneous_closing_brace);
713 Diag(Tok, diag::err_expected_external_declaration);
715 case tok::kw___extension__: {
719 return ParseExternalDeclaration(attrs);
722 ProhibitAttributes(attrs);
733 const auto *SL = cast<StringLiteral>(Result.
get());
734 if (!SL->getString().trim().empty())
735 Diag(StartLoc, diag::err_gnu_inline_asm_disabled);
738 ExpectAndConsume(tok::semi, diag::err_expected_after,
739 "top-level asm block");
747 return ParseObjCAtDirectives(attrs);
751 Diag(Tok, diag::err_expected_external_declaration);
755 SingleDecl = ParseObjCMethodDefinition();
757 case tok::code_completion:
758 if (CurParsedObjCImpl) {
774 SingleDecl = ParseExportDeclaration();
781 case tok::kw_namespace:
782 case tok::kw_typedef:
783 case tok::kw_template:
784 case tok::kw_static_assert:
785 case tok::kw__Static_assert:
808 if (NextKind == tok::kw_namespace) {
815 if (NextKind == tok::kw_template) {
830 diag::warn_cxx98_compat_extern_template :
831 diag::ext_extern_template) <<
SourceRange(ExternLoc, TemplateLoc);
835 TemplateLoc, DeclEnd, attrs));
839 case tok::kw___if_exists:
840 case tok::kw___if_not_exists:
841 ParseMicrosoftIfExistsExternalDeclaration();
845 Diag(Tok, diag::err_unexpected_module_decl);
856 return ParseDeclarationOrFunctionDefinition(attrs, DS);
866 bool Parser::isDeclarationAfterDeclarator() {
870 if (KW.
is(tok::kw_default) || KW.
is(tok::kw_delete))
874 return Tok.
is(tok::equal) ||
875 Tok.
is(tok::comma) ||
877 Tok.
is(tok::kw_asm) ||
878 Tok.
is(tok::kw___attribute) ||
880 Tok.
is(tok::l_paren));
887 if (Tok.
is(tok::l_brace))
893 return isDeclarationSpecifier();
897 return KW.
is(tok::kw_default) || KW.
is(tok::kw_delete);
900 return Tok.
is(tok::colon) ||
920 Parser::ParseDeclOrFunctionDefInternal(ParsedAttributesWithRange &attrs,
925 ParseDeclarationSpecifiers(DS, ParsedTemplateInfo(), AS,
926 DeclSpecContext::DSC_top_level);
931 DS, AS, DeclSpecContext::DSC_top_level))
936 if (Tok.
is(tok::semi)) {
951 llvm_unreachable(
"we only expect to get the length of the class/struct/union/enum");
961 ProhibitAttributes(attrs, CorrectLocationForAttributes);
970 Decl* decls[] = {AnonRecord, TheDecl};
985 Diag(Tok, diag::err_objc_unexpected_attr);
992 const char *PrevSpec =
nullptr;
996 Diag(AtLoc, DiagID) << PrevSpec;
999 return ParseObjCAtProtocolDeclaration(AtLoc, DS.
getAttributes());
1002 ParseObjCAtInterfaceDeclaration(AtLoc, DS.
getAttributes()));
1008 if (
getLangOpts().CPlusPlus && isTokenStringLiteral() &&
1019 Parser::ParseDeclarationOrFunctionDefinition(ParsedAttributesWithRange &attrs,
1023 return ParseDeclOrFunctionDefInternal(attrs, *DS, AS);
1031 return ParseDeclOrFunctionDefInternal(attrs, PDS, AS);
1050 const ParsedTemplateInfo &TemplateInfo,
1051 LateParsedAttrList *LateParsedAttrs) {
1060 const char *PrevSpec;
1074 ParseKNRParamDeclarations(D);
1078 if (Tok.
isNot(tok::l_brace) &&
1080 (Tok.
isNot(tok::colon) && Tok.
isNot(tok::kw_try) &&
1081 Tok.
isNot(tok::equal)))) {
1082 Diag(Tok, diag::err_expected_fn_body);
1088 if (Tok.
isNot(tok::l_brace))
1094 if (Tok.
isNot(tok::equal)) {
1096 if (AL.isKnownToGCC() && !AL.isCXX11Attribute())
1097 Diag(AL.getLoc(), diag::warn_attribute_on_function_definition)
1104 TemplateInfo.Kind == ParsedTemplateInfo::Template &&
1114 TemplateParameterLists);
1119 trySkippingFunctionBody()) {
1125 LexTemplateFunctionForLateParsing(Toks);
1134 else if (CurParsedObjCImpl &&
1135 !TemplateInfo.TemplateParams &&
1136 (Tok.
is(tok::l_brace) || Tok.
is(tok::kw_try) ||
1137 Tok.
is(tok::colon)) &&
1150 StashAwayMethodOrFunctionBodyTokens(FuncDecl);
1151 CurParsedObjCImpl->HasCFunction =
true;
1165 TemplateInfo.TemplateParams
1166 ? *TemplateInfo.TemplateParams
1189 ? diag::warn_cxx98_compat_defaulted_deleted_function
1190 : diag::ext_defaulted_deleted_function)
1196 ? diag::warn_cxx98_compat_defaulted_deleted_function
1197 : diag::ext_defaulted_deleted_function)
1201 llvm_unreachable(
"function definition after = not 'delete' or 'default'");
1204 if (Tok.
is(tok::comma)) {
1205 Diag(KWLoc, diag::err_default_delete_in_multiple_declaration)
1208 }
else if (ExpectAndConsume(tok::semi, diag::err_expected_after,
1209 Delete ?
"delete" :
"default")) {
1213 Stmt *GeneratedBody = Res ? Res->
getBody() :
nullptr;
1219 trySkippingFunctionBody()) {
1225 if (Tok.
is(tok::kw_try))
1226 return ParseFunctionTryBlock(Res, BodyScope);
1230 if (Tok.
is(tok::colon)) {
1231 ParseConstructorInitializer(Res);
1234 if (!Tok.
is(tok::l_brace)) {
1243 if (LateParsedAttrs)
1244 ParseLexedAttributeList(*LateParsedAttrs, Res,
false,
true);
1246 return ParseFunctionStatementBody(Res, BodyScope);
1249 void Parser::SkipFunctionBody() {
1250 if (Tok.
is(tok::equal)) {
1255 bool IsFunctionTryBlock = Tok.
is(tok::kw_try);
1256 if (IsFunctionTryBlock)
1260 if (ConsumeAndStoreFunctionPrologue(Skipped))
1264 while (IsFunctionTryBlock && Tok.
is(tok::kw_catch)) {
1273 void Parser::ParseKNRParamDeclarations(Declarator &D) {
1283 while (isDeclarationSpecifier()) {
1288 ParseDeclarationSpecifiers(DS);
1296 Diag(DSStart, diag::err_declaration_does_not_declare_param);
1305 diag::err_invalid_storage_class_in_func_decl);
1310 diag::err_invalid_storage_class_in_func_decl);
1316 ParseDeclarator(ParmDeclarator);
1321 MaybeParseGNUAttributes(ParmDeclarator);
1333 for (
unsigned i = 0; ; ++i) {
1346 diag::err_param_redefinition)
1358 if (Tok.
isNot(tok::comma))
1361 ParmDeclarator.
clear();
1367 ParseDeclarator(ParmDeclarator);
1371 if (!ExpectAndConsumeSemi(diag::err_expected_semi_declaration))
1392 if (!isTokenStringLiteral()) {
1393 Diag(Tok, diag::err_expected_string_literal)
1398 ExprResult AsmString(ParseStringLiteralExpression());
1400 const auto *SL = cast<StringLiteral>(AsmString.
get());
1401 if (!SL->isAscii()) {
1402 Diag(Tok, diag::err_asm_operand_wide_string_literal)
1404 << SL->getSourceRange();
1417 assert(Tok.
is(tok::kw_asm) &&
"Not an asm!");
1420 if (Tok.
is(tok::kw_volatile)) {
1425 Diag(Tok, diag::warn_file_asm_volatile)
1432 Diag(Tok, diag::err_expected_lparen_after) <<
"asm";
1456 assert(tok.
is(tok::annot_template_id) &&
"Expected template-id token");
1462 void Parser::AnnotateScopeToken(
CXXScopeSpec &SS,
bool IsNewAnnotation) {
1469 Tok.
setKind(tok::annot_cxxscope);
1476 if (IsNewAnnotation)
1489 Parser::AnnotatedNameKind
1490 Parser::TryAnnotateName(
bool IsAddressOfOperand,
1491 std::unique_ptr<CorrectionCandidateCallback> CCC) {
1492 assert(Tok.
is(tok::identifier) || Tok.
is(tok::annot_cxxscope));
1494 const bool EnteringContext =
false;
1495 const bool WasScopeAnnotation = Tok.
is(tok::annot_cxxscope);
1499 ParseOptionalCXXScopeSpecifier(SS,
nullptr, EnteringContext))
1505 return ANK_Unresolved;
1513 if (isTentativelyDeclared(Name)) {
1518 return Tok.
is(tok::annot_typename) ? ANK_Success : ANK_TentativeDecl;
1528 getCurScope(), SS, Name, NameLoc, Next, IsAddressOfOperand,
1529 SS.
isEmpty() ? std::move(CCC) :
nullptr);
1531 switch (Classification.
getKind()) {
1541 AnnotateScopeToken(SS, !WasScopeAnnotation);
1564 = parseObjCTypeArgsAndProtocolQualifiers(IdentifierLoc, Ty,
1573 Tok.
setKind(tok::annot_typename);
1574 setTypeAnnotation(Tok, Ty);
1582 Tok.
setKind(tok::annot_primary_expr);
1591 if (Next.
isNot(tok::less)) {
1594 AnnotateScopeToken(SS, !WasScopeAnnotation);
1595 return ANK_TemplateName;
1604 if (AnnotateTemplateIdToken(
1612 llvm_unreachable(
"already parsed nested name specifier");
1617 AnnotateScopeToken(SS, !WasScopeAnnotation);
1618 return ANK_Unresolved;
1621 bool Parser::TryKeywordIdentFallback(
bool DisableKeyword) {
1622 assert(Tok.
isNot(tok::identifier));
1623 Diag(Tok, diag::ext_keyword_as_ident)
1655 assert((Tok.
is(tok::identifier) || Tok.
is(tok::coloncolon) ||
1656 Tok.
is(tok::kw_typename) || Tok.
is(tok::annot_cxxscope) ||
1657 Tok.
is(tok::kw_decltype) || Tok.
is(tok::annot_template_id) ||
1658 Tok.
is(tok::kw___super)) &&
1659 "Cannot be a type or scope token!");
1661 if (Tok.
is(tok::kw_typename)) {
1670 PP.
Lex(TypedefToken);
1675 Diag(Tok.
getLocation(), diag::warn_expected_qualified_after_typename);
1687 if (ParseOptionalCXXScopeSpecifier(SS,
nullptr,
1692 if (Tok.
is(tok::identifier) || Tok.
is(tok::annot_template_id) ||
1693 Tok.
is(tok::annot_decltype)) {
1695 if (Tok.
is(tok::annot_decltype) ||
1697 unsigned DiagID = diag::err_expected_qualified_after_typename;
1701 DiagID = diag::warn_expected_qualified_after_typename;
1714 if (Tok.
is(tok::identifier)) {
1719 }
else if (Tok.
is(tok::annot_template_id)) {
1723 Diag(Tok, diag::err_typename_refers_to_non_type_template)
1740 Diag(Tok, diag::err_expected_type_name_after_typename)
1746 Tok.
setKind(tok::annot_typename);
1747 setTypeAnnotation(Tok, Ty.
isInvalid() ? nullptr : Ty.
get());
1755 bool WasScopeAnnotation = Tok.
is(tok::annot_cxxscope);
1759 if (ParseOptionalCXXScopeSpecifier(SS,
nullptr,
false))
1770 if (Tok.
is(tok::identifier)) {
1785 (Ty.get()->isObjCObjectType() ||
1786 Ty.get()->isObjCObjectPointerType())) {
1791 = parseObjCTypeArgsAndProtocolQualifiers(IdentifierLoc, Ty,
1802 Tok.
setKind(tok::annot_typename);
1803 setTypeAnnotation(Tok, Ty);
1825 bool MemberOfUnknownSpecialization;
1828 false, TemplateName,
1829 nullptr,
false, Template,
1830 MemberOfUnknownSpecialization)) {
1849 if (Tok.
is(tok::annot_template_id)) {
1856 AnnotateTemplateIdTokenAsType();
1865 AnnotateScopeToken(SS, IsNewScope);
1877 "Call sites of this function should be guarded by checking for C++");
1878 assert((Tok.
is(tok::identifier) || Tok.
is(tok::coloncolon) ||
1879 (Tok.
is(tok::annot_template_id) &&
NextToken().
is(tok::coloncolon)) ||
1880 Tok.
is(tok::kw_decltype) || Tok.
is(tok::kw___super)) &&
1881 "Cannot be a type or scope token!");
1884 if (ParseOptionalCXXScopeSpecifier(SS,
nullptr, EnteringContext))
1889 AnnotateScopeToken(SS,
true);
1893 bool Parser::isTokenEqualOrEqualTypo() {
1899 case tok::starequal:
1900 case tok::plusequal:
1901 case tok::minusequal:
1902 case tok::exclaimequal:
1903 case tok::slashequal:
1904 case tok::percentequal:
1905 case tok::lessequal:
1906 case tok::lesslessequal:
1907 case tok::greaterequal:
1908 case tok::greatergreaterequal:
1909 case tok::caretequal:
1910 case tok::pipeequal:
1911 case tok::equalequal:
1912 Diag(Tok, diag::err_invalid_token_after_declarator_suggest_equal)
1922 assert(Tok.
is(tok::code_completion));
1930 return PrevTokLocation;
1936 return PrevTokLocation;
1942 return PrevTokLocation;
1947 void Parser::CodeCompleteDirective(
bool InConditional) {
1951 void Parser::CodeCompleteInConditionalExclusion() {
1955 void Parser::CodeCompleteMacroName(
bool IsDefinition) {
1959 void Parser::CodeCompletePreprocessorExpression() {
1965 unsigned ArgumentIndex) {
1970 void Parser::CodeCompleteIncludedFile(llvm::StringRef Dir,
bool IsAngled) {
1974 void Parser::CodeCompleteNaturalLanguage() {
1978 bool Parser::ParseMicrosoftIfExistsCondition(IfExistsCondition&
Result) {
1979 assert((Tok.
is(tok::kw___if_exists) || Tok.
is(tok::kw___if_not_exists)) &&
1980 "Expected '__if_exists' or '__if_not_exists'");
1981 Result.IsIfExists = Tok.
is(tok::kw___if_exists);
1986 Diag(Tok, diag::err_expected_lparen_after)
1987 << (Result.IsIfExists?
"__if_exists" :
"__if_not_exists");
1993 ParseOptionalCXXScopeSpecifier(Result.SS,
nullptr,
1997 if (Result.SS.isInvalid()) {
2005 Result.SS,
false,
true,
2006 true,
false,
nullptr,
2007 &TemplateKWLoc, Result.Name)) {
2017 Result.IsIfExists, Result.SS,
2020 Result.Behavior = Result.IsIfExists ? IEB_Parse : IEB_Skip;
2024 Result.Behavior = !Result.IsIfExists ? IEB_Parse : IEB_Skip;
2028 Result.Behavior = IEB_Dependent;
2038 void Parser::ParseMicrosoftIfExistsExternalDeclaration() {
2039 IfExistsCondition
Result;
2040 if (ParseMicrosoftIfExistsCondition(Result))
2045 Diag(Tok, diag::err_expected) << tok::l_brace;
2049 switch (Result.Behavior) {
2055 llvm_unreachable(
"Cannot have a dependent external declaration");
2064 while (Tok.
isNot(tok::r_brace) && !isEofOrEom()) {
2065 ParsedAttributesWithRange attrs(AttrFactory);
2066 MaybeParseCXX11Attributes(attrs);
2089 assert(Tok.
is(tok::kw_module) &&
"not a module declaration");
2092 if (Tok.
is(tok::identifier) &&
NextToken().
is(tok::identifier) &&
2103 if (ParseModuleName(ModuleLoc, Path,
false))
2107 ParsedAttributesWithRange Attrs(AttrFactory);
2108 MaybeParseCXX11Attributes(Attrs);
2109 ProhibitCXX11Attributes(Attrs, diag::err_attribute_not_module_attr);
2111 ExpectAndConsumeSemi(diag::err_module_expected_semi);
2125 assert((AtLoc.
isInvalid() ? Tok.
is(tok::kw_import)
2127 "Improper start to module import");
2133 if (ParseModuleName(ImportLoc, Path,
true))
2136 ParsedAttributesWithRange Attrs(AttrFactory);
2137 MaybeParseCXX11Attributes(Attrs);
2139 ProhibitCXX11Attributes(Attrs, diag::err_attribute_not_import_attr);
2148 ExpectAndConsumeSemi(diag::err_module_expected_semi);
2154 if (IsObjCAtImport && AtLoc.
isValid()) {
2157 if (FE && llvm::sys::path::parent_path(FE->getDir()->getName())
2158 .endswith(
".framework"))
2159 Diags.
Report(AtLoc, diag::warn_atimport_in_framework_header);
2162 return Import.
get();
2172 bool Parser::ParseModuleName(
2178 if (!Tok.
is(tok::identifier)) {
2179 if (Tok.
is(tok::code_completion)) {
2185 Diag(Tok, diag::err_module_expected_ident) << IsImport;
2194 if (Tok.
isNot(tok::period))
2205 bool Parser::parseMisplacedModuleImport() {
2208 case tok::annot_module_end:
2212 if (MisplacedModuleBeginCount) {
2213 --MisplacedModuleBeginCount;
2215 reinterpret_cast<Module *
>(
2217 ConsumeAnnotationToken();
2224 case tok::annot_module_begin:
2227 reinterpret_cast<Module *
>(
2229 ConsumeAnnotationToken();
2230 ++MisplacedModuleBeginCount;
2232 case tok::annot_module_include:
2236 reinterpret_cast<Module *
>(
2238 ConsumeAnnotationToken();
2248 bool BalancedDelimiterTracker::diagnoseOverflow() {
2249 P.Diag(P.Tok, diag::err_bracket_depth_exceeded)
2250 << P.getLangOpts().BracketDepth;
2251 P.Diag(P.Tok, diag::note_bracket_depth);
2259 LOpen = P.Tok.getLocation();
2260 if (P.ExpectAndConsume(Kind, DiagID, Msg)) {
2261 if (SkipToTok != tok::unknown)
2266 if (getDepth() < P.getLangOpts().BracketDepth)
2269 return diagnoseOverflow();
2272 bool BalancedDelimiterTracker::diagnoseMissingClose() {
2273 assert(!P.Tok.is(Close) &&
"Should have consumed closing delimiter");
2275 if (P.Tok.is(tok::annot_module_end))
2276 P.Diag(P.Tok, diag::err_missing_before_module_end) << Close;
2278 P.Diag(P.Tok, diag::err_expected) << Close;
2279 P.Diag(LOpen, diag::note_matching) <<
Kind;
2283 if (P.Tok.isNot(tok::r_paren) && P.Tok.isNot(tok::r_brace) &&
2284 P.Tok.isNot(tok::r_square) &&
2285 P.SkipUntil(Close, FinalToken,
2288 LClose = P.ConsumeAnyToken();
Defines the clang::ASTContext interface.
Represents a function declaration or definition.
DeclaratorChunk::FunctionTypeInfo & getFunctionTypeInfo()
getFunctionTypeInfo - Retrieves the function type info object (looking through parentheses).
SourceLocation getLocWithOffset(int Offset) const
Return a source location with the specified offset from this SourceLocation.
ParseScope - Introduces a new scope for parsing.
This is a scope that corresponds to the parameters within a function prototype.
void clear()
Reset the contents of this Declarator.
void Initialize()
Perform initialization that occurs after the parser has been initialized but before it parses anythin...
void Initialize()
Initialize - Warm up the parser.
IdentifierInfo * getIdentifierInfo(StringRef Name) const
Return information about the specified preprocessor identifier token.
Code completion occurs within a class, struct, or union.
IdentifierInfo * Name
FIXME: Temporarily stores the name of a specialization.
virtual Stmt * getBody() const
getBody - If this Decl represents a declaration for a body of code, such as a function or method defi...
SourceLocation TemplateNameLoc
TemplateNameLoc - The location of the template name within the source.
Stmt - This represents one statement.
TemplateNameKind isTemplateName(Scope *S, CXXScopeSpec &SS, bool hasTemplateKeyword, const UnqualifiedId &Name, ParsedType ObjectType, bool EnteringContext, TemplateTy &Template, bool &MemberOfUnknownSpecialization)
bool is(tok::TokenKind K) const
is/isNot - Predicates to check if this token is a specific kind, as in "if (Tok.is(tok::l_brace)) {...
The name refers to a dependent template name:
bool isEmpty() const
No scope specifier.
Decl - This represents one declaration (or definition), e.g.
bool isCodeCompletionReached() const
Returns true if code-completion is enabled and we have hit the code-completion point.
Defines the C++ template declaration subclasses.
void CodeCompleteNaturalLanguage()
bool TryAnnotateCXXScopeToken(bool EnteringContext=false)
TryAnnotateScopeToken - Like TryAnnotateTypeOrScopeToken but only annotates C++ scope specifiers and ...
SourceLocation getCloseLocation() const
This indicates that the scope corresponds to a function, which means that labels are set here...
DiagnosticBuilder Report(SourceLocation Loc, unsigned DiagID)
Issue the message to the client.
bool isFunctionDeclarator(unsigned &idx) const
isFunctionDeclarator - This method returns true if the declarator is a function declarator (looking t...
static const char * getSpecifierName(DeclSpec::TST T, const PrintingPolicy &Policy)
Turn a type-specifier-type into a string like "_Bool" or "union".
AccessSpecifier
A C++ access specifier (public, private, protected), plus the special value "none" which means differ...
TemplateNameKind Kind
The kind of template that Template refers to.
Wrapper for void* pointer.
Parser - This implements a parser for the C family of languages.
TSCS getThreadStorageClassSpec() const
NameClassificationKind getKind() const
const ParsedAttributes & getAttributes() const
void * SaveNestedNameSpecifierAnnotation(CXXScopeSpec &SS)
Given a C++ nested-name-specifier, produce an annotation value that the parser can use later to recon...
bool isKNRPrototype() const
isKNRPrototype - Return true if this is a K&R style identifier list, like "void foo(a,b,c)".
bool hadModuleLoaderFatalFailure() const
void ActOnDefaultCtorInitializers(Decl *CDtorDecl)
static const TSCS TSCS_unspecified
void EnterToken(const Token &Tok)
Enters a token in the token stream to be lexed next.
Information about one declarator, including the parsed type information and the identifier.
unsigned getParsedSpecifiers() const
Return a bitmask of which flavors of specifiers this DeclSpec includes.
TypeSpecifierType
Specifies the kind of type.
static const TST TST_interface
RAIIObject to destroy the contents of a SmallVector of TemplateIdAnnotation pointers and clear the ve...
Describes how types, statements, expressions, and declarations should be printed. ...
Code completion occurs within an Objective-C implementation or category implementation.
RAII object that makes sure paren/bracket/brace count is correct after declaration/statement parsing...
Decl * ActOnParamDeclarator(Scope *S, Declarator &D)
ActOnParamDeclarator - Called from Parser::ParseFunctionDeclarator() to introduce parameters into fun...
ParsedType getTypeName(const IdentifierInfo &II, SourceLocation NameLoc, Scope *S, CXXScopeSpec *SS=nullptr, bool isClassName=false, bool HasTrailingDot=false, ParsedType ObjectType=nullptr, bool IsCtorOrDtorName=false, bool WantNontrivialTypeSourceInfo=false, bool IsClassTemplateDeductionContext=true, IdentifierInfo **CorrectedII=nullptr)
If the identifier refers to a type name within this scope, return the declaration of that type...
bool canSkipFunctionBody(Decl *D)
Determine whether we can skip parsing the body of a function definition, assuming we don't care about...
bool isAnnotation() const
Return true if this is any of tok::annot_* kind tokens.
friend class ObjCDeclContextSwitch
void MarkAsLateParsedTemplate(FunctionDecl *FD, Decl *FnD, CachedTokens &Toks)
tok::TokenKind getKind() const
bool SkipUntil(tok::TokenKind T, SkipUntilFlags Flags=static_cast< SkipUntilFlags >(0))
SkipUntil - Read tokens until we get to the specified token, then consume it (unless StopBeforeMatch ...
Information about a template-id annotation token.
void CodeCompleteOrdinaryName(Scope *S, ParserCompletionContext CompletionContext)
Decl * ActOnStartOfFunctionDef(Scope *S, Declarator &D, MultiTemplateParamsArg TemplateParamLists, SkipBodyInfo *SkipBody=nullptr)
Represents a struct/union/class.
const Token & NextToken()
NextToken - This peeks ahead one token and returns it without consuming it.
SourceLocation getLocForEndOfToken(SourceLocation Loc, unsigned Offset=0)
Computes the source location just past the end of the token at this source location.
bool TryConsumeToken(tok::TokenKind Expected)
void SetPoisonReason(IdentifierInfo *II, unsigned DiagID)
Specifies the reason for poisoning an identifier.
One of these records is kept for each identifier that is lexed.
bool isStr(const char(&Str)[StrLen]) const
Return true if this is the identifier for the specified string.
SourceLocation getTypeSpecTypeLoc() const
static const TST TST_class
DeclGroupPtrTy ConvertDeclToDeclGroup(Decl *Ptr, Decl *OwnedType=nullptr)
void ActOnEndOfTranslationUnit()
ActOnEndOfTranslationUnit - This is called at the very end of the translation unit when EOF is reache...
Token - This structure provides full information about a lexed token.
static const TST TST_enum
void CodeCompletePreprocessorMacroArgument(Scope *S, IdentifierInfo *Macro, MacroInfo *MacroInfo, unsigned Argument)
TemplateName getTemplateName() const
void setKind(tok::TokenKind K)
RAII class that helps handle the parsing of an open/close delimiter pair, such as braces { ...
void removeCommentHandler(CommentHandler *Handler)
Remove the specified comment handler.
bool hasTagDefinition() const
void ClearStorageClassSpecs()
Describes a module or submodule.
Code completion occurs at top-level or namespace context.
static bool HasFlagsSet(Parser::SkipUntilFlags L, Parser::SkipUntilFlags R)
bool isObjCAtKeyword(tok::ObjCKeywordKind objcKey) const
Return true if we have an ObjC keyword identifier.
Code completion occurs within the body of a function on a recovery path, where we do not have a speci...
Represents a C++ unqualified-id that has been parsed.
StringRef getSpelling(SourceLocation loc, SmallVectorImpl< char > &buffer, bool *invalid=nullptr) const
Return the 'spelling' of the token at the given location; does not go up to the spelling location or ...
tok::TokenKind getTokenID() const
If this is a source-language token (e.g.
void SetRangeBegin(SourceLocation Loc)
SetRangeBegin - Set the start of the source range to Loc, unless it's invalid.
DeclGroupPtrTy BuildDeclaratorGroup(MutableArrayRef< Decl *> Group)
BuildDeclaratorGroup - convert a list of declarations into a declaration group, performing any necess...
ExprResult getExpression() const
const clang::PrintingPolicy & getPrintingPolicy() const
void CodeCompleteModuleImport(SourceLocation ImportLoc, ModuleIdPath Path)
SourceRange getAnnotationRange() const
SourceRange of the group of tokens that this annotation token represents.
ParsedTemplateArgument * getTemplateArgs()
Retrieves a pointer to the template arguments.
MutableArrayRef< TemplateParameterList * > MultiTemplateParamsArg
IfExistsResult CheckMicrosoftIfExistsSymbol(Scope *S, CXXScopeSpec &SS, const DeclarationNameInfo &TargetNameInfo)
const ParsingDeclSpec & getDeclSpec() const
void ActOnModuleInclude(SourceLocation DirectiveLoc, Module *Mod)
The parser has processed a module import translated from a #include or similar preprocessing directiv...
Scope - A scope is a transient data structure that is used while parsing the program.
Decl * ActOnFileScopeAsmDecl(Expr *expr, SourceLocation AsmLoc, SourceLocation RParenLoc)
Represents a C++ nested-name-specifier or a global scope specifier.
void CodeCompleteIncludedFile(llvm::StringRef Dir, bool IsAngled)
void setCodeCompletionHandler(CodeCompletionHandler &Handler)
Set the code completion handler to the given object.
SourceLocation ConsumeAnyToken(bool ConsumeCodeCompletionTok=false)
ConsumeAnyToken - Dispatch to the right Consume* method based on the current token type...
Decl * ActOnFinishFunctionBody(Decl *Decl, Stmt *Body)
SourceRange getSourceRange() const LLVM_READONLY
IdentifierInfo * getIdentifier() const
SourceLocation TemplateKWLoc
TemplateKWLoc - The location of the template keyword.
SourceLocation LAngleLoc
The location of the '<' before the template argument list.
Sema - This implements semantic analysis and AST building for C.
A little helper class used to produce diagnostics.
A class for parsing a declarator.
void clearCodeCompletionHandler()
Clear out the code completion handler.
unsigned NumParams
NumParams - This is the number of formal parameters specified by the declarator.
SourceRange getRange() const
void setAnnotationRange(SourceRange R)
TST getTypeSpecType() const
static bool isDeclRep(TST T)
'module partition X;'
void setAnnotationValue(void *val)
bool isCodeCompletionEnabled() const
Determine if we are performing code completion.
void CodeCompleteObjCMethodDecl(Scope *S, Optional< bool > IsInstanceMethod, ParsedType ReturnType)
Decl * ParsedFreeStandingDeclSpec(Scope *S, AccessSpecifier AS, DeclSpec &DS, RecordDecl *&AnonRecord)
ParsedFreeStandingDeclSpec - This method is invoked when a declspec with no declarator (e...
void AnnotateCachedTokens(const Token &Tok)
We notify the Preprocessor that if it is caching tokens (because backtrack is enabled) it should repl...
const FileEntry * getFileEntryForID(FileID FID) const
Returns the FileEntry record for the provided FileID.
This file defines the classes used to store parsed information about declaration-specifiers and decla...
void SkipMalformedDecl()
SkipMalformedDecl - Read tokens until we get to some likely good stopping point for skipping past a s...
bool ParseUnqualifiedId(CXXScopeSpec &SS, bool EnteringContext, bool AllowDestructorName, bool AllowConstructorName, bool AllowDeductionGuide, ParsedType ObjectType, SourceLocation *TemplateKWLoc, UnqualifiedId &Result)
Parse a C++ unqualified-id (or a C identifier), which describes the name of an entity.
void RevertCachedTokens(unsigned N)
When backtracking is enabled and tokens are cached, this allows to revert a specific number of tokens...
DeclGroupPtrTy ActOnModuleDecl(SourceLocation StartLoc, SourceLocation ModuleLoc, ModuleDeclKind MDK, ModuleIdPath Path)
The parser has processed a module-declaration that begins the definition of a module interface or imp...
SourceLocation getThreadStorageClassSpecLoc() const
SourceLocation getLocation() const
Return a source location identifier for the specified offset in the current file. ...
SourceLocation getBeginLoc() const
Decl * ActOnEmptyDeclaration(Scope *S, const ParsedAttributesView &AttrList, SourceLocation SemiLoc)
Handle a C++11 empty-declaration and attribute-declaration.
Represents a C++ template name within the type system.
This is a compound statement scope.
void CodeCompletePreprocessorExpression()
SourceLocation getStorageClassSpecLoc() const
const char * getPunctuatorSpelling(TokenKind Kind) LLVM_READNONE
Determines the spelling of simple punctuation tokens like '!' or '', and returns NULL for literal and...
TemplateNameKind
Specifies the kind of template name that an identifier refers to.
void SetDeclDeleted(Decl *dcl, SourceLocation DelLoc)
SourceLocation getEnd() const
Wraps an identifier and optional source location for the identifier.
The result type of a method or function.
const LangOptions & getLangOpts() const
NameClassification ClassifyName(Scope *S, CXXScopeSpec &SS, IdentifierInfo *&Name, SourceLocation NameLoc, const Token &NextToken, bool IsAddressOfOperand, std::unique_ptr< CorrectionCandidateCallback > CCC=nullptr)
Perform name lookup on the given name, classifying it based on the results of name lookup and the fol...
This is a scope that corresponds to the parameters within a function prototype for a function declara...
SourceManager & getSourceManager() const
void CheckForFunctionRedefinition(FunctionDecl *FD, const FunctionDecl *EffectiveDefinition=nullptr, SkipBodyInfo *SkipBody=nullptr)
Decl * ActOnSkippedFunctionBody(Decl *Decl)
A class for parsing a DeclSpec.
void ActOnFinishKNRParamDeclarations(Scope *S, Declarator &D, SourceLocation LocAfterDecls)
Stop skipping at semicolon.
ActionResult - This structure is used while parsing/acting on expressions, stmts, etc...
void TypoCorrectToken(const Token &Tok)
Update the current token to represent the provided identifier, in order to cache an action performed ...
FunctionDecl * getAsFunction() LLVM_READONLY
Returns the function itself, or the templated function if this is a function template.
SCS getStorageClassSpec() const
ASTContext & getASTContext() const
Encodes a location in the source.
IdentifierInfo & get(StringRef Name)
Return the identifier token info for the specified named identifier.
void setCurrentOpenCLExtensionForDecl(Decl *FD)
Set current OpenCL extensions for a declaration which can only be used when these OpenCL extensions a...
bool TryAnnotateTypeOrScopeToken()
TryAnnotateTypeOrScopeToken - If the current token position is on a typename (possibly qualified in C...
void ExitScope()
ExitScope - Pop a scope off the scope stack.
static const TST TST_union
IdentifierInfo * getIdentifierInfo() const
Parser(Preprocessor &PP, Sema &Actions, bool SkipFunctionBodies)
void setAnnotationEndLoc(SourceLocation L)
IdentifierTable & getIdentifierTable()
ParsedAttr - Represents a syntactic attribute.
ParsedType getType() const
void setIdentifierInfo(IdentifierInfo *II)
ExtensionRAIIObject - This saves the state of extension warnings when constructed and disables them...
void SetDeclDefaulted(Decl *dcl, SourceLocation DefaultLoc)
bool canDelayFunctionBody(const Declarator &D)
Determine whether we can delay parsing the body of a function or function template until it is used...
void CodeCompletePreprocessorDirective(bool InConditional)
void Lex(Token &Result)
Lex the next token for this preprocessor.
TemplateNameKind getTemplateNameKind() const
void EnterScope(unsigned ScopeFlags)
EnterScope - Start a new scope.
bool isAtStartOfLine() const
isAtStartOfLine - Return true if this token is at the start of a line.
TokenKind
Provides a simple uniform namespace for tokens from all C languages.
bool isObjCObjectPointerType() const
static bool IsCommonTypo(tok::TokenKind ExpectedTok, const Token &Tok)
void SetLateTemplateParser(LateTemplateParserCB *LTP, LateTemplateParserCleanupCB *LTPCleanup, void *P)
Scope * getCurScope() const
The name is a dependent name, so the results will differ from one instantiation to the next...
void Init(Scope *parent, unsigned flags)
Init - This is used by the parser to implement scope caching.
void ActOnStartOfTranslationUnit()
This is called before the very first declaration in the translation unit is parsed.
ASTConsumer & getASTConsumer() const
bool isInvalid() const
An error occurred during parsing of the scope specifier.
void CodeCompletePreprocessorMacroName(bool IsDefinition)
void setFunctionDefinitionKind(FunctionDefinitionKind Val)
The scope of a struct/union/class definition.
bool isNot(tok::TokenKind K) const
bool isBacktrackEnabled() const
True if EnableBacktrackAtThisPos() was called and caching of tokens is on.
Dataflow Directional Tag Classes.
bool isValid() const
Return true if this is a valid SourceLocation object.
'export module X;'
bool expectAndConsume(unsigned DiagID=diag::err_expected, const char *Msg="", tok::TokenKind SkipToTok=tok::unknown)
const Scope * getParent() const
getParent - Return the scope that this is nested in.
void addCommentHandler(CommentHandler *Handler)
Add the specified comment handler to the preprocessor.
SourceLocation RAngleLoc
The location of the '>' after the template argument list.
void ActOnModuleBegin(SourceLocation DirectiveLoc, Module *Mod)
The parsed has entered a submodule.
static FixItHint CreateRemoval(CharSourceRange RemoveRange)
Create a code modification hint that removes the given source range.
SkipUntilFlags
Control flags for SkipUntil functions.
TypeResult ActOnTypenameType(Scope *S, SourceLocation TypenameLoc, const CXXScopeSpec &SS, const IdentifierInfo &II, SourceLocation IdLoc)
Called when the parser has parsed a C++ typename specifier, e.g., "typename T::type".
ExternalASTSource * getExternalSource() const
Retrieve a pointer to the external AST source associated with this AST context, if any...
The name refers to a template whose specialization produces a type.
static const TST TST_unspecified
Encapsulates the data about a macro definition (e.g.
void takeAttributesFrom(ParsedAttributes &attrs)
DiagnosticBuilder Diag(SourceLocation Loc, unsigned DiagID)
static FixItHint CreateInsertion(SourceLocation InsertionLoc, StringRef Code, bool BeforePreviousInsertions=false)
Create a code modification hint that inserts the given code string at a specific location.
SmallVector< TemplateParameterList *, 4 > TemplateParameterLists
DeclResult ActOnModuleImport(SourceLocation AtLoc, SourceLocation ImportLoc, ModuleIdPath Path)
The parser has processed a module import declaration.
This is a scope that can contain a declaration.
bool isObjCObjectType() const
bool SetTypeSpecType(TST T, SourceLocation Loc, const char *&PrevSpec, unsigned &DiagID, const PrintingPolicy &Policy)
NamedDecl * HandleDeclarator(Scope *S, Declarator &D, MultiTemplateParamsArg TemplateParameterLists)
SourceLocation getIdentifierLoc() const
void ActOnTranslationUnitScope(Scope *S)
bool isSet() const
Deprecated.
ParsingDeclSpec & getMutableDeclSpec() const
Captures information about "declaration specifiers".
SourceLocation ConsumeToken()
ConsumeToken - Consume the current 'peek token' and lex the next one.
SourceLocation getLastLoc() const
bool isNotEmpty() const
A scope specifier is present, but may be valid or invalid.
DeclContext * CurContext
CurContext - This is the current declaration context of parsing.
void ActOnPopScope(SourceLocation Loc, Scope *S)
Scope actions.
static FixItHint CreateReplacement(CharSourceRange RemoveRange, StringRef Code)
Create a code modification hint that replaces the given source range with the given code string...
void revertTokenIDToIdentifier()
Revert TokenID to tok::identifier; used for GNU libstdc++ 4.2 compatibility.
bool isIncrementalProcessingEnabled() const
Returns true if incremental processing is enabled.
bool TryAnnotateTypeOrScopeTokenAfterScopeSpec(CXXScopeSpec &SS, bool IsNewScope)
Try to annotate a type or scope token, having already parsed an optional scope specifier.
void ActOnComment(SourceRange Comment)
static OpaquePtr make(TemplateName P)
bool isEmpty() const
isEmpty - Return true if this declaration specifier is completely empty: no tokens were parsed in the...
virtual bool HandleTopLevelDecl(DeclGroupRef D)
HandleTopLevelDecl - Handle the specified top-level declaration.
static const TST TST_struct
void setLocation(SourceLocation L)
void ActOnModuleEnd(SourceLocation DirectiveLoc, Module *Mod)
The parser has left a submodule.
A trivial tuple used to represent a source range.
void setIdentifier(const IdentifierInfo *Id, SourceLocation IdLoc)
Specify that this unqualified-id was parsed as an identifier.
bool isTranslationUnit() const
bool isEditorPlaceholder() const
Returns true if this token is an editor placeholder.
unsigned NumArgs
NumArgs - The number of template arguments.
void * getAnnotationValue() const
SourceLocation getBegin() const
void setCommaLoc(SourceLocation CL)
The symbol does not exist.
void CodeCompleteInPreprocessorConditionalExclusion(Scope *S)
ParamInfo * Params
Params - This is a pointer to a new[]'d array of ParamInfo objects that describe the parameters speci...
ParsedAttributes & getAttributes()
void startToken()
Reset all flags to cleared.
ParsedTemplateTy Template
The declaration of the template corresponding to the template-name.
bool ParseFirstTopLevelDecl(DeclGroupPtrTy &Result)
Parse the first top-level declaration in a translation unit.
Engages in a tight little dance with the lexer to efficiently preprocess tokens.
Stop skipping at specified token, but don't skip the token itself.