20 #include "llvm/ADT/STLExtras.h" 21 #include "llvm/Support/Debug.h" 24 #define DEBUG_TYPE "format-token-breaker" 29 static const char *
const Blanks =
" \t\v\f\r";
45 static const char *
const KnownCStylePrefixes[] = {
"///<",
"//!<",
"///",
"//",
47 static const char *
const KnownTextProtoPrefixes[] = {
"//",
"#",
"##",
"###",
51 KnownPrefixes = KnownTextProtoPrefixes;
53 StringRef LongestPrefix;
54 for (StringRef KnownPrefix : KnownPrefixes) {
55 if (Comment.startswith(KnownPrefix)) {
56 size_t PrefixLength = KnownPrefix.size();
57 while (PrefixLength < Comment.size() && Comment[PrefixLength] ==
' ')
59 if (PrefixLength > LongestPrefix.size())
60 LongestPrefix = Comment.substr(0, PrefixLength);
67 unsigned ContentStartColumn,
72 LLVM_DEBUG(llvm::dbgs() <<
"Comment split: \"" << Text
73 <<
"\", Column limit: " << ColumnLimit
74 <<
", Content start: " << ContentStartColumn <<
"\n");
75 if (ColumnLimit <= ContentStartColumn + 1)
76 return BreakableToken::Split(StringRef::npos, 0);
78 unsigned MaxSplit = ColumnLimit - ContentStartColumn + 1;
79 unsigned MaxSplitBytes = 0;
81 for (
unsigned NumChars = 0;
82 NumChars < MaxSplit && MaxSplitBytes < Text.size();) {
83 unsigned BytesInChar =
87 ContentStartColumn, TabWidth,
Encoding);
88 MaxSplitBytes += BytesInChar;
91 StringRef::size_type SpaceOffset = Text.find_last_of(Blanks, MaxSplitBytes);
93 static auto *
const kNumberedListRegexp =
new llvm::Regex(
"^[1-9][0-9]?\\.");
94 while (SpaceOffset != StringRef::npos) {
97 if (kNumberedListRegexp->match(Text.substr(SpaceOffset).ltrim(Blanks)))
98 SpaceOffset = Text.find_last_of(Blanks, SpaceOffset);
103 SpaceOffset + 1 < Text.size() && Text[SpaceOffset + 1] ==
'{')
104 SpaceOffset = Text.find_last_of(Blanks, SpaceOffset);
109 if (SpaceOffset == StringRef::npos ||
111 Text.find_last_not_of(Blanks, SpaceOffset) == StringRef::npos) {
114 StringRef::size_type FirstNonWhitespace = Text.find_first_not_of(Blanks);
115 if (FirstNonWhitespace == StringRef::npos)
117 return BreakableToken::Split(StringRef::npos, 0);
118 SpaceOffset = Text.find_first_of(
119 Blanks, std::max<unsigned>(MaxSplitBytes, FirstNonWhitespace));
121 if (SpaceOffset != StringRef::npos && SpaceOffset != 0) {
126 if (SpaceOffset == 1 && Text[SpaceOffset - 1] ==
'*')
127 return BreakableToken::Split(StringRef::npos, 0);
128 StringRef BeforeCut = Text.substr(0, SpaceOffset).rtrim(Blanks);
129 StringRef AfterCut = Text.substr(SpaceOffset).ltrim(Blanks);
130 return BreakableToken::Split(BeforeCut.size(),
131 AfterCut.begin() - BeforeCut.end());
133 return BreakableToken::Split(StringRef::npos, 0);
136 static BreakableToken::Split
141 return BreakableToken::Split(StringRef::npos, 0);
142 if (ColumnLimit <= UsedColumns)
143 return BreakableToken::Split(StringRef::npos, 0);
144 unsigned MaxSplit = ColumnLimit - UsedColumns;
145 StringRef::size_type SpaceOffset = 0;
146 StringRef::size_type SlashOffset = 0;
147 StringRef::size_type WordStartOffset = 0;
148 StringRef::size_type SplitPoint = 0;
149 for (
unsigned Chars = 0;;) {
151 if (Text[0] ==
'\\') {
157 Text.substr(0, Advance), UsedColumns + Chars, TabWidth,
Encoding);
160 if (Chars > MaxSplit || Text.size() <= Advance)
164 SpaceOffset = SplitPoint;
166 SlashOffset = SplitPoint;
168 WordStartOffset = SplitPoint;
170 SplitPoint += Advance;
171 Text = Text.substr(Advance);
174 if (SpaceOffset != 0)
175 return BreakableToken::Split(SpaceOffset + 1, 0);
176 if (SlashOffset != 0)
177 return BreakableToken::Split(SlashOffset + 1, 0);
178 if (WordStartOffset != 0)
179 return BreakableToken::Split(WordStartOffset + 1, 0);
181 return BreakableToken::Split(SplitPoint, 0);
182 return BreakableToken::Split(StringRef::npos, 0);
186 assert((Token.
is(TT_BlockComment) || Token.
is(TT_LineComment)) &&
187 "formatting regions are switched by comment tokens");
188 StringRef Content = Token.
TokenText.substr(2).ltrim();
189 return Content.startswith(
"clang-format on") ||
190 Content.startswith(
"clang-format off");
207 return RemainingTokenColumns + 1 - Split.second;
214 StringRef::size_type Length,
216 llvm_unreachable(
"Getting the length of a part of the string literal " 217 "indicates that the code tries to reflow it.");
238 StartColumn(StartColumn), Prefix(Prefix), Postfix(Postfix),
239 UnbreakableTailLength(UnbreakableTailLength) {
242 Prefix.size(), Tok.
TokenText.size() - Prefix.size() - Postfix.size());
246 unsigned LineIndex,
unsigned TailOffset,
unsigned ColumnLimit,
247 unsigned ContentStartColumn, llvm::Regex &CommentPragmasRegex)
const {
253 unsigned TailOffset, Split Split,
254 unsigned ContentIndent,
266 StartColumn(StartColumn) {}
270 BreakableToken::Split
272 unsigned ColumnLimit,
unsigned ContentStartColumn,
273 llvm::Regex &CommentPragmasRegex)
const {
275 if (CommentPragmasRegex.match(
Content[LineIndex]))
276 return Split(StringRef::npos, 0);
283 unsigned LineIndex,
unsigned TailOffset, Split Split,
285 StringRef
Text =
Content[LineIndex].substr(TailOffset);
290 unsigned BreakOffsetInToken =
292 unsigned CharsToRemove = Split.second;
294 tokenAt(LineIndex), BreakOffsetInToken, CharsToRemove,
"",
"",
303 Content = Content.trim(Blanks);
306 bool hasSpecialMeaningPrefix =
false;
307 for (StringRef Prefix :
308 {
"@",
"TODO",
"FIXME",
"XXX",
"-# ",
"- ",
"+ ",
"* "}) {
309 if (Content.startswith(Prefix)) {
310 hasSpecialMeaningPrefix =
true;
318 static auto *
const kNumberedListRegexp =
new llvm::Regex(
"^[1-9][0-9]?\\. ");
319 hasSpecialMeaningPrefix =
320 hasSpecialMeaningPrefix || kNumberedListRegexp->match(Content);
325 return Content.size() >= 2 && !hasSpecialMeaningPrefix &&
326 !Content.endswith(
"\\") &&
334 unsigned OriginalStartColumn,
bool FirstInLine,
bool InPPDirective,
337 DelimitersOnNewline(
false),
338 UnbreakableTailLength(Token.UnbreakableTailLength) {
339 assert(
Tok.
is(TT_BlockComment) &&
340 "block comment section must start with a block comment");
343 assert(TokenText.startswith(
"/*") && TokenText.endswith(
"*/"));
344 TokenText.substr(2, TokenText.size() - 4).split(
Lines,
"\n");
346 int IndentDelta = StartColumn - OriginalStartColumn;
353 for (
size_t i = 1; i <
Lines.size(); ++i)
354 adjustWhitespace(i, IndentDelta);
358 DecorationColumn = StartColumn + 1;
371 if (
Lines.size() == 1 && !FirstInLine) {
379 for (
size_t i = 1, e =
Lines.size(); i < e && !Decoration.empty(); ++i) {
381 if (i + 1 == e &&
Content[i].empty())
383 if (!
Content[i].empty() && i + 1 != e && Decoration.startswith(
Content[i]))
385 while (!
Content[i].startswith(Decoration))
386 Decoration = Decoration.substr(0, Decoration.size() - 1);
389 LastLineNeedsDecoration =
true;
391 for (
size_t i = 1, e =
Lines.size(); i < e; ++i) {
397 LastLineNeedsDecoration =
false;
399 if (e >= 2 && !Decoration.empty()) {
402 }
else if (Decoration.empty()) {
414 unsigned DecorationSize = Decoration.startswith(
Content[i])
417 if (DecorationSize) {
421 if (!Decoration.startswith(
Content[i]))
425 IndentAtLineBreak = std::max<unsigned>(IndentAtLineBreak, Decoration.size());
430 if ((
Lines[0] ==
"*" ||
Lines[0].startswith(
"* ")) &&
Lines.size() > 1) {
432 DelimitersOnNewline =
true;
433 }
else if (
Lines[0].startswith(
"* ") &&
Lines.size() == 1) {
442 DelimitersOnNewline = EndColumn > Style.
ColumnLimit;
447 llvm::dbgs() <<
"IndentAtLineBreak " << IndentAtLineBreak <<
"\n";
448 llvm::dbgs() <<
"DelimitersOnNewline " << DelimitersOnNewline <<
"\n";
449 for (
size_t i = 0; i <
Lines.size(); ++i) {
450 llvm::dbgs() << i <<
" |" <<
Content[i] <<
"| " 452 <<
"IN=" << (
Content[i].data() -
Lines[i].data()) <<
"\n";
457 void BreakableBlockComment::adjustWhitespace(
unsigned LineIndex,
464 size_t EndOfPreviousLine =
Lines[LineIndex - 1].size();
470 Lines[LineIndex - 1].find_last_not_of(Blanks, EndOfPreviousLine);
471 if (EndOfPreviousLine == StringRef::npos)
472 EndOfPreviousLine = 0;
476 size_t StartOfLine =
Lines[LineIndex].find_first_not_of(Blanks);
477 if (StartOfLine == StringRef::npos)
478 StartOfLine =
Lines[LineIndex].rtrim(
"\r\n").size();
480 StringRef Whitespace =
Lines[LineIndex].substr(0, StartOfLine);
482 size_t PreviousContentOffset =
483 Content[LineIndex - 1].data() -
Lines[LineIndex - 1].data();
485 PreviousContentOffset, EndOfPreviousLine - PreviousContentOffset);
486 Content[LineIndex] =
Lines[LineIndex].substr(StartOfLine);
496 StringRef::size_type Length,
498 unsigned LineLength =
504 if (LineIndex + 1 ==
Lines.size()) {
509 if (
Content[LineIndex].substr(Offset, StringRef::npos).empty()) {
510 LineLength -= Decoration.size();
519 return UnbreakableTailLength +
526 return IndentAtLineBreak;
530 const llvm::StringSet<>
532 "@param",
"@return",
"@returns",
"@throws",
"@type",
"@template",
533 "@see",
"@deprecated",
"@define",
"@exports",
"@mods",
"@private",
543 StringRef ContentWithNoDecoration =
Content[LineIndex];
544 if (LineIndex == 0 && ContentWithNoDecoration.startswith(
"*")) {
545 ContentWithNoDecoration = ContentWithNoDecoration.substr(1).ltrim(Blanks);
547 StringRef FirstWord = ContentWithNoDecoration.substr(
548 0, ContentWithNoDecoration.find_first_of(Blanks));
556 Split Split,
unsigned ContentIndent,
558 StringRef
Text =
Content[LineIndex].substr(TailOffset);
559 StringRef Prefix = Decoration;
563 unsigned LocalIndentAtLineBreak = IndentAtLineBreak;
564 if (LineIndex + 1 ==
Lines.size() &&
565 Text.size() == Split.first + Split.second) {
568 if (LocalIndentAtLineBreak >= 2)
569 LocalIndentAtLineBreak -= 2;
573 unsigned BreakOffsetInToken =
575 unsigned CharsToRemove = Split.second;
576 assert(LocalIndentAtLineBreak >= Prefix.size());
577 std::string PrefixWithTrailingIndent = Prefix;
578 for (
unsigned I = 0; I < ContentIndent; ++I)
579 PrefixWithTrailingIndent +=
" ";
581 tokenAt(LineIndex), BreakOffsetInToken, CharsToRemove,
"",
583 LocalIndentAtLineBreak + ContentIndent -
584 PrefixWithTrailingIndent.size());
587 BreakableToken::Split
589 llvm::Regex &CommentPragmasRegex)
const {
590 if (!
mayReflow(LineIndex, CommentPragmasRegex))
591 return Split(StringRef::npos, 0);
595 size_t Trimmed =
Content[LineIndex].find_first_not_of(Blanks);
598 if (PreviousContentIndent && Trimmed != StringRef::npos &&
599 Trimmed != PreviousContentIndent)
600 return Split(StringRef::npos, 0);
603 return Split(0, Trimmed != StringRef::npos ? Trimmed : 0);
608 return DelimitersOnNewline &&
609 Lines[0].substr(1).find_first_not_of(Blanks) != StringRef::npos;
614 StringRef TrimmedContent =
Content[LineIndex].ltrim(Blanks);
617 "Reflowing whitespace within a token");
620 unsigned WhitespaceOffsetInToken =
Content[LineIndex - 1].data() +
621 Content[LineIndex - 1].size() -
623 unsigned WhitespaceLength = TrimmedContent.data() -
625 WhitespaceOffsetInToken;
627 tokenAt(LineIndex), WhitespaceOffsetInToken,
628 WhitespaceLength,
"",
635 if (LineIndex == 0) {
636 if (DelimitersOnNewline) {
641 size_t BreakLength =
Lines[0].substr(1).find_first_not_of(Blanks);
642 if (BreakLength != StringRef::npos)
650 StringRef Prefix = Decoration;
651 if (
Content[LineIndex].empty()) {
652 if (LineIndex + 1 ==
Lines.size()) {
653 if (!LastLineNeedsDecoration) {
658 }
else if (!Decoration.empty()) {
661 Prefix = Prefix.substr(0, 1);
666 Prefix = Prefix.substr(0, 1);
671 unsigned WhitespaceOffsetInToken =
Content[LineIndex - 1].data() +
672 Content[LineIndex - 1].size() -
674 unsigned WhitespaceLength =
Content[LineIndex].data() -
676 WhitespaceOffsetInToken;
678 tokenAt(LineIndex), WhitespaceOffsetInToken, WhitespaceLength,
"", Prefix,
682 BreakableToken::Split
684 if (DelimitersOnNewline) {
688 StringRef
Line =
Content.back().substr(TailOffset);
689 StringRef TrimmedLine = Line.rtrim(Blanks);
690 if (!TrimmedLine.empty())
691 return Split(TrimmedLine.size(), Line.size() - TrimmedLine.size());
693 return Split(StringRef::npos, 0);
697 llvm::Regex &CommentPragmasRegex)
const {
700 StringRef IndentContent =
Content[LineIndex];
701 if (
Lines[LineIndex].ltrim(Blanks).startswith(
"*")) {
702 IndentContent =
Lines[LineIndex].ltrim(Blanks).substr(1);
704 return LineIndex > 0 && !CommentPragmasRegex.match(IndentContent) &&
711 unsigned OriginalStartColumn,
bool FirstInLine,
bool InPPDirective,
714 assert(
Tok.
is(TT_LineComment) &&
715 "line comment section must start with a line comment");
718 CurrentTok && CurrentTok->
is(TT_LineComment);
719 CurrentTok = CurrentTok->Next) {
720 LastLineTok = LineTok;
721 StringRef TokenText(CurrentTok->TokenText);
722 assert((TokenText.startswith(
"//") || TokenText.startswith(
"#")) &&
723 "unsupported line comment prefix, '//' and '#' are supported");
724 size_t FirstLineIndex =
Lines.size();
725 TokenText.split(
Lines,
"\n");
728 OriginalContentColumn.resize(
Lines.size());
730 Prefix.resize(
Lines.size());
731 OriginalPrefix.resize(
Lines.size());
732 for (
size_t i = FirstLineIndex, e =
Lines.size(); i < e; ++i) {
736 StringRef IndentPrefix =
738 assert((TokenText.startswith(
"//") || TokenText.startswith(
"#")) &&
739 "unsupported line comment prefix, '//' and '#' are supported");
740 OriginalPrefix[i] = Prefix[i] = IndentPrefix;
741 if (
Lines[i].size() > Prefix[i].size() &&
743 if (Prefix[i] ==
"//")
745 else if (Prefix[i] ==
"///")
747 else if (Prefix[i] ==
"//!")
749 else if (Prefix[i] ==
"///<")
751 else if (Prefix[i] ==
"//!<")
753 else if (Prefix[i] ==
"#" &&
760 OriginalContentColumn[i] =
769 size_t EndOfLine =
Content[i].find_last_not_of(Blanks);
770 if (EndOfLine == StringRef::npos)
776 LineTok = CurrentTok->Next;
777 if (CurrentTok->Next && !CurrentTok->Next->ContinuesLineCommentSection) {
799 StringRef::size_type Length,
809 return OriginalContentColumn[LineIndex];
814 unsigned LineIndex,
unsigned TailOffset, Split Split,
816 StringRef
Text =
Content[LineIndex].substr(TailOffset);
819 unsigned BreakOffsetInToken =
821 unsigned CharsToRemove = Split.second;
824 unsigned IndentAtLineBreak = OriginalContentColumn[LineIndex] +
825 Prefix[LineIndex].size() -
826 OriginalPrefix[LineIndex].size();
827 assert(IndentAtLineBreak >= Prefix[LineIndex].size());
829 tokenAt(LineIndex), BreakOffsetInToken, CharsToRemove,
"",
831 IndentAtLineBreak - Prefix[LineIndex].size());
835 unsigned LineIndex, llvm::Regex &CommentPragmasRegex)
const {
836 if (!
mayReflow(LineIndex, CommentPragmasRegex))
837 return Split(StringRef::npos, 0);
839 size_t Trimmed =
Content[LineIndex].find_first_not_of(Blanks);
845 return Split(0, Trimmed != StringRef::npos ? Trimmed : 0);
850 if (LineIndex > 0 &&
Tokens[LineIndex] !=
Tokens[LineIndex - 1]) {
856 }
else if (LineIndex > 0) {
868 Lines[LineIndex - 1].size() -
872 unsigned WhitespaceLength =
887 unsigned WhitespaceLength =
907 if (LineIndex > 0 &&
Tokens[LineIndex] !=
Tokens[LineIndex - 1]) {
913 unsigned LineColumn =
916 (OriginalPrefix[LineIndex].size() - Prefix[LineIndex].size());
928 if (OriginalPrefix[LineIndex] != Prefix[LineIndex]) {
932 assert(Prefix[LineIndex] == (OriginalPrefix[LineIndex] +
" ").
str() &&
933 "Expecting a line comment prefix to differ from original by at most " 936 tokenAt(LineIndex), OriginalPrefix[LineIndex].size(), 0,
"",
"",
948 unsigned LineIndex, llvm::Regex &CommentPragmasRegex)
const {
951 StringRef IndentContent =
Content[LineIndex];
952 if (
Lines[LineIndex].startswith(
"//")) {
953 IndentContent =
Lines[LineIndex].substr(2);
961 return LineIndex > 0 && !CommentPragmasRegex.match(IndentContent) &&
964 OriginalPrefix[LineIndex] == OriginalPrefix[LineIndex - 1];
Declares BreakableToken, BreakableStringLiteral, BreakableComment, BreakableBlockComment and Breakabl...
Token - This structure provides full information about a lexed token.
const AnnotatedLine * Line
LLVM_READONLY bool isAlphanumeric(unsigned char c)
Return true if this character is an ASCII letter or digit: [a-zA-Z0-9].
Dataflow Directional Tag Classes.
LLVM_READONLY bool isPunctuation(unsigned char c)
Return true if this character is an ASCII punctuation character.
__DEVICE__ int max(int __a, int __b)
This file implements an indenter that manages the indentation of continuations.