head 1.1; branch 1.1.1; access; symbols netbsd-11-0-RC4:1.1.1.3 netbsd-11-0-RC3:1.1.1.3 netbsd-11-0-RC2:1.1.1.3 netbsd-11-0-RC1:1.1.1.3 perseant-exfatfs-base-20250801:1.1.1.3 netbsd-11:1.1.1.3.0.10 netbsd-11-base:1.1.1.3 netbsd-10-1-RELEASE:1.1.1.3 perseant-exfatfs-base-20240630:1.1.1.3 perseant-exfatfs:1.1.1.3.0.8 perseant-exfatfs-base:1.1.1.3 netbsd-8-3-RELEASE:1.1.1.1 netbsd-9-4-RELEASE:1.1.1.2 netbsd-10-0-RELEASE:1.1.1.3 netbsd-10-0-RC6:1.1.1.3 netbsd-10-0-RC5:1.1.1.3 netbsd-10-0-RC4:1.1.1.3 netbsd-10-0-RC3:1.1.1.3 netbsd-10-0-RC2:1.1.1.3 netbsd-10-0-RC1:1.1.1.3 netbsd-10:1.1.1.3.0.6 netbsd-10-base:1.1.1.3 netbsd-9-3-RELEASE:1.1.1.2 cjep_sun2x:1.1.1.3.0.4 cjep_sun2x-base:1.1.1.3 cjep_staticlib_x-base1:1.1.1.3 netbsd-9-2-RELEASE:1.1.1.2 cjep_staticlib_x:1.1.1.3.0.2 cjep_staticlib_x-base:1.1.1.3 netbsd-9-1-RELEASE:1.1.1.2 phil-wifi-20200421:1.1.1.3 phil-wifi-20200411:1.1.1.3 phil-wifi-20200406:1.1.1.3 netbsd-8-2-RELEASE:1.1.1.1 netbsd-9-0-RELEASE:1.1.1.2 netbsd-9-0-RC2:1.1.1.2 netbsd-9-0-RC1:1.1.1.2 netbsd-9:1.1.1.2.0.2 netbsd-9-base:1.1.1.2 phil-wifi-20190609:1.1.1.2 netbsd-8-1-RELEASE:1.1.1.1 netbsd-8-1-RC1:1.1.1.1 pgoyette-compat-merge-20190127:1.1.1.1.14.1 pgoyette-compat-20190127:1.1.1.2 pgoyette-compat-20190118:1.1.1.2 pgoyette-compat-1226:1.1.1.2 pgoyette-compat-1126:1.1.1.2 pgoyette-compat-1020:1.1.1.2 pgoyette-compat-0930:1.1.1.2 pgoyette-compat-0906:1.1.1.2 pgoyette-compat-0728:1.1.1.2 clang-337282:1.1.1.2 netbsd-8-0-RELEASE:1.1.1.1 phil-wifi:1.1.1.1.0.16 phil-wifi-base:1.1.1.1 pgoyette-compat-0625:1.1.1.1 netbsd-8-0-RC2:1.1.1.1 pgoyette-compat-0521:1.1.1.1 pgoyette-compat-0502:1.1.1.1 pgoyette-compat-0422:1.1.1.1 netbsd-8-0-RC1:1.1.1.1 pgoyette-compat-0415:1.1.1.1 pgoyette-compat-0407:1.1.1.1 pgoyette-compat-0330:1.1.1.1 pgoyette-compat-0322:1.1.1.1 pgoyette-compat-0315:1.1.1.1 pgoyette-compat:1.1.1.1.0.14 pgoyette-compat-base:1.1.1.1 clang-319952:1.1.1.1 matt-nb8-mediatek:1.1.1.1.0.12 matt-nb8-mediatek-base:1.1.1.1 clang-309604:1.1.1.1 perseant-stdc-iso10646:1.1.1.1.0.10 perseant-stdc-iso10646-base:1.1.1.1 netbsd-8:1.1.1.1.0.8 netbsd-8-base:1.1.1.1 prg-localcount2-base3:1.1.1.1 prg-localcount2-base2:1.1.1.1 prg-localcount2-base1:1.1.1.1 prg-localcount2:1.1.1.1.0.6 prg-localcount2-base:1.1.1.1 pgoyette-localcount-20170426:1.1.1.1 bouyer-socketcan-base1:1.1.1.1 pgoyette-localcount:1.1.1.1.0.4 pgoyette-localcount-20170320:1.1.1.1 clang-294123:1.1.1.1 bouyer-socketcan:1.1.1.1.0.2 bouyer-socketcan-base:1.1.1.1 clang-291444:1.1.1.1 LLVM:1.1.1; locks; strict; comment @// @; 1.1 date 2017.01.11.10.33.37; author joerg; state Exp; branches 1.1.1.1; next ; commitid CNnUNfII1jgNmxBz; 1.1.1.1 date 2017.01.11.10.33.37; author joerg; state Exp; branches 1.1.1.1.4.1 1.1.1.1.14.1 1.1.1.1.16.1; next 1.1.1.2; commitid CNnUNfII1jgNmxBz; 1.1.1.2 date 2018.07.17.18.30.58; author joerg; state Exp; branches; next 1.1.1.3; commitid wDzL46ALjrCZgwKA; 1.1.1.3 date 2019.11.13.22.19.23; author joerg; state dead; branches; next ; commitid QD8YATxuNG34YJKB; 1.1.1.1.4.1 date 2017.01.11.10.33.37; author pgoyette; state dead; branches; next 1.1.1.1.4.2; commitid jjw7cAwgyKq7RfKz; 1.1.1.1.4.2 date 2017.03.20.06.52.38; author pgoyette; state Exp; branches; next ; commitid jjw7cAwgyKq7RfKz; 1.1.1.1.14.1 date 2018.07.28.04.33.19; author pgoyette; state Exp; branches; next ; commitid 1UP1xAIUxv1ZgRLA; 1.1.1.1.16.1 date 2019.06.10.21.45.23; author christos; state Exp; branches; next 1.1.1.1.16.2; commitid jtc8rnCzWiEEHGqB; 1.1.1.1.16.2 date 2020.04.13.07.46.33; author martin; state dead; branches; next ; commitid X01YhRUPVUDaec4C; desc @@ 1.1 log @Initial revision @ text @//===--- TokenAnalyzer.cpp - Analyze Token Streams --------------*- C++ -*-===// // // The LLVM Compiler Infrastructure // // This file is distributed under the University of Illinois Open Source // License. See LICENSE.TXT for details. // //===----------------------------------------------------------------------===// /// /// \file /// \brief This file implements an abstract TokenAnalyzer and associated helper /// classes. TokenAnalyzer can be extended to generate replacements based on /// an annotated and pre-processed token stream. /// //===----------------------------------------------------------------------===// #include "TokenAnalyzer.h" #include "AffectedRangeManager.h" #include "Encoding.h" #include "FormatToken.h" #include "FormatTokenLexer.h" #include "TokenAnnotator.h" #include "UnwrappedLineParser.h" #include "clang/Basic/Diagnostic.h" #include "clang/Basic/DiagnosticOptions.h" #include "clang/Basic/FileManager.h" #include "clang/Basic/SourceManager.h" #include "clang/Format/Format.h" #include "llvm/ADT/STLExtras.h" #include "llvm/Support/Debug.h" #define DEBUG_TYPE "format-formatter" namespace clang { namespace format { // This sets up an virtual file system with file \p FileName containing \p // Code. std::unique_ptr Environment::CreateVirtualEnvironment(StringRef Code, StringRef FileName, ArrayRef Ranges) { // This is referenced by `FileMgr` and will be released by `FileMgr` when it // is deleted. IntrusiveRefCntPtr InMemoryFileSystem( new vfs::InMemoryFileSystem); // This is passed to `SM` as reference, so the pointer has to be referenced // in `Environment` so that `FileMgr` can out-live this function scope. std::unique_ptr FileMgr( new FileManager(FileSystemOptions(), InMemoryFileSystem)); // This is passed to `SM` as reference, so the pointer has to be referenced // by `Environment` due to the same reason above. std::unique_ptr Diagnostics(new DiagnosticsEngine( IntrusiveRefCntPtr(new DiagnosticIDs), new DiagnosticOptions)); // This will be stored as reference, so the pointer has to be stored in // due to the same reason above. std::unique_ptr VirtualSM( new SourceManager(*Diagnostics, *FileMgr)); InMemoryFileSystem->addFile( FileName, 0, llvm::MemoryBuffer::getMemBuffer( Code, FileName, /*RequiresNullTerminator=*/false)); FileID ID = VirtualSM->createFileID(FileMgr->getFile(FileName), SourceLocation(), clang::SrcMgr::C_User); assert(ID.isValid()); SourceLocation StartOfFile = VirtualSM->getLocForStartOfFile(ID); std::vector CharRanges; for (const tooling::Range &Range : Ranges) { SourceLocation Start = StartOfFile.getLocWithOffset(Range.getOffset()); SourceLocation End = Start.getLocWithOffset(Range.getLength()); CharRanges.push_back(CharSourceRange::getCharRange(Start, End)); } return llvm::make_unique(ID, std::move(FileMgr), std::move(VirtualSM), std::move(Diagnostics), CharRanges); } TokenAnalyzer::TokenAnalyzer(const Environment &Env, const FormatStyle &Style) : Style(Style), Env(Env), AffectedRangeMgr(Env.getSourceManager(), Env.getCharRanges()), UnwrappedLines(1), Encoding(encoding::detectEncoding( Env.getSourceManager().getBufferData(Env.getFileID()))) { DEBUG( llvm::dbgs() << "File encoding: " << (Encoding == encoding::Encoding_UTF8 ? "UTF8" : "unknown") << "\n"); DEBUG(llvm::dbgs() << "Language: " << getLanguageName(Style.Language) << "\n"); } tooling::Replacements TokenAnalyzer::process() { tooling::Replacements Result; FormatTokenLexer Tokens(Env.getSourceManager(), Env.getFileID(), Style, Encoding); UnwrappedLineParser Parser(Style, Tokens.getKeywords(), Tokens.lex(), *this); Parser.parse(); assert(UnwrappedLines.rbegin()->empty()); for (unsigned Run = 0, RunE = UnwrappedLines.size(); Run + 1 != RunE; ++Run) { DEBUG(llvm::dbgs() << "Run " << Run << "...\n"); SmallVector AnnotatedLines; TokenAnnotator Annotator(Style, Tokens.getKeywords()); for (unsigned i = 0, e = UnwrappedLines[Run].size(); i != e; ++i) { AnnotatedLines.push_back(new AnnotatedLine(UnwrappedLines[Run][i])); Annotator.annotate(*AnnotatedLines.back()); } tooling::Replacements RunResult = analyze(Annotator, AnnotatedLines, Tokens); DEBUG({ llvm::dbgs() << "Replacements for run " << Run << ":\n"; for (tooling::Replacements::const_iterator I = RunResult.begin(), E = RunResult.end(); I != E; ++I) { llvm::dbgs() << I->toString() << "\n"; } }); for (unsigned i = 0, e = AnnotatedLines.size(); i != e; ++i) { delete AnnotatedLines[i]; } for (const auto &R : RunResult) { auto Err = Result.add(R); // FIXME: better error handling here. For now, simply return an empty // Replacements to indicate failure. if (Err) { llvm::errs() << llvm::toString(std::move(Err)) << "\n"; return tooling::Replacements(); } } } return Result; } void TokenAnalyzer::consumeUnwrappedLine(const UnwrappedLine &TheLine) { assert(!UnwrappedLines.empty()); UnwrappedLines.back().push_back(TheLine); } void TokenAnalyzer::finishRun() { UnwrappedLines.push_back(SmallVector()); } } // end namespace format } // end namespace clang @ 1.1.1.1 log @Import Clang pre-4.0.0 r291444. @ text @@ 1.1.1.1.16.1 log @Sync with HEAD @ text @d11 1 a11 1 /// This file implements an abstract TokenAnalyzer and associated helper d37 30 a66 8 Environment::Environment(StringRef Code, StringRef FileName, ArrayRef Ranges, unsigned FirstStartColumn, unsigned NextStartColumn, unsigned LastStartColumn) : VirtualSM(new SourceManagerForFile(FileName, Code)), SM(VirtualSM->get()), ID(VirtualSM->get().getMainFileID()), FirstStartColumn(FirstStartColumn), NextStartColumn(NextStartColumn), LastStartColumn(LastStartColumn) { SourceLocation StartOfFile = SM.getLocForStartOfFile(ID); d72 3 d83 1 a83 1 LLVM_DEBUG( d87 2 a88 2 LLVM_DEBUG(llvm::dbgs() << "Language: " << getLanguageName(Style.Language) << "\n"); d91 1 a91 1 std::pair TokenAnalyzer::process() { d93 2 a94 2 FormatTokenLexer Tokens(Env.getSourceManager(), Env.getFileID(), Env.getFirstStartColumn(), Style, Encoding); d96 1 a96 2 UnwrappedLineParser Parser(Style, Tokens.getKeywords(), Env.getFirstStartColumn(), Tokens.lex(), *this); a98 1 unsigned Penalty = 0; d100 1 a100 1 LLVM_DEBUG(llvm::dbgs() << "Run " << Run << "...\n"); d109 1 a109 1 std::pair RunResult = d112 1 a112 1 LLVM_DEBUG({ d114 2 a115 2 for (tooling::Replacements::const_iterator I = RunResult.first.begin(), E = RunResult.first.end(); d123 1 a123 3 Penalty += RunResult.second; for (const auto &R : RunResult.first) { d129 1 a129 1 return {tooling::Replacements(), 0}; d133 1 a133 1 return {Result, Penalty}; @ 1.1.1.1.16.2 log @Mostly merge changes from HEAD upto 20200411 @ text @@ 1.1.1.1.14.1 log @Sync with HEAD @ text @d11 1 a11 1 /// This file implements an abstract TokenAnalyzer and associated helper d37 30 a66 8 Environment::Environment(StringRef Code, StringRef FileName, ArrayRef Ranges, unsigned FirstStartColumn, unsigned NextStartColumn, unsigned LastStartColumn) : VirtualSM(new SourceManagerForFile(FileName, Code)), SM(VirtualSM->get()), ID(VirtualSM->get().getMainFileID()), FirstStartColumn(FirstStartColumn), NextStartColumn(NextStartColumn), LastStartColumn(LastStartColumn) { SourceLocation StartOfFile = SM.getLocForStartOfFile(ID); d72 3 d83 1 a83 1 LLVM_DEBUG( d87 2 a88 2 LLVM_DEBUG(llvm::dbgs() << "Language: " << getLanguageName(Style.Language) << "\n"); d91 1 a91 1 std::pair TokenAnalyzer::process() { d93 2 a94 2 FormatTokenLexer Tokens(Env.getSourceManager(), Env.getFileID(), Env.getFirstStartColumn(), Style, Encoding); d96 1 a96 2 UnwrappedLineParser Parser(Style, Tokens.getKeywords(), Env.getFirstStartColumn(), Tokens.lex(), *this); a98 1 unsigned Penalty = 0; d100 1 a100 1 LLVM_DEBUG(llvm::dbgs() << "Run " << Run << "...\n"); d109 1 a109 1 std::pair RunResult = d112 1 a112 1 LLVM_DEBUG({ d114 2 a115 2 for (tooling::Replacements::const_iterator I = RunResult.first.begin(), E = RunResult.first.end(); d123 1 a123 3 Penalty += RunResult.second; for (const auto &R : RunResult.first) { d129 1 a129 1 return {tooling::Replacements(), 0}; d133 1 a133 1 return {Result, Penalty}; @ 1.1.1.2 log @Import clang r337282 from trunk @ text @d11 1 a11 1 /// This file implements an abstract TokenAnalyzer and associated helper d37 30 a66 8 Environment::Environment(StringRef Code, StringRef FileName, ArrayRef Ranges, unsigned FirstStartColumn, unsigned NextStartColumn, unsigned LastStartColumn) : VirtualSM(new SourceManagerForFile(FileName, Code)), SM(VirtualSM->get()), ID(VirtualSM->get().getMainFileID()), FirstStartColumn(FirstStartColumn), NextStartColumn(NextStartColumn), LastStartColumn(LastStartColumn) { SourceLocation StartOfFile = SM.getLocForStartOfFile(ID); d72 3 d83 1 a83 1 LLVM_DEBUG( d87 2 a88 2 LLVM_DEBUG(llvm::dbgs() << "Language: " << getLanguageName(Style.Language) << "\n"); d91 1 a91 1 std::pair TokenAnalyzer::process() { d93 2 a94 2 FormatTokenLexer Tokens(Env.getSourceManager(), Env.getFileID(), Env.getFirstStartColumn(), Style, Encoding); d96 1 a96 2 UnwrappedLineParser Parser(Style, Tokens.getKeywords(), Env.getFirstStartColumn(), Tokens.lex(), *this); a98 1 unsigned Penalty = 0; d100 1 a100 1 LLVM_DEBUG(llvm::dbgs() << "Run " << Run << "...\n"); d109 1 a109 1 std::pair RunResult = d112 1 a112 1 LLVM_DEBUG({ d114 2 a115 2 for (tooling::Replacements::const_iterator I = RunResult.first.begin(), E = RunResult.first.end(); d123 1 a123 3 Penalty += RunResult.second; for (const auto &R : RunResult.first) { d129 1 a129 1 return {tooling::Replacements(), 0}; d133 1 a133 1 return {Result, Penalty}; @ 1.1.1.3 log @Mark old LLVM instance as dead. @ text @@ 1.1.1.1.4.1 log @file TokenAnalyzer.cpp was added on branch pgoyette-localcount on 2017-03-20 06:52:38 +0000 @ text @d1 146 @ 1.1.1.1.4.2 log @Sync with HEAD @ text @a0 146 //===--- TokenAnalyzer.cpp - Analyze Token Streams --------------*- C++ -*-===// // // The LLVM Compiler Infrastructure // // This file is distributed under the University of Illinois Open Source // License. See LICENSE.TXT for details. // //===----------------------------------------------------------------------===// /// /// \file /// \brief This file implements an abstract TokenAnalyzer and associated helper /// classes. TokenAnalyzer can be extended to generate replacements based on /// an annotated and pre-processed token stream. /// //===----------------------------------------------------------------------===// #include "TokenAnalyzer.h" #include "AffectedRangeManager.h" #include "Encoding.h" #include "FormatToken.h" #include "FormatTokenLexer.h" #include "TokenAnnotator.h" #include "UnwrappedLineParser.h" #include "clang/Basic/Diagnostic.h" #include "clang/Basic/DiagnosticOptions.h" #include "clang/Basic/FileManager.h" #include "clang/Basic/SourceManager.h" #include "clang/Format/Format.h" #include "llvm/ADT/STLExtras.h" #include "llvm/Support/Debug.h" #define DEBUG_TYPE "format-formatter" namespace clang { namespace format { // This sets up an virtual file system with file \p FileName containing \p // Code. std::unique_ptr Environment::CreateVirtualEnvironment(StringRef Code, StringRef FileName, ArrayRef Ranges) { // This is referenced by `FileMgr` and will be released by `FileMgr` when it // is deleted. IntrusiveRefCntPtr InMemoryFileSystem( new vfs::InMemoryFileSystem); // This is passed to `SM` as reference, so the pointer has to be referenced // in `Environment` so that `FileMgr` can out-live this function scope. std::unique_ptr FileMgr( new FileManager(FileSystemOptions(), InMemoryFileSystem)); // This is passed to `SM` as reference, so the pointer has to be referenced // by `Environment` due to the same reason above. std::unique_ptr Diagnostics(new DiagnosticsEngine( IntrusiveRefCntPtr(new DiagnosticIDs), new DiagnosticOptions)); // This will be stored as reference, so the pointer has to be stored in // due to the same reason above. std::unique_ptr VirtualSM( new SourceManager(*Diagnostics, *FileMgr)); InMemoryFileSystem->addFile( FileName, 0, llvm::MemoryBuffer::getMemBuffer( Code, FileName, /*RequiresNullTerminator=*/false)); FileID ID = VirtualSM->createFileID(FileMgr->getFile(FileName), SourceLocation(), clang::SrcMgr::C_User); assert(ID.isValid()); SourceLocation StartOfFile = VirtualSM->getLocForStartOfFile(ID); std::vector CharRanges; for (const tooling::Range &Range : Ranges) { SourceLocation Start = StartOfFile.getLocWithOffset(Range.getOffset()); SourceLocation End = Start.getLocWithOffset(Range.getLength()); CharRanges.push_back(CharSourceRange::getCharRange(Start, End)); } return llvm::make_unique(ID, std::move(FileMgr), std::move(VirtualSM), std::move(Diagnostics), CharRanges); } TokenAnalyzer::TokenAnalyzer(const Environment &Env, const FormatStyle &Style) : Style(Style), Env(Env), AffectedRangeMgr(Env.getSourceManager(), Env.getCharRanges()), UnwrappedLines(1), Encoding(encoding::detectEncoding( Env.getSourceManager().getBufferData(Env.getFileID()))) { DEBUG( llvm::dbgs() << "File encoding: " << (Encoding == encoding::Encoding_UTF8 ? "UTF8" : "unknown") << "\n"); DEBUG(llvm::dbgs() << "Language: " << getLanguageName(Style.Language) << "\n"); } tooling::Replacements TokenAnalyzer::process() { tooling::Replacements Result; FormatTokenLexer Tokens(Env.getSourceManager(), Env.getFileID(), Style, Encoding); UnwrappedLineParser Parser(Style, Tokens.getKeywords(), Tokens.lex(), *this); Parser.parse(); assert(UnwrappedLines.rbegin()->empty()); for (unsigned Run = 0, RunE = UnwrappedLines.size(); Run + 1 != RunE; ++Run) { DEBUG(llvm::dbgs() << "Run " << Run << "...\n"); SmallVector AnnotatedLines; TokenAnnotator Annotator(Style, Tokens.getKeywords()); for (unsigned i = 0, e = UnwrappedLines[Run].size(); i != e; ++i) { AnnotatedLines.push_back(new AnnotatedLine(UnwrappedLines[Run][i])); Annotator.annotate(*AnnotatedLines.back()); } tooling::Replacements RunResult = analyze(Annotator, AnnotatedLines, Tokens); DEBUG({ llvm::dbgs() << "Replacements for run " << Run << ":\n"; for (tooling::Replacements::const_iterator I = RunResult.begin(), E = RunResult.end(); I != E; ++I) { llvm::dbgs() << I->toString() << "\n"; } }); for (unsigned i = 0, e = AnnotatedLines.size(); i != e; ++i) { delete AnnotatedLines[i]; } for (const auto &R : RunResult) { auto Err = Result.add(R); // FIXME: better error handling here. For now, simply return an empty // Replacements to indicate failure. if (Err) { llvm::errs() << llvm::toString(std::move(Err)) << "\n"; return tooling::Replacements(); } } } return Result; } void TokenAnalyzer::consumeUnwrappedLine(const UnwrappedLine &TheLine) { assert(!UnwrappedLines.empty()); UnwrappedLines.back().push_back(TheLine); } void TokenAnalyzer::finishRun() { UnwrappedLines.push_back(SmallVector()); } } // end namespace format } // end namespace clang @