diff --git a/clang/include/clang/ASTMatchers/ASTMatchFinder.h b/clang/include/clang/ASTMatchers/ASTMatchFinder.h index a387d9037b7da4..a559ba0fc8aed7 100644 --- a/clang/include/clang/ASTMatchers/ASTMatchFinder.h +++ b/clang/include/clang/ASTMatchers/ASTMatchFinder.h @@ -200,6 +200,8 @@ class MatchFinder { /// Finds all matches in the given AST. void matchAST(ASTContext &Context); + void matchDecl(clang::Decl *Decl, ASTContext &Context); + /// Registers a callback to notify the end of parsing. /// /// The provided closure is called after parsing is done, before the AST is diff --git a/clang/include/clang/Basic/DiagnosticCommonKinds.td b/clang/include/clang/Basic/DiagnosticCommonKinds.td index 08bb1d81ba29f1..cf87ef83d6d873 100644 --- a/clang/include/clang/Basic/DiagnosticCommonKinds.td +++ b/clang/include/clang/Basic/DiagnosticCommonKinds.td @@ -397,6 +397,17 @@ def note_mt_message : Note<"[rewriter] %0">; def warn_arcmt_nsalloc_realloc : Warning<"[rewriter] call returns pointer to GC managed memory; it will become unmanaged in ARC">; def err_arcmt_nsinvocation_ownership : Error<"NSInvocation's %0 is not safe to be used with an object with ownership other than __unsafe_unretained">; +// Clang rules +def err_clangrules_message : Error<"%0">; +def warn_clangrules_message : Warning<"%0">, InGroup>; +def note_clangrules_message : Note<"%0">; +def err_clangrules_rule_name_is_prefixed : Error<"Rule name '%0' must be unprefixed and not contain a '/' character.">; +def err_clangrules_ruleset_name_is_prefixed : Error<"Ruleset name '%0' must be unprefixed and not contain a '/' character.">; +def err_clangrules_ruleset_severity_is_notset : Error<"Ruleset '%0' severity must be a value other than 'NotSet'.">; +def err_clangrules_rule_name_conflict : Error<"Namespaced rule name '%0' is already declared in another .clang-rules file. Try using a different namespace to avoid conflicts.">; +def err_clangrules_rule_missing : Error<"Namespaced ruleset '%0' requested namespaced rule '%1' but it wasn't known at runtime. Make sure it is declared in either the same .clang-rules file, or in a .clang-rules file in a parent directory.">; +def err_clangrules_rule_matcher_parse_failure : Error<"Namespaced rule '%0' matcher failed to parse: %1">; + // API notes def err_apinotes_message : Error<"%0">; def warn_apinotes_message : Warning<"%0">, InGroup>; diff --git a/clang/lib/ASTMatchers/ASTMatchFinder.cpp b/clang/lib/ASTMatchers/ASTMatchFinder.cpp index 0bac2ed63a927e..562978f5d94667 100644 --- a/clang/lib/ASTMatchers/ASTMatchFinder.cpp +++ b/clang/lib/ASTMatchers/ASTMatchFinder.cpp @@ -1700,6 +1700,12 @@ void MatchFinder::matchAST(ASTContext &Context) { Visitor.onEndOfTranslationUnit(); } +void MatchFinder::matchDecl(clang::Decl *Decl, ASTContext &Context) { + internal::MatchASTVisitor Visitor(&Matchers, Options); + Visitor.set_active_ast_context(&Context); + Visitor.getDerived().TraverseDecl(Decl); +} + void MatchFinder::registerTestCallbackAfterParsing( MatchFinder::ParsingDoneTestCallback *NewParsingDone) { ParsingDone = NewParsingDone; diff --git a/clang/lib/Frontend/CMakeLists.txt b/clang/lib/Frontend/CMakeLists.txt index a9166672088459..23c6a0cf6be4ee 100644 --- a/clang/lib/Frontend/CMakeLists.txt +++ b/clang/lib/Frontend/CMakeLists.txt @@ -15,6 +15,7 @@ add_clang_library(clangFrontend ASTUnit.cpp ChainedDiagnosticConsumer.cpp ChainedIncludesSource.cpp + ClangRulesets.cpp CompilerInstance.cpp CompilerInvocation.cpp CreateInvocationFromCommandLine.cpp @@ -57,4 +58,6 @@ add_clang_library(clangFrontend clangParse clangSema clangSerialization + clangASTMatchers + clangDynamicASTMatchers ) diff --git a/clang/lib/Frontend/ClangRulesets.cpp b/clang/lib/Frontend/ClangRulesets.cpp new file mode 100644 index 00000000000000..8a7a6e632ec6a4 --- /dev/null +++ b/clang/lib/Frontend/ClangRulesets.cpp @@ -0,0 +1,872 @@ +#include "ClangRulesets.h" +#include "clang/AST/AST.h" +#include "clang/AST/ASTConsumer.h" +#include "clang/ASTMatchers/ASTMatchFinder.h" +#include "clang/ASTMatchers/Dynamic/Parser.h" +#include "clang/Basic/SourceMgrAdapter.h" +#include "clang/Lex/PPCallbacks.h" +#include "clang/Lex/Preprocessor.h" +#include "llvm/Support/RWMutex.h" +#include "llvm/Support/ThreadPool.h" +#include "llvm/Support/YAMLParser.h" +#include "llvm/Support/YAMLTraits.h" + +using namespace clang; + +#if 0 +#define RULESET_TRACE(x) llvm::errs() << x; +#else +#define RULESET_TRACE(x) +#endif + +namespace clang::rulesets::config { + +enum ClangRulesSeverity : int8_t { + CRS_NotSet, + CRS_Silence, + CRS_Info, + CRS_Warning, + CRS_Error, +}; + +struct ClangRulesRule { + std::string Name; + std::string Matcher; + std::string ErrorMessage; + std::string Callsite; + std::map Hints; + std::optional MatcherParsed; +}; + +struct ClangRulesRulesetRule { + std::string Name; + ClangRulesSeverity Severity; +}; + +struct ClangRulesRuleset { + std::string Name; + ClangRulesSeverity Severity; + std::vector Rules; +}; + +struct ClangRules { + std::string Namespace; + std::vector Rules; + std::vector Rulesets; +}; + +} // namespace clang::rulesets::config + +LLVM_YAML_IS_STRING_MAP(std::string); +LLVM_YAML_IS_SEQUENCE_VECTOR(clang::rulesets::config::ClangRulesRulesetRule); +LLVM_YAML_IS_SEQUENCE_VECTOR(clang::rulesets::config::ClangRulesRule); +LLVM_YAML_IS_SEQUENCE_VECTOR(clang::rulesets::config::ClangRulesRuleset); + +namespace llvm::yaml { + +template <> +struct ScalarEnumerationTraits { + static void enumeration(IO &IO, + clang::rulesets::config::ClangRulesSeverity &Value) { + IO.enumCase(Value, "NotSet", + clang::rulesets::config::ClangRulesSeverity::CRS_NotSet); + IO.enumCase(Value, "Silence", + clang::rulesets::config::ClangRulesSeverity::CRS_Silence); + IO.enumCase(Value, "Info", + clang::rulesets::config::ClangRulesSeverity::CRS_Info); + IO.enumCase(Value, "Warning", + clang::rulesets::config::ClangRulesSeverity::CRS_Warning); + IO.enumCase(Value, "Error", + clang::rulesets::config::ClangRulesSeverity::CRS_Error); + } +}; + +template <> struct MappingTraits { + static void mapping(IO &IO, clang::rulesets::config::ClangRulesRule &Rule) { + IO.mapRequired("Name", Rule.Name); + IO.mapRequired("Matcher", Rule.Matcher); + IO.mapRequired("ErrorMessage", Rule.ErrorMessage); + IO.mapRequired("Callsite", Rule.Callsite); + IO.mapOptional("Hints", Rule.Hints); + } +}; + +template <> +struct MappingTraits { + static void mapping(IO &IO, + clang::rulesets::config::ClangRulesRulesetRule &Rule) { + if (IO.getNodeKind() == NodeKind::Scalar) { + // Allow rules for rulesets to be encoded as plain strings. + llvm::StringRef RuleName = Rule.Name; + IO.scalarString(RuleName, QuotingType::Double); + Rule.Name = RuleName; + Rule.Severity = clang::rulesets::config::ClangRulesSeverity::CRS_NotSet; + } else { + // Allow rules for rulesets to specify name and severity. + IO.mapRequired("Name", Rule.Name); + IO.mapOptional("Severity", Rule.Severity, + clang::rulesets::config::ClangRulesSeverity::CRS_NotSet); + } + } +}; + +template <> struct MappingTraits { + static void mapping(IO &IO, + clang::rulesets::config::ClangRulesRuleset &Ruleset) { + IO.mapRequired("Name", Ruleset.Name); + IO.mapOptional("Severity", Ruleset.Severity, + clang::rulesets::config::ClangRulesSeverity::CRS_Warning); + IO.mapRequired("Rules", Ruleset.Rules); + } +}; + +template <> struct MappingTraits { + static void mapping(IO &IO, clang::rulesets::config::ClangRules &Rules) { + IO.mapRequired("Namespace", Rules.Namespace); + IO.mapOptional("Rulesets", Rules.Rulesets); + IO.mapOptional("Rules", Rules.Rules); + } +}; + +} // namespace llvm::yaml + +namespace clang::rulesets { + +struct ClangRulesetsEffectiveRule { + // Pointer to memory inside a loaded config::ClangRules. + config::ClangRulesRule *Rule; + config::ClangRulesSeverity Severity; +}; + +struct ClangRulesetsEffectiveConfig { + std::map EffectiveRules; +}; + +struct ClangRulesetsDirectoryState { +public: + // If this directory contained a .clang-rules file, this is the on-disk + // configuration that was loaded. + std::unique_ptr ActualOnDiskConfig; + // The parent directory of this directory, if this directory is not + // a root. + OptionalDirectoryEntryRef ParentDirectory; + // Have we materialised the resolution fields (even if they resolved + // to no config at all)? + bool Materialized; + // The effective rules config that applies to this directory. + ClangRulesetsEffectiveConfig *EffectiveConfig; +}; + +class ClangRulesetsState { +public: + llvm::DenseMap Dirs; + +private: + std::vector CreatedEffectiveConfigs; + llvm::StringMap RuleByNamespacedName; + llvm::ThreadPool ThreadPool; + llvm::sys::SmartRWMutex ThreadRWMutex; + + struct MissingClangRule { + std::string NamespacedRulesetName; + std::string NamespacedRuleName; + }; + std::vector MissingClangRules; + + struct DiagnosticToReport { + const ClangRulesetsEffectiveRule *EffectiveRule; + clang::SourceLocation CallsiteLoc; + std::vector> HintLocs; + }; + std::map> DiagnosticsToReport; + +public: + ClangRulesetsState() + : Dirs(), CreatedEffectiveConfigs(), RuleByNamespacedName(), + ThreadPool(){}; + ClangRulesetsState(const ClangRulesetsState &) = delete; + ClangRulesetsState(ClangRulesetsState &&) = delete; + ~ClangRulesetsState() { + for (const auto &Config : this->CreatedEffectiveConfigs) { + delete Config; + } + } + + std::unique_ptr + loadClangRules_fromPreprocessor(clang::FileID &FileID, + clang::SourceManager &SrcMgr) { + // Load the .clang-rules file. + SourceMgrAdapter SMAdapter( + SrcMgr, SrcMgr.getDiagnostics(), diag::err_clangrules_message, + diag::warn_clangrules_message, diag::note_clangrules_message, + SrcMgr.getFileEntryRefForID(FileID)); + std::unique_ptr LoadedRules = + std::make_unique(); + llvm::yaml::Input YamlParse(SrcMgr.getBufferData(FileID), nullptr, + SMAdapter.getDiagHandler(), + SMAdapter.getDiagContext()); + YamlParse >> *LoadedRules; + if (YamlParse.error()) { + return nullptr; + } + + // Track whether the loaded rules are still valid. + bool StillValid = true; + + // Go through rules, make sure they aren't already prefixed, and then update + // our in-memory version of the rules file to prefix them with the + // namespace. + for (auto &Rule : LoadedRules->Rules) { + if (Rule.Name.find('/') != std::string::npos) { + SrcMgr.getDiagnostics().Report( + SrcMgr.getLocForStartOfFile(FileID), + diag::err_clangrules_rule_name_is_prefixed) + << Rule.Name; + StillValid = false; + continue; + } + std::string NamespacedName = LoadedRules->Namespace; + NamespacedName.append("/"); + NamespacedName.append(Rule.Name); + Rule.Name = NamespacedName; + + // Make sure this namespaced rule name isn't already taken. + if (this->RuleByNamespacedName[Rule.Name] != nullptr) { + SrcMgr.getDiagnostics().Report(SrcMgr.getLocForStartOfFile(FileID), + diag::err_clangrules_rule_name_conflict) + << Rule.Name; + StillValid = false; + continue; + } + + // Attempt to parse the matcher expression. + { + clang::ast_matchers::dynamic::Diagnostics ParseDiag; + llvm::StringRef MatcherRef(Rule.Matcher); + Rule.MatcherParsed = + clang::ast_matchers::dynamic::Parser::parseMatcherExpression( + MatcherRef, &ParseDiag); + if (!Rule.MatcherParsed.has_value()) { + SrcMgr.getDiagnostics().Report( + SrcMgr.getLocForStartOfFile(FileID), + diag::err_clangrules_rule_matcher_parse_failure) + << NamespacedName << ParseDiag.toStringFull(); + StillValid = false; + continue; + } + } + } + + // Go through rulesets and namespace both their name and any unprefixed + // rules they enable, and normalize the severity on rules using the + // default severity if it's not set. + for (auto &Ruleset : LoadedRules->Rulesets) { + if (Ruleset.Name.find('/') != std::string::npos) { + SrcMgr.getDiagnostics().Report( + SrcMgr.getLocForStartOfFile(FileID), + diag::err_clangrules_ruleset_name_is_prefixed) + << Ruleset.Name; + StillValid = false; + } else { + std::string NamespacedName = LoadedRules->Namespace; + NamespacedName.append("/"); + NamespacedName.append(Ruleset.Name); + Ruleset.Name = NamespacedName; + } + if (Ruleset.Severity == + clang::rulesets::config::ClangRulesSeverity::CRS_NotSet) { + SrcMgr.getDiagnostics().Report( + SrcMgr.getLocForStartOfFile(FileID), + diag::err_clangrules_ruleset_severity_is_notset) + << Ruleset.Name; + StillValid = false; + } + for (auto &Rule : Ruleset.Rules) { + if (Rule.Name.find('/') == std::string::npos) { + std::string NamespacedName = LoadedRules->Namespace; + NamespacedName.append("/"); + NamespacedName.append(Rule.Name); + Rule.Name = NamespacedName; + } + if (Rule.Severity == + clang::rulesets::config::ClangRulesSeverity::CRS_NotSet) { + Rule.Severity = Ruleset.Severity; + } + } + } + + // If we have a fatal error in loading the rules, release the memory + // and treat the directory as if it has no rules at all. + if (!StillValid) { + return nullptr; + } + + // Map all of the namespaced rule names to their locations in memory (as + // part of LoadedRules). + for (auto &Rule : LoadedRules->Rules) { + this->RuleByNamespacedName[Rule.Name] = &Rule; + } + + // Return the loaded rules. + return LoadedRules; + } + +private: + void materializeDirectoryState_withinWriteLock( + ClangRulesetsDirectoryState &DirState) { + assert(!DirState.Materialized); + + // If we have an actual on-disk configuration, we need to merge that + // with our parent. + if (DirState.ActualOnDiskConfig && + DirState.ActualOnDiskConfig->Rulesets.size() > 0) { + // Create our new effective configuration. + auto *EffectiveConfig = new ClangRulesetsEffectiveConfig(); + + // Get the materialized effective config of the parent, if we're not + // a root directory, and then copy from that. + if (DirState.ParentDirectory) { + auto &ParentState = this->Dirs[*DirState.ParentDirectory]; + if (!ParentState.Materialized) { + RULESET_TRACE("materializing: " << DirState.ParentDirectory->getName() + << "\n"); + assert(&ParentState != &DirState); + this->materializeDirectoryState_withinWriteLock(ParentState); + RULESET_TRACE("materializing done: " + << DirState.ParentDirectory->getName() << "\n"); + } + if (ParentState.EffectiveConfig != nullptr) { + // Copy the effective rules (which are namespaced rule names plus the + // effective severity). + EffectiveConfig->EffectiveRules = + ParentState.EffectiveConfig->EffectiveRules; + } + } + + // For all of the rulesets in our rules, add them or update their existing + // severity in the effective rules. + bool StillValid = true; + for (const auto &Ruleset : DirState.ActualOnDiskConfig->Rulesets) { + for (const auto &RulesetRule : Ruleset.Rules) { + // Lookup the rule by namespaced name. If this doesn't exist, then the + // ruleset is referencing a rule that isn't known. + auto *Rule = this->RuleByNamespacedName[RulesetRule.Name]; + if (Rule == nullptr) { + MissingClangRules.push_back( + MissingClangRule{Ruleset.Name, RulesetRule.Name}); + StillValid = false; + } else { + EffectiveConfig->EffectiveRules[RulesetRule.Name] = + ClangRulesetsEffectiveRule{Rule, RulesetRule.Severity}; + } + } + } + if (!StillValid) { + // This directory doesn't have a valid .clang-rules effective state. + delete EffectiveConfig; + DirState.Materialized = true; + DirState.EffectiveConfig = nullptr; + return; + } + + // Remove any effective rules that are silence, since we don't need to run + // them at all. + for (auto It = EffectiveConfig->EffectiveRules.begin(); + It != EffectiveConfig->EffectiveRules.end(); ++It) { + if (It->second.Severity == config::ClangRulesSeverity::CRS_Silence) { + EffectiveConfig->EffectiveRules.erase(It); + } + } + + // If there are no effective rules remaining, materialize this directory + // as if there was no .clang-rules anywhere in the hierarchy. + if (EffectiveConfig->EffectiveRules.size() == 0) { + delete EffectiveConfig; + DirState.Materialized = true; + DirState.EffectiveConfig = nullptr; + return; + } + + // Otherwise, this is the effective config for this directory. + this->CreatedEffectiveConfigs.push_back(EffectiveConfig); + DirState.Materialized = true; + DirState.EffectiveConfig = EffectiveConfig; + return; + } + // Otherwise, we're going to ask our parent directory to be materialized + // if they aren't already, and then borrow their materialized values. + else { + if (DirState.ParentDirectory) { + // Materialize our parent if needed and get the config. + auto &ParentState = this->Dirs[*DirState.ParentDirectory]; + if (!ParentState.Materialized) { + RULESET_TRACE("materializing: " << DirState.ParentDirectory->getName() + << "\n"); + assert(&ParentState != &DirState); + this->materializeDirectoryState_withinWriteLock(ParentState); + RULESET_TRACE("materializing done: " + << DirState.ParentDirectory->getName() << "\n"); + } + DirState.EffectiveConfig = ParentState.EffectiveConfig; + DirState.Materialized = true; + } else { + // No parent directory. We are a root with no on-disk config. + DirState.EffectiveConfig = nullptr; + DirState.Materialized = true; + } + } + } + + class LockableClangRulesetsState { + private: + ClangRulesetsState *State; + + public: + LockableClangRulesetsState(ClangRulesetsState *InState) : State(InState){}; + LockableClangRulesetsState(const LockableClangRulesetsState &) = delete; + LockableClangRulesetsState(LockableClangRulesetsState &&) = delete; + ~LockableClangRulesetsState() = default; + + ClangRulesetsEffectiveConfig * + getEffectiveConfigForDirectoryEntry(const clang::DirectoryEntryRef &Dir) { + // Obtain a read lock. + RULESET_TRACE("getEffectiveConfigForDirectoryEntry: obtain read lock\n"); + this->State->ThreadRWMutex.lock_shared(); + auto DirState = State->Dirs.find(Dir); + if (DirState == State->Dirs.end()) { + // Release the read lock and return; this is not a tracked directory. + RULESET_TRACE( + "getEffectiveConfigForDirectoryEntry: release read lock\n"); + this->State->ThreadRWMutex.unlock_shared(); + return nullptr; + } + + // If we haven't materialized this directory, upgrade to a + // write lock and then materialize. + if (!DirState->second.Materialized) { + // Upgrade to write lock. + RULESET_TRACE( + "getEffectiveConfigForDirectoryEntry: release read lock\n"); + this->State->ThreadRWMutex.unlock_shared(); + RULESET_TRACE( + "getEffectiveConfigForDirectoryEntry: obtain write lock\n"); + this->State->ThreadRWMutex.lock(); + + // Check that another writer that was waiting didn't just + // materialize this directory state. + if (!DirState->second.Materialized) { + RULESET_TRACE("materializing inside lock: " + << DirState->first.getName() << "\n"); + State->materializeDirectoryState_withinWriteLock(DirState->second); + RULESET_TRACE("materializing inside lock done: " + << DirState->first.getName() << "\n"); + } + + // Read the effective config pointer while still in the write lock. + auto *EffectiveConfig = DirState->second.EffectiveConfig; + + // Now release the write lock and return. + RULESET_TRACE( + "getEffectiveConfigForDirectoryEntry: release write lock\n"); + this->State->ThreadRWMutex.unlock(); + return EffectiveConfig; + } else { + // Read the effective config pointer while still in the read lock. + auto *EffectiveConfig = DirState->second.EffectiveConfig; + + // Now release the read lock and return. + RULESET_TRACE( + "getEffectiveConfigForDirectoryEntry: release read lock\n"); + this->State->ThreadRWMutex.unlock_shared(); + return EffectiveConfig; + } + } + + void reportDiagnostic(ASTContext &AST, + const DiagnosticToReport &InDiagnosticToReport) { + this->State->ThreadRWMutex.lock(); + this->State->DiagnosticsToReport[&AST].push_back(InDiagnosticToReport); + this->State->ThreadRWMutex.unlock(); + } + }; + +public: + void receiveTranslationUnitForAnalysis(ASTContext &AST) { + // @note: Capturing AST by reference is safe here because it's the same AST + // that will be passed into the "wait consumer" when that runs after the + // main consumer is done with the AST. Since the "wait consumer" blocks on + // background threads, there's no way for the reference of AST to be + // released while it's being used on the background thread. + this->ThreadPool.async([this, &AST]() { + LockableClangRulesetsState Lockable(this); + RULESET_TRACE("background thread start\n"); + this->runTranslationUnitAnalysisOnBackgroundThread(Lockable, AST); + RULESET_TRACE("background thread end\n"); + }); + } + +private: + class InstantiatedMatcher { + private: + class InstantiatedMatcherCallback + : public clang::ast_matchers::MatchFinder::MatchCallback { + private: + LockableClangRulesetsState &State; + ASTContext &AST; + const ClangRulesetsEffectiveRule &EffectiveRule; + + public: + InstantiatedMatcherCallback( + LockableClangRulesetsState &InState, ASTContext &InAST, + const ClangRulesetsEffectiveRule &InEffectiveRule) + : State(InState), AST(InAST), EffectiveRule(InEffectiveRule){}; + + virtual void run(const clang::ast_matchers::MatchFinder::MatchResult + &Result) override { + RULESET_TRACE("run() called for match result\n"); + DiagnosticToReport Diagnostic = {}; + Diagnostic.EffectiveRule = &this->EffectiveRule; + { + auto CallsiteIt = + Result.Nodes.getMap().find(this->EffectiveRule.Rule->Callsite); + if (CallsiteIt == Result.Nodes.getMap().end()) { + Diagnostic.CallsiteLoc = + this->AST.getTranslationUnitDecl()->getBeginLoc(); + } else { + Diagnostic.CallsiteLoc = + CallsiteIt->second.getSourceRange().getBegin(); + } + } + for (const auto &HintKV : this->EffectiveRule.Rule->Hints) { + auto HintIt = Result.Nodes.getMap().find(HintKV.first); + if (HintIt != Result.Nodes.getMap().end()) { + Diagnostic.HintLocs.push_back( + std::pair( + HintIt->second.getSourceRange().getBegin(), + llvm::StringRef(HintKV.second))); + } + } + this->State.reportDiagnostic(this->AST, Diagnostic); + } + }; + + std::unique_ptr Finder; + llvm::DenseMap + Callbacks; + LockableClangRulesetsState &State; + ASTContext &AST; + + public: + InstantiatedMatcher(LockableClangRulesetsState &InState, ASTContext &InAST) + : Finder(std::make_unique()), Callbacks(), + State(InState), AST(InAST) {} + InstantiatedMatcher(const InstantiatedMatcher &) = delete; + InstantiatedMatcher(InstantiatedMatcher &&) = delete; + ~InstantiatedMatcher() { + for (const auto &KV : this->Callbacks) { + delete KV.second; + } + } + + void addRule(const ClangRulesetsEffectiveRule &EffectiveRule) { + if (this->Callbacks.contains(&EffectiveRule)) { + return; + } + auto &Rule = EffectiveRule.Rule; + if (Rule->MatcherParsed.has_value()) { + auto *Callback = new InstantiatedMatcherCallback(this->State, this->AST, + EffectiveRule); + RULESET_TRACE("adding dynamic matcher to finder\n"); + this->Finder->addDynamicMatcher(*Rule->MatcherParsed, Callback); + this->Callbacks[&EffectiveRule] = Callback; + } + } + + void match(clang::Decl *Decl) { + RULESET_TRACE("match called for decl\n"); + this->Finder->matchDecl(Decl, this->AST); + } + }; + + static void runTranslationUnitAnalysisOnBackgroundThread( + LockableClangRulesetsState &State, const ASTContext &AST) { + const auto *UnitDeclEntry = AST.getTranslationUnitDecl(); + if (UnitDeclEntry == nullptr) { + RULESET_TRACE( + "skipping AST analysis because there's no translation unit\n"); + return; + } + const SourceManager &SrcMgr = AST.getSourceManager(); + + RULESET_TRACE("starting AST analysis\n"); + + // Track the last file ID and last directory entry, so that as we go + // over decls in the same source file, we don't need to redo lookups. + FileID LastFileID; + ClangRulesetsEffectiveConfig *LastEffectiveConfig = nullptr; + + // Cached callbacks. + std::map + SharedEffectiveConfigToInstantiatedMatchers; + + // Iterate through all of the decls in the translation unit. + for (const auto &DeclEntry : UnitDeclEntry->decls()) { + // Get the location of this decl. + FileID NewFileID = SrcMgr.getFileID(DeclEntry->getLocation()); + if (NewFileID.isInvalid()) { + // Ignore any decls that have no file ID. + continue; + } + + // If we're not in the same file as we were previously... + if (NewFileID != LastFileID) { + // @note: We always update LastFileID, even if the calls below are + // unable to get valid info. This allows us to skip over decls quickly + // if we know the last file ID won't actually resolve anywhere. + LastFileID = NewFileID; + LastEffectiveConfig = nullptr; + + // Try to get the file entry for the file ID. + auto FileEntry = SrcMgr.getFileEntryRefForID(NewFileID); + if (!FileEntry) { + continue; + } + + // Get the new effective config via the lockable state. + LastEffectiveConfig = + State.getEffectiveConfigForDirectoryEntry(FileEntry->getDir()); + + // If there is a config, instantiate the matcher we will use. + if (LastEffectiveConfig != nullptr) { + auto Matcher = + new InstantiatedMatcher(State, const_cast(AST)); + for (const auto &EffectiveRule : + LastEffectiveConfig->EffectiveRules) { + RULESET_TRACE("adding rule to matcher: " << EffectiveRule.first + << "\n"); + Matcher->addRule(EffectiveRule.second); + } + RULESET_TRACE("instantiated matcher\n"); + SharedEffectiveConfigToInstantiatedMatchers[LastEffectiveConfig] = + Matcher; + } + } + + // If we have no .clang-rules effective config for this decl, skip. + if (LastEffectiveConfig == nullptr) { + continue; + } + + // Evaluate all of the matchers against this node. + RULESET_TRACE("executing matcher\n"); + SharedEffectiveConfigToInstantiatedMatchers[LastEffectiveConfig]->match( + DeclEntry); + } + + // Free all the matchers. + for (const auto &KV : SharedEffectiveConfigToInstantiatedMatchers) { + delete KV.second; + } + SharedEffectiveConfigToInstantiatedMatchers.clear(); + + RULESET_TRACE("ending AST analysis\n"); + } + +public: + void blockUntilAnalysisOnBackgroundThreadsIsCompleteAndFlushDiagnostics( + ASTContext &ReceivedAST) { + // Wait for all current analysis to finish. + this->ThreadPool.wait(); + + // @note: We don't need to obtain a write lock here because only the main + // thread calls receiveTranslationUnitForAnalysis to add new tasks to the + // thread pool, and this function only runs on the main thread. + + // Flush all pending "missing rule" diagnostics. + for (const auto &PendingError : this->MissingClangRules) { + ReceivedAST.getDiagnostics().Report(diag::err_clangrules_rule_missing) + << PendingError.NamespacedRulesetName + << PendingError.NamespacedRuleName; + } + this->MissingClangRules.clear(); + + // Flush diagnostics for this AST in particular. + for (const auto &Diagnostic : this->DiagnosticsToReport[&ReceivedAST]) { + // Report the error message. + { + clang::DiagnosticIDs::Level DiagnosticLevel = + clang::DiagnosticIDs::Level::Remark; + switch (Diagnostic.EffectiveRule->Severity) { + case config::ClangRulesSeverity::CRS_Silence: + case config::ClangRulesSeverity::CRS_Info: + DiagnosticLevel = clang::DiagnosticIDs::Level::Remark; + break; + case config::ClangRulesSeverity::CRS_Warning: + case config::ClangRulesSeverity::CRS_NotSet: + DiagnosticLevel = clang::DiagnosticIDs::Level::Warning; + break; + case config::ClangRulesSeverity::CRS_Error: + DiagnosticLevel = clang::DiagnosticIDs::Level::Error; + break; + } + auto CallsiteDiagID = + ReceivedAST.getDiagnostics().getDiagnosticIDs()->getCustomDiagID( + DiagnosticLevel, Diagnostic.EffectiveRule->Rule->ErrorMessage); + ReceivedAST.getDiagnostics().Report(Diagnostic.CallsiteLoc, + CallsiteDiagID); + } + + // Report any attached hints. + for (const auto &HintKV : Diagnostic.HintLocs) { + auto HintDiagID = + ReceivedAST.getDiagnostics().getDiagnosticIDs()->getCustomDiagID( + clang::DiagnosticIDs::Note, HintKV.second); + ReceivedAST.getDiagnostics().Report(HintKV.first, HintDiagID); + } + } + this->DiagnosticsToReport.erase(&ReceivedAST); + } +}; + +class ClangRulesetsPPCallbacks : public PPCallbacks { +private: + std::shared_ptr State; + CompilerInstance &CI; + +public: + ClangRulesetsPPCallbacks(std::shared_ptr InState, + CompilerInstance &InCI) + : State(InState), CI(InCI){}; + virtual ~ClangRulesetsPPCallbacks() override = default; + + virtual void LexedFileChanged(FileID FID, LexedFileChangeReason Reason, + SrcMgr::CharacteristicKind FileType, + FileID PrevFID, SourceLocation Loc) override { + auto &SrcMgr = CI.getSourceManager(); + auto OptionalFileEntryRef = SrcMgr.getFileEntryRefForID(FID); + if (!OptionalFileEntryRef.has_value()) { + // If there's no file entry for the new file, we don't process it. + return; + } + + auto ContainingDirectory = OptionalFileEntryRef->getDir(); + if (!State->Dirs.contains(ContainingDirectory)) { + // This leaf directory hasn't been seen before. We need to make an + // absolute path with '.' entries removed so that we can start traversing + // up the directory tree. + llvm::SmallString<256> LeafAbsolutePath(ContainingDirectory.getName()); + CI.getFileManager().makeAbsolutePath(LeafAbsolutePath); + llvm::sys::path::remove_dots(LeafAbsolutePath, true); + + // Track our current absolute path as we move upwards from the leaf. + llvm::StringRef CurrentAbsolutePath = LeafAbsolutePath; + + // Starting at the current directory, search upwards for .clang-rules + // files. + while (!State->Dirs.contains(ContainingDirectory)) { + // Convert to an absolute path, since we might need to traverse up out + // of the working directory to find our .clangrules files. + // Go check this directory for a .clangrules file. + llvm::SmallString<256> ClangRulesPath(CurrentAbsolutePath); + llvm::sys::path::append(ClangRulesPath, ".clang-rules"); + auto ClangRulesFile = + CI.getFileManager().getFileRef(ClangRulesPath, true, true); + if (ClangRulesFile) { + // We got a .clangrules file in this directory; load it into the + // Clang source manager so we can report diagnostics etc. + clang::FileID ClangRulesFileID = + CI.getSourceManager().getOrCreateFileID( + ClangRulesFile.get(), SrcMgr::CharacteristicKind::C_User); + State->Dirs[ContainingDirectory].ActualOnDiskConfig = + State->loadClangRules_fromPreprocessor(ClangRulesFileID, SrcMgr); + } else { + // We did not get a .clangrules file in this directory; cache that it + // is empty. + State->Dirs[ContainingDirectory].ActualOnDiskConfig = nullptr; + } + // Modify CurrentAbsolutePath so that it contains the next parent path + // to evaluate. + RULESET_TRACE("directory: " << CurrentAbsolutePath); + CurrentAbsolutePath = llvm::sys::path::parent_path(CurrentAbsolutePath); + RULESET_TRACE(" -> " << CurrentAbsolutePath << "\n"); + if (CurrentAbsolutePath.empty() || + (llvm::sys::path::is_style_windows( + llvm::sys::path::Style::native) && + CurrentAbsolutePath.ends_with(":"))) { + // No further parent directories. + break; + } else { + auto OptionalParentDirectory = + CI.getFileManager().getDirectoryRef(CurrentAbsolutePath, true); + if (!OptionalParentDirectory) { + // Can't get parent directory. + break; + } else { + // Loop again with the new parent directory. + State->Dirs[ContainingDirectory].ParentDirectory = + OptionalParentDirectory.get(); + ContainingDirectory = OptionalParentDirectory.get(); + } + } + } + } + } +}; + +class ClangRulesetsStartConsumer : public ASTConsumer { +private: + std::shared_ptr State; + +public: + ClangRulesetsStartConsumer(std::shared_ptr InState) + : State(InState){}; + virtual ~ClangRulesetsStartConsumer() override = default; + + void HandleTranslationUnit(ASTContext &AST) override { + RULESET_TRACE("Receiving translation unit for analysis\n"); + this->State->receiveTranslationUnitForAnalysis(AST); + } +}; + +class ClangRulesetsWaitConsumer : public ASTConsumer { +private: + std::shared_ptr State; + +public: + ClangRulesetsWaitConsumer(std::shared_ptr InState) + : State(InState){}; + virtual ~ClangRulesetsWaitConsumer() override = default; + + void HandleTranslationUnit(ASTContext &AST) override { + RULESET_TRACE("Blocking until translation unit analysis complete\n"); + this->State + ->blockUntilAnalysisOnBackgroundThreadsIsCompleteAndFlushDiagnostics( + AST); + } +}; + +void ClangRulesetsProvider::CreateAndAddASTConsumers( + clang::CompilerInstance &CI, + std::vector> &BeforeConsumers, + std::vector> &AfterConsumers) { + // Create our state that will be shared across consumers and the preprocessor. + RULESET_TRACE("Creating Clang rulesets state\n"); + std::shared_ptr State = + std::make_shared(); + + // Register our preprocessor callbacks, which are used to discover rulesets as + // files are included. + CI.getPreprocessor().addPPCallbacks( + std::make_unique(State, CI)); + + // Create our "start consumer" and "wait consumer". Because analysis can take + // a long time, we run the analysis on a background thread while CodeGen is + // happening, and then re-join the thread later with a "wait consumer". + RULESET_TRACE("Attaching AST consumers\n"); + BeforeConsumers.push_back( + std::make_unique(State)); + AfterConsumers.push_back(std::make_unique(State)); +} + +} // namespace clang::rulesets \ No newline at end of file diff --git a/clang/lib/Frontend/ClangRulesets.h b/clang/lib/Frontend/ClangRulesets.h new file mode 100644 index 00000000000000..cff6fde14e0d7b --- /dev/null +++ b/clang/lib/Frontend/ClangRulesets.h @@ -0,0 +1,16 @@ +#pragma once + +#include "clang/AST/ASTConsumer.h" +#include "clang/Frontend/CompilerInstance.h" + +namespace clang::rulesets { + +class ClangRulesetsProvider { +public: + static void CreateAndAddASTConsumers( + clang::CompilerInstance &CI, + std::vector> &BeforeConsumers, + std::vector> &AfterConsumers); +}; + +} // namespace clang::rulesets diff --git a/clang/lib/Frontend/FrontendAction.cpp b/clang/lib/Frontend/FrontendAction.cpp index eff785b99a09a4..33eb8db8cb2e95 100644 --- a/clang/lib/Frontend/FrontendAction.cpp +++ b/clang/lib/Frontend/FrontendAction.cpp @@ -7,6 +7,7 @@ //===----------------------------------------------------------------------===// #include "clang/Frontend/FrontendAction.h" +#include "ClangRulesets.h" #include "clang/AST/ASTConsumer.h" #include "clang/AST/ASTContext.h" #include "clang/AST/DeclGroup.h" @@ -71,8 +72,7 @@ class DelegatingDeserializationListener : public ASTDeserializationListener { if (Previous) Previous->ReaderInitialized(Reader); } - void IdentifierRead(serialization::IdentID ID, - IdentifierInfo *II) override { + void IdentifierRead(serialization::IdentID ID, IdentifierInfo *II) override { if (Previous) Previous->IdentifierRead(ID, II); } @@ -131,9 +131,8 @@ class DeserializedDeclsChecker : public DelegatingDeserializationListener { void DeclRead(serialization::DeclID ID, const Decl *D) override { if (const NamedDecl *ND = dyn_cast(D)) if (NamesToCheck.find(ND->getNameAsString()) != NamesToCheck.end()) { - unsigned DiagID - = Ctx.getDiagnostics().getCustomDiagID(DiagnosticsEngine::Error, - "%0 was deserialized"); + unsigned DiagID = Ctx.getDiagnostics().getCustomDiagID( + DiagnosticsEngine::Error, "%0 was deserialized"); Ctx.getDiagnostics().Report(Ctx.getFullLoc(D->getLocation()), DiagID) << ND; } @@ -157,7 +156,7 @@ void FrontendAction::setCurrentInput(const FrontendInputFile &CurrentInput, Module *FrontendAction::getCurrentModule() const { CompilerInstance &CI = getCompilerInstance(); return CI.getPreprocessor().getHeaderSearchInfo().lookupModule( - CI.getLangOpts().CurrentModule, SourceLocation(), /*AllowSearch*/false); + CI.getLangOpts().CurrentModule, SourceLocation(), /*AllowSearch*/ false); } std::unique_ptr @@ -184,10 +183,6 @@ FrontendAction::CreateWrappedASTConsumer(CompilerInstance &CI, if (!FoundAllPlugins) return nullptr; - // If there are no registered plugins we don't need to wrap the consumer - if (FrontendPluginRegistry::begin() == FrontendPluginRegistry::end()) - return Consumer; - // If this is a code completion run, avoid invoking the plugin consumers if (CI.hasCodeCompletionConsumer()) return Consumer; @@ -217,7 +212,8 @@ FrontendAction::CreateWrappedASTConsumer(CompilerInstance &CI, P->ParseArgs( CI, CI.getFrontendOpts().PluginArgs[std::string(Plugin.getName())])) { - std::unique_ptr PluginConsumer = P->CreateASTConsumer(CI, InFile); + std::unique_ptr PluginConsumer = + P->CreateASTConsumer(CI, InFile); if (ActionType == PluginASTAction::AddBeforeMainAction) { Consumers.push_back(std::move(PluginConsumer)); } else { @@ -226,6 +222,10 @@ FrontendAction::CreateWrappedASTConsumer(CompilerInstance &CI, } } + // Add rulesets consumer around the main consumer. + clang::rulesets::ClangRulesetsProvider::CreateAndAddASTConsumers( + CI, Consumers, AfterConsumers); + // Add to Consumers the main consumer, then all the plugins that go after it Consumers.push_back(std::move(Consumer)); if (!AfterConsumers.empty()) { @@ -303,16 +303,15 @@ static SourceLocation ReadOriginalFileName(CompilerInstance &CI, return T.getLocation(); } -static SmallVectorImpl & -operator+=(SmallVectorImpl &Includes, StringRef RHS) { +static SmallVectorImpl &operator+=(SmallVectorImpl &Includes, + StringRef RHS) { Includes.append(RHS.begin(), RHS.end()); return Includes; } static void addHeaderInclude(StringRef HeaderName, SmallVectorImpl &Includes, - const LangOptions &LangOpts, - bool IsExternC) { + const LangOptions &LangOpts, bool IsExternC) { if (IsExternC && LangOpts.CPlusPlus) Includes += "extern \"C\" {\n"; if (LangOpts.ObjC) @@ -352,7 +351,7 @@ static std::error_code collectModuleHeaderIncludes( if (!Module->MissingHeaders.empty()) { auto &MissingHeader = Module->MissingHeaders.front(); Diag.Report(MissingHeader.FileNameLoc, diag::err_module_header_missing) - << MissingHeader.IsUmbrella << MissingHeader.FileName; + << MissingHeader.IsUmbrella << MissingHeader.FileName; return std::error_code(); } @@ -504,7 +503,7 @@ static Module *prepareToBuildModule(CompilerInstance &CI, /*AllowSearch=*/true); if (!M) { CI.getDiagnostics().Report(diag::err_missing_module) - << CI.getLangOpts().CurrentModule << ModuleMapFilename; + << CI.getLangOpts().CurrentModule << ModuleMapFilename; return nullptr; } @@ -529,14 +528,16 @@ static Module *prepareToBuildModule(CompilerInstance &CI, /*openFile*/ true); if (!OriginalModuleMap) { CI.getDiagnostics().Report(diag::err_module_map_not_found) - << OriginalModuleMapName; + << OriginalModuleMapName; return nullptr; } if (*OriginalModuleMap != CI.getSourceManager().getFileEntryRefForID( CI.getSourceManager().getMainFileID())) { M->IsInferred = true; - CI.getPreprocessor().getHeaderSearchInfo().getModuleMap() - .setInferredModuleAllowedBy(M, *OriginalModuleMap); + CI.getPreprocessor() + .getHeaderSearchInfo() + .getModuleMap() + .setInferredModuleAllowedBy(M, *OriginalModuleMap); } } @@ -569,7 +570,7 @@ getInputBufferForModule(CompilerInstance &CI, Module *M) { if (Err) { CI.getDiagnostics().Report(diag::err_module_cannot_create_includes) - << M->getFullModuleName() << Err.message(); + << M->getFullModuleName() << Err.message(); return nullptr; } @@ -610,10 +611,9 @@ bool FrontendAction::BeginSourceFile(CompilerInstance &CI, IntrusiveRefCntPtr Diags(&CI.getDiagnostics()); // The AST unit populates its own diagnostics engine rather than ours. - IntrusiveRefCntPtr ASTDiags( - new DiagnosticsEngine(Diags->getDiagnosticIDs(), - &Diags->getDiagnosticOptions())); - ASTDiags->setClient(Diags->getClient(), /*OwnsClient*/false); + IntrusiveRefCntPtr ASTDiags(new DiagnosticsEngine( + Diags->getDiagnosticIDs(), &Diags->getDiagnosticOptions())); + ASTDiags->setClient(Diags->getClient(), /*OwnsClient*/ false); // FIXME: What if the input is a memory buffer? StringRef InputFile = Input.getFile(); @@ -741,7 +741,7 @@ bool FrontendAction::BeginSourceFile(CompilerInstance &CI, // Set up embedding for any specified files. Do this before we load any // source files, including the primary module map for the compilation. for (const auto &F : CI.getFrontendOpts().ModulesEmbedFiles) { - if (auto FE = CI.getFileManager().getOptionalFileRef(F, /*openFile*/true)) + if (auto FE = CI.getFileManager().getOptionalFileRef(F, /*openFile*/ true)) CI.getSourceManager().setFileIsTransient(*FE); else CI.getDiagnostics().Report(diag::err_modules_embed_file_not_found) << F; @@ -751,8 +751,7 @@ bool FrontendAction::BeginSourceFile(CompilerInstance &CI, // IR files bypass the rest of initialization. if (Input.getKind().getLanguage() == Language::LLVM_IR) { - assert(hasIRSupport() && - "This action does not have IR file support!"); + assert(hasIRSupport() && "This action does not have IR file support!"); // Inform the diagnostic client we are processing a source file. CI.getDiagnosticClient().BeginSourceFile(CI.getLangOpts(), nullptr); @@ -917,7 +916,7 @@ bool FrontendAction::BeginSourceFile(CompilerInstance &CI, for (const auto &Filename : CI.getFrontendOpts().ModuleMapFiles) { if (auto File = CI.getFileManager().getOptionalFileRef(Filename)) CI.getPreprocessor().getHeaderSearchInfo().loadModuleMapFile( - *File, /*IsSystem*/false); + *File, /*IsSystem*/ false); else CI.getDiagnostics().Report(diag::err_module_map_not_found) << Filename; } @@ -952,7 +951,8 @@ bool FrontendAction::BeginSourceFile(CompilerInstance &CI, // FIXME: should not overwrite ASTMutationListener when parsing model files? if (!isModelParsingAction()) - CI.getASTContext().setASTMutationListener(Consumer->GetASTMutationListener()); + CI.getASTContext().setASTMutationListener( + Consumer->GetASTMutationListener()); if (!CI.getPreprocessorOpts().ChainedIncludes.empty()) { // Convert headers to PCH and chain them. @@ -1036,9 +1036,8 @@ bool FrontendAction::BeginSourceFile(CompilerInstance &CI, // provides the layouts from that file. if (!CI.getFrontendOpts().OverrideRecordLayoutsFile.empty() && CI.hasASTContext() && !CI.getASTContext().getExternalSource()) { - IntrusiveRefCntPtr - Override(new LayoutOverrideSource( - CI.getFrontendOpts().OverrideRecordLayoutsFile)); + IntrusiveRefCntPtr Override(new LayoutOverrideSource( + CI.getFrontendOpts().OverrideRecordLayoutsFile)); CI.getASTContext().setExternalSource(Override); } @@ -1065,8 +1064,8 @@ llvm::Error FrontendAction::Execute() { if (CI.hasFrontendTimer()) { llvm::TimeRegion Timer(CI.getFrontendTimer()); ExecuteAction(); - } - else ExecuteAction(); + } else + ExecuteAction(); // If we are supposed to rebuild the global module index, do so now unless // there were any module-build failures. @@ -1116,7 +1115,8 @@ void FrontendAction::EndSourceFile() { } if (CI.getFrontendOpts().ShowStats) { - llvm::errs() << "\nSTATISTICS FOR '" << getCurrentFileOrBufferName() << "':\n"; + llvm::errs() << "\nSTATISTICS FOR '" << getCurrentFileOrBufferName() + << "':\n"; CI.getPreprocessor().PrintStats(); CI.getPreprocessor().getIdentifierTable().PrintStats(); CI.getPreprocessor().getHeaderSearchInfo().PrintStats(); @@ -1184,7 +1184,7 @@ void ASTFrontendAction::ExecuteAction() { CI.getFrontendOpts().SkipFunctionBodies); } -void PluginASTAction::anchor() { } +void PluginASTAction::anchor() {} std::unique_ptr PreprocessorFrontendAction::CreateASTConsumer(CompilerInstance &CI, @@ -1211,9 +1211,7 @@ bool WrapperFrontendAction::BeginSourceFileAction(CompilerInstance &CI) { setCurrentInput(WrappedAction->getCurrentInput()); return Ret; } -void WrapperFrontendAction::ExecuteAction() { - WrappedAction->ExecuteAction(); -} +void WrapperFrontendAction::ExecuteAction() { WrappedAction->ExecuteAction(); } void WrapperFrontendAction::EndSourceFile() { WrappedAction->EndSourceFile(); } void WrapperFrontendAction::EndSourceFileAction() { WrappedAction->EndSourceFileAction(); @@ -1243,4 +1241,4 @@ bool WrapperFrontendAction::hasCodeCompletionSupport() const { WrapperFrontendAction::WrapperFrontendAction( std::unique_ptr WrappedAction) - : WrappedAction(std::move(WrappedAction)) {} + : WrappedAction(std::move(WrappedAction)) {}