diff --git a/Makefile b/Makefile index 42b754733ec..5e7191dab39 100644 --- a/Makefile +++ b/Makefile @@ -669,7 +669,7 @@ $(libcppdir)/summaries.o: lib/summaries.cpp lib/analyzerinfo.h lib/checkers.h li $(libcppdir)/suppressions.o: lib/suppressions.cpp externals/tinyxml2/tinyxml2.h lib/addoninfo.h lib/checkers.h lib/config.h lib/errorlogger.h lib/errortypes.h lib/filesettings.h lib/library.h lib/mathlib.h lib/path.h lib/pathmatch.h lib/platform.h lib/settings.h lib/smallvector.h lib/standards.h lib/suppressions.h lib/templatesimplifier.h lib/token.h lib/tokenlist.h lib/utils.h lib/vfvalue.h lib/xml.h $(CXX) ${INCLUDE_FOR_LIB} $(CPPFLAGS) $(CXXFLAGS) -c -o $@ $(libcppdir)/suppressions.cpp -$(libcppdir)/templatesimplifier.o: lib/templatesimplifier.cpp lib/checkers.h lib/config.h lib/errorlogger.h lib/errortypes.h lib/library.h lib/mathlib.h lib/platform.h lib/settings.h lib/smallvector.h lib/standards.h lib/templatesimplifier.h lib/token.h lib/tokenize.h lib/tokenlist.h lib/utils.h lib/vfvalue.h +$(libcppdir)/templatesimplifier.o: lib/templatesimplifier.cpp lib/astutils.h lib/checkers.h lib/config.h lib/errorlogger.h lib/errortypes.h lib/library.h lib/mathlib.h lib/platform.h lib/settings.h lib/smallvector.h lib/sourcelocation.h lib/standards.h lib/symboldatabase.h lib/templatesimplifier.h lib/token.h lib/tokenize.h lib/tokenlist.h lib/utils.h lib/vfvalue.h $(CXX) ${INCLUDE_FOR_LIB} $(CPPFLAGS) $(CXXFLAGS) -c -o $@ $(libcppdir)/templatesimplifier.cpp $(libcppdir)/timer.o: lib/timer.cpp lib/config.h lib/timer.h diff --git a/cli/cmdlineparser.cpp b/cli/cmdlineparser.cpp index f2d2254f833..397f21e0c35 100644 --- a/cli/cmdlineparser.cpp +++ b/cli/cmdlineparser.cpp @@ -1548,6 +1548,11 @@ CmdLineParser::Result CmdLineParser::parseFromArgs(int argc, const char* const a // TODO: bail out when no placeholders are found? } + // Recreate the symbol database etc. after each template type deduction round + // instead of updating them incrementally - for testing and debugging + else if (std::strcmp(argv[i], "--template-full-rebuild") == 0) + mSettings.templateFullRebuild = true; + else if (std::strncmp(argv[i], "--template-location=", 20) == 0) { mSettings.templateLocation = argv[i] + 20; // TODO: bail out when no template is provided? diff --git a/lib/settings.h b/lib/settings.h index ce4a32e1690..4f97e293ac0 100644 --- a/lib/settings.h +++ b/lib/settings.h @@ -466,6 +466,11 @@ class CPPCHECKLIB WARN_UNUSED Settings { e.g. "{severity} {file}:{line} {message} {id}" */ std::string templateFormat; + /** @brief Is --template-full-rebuild given? Recreate the symbol database etc. from + * scratch after each template type deduction round instead of updating them + * incrementally. For testing and debugging. */ + bool templateFullRebuild = false; + /** @brief The output format in which the error locations are printed in * text mode, e.g. "{file}:{line} {info}" */ std::string templateLocation; diff --git a/lib/symboldatabase.cpp b/lib/symboldatabase.cpp index b687c9f77a4..e2e1a66841d 100644 --- a/lib/symboldatabase.cpp +++ b/lib/symboldatabase.cpp @@ -56,6 +56,8 @@ #include //--------------------------------------------------------------------------- +static const Type* findVariableTypeInBase(const Scope* scope, const Token* typeTok); + SymbolDatabase::SymbolDatabase(Tokenizer& tokenizer) : mTokenizer(tokenizer) , mSettings(tokenizer.getSettings()) @@ -74,11 +76,9 @@ SymbolDatabase::SymbolDatabase(Tokenizer& tokenizer) createSymbolDatabaseFindAllScopes(); createSymbolDatabaseClassInfo(); createSymbolDatabaseVariableInfo(); - createSymbolDatabaseCopyAndMoveConstructors(); createSymbolDatabaseFunctionScopes(); createSymbolDatabaseClassAndStructScopes(); createSymbolDatabaseFunctionReturnTypes(); - createSymbolDatabaseNeedInitialization(); createSymbolDatabaseVariableSymbolTable(); createSymbolDatabaseSetScopePointers(); createSymbolDatabaseSetVariablePointers(); @@ -88,6 +88,15 @@ SymbolDatabase::SymbolDatabase(Tokenizer& tokenizer) createSymbolDatabaseSetSmartPointerType(); setValueTypeInTokenList(false); createSymbolDatabaseEnums(); +} + +void SymbolDatabase::finalize() +{ + if (!mTokenizer.tokens()) + return; + + createSymbolDatabaseCopyAndMoveConstructors(); + createSymbolDatabaseNeedInitialization(); createSymbolDatabaseEscapeFunctions(); createSymbolDatabaseIncompleteVars(); createSymbolDatabaseExprIds(); @@ -148,8 +157,13 @@ void SymbolDatabase::createSymbolDatabaseFindAllScopes() // create global scope scopeList.emplace_back(*this, nullptr, nullptr); + findAllScopes(mTokenizer.tokens(), nullptr, &scopeList.back()); +} + +void SymbolDatabase::findAllScopes(const Token* startToken, const Token* endToken, Scope* startScope) +{ // pointer to current scope - Scope *scope = &scopeList.back(); + Scope *scope = startScope; // Store the ending of init lists std::stack> endInitList; @@ -174,6 +188,8 @@ void SymbolDatabase::createSymbolDatabaseFindAllScopes() // Store current access in each scope (depends on evaluation progress) std::map access; + if (startScope->isClassOrStructOrUnion()) + access[startScope] = startScope->type == ScopeType::eClass ? AccessControl::Private : AccessControl::Public; std::map> forwardDecls; @@ -202,7 +218,7 @@ void SymbolDatabase::createSymbolDatabaseFindAllScopes() ProgressReporter progressReporter(mErrorLogger, mSettings.reportProgress, mTokenizer.list.getSourceFilePath(), "SymbolDatabase (find all scopes)"); // find all scopes - for (const Token *tok = mTokenizer.tokens(); tok; tok = tok ? tok->next() : nullptr) { + for (const Token *tok = startToken; tok && tok != endToken; tok = tok ? tok->next() : nullptr) { // #5593 suggested to add here: progressReporter.report(tok->progressValue()); @@ -942,23 +958,26 @@ void SymbolDatabase::createSymbolDatabaseClassAndStructScopes() } } +void SymbolDatabase::setFunctionReturnType(Function& function, const Scope& scope) const +{ + if (!function.retDef) + return; + const Token *type = function.retDef; + while (Token::Match(type, "static|const|struct|union|enum")) + type = type->next(); + if (type) { + function.retType = findVariableTypeInBase(&scope, type); + if (!function.retType) + function.retType = findTypeInNested(type, function.nestedIn); + } +} + void SymbolDatabase::createSymbolDatabaseFunctionReturnTypes() { // fill in function return types for (Scope& scope : scopeList) { - for (auto func = scope.functionList.begin(); func != scope.functionList.end(); ++func) { - // add return types - if (func->retDef) { - const Token *type = func->retDef; - while (Token::Match(type, "static|const|struct|union|enum")) - type = type->next(); - if (type) { - func->retType = findVariableTypeInBase(&scope, type); - if (!func->retType) - func->retType = findTypeInNested(type, func->nestedIn); - } - } - } + for (Function& function : scope.functionList) + setFunctionReturnType(function, scope); } } @@ -1063,69 +1082,79 @@ void SymbolDatabase::createSymbolDatabaseNeedInitialization() } } -void SymbolDatabase::createSymbolDatabaseVariableSymbolTable() +void SymbolDatabase::addVariablesToSymbolTable(Scope& scope) { - // create variable symbol table - mVariableList.resize(mTokenizer.varIdCount() + 1); - std::fill_n(mVariableList.begin(), mVariableList.size(), nullptr); + for (Variable& var: scope.varlist) { + const int varId = var.declarationId(); + if (varId) + mVariableList[varId] = &var; + // fix up variables without type + if (!var.type() && !var.typeStartToken()->isStandardType()) { + const Type *type = findType(var.typeStartToken(), &scope); + if (type) + var.type(type); + } + } +} - // check all scopes for variables - for (Scope& scope : scopeList) { - // add all variables - for (Variable& var: scope.varlist) { - const int varId = var.declarationId(); - if (varId) - mVariableList[varId] = &var; - // fix up variables without type - if (!var.type() && !var.typeStartToken()->isStandardType()) { - const Type *type = findType(var.typeStartToken(), &scope); +void SymbolDatabase::addArgumentsToSymbolTable(Function& function, const Scope& scope) +{ + for (Variable& arg: function.argumentList) { + // check for named parameters + if (arg.nameToken() && arg.declarationId()) { + const int declarationId = arg.declarationId(); + mVariableList[declarationId] = &arg; + // fix up parameters without type + if (!arg.type() && !arg.typeStartToken()->isStandardType()) { + const Type *type = findTypeInNested(arg.typeStartToken(), &scope); if (type) - var.type(type); + arg.type(type); } } + } +} - // add all function parameters - for (Function& func : scope.functionList) { - for (Variable& arg: func.argumentList) { - // check for named parameters - if (arg.nameToken() && arg.declarationId()) { - const int declarationId = arg.declarationId(); - mVariableList[declarationId] = &arg; - // fix up parameters without type - if (!arg.type() && !arg.typeStartToken()->isStandardType()) { - const Type *type = findTypeInNested(arg.typeStartToken(), &scope); - if (type) - arg.type(type); - } +void SymbolDatabase::fillMissingVariables(const Scope& functionScope) +{ + for (const Token *tok = functionScope.bodyStart->next(); tok && tok != functionScope.bodyEnd; tok = tok->next()) { + // check for member variable + if (!Token::Match(tok, "%var% .|[")) + continue; + const Token* tokDot = tok->next(); + while (Token::simpleMatch(tokDot, "[")) + tokDot = tokDot->link()->next(); + if (!Token::Match(tokDot, ". %var%")) + continue; + const Token *member = tokDot->next(); + if (mVariableList[member->varId()] == nullptr) { + const Variable *var1 = mVariableList[tok->varId()]; + if (var1 && var1->typeScope()) { + const Variable* memberVar = var1->typeScope()->getVariable(member->str()); + if (memberVar) { + // add this variable to the look up table + mVariableList[member->varId()] = memberVar; } } } } +} - // fill in missing variables if possible - for (const Scope *func: functionScopes) { - for (const Token *tok = func->bodyStart->next(); tok && tok != func->bodyEnd; tok = tok->next()) { - // check for member variable - if (!Token::Match(tok, "%var% .|[")) - continue; - const Token* tokDot = tok->next(); - while (Token::simpleMatch(tokDot, "[")) - tokDot = tokDot->link()->next(); - if (!Token::Match(tokDot, ". %var%")) - continue; - const Token *member = tokDot->next(); - if (mVariableList[member->varId()] == nullptr) { - const Variable *var1 = mVariableList[tok->varId()]; - if (var1 && var1->typeScope()) { - const Variable* memberVar = var1->typeScope()->getVariable(member->str()); - if (memberVar) { - // add this variable to the look up table - mVariableList[member->varId()] = memberVar; - } - } - } - } +void SymbolDatabase::createSymbolDatabaseVariableSymbolTable() +{ + // create variable symbol table + mVariableList.resize(mTokenizer.varIdCount() + 1); + std::fill_n(mVariableList.begin(), mVariableList.size(), nullptr); + + // check all scopes for variables and function parameters + for (Scope& scope : scopeList) { + addVariablesToSymbolTable(scope); + for (Function& function : scope.functionList) + addArgumentsToSymbolTable(function, scope); } + + // fill in missing variables if possible + for (const Scope *func: functionScopes) + fillMissingVariables(*func); } void SymbolDatabase::createSymbolDatabaseSetScopePointers() @@ -1477,6 +1506,10 @@ void SymbolDatabase::createSymbolDatabaseEnums() // find enumerators for (Token* tok = mTokenizer.list.front(); tok != mTokenizer.list.back(); tok = tok->next()) { + // an eEnumerator token without an enumerator is a copy that the template + // simplifier made of an already resolved token - resolve it again + if (tok->tokType() == Token::eEnumerator && !tok->enumerator()) + tok->tokType(Token::eName); const bool isVariable = (tok->tokType() == Token::eVariable && !tok->variable()); if (tok->tokType() != Token::eName && !isVariable) continue; @@ -1835,6 +1868,212 @@ void SymbolDatabase::createSymbolDatabaseExprIds() } } +void SymbolDatabase::removeSymbolsForTokens(const std::unordered_set& removedTokens) +{ + if (removedTokens.empty()) + return; + + // the scopes, functions, variables, types and enumerators declared by those tokens + std::unordered_set removedScopes; + for (const Scope& scope : scopeList) { + if ((scope.classDef && removedTokens.count(scope.classDef) != 0) || + (scope.bodyStart && removedTokens.count(scope.bodyStart) != 0)) + removedScopes.insert(&scope); + } + std::unordered_set removedFunctions; + for (Scope& scope : scopeList) { + for (Function& function : scope.functionList) { + if (function.tokenDef && removedTokens.count(function.tokenDef) != 0) + removedFunctions.insert(&function); + else if (function.token && removedTokens.count(function.token) != 0) { + // only the function definition is removed - the declaration remains + function.token = nullptr; + function.arg = function.argDef; + function.functionScope = nullptr; + function.hasBody(false); + } + } + } + std::unordered_set removedVariables; + for (const Scope* scope : removedScopes) { + std::transform(scope->varlist.cbegin(), scope->varlist.cend(), std::inserter(removedVariables, removedVariables.end()), [](const Variable& var) { + return &var; + }); + } + for (const Function* function : removedFunctions) { + std::transform(function->argumentList.cbegin(), function->argumentList.cend(), std::inserter(removedVariables, removedVariables.end()), [](const Variable& arg) { + return &arg; + }); + } + std::unordered_set removedTypes; + for (const Type& type : typeList) { + if (type.classDef && removedTokens.count(type.classDef) != 0) + removedTypes.insert(&type); + } + std::unordered_set removedEnumerators; + for (const Scope* scope : removedScopes) { + std::transform(scope->enumeratorList.cbegin(), scope->enumeratorList.cend(), std::inserter(removedEnumerators, removedEnumerators.end()), [](const Enumerator& enumerator) { + return &enumerator; + }); + } + + // clear the references from the surviving tokens + for (Token* tok = mTokenizer.list.front(); tok; tok = tok->next()) { + if (removedTokens.count(tok) != 0) + continue; + if (tok->function() && removedFunctions.count(tok->function()) != 0) + tok->function(nullptr); + if (tok->variable() && removedVariables.count(tok->variable()) != 0) + tok->variable(nullptr); + if (tok->type() && removedTypes.count(tok->type()) != 0) + tok->type(nullptr); + if (tok->enumerator() && removedEnumerators.count(tok->enumerator()) != 0) + tok->enumerator(nullptr); + if (tok->scope() && removedScopes.count(tok->scope()) != 0) + tok->scope(nullptr); + } + + // remove from the indexes + for (std::size_t i = 0; i < mVariableList.size(); ++i) { + if (mVariableList[i] && removedVariables.count(mVariableList[i]) != 0) + mVariableList[i] = nullptr; + } + functionScopes.erase(std::remove_if(functionScopes.begin(), functionScopes.end(), [&](const Scope* scope) { + return removedScopes.count(scope) != 0; + }), functionScopes.end()); + classAndStructScopes.erase(std::remove_if(classAndStructScopes.begin(), classAndStructScopes.end(), [&](const Scope* scope) { + return removedScopes.count(scope) != 0; + }), classAndStructScopes.end()); + + // remove from the owning scopes + for (Scope& scope : scopeList) { + if (removedScopes.count(&scope) != 0) + continue; + scope.nestedList.erase(std::remove_if(scope.nestedList.begin(), scope.nestedList.end(), [&](const Scope* nested) { + return removedScopes.count(nested) != 0; + }), scope.nestedList.end()); + for (auto it = scope.functionMap.begin(); it != scope.functionMap.end();) { + if (removedFunctions.count(it->second) != 0) + it = scope.functionMap.erase(it); + else + ++it; + } + for (auto it = scope.functionList.begin(); it != scope.functionList.end();) { + if (removedFunctions.count(&(*it)) != 0) + it = scope.functionList.erase(it); + else + ++it; + } + for (auto it = scope.definedTypesMap.begin(); it != scope.definedTypesMap.end();) { + if (removedTypes.count(it->second) != 0) + it = scope.definedTypesMap.erase(it); + else + ++it; + } + } + typeList.remove_if([&](const Type& type) { + return removedTypes.count(&type) != 0; + }); + scopeList.remove_if([&](const Scope& scope) { + return removedScopes.count(&scope) != 0; + }); +} + +void SymbolDatabase::addSymbolsForNewTokenRanges(const std::vector>& newRanges) +{ + if (newRanges.empty()) + return; + + const std::size_t oldScopeCount = scopeList.size(); + + // all new tokens - the new functions are found through them afterwards + std::unordered_set newTokens; + for (const auto& range : newRanges) { + for (const Token* tok = range.first; tok; tok = tok->next()) { + newTokens.insert(tok); + if (tok == range.second) + break; + } + } + + for (const auto& range : newRanges) { + // the scope the new tokens are in is the scope of the nearest preceding token + const Token* anchor = range.first->previous(); + while (anchor && !anchor->scope()) + anchor = anchor->previous(); + const Scope* enclosing = anchor ? anchor->scope() : &scopeList.front(); + if (!enclosing) + continue; + if (anchor && anchor == enclosing->bodyEnd) + enclosing = enclosing->nestedIn ? enclosing->nestedIn : &scopeList.front(); + findAllScopes(range.first, range.second->next(), const_cast(enclosing)); + } + + // the scopes and functions that were added + std::vector newScopes; + { + auto it = scopeList.begin(); + std::advance(it, oldScopeCount); + for (; it != scopeList.end(); ++it) + newScopes.push_back(&(*it)); + } + std::vector> newFunctions; // the function and the scope it is declared in + for (Scope& scope : scopeList) { + for (Function& function : scope.functionList) { + if (function.tokenDef && newTokens.count(function.tokenDef) != 0) + newFunctions.emplace_back(&function, &scope); + } + } + + // set the scope pointers - the old tokens keep their scopes + createSymbolDatabaseSetScopePointers(); + + // the variables of the new scopes, the arguments and return types of the new functions + for (Scope* scope : newScopes) + scope->getVariableList(); + for (const auto& f : newFunctions) { + if (f.first->argumentList.empty()) + f.first->addArguments(f.second); + setFunctionReturnType(*f.first, *f.second); + } + + // fast access vectors + for (const Scope* scope : newScopes) { + if (scope->type == ScopeType::eFunction) + functionScopes.push_back(scope); + if (scope->isClassOrStruct()) + classAndStructScopes.push_back(scope); + } + + // variable symbol table: add the new variables (incremental version of + // createSymbolDatabaseVariableSymbolTable) + mVariableList.resize(mTokenizer.varIdCount() + 1); + for (Scope* scope : newScopes) + addVariablesToSymbolTable(*scope); + for (const auto& f : newFunctions) + addArgumentsToSymbolTable(*f.first, *f.second); + // fill in missing variables in the new code if possible + for (const Scope* scope : newScopes) { + if (scope->type == ScopeType::eFunction) + fillMissingVariables(*scope); + } + + // set the definition and declaration pointers of the new functions. The call + // pointers are updated afterwards by updateFunctionAndVariablePointers(). + for (const auto& f : newFunctions) { + if (f.first->tokenDef) + const_cast(f.first->tokenDef)->function(f.first); + if (f.first->token) + const_cast(f.first->token)->function(f.first); + } + + // type/enumerator pointers - these only set pointers that are not set, so the old + // tokens are not affected + createSymbolDatabaseSetTypePointers(); + createSymbolDatabaseSetSmartPointerType(); + createSymbolDatabaseEnums(); +} + // cppcheck-suppress functionConst - has side effects void SymbolDatabase::setArrayDimensionsUsingValueFlow() { @@ -1947,6 +2186,7 @@ SymbolDatabase::~SymbolDatabase() tok->variable(nullptr); tok->enumerator(nullptr); tok->setValueType(nullptr); + tok->exprId(0); } } @@ -5668,7 +5908,7 @@ const Enumerator * SymbolDatabase::findEnumerator(const Token * tok, std::setdefinedType && !scope->definedType->derivedFrom.empty()) { const std::vector &derivedFrom = scope->definedType->derivedFrom; @@ -7761,15 +8001,15 @@ static int getIntegerConstantMacroWidth(const Token* tok) { return intnum; } -void SymbolDatabase::setValueTypeInTokenList(bool reportDebugWarnings, Token *tokens) +void SymbolDatabase::setValueTypeInTokenList(bool reportDebugWarnings, Token *tokens, const Token *endToken) { if (!tokens) tokens = mTokenizer.list.front(); - for (Token *tok = tokens; tok; tok = tok->next()) + for (Token *tok = tokens; tok && tok != endToken; tok = tok->next()) tok->setValueType(nullptr); - for (Token *tok = tokens; tok; tok = tok->next()) { + for (Token *tok = tokens; tok && tok != endToken; tok = tok->next()) { if (tok->isNumber()) { if (MathLib::isFloat(tok->str())) { ValueType::Type type = ValueType::Type::DOUBLE; @@ -8204,7 +8444,7 @@ void SymbolDatabase::setValueTypeInTokenList(bool reportDebugWarnings, Token *to } if (reportDebugWarnings && mSettings.debugwarnings) { - for (Token *tok = tokens; tok; tok = tok->next()) { + for (Token *tok = tokens; tok && tok != endToken; tok = tok->next()) { if (tok->str() == "auto" && !tok->valueType()) { if (Token::Match(tok->next(), "%name% ; %name% = [") && isLambdaCaptureList(tok->tokAt(5))) continue; @@ -8215,6 +8455,10 @@ void SymbolDatabase::setValueTypeInTokenList(bool reportDebugWarnings, Token *to } } + // when only a range was set the caller is responsible for updating the pointers + if (endToken) + return; + // Update functions with new type information. createSymbolDatabaseSetFunctionPointers(false); @@ -8222,6 +8466,12 @@ void SymbolDatabase::setValueTypeInTokenList(bool reportDebugWarnings, Token *to createSymbolDatabaseSetVariablePointers(); } +void SymbolDatabase::updateFunctionAndVariablePointers() +{ + createSymbolDatabaseSetFunctionPointers(false); + createSymbolDatabaseSetVariablePointers(); +} + ValueType ValueType::parseDecl(const Token *type, const Settings &settings) { ValueType vt; diff --git a/lib/symboldatabase.h b/lib/symboldatabase.h index 166f228ad99..c9a38e7203a 100644 --- a/lib/symboldatabase.h +++ b/lib/symboldatabase.h @@ -38,6 +38,7 @@ #include #include #include +#include #include #include @@ -1338,6 +1339,11 @@ class CPPCHECKLIB ValueType { class CPPCHECKLIB SymbolDatabase { friend class TestSymbolDatabase; public: + /** + * The phases that only later analysis (ValueFlow, checks) needs are not run by the + * constructor - call finalize() when the token list is final, e.g. after the + * template simplifier can no longer change it. + */ explicit SymbolDatabase(Tokenizer& tokenizer); ~SymbolDatabase(); @@ -1411,8 +1417,14 @@ class CPPCHECKLIB SymbolDatabase { */ void validate() const; - /** Set valuetype in provided tokenlist */ - void setValueTypeInTokenList(bool reportDebugWarnings, Token *tokens=nullptr); + /** Set valuetype in provided tokenlist. When endToken (exclusive) is given, only + * that token range is set and the caller is responsible for updating the function + * and variable pointers (updateFunctionAndVariablePointers()). */ + void setValueTypeInTokenList(bool reportDebugWarnings, Token *tokens=nullptr, const Token *endToken=nullptr); + + /** Update the function pointers of calls and the variable pointers in the whole + * token list, e.g. after tokens were added or renamed. */ + void updateFunctionAndVariablePointers(); /** * Calculates sizeof value for given type. @@ -1427,6 +1439,29 @@ class CPPCHECKLIB SymbolDatabase { void clangSetVariables(const std::vector &vars); void createSymbolDatabaseExprIds(); + /** + * Remove the symbols (scopes, functions, variables, types) that are declared by the + * given tokens and clear the references to them. Called before the tokens are + * removed from the token list, e.g. when the template simplifier removes + * instantiated template declarations. + */ + void removeSymbolsForTokens(const std::unordered_set& removedTokens); + + /** + * Add the symbols for token ranges (both range ends are inclusive) that were added + * to the token list after the symbol database was created, e.g. by template + * instantiations. This is an incremental alternative to recreating the database. + * Only new function declarations/definitions are supported - not new classes. + * setValueTypeInTokenList() should be called afterwards. + */ + void addSymbolsForNewTokenRanges(const std::vector>& newRanges); + + /** + * Run the phases that only later analysis (ValueFlow, checks) needs. Call exactly + * once, when the token list is final. + */ + void finalize(); + /* returns the opening { if tok points to enum */ static const Token* isEnumDefinition(const Token* tok); @@ -1445,12 +1480,28 @@ class CPPCHECKLIB SymbolDatabase { * @throws InternalError thrown on unhandled code */ void createSymbolDatabaseFindAllScopes(); + /** + * Find the scopes in the given token range (endToken is exclusive, nullptr means + * the end of the token list) and add the symbols to the database. startScope is + * the scope the range is in. + * @throws InternalError thrown on unhandled code + */ + void findAllScopes(const Token* startToken, const Token* endToken, Scope* startScope); void createSymbolDatabaseClassInfo(); void createSymbolDatabaseVariableInfo(); void createSymbolDatabaseCopyAndMoveConstructors(); void createSymbolDatabaseFunctionScopes(); void createSymbolDatabaseClassAndStructScopes(); void createSymbolDatabaseFunctionReturnTypes(); + /** fill in the return type of one function (used by the full build and by + * addSymbolsForNewTokenRanges) */ + void setFunctionReturnType(Function& function, const Scope& scope) const; + /** add the variables of one scope to the variable symbol table */ + void addVariablesToSymbolTable(Scope& scope); + /** add the parameters of one function to the variable symbol table */ + void addArgumentsToSymbolTable(Function& function, const Scope& scope); + /** fill in the missing member variables in one function scope */ + void fillMissingVariables(const Scope& functionScope); void createSymbolDatabaseNeedInitialization(); void createSymbolDatabaseVariableSymbolTable(); void createSymbolDatabaseSetScopePointers(); @@ -1475,7 +1526,6 @@ class CPPCHECKLIB SymbolDatabase { const Type *findTypeInNested(const Token *startTok, const Scope *startScope) const; const Scope *findNamespace(const Token * tok, const Scope * scope) const; static Function *findFunctionInScope(const Token *func, const Scope *ns, const std::string & path, nonneg int path_length); - static const Type *findVariableTypeInBase(const Scope *scope, const Token *typeTok); using MemberIdMap = std::map; using VarIdMap = std::map; diff --git a/lib/templatesimplifier.cpp b/lib/templatesimplifier.cpp index 6fdd423628b..47245c729d8 100644 --- a/lib/templatesimplifier.cpp +++ b/lib/templatesimplifier.cpp @@ -18,11 +18,13 @@ #include "templatesimplifier.h" +#include "astutils.h" #include "errorlogger.h" #include "errortypes.h" #include "mathlib.h" #include "settings.h" #include "standards.h" +#include "symboldatabase.h" #include "token.h" #include "tokenize.h" #include "tokenlist.h" @@ -31,10 +33,12 @@ #include #include #include +#include #include #include #include #include +#include #include static Token *skipRequires(Token *tok) @@ -792,9 +796,574 @@ static bool isTemplateInstantion(const Token* tok) return Token::Match(tok->tokAt(-2), "(|{|}|;|=|<<|:|.|*|&|return|<|,|!|[ :: %name% ::|<|("); } +namespace { + // A function template parameter with a form that is supported by type deduction: + // "T", "T &", "const T &", "T *", "const T *" or a concrete single token type + struct ParameterShape { + const Token* typeTok = nullptr; + int templateParameterIndex = -1; // -1 => a concrete type, nothing is deduced + bool isConst = false; + bool isPointer = false; + bool isReference = false; + }; + + // The type deduced for a template parameter from a function call argument + struct DeducedType { + std::string typeStr; // base type, e.g. "int" or "MyClass" + std::vector qualification; // enclosing scopes for record types, outermost first + unsigned int constness = 0; // bit 0 = data const, bit 1 = first '*' const, .. + int pointer = 0; // number of '*' + bool isUnsigned = false; + bool isLong = false; + + bool operator==(const DeducedType& other) const { + return typeStr == other.typeStr && + qualification == other.qualification && + constness == other.constness && + pointer == other.pointer && + isUnsigned == other.isUnsigned && + isLong == other.isLong; + } + }; + + // A parsed function template declaration that type deduction supports. Invalid + // (typeParameters is empty) when the declaration is not supported. + struct DeductionCandidate { + std::vector typeParameters; + std::vector parameterShapes; + std::string signature; + }; +} + +// Parse a function parameter of a template declaration. The returned shape has no +// typeTok when the parameter does not have a supported form. +static ParameterShape parseParameterShape(const Token* start, const std::vector& typeParameters) +{ + ParameterShape shape; + const Token* tok = start; + if (Token::simpleMatch(tok, "const")) { + shape.isConst = true; + tok = tok->next(); + } + if (!Token::Match(tok, "%type%") || Token::Match(tok, "const|struct|class|union|enum")) + return ParameterShape(); + shape.typeTok = tok; + tok = tok->next(); + if (Token::simpleMatch(tok, "*")) { + shape.isPointer = true; + tok = tok->next(); + } else if (Token::simpleMatch(tok, "&")) { + shape.isReference = true; + tok = tok->next(); + } + if (Token::Match(tok, "%name% ,|)")) + tok = tok->next(); + if (!Token::Match(tok, ",|)")) + return ParameterShape(); + const auto it = std::find_if(typeParameters.cbegin(), typeParameters.cend(), [&](const Token* typeParameter) { + return typeParameter->str() == shape.typeTok->str(); + }); + if (it != typeParameters.cend()) + shape.templateParameterIndex = static_cast(it - typeParameters.cbegin()); + return shape; +} + +// Convert the ValueType of an argument expression to the type that the parameter shape +// deduces for its template parameter. The returned type has an empty typeStr when the +// type is not supported. +static DeducedType valueTypeToDeducedType(const ValueType* vt, const ParameterShape& shape) +{ + if (!vt) + return DeducedType(); + if (vt->bits > 0 || vt->volatileness != 0) + return DeducedType(); + if (vt->smartPointer || vt->smartPointerType || vt->smartPointerTypeToken || vt->container || vt->containerTypeToken) + return DeducedType(); + + int pointer = vt->pointer; + unsigned int constness = vt->constness; + if (shape.isPointer) { + // "T *": the argument must be a pointer and T is the pointee type + if (pointer < 1) + return DeducedType(); + constness &= ~(1U << pointer); + --pointer; + } else if (!shape.isReference || shape.isConst) { + // "T" and "const T &" deduction drops the top level const + constness &= ~(1U << pointer); + } + // else "T &": the constness is part of the deduced type + + if (pointer > 2) + return DeducedType(); + if (constness >= (1U << (pointer + 1))) + return DeducedType(); + DeducedType deduced; + deduced.pointer = pointer; + deduced.constness = constness; + + switch (vt->type) { + case ValueType::BOOL: + deduced.typeStr = "bool"; + break; + case ValueType::CHAR: + deduced.typeStr = "char"; + deduced.isUnsigned = vt->sign == ValueType::UNSIGNED; + break; + case ValueType::SHORT: + deduced.typeStr = "short"; + deduced.isUnsigned = vt->sign == ValueType::UNSIGNED; + break; + case ValueType::WCHAR_T: + deduced.typeStr = "wchar_t"; + break; + case ValueType::INT: + deduced.typeStr = "int"; + deduced.isUnsigned = vt->sign == ValueType::UNSIGNED; + break; + case ValueType::LONG: + deduced.typeStr = "long"; + deduced.isUnsigned = vt->sign == ValueType::UNSIGNED; + break; + case ValueType::LONGLONG: + deduced.typeStr = "long"; + deduced.isLong = true; + deduced.isUnsigned = vt->sign == ValueType::UNSIGNED; + break; + case ValueType::FLOAT: + deduced.typeStr = "float"; + break; + case ValueType::DOUBLE: + deduced.typeStr = "double"; + break; + case ValueType::LONGDOUBLE: + deduced.typeStr = "double"; + deduced.isLong = true; + break; + case ValueType::RECORD: { + const Scope* typeScope = vt->typeScope; + if (!typeScope || typeScope->className.empty() || !typeScope->isClassOrStructOrUnion()) + return DeducedType(); + deduced.typeStr = typeScope->className; + for (const Scope* nested = typeScope->nestedIn; nested && nested->type != ScopeType::eGlobal; nested = nested->nestedIn) { + if (nested->type != ScopeType::eNamespace && !nested->isClassOrStructOrUnion()) + return DeducedType(); // local types etc. cannot be named - bail out + if (nested->className.empty()) + return DeducedType(); + deduced.qualification.insert(deduced.qualification.cbegin(), nested->className); + } + break; + } + case ValueType::UNKNOWN_TYPE: + case ValueType::POD: + case ValueType::NONSTD: + case ValueType::SMART_POINTER: + case ValueType::CONTAINER: + case ValueType::ITERATOR: + case ValueType::VOID: + case ValueType::UNKNOWN_INT: + // not deducible + return DeducedType(); + } + + return deduced; +} + +// Is there a non template function with the given name in the scope? Overload +// resolution is not implemented, so a call is not deduced when such a function might +// be a better overload match. +static bool scopeHasNonTemplateFunction(const Scope* scope, const std::string& name) +{ + const auto range = scope->functionMap.equal_range(name); + return std::any_of(range.first, range.second, [](const std::pair& func) { + return !func.second->templateDef; + }); +} + +// The qualified name of a scope ("N :: A"). Empty when the scope or an enclosing scope +// cannot be named (anonymous namespaces, local classes, ..). +static std::string scopeQualifiedName(const Scope* scope) +{ + std::string name; + for (const Scope* it = scope; it && it->type != ScopeType::eGlobal; it = it->nestedIn) { + if (it->className.empty() || (!it->isClassOrStructOrUnion() && it->type != ScopeType::eNamespace)) + return ""; + name = name.empty() ? it->className : (it->className + " :: " + name); + } + return name; +} + +// Insert the tokens of a deduced type after tok (the tokens are inserted in reverse order) +static void insertDeducedType(Token* tok, const DeducedType& deduced) +{ + for (int level = deduced.pointer; level >= 1; --level) { + if (deduced.constness & (1U << level)) + tok->insertToken("const"); + tok->insertToken("*"); + } + tok->insertToken(deduced.typeStr); + tok->next()->isUnsigned(deduced.isUnsigned); + tok->next()->isLong(deduced.isLong); + for (auto it = deduced.qualification.crbegin(); it != deduced.qualification.crend(); ++it) { + tok->insertToken("::"); + tok->insertToken(*it); + } + if (deduced.constness & 1U) + tok->insertToken("const"); +} + +std::string TemplateSimplifier::deduceFunctionTemplateArguments(Token* tok, + std::string qualification, + const std::string& scopeName, + const std::multimap& functionNameMap, + std::map& parameterCountCache) +{ + const auto range = functionNameMap.equal_range(tok->str()); + if (range.first == range.second) + return qualification; + + std::string fullName; + if (!qualification.empty()) + fullName = qualification + " :: " + tok->str(); + else if (!scopeName.empty()) + fullName = scopeName + " :: " + tok->str(); + else + fullName = tok->str(); + + std::vector instantiationArgs; + getFunctionArguments(tok, instantiationArgs); + + // deduction from literal arguments - works without type information + for (auto pos = range.first; pos != range.second; ++pos) { + // look for declaration with same qualification or constructor with same qualification + if (pos->second->fullName() == fullName || + (pos->second->scope() == fullName && tok->str() == pos->second->name())) { + std::vector templateParams; + getTemplateParametersInDeclaration(pos->second->token()->tokAt(2), templateParams); + + // todo: handle more than one template parameter + if (templateParams.size() != 1 || !areAllParamsTypes(templateParams)) + continue; + + std::vector declarationParams; + getFunctionArguments(pos->second->nameToken(), declarationParams); + + // function argument counts must match + if (instantiationArgs.empty() || instantiationArgs.size() != declarationParams.size()) + continue; + + size_t match = 0; + size_t argMatch = 0; + for (size_t i = 0; i < declarationParams.size(); ++i) { + const bool isArgLiteral = Token::Match(instantiationArgs[i], "%num%|%str%|%char%|%bool% ,|)"); + if (isArgLiteral && Token::Match(declarationParams[i], "const| %type% &| %name%| ,|)")) { + match++; + + // check if parameter types match + if (templateParams[0]->str() == declarationParams[i]->str()) + argMatch = i; + else { + // todo: check if non-template args match for function overloads + } + } + } + + if (match == declarationParams.size()) { + const Token *arg = instantiationArgs[argMatch]; + tok->insertToken(">"); + switch (arg->tokType()) { + case Token::eBoolean: + tok->insertToken("bool"); + break; + case Token::eChar: + if (arg->isLong()) + tok->insertToken("wchar_t"); + else + tok->insertToken("char"); + break; + case Token::eString: + tok->insertToken("*"); + if (arg->isLong()) + tok->insertToken("wchar_t"); + else + tok->insertToken("char"); + tok->insertToken("const"); + break; + case Token::eNumber: { + MathLib::value num(arg->str()); + if (num.isFloat()) { + // MathLib::getSuffix doesn't work for floating point numbers + const char suffix = arg->str().back(); + if (suffix == 'f' || suffix == 'F') + tok->insertToken("float"); + else if (suffix == 'l' || suffix == 'L') { + tok->insertToken("double"); + tok->next()->isLong(true); + } else + tok->insertToken("double"); + } else if (num.isInt()) { + std::string suffix = MathLib::getSuffix(tok->strAt(3)); + if (suffix.find("LL") != std::string::npos) { + tok->insertToken("long"); + tok->next()->isLong(true); + } else if (suffix.find('L') != std::string::npos) + tok->insertToken("long"); + else + tok->insertToken("int"); + if (suffix.find('U') != std::string::npos) + tok->next()->isUnsigned(true); + } + break; + } + default: + // the arguments matched %num%|%str%|%char%|%bool% + cppcheck::unreachable(); + } + tok->insertToken("<"); + return qualification; + } + } + } + + // the remaining deduction needs the types of the argument expressions + if (instantiationArgs.empty()) + return qualification; + + // a call through a variable (function object, function pointer) is not a template instantiation + if (tok->varId() != 0) + return qualification; + // a call that the symbol database resolved to a real (non template) function must not + // be hijacked - overload resolution against non templates is not implemented here + if (tok->function() && !tok->function()->templateDef) + return qualification; + + // Parse a candidate declaration: all parameters must have a supported form and + // every template parameter must be deducible from the function parameters. The + // signature identifies the parameter list, so a forward declaration and the + // definition of the same template can be recognized. + auto parseDeductionCandidate = [&](const TokenAndName& candidate) -> DeductionCandidate { + if (candidate.isVariadic()) + return DeductionCandidate(); + + DeductionCandidate parsed; + getTemplateParametersInDeclaration(candidate.token()->tokAt(2), parsed.typeParameters); + if (parsed.typeParameters.empty() || !areAllParamsTypes(parsed.typeParameters)) + return DeductionCandidate(); + + std::vector declarationParams; + getFunctionArguments(candidate.nameToken(), declarationParams); + + std::vector deducible(parsed.typeParameters.size(), false); + for (const Token* param : declarationParams) { + const ParameterShape shape = parseParameterShape(param, parsed.typeParameters); + if (!shape.typeTok) + return DeductionCandidate(); + if (shape.templateParameterIndex >= 0) + deducible[shape.templateParameterIndex] = true; + parsed.parameterShapes.push_back(shape); + parsed.signature += (shape.isConst ? " const" : ""); + // template parameters are compared by position - their names may differ + // between a forward declaration and the definition + if (shape.templateParameterIndex >= 0) + parsed.signature += " $" + std::to_string(shape.templateParameterIndex); + else + parsed.signature += " " + shape.typeTok->str(); + parsed.signature += (shape.isPointer ? " *" : shape.isReference ? " &" : ""); + parsed.signature += ","; + } + if (!std::all_of(deducible.cbegin(), deducible.cend(), [](bool b) { + return b; + })) + return DeductionCandidate(); + return parsed; + }; + + // The parse result only depends on the declaration but this function runs for every + // call site, so cache the outcome per declaration: the number of function parameters + // when deduction can handle the declaration, -1 when it can not. + auto supportedParameterCount = [&](const TokenAndName& candidate) -> int { + const auto it = parameterCountCache.find(&candidate); + if (it != parameterCountCache.cend()) + return it->second; + const DeductionCandidate parsed = parseDeductionCandidate(candidate); + const int count = parsed.typeParameters.empty() ? -1 : static_cast(parsed.parameterShapes.size()); + parameterCountCache[&candidate] = count; + return count; + }; + + if (!mUseTypeInformation) { + // Can the deduction succeed later, when type information is available? Scope + // matching is not possible yet - the declaration may be in a base class - so + // consider all declarations with this name. + for (auto pos = range.first; pos != range.second; ++pos) { + if (supportedParameterCount(*pos->second) == static_cast(instantiationArgs.size())) { + mPendingTypeDeductions = true; + return qualification; + } + } + return qualification; + } + + if (!tok->scope()) + return qualification; + + // The scopes where the call name is looked up, in lookup order: the innermost scope + // with matching declarations wins. At class scopes the transitive base classes are + // searched too; when a base class declaration wins, explicit qualification is + // inserted at the call site so the instantiation is matched with that declaration. + struct LookupLevel { + std::string scopeName; // "" is the global scope + const Scope* scope; + bool isBaseClass; + }; + std::vector levels; + if (!qualification.empty()) + levels.push_back(LookupLevel{qualification, nullptr, false}); + else { + // use the symbol database scopes + std::vector visited; + auto addLevel = [&](const Scope* scope, bool isBaseClass) { + if (std::find(visited.cbegin(), visited.cend(), scope) != visited.cend()) + return; + visited.push_back(scope); + if (scope->type == ScopeType::eGlobal) { + levels.push_back(LookupLevel{"", scope, false}); + return; + } + const std::string name = scopeQualifiedName(scope); + if (!name.empty()) + levels.push_back(LookupLevel{name, scope, isBaseClass}); + }; + auto addClassLevels = [&](const Scope* classScope) { + // the class itself, then its transitive base classes + for (const Scope* scope : classScope->findAssociatedScopes()) + addLevel(scope, scope != classScope); + }; + for (const Scope* scope = tok->scope(); scope; scope = scope->nestedIn) { + if (scope->functionOf && scope->functionOf->isClassOrStructOrUnion()) { + // out of class member function body: the class, its base classes and + // its enclosing scopes come before the lexically enclosing scopes + for (const Scope* classScope = scope->functionOf; classScope && classScope->type != ScopeType::eGlobal; classScope = classScope->nestedIn) { + if (classScope->isClassOrStructOrUnion()) + addClassLevels(classScope); + else if (classScope->type == ScopeType::eNamespace) + addLevel(classScope, false); + } + } + if (scope->isClassOrStructOrUnion()) + addClassLevels(scope); + else if (scope->type == ScopeType::eNamespace || scope->type == ScopeType::eGlobal) + addLevel(scope, false); + } + } + + // find the function template declarations this call could instantiate + std::vector candidates; + std::string baseClassQualification; + for (const LookupLevel& level : levels) { + // a non template function with the same name might be a better overload match + if (level.scope && scopeHasNonTemplateFunction(level.scope, tok->str())) + return qualification; + const std::string lookupName = level.scopeName.empty() ? tok->str() : (level.scopeName + " :: " + tok->str()); + for (auto pos = range.first; pos != range.second; ++pos) { + if (pos->second->fullName() == lookupName) + candidates.push_back(pos->second); + } + if (!candidates.empty()) { + if (level.isBaseClass) + baseClassQualification = level.scopeName; + break; + } + } + + // parse the candidate declarations + const TokenAndName* declaration = nullptr; + DeductionCandidate parsedDeclaration; + for (const TokenAndName* candidate : candidates) { + if (supportedParameterCount(*candidate) != static_cast(instantiationArgs.size())) + continue; + DeductionCandidate parsed = parseDeductionCandidate(*candidate); + + if (declaration) { + // multiple overloads could match: overload resolution is not implemented => bail out + if (parsed.signature != parsedDeclaration.signature) + return qualification; + continue; // forward declaration and definition of the same template + } + declaration = candidate; + parsedDeclaration = std::move(parsed); + } + if (!declaration) + return qualification; + + // deduce the template parameters from the ValueTypes of the argument expressions + const std::vector argRoots = getArguments(tok); + if (argRoots.size() != instantiationArgs.size()) + return qualification; + std::vector deducedTypes(parsedDeclaration.typeParameters.size()); + std::vector deducedSet(parsedDeclaration.typeParameters.size(), false); + for (std::size_t i = 0; i < parsedDeclaration.parameterShapes.size(); ++i) { + const ParameterShape& shape = parsedDeclaration.parameterShapes[i]; + if (shape.templateParameterIndex < 0) + continue; + const ValueType* vt = argRoots[i]->valueType(); + if (!vt) { + // type information is not available (yet) for this argument + mPendingTypeDeductions = true; + return qualification; + } + DeducedType deduced = valueTypeToDeducedType(vt, shape); + if (deduced.typeStr.empty()) + return qualification; + if (deducedSet[shape.templateParameterIndex]) { + // inconsistent deduction => don't deduce + if (!(deducedTypes[shape.templateParameterIndex] == deduced)) + return qualification; + } else { + deducedTypes[shape.templateParameterIndex] = std::move(deduced); + deducedSet[shape.templateParameterIndex] = true; + } + } + + // when the declaration is in a base class, qualify the call site explicitly so the + // instantiation is matched with that declaration: "btf ( x )" => "Base :: btf ( x )" + if (!baseClassQualification.empty()) { + Token* const beforeQualification = tok->previous(); + std::string::size_type offset = 0; + std::string::size_type pos = 0; + while ((pos = baseClassQualification.find(' ', offset)) != std::string::npos) { + tok->insertTokenBefore(baseClassQualification.substr(offset, pos - offset)); + offset = pos + 1; + } + tok->insertTokenBefore(baseClassQualification.substr(offset)); + tok->insertTokenBefore("::"); + qualification = baseClassQualification; + rememberNewTokens(beforeQualification ? beforeQualification->next() : mTokenList.front(), tok->previous()); + } + + // insert the deduced template arguments: "f ( x )" => "f < int > ( x )" + tok->insertToken(">"); + Token* const closingBracket = tok->next(); + for (std::size_t i = deducedTypes.size(); i > 0; --i) { + insertDeducedType(tok, deducedTypes[i - 1]); + if (i > 1) + tok->insertToken(","); + } + tok->insertToken("<"); + rememberNewTokens(tok->next(), closingBracket); + rememberDirtyCallSite(tok); + mTypeDeductionsMade = true; + mChanged = true; + return qualification; +} + void TemplateSimplifier::getTemplateInstantiations() { + mPendingTypeDeductions = false; + std::multimap functionNameMap; + // deduceFunctionTemplateArguments() parse results, cached per declaration + std::map deductionParameterCounts; for (const auto & decl : mTemplateDeclarations) { if (decl.isFunction()) @@ -866,111 +1435,8 @@ void TemplateSimplifier::getTemplateInstantiations() } // look for function instantiation with type deduction - if (tok->strAt(1) == "(") { - std::vector instantiationArgs; - getFunctionArguments(tok, instantiationArgs); - - std::string fullName; - if (!qualification.empty()) - fullName = qualification + " :: " + tok->str(); - else if (!scopeName.empty()) - fullName = scopeName + " :: " + tok->str(); - else - fullName = tok->str(); - - // get all declarations with this name - auto range = functionNameMap.equal_range(tok->str()); - for (auto pos = range.first; pos != range.second; ++pos) { - // look for declaration with same qualification or constructor with same qualification - if (pos->second->fullName() == fullName || - (pos->second->scope() == fullName && tok->str() == pos->second->name())) { - std::vector templateParams; - getTemplateParametersInDeclaration(pos->second->token()->tokAt(2), templateParams); - - // todo: handle more than one template parameter - if (templateParams.size() != 1 || !areAllParamsTypes(templateParams)) - continue; - - std::vector declarationParams; - getFunctionArguments(pos->second->nameToken(), declarationParams); - - // function argument counts must match - if (instantiationArgs.empty() || instantiationArgs.size() != declarationParams.size()) - continue; - - size_t match = 0; - size_t argMatch = 0; - for (size_t i = 0; i < declarationParams.size(); ++i) { - // fixme: only type deduction from literals is supported - const bool isArgLiteral = Token::Match(instantiationArgs[i], "%num%|%str%|%char%|%bool% ,|)"); - if (isArgLiteral && Token::Match(declarationParams[i], "const| %type% &| %name%| ,|)")) { - match++; - - // check if parameter types match - if (templateParams[0]->str() == declarationParams[i]->str()) - argMatch = i; - else { - // todo: check if non-template args match for function overloads - } - } - } - - if (match == declarationParams.size()) { - const Token *arg = instantiationArgs[argMatch]; - tok->insertToken(">"); - switch (arg->tokType()) { - case Token::eBoolean: - tok->insertToken("bool"); - break; - case Token::eChar: - if (arg->isLong()) - tok->insertToken("wchar_t"); - else - tok->insertToken("char"); - break; - case Token::eString: - tok->insertToken("*"); - if (arg->isLong()) - tok->insertToken("wchar_t"); - else - tok->insertToken("char"); - tok->insertToken("const"); - break; - case Token::eNumber: { - MathLib::value num(arg->str()); - if (num.isFloat()) { - // MathLib::getSuffix doesn't work for floating point numbers - const char suffix = arg->str().back(); - if (suffix == 'f' || suffix == 'F') - tok->insertToken("float"); - else if (suffix == 'l' || suffix == 'L') { - tok->insertToken("double"); - tok->next()->isLong(true); - } else - tok->insertToken("double"); - } else if (num.isInt()) { - std::string suffix = MathLib::getSuffix(tok->strAt(3)); - if (suffix.find("LL") != std::string::npos) { - tok->insertToken("long"); - tok->next()->isLong(true); - } else if (suffix.find('L') != std::string::npos) - tok->insertToken("long"); - else - tok->insertToken("int"); - if (suffix.find('U') != std::string::npos) - tok->next()->isUnsigned(true); - } - break; - } - default: - break; - } - tok->insertToken("<"); - break; - } - } - } - } + if (tok->strAt(1) == "(") + qualification = deduceFunctionTemplateArguments(tok, qualification, scopeName, functionNameMap, deductionParameterCounts); if (!Token::Match(tok, "%name% <") || Token::Match(tok, "const_cast|dynamic_cast|reinterpret_cast|static_cast")) @@ -1352,6 +1818,10 @@ void TemplateSimplifier::simplifyTemplateAliases() } mChanged = true; + // type alias simplification restructures tokens in a way the incremental + // symbol database update can not handle + if (mUseTypeInformation) + mIncrementalUpdatePossible = false; // copy template-id from declaration to after instantiation Token * dst = aliasUsage.token()->next()->findClosingBracket(); @@ -1654,6 +2124,13 @@ void TemplateSimplifier::expandTemplate( const bool isSpecialization = templateDeclaration.isSpecialization(); const bool isVariable = templateDeclaration.isVariable(); + // the incremental symbol database update only supports expansions that copy a + // function template (see SymbolDatabase::addSymbolsForNewTokenRanges) - everything + // else falls back to recreating the database + if (mUseTypeInformation && (!copy || !isFunction)) + mIncrementalUpdatePossible = false; + Token* const lastTokenBeforeExpansion = mTokenList.back(); + std::vector newInstantiations; for (const Token* tok = templateInstantiation.token()->next()->findClosingBracket(); @@ -1893,6 +2370,11 @@ void TemplateSimplifier::expandTemplate( if (isVariable || isFunction) simplifyTemplateArgs(dstStart, dst); + + // remember the new declaration tokens + Token* const firstNew = dstStart ? dstStart->next() : mTokenList.front(); + if (firstNew != dst) + rememberNewTokens(firstNew, dst->previous()); } if (copy && (isClass || isFunction)) { @@ -2356,6 +2838,10 @@ void TemplateSimplifier::expandTemplate( (inst.token->tokAt(2)->isNumber() || inst.token->tokAt(2)->isStandardType())) mTemplateInstantiations.emplace_back(inst.token, inst.scope); } + + // remember the tokens that were appended at the end of the token list + if (mTokenList.back() != lastTokenBeforeExpansion) + rememberNewTokens(lastTokenBeforeExpansion->next(), mTokenList.back()); } static bool isLowerThanLogicalAnd(const Token *lower) @@ -3298,7 +3784,9 @@ bool TemplateSimplifier::simplifyTemplateInstantiations( const std::string newName(templateDeclaration.name() + " < " + typeForNewName + " >"); std::string newFullName(templateDeclaration.scope() + (templateDeclaration.scope().empty() ? "" : " :: ") + newName); - if (expandedtemplates.insert(std::move(newFullName)).second) { + if (mExpandedTemplateNames.find(newFullName) != mExpandedTemplateNames.cend()) { + // expanded during an earlier simplifyTemplates() call - just replace the usages + } else if (expandedtemplates.insert(std::move(newFullName)).second) { expandTemplate(templateDeclaration, instantiation, typeParametersInDeclaration, newName, !specialized && !isVar); instantiated = true; mChanged = true; @@ -3373,7 +3861,9 @@ bool TemplateSimplifier::simplifyTemplateInstantiations( const std::string newName(templateDeclaration.name() + " < " + typeForNewName + " >"); std::string newFullName(templateDeclaration.scope() + (templateDeclaration.scope().empty() ? "" : " :: ") + newName); - if (expandedtemplates.insert(std::move(newFullName)).second) { + if (mExpandedTemplateNames.find(newFullName) != mExpandedTemplateNames.cend()) { + // expanded during an earlier simplifyTemplates() call - just replace the usages + } else if (expandedtemplates.insert(std::move(newFullName)).second) { expandTemplate(templateDeclaration, templateDeclaration, typeParametersInDeclaration, newName, !specialized && !isVar); instantiated = true; mChanged = true; @@ -3465,6 +3955,7 @@ void TemplateSimplifier::replaceTemplateUsage( const Token * const nameTok1 = nameTok; nameTok->str(newName); + rememberDirtyCallSite(nameTok); // matching template usage => replace tokens.. // Foo < int > => Foo @@ -3486,6 +3977,17 @@ void TemplateSimplifier::replaceTemplateUsage( nameTok = tok2; } + // remembered new token ranges that start in the erased tokens are gone + if (!mNewTokenRanges.empty() && !removeTokens.empty()) { + std::unordered_set erasedTokens; + for (const auto& removeToken : removeTokens) { + for (const Token* tok = removeToken.first->next(); tok && tok != removeToken.second; tok = tok->next()) + erasedTokens.insert(tok); + } + mNewTokenRanges.erase(std::remove_if(mNewTokenRanges.begin(), mNewTokenRanges.end(), [&](const std::pair& range) { + return erasedTokens.count(range.first) != 0; + }), mNewTokenRanges.end()); + } while (!removeTokens.empty()) { eraseTokens(removeTokens.back().first, removeTokens.back().second); removeTokens.pop_back(); @@ -3851,14 +4353,22 @@ void TemplateSimplifier::simplifyTemplates(const std::time_t maxtime) mTokenizer.calculateScopes(); + // all template names expanded during this call; merged into mExpandedTemplateNames + // when this call is done so later calls can reuse the expansions + std::set expandedTemplateNamesThisCall; + unsigned int passCount = 0; constexpr unsigned int passCountMax = 10; for (; passCount < passCountMax; ++passCount) { if (passCount) { // it may take more than one pass to simplify type aliases bool usingChanged = false; - while (mTokenizer.simplifyUsing()) - usingChanged = true; + // don't simplify type aliases again when deducing template arguments with + // type information - the using declarations have already been simplified + if (!mUseTypeInformation) { + while (mTokenizer.simplifyUsing()) + usingChanged = true; + } if (!usingChanged && !mChanged) break; @@ -3877,7 +4387,7 @@ void TemplateSimplifier::simplifyTemplates(const std::time_t maxtime) getTemplateDeclarations(); - if (passCount == 0) { + if (passCount == 0 && !mUseTypeInformation) { mDump.clear(); for (const TokenAndName& t: mTemplateDeclarations) mDump += t.dump(mTokenizer.list.getFiles()); @@ -3889,7 +4399,7 @@ void TemplateSimplifier::simplifyTemplates(const std::time_t maxtime) // Make sure there is something to simplify. if (mTemplateDeclarations.empty() && mTemplateForwardDeclarations.empty()) - return; + break; if (mSettings.debugtemplate && mSettings.debugnormal) { std::string title("Template Simplifier pass " + std::to_string(passCount + 1)); @@ -3995,16 +4505,31 @@ void TemplateSimplifier::simplifyTemplates(const std::time_t maxtime) } } + expandedTemplateNamesThisCall.insert(expandedtemplates.cbegin(), expandedtemplates.cend()); + for (auto it = mInstantiatedTemplates.cbegin(); it != mInstantiatedTemplates.cend(); ++it) { auto decl = std::find_if(mTemplateDeclarations.begin(), mTemplateDeclarations.end(), [&it](const TokenAndName& decl) { return decl.token() == it->token(); }); if (decl != mTemplateDeclarations.end()) { if (it->isSpecialization()) { + // the specialization becomes a normal function - remember it so the + // symbol database can clear its templateDef (incremental update) + if (mUseTypeInformation) + mConvertedSpecializations.push_back(it->nameToken()); // delete the "template < >" Token * tok = it->token(); tok->deleteNext(2); tok->deleteThis(); + } else if (mUseTypeInformation || (mPendingTypeDeductions && it->isFunction())) { + // don't remove the declaration yet: pending type deductions may + // instantiate this function template again, and during the type + // deduction rounds the symbol database has references to these + // tokens. It is removed later by removeDeferredTemplateDeclarations(). + auto it1 = mTemplateForwardDeclarationsMap.find(it->token()); + if (it1 != mTemplateForwardDeclarationsMap.end()) + deferRemoval(it1->second); + deferRemoval(it->token()); } else { // remove forward declaration if found auto it1 = mTemplateForwardDeclarationsMap.find(it->token()); @@ -4023,7 +4548,10 @@ void TemplateSimplifier::simplifyTemplates(const std::time_t maxtime) FindToken(mMemberFunctionsToDelete.cbegin()->token())); // multiple functions can share the same declaration so make sure it hasn't already been deleted if (it != mTemplateDeclarations.end()) { - removeTemplate(it->token()); + if (mUseTypeInformation || (mPendingTypeDeductions && it->isFunction())) + deferRemoval(it->token()); + else + removeTemplate(it->token()); mTemplateDeclarations.erase(it); } else { const auto it1 = std::find_if(mTemplateForwardDeclarations.begin(), @@ -4031,7 +4559,10 @@ void TemplateSimplifier::simplifyTemplates(const std::time_t maxtime) FindToken(mMemberFunctionsToDelete.cbegin()->token())); // multiple functions can share the same declaration so make sure it hasn't already been deleted if (it1 != mTemplateForwardDeclarations.end()) { - removeTemplate(it1->token()); + if (mUseTypeInformation || (mPendingTypeDeductions && it1->isFunction())) + deferRemoval(it1->token()); + else + removeTemplate(it1->token()); mTemplateForwardDeclarations.erase(it1); } } @@ -4054,6 +4585,8 @@ void TemplateSimplifier::simplifyTemplates(const std::time_t maxtime) } } + mExpandedTemplateNames.insert(expandedTemplateNamesThisCall.cbegin(), expandedTemplateNamesThisCall.cend()); + if (passCount == passCountMax) { if (mSettings.debugwarnings) { const std::list locationList(1, mTokenList.front()); @@ -4067,7 +4600,7 @@ void TemplateSimplifier::simplifyTemplates(const std::time_t maxtime) } // Tweak uninstantiated C++17 fold expressions (... && args) - if (mSettings.standards.cpp >= Standards::CPP17) { + if (!mUseTypeInformation && mSettings.standards.cpp >= Standards::CPP17) { bool simplify = false; for (Token *tok = mTokenList.front(); tok; tok = tok->next()) { if (tok->str() == "template") @@ -4110,6 +4643,104 @@ void TemplateSimplifier::simplifyTemplates(const std::time_t maxtime) } } +bool TemplateSimplifier::simplifyTemplatesUsingTypeInformation(std::time_t maxtime) +{ + mUseTypeInformation = true; + mTypeDeductionsMade = false; + mPendingTypeDeductions = false; + mNewTokenRanges.clear(); + mConvertedSpecializations.clear(); + mDirtyCallSites.clear(); + mIncrementalUpdatePossible = true; + // start from a clean state - the state left behind by the previous + // simplifyTemplates() call refers to a token list that has changed since + mTemplateDeclarations.clear(); + mTemplateForwardDeclarations.clear(); + mTemplateForwardDeclarationsMap.clear(); + mTemplateSpecializationMap.clear(); + mTemplatePartialSpecializationMap.clear(); + mTemplateInstantiations.clear(); + mInstantiatedTemplates.clear(); + mExplicitInstantiationsToDelete.clear(); + mTemplateNamePos.clear(); + mChanged = false; + + simplifyTemplates(maxtime); + + mUseTypeInformation = false; + return mTypeDeductionsMade; +} + +void TemplateSimplifier::deferRemoval(Token* declTok) +{ + if (!declTok) + return; + if (std::find(mDeferredRemovals.cbegin(), mDeferredRemovals.cend(), declTok) == mDeferredRemovals.cend()) + mDeferredRemovals.push_back(declTok); +} + +void TemplateSimplifier::rememberNewTokens(Token* first, Token* last) +{ + if (!mUseTypeInformation || !first || !last) + return; + mNewTokenRanges.emplace_back(first, last); +} + +void TemplateSimplifier::rememberDirtyCallSite(Token* nameTok) +{ + if (!mUseTypeInformation || !nameTok) + return; + // duplicates are harmless - the regions around the call sites are merged + mDirtyCallSites.push_back(nameTok); +} + +bool TemplateSimplifier::removeDeferredTemplateDeclarations(SymbolDatabase* symbolDatabase) +{ + if (mDeferredRemovals.empty()) + return false; + + // the declarations could have been removed by a later simplification - make sure + // the tokens still exist before removing them + std::unordered_set liveTokens; + for (const Token* tok = mTokenList.front(); tok; tok = tok->next()) + liveTokens.insert(tok); + std::vector declarations; + std::copy_if(mDeferredRemovals.cbegin(), mDeferredRemovals.cend(), std::back_inserter(declarations), [&](const Token* declTok) { + return liveTokens.count(declTok) != 0; + }); + mDeferredRemovals.clear(); + + // all tokens that are going to be removed + std::unordered_set removedTokens; + for (Token* declTok : declarations) { + const Token* end = findTemplateDeclarationEnd(declTok); + if (!end) + continue; + for (const Token* tok = declTok; tok; tok = tok->next()) { + removedTokens.insert(tok); + if (tok == end) + break; + } + } + + // dirty call sites inside the removed declarations are gone + mDirtyCallSites.erase(std::remove_if(mDirtyCallSites.begin(), mDirtyCallSites.end(), [&](const Token* site) { + return removedTokens.count(site) != 0; + }), mDirtyCallSites.end()); + + if (symbolDatabase) { + // remove the symbols before the tokens are removed + symbolDatabase->removeSymbolsForTokens(removedTokens); + } + + bool removed = false; + for (Token* declTok : declarations) { + if (removeTemplate(declTok)) + removed = true; + } + return removed; +} + void TemplateSimplifier::syntaxError(const Token *tok) { throw InternalError(tok, "syntax error", InternalError::SYNTAX); diff --git a/lib/templatesimplifier.h b/lib/templatesimplifier.h index 3c5fed3ea1c..4457ac9d409 100644 --- a/lib/templatesimplifier.h +++ b/lib/templatesimplifier.h @@ -31,10 +31,12 @@ #include #include #include +#include #include class ErrorLogger; class Settings; +class SymbolDatabase; class Token; class Tokenizer; class TokenList; @@ -317,6 +319,66 @@ class CPPCHECKLIB TemplateSimplifier { */ void simplifyTemplates(std::time_t maxtime); + /** + * Simplify templates again, using type information (AST, SymbolDatabase, ValueType) + * to deduce the template arguments of function template calls such as "f(x)", + * "f(x+y)" or "f(x.g())" from the types of the argument expressions. + * @param maxtime time when the simplification should be stopped + * @return true if the token list was changed + */ + bool simplifyTemplatesUsingTypeInformation(std::time_t maxtime); + + /** + * @return true when there are function template calls where the template arguments + * could not be deduced yet but deduction may succeed once type information is + * available (see simplifyTemplatesUsingTypeInformation). + */ + bool hasPendingTypeDeductions() const { + return mPendingTypeDeductions; + } + + /** + * Remove the instantiated function template declarations whose removal was deferred + * while type deductions were pending. + * @param symbolDatabase when given, its symbols for the removed declarations are + * removed too (incremental update) + * @return true if any tokens were removed + */ + bool removeDeferredTemplateDeclarations(SymbolDatabase* symbolDatabase = nullptr); + + /** token ranges (first and last token) that were added to the token list by the + * last simplifyTemplatesUsingTypeInformation() call */ + const std::vector>& newTokenRanges() const { + return mNewTokenRanges; + } + + /** name tokens of the specializations that the last + * simplifyTemplatesUsingTypeInformation() call turned into normal functions */ + const std::vector& convertedSpecializations() const { + return mConvertedSpecializations; + } + + /** name tokens of the call sites that the last simplifyTemplatesUsingTypeInformation() + * call changed (deduced template arguments and qualifications were inserted, or the + * call was renamed to an instantiation) - the AST and the valuetypes of the + * statements around them must be recreated */ + const std::vector& dirtyCallSites() const { + return mDirtyCallSites; + } + + /** forget the recorded token changes, called when they have been consumed */ + void clearTokenChanges() { + mNewTokenRanges.clear(); + mConvertedSpecializations.clear(); + mDirtyCallSites.clear(); + } + + /** true when the token changes made by the last simplifyTemplatesUsingTypeInformation() + * call can be applied to the symbol database incrementally */ + bool incrementalUpdatePossible() const { + return mIncrementalUpdatePossible; + } + /** * Simplify constant calculations such as "1+2" => "3" * @param tok start token @@ -353,6 +415,43 @@ class CPPCHECKLIB TemplateSimplifier { */ void addInstantiation(Token *token, const std::string &scope); + /** + * Deduce the template arguments of a function template call from the function + * arguments and insert them after the name token: "f ( 1 )" => "f < int > ( 1 )". + * Literal arguments are always handled; arbitrary argument expressions are handled + * when type information is available (mUseTypeInformation). Sets + * mPendingTypeDeductions when a deduction could succeed later with type information. + * @param tok name token of the function call + * @param qualification qualification of the call ("A :: B" for "A :: B :: f ( 1 )") + * @param scopeName name of the scope the call is in + * @param functionNameMap map with all function template declarations + * @param parameterCountCache cache of the per declaration parse results, shared by + * the call sites of one pass + * @return the qualification of the call: the given qualification, or the scope of + * the deduced declaration when it is in a base class and explicit qualification + * was inserted at the call site + */ + std::string deduceFunctionTemplateArguments(Token* tok, + std::string qualification, + const std::string& scopeName, + const std::multimap& functionNameMap, + std::map& parameterCountCache); + + /** + * Remember that the given instantiated function template declaration should not be + * removed from the token list yet because pending type deductions may instantiate + * it again. It is removed later by removeDeferredTemplateDeclarations(). + */ + void deferRemoval(Token* declTok); + + /** remember tokens that were added during a type-information round so the symbol + * database can be updated incrementally */ + void rememberNewTokens(Token* first, Token* last); + + /** remember a call site whose statement was changed during a type-information round + * so its AST and valuetypes can be recreated */ + void rememberDirtyCallSite(Token* nameTok); + /** * Get template instantiations */ @@ -509,6 +608,28 @@ class CPPCHECKLIB TemplateSimplifier { const Settings &mSettings; ErrorLogger &mErrorLogger; bool mChanged{}; + /** true when type information (AST, SymbolDatabase, ValueType) is available for deduction */ + bool mUseTypeInformation = false; + /** true when there are calls where deduction may succeed once type information is available */ + bool mPendingTypeDeductions = false; + /** true when the current simplifyTemplatesUsingTypeInformation() call deduced something */ + bool mTypeDeductionsMade = false; + /** names of all expanded instantiations; persists between simplifyTemplates() calls so a + * later deduced instantiation reuses the existing expansion instead of duplicating it */ + std::set mExpandedTemplateNames; + /** instantiated function template declarations whose removal has been deferred */ + std::vector mDeferredRemovals; + /** token ranges added during type-information rounds (see newTokenRanges()) */ + std::vector> mNewTokenRanges; + /** specializations that were turned into normal functions during type-information + * rounds - their Function::templateDef must be cleared */ + std::vector mConvertedSpecializations; + /** call sites changed during type-information rounds (see dirtyCallSites()) */ + std::vector mDirtyCallSites; + /** false when the current round made token changes that the incremental symbol + * database update can not handle (class/variable/specialization instantiations, + * type alias changes) */ + bool mIncrementalUpdatePossible = true; std::list mTemplateDeclarations; std::list mTemplateForwardDeclarations; diff --git a/lib/token.h b/lib/token.h index 3328af31fa0..4cb0d3562eb 100644 --- a/lib/token.h +++ b/lib/token.h @@ -1652,6 +1652,14 @@ class CPPCHECKLIB Token { mImpl->mValues = nullptr; } + /** Clear the AST links (operands, parent and the cached top token) so the AST can be created again */ + void clearAst() { + mImpl->mAstOperand1 = nullptr; + mImpl->mAstOperand2 = nullptr; + mImpl->mAstParent = nullptr; + mImpl->mAstTop = nullptr; + } + // cppcheck-suppress unusedFunction - used in tests only std::string astString(const char *sep = "") const { std::string ret; diff --git a/lib/tokenize.cpp b/lib/tokenize.cpp index d2ad6f581e7..04c50cb258b 100644 --- a/lib/tokenize.cpp +++ b/lib/tokenize.cpp @@ -3546,6 +3546,18 @@ bool Tokenizer::simplifyTokens1(const std::string &configuration, int fileIndex) mSymbolDatabase->setValueTypeInTokenList(true); }); + if (isCPP() && mTemplateSimplifier->hasPendingTypeDeductions()) { + Timer::run("Tokenizer::simplifyTokens1::simplifyTemplatesUsingTypeInformation", mTimerResults, [&]() { + simplifyTemplatesUsingTypeInformation(); + }); + } + + // the phases that only later analysis needs run when the template simplifier can + // no longer change the token list + Timer::run("Tokenizer::simplifyTokens1::finalizeSymbolDatabase", mTimerResults, [&]() { + mSymbolDatabase->finalize(); + }); + if (!mSettings.buildDir.empty()) Summaries::create(*this, configuration, fileIndex); @@ -4264,6 +4276,171 @@ void Tokenizer::simplifyTemplates() mTemplateSimplifier->simplifyTemplates( maxTime); } + +void Tokenizer::simplifyTemplatesUsingTypeInformation() +{ + if (isC()) + return; + + const std::time_t maxTime = mSettings.templateMaxTime > 0 ? std::time(nullptr) + mSettings.templateMaxTime : 0; + + // for testing and debugging: recreate the symbol database etc. from scratch after + // each deduction round instead of updating them incrementally + const bool allowIncremental = !mSettings.templateFullRebuild; + + bool finalized = false; + + auto updateOrRebuild = [&](bool incremental) { + if (incremental) + updateTokenDataAfterTemplateSimplification(); + else { + mTemplateSimplifier->clearTokenChanges(); + rebuildTokenDataAfterTemplateSimplification(); + } + }; + + constexpr int maxRounds = 5; + for (int round = 0; round < maxRounds; ++round) { + if (Settings::terminated()) + return; + if (!mTemplateSimplifier->simplifyTemplatesUsingTypeInformation(maxTime)) + break; + const bool incremental = allowIncremental && mTemplateSimplifier->incrementalUpdatePossible(); + // when no deductions are pending anymore the deferred template declarations can + // be removed now, so that the updated token information is already final + if (!mTemplateSimplifier->hasPendingTypeDeductions()) { + mTemplateSimplifier->removeDeferredTemplateDeclarations(incremental ? mSymbolDatabase : nullptr); + finalized = true; + } + updateOrRebuild(incremental); + if (finalized) + break; + } + if (!finalized) { + const bool incremental = allowIncremental && mTemplateSimplifier->incrementalUpdatePossible(); + if (mTemplateSimplifier->removeDeferredTemplateDeclarations(incremental ? mSymbolDatabase : nullptr)) + updateOrRebuild(incremental); + } +} + +void Tokenizer::rebuildTokenDataAfterTemplateSimplification() +{ + // recreate the supporting token information for the added and removed tokens, in + // the same order the information was created initially + createLinks(); + setVarId(); + createLinks2(); + Token::assignProgressValues(list.front()); + list.front()->assignIndexes(); + list.clearAst(); + list.createAst(); + list.validateAst(mSettings.debugnormal); + delete mSymbolDatabase; + mSymbolDatabase = nullptr; + // the phases that only later analysis needs are run by finalize() after the + // template simplification is done + createSymbolDatabase(); + mSymbolDatabase->setValueTypeInTokenList(false); + mSymbolDatabase->setValueTypeInTokenList(true); +} + +void Tokenizer::updateTokenDataAfterTemplateSimplification() +{ + // update the supporting token information incrementally: the information of the + // unchanged tokens - variable ids, symbols, AST, valuetypes - stays valid + setVarId(/*incremental*/ true); + createLinks2(); // adds the "<>" links of the new tokens + Token::assignProgressValues(list.front()); + list.front()->assignIndexes(); + + // The token regions whose AST and valuetypes must be recreated: the new tokens and + // the function bodies around the changed call sites. + bool rebuildAll = false; + // the regions (first and last token, inclusive) whose AST and valuetypes must be recreated + std::vector> regions = mTemplateSimplifier->newTokenRanges(); + for (const Token* siteTok : mTemplateSimplifier->dirtyCallSites()) { + const Scope* scope = siteTok->scope(); + if (!scope) + continue; // a new token - covered by the new token ranges + // the outermost executable scope is the function body around the call + const Scope* functionScope = nullptr; + for (const Scope* s = scope; s; s = s->nestedIn) { + if (s->isExecutable()) + functionScope = s; + } + if (!functionScope || !functionScope->bodyStart || !functionScope->bodyEnd || + functionScope->bodyStart->astParent()) { + // no function body around the call (global initializer, lambda body that is + // part of an expression, ..) - recreate everything + rebuildAll = true; + break; + } + regions.emplace_back(const_cast(functionScope->bodyStart), const_cast(functionScope->bodyEnd)); + } + + if (!rebuildAll && !regions.empty()) { + // merge the overlapping regions - Token::index() is up to date + std::sort(regions.begin(), regions.end(), [](const std::pair& lhs, const std::pair& rhs) { + return lhs.first->index() < rhs.first->index(); + }); + std::vector> merged; + for (const auto& region : regions) { + if (!merged.empty() && region.first->index() <= merged.back().second->index()) { + if (region.second->index() > merged.back().second->index()) + merged.back().second = region.second; + } else + merged.push_back(region); + } + regions = std::move(merged); + } + + if (rebuildAll) { + list.clearAst(); + list.createAst(); + } else { + for (const auto& region : regions) { + for (Token* tok = region.first; tok; tok = tok->next()) { + tok->clearAst(); + if (tok == region.second) + break; + } + list.createAst(region.first, region.second->next()); + } + } + list.validateAst(mSettings.debugnormal); + + for (const Token* nameTok : mTemplateSimplifier->convertedSpecializations()) { + // the specialization became a normal function: its "template < >" tokens are gone + if (nameTok->function()) + const_cast(nameTok->function())->templateDef = nullptr; + } + mSymbolDatabase->addSymbolsForNewTokenRanges(mTemplateSimplifier->newTokenRanges()); + // safety net: every token needs a scope. A token that was added by a simplification + // that is not covered by the new token ranges gets the scope of the preceding token. + const Scope* lastScope = nullptr; + for (Token* tok = list.front(); tok; tok = tok->next()) { + if (tok->scope()) { + lastScope = tok->scope(); + if (tok == lastScope->bodyEnd && lastScope->nestedIn) + lastScope = lastScope->nestedIn; + } else if (lastScope) + tok->scope(lastScope); + } + + // recreate the valuetypes of the changed regions + if (rebuildAll) { + mSymbolDatabase->setValueTypeInTokenList(false); + mSymbolDatabase->setValueTypeInTokenList(true); + } else { + mSymbolDatabase->updateFunctionAndVariablePointers(); + for (const auto& region : regions) { + mSymbolDatabase->setValueTypeInTokenList(false, region.first, region.second->next()); + mSymbolDatabase->setValueTypeInTokenList(true, region.first, region.second->next()); + } + } + mTemplateSimplifier->clearTokenChanges(); + mSymbolDatabase->validate(); +} //--------------------------------------------------------------------------- @@ -4282,7 +4459,7 @@ namespace { VariableMap() = default; void enterScope(); bool leaveScope(); - void addVariable(const std::string& varname, bool globalNamespace); + void addVariable(const std::string& varname, bool globalNamespace, nonneg int existingVarId = 0); bool hasVariable(const std::string& varname) const { return mVariableId.find(varname) != mVariableId.end(); } @@ -4320,10 +4497,13 @@ bool VariableMap::leaveScope() return true; } -void VariableMap::addVariable(const std::string& varname, bool globalNamespace) +void VariableMap::addVariable(const std::string& varname, bool globalNamespace, nonneg int existingVarId) { + // a variable that already has an id keeps it, so ids are stable when the pass is + // run again after the template simplifier added tokens (incremental) + const nonneg int id = (existingVarId != 0) ? existingVarId : ++mVarId; if (mScopeInfo.empty()) { - mVariableId[varname].id = ++mVarId; + mVariableId[varname].id = id; if (globalNamespace) mVariableId_global[varname] = mVariableId[varname]; return; @@ -4331,13 +4511,13 @@ void VariableMap::addVariable(const std::string& varname, bool globalNamespace) const auto it = mVariableId.find(varname); if (it == mVariableId.end()) { mScopeInfo.top().emplace_back(varname, VarInfo{}); - mVariableId[varname].id = ++mVarId; + mVariableId[varname].id = id; if (globalNamespace) mVariableId_global[varname] = mVariableId[varname]; return; } mScopeInfo.top().emplace_back(varname, it->second); - it->second.id = ++mVarId; + it->second.id = id; it->second.assigned = false; } @@ -4509,8 +4689,12 @@ static void setVarIdStructMembers(Token *&tok1, tok = tok->next(); const auto it = utils::as_const(members).find(tok->str()); if (it == members.cend()) { - members[tok->str()] = ++varId; - tok->varId(varId); + if (tok->varId() != 0) + members[tok->str()] = tok->varId(); // keep the id (incremental pass) + else { + members[tok->str()] = ++varId; + tok->varId(varId); + } } else { tok->varId(it->second); } @@ -4547,8 +4731,12 @@ static void setVarIdStructMembers(Token *&tok1, std::map& members = structMembers[struct_varid]; const auto it = utils::as_const(members).find(tok->str()); if (it == members.cend()) { - members[tok->str()] = ++varId; - tok->varId(varId); + if (tok->varId() != 0) + members[tok->str()] = tok->varId(); // keep the id (incremental pass) + else { + members[tok->str()] = ++varId; + tok->varId(varId); + } } else { tok->varId(it->second); } @@ -4689,15 +4877,20 @@ void Tokenizer::setVarIdClassFunction(const std::string &classname, -void Tokenizer::setVarId() +void Tokenizer::setVarId(bool incremental) { - // Clear all variable ids - for (Token *tok = list.front(); tok; tok = tok->next()) { - if (tok->isName()) - tok->varId(0); + if (!incremental) { + // Clear all variable ids + for (Token *tok = list.front(); tok; tok = tok->next()) { + if (tok->isName()) + tok->varId(0); + } } + // incremental: tokens keep their variable ids and only tokens without an id get + // one, continuing above the previous maximum. This keeps the ids stable so the + // symbol database can be updated instead of recreated. - setVarIdPass1(); + setVarIdPass1(incremental); setPodTypes(); @@ -4712,13 +4905,15 @@ static const std::unordered_set notstart_cpp = { NOTSTART_C, "delete", "friend", "new", "throw", "using", "virtual", "explicit", "const_cast", "dynamic_cast", "reinterpret_cast", "static_cast", "template" }; -void Tokenizer::setVarIdPass1() +void Tokenizer::setVarIdPass1(bool incremental) { const bool cpp = isCPP(); // Variable declarations can't start with "return" etc. const std::unordered_set& notstart = (isC()) ? notstart_c : notstart_cpp; VariableMap variableMap; + if (incremental) + variableMap.getVarId() = mVarId; // new ids continue above the previous maximum std::map> structMembers; std::stack scopeStack; @@ -4895,7 +5090,7 @@ void Tokenizer::setVarIdPass1() Token::simpleMatch(tok2->link(), "] =")) { while (tok2 && tok2->str() != "]") { if (Token::Match(tok2, "%name% [,]]")) - variableMap.addVariable(tok2->str(), false); + variableMap.addVariable(tok2->str(), false, tok2->varId()); tok2 = tok2->next(); } continue; @@ -4995,14 +5190,14 @@ void Tokenizer::setVarIdPass1() if (decl) { if (isC() && Token::Match(prev2->previous(), "&|&&")) syntaxErrorC(prev2, prev2->strAt(-2) + prev2->strAt(-1) + " " + prev2->str()); - variableMap.addVariable(prev2->str(), scopeStack.size() <= 1); + variableMap.addVariable(prev2->str(), scopeStack.size() <= 1, prev2->varId()); if (Token::simpleMatch(tok->previous(), "for (") && Token::Match(prev2, "%name% [=[({,]")) { for (const Token *tok3 = prev2->next(); tok3 && tok3->str() != ";"; tok3 = tok3->next()) { if (Token::Match(tok3, "[([]")) tok3 = tok3->link(); if (Token::Match(tok3, ", %name% [=[({,;]")) - variableMap.addVariable(tok3->strAt(1), false); + variableMap.addVariable(tok3->strAt(1), false, tok3->next()->varId()); } } diff --git a/lib/tokenize.h b/lib/tokenize.h index f7f6ddbecae..68e7220dba8 100644 --- a/lib/tokenize.h +++ b/lib/tokenize.h @@ -83,8 +83,14 @@ class CPPCHECKLIB Tokenizer { private: /** Set variable id */ - void setVarId(); - void setVarIdPass1(); + /** + * Set variable ids. + * @param incremental don't clear the existing variable ids: tokens keep their ids + * and tokens without an id get a new id above the previous maximum, so the ids are + * stable. Used when the template simplifier has added tokens. + */ + void setVarId(bool incremental = false); + void setVarIdPass1(bool incremental = false); void setVarIdPass2(); /** @@ -297,6 +303,28 @@ class CPPCHECKLIB Tokenizer { */ void simplifyTemplates(); + /** + * Simplify templates again, this time using type information (AST, SymbolDatabase, + * ValueType) to deduce the template arguments of function template calls from the + * argument expressions. Runs in a loop: after each round of new instantiations the + * supporting token information (varid, links, AST, SymbolDatabase, ValueTypes) is + * updated so the new tokens can be used by the next round. + */ + void simplifyTemplatesUsingTypeInformation(); + + /** + * Recreate varid, links, AST, SymbolDatabase and ValueTypes from scratch after the + * template simplifier changed the token list. + */ + void rebuildTokenDataAfterTemplateSimplification(); + + /** + * Incrementally update varid, links, AST, SymbolDatabase and ValueTypes after the + * template simplifier changed the token list: the information of the unchanged + * tokens stays valid. + */ + void updateTokenDataAfterTemplateSimplification(); + void simplifyDoublePlusAndDoubleMinus(); void simplifyRedundantConsecutiveBraces(); diff --git a/lib/tokenlist.cpp b/lib/tokenlist.cpp index 5921621fed2..07bc16c060b 100644 --- a/lib/tokenlist.cpp +++ b/lib/tokenlist.cpp @@ -1896,15 +1896,26 @@ static Token * createAstAtToken(Token *tok) return tok; } +void TokenList::clearAst() const +{ + for (Token *tok = mTokensFrontBack->front; tok; tok = tok->next()) + tok->clearAst(); +} + void TokenList::createAst() const { - for (Token *tok = mTokensFrontBack->front; tok; tok = tok ? tok->next() : nullptr) { + createAst(mTokensFrontBack->front, nullptr); +} + +void TokenList::createAst(Token* startToken, const Token* endToken) +{ + for (Token *tok = startToken; tok && tok != endToken; tok = tok ? tok->next() : nullptr) { Token* const nextTok = createAstAtToken(tok); if (precedes(nextTok, tok)) throw InternalError(tok, "Syntax Error: Infinite loop when creating AST.", InternalError::AST); tok = nextTok; } - for (Token *tok = mTokensFrontBack->front; tok; tok = tok ? tok->next() : nullptr) { + for (Token *tok = startToken; tok && tok != endToken; tok = tok ? tok->next() : nullptr) { if (tok->astParent()) continue; if (!tok->astOperand1() && !tok->astOperand2()) diff --git a/lib/tokenlist.h b/lib/tokenlist.h index f5b652f3506..318f07fb568 100644 --- a/lib/tokenlist.h +++ b/lib/tokenlist.h @@ -172,6 +172,19 @@ class CPPCHECKLIB TokenList { */ void createAst() const; + /** + * Create the abstract syntax tree for the tokens in the given range. endToken is + * exclusive, nullptr means the end of the token list. + * @throws InternalError thrown if encountering an infinite loop in AST creation + */ + static void createAst(Token* startToken, const Token* endToken); + + /** + * Remove the abstract syntax tree from all tokens so createAst() can be called again, + * for instance after the token list has been modified. + */ + void clearAst() const; + /** * Check abstract syntax tree. * @throws InternalError thrown if validation fails diff --git a/oss-fuzz/Makefile b/oss-fuzz/Makefile index eeb795bd35c..a20ac6f08ec 100644 --- a/oss-fuzz/Makefile +++ b/oss-fuzz/Makefile @@ -339,7 +339,7 @@ $(libcppdir)/summaries.o: ../lib/summaries.cpp ../lib/analyzerinfo.h ../lib/chec $(libcppdir)/suppressions.o: ../lib/suppressions.cpp ../externals/tinyxml2/tinyxml2.h ../lib/addoninfo.h ../lib/checkers.h ../lib/config.h ../lib/errorlogger.h ../lib/errortypes.h ../lib/filesettings.h ../lib/library.h ../lib/mathlib.h ../lib/path.h ../lib/pathmatch.h ../lib/platform.h ../lib/settings.h ../lib/smallvector.h ../lib/standards.h ../lib/suppressions.h ../lib/templatesimplifier.h ../lib/token.h ../lib/tokenlist.h ../lib/utils.h ../lib/vfvalue.h ../lib/xml.h $(CXX) ${LIB_FUZZING_ENGINE} $(CPPFLAGS) $(CXXFLAGS) -c -o $@ $(libcppdir)/suppressions.cpp -$(libcppdir)/templatesimplifier.o: ../lib/templatesimplifier.cpp ../lib/checkers.h ../lib/config.h ../lib/errorlogger.h ../lib/errortypes.h ../lib/library.h ../lib/mathlib.h ../lib/platform.h ../lib/settings.h ../lib/smallvector.h ../lib/standards.h ../lib/templatesimplifier.h ../lib/token.h ../lib/tokenize.h ../lib/tokenlist.h ../lib/utils.h ../lib/vfvalue.h +$(libcppdir)/templatesimplifier.o: ../lib/templatesimplifier.cpp ../lib/astutils.h ../lib/checkers.h ../lib/config.h ../lib/errorlogger.h ../lib/errortypes.h ../lib/library.h ../lib/mathlib.h ../lib/platform.h ../lib/settings.h ../lib/smallvector.h ../lib/sourcelocation.h ../lib/standards.h ../lib/symboldatabase.h ../lib/templatesimplifier.h ../lib/token.h ../lib/tokenize.h ../lib/tokenlist.h ../lib/utils.h ../lib/vfvalue.h $(CXX) ${LIB_FUZZING_ENGINE} $(CPPFLAGS) $(CXXFLAGS) -c -o $@ $(libcppdir)/templatesimplifier.cpp $(libcppdir)/timer.o: ../lib/timer.cpp ../lib/config.h ../lib/timer.h diff --git a/test/cli/other_test.py b/test/cli/other_test.py index c79fd0c8e50..e13fe5d9703 100644 --- a/test/cli/other_test.py +++ b/test/cli/other_test.py @@ -1045,7 +1045,7 @@ def test_showtime_top5_summary_compdb(tmp_path): def __test_showtime_file(tmp_path, use_compdb=False, use_addons=False, use_clang_tidy=False): - exp_res = 79 + exp_res = 80 # project analysis does not call Preprocessor::getConfig() if use_compdb: exp_res -= 1 @@ -1079,7 +1079,7 @@ def test_showtime_file_clang_tidy_compdb(tmp_path): def __test_showtime_summary(tmp_path, use_compdb=False, use_addons=False, use_clang_tidy=False): - exp_res = 79 + exp_res = 80 # project analysis does not call Preprocessor::getConfig() if use_compdb: exp_res -= 1 diff --git a/test/testcmdlineparser.cpp b/test/testcmdlineparser.cpp index 326b7ad613d..94c60cf69e8 100644 --- a/test/testcmdlineparser.cpp +++ b/test/testcmdlineparser.cpp @@ -383,6 +383,7 @@ class TestCmdlineParser : public TestFixture { TEST_CASE(performanceValueflowMaxTimeInvalid); TEST_CASE(performanceValueFlowMaxIfCount); TEST_CASE(performanceValueFlowMaxIfCountInvalid); + TEST_CASE(templateFullRebuild); TEST_CASE(templateMaxTime); TEST_CASE(templateMaxTimeInvalid); TEST_CASE(templateMaxTimeInvalid2); @@ -2535,6 +2536,13 @@ class TestCmdlineParser : public TestFixture { ASSERT_EQUALS("cppcheck: error: argument to '--performance-valueflow-max-if-count=' is not valid - not an integer (invalid_argument).\n", logger->str()); } + void templateFullRebuild() { + REDIRECT; + const char * const argv[] = {"cppcheck", "--template-full-rebuild", "file.cpp"}; + ASSERT_EQUALS_ENUM(CmdLineParser::Result::Success, parseFromArgs(argv)); + ASSERT_EQUALS(true, settings->templateFullRebuild); + } + void templateMaxTime() { REDIRECT; const char * const argv[] = {"cppcheck", "--template-max-time=12", "file.cpp"}; diff --git a/test/testsimplifytemplate.cpp b/test/testsimplifytemplate.cpp index 6960b791426..e9426c7063a 100644 --- a/test/testsimplifytemplate.cpp +++ b/test/testsimplifytemplate.cpp @@ -42,6 +42,13 @@ class TestSimplifyTemplate : public TestFixture { const Settings settings = settingsBuilder().severity(Severity::portability).build(); const Settings settings1 = settingsBuilder(settings).library("std.cfg").build(); const Settings settings1_d = settingsBuilder(settings1).debugwarnings().build(); + const Settings settings1_fr = makeFullRebuildSettings(settings1); + + static Settings makeFullRebuildSettings(const Settings& base) { + Settings s = base; + s.templateFullRebuild = true; + return s; + } void run() override { TEST_CASE(template1); @@ -284,6 +291,17 @@ class TestSimplifyTemplate : public TestFixture { TEST_CASE(templateTypeDeduction3); TEST_CASE(templateTypeDeduction4); // #9983 TEST_CASE(templateTypeDeduction5); + TEST_CASE(templateTypeDeduction6); // deduction from variables + TEST_CASE(templateTypeDeduction7); // parameter forms: T, T&, const T&, T* + TEST_CASE(templateTypeDeduction8); // deduction from expressions + TEST_CASE(templateTypeDeduction9); // multiple parameters, scopes, bailouts + TEST_CASE(templateTypeDeduction10); // parameter visibility: init list, const method + TEST_CASE(templateTypeDeduction11); // unqualified lookup: enclosing scopes, shadowing + TEST_CASE(templateTypeDeduction12); // base class member templates, out of class method bodies + TEST_CASE(templateTypeDeduction13); // members declared after the member function are visible + TEST_CASE(templateTypeDeduction14); // a non template overload might be a better match + TEST_CASE(templateTypeDeduction15); // final classes + TEST_CASE(templateTypeDeductionFullRebuild); // --template-full-rebuild gives the same result TEST_CASE(simplifyTemplateArgs1); TEST_CASE(simplifyTemplateArgs2); @@ -342,12 +360,19 @@ class TestSimplifyTemplate : public TestFixture { struct CheckOptions { bool debugwarnings = false; + bool templateFullRebuild = false; }; + const Settings& checkOptionsSettings(const CheckOptions& options) const { + if (options.templateFullRebuild) + return settings1_fr; + return options.debugwarnings ? settings1_d : settings1; + } + #define tok(...) tok_(__FILE__, __LINE__, __VA_ARGS__) template std::string tok_(const char* file, int line, const char (&code)[size], const CheckOptions& options = make_default_obj()) { - const Settings& s = options.debugwarnings ? settings1_d : settings1; + const Settings& s = checkOptionsSettings(options); SimpleTokenizer tokenizer(s, *this); ASSERT_LOC(tokenizer.tokenize(code), file, line); @@ -4715,16 +4740,16 @@ class TestSimplifyTemplate : public TestFixture { " g(1.0f);\n" "}\n"; const char exp[] = "float g ( float x ) ; " - "template < typename T > " - "T g ( T x ) { return x ; } " + "int g ( int x ) ; " "float g ( float x ) {" " return x + 1.0f ; " "} " "void f ( int i ) {" - " g ( i ) ;" + " g ( i ) ;" " g ( 1.0f ) ; " - "}"; - ASSERT_EQUALS(exp, tok(code)); // TODO: instantiate g(int) + "} " + "int g ( int x ) { return x ; }"; + ASSERT_EQUALS(exp, tok(code)); } void template_specialization_1() { // #7868 - template specialization template struct S> {..}; @@ -6126,18 +6151,12 @@ class TestSimplifyTemplate : public TestFixture { "void f ( double t , int u ) ; " "static void func ( ) { " "f ( 0 , 0.0 ) ; " - "f ( 0.0, 0 ) ; " + "f ( 0.0 , 0 ) ; " + "} " "void f ( int t , double u ) { } " - "void f ( double t , int u ) { } "; - - const char actual[] = "template < typename T , typename U > " - "void f ( T t , U u ) { } " - "static void func ( ) { " - "f ( 0 , 0.0 ) ; " - "f ( 0.0 , 0 ) ; " - "}"; + "void f ( double t , int u ) { }"; - TODO_ASSERT_EQUALS(expected, actual, tok(code)); + ASSERT_EQUALS(expected, tok(code)); } void templateTypeDeduction3() { // #9975 @@ -6241,10 +6260,7 @@ class TestSimplifyTemplate : public TestFixture { "void f ( int x , double y ) ; " "void test ( ) { f ( 0 , 0.0 ) ; } " "void f ( int x , double y ) { a = x + y ; }"; - const char act[] = "int a ; a = 1 ; " - "template < typename T , typename U > void f ( T x , U y ) { a = x + y ; } " - "void test ( ) { f ( 0 , 0.0 ) ; }"; - TODO_ASSERT_EQUALS(exp, act, tok(code)); + ASSERT_EQUALS(exp, tok(code)); } } @@ -6315,6 +6331,635 @@ class TestSimplifyTemplate : public TestFixture { } } + void templateTypeDeduction6() + { // type deduction from variables + const char code[] = "template void f(T n) { (void)n; }\n" + "struct MyClass { int m; };\n" + "void func() {\n" + " unsigned long long ull = 0;\n" + " const char* p = \"abc\";\n" + " double d = 1.0;\n" + " int arr[3];\n" + " MyClass mc;\n" + " f(ull);\n" + " f(p);\n" + " f(d);\n" + " f(arr);\n" + " f(mc);\n" + "}"; + const char exp[] = "void f ( unsigned long long n ) ; " + "void f ( const char * n ) ; " + "void f ( double n ) ; " + "void f ( int * n ) ; " + "void f ( MyClass n ) ; " + "struct MyClass { int m ; } ; " + "void func ( ) { " + "unsigned long long ull ; ull = 0 ; " + "const char * p ; p = \"abc\" ; " + "double d ; d = 1.0 ; " + "int arr [ 3 ] ; " + "MyClass mc ; " + "f ( ull ) ; " + "f ( p ) ; " + "f ( d ) ; " + "f ( arr ) ; " + "f ( mc ) ; " + "} " + "void f ( unsigned long long n ) { ( void ) n ; } " + "void f ( const char * n ) { ( void ) n ; } " + "void f ( double n ) { ( void ) n ; } " + "void f ( int * n ) { ( void ) n ; } " + "void f ( MyClass n ) { ( void ) n ; }"; + ASSERT_EQUALS(exp, tok(code)); + ASSERT_EQUALS("", errout_str()); + } + + void templateTypeDeduction7() + { // parameter forms: T, T&, const T&, T* + const char code[] = "template void byval(T x) { (void)x; }\n" + "template void byref(T& x) { (void)x; }\n" + "template void bycref(const T& x) { (void)x; }\n" + "template void byptr(T* x) { (void)x; }\n" + "void func() {\n" + " const int ci = 1;\n" + " int i = 2;\n" + " int& r = i;\n" + " const char* s = \"x\";\n" + " byval(ci);\n" + " byref(ci);\n" + " bycref(ci);\n" + " byptr(s);\n" + " byval(r);\n" + "}"; + const char exp[] = "void byval ( int x ) ; " + "void byref ( const int & x ) ; " + "void bycref ( const int & x ) ; " + "void byptr ( const char * x ) ; " + "void func ( ) { " + "const int ci = 1 ; " + "int i ; i = 2 ; " + "int & r = i ; " + "const char * s ; s = \"x\" ; " + "byval ( ci ) ; " + "byref ( ci ) ; " + "bycref ( ci ) ; " + "byptr ( s ) ; " + "byval ( r ) ; " + "} " + "void byptr ( const char * x ) { ( void ) x ; } " + "void bycref ( const int & x ) { ( void ) x ; } " + "void byref ( const int & x ) { ( void ) x ; } " + "void byval ( int x ) { ( void ) x ; }"; + ASSERT_EQUALS(exp, tok(code)); + ASSERT_EQUALS("", errout_str()); + } + + void templateTypeDeduction8() + { // type deduction from expressions + const char code[] = "template void f(T n) { (void)n; }\n" + "int geti();\n" + "void func() {\n" + " short s = 2;\n" + " double d = 1.0;\n" + " const char* p = \"abc\";\n" + " f(s + 1);\n" + " f(&d);\n" + " f(*p);\n" + " f((unsigned char)s);\n" + " f(static_cast(d));\n" + " f(geti());\n" + " f(s == 1);\n" + "}"; + const char exp[] = "void f ( int n ) ; " + "void f ( double * n ) ; " + "void f ( char n ) ; " + "void f ( unsigned char n ) ; " + "void f ( float n ) ; " + "void f ( bool n ) ; " + "int geti ( ) ; " + "void func ( ) { " + "short s ; s = 2 ; " + "double d ; d = 1.0 ; " + "const char * p ; p = \"abc\" ; " + "f ( s + 1 ) ; " + "f ( & d ) ; " + "f ( * p ) ; " + "f ( ( unsigned char ) s ) ; " + "f ( static_cast < float > ( d ) ) ; " + "f ( geti ( ) ) ; " + "f ( s == 1 ) ; " + "} " + "void f ( int n ) { ( void ) n ; } " + "void f ( double * n ) { ( void ) n ; } " + "void f ( char n ) { ( void ) n ; } " + "void f ( unsigned char n ) { ( void ) n ; } " + "void f ( float n ) { ( void ) n ; } " + "void f ( bool n ) { ( void ) n ; }"; + ASSERT_EQUALS(exp, tok(code)); + ASSERT_EQUALS("", errout_str()); + } + + void templateTypeDeduction9() + { // multiple parameters, scopes, bailouts + { + // multiple template parameters, deduction from variables + const char code[] = "template void f(T t, U u) { }\n" + "void func() {\n" + " int i = 1;\n" + " double d = 2.0;\n" + " f(i, d);\n" + "}"; + const char exp[] = "void f ( int t , double u ) ; " + "void func ( ) { " + "int i ; i = 1 ; " + "double d ; d = 2.0 ; " + "f ( i , d ) ; " + "} " + "void f ( int t , double u ) { }"; + ASSERT_EQUALS(exp, tok(code)); + } + { + // inconsistent deduction => don't deduce + const char code[] = "template void f(T a, T b) { }\n" + "void func() {\n" + " int i = 1;\n" + " long l0 = 2;\n" + " f(i, l0);\n" + "}"; + const char exp[] = "template < typename T > void f ( T a , T b ) { } " + "void func ( ) { " + "int i ; i = 1 ; " + "long l0 ; l0 = 2 ; " + "f ( i , l0 ) ; " + "}"; + ASSERT_EQUALS(exp, tok(code)); + } + { + // unknown variable => don't deduce + const char code[] = "template void f(T n) { }\n" + "void func() {\n" + " f(unknown);\n" + "}"; + const char exp[] = "template < typename T > void f ( T n ) { } " + "void func ( ) { " + "f ( unknown ) ; " + "}"; + ASSERT_EQUALS(exp, tok(code)); + } + { + // the innermost declaration shadows outer scopes + const char code[] = "template void f(T n) { }\n" + "int x;\n" + "void func1() {\n" + " double x = 1.0;\n" + " f(x);\n" + "}\n" + "void func2(float x) {\n" + " f(x);\n" + "}\n" + "void func3() {\n" + " f(x);\n" + "}"; + const char exp[] = "void f ( double n ) ; " + "void f ( float n ) ; " + "void f ( int n ) ; " + "int x ; " + "void func1 ( ) { " + "double x ; x = 1.0 ; " + "f ( x ) ; " + "} " + "void func2 ( float x ) { " + "f ( x ) ; " + "} " + "void func3 ( ) { " + "f ( x ) ; " + "} " + "void f ( double n ) { } " + "void f ( float n ) { } " + "void f ( int n ) { }"; + ASSERT_EQUALS(exp, tok(code)); + } + } + + void templateTypeDeduction10() + { // parameters are visible in constructor + // initializer lists and const methods + const char code[] = "struct A {\n" + " template void tf(T t) { (void)t; }\n" + " int m;\n" + " A(long n) : m(0) { tf(n); }\n" + " void g(float v) const { tf(v); }\n" + "};"; + const char exp[] = "struct A { " + "void tf ( long t ) ; " + "void tf ( float t ) ; " + "int m ; " + "A ( long n ) : m ( 0 ) { tf ( n ) ; } " + "void g ( float v ) const { tf ( v ) ; } " + "} ; " + "void A :: tf ( long t ) { ( void ) t ; } " + "void A :: tf ( float t ) { ( void ) t ; }"; + ASSERT_EQUALS(exp, tok(code)); + ASSERT_EQUALS("", errout_str()); + } + + void templateTypeDeduction11() + { // unqualified lookup: enclosing scopes, shadowing + { + // global function template called from a class scope + const char code[] = "template void f(T n) { (void)n; }\n" + "struct A {\n" + " int m;\n" + " A(long n) : m(0) { f(n); }\n" + " void g(float v) const { f(v); }\n" + "};"; + const char exp[] = "void f ( long n ) ; " + "void f ( float n ) ; " + "struct A { " + "int m ; " + "A ( long n ) : m ( 0 ) { f ( n ) ; } " + "void g ( float v ) const { f ( v ) ; } " + "} ; " + "void f ( long n ) { ( void ) n ; } " + "void f ( float n ) { ( void ) n ; }"; + ASSERT_EQUALS(exp, tok(code)); + } + { + // global function template called from a class nested in a namespace + const char code[] = "template T twice(T v) { return v + v; }\n" + "namespace N {\n" + " struct Inner {\n" + " long go(int a) { return twice(a); }\n" + " };\n" + "}"; + const char exp[] = "int twice ( int v ) ; " + "namespace N { " + "struct Inner { " + "long go ( int a ) { return twice ( a ) ; } " + "} ; " + "} " + "int twice ( int v ) { return v + v ; }"; + ASSERT_EQUALS(exp, tok(code)); + } + { + // a member function template shadows a global function template + const char code[] = "template void dup(T t) { (void)t; }\n" + "struct B {\n" + " template void dup(T t) { (void)t; }\n" + " void m() { dup(1L); }\n" + "};"; + const char exp[] = "template < typename T > void dup ( T t ) { ( void ) t ; } " + "struct B { " + "void dup ( long t ) ; " + "void m ( ) { dup ( 1L ) ; } " + "} ; " + "void B :: dup ( long t ) { ( void ) t ; }"; + ASSERT_EQUALS(exp, tok(code)); + } + } + + void templateTypeDeduction12() + { // base class member templates, out of class method bodies + { + // class members are visible in out of class member function bodies + const char code[] = "struct A {\n" + " int m;\n" + " template void tf(T t) { (void)t; }\n" + " void g();\n" + " long getl();\n" + "};\n" + "void A::g() {\n" + " tf(m);\n" + " tf(getl());\n" + "}"; + const char exp[] = "struct A { " + "int m ; " + "void tf ( int t ) ; " + "void tf ( long t ) ; " + "void g ( ) ; " + "long getl ( ) ; " + "} ; " + "void A :: g ( ) { " + "tf ( m ) ; " + "tf ( getl ( ) ) ; " + "} " + "void A :: tf ( int t ) { ( void ) t ; } " + "void A :: tf ( long t ) { ( void ) t ; }"; + ASSERT_EQUALS(exp, tok(code)); + } + { + // a member template of a base class is called with an inherited + // member variable as the argument + const char code[] = "struct Base {\n" + " template void btf(T t) { (void)t; }\n" + " int bm;\n" + "};\n" + "struct Derived : public Base {\n" + " void use() { btf(bm); }\n" + "};"; + const char exp[] = "struct Base { " + "void btf ( int t ) ; " + "int bm ; " + "} ; " + "struct Derived : public Base { " + "void use ( ) { Base :: btf ( bm ) ; } " + "} ; " + "void Base :: btf ( int t ) { ( void ) t ; }"; + ASSERT_EQUALS(exp, tok(code)); + } + { + // transitive base classes across namespaces, in inline and in + // out of class member function bodies; a derived class member + // template shadows the base class one + const char code[] = "namespace N {\n" + " struct GrandBase {\n" + " template void gtf(T t) { (void)t; }\n" + " double gm;\n" + " };\n" + " struct Mid : public GrandBase {};\n" + "}\n" + "struct Leaf : N::Mid {\n" + " void inlineUse() { gtf(gm); }\n" + " void outOfClassUse();\n" + "};\n" + "void Leaf::outOfClassUse() { gtf(gm); }\n" + "struct Shadowing : N::GrandBase {\n" + " template void gtf(T t) { (void)t; }\n" + " void s() { gtf(1); }\n" + "};"; + const char exp[] = "namespace N { " + "struct GrandBase { " + "void gtf ( double t ) ; " + "double gm ; " + "} ; " + "struct Mid : public GrandBase { } ; " + "} " + "struct Leaf : N :: Mid { " + "void inlineUse ( ) { N :: GrandBase :: gtf ( gm ) ; } " + "void outOfClassUse ( ) ; " + "} ; " + "void Leaf :: outOfClassUse ( ) { N :: GrandBase :: gtf ( gm ) ; } " + "struct Shadowing : N :: GrandBase { " + "void gtf ( int t ) ; " + "void s ( ) { gtf ( 1 ) ; } " + "} ; " + "void Shadowing :: gtf ( int t ) { ( void ) t ; } " + "void N :: GrandBase :: gtf ( double t ) { ( void ) t ; }"; + ASSERT_EQUALS(exp, tok(code)); + } + } + + void templateTypeDeduction13() + { // members declared after the member function are visible + { + const char code[] = "struct A {\n" + " template void tf(T t) { (void)t; }\n" + " void m() { tf(later); tf(getl()); }\n" + " int later;\n" + " long getl();\n" + "};"; + const char exp[] = "struct A { " + "void tf ( int t ) ; " + "void tf ( long t ) ; " + "void m ( ) { tf ( later ) ; tf ( getl ( ) ) ; } " + "int later ; " + "long getl ( ) ; " + "} ; " + "void A :: tf ( int t ) { ( void ) t ; } " + "void A :: tf ( long t ) { ( void ) t ; }"; + ASSERT_EQUALS(exp, tok(code)); + } + { + // constructor body; the member template is also declared later + const char code[] = "struct B {\n" + " B() : v(1.0f) { tf(later); }\n" + " template void tf(T t) { (void)t; }\n" + " float v;\n" + " double later;\n" + "};"; + const char exp[] = "struct B { " + "B ( ) : v ( 1.0f ) { tf ( later ) ; } " + "void tf ( double t ) ; " + "float v ; " + "double later ; " + "} ; " + "void B :: tf ( double t ) { ( void ) t ; }"; + ASSERT_EQUALS(exp, tok(code)); + } + { + // a parameter shadows a member that is declared later + const char code[] = "struct C {\n" + " template void tf(T t) { (void)t; }\n" + " void n(short x) { tf(x); }\n" + " void o() { tf(x); }\n" + " long x;\n" + "};"; + const char exp[] = "struct C { " + "void tf ( short t ) ; " + "void tf ( long t ) ; " + "void n ( short x ) { tf ( x ) ; } " + "void o ( ) { tf ( x ) ; } " + "long x ; " + "} ; " + "void C :: tf ( short t ) { ( void ) t ; } " + "void C :: tf ( long t ) { ( void ) t ; }"; + ASSERT_EQUALS(exp, tok(code)); + } + } + + void templateTypeDeduction14() + { // a non template overload might be a better match: don't deduce + { + // the non template h(char) wins the overload resolution + const char code[] = "template T h(T t) { return t; }\n" + "long h(char);\n" + "void use1() {\n" + " char c = 'x';\n" + " h(c);\n" + "}"; + const char exp[] = "template < class T > T h ( T t ) { return t ; } " + "long h ( char ) ; " + "void use1 ( ) { " + "char c ; c = 'x' ; " + "h ( c ) ; " + "}"; + ASSERT_EQUALS(exp, tok(code)); + } + { + // member function overload set + const char code[] = "struct M {\n" + " template void mf(T t) { (void)t; }\n" + " void mf(int);\n" + " void go(long l) { mf(l); }\n" + "};"; + const char exp[] = "struct M { " + "template < class T > void mf ( T t ) { ( void ) t ; } " + "void mf ( int ) ; " + "void go ( long l ) { mf ( l ) ; } " + "} ;"; + ASSERT_EQUALS(exp, tok(code)); + } + { + // non template overload in a base class + const char code[] = "struct Base {\n" + " template void bf(T t) { (void)t; }\n" + " void bf(int);\n" + "};\n" + "struct Derived : public Base {\n" + " void use(long l) { bf(l); }\n" + "};"; + const char exp[] = "struct Base { " + "template < class T > void bf ( T t ) { ( void ) t ; } " + "void bf ( int ) ; " + "} ; " + "struct Derived : public Base { " + "void use ( long l ) { bf ( l ) ; } " + "} ;"; + ASSERT_EQUALS(exp, tok(code)); + } + { + // overloaded function templates => don't deduce + const char code[] = "template void g(T t) { (void)t; }\n" + "template void g(T* t) { (void)t; }\n" + "void use(long l) { g(l); }"; + const char exp[] = "template < class T > void g ( T t ) { ( void ) t ; } " + "template < class T > void g ( T * t ) { ( void ) t ; } " + "void use ( long l ) { g ( l ) ; }"; + ASSERT_EQUALS(exp, tok(code)); + } + { + // substitution of the template would fail (SFINAE) and the non template + // overload wins the overload resolution => don't deduce + const char code[] = "template\n" + "typename T::type f(T x) { return x; }\n" + "int f(int x) { return x; }\n" + "int use(int y) { return f(y); }"; + const char exp[] = "template < class T > " + "T :: type f ( T x ) { return x ; } " + "int f ( int x ) { return x ; } " + "int use ( int y ) { return f ( y ) ; }"; + ASSERT_EQUALS(exp, tok(code)); + } + { + // same overload set with a literal argument: the literal deduction runs + // before any type information exists and has no overload guards, so it + // wrongly instantiates the template instead of keeping the call to the + // non template overload + const char code[] = "template\n" + "typename T::type f(T x) { return x; }\n" + "int f(int x) { return x; }\n" + "int use() { return f(1); }"; + const char exp[] = "template < class T > " + "T :: type f ( T x ) { return x ; } " + "int f ( int x ) { return x ; } " + "int use ( ) { return f ( 1 ) ; }"; + const char act[] = "int :: type f ( int x ) ; " + "int f ( int x ) { return x ; } " + "int use ( ) { return f ( 1 ) ; } " + "int :: type f ( int x ) { return x ; }"; + TODO_ASSERT_EQUALS(exp, act, tok(code)); + } + } + + void templateTypeDeduction15() + { // deduction is not affected by "final" on the class + { + // members are visible in the out of class member function bodies + const char code[] = "struct A final {\n" + " template void tf(T t) { (void)t; }\n" + " int m;\n" + " void g();\n" + "};\n" + "void A::g() { tf(m); }"; + const char exp[] = "struct A { " + "void tf ( int t ) ; " + "int m ; " + "void g ( ) ; " + "} ; " + "void A :: g ( ) { tf ( m ) ; } " + "void A :: tf ( int t ) { ( void ) t ; }"; + ASSERT_EQUALS(exp, tok(code)); + } + { + // the base classes of a final class are searched + const char code[] = "struct Base {\n" + " template void btf(T t) { (void)t; }\n" + " long bm;\n" + "};\n" + "struct D final : public Base {\n" + " void use();\n" + "};\n" + "void D::use() { btf(bm); }"; + const char exp[] = "struct Base { " + "void btf ( long t ) ; " + "long bm ; " + "} ; " + "struct D : public Base { " + "void use ( ) ; " + "} ; " + "void D :: use ( ) { Base :: btf ( bm ) ; } " + "void Base :: btf ( long t ) { ( void ) t ; }"; + ASSERT_EQUALS(exp, tok(code)); + } + { + // inline member function body of a final class + const char code[] = "struct E final {\n" + " template void etf(T t) { (void)t; }\n" + " void m() { etf(later); }\n" + " double later;\n" + "};"; + const char exp[] = "struct E { " + "void etf ( double t ) ; " + "void m ( ) { etf ( later ) ; } " + "double later ; " + "} ; " + "void E :: etf ( double t ) { ( void ) t ; }"; + ASSERT_EQUALS(exp, tok(code)); + } + } + + void templateTypeDeductionFullRebuild() + { // --template-full-rebuild recreates the symbol database etc. after each type + // deduction round instead of updating them incrementally - the result must be + // the same + { + // deduction from variables and expressions + const char code[] = "template void f(T n) { (void)n; }\n" + "struct MyClass { int m; };\n" + "void func(MyClass mc, unsigned long long ull, double d) {\n" + " f(mc);\n" + " f(ull);\n" + " f(d + 1.5);\n" + "}"; + const std::string expected = tok(code); + ASSERT(expected.find("f") != std::string::npos); + ASSERT_EQUALS(expected, tok(code, dinit(CheckOptions, $.templateFullRebuild = true))); + } + { + // base class member template: the instantiated declaration is removed in a + // finalize step + const char code[] = "struct Base {\n" + " template void btf(T t) { (void)t; }\n" + " int bm;\n" + "};\n" + "struct Derived : public Base {\n" + " void use() { btf(bm); }\n" + "};"; + const std::string expected = tok(code); + ASSERT(expected.find("Base :: btf") != std::string::npos); + ASSERT_EQUALS(expected, tok(code, dinit(CheckOptions, $.templateFullRebuild = true))); + } + { + // nested calls need multiple deduction rounds + const char code[] = "template T pass(T x) { return x; }\n" + "template void sink(T x) { (void)x; }\n" + "void use(long v) { sink(pass(v)); }"; + const std::string expected = tok(code); + ASSERT(expected.find("sink") != std::string::npos); + ASSERT_EQUALS(expected, tok(code, dinit(CheckOptions, $.templateFullRebuild = true))); + } + } + void simplifyTemplateArgs1() { ASSERT_EQUALS("foo<2> = 2 ; foo<2> ;", tok("template foo = N; foo < ( 2 ) >;")); ASSERT_EQUALS("foo<2> = 2 ; foo<2> ;", tok("template foo = N; foo < 1 + 1 >;"));