new loplugin:unusedfields

run it over the framework module

Change-Id: I1220a4be0936ba30136ce22ffd78633c8a7b9d35

compilerplugins/clang/unusedfields.cxx

@@ -0,0 +1,202 @@
+/* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */
+/*
+ * This file is part of the LibreOffice project.
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ */
+
+#include <cassert>
+#include <string>
+#include <iostream>
+#include <fstream>
+#include <set>
+#include "plugin.hxx"
+#include "compat.hxx"
+
+/**
+Dump a list of calls to methods, and a list of field definitions.
+Then we will post-process the 2 lists and find the set of unused methods.
+
+Be warned that it produces around 5G of log file.
+
+The process goes something like this:
+  $ make check
+  $ make FORCE_COMPILE_ALL=1 COMPILER_PLUGIN_TOOL='unusedfields' check
+  $ ./compilerplugins/clang/unusedfields.py unusedfields.log > result.txt
+
+and then
+  $ for dir in *; do make FORCE_COMPILE_ALL=1 UPDATE_FILES=$dir COMPILER_PLUGIN_TOOL='unusedfieldsremove' $dir; done
+to auto-remove the method declarations
+
+Note that the actual process may involve a fair amount of undoing, hand editing, and general messing around
+to get it to work :-)
+
+*/
+
+namespace {
+
+struct MyFieldInfo
+{
+    std::string parentClass;
+    std::string fieldName;
+    std::string sourceLocation;
+
+    bool operator < (const MyFieldInfo &other) const
+    {
+        if (parentClass < other.parentClass)
+            return true;
+        else if (parentClass == other.parentClass)
+            return fieldName < other.fieldName;
+        else
+            return false;
+    }
+};
+
+
+// try to limit the voluminous output a little
+static std::set<MyFieldInfo> touchedSet;
+static std::set<MyFieldInfo> definitionSet;
+
+
+class UnusedFields:
+    public RecursiveASTVisitor<UnusedFields>, public loplugin::Plugin
+{
+public:
+    explicit UnusedFields(InstantiationData const & data): Plugin(data) {}
+
+    virtual void run() override
+    {
+        TraverseDecl(compiler.getASTContext().getTranslationUnitDecl());
+
+        // dump all our output in one write call - this is to try and limit IO "crosstalk" between multiple processes
+        // writing to the same logfile
+        std::string output;
+        for (const MyFieldInfo & s : touchedSet)
+            output += "touch:\t" + s.parentClass + "\t" + s.fieldName + "\n";
+        for (const MyFieldInfo & s : definitionSet)
+        {
+            output += "definition:\t" + s.parentClass + "\t" + s.fieldName + "\t" + s.sourceLocation + "\n";
+        }
+        ofstream myfile;
+        myfile.open( SRCDIR "/unusedfields.log", ios::app | ios::out);
+        myfile << output;
+        myfile.close();
+    }
+
+    bool shouldVisitTemplateInstantiations () const { return true; }
+
+    bool VisitCallExpr(CallExpr* );
+    bool VisitFieldDecl( const FieldDecl* );
+    bool VisitMemberExpr( const MemberExpr* );
+    bool VisitDeclRefExpr( const DeclRefExpr* );
+private:
+    MyFieldInfo niceName(const FieldDecl*);
+    std::string fullyQualifiedName(const FunctionDecl*);
+};
+
+MyFieldInfo UnusedFields::niceName(const FieldDecl* fieldDecl)
+{
+    MyFieldInfo aInfo;
+    aInfo.parentClass = fieldDecl->getParent()->getQualifiedNameAsString();
+    aInfo.fieldName = fieldDecl->getNameAsString();
+
+    SourceLocation expansionLoc = compiler.getSourceManager().getExpansionLoc( fieldDecl->getLocation() );
+    StringRef name = compiler.getSourceManager().getFilename(expansionLoc);
+    aInfo.sourceLocation = std::string(name.substr(strlen(SRCDIR)+1)) + ":" + std::to_string(compiler.getSourceManager().getSpellingLineNumber(expansionLoc));
+
+    return aInfo;
+}
+
+std::string UnusedFields::fullyQualifiedName(const FunctionDecl* functionDecl)
+{
+    std::string ret = compat::getReturnType(*functionDecl).getCanonicalType().getAsString();
+    ret += " ";
+    if (isa<CXXMethodDecl>(functionDecl)) {
+        const CXXRecordDecl* recordDecl = dyn_cast<CXXMethodDecl>(functionDecl)->getParent();
+        ret += recordDecl->getQualifiedNameAsString();
+        ret += "::";
+    }
+    ret += functionDecl->getNameAsString() + "(";
+    bool bFirst = true;
+    for (const ParmVarDecl *pParmVarDecl : functionDecl->params()) {
+        if (bFirst)
+            bFirst = false;
+        else
+            ret += ",";
+        ret += pParmVarDecl->getType().getCanonicalType().getAsString();
+    }
+    ret += ")";
+    if (isa<CXXMethodDecl>(functionDecl) && dyn_cast<CXXMethodDecl>(functionDecl)->isConst()) {
+        ret += " const";
+    }
+
+    return ret;
+}
+
+// prevent recursive templates from blowing up the stack
+static std::set<std::string> traversedFunctionSet;
+
+bool UnusedFields::VisitCallExpr(CallExpr* expr)
+{
+    // Note that I don't ignore ANYTHING here, because I want to get calls to my code that result
+    // from template instantiation deep inside the STL and other external code
+
+    FunctionDecl* calleeFunctionDecl = expr->getDirectCallee();
+    if (calleeFunctionDecl == nullptr) {
+        Expr* callee = expr->getCallee()->IgnoreParenImpCasts();
+        DeclRefExpr* dr = dyn_cast<DeclRefExpr>(callee);
+        if (dr) {
+            calleeFunctionDecl = dyn_cast<FunctionDecl>(dr->getDecl());
+            if (calleeFunctionDecl)
+                goto gotfunc;
+        }
+        return true;
+    }
+
+gotfunc:
+    // if we see a call to a function, it may effectively create new code,
+    // if the function is templated. However, if we are inside a template function,
+    // calling another function on the same template, the same problem occurs.
+    // Rather than tracking all of that, just traverse anything we have not already traversed.
+    if (traversedFunctionSet.insert(fullyQualifiedName(calleeFunctionDecl)).second)
+        TraverseFunctionDecl(calleeFunctionDecl);
+
+    return true;
+}
+
+bool UnusedFields::VisitFieldDecl( const FieldDecl* fieldDecl )
+{
+    fieldDecl = fieldDecl->getCanonicalDecl();
+
+    if( !ignoreLocation( fieldDecl ))
+        definitionSet.insert(niceName(fieldDecl));
+    return true;
+}
+
+bool UnusedFields::VisitMemberExpr( const MemberExpr* memberExpr )
+{
+    const ValueDecl* decl = memberExpr->getMemberDecl();
+    if (!isa<FieldDecl>(decl)) {
+        return true;
+    }
+    touchedSet.insert(niceName(dyn_cast<FieldDecl>(decl)));
+    return true;
+}
+
+bool UnusedFields::VisitDeclRefExpr( const DeclRefExpr* declRefExpr )
+{
+    const Decl* decl = declRefExpr->getDecl();
+    if (!isa<FieldDecl>(decl)) {
+        return true;
+    }
+    touchedSet.insert(niceName(dyn_cast<FieldDecl>(decl)));
+    return true;
+}
+
+loplugin::Plugin::Registration< UnusedFields > X("unusedfields", false);
+
+}
+
+/* vim:set shiftwidth=4 softtabstop=4 expandtab: */

compilerplugins/clang/unusedfields.py

@@ -0,0 +1,79 @@
+#!/usr/bin/python
+
+import sys
+import re
+import io
+
+definitionSet = set()
+definitionToSourceLocationMap = dict()
+callSet = set()
+sourceLocationSet = set()
+# things we need to exclude for reasons like :
+# - it's a weird template thingy that confuses the plugin
+exclusionSet = set([
+    ])
+
+# clang does not always use exactly the same numbers in the type-parameter vars it generates
+# so I need to substitute them to ensure we can match correctly.
+normalizeTypeParamsRegex = re.compile(r"type-parameter-\d+-\d+")
+def normalizeTypeParams( line ):
+    return normalizeTypeParamsRegex.sub("type-parameter-?-?", line)
+
+# The parsing here is designed to avoid grabbing stuff which is mixed in from gbuild.
+# I have not yet found a way of suppressing the gbuild output.
+with io.open(sys.argv[1], "rb", buffering=1024*1024) as txt:
+    for line in txt:
+        if line.startswith("definition:\t"):
+            idx1 = line.find("\t",12)
+            idx2 = line.find("\t",idx1+1)
+            funcInfo = (normalizeTypeParams(line[12:idx1]), normalizeTypeParams(line[idx1+1:idx2]))
+            definitionSet.add(funcInfo)
+            definitionToSourceLocationMap[funcInfo] = line[idx2+1:].strip()
+        elif line.startswith("touch:\t"):
+            idx1 = line.find("\t",7)
+            callInfo = (normalizeTypeParams(line[7:idx1]), normalizeTypeParams(line[idx1+1:].strip()))
+            callSet.add(callInfo)
+
+# Invert the definitionToSourceLocationMap
+# If we see more than one method at the same sourceLocation, it's being autogenerated as part of a template
+# and we should just ignore
+sourceLocationToDefinitionMap = {}
+for k, v in definitionToSourceLocationMap.iteritems():
+    sourceLocationToDefinitionMap[v] = sourceLocationToDefinitionMap.get(v, [])
+    sourceLocationToDefinitionMap[v].append(k)
+for k, definitions in sourceLocationToDefinitionMap.iteritems():
+    if len(definitions) > 1:
+        for d in definitions:
+            definitionSet.remove(d)
+    
+tmp1set = set()
+for d in definitionSet:
+    clazz = d[0] + " " + d[1]
+    if clazz in exclusionSet:
+        continue
+    if d in callSet:
+        continue
+    if (definitionToSourceLocationMap[d].startswith("include/")):
+        continue
+
+    tmp1set.add((clazz, definitionToSourceLocationMap[d]))
+
+# sort the results using a "natural order" so sequences like [item1,item2,item10] sort nicely
+def natural_sort_key(s, _nsre=re.compile('([0-9]+)')):
+    return [int(text) if text.isdigit() else text.lower()
+            for text in re.split(_nsre, s)]
+
+# sort results by name and line number
+tmp1list = sorted(tmp1set, key=lambda v: natural_sort_key(v[1]))
+
+# print out the results
+for t in tmp1list:
+    print t[1]
+    print "    ", t[0]
+
+    
+
+# add an empty line at the end to make it easier for the unusedFieldsremove plugin to mmap() the output file 
+print
+        
+

compilerplugins/clang/unusedfieldsremove.cxx

@@ -0,0 +1,136 @@
+/* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */
+/*
+ * This file is part of the LibreOffice project.
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ */
+
+#include <cassert>
+#include <string>
+#include <iostream>
+#include "plugin.hxx"
+#include "compat.hxx"
+#include <sys/mman.h>
+#include <sys/types.h>
+#include <fcntl.h>
+#include <unistd.h>
+#include <sys/stat.h>
+#include <assert.h>
+#include <cstring>
+
+/**
+  This is intended to be run as the second stage of the "unusedfields" clang plugin.
+*/
+
+namespace {
+
+class UnusedFieldsRemove:
+    public RecursiveASTVisitor<UnusedFieldsRemove>, public loplugin::RewritePlugin
+{
+public:
+    explicit UnusedFieldsRemove(InstantiationData const & data);
+    ~UnusedFieldsRemove();
+
+    virtual void run() override { TraverseDecl(compiler.getASTContext().getTranslationUnitDecl()); }
+
+    bool VisitFieldDecl( const FieldDecl* var );
+private:
+    // I use a brute-force approach - mmap the results file and do a linear search on it
+    // It works surprisingly well, because the file is small enough to fit into L2 cache on modern CPU's
+    size_t mmapFilesize;
+    int mmapFD;
+    char* mmappedData;
+};
+
+static size_t getFilesize(const char* filename)
+{
+    struct stat st;
+    stat(filename, &st);
+    return st.st_size;
+}
+
+UnusedFieldsRemove::UnusedFieldsRemove(InstantiationData const & data): RewritePlugin(data)
+{
+    static const char sInputFile[] = SRCDIR "/result.txt";
+    mmapFilesize = getFilesize(sInputFile);
+    //Open file
+    mmapFD = open(sInputFile, O_RDONLY, 0);
+    assert(mmapFD != -1);
+    //Execute mmap
+    mmappedData = static_cast<char*>(mmap(NULL, mmapFilesize, PROT_READ, MAP_PRIVATE, mmapFD, 0));
+    assert(mmappedData != NULL);
+}
+
+UnusedFieldsRemove::~UnusedFieldsRemove()
+{
+    //Cleanup
+    int rc = munmap(mmappedData, mmapFilesize);
+    assert(rc == 0);
+    close(mmapFD);
+}
+
+static std::string niceName(const FieldDecl* fieldDecl)
+{
+    std::string s = fieldDecl->getParent()->getQualifiedNameAsString() + " " +
+        fieldDecl->getNameAsString();
+    if (s.find("m_xExternalProgress") != std::string::npos)
+            cout << s << endl;
+    return s;
+}
+
+bool UnusedFieldsRemove::VisitFieldDecl( const FieldDecl* fieldDecl )
+{
+    if (rewriter == nullptr) {
+        return true;
+    }
+    if (ignoreLocation(fieldDecl)) {
+        return true;
+    }
+    // ignore stuff that forms part of the stable URE interface
+    if (isInUnoIncludeFile(compiler.getSourceManager().getSpellingLoc(
+                              fieldDecl->getCanonicalDecl()->getLocation()))) {
+        return true;
+    }
+
+    // don't mess with templates
+/*    if (isa<CXXRecordDecl>(fieldDecl->getParent())) {
+        if (dyn_cast<CXXRecordDecl>(fieldDecl->getParent())->getDescribedClassTemplate() != nullptr) {
+            return true;
+        }
+    }
+*/
+    std::string aNiceName = " " + niceName(fieldDecl) + "\n";
+    const char *aNiceNameStr = aNiceName.c_str();
+    char* found = std::search(mmappedData, mmappedData + mmapFilesize, aNiceNameStr, aNiceNameStr + strlen(aNiceNameStr));
+    if(!(found < mmappedData + mmapFilesize)) {
+        return true;
+    }
+    SourceRange replaceRange(fieldDecl->getSourceRange());
+    // sometimes the declaration has a semicolon just after it, and it's much neater to remove that too.
+    if (rewriter->getRewrittenText(SourceRange(replaceRange.getEnd(), replaceRange.getEnd().getLocWithOffset(1))) == ";") {
+        replaceRange.setEnd(replaceRange.getEnd().getLocWithOffset(1));
+    }
+    // remove leading spaces
+    while (rewriter->getRewrittenText(SourceRange(replaceRange.getBegin().getLocWithOffset(-1), replaceRange.getBegin())) == " ")
+    {
+        replaceRange.setBegin(replaceRange.getBegin().getLocWithOffset(-1));
+    }
+    if (!replaceText(replaceRange, "")) {
+        report(
+            DiagnosticsEngine::Warning,
+            "Could not remove unused field (" + niceName(fieldDecl) + ")",
+            fieldDecl->getLocStart())
+          << fieldDecl->getSourceRange();
+    }
+    return true;
+}
+
+
+
+loplugin::Plugin::Registration< UnusedFieldsRemove > X("unusedfieldsremove", false);
+
+}
+
+/* vim:set shiftwidth=4 softtabstop=4 expandtab: */