Add diff support between files to heaptrack_{print,gui}.

Both executables get a new `--diff <file>` argument. The file
that is passed gets parsed in addition to the primary data file.
The allocation data therein will then be subtracted from the data
found in the primary data file.

The algorithm is somewhat complicated, because we have to ignore
instruction pointer addresses, as these will vary between runs
due to address space randomization. Paired with (partial) lack of
debug symbols, we end up with non-trivial situation to map allocations
from one run to that of another.

Essentially, we need to find the correct allocation that shares the
same backtrace while ignoring instruction pointer addresses.

diff --git a/accumulatedtracedata.cpp b/accumulatedtracedata.cpp
index 2d589be..7412425 100644 (file)
--- a/accumulatedtracedata.cpp
+++ b/accumulatedtracedata.cpp
@@ -32,6 +32,12 @@
 #include "config.h"
 #include "pointermap.h"
 
+#ifdef __GNUC__
+#define POTENTIALLY_UNUSED __attribute__ ((unused))
+#else
+#define POTENTIALLY_UNUSED
+#endif
+
 using namespace std;
 
 namespace {
@@ -359,6 +365,234 @@ bool AccumulatedTraceData::read(istream& in)
     return true;
 }
 
+namespace {
+
+template<typename IndexT, typename SortF>
+vector<IndexT> sortedIndices(size_t numIndices, SortF sorter)
+{
+    vector<IndexT> indices;
+    indices.resize(numIndices);
+    for (size_t i = 0; i < numIndices; ++i) {
+        indices[i].index = (i + 1);
+    }
+    sort(indices.begin(), indices.end(), sorter);
+    return indices;
+}
+
+vector<StringIndex> remapStrings(vector<string>& lhs, const vector<string>& rhs)
+{
+    unordered_map<string, StringIndex> stringRemapping;
+    StringIndex stringIndex;
+    {
+        stringRemapping.reserve(lhs.size());
+        for (const auto& string : lhs) {
+            ++stringIndex.index;
+            stringRemapping.insert(make_pair(string, stringIndex));
+        }
+    }
+
+    vector<StringIndex> map;
+    {
+        map.reserve(rhs.size() + 1);
+        map.push_back({});
+        for (const auto& string : rhs) {
+            auto it = stringRemapping.find(string);
+            if (it == stringRemapping.end()) {
+                ++stringIndex.index;
+                lhs.push_back(string);
+                map.push_back(stringIndex);
+            } else {
+                map.push_back(it->second);
+            }
+        }
+    }
+    return map;
+}
+
+template<typename T>
+inline const T& identity(const T& t)
+{
+    return t;
+}
+
+template<typename IpMapper>
+int compareTraceIndices(const TraceIndex &lhs, const AccumulatedTraceData& lhsData,
+                          const TraceIndex &rhs, const AccumulatedTraceData& rhsData,
+                          IpMapper ipMapper)
+{
+    if (!lhs && !rhs) {
+        return 0;
+    } else if (lhs && !rhs) {
+        return 1;
+    } else if (rhs && !lhs) {
+        return -1;
+    } else if (&lhsData == &rhsData && lhs == rhs) {
+        // fast-path if both indices are equal and we compare the same data
+        return 0;
+    }
+
+    const auto& lhsTrace = lhsData.findTrace(lhs);
+    const auto& rhsTrace = rhsData.findTrace(rhs);
+
+    const int parentComparsion = compareTraceIndices(lhsTrace.parentIndex, lhsData, rhsTrace.parentIndex, rhsData, ipMapper);
+    if (parentComparsion != 0) {
+        return parentComparsion;
+    } // else fall-through to below, parents are equal
+
+    const auto& lhsIp = lhsData.findIp(lhsTrace.ipIndex);
+    const auto& rhsIp = ipMapper(rhsData.findIp(rhsTrace.ipIndex));
+    if (lhsIp.equalWithoutAddress(rhsIp)) {
+        return 0;
+    }
+    return lhsIp.compareWithoutAddress(rhsIp) ? -1 : 1;
+}
+
+POTENTIALLY_UNUSED void printTrace(const AccumulatedTraceData& data, TraceIndex index)
+{
+    do {
+        const auto trace = data.findTrace(index);
+        const auto& ip = data.findIp(trace.ipIndex);
+        cerr << index << " (" << trace.ipIndex << ", " << trace.parentIndex << ")"
+            << '\t' << data.stringify(ip.functionIndex)
+            << " in " << data.stringify(ip.moduleIndex)
+            << " at " << data.stringify(ip.fileIndex) << ':' << ip.line
+            << '\n';
+        index = trace.parentIndex;
+    } while (index);
+    cerr << "---\n";
+}
+}
+
+void AccumulatedTraceData::diff(const AccumulatedTraceData& base)
+{
+    totalCost -= base.totalCost;
+
+    // step 0: remap strings and ips
+    const auto& stringMap = remapStrings(strings, base.strings);
+    auto remapString = [&stringMap] (StringIndex& index) {
+        if (index) {
+            index.index = stringMap[index.index].index;
+        }
+    };
+    auto remapIp = [&remapString] (InstructionPointer ip) -> InstructionPointer {
+        remapString(ip.moduleIndex);
+        remapString(ip.functionIndex);
+        remapString(ip.fileIndex);
+        return ip;
+    };
+
+    // step 1: sort trace indices of allocations for efficient lookup
+    // step 2: while at it, also merge equal allocations
+    vector<TraceIndex> allocationTraceNodes;
+    allocationTraceNodes.reserve(allocations.size());
+    for (auto it = allocations.begin(); it != allocations.end();) {
+        const auto& allocation = *it;
+        auto sortedIt = lower_bound(allocationTraceNodes.begin(), allocationTraceNodes.end(), allocation.traceIndex,
+            [this] (const TraceIndex& lhs, const TraceIndex& rhs) -> bool {
+                return compareTraceIndices(lhs, *this,
+                                           rhs, *this,
+                                           identity<InstructionPointer>) < 0;
+            });
+        if (sortedIt == allocationTraceNodes.end()
+            || compareTraceIndices(allocation.traceIndex, *this, *sortedIt, *this, identity<InstructionPointer>) != 0)
+        {
+            allocationTraceNodes.insert(sortedIt, allocation.traceIndex);
+            ++it;
+        } else if (*sortedIt != allocation.traceIndex) {
+            findAllocation(*sortedIt) += allocation;
+            it = allocations.erase(it);
+        } else {
+            ++it;
+        }
+    }
+
+    // step 3: iterate over rhs data and find matching traces
+    //         if no match is found, copy the data over
+
+    auto sortedIps = sortedIndices<IpIndex>(instructionPointers.size(),
+        [this] (const IpIndex &lhs, const IpIndex &rhs) {
+            return findIp(lhs).compareWithoutAddress(findIp(rhs));
+        });
+
+    auto remapIpIndex = [&sortedIps, this, &base, &remapIp] (IpIndex rhsIndex) -> IpIndex {
+        if (!rhsIndex) {
+            return rhsIndex;
+        }
+
+        const auto& rhsIp = base.findIp(rhsIndex);
+        const auto& lhsIp = remapIp(rhsIp);
+
+        auto it = lower_bound(sortedIps.begin(), sortedIps.end(), lhsIp,
+                              [this] (const IpIndex &lhs, const InstructionPointer &rhs) {
+                                  return findIp(lhs).compareWithoutAddress(rhs);
+                              });
+        if (it != sortedIps.end() && findIp(*it).equalWithoutAddress(lhsIp)) {
+            return *it;
+        }
+
+        instructionPointers.push_back(lhsIp);
+
+        IpIndex ret;
+        ret.index = instructionPointers.size();
+        sortedIps.insert(it, ret);
+
+        return ret;
+    };
+
+    function<TraceIndex (TraceIndex)> copyTrace = [this, &base, remapIpIndex, &copyTrace] (TraceIndex rhsIndex) -> TraceIndex {
+        if (!rhsIndex) {
+            return rhsIndex;
+        }
+
+        // new location, add it
+        const auto& rhsTrace = base.findTrace(rhsIndex);
+
+        TraceNode node;
+        node.parentIndex = copyTrace(rhsTrace.parentIndex);
+        node.ipIndex = remapIpIndex(rhsTrace.ipIndex);
+
+        traces.push_back(node);
+        TraceIndex ret;
+        ret.index = traces.size();
+
+        return ret;
+    };
+
+    auto remapTrace = [&base, &allocationTraceNodes, this, remapIp, copyTrace] (TraceIndex rhsIndex) -> TraceIndex {
+        if (!rhsIndex) {
+            return rhsIndex;
+        }
+
+        auto it = lower_bound(allocationTraceNodes.begin(), allocationTraceNodes.end(), rhsIndex,
+            [&base, this, remapIp] (const TraceIndex& lhs, const TraceIndex& rhs) -> bool {
+                return compareTraceIndices(lhs, *this, rhs, base, remapIp) < 0;
+            });
+
+        if (it != allocationTraceNodes.end()
+            && compareTraceIndices(*it, *this, rhsIndex, base, remapIp) == 0)
+        {
+            return *it;
+        }
+
+        TraceIndex ret = copyTrace(rhsIndex);
+        allocationTraceNodes.insert(it, ret);
+        return ret;
+    };
+
+    for (const auto& rhsAllocation : base.allocations) {
+        const auto lhsTrace = remapTrace(rhsAllocation.traceIndex);
+        assert(base.findIp(base.findTrace(rhsAllocation.traceIndex).ipIndex).equalWithoutAddress(findIp(findTrace(lhsTrace).ipIndex)));
+        findAllocation(lhsTrace) -= rhsAllocation;
+    }
+
+    // step 4: remove allocations that don't show any differences
+    allocations.erase(remove_if(allocations.begin(), allocations.end(),
+        [] (const Allocation& allocation) -> bool {
+            return !allocation.allocated && !allocation.allocations && !allocation.leaked
+                && !allocation.peak && !allocation.temporary;
+        }), allocations.end());
+}
+
 Allocation& AccumulatedTraceData::findAllocation(const TraceIndex traceIndex)
 {
     if (traceIndex < m_maxAllocationTraceIndex) {

diff --git a/accumulatedtracedata.h b/accumulatedtracedata.h
index 6418375..bd390aa 100644 (file)
--- a/accumulatedtracedata.h
+++ b/accumulatedtracedata.h
@@ -95,6 +95,8 @@ struct AccumulatedTraceData
     bool read(const std::string& inputFile);
     bool read(std::istream& in);
 
+    void diff(const AccumulatedTraceData& base);
+
     bool shortenTemplates = false;
     bool fromAttached = false;
 

diff --git a/gui/gui.cpp b/gui/gui.cpp
index 4667904..abf53eb 100644 (file)
--- a/gui/gui.cpp
+++ b/gui/gui.cpp
@@ -53,6 +53,8 @@ int main(int argc, char** argv)
     parser.addHelpOption();
     aboutData.setupCommandLine(&parser);
 
+    QCommandLineOption diffOption{QStringLiteral("diff"), i18n("Base <file> to calculate differences to."), QStringLiteral("diff")};
+    parser.addOption(diffOption);
     parser.addPositionalArgument(QStringLiteral("files"), i18n( "Files to load" ), i18n("[FILE...]"));
 
     parser.process(app);
@@ -66,7 +68,7 @@ int main(int argc, char** argv)
     };
 
     foreach (const QString &file, parser.positionalArguments()) {
-        createWindow()->loadFile(file);
+        createWindow()->loadFile(file, parser.value(diffOption));
     }
 
     if (parser.positionalArguments().isEmpty()) {

diff --git a/gui/mainwindow.cpp b/gui/mainwindow.cpp
index 9ce6d59..1e87dd3 100644 (file)
--- a/gui/mainwindow.cpp
+++ b/gui/mainwindow.cpp
@@ -329,12 +329,12 @@ MainWindow::~MainWindow()
     group.writeEntry(Config::Entries::State, state);
 }
 
-void MainWindow::loadFile(const QString& file)
+void MainWindow::loadFile(const QString& file, const QString& diffBase)
 {
     m_ui->loadingLabel->setText(i18n("Loading file %1, please wait...", file));
     setWindowTitle(i18nc("%1: file name that is open", "Heaptrack - %1", file));
     m_ui->pages->setCurrentWidget(m_ui->loadingPage);
-    m_parser->parse(file);
+    m_parser->parse(file, diffBase);
 }
 
 void MainWindow::openFile()
@@ -343,7 +343,9 @@ void MainWindow::openFile()
     dialog->setAttribute(Qt::WA_DeleteOnClose, true);
     dialog->setFileMode(QFileDialog::ExistingFile);
     connect(dialog, &QFileDialog::fileSelected,
-            this, &MainWindow::loadFile);
+            this, [this] (const QString& file) {
+                loadFile(file);
+            });
     dialog->show();
 }
 

diff --git a/gui/mainwindow.h b/gui/mainwindow.h
index d3e2d31..b43109a 100644 (file)
--- a/gui/mainwindow.h
+++ b/gui/mainwindow.h
@@ -39,7 +39,7 @@ public:
     virtual ~MainWindow();
 
 public slots:
-    void loadFile(const QString& path);
+    void loadFile(const QString& path, const QString& diffBase = {});
     void openFile();
 
 private:

diff --git a/gui/parser.cpp b/gui/parser.cpp
index 7194eaf..b009a31 100644 (file)
--- a/gui/parser.cpp
+++ b/gui/parser.cpp
@@ -46,6 +46,8 @@ struct StringCache
         if (ip.functionIndex) {
             // TODO: support removal of template arguments
             return stringify(ip.functionIndex);
+        } else if (diffMode) {
+            return i18n("<unresolved function>");
         } else {
             auto& ipAddr = m_ipAddresses[ip.instructionPointer];
             if (ipAddr.isEmpty()) {
@@ -100,6 +102,8 @@ struct StringCache
     vector<QString> m_strings;
     mutable QHash<uint64_t, QString> m_ipAddresses;
     mutable vector<shared_ptr<LocationData>> m_locations;
+
+    bool diffMode = false;
 };
 
 struct ChartMergeData
@@ -474,10 +478,10 @@ Parser::Parser(QObject* parent)
 
 Parser::~Parser() = default;
 
-void Parser::parse(const QString& path)
+void Parser::parse(const QString& path, const QString& diffBase)
 {
     using namespace ThreadWeaver;
-    stream() << make_job([this, path]() {
+    stream() << make_job([this, path, diffBase]() {
         const auto stdPath = path.toStdString();
         auto data = make_shared<ParserData>();
         emit progressMessageAvailable(i18n("parsing data..."));
@@ -486,6 +490,16 @@ void Parser::parse(const QString& path)
             return;
         }
 
+        if (!diffBase.isEmpty()) {
+            ParserData diffData;
+            if (!diffData.read(diffBase.toStdString())) {
+                emit failedToOpen(diffBase);
+                return;
+            }
+            data->diff(diffData);
+            data->stringCache.diffMode = true;
+        }
+
         data->updateStringCache();
 
         emit summaryAvailable({

diff --git a/gui/parser.h b/gui/parser.h
index eec9d29..4041731 100644 (file)
--- a/gui/parser.h
+++ b/gui/parser.h
@@ -35,7 +35,7 @@ public:
     virtual ~Parser();
 
 public slots:
-    void parse(const QString& path);
+    void parse(const QString& path, const QString& diffBase);
 
 signals:
     void progressMessageAvailable(const QString& progress);

diff --git a/heaptrack_print.cpp b/heaptrack_print.cpp
index a025021..01ef492 100644 (file)
--- a/heaptrack_print.cpp
+++ b/heaptrack_print.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright 2014 Milian Wolff <mail@milianw.de>
+ * Copyright 2014-2016 Milian Wolff <mail@milianw.de>
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU Library General Public License as
@@ -498,6 +498,8 @@ int main(int argc, char** argv)
     desc.add_options()
         ("file,f", po::value<string>(),
             "The heaptrack data file to print.")
+        ("diff,d", po::value<string>()->default_value({}),
+            "Find the differences to this file.")
         ("shorten-templates,t", po::value<bool>()->default_value(true)->implicit_value(true),
             "Shorten template identifiers.")
         ("merge-backtraces,m", po::value<bool>()->default_value(true)->implicit_value(true),
@@ -577,6 +579,7 @@ int main(int argc, char** argv)
     Printer data;
 
     const auto inputFile = vm["file"].as<string>();
+    const auto diffFile = vm["diff"].as<string>();
     data.shortenTemplates = vm["shorten-templates"].as<bool>();
     data.mergeBacktraces = vm["merge-backtraces"].as<bool>();
     data.filterBtFunction = vm["filter-bt-function"].as<string>();
@@ -605,6 +608,16 @@ int main(int argc, char** argv)
     if (!data.read(inputFile)) {
         return 1;
     }
+
+    if (!diffFile.empty()) {
+        cout << "reading diff file \"" << diffFile << "\" - please wait, this might take some time..." << endl;
+        Printer diffData;
+        if (!diffData.read(diffFile)) {
+            return 1;
+        }
+        data.diff(diffData);
+    }
+
     data.finalize();
 
     cout << "finished reading file, now analyzing data:\n" << endl;

diff --git a/indices.h b/indices.h
index 7242bcd..22749b1 100644 (file)
--- a/indices.h
+++ b/indices.h
@@ -38,6 +38,21 @@ struct Index
         return index < o.index;
     }
 
+    bool operator<=(Index o) const
+    {
+        return index <= o.index;
+    }
+
+    bool operator>(Index o) const
+    {
+        return index > o.index;
+    }
+
+    bool operator>=(Index o) const
+    {
+        return index >= o.index;
+    }
+
     bool operator!=(Index o) const
     {
         return index != o.index;