Report retained size

shark/shark/src/main/java/shark/AndroidObjectSizeCalculator.kt

@@ -0,0 +1,16 @@
+package shark
+
+import shark.DominatorTree.ObjectSizeCalculator
+import shark.internal.ShallowSizeCalculator
+
+class AndroidObjectSizeCalculator(graph: HeapGraph) : ObjectSizeCalculator {
+
+  private val nativeSizes = AndroidNativeSizeMapper(graph).mapNativeSizes()
+  private val shallowSizeCalculator = ShallowSizeCalculator(graph)
+
+  override fun computeSize(objectId: Long): Int {
+    val nativeSize = nativeSizes[objectId] ?: 0
+    val shallowSize = shallowSizeCalculator.computeShallowSize(objectId)
+    return nativeSize + shallowSize
+  }
+}

shark/shark/src/main/java/shark/DominatorTree.kt

@@ -8,6 +8,10 @@ import shark.internal.hppc.LongScatterSet
 
 class DominatorTree(expectedElements: Int = 4) {
 
+  fun interface ObjectSizeCalculator {
+    fun computeSize(objectId: Long): Int
+  }
+
   /**
    * Map of objects to their dominator.
    *
@@ -16,6 +20,8 @@ class DominatorTree(expectedElements: Int = 4) {
    */
   private val dominated = LongLongScatterMap(expectedElements)
 
+  operator fun contains(objectId: Long): Boolean = dominated.containsKey(objectId)
+
   /**
    * Records that [objectId] is a root.
    */
@@ -98,7 +104,7 @@ class DominatorTree(expectedElements: Int = 4) {
     val dominated = mutableListOf<Long>()
   }
 
-  fun buildFullDominatorTree(computeSize: (Long) -> Int): Map<Long, DominatorNode> {
+  fun buildFullDominatorTree(objectSizeCalculator: ObjectSizeCalculator): Map<Long, DominatorNode> {
     val dominators = mutableMapOf<Long, MutableDominatorNode>()
     dominated.forEach(ForEachCallback {key, value ->
       // create entry for dominated
@@ -115,7 +121,7 @@ class DominatorTree(expectedElements: Int = 4) {
     val allReachableObjectIds = dominators.keys.toSet() - ValueHolder.NULL_REFERENCE
 
     val retainedSizes = computeRetainedSizes(allReachableObjectIds) { objectId ->
-      val shallowSize = computeSize(objectId)
+      val shallowSize = objectSizeCalculator.computeSize(objectId)
       dominators.getValue(objectId).shallowSize = shallowSize
       shallowSize
     }
@@ -147,12 +153,12 @@ class DominatorTree(expectedElements: Int = 4) {
 
   /**
    * Computes the size retained by [retainedObjectIds] using the dominator tree built using
-   * [updateDominatedAsRoot]. The shallow size of each object is provided by [computeSize].
+   * [updateDominated]. The shallow size of each object is provided by [objectSizeCalculator].
    * @return a map of object id to retained size.
    */
   fun computeRetainedSizes(
     retainedObjectIds: Set<Long>,
-    computeSize: (Long) -> Int
+    objectSizeCalculator: ObjectSizeCalculator
   ): Map<Long, Pair<Int, Int>> {
     val nodeRetainedSizes = mutableMapOf<Long, Pair<Int, Int>>()
     retainedObjectIds.forEach { objectId ->
@@ -169,7 +175,7 @@ class DominatorTree(expectedElements: Int = 4) {
 
         // If the entry is a node, add its size to nodeRetainedSizes
         nodeRetainedSizes[key]?.let { (currentRetainedSize, currentRetainedCount) ->
-          instanceSize = computeSize(key)
+          instanceSize = objectSizeCalculator.computeSize(key)
           nodeRetainedSizes[key] = currentRetainedSize + instanceSize to currentRetainedCount + 1
         }
 
@@ -186,7 +192,7 @@ class DominatorTree(expectedElements: Int = 4) {
                 dominated[objectId] = dominator
               }
               if (instanceSize == -1) {
-                instanceSize = computeSize(key)
+                instanceSize = objectSizeCalculator.computeSize(key)
               }
               // Update retained size for that node
               val (currentRetainedSize, currentRetainedCount) = nodeRetainedSizes.getValue(

shark/shark/src/main/java/shark/HeapGraphObjectGrowthDetector.kt

@@ -4,12 +4,14 @@ package shark
 
 import java.util.ArrayDeque
 import java.util.Deque
+import shark.ByteSize.Companion.bytes
 import shark.HeapObject.HeapClass
 import shark.HeapObject.HeapInstance
 import shark.HeapObject.HeapObjectArray
 import shark.HeapObject.HeapPrimitiveArray
 import shark.ReferenceLocationType.ARRAY_ENTRY
 import shark.ReferenceReader.Factory
+import shark.ShortestPathObjectNode.Retained
 import shark.internal.hppc.LongScatterSet
 
 class HeapGraphObjectGrowthDetector(
@@ -20,15 +22,29 @@ class HeapGraphObjectGrowthDetector(
   fun findGrowingObjects(
     heapGraph: CloseableHeapGraph,
     scenarioLoops: Int,
-    previousTraversal: InputHeapTraversal = NoHeapTraversalYet,
+    previousTraversal: InputHeapTraversal = NoHeapTraversalYet
   ): HeapTraversal {
-    val state = TraversalState()
+    val computeRetainedHeapSize = previousTraversal !is NoHeapTraversalYet
+    // Estimate of how many objects we'll visit. This is a conservative estimate, we should always
+    // visit more than that but this limits the number of early array growths.
+    val estimatedVisitedObjects = (heapGraph.instanceCount / 2).coerceAtLeast(4)
+    val state = TraversalState(
+      estimatedVisitedObjects = estimatedVisitedObjects,
+      computeRetainedHeapSize = computeRetainedHeapSize
+    )
     return heapGraph.use {
-      state.traverseHeapDiffingShortestPaths(heapGraph, scenarioLoops, previousTraversal)
+      state.traverseHeapDiffingShortestPaths(
+        heapGraph,
+        scenarioLoops,
+        previousTraversal
+      )
     }
   }
 
-  private class TraversalState {
+  private class TraversalState(
+    estimatedVisitedObjects: Int,
+    computeRetainedHeapSize: Boolean
+  ) {
     var visitingLast = false
 
     /** Set of objects to visit */
@@ -39,7 +55,9 @@ class HeapGraphObjectGrowthDetector(
      */
     val toVisitLastQueue: Deque<Node> = ArrayDeque()
 
-    val visitedSet = LongScatterSet()
+    val visitedSet = LongScatterSet(estimatedVisitedObjects)
+    val dominatorTree =
+      if (computeRetainedHeapSize) DominatorTree(estimatedVisitedObjects) else null
 
     val tree = ShortestPathObjectNode("root", null, newNode = false).apply {
       selfObjectCount = 1
@@ -52,7 +70,7 @@ class HeapGraphObjectGrowthDetector(
   private fun TraversalState.traverseHeapDiffingShortestPaths(
     graph: CloseableHeapGraph,
     detectedGrowth: Int,
-    previousTraversal: InputHeapTraversal,
+    previousTraversal: InputHeapTraversal
   ): HeapTraversal {
 
     // First iteration, all nodes are growing.
@@ -94,6 +112,7 @@ class HeapGraphObjectGrowthDetector(
         val isLeafObject: Boolean
       )
 
+      // Note: this is different from visitedSet.size(), which includes gc roots.
       var visitedObjectCount = 0
 
       val edges = node.objectIds.flatMap { objectId ->
@@ -106,33 +125,41 @@ class HeapGraphObjectGrowthDetector(
           if (node.isLeafObject) {
             emptySequence()
           } else {
-          val heapObject = graph.findObjectById(objectId)
-          val refs = objectReferenceReader.read(heapObject)
-          refs.mapNotNull { reference ->
-            if (reference.valueObjectId in visitedSet) {
-              null
-            } else {
-              val details = reference.lazyDetailsResolver.resolve()
-              val refType = details.locationType.name
-              val owningClassSimpleName =
-                graph.findObjectById(details.locationClassObjectId).asClass!!.simpleName
-              val refName = if (details.locationType == ARRAY_ENTRY) "[x]" else details.name
-              val referencedObjectName =
-                when (val referencedObject = graph.findObjectById(reference.valueObjectId)) {
-                  is HeapClass -> "class ${referencedObject.name}"
-                  is HeapInstance -> "instance of ${referencedObject.instanceClassName}"
-                  is HeapObjectArray -> "array of ${referencedObject.arrayClassName}"
-                  is HeapPrimitiveArray -> "array of ${referencedObject.primitiveType.name.lowercase()}"
-                }
-              val nodeAndEdgeName =
-                "$refType ${owningClassSimpleName}.${refName} -> $referencedObjectName"
-              ExpandedObject(
-                reference.valueObjectId, nodeAndEdgeName, reference.isLowPriority,
-                reference.isLeafObject
+            val heapObject = graph.findObjectById(objectId)
+            val refs = objectReferenceReader.read(heapObject)
+            refs.mapNotNull { reference ->
+              // dominatorTree is updated prior to enqueueing, because that's where we have the
+              // parent object id information. visitedSet is updated on dequeuing, because bumping
+              // node priority would be complex when as we'd need to move object ids between nodes
+              // rather than just move nodes.
+              dominatorTree?.updateDominated(
+                objectId = reference.valueObjectId,
+                parentObjectId = objectId
               )
+              if (reference.valueObjectId in visitedSet) {
+                null
+              } else {
+                val details = reference.lazyDetailsResolver.resolve()
+                val refType = details.locationType.name
+                val owningClassSimpleName =
+                  graph.findObjectById(details.locationClassObjectId).asClass!!.simpleName
+                val refName = if (details.locationType == ARRAY_ENTRY) "[x]" else details.name
+                val referencedObjectName =
+                  when (val referencedObject = graph.findObjectById(reference.valueObjectId)) {
+                    is HeapClass -> "class ${referencedObject.name}"
+                    is HeapInstance -> "instance of ${referencedObject.instanceClassName}"
+                    is HeapObjectArray -> "array of ${referencedObject.arrayClassName}"
+                    is HeapPrimitiveArray -> "array of ${referencedObject.primitiveType.name.lowercase()}"
+                  }
+                val nodeAndEdgeName =
+                  "$refType ${owningClassSimpleName}.${refName} -> $referencedObjectName"
+                ExpandedObject(
+                  reference.valueObjectId, nodeAndEdgeName, reference.isLowPriority,
+                  reference.isLeafObject
+                )
+              }
             }
           }
-          }
         }
       }.groupBy {
         it.nodeAndEdgeName + if (it.isLowPriority) "low-priority" else ""
@@ -183,7 +210,8 @@ class HeapGraphObjectGrowthDetector(
     }
 
     val growingNodes = if (previousTree != null) {
-      nodesMaybeGrowing.mapNotNull { node ->
+      val growingNodePairs = mutableListOf<Pair<Node, ShortestPathObjectNode>>()
+      val growingNodes = nodesMaybeGrowing.mapNotNull { node ->
         val shortestPathNode = node.shortestPathNode
         val growing = if (node.previousPathNode != null) {
           // Existing node. Growing if was growing (already true) and edges increased at least detectedGrowth.
@@ -216,14 +244,56 @@ class HeapGraphObjectGrowthDetector(
               child.selfObjectCountIncrease = child.selfObjectCount
             }
           }
-          // Mark as growing in the tree(useful for next iteration)
+          // Mark as growing in the tree (useful for next iteration)
           shortestPathNode.growing = true
+
+          val previouslyGrowing = !shortestPathNode.newNode
+          val parentAlreadyReported = (shortestPathNode.parent?.growing) ?: false
+
+          val repeatedlyGrowingNode = previouslyGrowing && !parentAlreadyReported
           // Return in list of growing nodes.
-          shortestPathNode
+          if (repeatedlyGrowingNode) {
+            if (dominatorTree != null) {
+              growingNodePairs += node to shortestPathNode
+            }
+            shortestPathNode
+          } else {
+            null
+          }
         } else {
           null
         }
       }
+      dominatorTree?.let { dominatorTree ->
+        val growingNodeObjectIds = growingNodePairs.flatMapTo(LinkedHashSet()) { (node, _) ->
+          node.objectIds
+        }
+        val objectSizeCalculator = AndroidObjectSizeCalculator(graph)
+        val retainedMap =
+          dominatorTree.computeRetainedSizes(growingNodeObjectIds, objectSizeCalculator)
+        growingNodePairs.forEach { (node, shortestPathNode) ->
+          var heapSize = ByteSize.ZERO
+          var objectCount = 0
+          for (objectId in node.objectIds) {
+            val (additionalByteSize, additionalObjectCount) = retainedMap.getValue(objectId)
+            heapSize += additionalByteSize.bytes
+            objectCount += additionalObjectCount
+          }
+          shortestPathNode.retainedOrNull = Retained(
+            heapSize = heapSize,
+            objectCount = objectCount
+          )
+          val previousRetained = node.previousPathNode?.retainedOrNull
+          shortestPathNode.retainedIncreaseOrNull = if (previousRetained == null) {
+            Retained(ByteSize.ZERO, 0)
+          } else {
+            Retained(
+              heapSize - previousRetained.heapSize, objectCount - previousRetained.objectCount
+            )
+          }
+        }
+      }
+      growingNodes
     } else {
       null
     }
@@ -233,13 +303,7 @@ class HeapGraphObjectGrowthDetector(
       InitialHeapTraversal(tree)
     } else {
       check(previousTraversal !is NoHeapTraversalYet)
-      val repeatedlyGrowingNodes =
-        growingNodes.filter {
-          val previouslyGrowing = !it.newNode
-          val parentAlreadyReported = (it.parent?.growing) ?: false
-          previouslyGrowing && !parentAlreadyReported
-        }
-      HeapTraversalWithDiff(tree, repeatedlyGrowingNodes)
+      HeapTraversalWithDiff(tree, growingNodes)
     }
   }
 
@@ -276,10 +340,17 @@ class HeapGraphObjectGrowthDetector(
       } else {
         null
       }
+      val objectIds = gcRootReferences.map { it.second.gcRoot.id }
+      dominatorTree?.let {
+        objectIds.forEach { objectId ->
+          it.updateDominatedAsRoot(objectId)
+        }
+      }
+
       enqueue(
         parentPathNode = tree,
         previousPathNode = previousPathNode,
-        objectIds = gcRootReferences.map { it.second.gcRoot.id },
+        objectIds = objectIds,
         nodeAndEdgeName = nodeAndEdgeName,
         isLowPriority = firstOfGroup.second.isLowPriority,
         isLeafObject = false
@@ -296,6 +367,7 @@ class HeapGraphObjectGrowthDetector(
     isLeafObject: Boolean
   ) {
     // TODO Maybe the filtering should happen at the callsite.
+    // TODO we already filter visited on the traversal side. maybe crash?
     val filteredObjectIds = objectIds.filter { objectId ->
       objectId != ValueHolder.NULL_REFERENCE &&
         // note: we only update visitedSet once dequeued. This could lead

shark/shark/src/main/java/shark/ObjectDominators.kt

@@ -6,7 +6,6 @@ import shark.HeapObject.HeapClass
 import shark.HeapObject.HeapInstance
 import shark.HeapObject.HeapObjectArray
 import shark.HeapObject.HeapPrimitiveArray
-import shark.internal.ShallowSizeCalculator
 
 /**
  * Exposes high level APIs to compute and render a dominator tree. This class
@@ -169,15 +168,9 @@ class ObjectDominators {
         computeRetainedHeapSize = true,
     ).createFor(graph)
 
-    val nativeSizeMapper = AndroidNativeSizeMapper(graph)
-    val nativeSizes = nativeSizeMapper.mapNativeSizes()
-    val shallowSizeCalculator = ShallowSizeCalculator(graph)
+    val objectSizeCalculator = AndroidObjectSizeCalculator(graph)
 
     val result = pathFinder.findShortestPathsFromGcRoots(setOf())
-    return result.dominatorTree!!.buildFullDominatorTree { objectId ->
-      val nativeSize = nativeSizes[objectId] ?: 0
-      val shallowSize = shallowSizeCalculator.computeShallowSize(objectId)
-      nativeSize + shallowSize
-    }
+    return result.dominatorTree!!.buildFullDominatorTree(objectSizeCalculator)
   }
 }

shark/shark/src/main/java/shark/RealLeakTracerFactory.kt

@@ -57,6 +57,7 @@ class RealLeakTracerFactory constructor(
   sealed interface Event {
     object StartedBuildingLeakTraces : Event
     object StartedInspectingObjects : Event
+    @Deprecated("Event not sent anymore")
     object StartedComputingNativeRetainedSize: Event
     object StartedComputingJavaHeapRetainedSize: Event
 
@@ -350,16 +351,9 @@ class RealLeakTracerFactory constructor(
       inspectedObjects.filter { it.leakingStatus == UNKNOWN || it.leakingStatus == LEAKING }
         .map { it.heapObject.objectId }
     }.toSet()
-    listener.onEvent(StartedComputingNativeRetainedSize)
-    val nativeSizeMapper = AndroidNativeSizeMapper(graph)
-    val nativeSizes = nativeSizeMapper.mapNativeSizes()
     listener.onEvent(StartedComputingJavaHeapRetainedSize)
-    val shallowSizeCalculator = ShallowSizeCalculator(graph)
-    return dominatorTree.computeRetainedSizes(nodeObjectIds) { objectId ->
-      val nativeSize = nativeSizes[objectId] ?: 0
-      val shallowSize = shallowSizeCalculator.computeShallowSize(objectId)
-      nativeSize + shallowSize
-    }
+    val objectSizeCalculator = AndroidObjectSizeCalculator(graph)
+    return dominatorTree.computeRetainedSizes(nodeObjectIds, objectSizeCalculator)
   }
 
   private fun buildLeakTraceObjects(