Executor cache metrics (backport #22332)

runtime/src/bank.rs

@@ -242,6 +242,53 @@ pub struct SquashTiming {
 
 type EpochCount = u64;
 
+mod executor_cache {
+    use super::*;
+    use log;
+
+    #[derive(Debug, Default)]
+    pub struct Stats {
+        pub hits: u64,
+        pub misses: u64,
+        pub evictions: HashMap<Pubkey, u64>,
+    }
+
+    impl Stats {
+        pub fn submit(&self, slot: Slot) {
+            let evictions: u64 = self.evictions.values().sum();
+            datapoint_info!(
+                "bank-executor-cache-stats",
+                ("slot", slot, i64),
+                ("hits", self.hits, i64),
+                ("misses", self.misses, i64),
+                ("evictions", evictions, i64),
+            );
+            debug!(
+                "Executor Cache Stats -- Hits: {}, Misses: {}, Evictions: {}",
+                self.hits, self.misses, evictions
+            );
+            if log_enabled!(log::Level::Trace) && !self.evictions.is_empty() {
+                let mut evictions = self.evictions.iter().collect::<Vec<_>>();
+                evictions.sort_by_key(|e| e.1);
+                let evictions = evictions
+                    .into_iter()
+                    .rev()
+                    .map(|(program_id, evictions)| {
+                        format!("  {:<44}  {}", program_id.to_string(), evictions)
+                    })
+                    .collect::<Vec<_>>();
+                let evictions = evictions.join("\n");
+                trace!(
+                    "Eviction Details:\n  {:<44}  {}\n{}",
+                    "Program",
+                    "Count",
+                    evictions
+                );
+            }
+        }
+    }
+}
+
 const MAX_CACHED_EXECUTORS: usize = 100; // 10 MB assuming programs are around 100k
 #[derive(Debug)]
 struct CachedExecutorsEntry {
@@ -255,13 +302,15 @@ struct CachedExecutors {
     max: usize,
     current_epoch: Epoch,
     executors: HashMap<Pubkey, CachedExecutorsEntry>,
+    stats: executor_cache::Stats,
 }
 impl Default for CachedExecutors {
     fn default() -> Self {
         Self {
             max: MAX_CACHED_EXECUTORS,
             current_epoch: Epoch::default(),
             executors: HashMap::default(),
+            stats: executor_cache::Stats::default(),
         }
     }
 }
@@ -290,6 +339,7 @@ impl Clone for CachedExecutors {
             max: self.max,
             current_epoch: self.current_epoch,
             executors: executors.collect(),
+            stats: executor_cache::Stats::default(),
         }
     }
 }
@@ -313,6 +363,7 @@ impl CachedExecutors {
             max: self.max,
             current_epoch: epoch,
             executors: executors.collect(),
+            stats: executor_cache::Stats::default(),
         })
     }
 
@@ -321,6 +372,7 @@ impl CachedExecutors {
             max,
             current_epoch,
             executors: HashMap::new(),
+            stats: executor_cache::Stats::default(),
         }
     }
 
@@ -333,9 +385,11 @@ impl CachedExecutors {
 
     fn put(&mut self, pubkey: &Pubkey, executor: Arc<dyn Executor>) {
         let entry = if let Some(mut entry) = self.executors.remove(pubkey) {
+            saturating_add_assign!(self.stats.hits, 1);
             entry.executor = executor;
             entry
         } else {
+            saturating_add_assign!(self.stats.misses, 1);
             if self.executors.len() >= self.max {
                 let mut least = u64::MAX;
                 let default_key = Pubkey::default();
@@ -350,6 +404,11 @@ impl CachedExecutors {
                 }
                 let least_key = *least_key;
                 let _ = self.executors.remove(&least_key);
+                self.stats
+                    .evictions
+                    .entry(least_key)
+                    .and_modify(|c| saturating_add_assign!(*c, 1))
+                    .or_insert(1);
             }
             CachedExecutorsEntry {
                 prev_epoch_count: 0,
@@ -1507,6 +1566,13 @@ impl Bank {
             ("time_us", time.as_us(), i64),
         );
 
+        parent
+            .cached_executors
+            .read()
+            .unwrap()
+            .stats
+            .submit(parent.slot());
+
         new
     }