Allow Stressors to implement their own Stats

Summary:
The current stats implementation in cachebench assumes the old model where you have a CacheAllocator that has a RAM and flash cache.  This doesn't work for CacheComponent (which explicitly breaks these apart) and trying to force CacheComponents into the existing stats reporting is too cumbersome.

Instead, make this interface generic instead to allow stressors to implement their own stats collection & reporting.  Follow-on diffs will implement CacheComponent stats reporting.

Reviewed By: AlnisM

Differential Revision: D92316026

fbshipit-source-id: 55fe502f07e05ef343b0cef86b2b8365806e3568

Author: rlyerly

cachelib/cachebench/cache/Cache.h

@@ -322,7 +322,7 @@ class Cache {
   }
 
   // Get overall stats on the whole cache allocator
-  Stats getStats() const;
+  std::unique_ptr<StatsBase> getStats() const;
 
   // Get number of bytes written to NVM.
   double getNvmBytesWritten() const;
@@ -1114,12 +1114,14 @@ double Cache<Allocator>::getNvmBytesWritten() const {
 }
 
 template <typename Allocator>
-Stats Cache<Allocator>::getStats() const {
+std::unique_ptr<StatsBase> Cache<Allocator>::getStats() const {
+  auto retPtr = std::make_unique<Stats>();
+  auto& ret = *retPtr;
+
   PoolStats aggregate = cache_->getPoolStats(pools_[0]);
   auto usageFraction =
       1.0 - (static_cast<double>(aggregate.freeMemoryBytes())) /
                 aggregate.poolUsableSize;
-  Stats ret;
   ret.poolUsageFraction.push_back(usageFraction);
   for (size_t pid = 1; pid < pools_.size(); pid++) {
     auto poolStats = cache_->getPoolStats(static_cast<PoolId>(pid));
@@ -1292,7 +1294,7 @@ Stats Cache<Allocator>::getStats() const {
     }
   }
 
-  return ret;
+  return retPtr;
 }
 
 template <typename Allocator>

cachelib/cachebench/cache/CacheStats.h

@@ -20,6 +20,7 @@
 
 #include "cachelib/allocator/memory/MemoryAllocatorStats.h"
 #include "cachelib/allocator/memory/Slab.h"
+#include "cachelib/cachebench/cache/StatsBase.h"
 #include "cachelib/common/PercentileStats.h"
 
 DECLARE_bool(report_api_latency);
@@ -51,7 +52,8 @@ struct BackgroundPromotionStats {
   uint64_t nTraversals{0};
 };
 
-struct Stats {
+class Stats : public StatsBase {
+ public:
   BackgroundEvictionStats backgndEvicStats;
   BackgroundPromotionStats backgndPromoStats;
 
@@ -169,7 +171,10 @@ struct Stats {
 
   // Aggregate throughput stats from another instance. DOES NOT HANDLE
   // LATENCY STATS!
-  Stats& operator+=(const Stats& other) {
+  Stats& operator+=(const StatsBase& otherBase) override {
+    aggregated_ = true;
+    auto& other = otherBase.as<Stats>();
+
     backgndEvicStats.nEvictedItems += other.backgndEvicStats.nEvictedItems;
     backgndEvicStats.nTraversals += other.backgndEvicStats.nTraversals;
     backgndEvicStats.nClasses += other.backgndEvicStats.nClasses;
@@ -241,7 +246,21 @@ struct Stats {
     return *this;
   }
 
-  void render(std::ostream& out, bool isAggregate = false) const {
+  std::string progress(const StatsBase& prevStatsBase) const override {
+    const auto& prevStats = prevStatsBase.as<Stats>();
+    auto hitRates = getHitRatios(prevStats);
+    return folly::sformat(
+        "{} items in cache. {} items in nvm cache. {} items evicted from nvm "
+        "cache. Hit Ratio {:6.2f}% (RAM {:6.2f}%, NVM {:6.2f}%).",
+        numItems,
+        numNvmItems,
+        numNvmEvictions,
+        hitRates["overall"],
+        hitRates["ram"],
+        hitRates["nvm"]);
+  }
+
+  void render(std::ostream& out) const override {
     auto totalMisses = getTotalMisses();
     const double overallHitRatio = invertPctFn(totalMisses, numCacheGets);
     out << folly::sformat("Items in RAM  : {:,}", numItems) << std::endl;
@@ -265,7 +284,7 @@ struct Stats {
       }
     };
 
-    if (!isAggregate) {
+    if (!aggregated_) {
       for (auto pid = 0U; pid < poolUsageFraction.size(); pid++) {
         out << folly::sformat("Fraction of pool {:,} used : {:.2f}", pid,
                               poolUsageFraction[pid])
@@ -334,7 +353,7 @@ struct Stats {
       out << folly::sformat("Hit Ratio     : {:6.2f}%", overallHitRatio)
           << std::endl;
 
-      if (FLAGS_report_api_latency && !isAggregate) {
+      if (FLAGS_report_api_latency && !aggregated_) {
         auto printLatencies =
             [&out](folly::StringPiece cat,
                    const util::PercentileStats::Estimates& latency) {
@@ -362,7 +381,7 @@ struct Stats {
     if (!backgroundEvictionClasses.empty() &&
         backgndEvicStats.nEvictedItems > 0) {
       out << "== Class Background Eviction Counters Map ==" << std::endl;
-      if (!isAggregate) {
+      if (!aggregated_) {
         foreachAC(backgroundEvictionClasses,
                   [&](auto pid, auto cid, auto evicted) {
                     out << folly::sformat("pid{:2} cid{:4} evicted: {:4}", pid,
@@ -382,7 +401,7 @@ struct Stats {
     if (!backgroundPromotionClasses.empty() &&
         backgndPromoStats.nPromotedItems > 0) {
       out << "== Class Background Promotion Counters Map ==" << std::endl;
-      if (!isAggregate) {
+      if (!aggregated_) {
         foreachAC(backgroundPromotionClasses,
                   [&](auto pid, auto cid, auto promoted) {
                     out << folly::sformat("pid{:2} cid{:4} promoted: {:4}", pid,
@@ -413,7 +432,7 @@ struct Stats {
           "{:,}\n",
           numNvmRejectsByExpiry, numNvmRejectsByClean);
 
-      if (!isAggregate) {
+      if (!aggregated_) {
         folly::StringPiece readCat = "NVM Read  Latency";
         folly::StringPiece writeCat = "NVM Write Latency";
         auto fmtLatency = [&](folly::StringPiece cat, folly::StringPiece pct,
@@ -575,51 +594,55 @@ struct Stats {
     return numNvmGets > 0 ? numNvmGetMiss : numCacheGetMiss;
   }
 
-  std::tuple<double, double, double> getHitRatios(
-      const Stats& prevStats) const {
-    double overallHitRatio = 0.0;
-    double ramHitRatio = 0.0;
-    double nvmHitRatio = 0.0;
+  std::map<std::string, double> getHitRatios(
+      const StatsBase& prevStatsBase) const override {
+    auto& prevStats = prevStatsBase.as<Stats>();
+    std::map<std::string, double> rates;
+    rates["overall"] = 0.0;
+    rates["ram"] = 0.0;
+    rates["nvm"] = 0.0;
 
     if (numCacheGets > prevStats.numCacheGets) {
       auto totalMisses = getTotalMisses();
       auto prevTotalMisses = prevStats.getTotalMisses();
 
-      overallHitRatio = invertPctFn(totalMisses - prevTotalMisses,
-                                    numCacheGets - prevStats.numCacheGets);
+      rates["overall"] = invertPctFn(totalMisses - prevTotalMisses,
+                                     numCacheGets - prevStats.numCacheGets);
 
-      ramHitRatio = invertPctFn(numCacheGetMiss - prevStats.numCacheGetMiss,
-                                numCacheGets - prevStats.numCacheGets);
+      rates["ram"] = invertPctFn(numCacheGetMiss - prevStats.numCacheGetMiss,
+                                 numCacheGets - prevStats.numCacheGets);
     }
 
     if (numNvmGets > prevStats.numNvmGets) {
-      nvmHitRatio = invertPctFn(numNvmGetMiss - prevStats.numNvmGetMiss,
-                                numNvmGets - prevStats.numNvmGets);
+      rates["nvm"] = invertPctFn(numNvmGetMiss - prevStats.numNvmGetMiss,
+                                 numNvmGets - prevStats.numNvmGets);
     }
 
-    return std::make_tuple(overallHitRatio, ramHitRatio, nvmHitRatio);
+    return rates;
   }
 
   // Render the stats based on the delta between overall stats and previous
   // stats. It can be used to render the stats in the last time period.
-  void render(const Stats& prevStats, std::ostream& out) const {
+  void render(const StatsBase& prevStatsBase,
+              std::ostream& out) const override {
+    auto& prevStats = prevStatsBase.as<Stats>();
+
     if (numCacheGets > prevStats.numCacheGets) {
-      auto [overallHitRatio, ramHitRatio, nvmHitRatio] =
-          getHitRatios(prevStats);
+      auto rates = getHitRatios(prevStatsBase);
       out << folly::sformat("Cache Gets    : {:,}",
                             numCacheGets - prevStats.numCacheGets)
           << std::endl;
-      out << folly::sformat("Hit Ratio     : {:6.2f}%", overallHitRatio)
+      out << folly::sformat("Hit Ratio     : {:6.2f}%", rates["overall"])
           << std::endl;
 
       out << folly::sformat(
           "RAM Hit Ratio : {:6.2f}%\n"
           "NVM Hit Ratio : {:6.2f}%\n",
-          ramHitRatio, nvmHitRatio);
+          rates["ram"], rates["nvm"]);
     }
   }
 
-  void render(folly::UserCounters& counters) {
+  void render(folly::UserCounters& counters) const override {
     auto calcInvertPctFn = [](uint64_t ops, uint64_t total) {
       return static_cast<int64_t>(invertPctFn(ops, total) * 100);
     };
@@ -657,7 +680,7 @@ struct Stats {
     counters["nvm_dev_write_amp"] = static_cast<int64_t>(devWriteAmp);
   }
 
-  bool renderIsTestPassed(std::ostream& out) {
+  bool renderIsTestPassed(std::ostream& out) const override {
     bool pass = true;
 
     if (isNvmCacheDisabled) {
@@ -697,6 +720,8 @@ struct Stats {
   }
 
  private:
+  bool aggregated_{false};
+
   static double pctFn(uint64_t ops, uint64_t total) {
     return total == 0
                ? 0
@@ -706,8 +731,7 @@ struct Stats {
   static double invertPctFn(uint64_t ops, uint64_t total) {
     return 100 - pctFn(ops, total);
   }
-
-}; // namespace cachebench
+};
 
 } // namespace cachebench
 } // namespace cachelib

cachelib/cachebench/cache/StatsBase.h

@@ -0,0 +1,116 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include <folly/Benchmark.h>
+#include <folly/logging/xlog.h>
+
+#include <map>
+#include <ostream>
+#include <string>
+
+namespace facebook {
+namespace cachelib {
+namespace cachebench {
+
+/**
+ * Interface for cache statistics. Different stressors can use different stats
+ * implementations while providing a common interface for rendering and
+ * calculating hit rates.
+ */
+class StatsBase {
+ public:
+  virtual ~StatsBase() = default;
+
+  /**
+   * Add the stats from another stats object to this one. It should only ever be
+   * called with two of the same sub-types, i.e., you don't have to deal with
+   * trying to aggregate child class A and child class B.
+   *
+   * @param otherBase other stats object
+   * @return reference to this
+   */
+  virtual StatsBase& operator+=(const StatsBase& otherBase) = 0;
+
+  /**
+   * Get a string with some brief summary statistics for printing progress.
+   * @return progress string
+   */
+  virtual std::string progress(const StatsBase& prevStats) const = 0;
+
+  /**
+   * Render the current stats to the output stream.
+   * @param out The output stream to write to.
+   */
+  virtual void render(std::ostream& out) const = 0;
+
+  /**
+   * Render stats as a delta from previous stats to show changes over time.
+   * @param prevStats The previous stats snapshot to calculate deltas from.
+   * @param out The output stream to write to.
+   */
+  virtual void render(const StatsBase& prevStats, std::ostream& out) const = 0;
+
+  /**
+   * Render stats to folly::UserCounters for benchmark integration.
+   * @param counters The counter map to populate.
+   */
+  virtual void render(folly::UserCounters& counters) const = 0;
+
+  /**
+   * Calculate hit rates as deltas from previous stats.
+   * @param prevStats The previous stats snapshot to calculate deltas from.
+   * @return Map of hit rate metric names to their delta values (0.0 - 100.0).
+   */
+  virtual std::map<std::string, double> getHitRatios(
+      const StatsBase& prevStats) const = 0;
+
+  /**
+   * Render test pass/fail status and return whether the test passed.
+   * @param out The output stream to write status messages to.
+   * @return true if the test passed, false otherwise.
+   */
+  virtual bool renderIsTestPassed(std::ostream& out) const = 0;
+
+  /**
+   * Helper to cast a StatsBase to a child. Returns a nullptr if not casting to
+   * the correct type.
+   *
+   * @return this cast to a child type pointer
+   */
+  template <typename ChildT>
+  ChildT* asPtr() const {
+    return dynamic_cast<ChildT*>(this);
+  }
+
+  /**
+   * Helper to cast a StatsBase to a child. Will crash if not casting to the
+   * correct type.
+   *
+   * @return this cast to a child type reference
+   */
+  template <typename ChildT>
+  const ChildT& as() const {
+    auto* ret = asPtr<const ChildT>();
+    XCHECK_NE(ret, nullptr);
+    return *ret;
+  }
+};
+
+} // namespace cachebench
+} // namespace cachelib
+} // namespace facebook

cachelib/cachebench/main.cpp

@@ -23,6 +23,7 @@
 #include <thread>
 #include <vector>
 
+#include "cachelib/cachebench/cache/Cache.h"
 #include "cachelib/cachebench/runner/Runner.h"
 #include "cachelib/cachebench/util/AggregateStats.h"
 #include "cachelib/cachebench/util/Sleep.h"

cachelib/cachebench/runner/AsyncCacheStressor.h

@@ -171,7 +171,9 @@ class AsyncCacheStressor : public Stressor {
   }
 
   // obtain stats from the cache instance.
-  Stats getCacheStats() const override { return cache_->getStats(); }
+  std::unique_ptr<StatsBase> getCacheStats() const override {
+    return cache_->getStats();
+  }
 
   // obtain aggregated throughput stats for the stress run so far.
   ThroughputStats aggregateThroughputStats() const override {