From 71213506ee0dd109390b01c05b4eb4fe90eb9bcc Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Sat, 5 Nov 2022 11:36:45 -0700
Subject: [PATCH 001/468] Fix transaction_too_old error when version vector is
 enabled

When VV is enabled, the comparison of storage server version and read version
should use the original read version, otherwise, the client may get the wrong
transaction_too_old error.
---
 fdbserver/storageserver.actor.cpp | 40 +++++++++++++++++++++++++++----
 1 file changed, 36 insertions(+), 4 deletions(-)

diff --git a/fdbserver/storageserver.actor.cpp b/fdbserver/storageserver.actor.cpp
index edda4be1901..5aa75b5eee6 100644
--- a/fdbserver/storageserver.actor.cpp
+++ b/fdbserver/storageserver.actor.cpp
@@ -2920,8 +2920,10 @@ ACTOR Future<GetValueReqAndResultRef> quickGetValue(StorageServer* data,
 
 // If limit>=0, it returns the first rows in the range (sorted ascending), otherwise the last rows (sorted descending).
 // readRange has O(|result|) + O(log |data|) cost
+// origVersion: original read version when version vector is enabled
 ACTOR Future<GetKeyValuesReply> readRange(StorageServer* data,
                                           Version version,
+                                          Version origVersion,
                                           KeyRange range,
                                           int limit,
                                           int* pLimitBytes,
@@ -3006,8 +3008,15 @@ ACTOR Future<GetKeyValuesReply> readRange(StorageServer* data,
 			data->counters.kvScanBytes += atStorageVersion.logicalSize();
 
 			ASSERT(atStorageVersion.size() <= limit);
-			if (data->storageVersion() > version)
+			if ((!SERVER_KNOBS->ENABLE_VERSION_VECTOR && data->storageVersion() > version) ||
+			    (SERVER_KNOBS->ENABLE_VERSION_VECTOR && data->storageVersion() > origVersion)) {
+				DisabledTraceEvent("SS_TTO", data->thisServerID)
+				    .detail("StorageVersion", data->storageVersion())
+				    .detail("Version", version)
+				    .detail("OrigVersion", origVersion)
+				    .detail("Range", range);
 				throw transaction_too_old();
+			}
 
 			// merge the sets in resultCache with the sets on disk, stopping at the last key from disk if there is
 			// 'more'
@@ -3099,8 +3108,15 @@ ACTOR Future<GetKeyValuesReply> readRange(StorageServer* data,
 			data->counters.kvScanBytes += atStorageVersion.logicalSize();
 
 			ASSERT(atStorageVersion.size() <= -limit);
-			if (data->storageVersion() > version)
+			if ((!SERVER_KNOBS->ENABLE_VERSION_VECTOR && data->storageVersion() > version) ||
+			    (SERVER_KNOBS->ENABLE_VERSION_VECTOR && data->storageVersion() > origVersion)) {
+				DisabledTraceEvent("SS_TTO", data->thisServerID)
+				    .detail("StorageVersion", data->storageVersion())
+				    .detail("Version", version)
+				    .detail("OrigVersion", origVersion)
+				    .detail("Range", range);
 				throw transaction_too_old();
+			}
 
 			int prevSize = result.data.size();
 			merge(result.arena,
@@ -3199,6 +3215,7 @@ ACTOR Future<Key> findKey(StorageServer* data,
 
 	state GetKeyValuesReply rep = wait(
 	    readRange(data,
+	              version,
 	              version,
 	              forward ? KeyRangeRef(sel.getKey(), range.end) : KeyRangeRef(range.begin, keyAfter(sel.getKey())),
 	              (distance + skipEqualKey) * sign,
@@ -3214,6 +3231,7 @@ ACTOR Future<Key> findKey(StorageServer* data,
 		TEST(true); // Reverse key selector returned only one result in range read
 		maxBytes = std::numeric_limits<int>::max();
 		GetKeyValuesReply rep2 = wait(readRange(data,
+		                                        version,
 		                                        version,
 		                                        KeyRangeRef(range.begin, keyAfter(sel.getKey())),
 		                                        -2,
@@ -3318,6 +3336,11 @@ ACTOR Future<Void> getKeyValuesQ(StorageServer* data, GetKeyValuesRequest req)
 
 		Version commitVersion = getLatestCommitVersion(req.ssLatestCommitVersions, data->tag);
 		state Version version = wait(waitForVersion(data, commitVersion, req.version, span.context));
+		DisabledTraceEvent("VVV", data->thisServerID)
+		    .detail("Version", version)
+		    .detail("ReqVersion", req.version)
+		    .detail("VV", req.ssLatestCommitVersions.toString())
+		    .detail("DebugID", req.debugID.present() ? req.debugID.get() : UID());
 		data->counters.readVersionWaitSample.addMeasurement(g_network->timer() - queueWaitEnd);
 
 		state Optional<TenantMapEntry> tenantEntry = data->getTenantEntry(version, req.tenantInfo);
@@ -3398,6 +3421,7 @@ ACTOR Future<Void> getKeyValuesQ(StorageServer* data, GetKeyValuesRequest req)
 			state double kvReadRange = g_network->timer();
 			GetKeyValuesReply _r = wait(readRange(data,
 			                                      version,
+			                                      req.version,
 			                                      KeyRangeRef(begin, end),
 			                                      req.limit,
 			                                      &remainingLimitBytes,
@@ -4029,6 +4053,7 @@ ACTOR Future<Void> getMappedKeyValuesQ(StorageServer* data, GetMappedKeyValuesRe
 
 			GetKeyValuesReply getKeyValuesReply = wait(readRange(data,
 			                                                     version,
+			                                                     req.version,
 			                                                     KeyRangeRef(begin, end),
 			                                                     req.limit,
 			                                                     &remainingLimitBytes,
@@ -4232,8 +4257,15 @@ ACTOR Future<Void> getKeyValuesStreamQ(StorageServer* data, GetKeyValuesStreamRe
 				                       !data->isTss() && !data->isSSWithTSSPair())
 				                          ? 1
 				                          : CLIENT_KNOBS->REPLY_BYTE_LIMIT;
-				GetKeyValuesReply _r = wait(readRange(
-				    data, version, KeyRangeRef(begin, end), req.limit, &byteLimit, span.context, type, tenantPrefix));
+				GetKeyValuesReply _r = wait(readRange(data,
+				                                      version,
+				                                      req.version,
+				                                      KeyRangeRef(begin, end),
+				                                      req.limit,
+				                                      &byteLimit,
+				                                      span.context,
+				                                      type,
+				                                      tenantPrefix));
 				GetKeyValuesStreamReply r(_r);
 
 				if (req.debugID.present())

From 3371992e07e5197e77d157f9eb875d4a4f43f52f Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Sat, 5 Nov 2022 12:02:50 -0700
Subject: [PATCH 002/468] Fix assertions w.r.t. VV

---
 fdbserver/storageserver.actor.cpp | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/fdbserver/storageserver.actor.cpp b/fdbserver/storageserver.actor.cpp
index 5aa75b5eee6..bafeee51da1 100644
--- a/fdbserver/storageserver.actor.cpp
+++ b/fdbserver/storageserver.actor.cpp
@@ -2978,7 +2978,8 @@ ACTOR Future<GetKeyValuesReply> readRange(StorageServer* data,
 
 		while (limit > 0 && *pLimitBytes > 0 && readBegin < range.end) {
 			ASSERT(!vCurrent || vCurrent.key() >= readBegin);
-			ASSERT(data->storageVersion() <= version);
+			ASSERT((!SERVER_KNOBS->ENABLE_VERSION_VECTOR && data->storageVersion() <= version) ||
+			       (SERVER_KNOBS->ENABLE_VERSION_VECTOR && data->storageVersion() <= origVersion));
 
 			/* Traverse the PTree further, if thare are no unconsumed resultCache items */
 			if (pos == resultCache.size()) {
@@ -3078,7 +3079,8 @@ ACTOR Future<GetKeyValuesReply> readRange(StorageServer* data,
 
 		while (limit < 0 && *pLimitBytes > 0 && readEnd > range.begin) {
 			ASSERT(!vCurrent || vCurrent.key() < readEnd);
-			ASSERT(data->storageVersion() <= version);
+			ASSERT((!SERVER_KNOBS->ENABLE_VERSION_VECTOR && data->storageVersion() <= version) ||
+			       (SERVER_KNOBS->ENABLE_VERSION_VECTOR && data->storageVersion() <= origVersion));
 
 			/* Traverse the PTree further, if thare are no unconsumed resultCache items */
 			if (pos == resultCache.size()) {

From c3f25a2edfbb90896b69dec5d88976132e13e66f Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Mon, 7 Nov 2022 14:29:21 -0800
Subject: [PATCH 003/468] Avoid using oldest version as read version for VV

---
 fdbserver/storageserver.actor.cpp             | 44 +++++++------------
 .../workloads/ConsistencyCheck.actor.cpp      |  2 +
 2 files changed, 19 insertions(+), 27 deletions(-)

diff --git a/fdbserver/storageserver.actor.cpp b/fdbserver/storageserver.actor.cpp
index bafeee51da1..91477d05d11 100644
--- a/fdbserver/storageserver.actor.cpp
+++ b/fdbserver/storageserver.actor.cpp
@@ -1521,7 +1521,14 @@ Future<Version> waitForVersion(StorageServer* data, Version commitVersion, Versi
 		return transaction_too_old();
 	} else {
 		if (commitVersion < data->oldestVersion.get()) {
-			return data->oldestVersion.get();
+			if (data->version.get() < readVersion) {
+				// Majority of the case, try using higher version to avoid
+				// transaction_too_old error when oldestVersion advances.
+				return data->version.get(); // majority of cases
+			} else {
+				ASSERT(readVersion >= data->oldestVersion.get());
+				return readVersion;
+			}
 		} else if (commitVersion <= data->version.get()) {
 			return commitVersion;
 		}
@@ -2920,10 +2927,8 @@ ACTOR Future<GetValueReqAndResultRef> quickGetValue(StorageServer* data,
 
 // If limit>=0, it returns the first rows in the range (sorted ascending), otherwise the last rows (sorted descending).
 // readRange has O(|result|) + O(log |data|) cost
-// origVersion: original read version when version vector is enabled
 ACTOR Future<GetKeyValuesReply> readRange(StorageServer* data,
                                           Version version,
-                                          Version origVersion,
                                           KeyRange range,
                                           int limit,
                                           int* pLimitBytes,
@@ -2978,8 +2983,7 @@ ACTOR Future<GetKeyValuesReply> readRange(StorageServer* data,
 
 		while (limit > 0 && *pLimitBytes > 0 && readBegin < range.end) {
 			ASSERT(!vCurrent || vCurrent.key() >= readBegin);
-			ASSERT((!SERVER_KNOBS->ENABLE_VERSION_VECTOR && data->storageVersion() <= version) ||
-			       (SERVER_KNOBS->ENABLE_VERSION_VECTOR && data->storageVersion() <= origVersion));
+			ASSERT(data->storageVersion() <= version);
 
 			/* Traverse the PTree further, if thare are no unconsumed resultCache items */
 			if (pos == resultCache.size()) {
@@ -3009,12 +3013,11 @@ ACTOR Future<GetKeyValuesReply> readRange(StorageServer* data,
 			data->counters.kvScanBytes += atStorageVersion.logicalSize();
 
 			ASSERT(atStorageVersion.size() <= limit);
-			if ((!SERVER_KNOBS->ENABLE_VERSION_VECTOR && data->storageVersion() > version) ||
-			    (SERVER_KNOBS->ENABLE_VERSION_VECTOR && data->storageVersion() > origVersion)) {
+			if (data->storageVersion() > version) {
 				DisabledTraceEvent("SS_TTO", data->thisServerID)
 				    .detail("StorageVersion", data->storageVersion())
+				    .detail("Oldest", data->oldestVersion.get())
 				    .detail("Version", version)
-				    .detail("OrigVersion", origVersion)
 				    .detail("Range", range);
 				throw transaction_too_old();
 			}
@@ -3079,8 +3082,7 @@ ACTOR Future<GetKeyValuesReply> readRange(StorageServer* data,
 
 		while (limit < 0 && *pLimitBytes > 0 && readEnd > range.begin) {
 			ASSERT(!vCurrent || vCurrent.key() < readEnd);
-			ASSERT((!SERVER_KNOBS->ENABLE_VERSION_VECTOR && data->storageVersion() <= version) ||
-			       (SERVER_KNOBS->ENABLE_VERSION_VECTOR && data->storageVersion() <= origVersion));
+			ASSERT(data->storageVersion() <= version);
 
 			/* Traverse the PTree further, if thare are no unconsumed resultCache items */
 			if (pos == resultCache.size()) {
@@ -3110,12 +3112,10 @@ ACTOR Future<GetKeyValuesReply> readRange(StorageServer* data,
 			data->counters.kvScanBytes += atStorageVersion.logicalSize();
 
 			ASSERT(atStorageVersion.size() <= -limit);
-			if ((!SERVER_KNOBS->ENABLE_VERSION_VECTOR && data->storageVersion() > version) ||
-			    (SERVER_KNOBS->ENABLE_VERSION_VECTOR && data->storageVersion() > origVersion)) {
-				DisabledTraceEvent("SS_TTO", data->thisServerID)
+			if (data->storageVersion() > version) {
+				TraceEvent("SS_TTO", data->thisServerID)
 				    .detail("StorageVersion", data->storageVersion())
 				    .detail("Version", version)
-				    .detail("OrigVersion", origVersion)
 				    .detail("Range", range);
 				throw transaction_too_old();
 			}
@@ -3217,7 +3217,6 @@ ACTOR Future<Key> findKey(StorageServer* data,
 
 	state GetKeyValuesReply rep = wait(
 	    readRange(data,
-	              version,
 	              version,
 	              forward ? KeyRangeRef(sel.getKey(), range.end) : KeyRangeRef(range.begin, keyAfter(sel.getKey())),
 	              (distance + skipEqualKey) * sign,
@@ -3233,7 +3232,6 @@ ACTOR Future<Key> findKey(StorageServer* data,
 		TEST(true); // Reverse key selector returned only one result in range read
 		maxBytes = std::numeric_limits<int>::max();
 		GetKeyValuesReply rep2 = wait(readRange(data,
-		                                        version,
 		                                        version,
 		                                        KeyRangeRef(range.begin, keyAfter(sel.getKey())),
 		                                        -2,
@@ -3341,6 +3339,7 @@ ACTOR Future<Void> getKeyValuesQ(StorageServer* data, GetKeyValuesRequest req)
 		DisabledTraceEvent("VVV", data->thisServerID)
 		    .detail("Version", version)
 		    .detail("ReqVersion", req.version)
+		    .detail("Oldest", data->oldestVersion.get())
 		    .detail("VV", req.ssLatestCommitVersions.toString())
 		    .detail("DebugID", req.debugID.present() ? req.debugID.get() : UID());
 		data->counters.readVersionWaitSample.addMeasurement(g_network->timer() - queueWaitEnd);
@@ -3423,7 +3422,6 @@ ACTOR Future<Void> getKeyValuesQ(StorageServer* data, GetKeyValuesRequest req)
 			state double kvReadRange = g_network->timer();
 			GetKeyValuesReply _r = wait(readRange(data,
 			                                      version,
-			                                      req.version,
 			                                      KeyRangeRef(begin, end),
 			                                      req.limit,
 			                                      &remainingLimitBytes,
@@ -4055,7 +4053,6 @@ ACTOR Future<Void> getMappedKeyValuesQ(StorageServer* data, GetMappedKeyValuesRe
 
 			GetKeyValuesReply getKeyValuesReply = wait(readRange(data,
 			                                                     version,
-			                                                     req.version,
 			                                                     KeyRangeRef(begin, end),
 			                                                     req.limit,
 			                                                     &remainingLimitBytes,
@@ -4259,15 +4256,8 @@ ACTOR Future<Void> getKeyValuesStreamQ(StorageServer* data, GetKeyValuesStreamRe
 				                       !data->isTss() && !data->isSSWithTSSPair())
 				                          ? 1
 				                          : CLIENT_KNOBS->REPLY_BYTE_LIMIT;
-				GetKeyValuesReply _r = wait(readRange(data,
-				                                      version,
-				                                      req.version,
-				                                      KeyRangeRef(begin, end),
-				                                      req.limit,
-				                                      &byteLimit,
-				                                      span.context,
-				                                      type,
-				                                      tenantPrefix));
+				GetKeyValuesReply _r = wait(readRange(
+				    data, version, KeyRangeRef(begin, end), req.limit, &byteLimit, span.context, type, tenantPrefix));
 				GetKeyValuesStreamReply r(_r);
 
 				if (req.debugID.present())
diff --git a/fdbserver/workloads/ConsistencyCheck.actor.cpp b/fdbserver/workloads/ConsistencyCheck.actor.cpp
index 3d57e9fc5b4..65a5592b450 100644
--- a/fdbserver/workloads/ConsistencyCheck.actor.cpp
+++ b/fdbserver/workloads/ConsistencyCheck.actor.cpp
@@ -1343,6 +1343,8 @@ struct ConsistencyCheckWorkload : TestWorkload {
 						req.limitBytes = CLIENT_KNOBS->REPLY_BYTE_LIMIT;
 						req.version = version;
 						req.tags = TagSet();
+						req.debugID = debugRandom()->randomUniqueID();
+						DisabledTraceEvent("CCD", req.debugID.get()).detail("Version", version);
 
 						// Try getting the entries in the specified range
 						state std::vector<Future<ErrorOr<GetKeyValuesReply>>> keyValueFutures;

From 18add2ad7fb607f6c5a705b9f0a6e6459f26bae8 Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Mon, 7 Nov 2022 14:49:37 -0800
Subject: [PATCH 004/468] Disable a debugging trace event

---
 fdbserver/storageserver.actor.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/fdbserver/storageserver.actor.cpp b/fdbserver/storageserver.actor.cpp
index 91477d05d11..48705d31977 100644
--- a/fdbserver/storageserver.actor.cpp
+++ b/fdbserver/storageserver.actor.cpp
@@ -3113,7 +3113,7 @@ ACTOR Future<GetKeyValuesReply> readRange(StorageServer* data,
 
 			ASSERT(atStorageVersion.size() <= -limit);
 			if (data->storageVersion() > version) {
-				TraceEvent("SS_TTO", data->thisServerID)
+				DisabledTraceEvent("SS_TTO", data->thisServerID)
 				    .detail("StorageVersion", data->storageVersion())
 				    .detail("Version", version)
 				    .detail("Range", range);

From 339543a8a4bff550a4e7c4ab756d9561eda3258b Mon Sep 17 00:00:00 2001
From: Dan Lambright <dlambrig@gmail.com>
Date: Tue, 8 Nov 2022 11:06:25 -0500
Subject: [PATCH 005/468] Cherry pick 8630

---
 bindings/c/test/apitester/TesterOptions.h      |  1 +
 bindings/c/test/apitester/fdb_c_api_tester.cpp | 12 +++++++++++-
 bindings/c/test/apitester/run_c_api_tests.py   | 10 +++++++++-
 fdbclient/ClientKnobs.cpp                      |  1 -
 fdbclient/MultiVersionTransaction.actor.cpp    | 11 ++++++-----
 fdbclient/MultiVersionTransaction.h            |  2 ++
 fdbclient/vexillographer/fdb.options           |  2 ++
 7 files changed, 31 insertions(+), 8 deletions(-)

diff --git a/bindings/c/test/apitester/TesterOptions.h b/bindings/c/test/apitester/TesterOptions.h
index 0f60ae436fa..8c73f5d3e1a 100644
--- a/bindings/c/test/apitester/TesterOptions.h
+++ b/bindings/c/test/apitester/TesterOptions.h
@@ -42,6 +42,7 @@ class TesterOptions {
 	int numClients;
 	std::vector<std::pair<std::string, std::string>> knobs;
 	TestSpec testSpec;
+	bool retainClientLibCopies = false;
 };
 
 } // namespace FdbApiTester
diff --git a/bindings/c/test/apitester/fdb_c_api_tester.cpp b/bindings/c/test/apitester/fdb_c_api_tester.cpp
index 062ffb95f71..4ee484db9aa 100644
--- a/bindings/c/test/apitester/fdb_c_api_tester.cpp
+++ b/bindings/c/test/apitester/fdb_c_api_tester.cpp
@@ -45,7 +45,8 @@ enum TesterOptionId {
 	OPT_TRACE_FORMAT,
 	OPT_KNOB,
 	OPT_EXTERNAL_CLIENT_LIBRARY,
-	OPT_TEST_FILE
+	OPT_TEST_FILE,
+	OPT_RETAIN_CLIENT_LIB_COPIES
 };
 
 CSimpleOpt::SOption TesterOptionDefs[] = //
@@ -61,6 +62,7 @@ CSimpleOpt::SOption TesterOptionDefs[] = //
 	  { OPT_EXTERNAL_CLIENT_LIBRARY, "--external-client-library", SO_REQ_SEP },
 	  { OPT_TEST_FILE, "-f", SO_REQ_SEP },
 	  { OPT_TEST_FILE, "--test-file", SO_REQ_SEP },
+	  { OPT_RETAIN_CLIENT_LIB_COPIES, "--retain-client-lib-copies", SO_NONE },
 	  SO_END_OF_OPTIONS };
 
 void printProgramUsage(const char* execName) {
@@ -144,6 +146,10 @@ bool processArg(TesterOptions& options, const CSimpleOpt& args) {
 		options.testFile = args.OptionArg();
 		options.testSpec = readTomlTestSpec(options.testFile);
 		break;
+
+	case OPT_RETAIN_CLIENT_LIB_COPIES:
+		options.retainClientLibCopies = true;
+		break;
 	}
 	return true;
 }
@@ -205,6 +211,10 @@ void applyNetworkOptions(TesterOptions& options) {
 		fdb_check(FdbApi::setOption(FDBNetworkOption::FDB_NET_OPTION_TRACE_LOG_GROUP, options.logGroup));
 	}
 
+	if (options.retainClientLibCopies) {
+		fdb_check(FdbApi::setOption(FDBNetworkOption::FDB_NET_OPTION_RETAIN_CLIENT_LIBRARY_COPIES));
+	}
+
 	for (auto knob : options.knobs) {
 		fdb_check(FdbApi::setOption(FDBNetworkOption::FDB_NET_OPTION_KNOB,
 		                            fmt::format("{}={}", knob.first.c_str(), knob.second.c_str())));
diff --git a/bindings/c/test/apitester/run_c_api_tests.py b/bindings/c/test/apitester/run_c_api_tests.py
index 8f79e6d8b11..39067ae4a86 100755
--- a/bindings/c/test/apitester/run_c_api_tests.py
+++ b/bindings/c/test/apitester/run_c_api_tests.py
@@ -54,6 +54,9 @@ def run_tester(args, test_file):
     if args.external_client_library is not None:
         cmd += ["--external-client-library", args.external_client_library]
 
+    if args.retain_client_lib_copies:
+        cmd += ["--retain-client-lib-copies"]
+
     get_logger().info('\nRunning tester \'%s\'...' % ' '.join(cmd))
     proc = Popen(cmd, stdout=sys.stdout, stderr=sys.stderr)
     timed_out = False
@@ -111,7 +114,12 @@ def parse_args(argv):
                         help='The timeout in seconds for running each individual test. (default 300)')
     parser.add_argument('--logging-level', type=str, default='INFO',
                         choices=['ERROR', 'WARNING', 'INFO', 'DEBUG'], help='Specifies the level of detail in the tester output (default=\'INFO\').')
-
+    parser.add_argument(
+                        "--retain-client-lib-copies",
+                        action="store_true",
+                        default=False,
+                        help="Retain temporary external client library copies.",
+    )
     return parser.parse_args(argv)
 
 
diff --git a/fdbclient/ClientKnobs.cpp b/fdbclient/ClientKnobs.cpp
index 7fa8d890294..16c8191cee4 100644
--- a/fdbclient/ClientKnobs.cpp
+++ b/fdbclient/ClientKnobs.cpp
@@ -203,7 +203,6 @@ void ClientKnobs::initialize(Randomize randomize) {
 	init( DEFAULT_AUTO_LOGS,                         3 );
 	init( DEFAULT_COMMIT_GRV_PROXIES_RATIO,          3 );
 	init( DEFAULT_MAX_GRV_PROXIES,                   4 );
-	init( UNLINKONLOAD_FDBCLIB,                   true ); // if false, don't delete libfdb_c in tmp directory on client connect.
 
 	init( IS_ACCEPTABLE_DELAY,                     1.5 );
 
diff --git a/fdbclient/MultiVersionTransaction.actor.cpp b/fdbclient/MultiVersionTransaction.actor.cpp
index c21273975ca..d5cbae914f8 100644
--- a/fdbclient/MultiVersionTransaction.actor.cpp
+++ b/fdbclient/MultiVersionTransaction.actor.cpp
@@ -2134,6 +2134,9 @@ void MultiVersionApi::setNetworkOptionInternal(FDBNetworkOptions::Option option,
 		// multiple client threads are not supported on windows.
 		threadCount = extractIntOption(value, 1, 1);
 #endif
+	} else if (option == FDBNetworkOptions::RETAIN_CLIENT_LIBRARY_COPIES) {
+		validateOption(value, false, true);
+		retainClientLibCopies = true;
 	} else {
 		forwardOption = true;
 	}
@@ -2177,10 +2180,7 @@ void MultiVersionApi::setupNetwork() {
 			if (externalClients.count(filename) == 0) {
 				externalClients[filename] = {};
 				for (const auto& tmp : copyExternalLibraryPerThread(path)) {
-					bool unlinkOnLoad = tmp.second;
-					if (!CLIENT_KNOBS->UNLINKONLOAD_FDBCLIB) {
-						unlinkOnLoad = false;
-					}
+					bool unlinkOnLoad = tmp.second && !retainClientLibCopies;
 					externalClients[filename].push_back(Reference<ClientInfo>(
 					    new ClientInfo(new DLApi(tmp.first, unlinkOnLoad /*unlink on load*/), path)));
 				}
@@ -2513,7 +2513,8 @@ void MultiVersionApi::loadEnvironmentVariableNetworkOptions() {
 
 MultiVersionApi::MultiVersionApi()
   : callbackOnMainThread(true), localClientDisabled(false), networkStartSetup(false), networkSetup(false),
-    bypassMultiClientApi(false), externalClient(false), apiVersion(0), threadCount(0), envOptionsLoaded(false) {}
+    bypassMultiClientApi(false), externalClient(false), retainClientLibCopies(false), apiVersion(0), threadCount(0),
+    envOptionsLoaded(false) {}
 
 MultiVersionApi* MultiVersionApi::api = new MultiVersionApi();
 
diff --git a/fdbclient/MultiVersionTransaction.h b/fdbclient/MultiVersionTransaction.h
index e2b0c7e027c..6e172d33513 100644
--- a/fdbclient/MultiVersionTransaction.h
+++ b/fdbclient/MultiVersionTransaction.h
@@ -894,6 +894,8 @@ class MultiVersionApi : public IClientApi {
 	volatile bool networkSetup;
 	volatile bool bypassMultiClientApi;
 	volatile bool externalClient;
+	bool retainClientLibCopies;
+
 	int apiVersion;
 
 	int nextThread = 0;
diff --git a/fdbclient/vexillographer/fdb.options b/fdbclient/vexillographer/fdb.options
index 9f2f9e52f93..bb9c5d2cff2 100644
--- a/fdbclient/vexillographer/fdb.options
+++ b/fdbclient/vexillographer/fdb.options
@@ -30,6 +30,8 @@ description is not currently required but encouraged.
 
 <Options>
   <Scope name="NetworkOption">
+    <Option name="retain_client_library_copies" code="67"
+            description="Retain temporary external client library copies that are created for enabling multi-threading." />
     <Option name="local_address"  code="10" 
             paramType="String" paramDescription="IP:PORT" 
             description="Deprecated"/>

From 4173acffcbdab4139f134380c11731f3603fe353 Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Tue, 8 Nov 2022 08:59:55 -0800
Subject: [PATCH 006/468] Address review comments

---
 fdbserver/storageserver.actor.cpp | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/fdbserver/storageserver.actor.cpp b/fdbserver/storageserver.actor.cpp
index 48705d31977..57a68c8298a 100644
--- a/fdbserver/storageserver.actor.cpp
+++ b/fdbserver/storageserver.actor.cpp
@@ -1520,11 +1520,14 @@ Future<Version> waitForVersion(StorageServer* data, Version commitVersion, Versi
 	if (readVersion < data->oldestVersion.get() || readVersion <= 0) {
 		return transaction_too_old();
 	} else {
+		// It is correct to read any version between [commitVersion, readVersion],
+		// because version vector guarantees no mutations between them.
 		if (commitVersion < data->oldestVersion.get()) {
 			if (data->version.get() < readVersion) {
 				// Majority of the case, try using higher version to avoid
 				// transaction_too_old error when oldestVersion advances.
-				return data->version.get(); // majority of cases
+				// BTW, any version in the range [oldestVersion, data->version.get()] is valid in this case.
+				return data->version.get();
 			} else {
 				ASSERT(readVersion >= data->oldestVersion.get());
 				return readVersion;
@@ -3115,6 +3118,7 @@ ACTOR Future<GetKeyValuesReply> readRange(StorageServer* data,
 			if (data->storageVersion() > version) {
 				DisabledTraceEvent("SS_TTO", data->thisServerID)
 				    .detail("StorageVersion", data->storageVersion())
+				    .detail("Oldest", data->oldestVersion.get())
 				    .detail("Version", version)
 				    .detail("Range", range);
 				throw transaction_too_old();

From b4bd84bd19859a36ef83d11dab203b3e11ce73dc Mon Sep 17 00:00:00 2001
From: FoundationDB CI <foundationdb_ci@apple.com>
Date: Mon, 7 Nov 2022 23:12:42 +0000
Subject: [PATCH 007/468] enable AVX and update version for 7.1.25 release

---
 CMakeLists.txt                | 2 +-
 cmake/ConfigureCompiler.cmake | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index c712cdfae0a..d160b7e70e4 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -23,7 +23,7 @@ else()
 endif()
 
 project(foundationdb
-  VERSION 7.1.24
+  VERSION 7.1.25
   DESCRIPTION "FoundationDB is a scalable, fault-tolerant, ordered key-value store with full ACID transactions."
   HOMEPAGE_URL "http://www.foundationdb.org/"
   LANGUAGES C CXX ASM)
diff --git a/cmake/ConfigureCompiler.cmake b/cmake/ConfigureCompiler.cmake
index cbd7935e464..398dfe5fb61 100644
--- a/cmake/ConfigureCompiler.cmake
+++ b/cmake/ConfigureCompiler.cmake
@@ -253,7 +253,7 @@ else()
       set(USE_AVX512F OFF)
     endif()
   endif()
-  set(USE_AVX OFF CACHE BOOL "Enable AVX instructions")
+  set(USE_AVX ON CACHE BOOL "Enable AVX instructions")
   if (USE_AVX)
     if (CMAKE_HOST_SYSTEM_PROCESSOR MATCHES "^x86")
       add_compile_options(-mavx)

From 6cb7ac4f04267d92c04be3f913ea880bcaf88340 Mon Sep 17 00:00:00 2001
From: neethuhaneesha <nbingi@apple.com>
Date: Mon, 7 Nov 2022 16:49:22 -0800
Subject: [PATCH 008/468] skip proxy when fetching kubectl

---
 packaging/docker/Dockerfile     | 2 +-
 packaging/docker/Dockerfile.eks | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/packaging/docker/Dockerfile b/packaging/docker/Dockerfile
index 5075e81054e..75dbda3e9da 100644
--- a/packaging/docker/Dockerfile
+++ b/packaging/docker/Dockerfile
@@ -178,7 +178,7 @@ RUN yum -y install \
     rm -rf /var/cache/yum
 
 WORKDIR /tmp
-RUN curl -Ls https://s3.us-west-2.amazonaws.com/amazon-eks/1.22.6/2022-03-09/bin/linux/amd64/kubectl -o kubectl && \
+RUN NO_PROXY="" no_proxy="" curl -Ls https://s3.us-west-2.amazonaws.com/amazon-eks/1.22.6/2022-03-09/bin/linux/amd64/kubectl -o kubectl && \
     echo "860c3d37a5979491895767e7332404d28dc0d7797c7673c33df30ca80e215a07  kubectl" > kubectl.txt && \
     sha256sum --quiet -c kubectl.txt && \
     mv kubectl /usr/local/bin/kubectl && \
diff --git a/packaging/docker/Dockerfile.eks b/packaging/docker/Dockerfile.eks
index 0aae38dcc57..f8dec2421a2 100644
--- a/packaging/docker/Dockerfile.eks
+++ b/packaging/docker/Dockerfile.eks
@@ -53,7 +53,7 @@ RUN curl -Ls https://github.com/krallin/tini/releases/download/v0.19.0/tini-amd6
     mv tini /usr/bin/ && \
     rm -rf /tmp/*
 
-RUN curl -Ls https://s3.us-west-2.amazonaws.com/amazon-eks/1.22.6/2022-03-09/bin/linux/amd64/kubectl -o kubectl && \
+RUN NO_PROXY="" no_proxy="" curl -Ls https://s3.us-west-2.amazonaws.com/amazon-eks/1.22.6/2022-03-09/bin/linux/amd64/kubectl -o kubectl && \
     echo "860c3d37a5979491895767e7332404d28dc0d7797c7673c33df30ca80e215a07  kubectl" > kubectl.txt && \
     sha256sum --quiet -c kubectl.txt && \
     mv kubectl /usr/local/bin/kubectl && \

From 1439943eceb9bd5f6a40e46340f95373eb1ba9cf Mon Sep 17 00:00:00 2001
From: Dan Lambright <dlambrig@gmail.com>
Date: Tue, 8 Nov 2022 21:22:41 -0500
Subject: [PATCH 009/468] add generated.go

---
 bindings/go/src/fdb/generated.go | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/bindings/go/src/fdb/generated.go b/bindings/go/src/fdb/generated.go
index b636484408b..f319208dba5 100644
--- a/bindings/go/src/fdb/generated.go
+++ b/bindings/go/src/fdb/generated.go
@@ -239,6 +239,11 @@ func (o NetworkOptions) SetClientThreadsPerVersion(param int64) error {
 	return o.setOpt(65, int64ToBytes(param))
 }
 
+// Retain temporary external client library copies that are created for enabling multi-threading.
+func (o NetworkOptions) SetRetainClientLibraryCopies() error {
+	return o.setOpt(67, nil)
+}
+
 // Disables logging of client statistics, such as sampled transaction activity.
 func (o NetworkOptions) SetDisableClientStatisticsLogging() error {
 	return o.setOpt(70, nil)

From cf0aed4e08ba3ac68b2d0c7d68f4ac615f31f170 Mon Sep 17 00:00:00 2001
From: FoundationDB CI <foundationdb_ci@apple.com>
Date: Wed, 9 Nov 2022 16:29:10 +0000
Subject: [PATCH 010/468] update version after 7.1.25 release

---
 CMakeLists.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index d160b7e70e4..2e7e4cda729 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -23,7 +23,7 @@ else()
 endif()
 
 project(foundationdb
-  VERSION 7.1.25
+  VERSION 7.1.26
   DESCRIPTION "FoundationDB is a scalable, fault-tolerant, ordered key-value store with full ACID transactions."
   HOMEPAGE_URL "http://www.foundationdb.org/"
   LANGUAGES C CXX ASM)

From e58d3884a03bd89643da345aee2c682171cc0cc2 Mon Sep 17 00:00:00 2001
From: Dan Lambright <dlambrig@gmail.com>
Date: Wed, 9 Nov 2022 12:36:26 -0500
Subject: [PATCH 011/468] Add changes to generated.go from PR8761, and remove
 change to ConfigureCompiler.cmake

---
 bindings/go/src/fdb/generated.go | 5 -----
 cmake/ConfigureCompiler.cmake    | 2 +-
 2 files changed, 1 insertion(+), 6 deletions(-)

diff --git a/bindings/go/src/fdb/generated.go b/bindings/go/src/fdb/generated.go
index f319208dba5..b636484408b 100644
--- a/bindings/go/src/fdb/generated.go
+++ b/bindings/go/src/fdb/generated.go
@@ -239,11 +239,6 @@ func (o NetworkOptions) SetClientThreadsPerVersion(param int64) error {
 	return o.setOpt(65, int64ToBytes(param))
 }
 
-// Retain temporary external client library copies that are created for enabling multi-threading.
-func (o NetworkOptions) SetRetainClientLibraryCopies() error {
-	return o.setOpt(67, nil)
-}
-
 // Disables logging of client statistics, such as sampled transaction activity.
 func (o NetworkOptions) SetDisableClientStatisticsLogging() error {
 	return o.setOpt(70, nil)
diff --git a/cmake/ConfigureCompiler.cmake b/cmake/ConfigureCompiler.cmake
index 398dfe5fb61..cbd7935e464 100644
--- a/cmake/ConfigureCompiler.cmake
+++ b/cmake/ConfigureCompiler.cmake
@@ -253,7 +253,7 @@ else()
       set(USE_AVX512F OFF)
     endif()
   endif()
-  set(USE_AVX ON CACHE BOOL "Enable AVX instructions")
+  set(USE_AVX OFF CACHE BOOL "Enable AVX instructions")
   if (USE_AVX)
     if (CMAKE_HOST_SYSTEM_PROCESSOR MATCHES "^x86")
       add_compile_options(-mavx)

From 91f452eb099c836785b316d0555886e1107e01a0 Mon Sep 17 00:00:00 2001
From: Dan Lambright <dlambrig@gmail.com>
Date: Wed, 9 Nov 2022 14:44:07 -0500
Subject: [PATCH 012/468] Update generated.go

---
 bindings/go/src/fdb/generated.go | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/bindings/go/src/fdb/generated.go b/bindings/go/src/fdb/generated.go
index b636484408b..d8d89de8b0c 100644
--- a/bindings/go/src/fdb/generated.go
+++ b/bindings/go/src/fdb/generated.go
@@ -39,6 +39,11 @@ func int64ToBytes(i int64) []byte {
 	return buf
 }
 
+// Retain temporary external client library copies that are created for enabling multi-threading.
+func (o NetworkOptions) SetRetainClientLibraryCopies() error {
+	return o.setOpt(67, nil)
+}
+
 // Deprecated
 //
 // Parameter: IP:PORT

From 3b52fc48d92ecfd4d4f923c3518f5977a68e54c3 Mon Sep 17 00:00:00 2001
From: Dan Lambright <dlambrig@gmail.com>
Date: Wed, 9 Nov 2022 16:19:11 -0500
Subject: [PATCH 013/468] Update generated.go with 8761

---
 bindings/go/src/fdb/generated.go | 30 +++++++++++++++---------------
 1 file changed, 15 insertions(+), 15 deletions(-)

diff --git a/bindings/go/src/fdb/generated.go b/bindings/go/src/fdb/generated.go
index d8d89de8b0c..f8bb231980c 100644
--- a/bindings/go/src/fdb/generated.go
+++ b/bindings/go/src/fdb/generated.go
@@ -622,12 +622,12 @@ const (
 	StreamingModeWantAll StreamingMode = -1
 
 	// The default. The client doesn't know how much of the range it is likely
-	// to used and wants different performance concerns to be balanced. Only a
-	// small portion of data is transferred to the client initially (in order to
-	// minimize costs if the client doesn't read the entire range), and as the
-	// caller iterates over more items in the range larger batches will be
-	// transferred in order to minimize latency. After enough iterations, the
-	// iterator mode will eventually reach the same byte limit as “WANT_ALL“
+	// to used and wants different performance concerns to be balanced.
+	// Only a small portion of data is transferred to the client initially (in
+	// order to minimize costs if the client doesn't read the entire range), and
+	// as the caller iterates over more items in the range larger batches will
+	// be transferred in order to minimize latency. After enough iterations,
+	// the iterator mode will eventually reach the same byte limit as “WANT_ALL“
 	StreamingModeIterator StreamingMode = 0
 
 	// Infrequently used. The client has passed a specific row limit and wants
@@ -637,8 +637,8 @@ const (
 	// mode is used.
 	StreamingModeExact StreamingMode = 1
 
-	// Infrequently used. Transfer data in batches small enough to not be much
-	// more expensive than reading individual rows, to minimize cost if
+	// Infrequently used. Transfer data in batches small enough to not be
+	// much more expensive than reading individual rows, to minimize cost if
 	// iteration stops early.
 	StreamingModeSmall StreamingMode = 2
 
@@ -646,16 +646,16 @@ const (
 	// large.
 	StreamingModeMedium StreamingMode = 3
 
-	// Infrequently used. Transfer data in batches large enough to be, in a
-	// high-concurrency environment, nearly as efficient as possible. If the
-	// client stops iteration early, some disk and network bandwidth may be
-	// wasted. The batch size may still be too small to allow a single client to
-	// get high throughput from the database, so if that is what you need
+	// Infrequently used. Transfer data in batches large enough to be,
+	// in a high-concurrency environment, nearly as efficient as possible.
+	// If the client stops iteration early, some disk and network bandwidth may
+	// be wasted. The batch size may still be too small to allow a single client
+	// to get high throughput from the database, so if that is what you need
 	// consider the SERIAL StreamingMode.
 	StreamingModeLarge StreamingMode = 4
 
-	// Transfer data in batches large enough that an individual client can get
-	// reasonable read bandwidth from the database. If the client stops
+	// Transfer data in batches large enough that an individual client can
+	// get reasonable read bandwidth from the database. If the client stops
 	// iteration early, considerable disk and network bandwidth may be wasted.
 	StreamingModeSerial StreamingMode = 5
 )

From 347bd929facf89929b5d7be636cc96042c8b92a1 Mon Sep 17 00:00:00 2001
From: neethuhaneesha <nbingi@apple.com>
Date: Fri, 11 Nov 2022 10:39:02 -0800
Subject: [PATCH 014/468] Rocksdb stats level knob. (#8713)

---
 fdbclient/ServerKnobs.cpp                | 14 +++---
 fdbclient/ServerKnobs.h                  |  1 +
 fdbserver/KeyValueStoreRocksDB.actor.cpp | 58 ++++++++++++------------
 3 files changed, 38 insertions(+), 35 deletions(-)

diff --git a/fdbclient/ServerKnobs.cpp b/fdbclient/ServerKnobs.cpp
index c479e293bae..b74124ee919 100644
--- a/fdbclient/ServerKnobs.cpp
+++ b/fdbclient/ServerKnobs.cpp
@@ -388,8 +388,8 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi
 	init( ROCKSDB_FETCH_QUEUE_SOFT_MAX,                           50 );
 	init( ROCKSDB_HISTOGRAMS_SAMPLE_RATE,                      0.001 ); if( randomize && BUGGIFY ) ROCKSDB_HISTOGRAMS_SAMPLE_RATE = 0;
 	init( ROCKSDB_READ_RANGE_ITERATOR_REFRESH_TIME,             30.0 ); if( randomize && BUGGIFY ) ROCKSDB_READ_RANGE_ITERATOR_REFRESH_TIME = 0.1;
-	init( ROCKSDB_READ_RANGE_REUSE_ITERATORS,                   true ); if( randomize && BUGGIFY ) ROCKSDB_READ_RANGE_REUSE_ITERATORS = deterministicRandom()->coinflip() ? true : false;
-	init( ROCKSDB_READ_RANGE_REUSE_BOUNDED_ITERATORS,          false ); if( randomize && BUGGIFY ) ROCKSDB_READ_RANGE_REUSE_BOUNDED_ITERATORS = deterministicRandom()->coinflip() ? true : false;
+	init( ROCKSDB_READ_RANGE_REUSE_ITERATORS,                   true ); if( randomize && BUGGIFY ) ROCKSDB_READ_RANGE_REUSE_ITERATORS = deterministicRandom()->coinflip();
+	init( ROCKSDB_READ_RANGE_REUSE_BOUNDED_ITERATORS,          false ); if( randomize && BUGGIFY ) ROCKSDB_READ_RANGE_REUSE_BOUNDED_ITERATORS = deterministicRandom()->coinflip();
 	init( ROCKSDB_READ_RANGE_BOUNDED_ITERATORS_MAX_LIMIT,        200 );
 	// Set to 0 to disable rocksdb write rate limiting. Rate limiter unit: bytes per second.
 	init( ROCKSDB_WRITE_RATE_LIMITER_BYTES_PER_SEC,                0 );
@@ -398,8 +398,8 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi
 	init( DEFAULT_FDB_ROCKSDB_COLUMN_FAMILY,                    "fdb");
 	init( ROCKSDB_DISABLE_AUTO_COMPACTIONS,                    false ); // RocksDB default
 
-	init( ROCKSDB_PERFCONTEXT_ENABLE,                          false ); if( randomize && BUGGIFY ) ROCKSDB_PERFCONTEXT_ENABLE = deterministicRandom()->coinflip() ? false : true;
-	init( ROCKSDB_PERFCONTEXT_SAMPLE_RATE, 					  0.0001 );
+	init( ROCKSDB_PERFCONTEXT_ENABLE,                          false ); if( randomize && BUGGIFY ) ROCKSDB_PERFCONTEXT_ENABLE = deterministicRandom()->coinflip();
+	init( ROCKSDB_PERFCONTEXT_SAMPLE_RATE,                    0.0001 );
 	init( ROCKSDB_MAX_SUBCOMPACTIONS,                              2 );
 	init( ROCKSDB_SOFT_PENDING_COMPACT_BYTES_LIMIT,      64000000000 ); // 64GB, Rocksdb option, Writes will slow down.
 	init( ROCKSDB_HARD_PENDING_COMPACT_BYTES_LIMIT,     100000000000 ); // 100GB, Rocksdb option, Writes will stall.
@@ -409,9 +409,11 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi
 	init( ROCKSDB_DISABLE_WAL_EXPERIMENTAL,                    false );
 	// If ROCKSDB_SINGLEKEY_DELETES_ON_CLEARRANGE is enabled, disable ROCKSDB_ENABLE_CLEAR_RANGE_EAGER_READS knob.
 	// These knobs have contrary functionality.
-	init( ROCKSDB_SINGLEKEY_DELETES_ON_CLEARRANGE,             false ); if( randomize && BUGGIFY ) ROCKSDB_SINGLEKEY_DELETES_ON_CLEARRANGE = deterministicRandom()->coinflip() ? false : true;
+	init( ROCKSDB_SINGLEKEY_DELETES_ON_CLEARRANGE,             false ); if( randomize && BUGGIFY ) ROCKSDB_SINGLEKEY_DELETES_ON_CLEARRANGE = deterministicRandom()->coinflip();
 	init( ROCKSDB_SINGLEKEY_DELETES_BYTES_LIMIT,              200000 ); // 200KB
-	init( ROCKSDB_ENABLE_CLEAR_RANGE_EAGER_READS,               true ); if( randomize && BUGGIFY ) ROCKSDB_ENABLE_CLEAR_RANGE_EAGER_READS = deterministicRandom()->coinflip() ? false : true;
+	init( ROCKSDB_ENABLE_CLEAR_RANGE_EAGER_READS,               true ); if( randomize && BUGGIFY ) ROCKSDB_ENABLE_CLEAR_RANGE_EAGER_READS = deterministicRandom()->coinflip();
+	// ROCKSDB_STATS_LEVEL=1 indicates rocksdb::StatsLevel::kExceptHistogramOrTimers
+	init( ROCKSDB_STATS_LEVEL,                                     1 ); if( randomize && BUGGIFY ) ROCKSDB_STATS_LEVEL = deterministicRandom()->randomInt(0, 6);
 	// Can commit will delay ROCKSDB_CAN_COMMIT_DELAY_ON_OVERLOAD seconds for
 	// ROCKSDB_CAN_COMMIT_DELAY_TIMES_ON_OVERLOAD times, if rocksdb overloaded.
 	// Set ROCKSDB_CAN_COMMIT_DELAY_TIMES_ON_OVERLOAD to 0, to disable
diff --git a/fdbclient/ServerKnobs.h b/fdbclient/ServerKnobs.h
index 2034b61f4e7..26b7b787c73 100644
--- a/fdbclient/ServerKnobs.h
+++ b/fdbclient/ServerKnobs.h
@@ -336,6 +336,7 @@ class ServerKnobs : public KnobsImpl<ServerKnobs> {
 	bool ROCKSDB_SINGLEKEY_DELETES_ON_CLEARRANGE;
 	int64_t ROCKSDB_SINGLEKEY_DELETES_BYTES_LIMIT;
 	bool ROCKSDB_ENABLE_CLEAR_RANGE_EAGER_READS;
+	int ROCKSDB_STATS_LEVEL;
 	int64_t ROCKSDB_COMPACTION_READAHEAD_SIZE;
 	int64_t ROCKSDB_BLOCK_SIZE;
 
diff --git a/fdbserver/KeyValueStoreRocksDB.actor.cpp b/fdbserver/KeyValueStoreRocksDB.actor.cpp
index 121812776d8..7bad3874a88 100644
--- a/fdbserver/KeyValueStoreRocksDB.actor.cpp
+++ b/fdbserver/KeyValueStoreRocksDB.actor.cpp
@@ -373,7 +373,7 @@ rocksdb::Options getOptions() {
 	}
 
 	options.statistics = rocksdb::CreateDBStatistics();
-	options.statistics->set_stats_level(rocksdb::kExceptHistogramOrTimers);
+	options.statistics->set_stats_level(rocksdb::StatsLevel(SERVER_KNOBS->ROCKSDB_STATS_LEVEL));
 
 	options.db_log_dir = SERVER_KNOBS->LOG_DIRECTORY;
 	return options;
@@ -477,10 +477,7 @@ class ReadIteratorPool {
 			iteratorsMap.insert({ readIteratorIndex, iter });
 			mutex.unlock();
 			return iter;
-		} else if (SERVER_KNOBS->ROCKSDB_READ_RANGE_REUSE_BOUNDED_ITERATORS &&
-		           iteratorsMap.size() < SERVER_KNOBS->ROCKSDB_READ_RANGE_BOUNDED_ITERATORS_MAX_LIMIT) {
-			// Not storing more than ROCKSDB_READ_RANGE_BOUNDED_ITERATORS_MAX_LIMIT of iterators
-			// to avoid 'out of memory' issues.
+		} else if (SERVER_KNOBS->ROCKSDB_READ_RANGE_REUSE_BOUNDED_ITERATORS) {
 			// TODO: Based on the datasize in the keyrange, decide whether to store the iterator for reuse.
 			mutex.lock();
 			for (it = iteratorsMap.begin(); it != iteratorsMap.end(); it++) {
@@ -497,18 +494,13 @@ class ReadIteratorPool {
 			mutex.unlock();
 
 			ReadIterator iter(cf, readIteratorIndex, db, readRangeOptions, keyRange);
-			mutex.lock();
-			iteratorsMap.insert({ readIteratorIndex, iter });
-			mutex.unlock();
-			return iter;
-		} else if (SERVER_KNOBS->ROCKSDB_READ_RANGE_REUSE_BOUNDED_ITERATORS) {
-			// If the iterator size reaches the ROCKSDB_READ_RANGE_BOUNDED_ITERATORS_MAX_LIMIT.
-			mutex.lock();
-			index++;
-			uint64_t readIteratorIndex = index;
-			mutex.unlock();
-
-			ReadIterator iter(cf, readIteratorIndex, db, readRangeOptions, keyRange);
+			if (iteratorsMap.size() < SERVER_KNOBS->ROCKSDB_READ_RANGE_BOUNDED_ITERATORS_MAX_LIMIT) {
+				// Not storing more than ROCKSDB_READ_RANGE_BOUNDED_ITERATORS_MAX_LIMIT of iterators
+				// to avoid 'out of memory' issues.
+				mutex.lock();
+				iteratorsMap.insert({ readIteratorIndex, iter });
+				mutex.unlock();
+			}
 			return iter;
 		} else {
 			index++;
@@ -899,14 +891,19 @@ ACTOR Future<Void> rocksDBMetricLogger(UID id,
 		{ "CountIterSkippedKeys", rocksdb::NUMBER_ITER_SKIP, 0 },
 	};
 
+	// To control the rocksdb::StatsLevel, use ROCKSDB_STATS_LEVEL knob.
 	state std::vector<std::pair<const char*, uint32_t>> histogramStats = {
-		{ "CompactionTime", rocksdb::COMPACTION_TIME },
-		{ "CompactionCPUTime", rocksdb::COMPACTION_CPU_TIME },
-		{ "CompressionTimeNanos", rocksdb::COMPRESSION_TIMES_NANOS },
-		{ "DecompressionTimeNanos", rocksdb::DECOMPRESSION_TIMES_NANOS },
-		{ "HardRateLimitDelayCount", rocksdb::HARD_RATE_LIMIT_DELAY_COUNT },
-		{ "SoftRateLimitDelayCount", rocksdb::SOFT_RATE_LIMIT_DELAY_COUNT },
-		{ "WriteStall", rocksdb::WRITE_STALL },
+		{ "CompactionTime", rocksdb::COMPACTION_TIME }, // enabled if rocksdb::StatsLevel > kExceptTimers(2)
+		{ "CompactionCPUTime", rocksdb::COMPACTION_CPU_TIME }, // enabled if rocksdb::StatsLevel > kExceptTimers(2)
+		{ "CompressionTimeNanos",
+		  rocksdb::COMPRESSION_TIMES_NANOS }, // enabled if rocksdb::StatsLevel > kExceptDetailedTimers(3)
+		{ "DecompressionTimeNanos",
+		  rocksdb::DECOMPRESSION_TIMES_NANOS }, // enabled if rocksdb::StatsLevel > kExceptDetailedTimers(3)
+		{ "HardRateLimitDelayCount",
+		  rocksdb::HARD_RATE_LIMIT_DELAY_COUNT }, // enabled if rocksdb::StatsLevel > kExceptHistogramOrTimers(1)
+		{ "SoftRateLimitDelayCount",
+		  rocksdb::SOFT_RATE_LIMIT_DELAY_COUNT }, // enabled if rocksdb::StatsLevel > kExceptHistogramOrTimers(1)
+		{ "WriteStall", rocksdb::WRITE_STALL }, // enabled if rocksdb::StatsLevel > kExceptHistogramOrTimers(1)
 	};
 	state std::vector<std::pair<const char*, std::string>> intPropertyStats = {
 		{ "NumImmutableMemtables", rocksdb::DB::Properties::kNumImmutableMemTable },
@@ -959,11 +956,14 @@ ACTOR Future<Void> rocksDBMetricLogger(UID id,
 			cum = stat;
 		}
 
-		for (auto& [name, histogram] : histogramStats) {
-			rocksdb::HistogramData histogram_data;
-			statistics->histogramData(histogram, &histogram_data);
-			e.detail(format("%s%d", name, "P95"), histogram_data.percentile95);
-			e.detail(format("%s%d", name, "P99"), histogram_data.percentile99);
+		// None of the histogramStats are enabled unless the ROCKSDB_STATS_LEVEL > kExceptHistogramOrTimers(1)
+		if (SERVER_KNOBS->ROCKSDB_STATS_LEVEL > rocksdb::kExceptHistogramOrTimers) {
+			for (auto& [name, histogram] : histogramStats) {
+				rocksdb::HistogramData histogram_data;
+				statistics->histogramData(histogram, &histogram_data);
+				e.detail(format("%s%s", name, "P95"), histogram_data.percentile95);
+				e.detail(format("%s%s", name, "P99"), histogram_data.percentile99);
+			}
 		}
 
 		for (const auto& [name, property] : intPropertyStats) {

From c2321bf8bef06a0a0002594341e25d40bfbd01a1 Mon Sep 17 00:00:00 2001
From: neethuhaneesha <nbingi@apple.com>
Date: Fri, 11 Nov 2022 18:02:07 -0800
Subject: [PATCH 015/468] Adding counters for singlekey clear requests (#8792)

---
 fdbserver/KeyValueStoreRocksDB.actor.cpp | 16 +++++++++++++++-
 fdbserver/storageserver.actor.cpp        | 17 +++++++++++++++--
 2 files changed, 30 insertions(+), 3 deletions(-)

diff --git a/fdbserver/KeyValueStoreRocksDB.actor.cpp b/fdbserver/KeyValueStoreRocksDB.actor.cpp
index 7bad3874a88..ebab54bca49 100644
--- a/fdbserver/KeyValueStoreRocksDB.actor.cpp
+++ b/fdbserver/KeyValueStoreRocksDB.actor.cpp
@@ -390,9 +390,16 @@ struct Counters {
 	CounterCollection cc;
 	Counter immediateThrottle;
 	Counter failedToAcquire;
+	Counter deleteKeyReqs;
+	Counter deleteRangeReqs;
+	Counter convertedDeleteKeyReqs;
+	Counter convertedDeleteRangeReqs;
 
 	Counters()
-	  : cc("RocksDBThrottle"), immediateThrottle("ImmediateThrottle", cc), failedToAcquire("FailedToAcquire", cc) {}
+	  : cc("RocksDBThrottle"), immediateThrottle("ImmediateThrottle", cc), failedToAcquire("FailedToAcquire", cc),
+	    deleteKeyReqs("DeleteKeyRequests", cc), deleteRangeReqs("DeleteRangeRequests", cc),
+	    convertedDeleteKeyReqs("ConvertedDeleteKeyRequests", cc),
+	    convertedDeleteRangeReqs("ConvertedDeleteRangeRequests", cc) {}
 };
 
 struct ReadIterator {
@@ -2002,12 +2009,17 @@ struct RocksDBKeyValueStore : IKeyValueStore {
 		}
 
 		ASSERT(defaultFdbCF != nullptr);
+		// Number of deletes to rocksdb = counters.deleteKeyReqs + convertedDeleteKeyReqs;
+		// Number of deleteRanges to rocksdb = counters.deleteRangeReqs - counters.convertedDeleteRangeReqs;
 		if (keyRange.singleKeyRange()) {
 			writeBatch->Delete(defaultFdbCF, toSlice(keyRange.begin));
+			++counters.deleteKeyReqs;
 		} else {
+			++counters.deleteRangeReqs;
 			if (SERVER_KNOBS->ROCKSDB_SINGLEKEY_DELETES_ON_CLEARRANGE && storageMetrics != nullptr &&
 			    storageMetrics->byteSample.getEstimate(keyRange) <
 			        SERVER_KNOBS->ROCKSDB_SINGLEKEY_DELETES_BYTES_LIMIT) {
+				++counters.convertedDeleteRangeReqs;
 				rocksdb::ReadOptions options = getReadOptions();
 				auto beginSlice = toSlice(keyRange.begin);
 				auto endSlice = toSlice(keyRange.end);
@@ -2017,6 +2029,7 @@ struct RocksDBKeyValueStore : IKeyValueStore {
 				cursor->Seek(toSlice(keyRange.begin));
 				while (cursor->Valid() && toStringRef(cursor->key()) < keyRange.end) {
 					writeBatch->Delete(defaultFdbCF, cursor->key());
+					++counters.convertedDeleteKeyReqs;
 					cursor->Next();
 				}
 				if (!cursor->status().ok()) {
@@ -2026,6 +2039,7 @@ struct RocksDBKeyValueStore : IKeyValueStore {
 					auto it = keysSet.lower_bound(keyRange.begin);
 					while (it != keysSet.end() && *it < keyRange.end) {
 						writeBatch->Delete(defaultFdbCF, toSlice(*it));
+						++counters.convertedDeleteKeyReqs;
 						it++;
 					}
 				}
diff --git a/fdbserver/storageserver.actor.cpp b/fdbserver/storageserver.actor.cpp
index 57a68c8298a..2dd73443302 100644
--- a/fdbserver/storageserver.actor.cpp
+++ b/fdbserver/storageserver.actor.cpp
@@ -311,6 +311,7 @@ struct StorageServerDisk {
 	// The following are pointers to the Counters in StorageServer::counters of the same names.
 	Counter* kvCommitLogicalBytes;
 	Counter* kvClearRanges;
+	Counter* kvClearSingleKey;
 	Counter* kvGets;
 	Counter* kvScans;
 	Counter* kvCommits;
@@ -893,6 +894,8 @@ struct StorageServer {
 		Counter kvCommitLogicalBytes;
 		// Count of all clearRange operatons to the storage engine.
 		Counter kvClearRanges;
+		// Count of all clearRange operations on a singlekeyRange(key delete) to the storage engine.
+		Counter kvClearSingleKey;
 		// ClearRange operations issued by FDB, instead of from users, e.g., ClearRange operations to remove a shard
 		// from a storage server, as in removeDataRange().
 		Counter kvSystemClearRanges;
@@ -965,8 +968,8 @@ struct StorageServer {
 		    bytesInput("BytesInput", cc), logicalBytesInput("LogicalBytesInput", cc),
 		    logicalBytesMoveInOverhead("LogicalBytesMoveInOverhead", cc),
 		    kvCommitLogicalBytes("KVCommitLogicalBytes", cc), kvClearRanges("KVClearRanges", cc),
-		    kvSystemClearRanges("KVSystemClearRanges", cc), bytesDurable("BytesDurable", cc),
-		    bytesFetched("BytesFetched", cc), mutationBytes("MutationBytes", cc),
+		    kvClearSingleKey("KVClearSingleKey", cc), kvSystemClearRanges("KVSystemClearRanges", cc),
+		    bytesDurable("BytesDurable", cc), bytesFetched("BytesFetched", cc), mutationBytes("MutationBytes", cc),
 		    feedBytesFetched("FeedBytesFetched", cc), sampledBytesCleared("SampledBytesCleared", cc),
 		    kvFetched("KVFetched", cc), mutations("Mutations", cc), setMutations("SetMutations", cc),
 		    clearRangeMutations("ClearRangeMutations", cc), atomicMutations("AtomicMutations", cc),
@@ -1114,6 +1117,7 @@ struct StorageServer {
 
 		this->storage.kvCommitLogicalBytes = &counters.kvCommitLogicalBytes;
 		this->storage.kvClearRanges = &counters.kvClearRanges;
+		this->storage.kvClearSingleKey = &counters.kvClearSingleKey;
 		this->storage.kvGets = &counters.kvGets;
 		this->storage.kvScans = &counters.kvScans;
 		this->storage.kvCommits = &counters.kvCommits;
@@ -7813,6 +7817,9 @@ void setAssignedStatus(StorageServer* self, KeyRangeRef keys, bool nowAssigned)
 void StorageServerDisk::clearRange(KeyRangeRef keys) {
 	storage->clear(keys, &data->metrics);
 	++(*kvClearRanges);
+	if (keys.singleKeyRange()) {
+		++(*kvClearSingleKey);
+	}
 }
 
 void StorageServerDisk::writeKeyValue(KeyValueRef kv) {
@@ -7827,6 +7834,9 @@ void StorageServerDisk::writeMutation(MutationRef mutation) {
 	} else if (mutation.type == MutationRef::ClearRange) {
 		storage->clear(KeyRangeRef(mutation.param1, mutation.param2), &data->metrics);
 		++(*kvClearRanges);
+		if (KeyRangeRef(mutation.param1, mutation.param2).singleKeyRange()) {
+			++(*kvClearSingleKey);
+		}
 	} else
 		ASSERT(false);
 }
@@ -7842,6 +7852,9 @@ void StorageServerDisk::writeMutations(const VectorRef<MutationRef>& mutations,
 		} else if (m.type == MutationRef::ClearRange) {
 			storage->clear(KeyRangeRef(m.param1, m.param2), &data->metrics);
 			++(*kvClearRanges);
+			if (KeyRangeRef(m.param1, m.param2).singleKeyRange()) {
+				++(*kvClearSingleKey);
+			}
 		}
 	}
 }

From 32a9c5a797067e71f216e7bc7930140bb368cd79 Mon Sep 17 00:00:00 2001
From: Hao Fu <77984096+hfu94@users.noreply.github.com>
Date: Fri, 11 Nov 2022 13:36:06 -0800
Subject: [PATCH 016/468] add bytelimit for prefetch

This is a patch to release-7.1 after resolving conflicts from commit in
main branch, in order to enable byteLimit in release-7.1

A fraction of byteLimit will be used as the limit to fetch index.
For the indexes fetched, fetch records for them in batch.

byteLimit always count the index size, it also count record if exist,
it at least return 1 index-record entry and always include the last entry
despite that adding the last entry despite it might exceed limit.

There is a Knob STRICTLY_ENFORCE_BYTE_LIMIT, when it is set, records
will be discarded once the byteLimit is hit, despite they are fetched.
Otherwise, return the whole batch.
---
 fdbclient/NativeAPI.actor.cpp                |  4 +-
 fdbclient/ServerKnobs.cpp                    |  6 +-
 fdbclient/ServerKnobs.h                      |  2 +
 fdbserver/storageserver.actor.cpp            | 47 ++++++++++----
 fdbserver/workloads/GetMappedRange.actor.cpp | 64 ++++++++++++++++++--
 5 files changed, 101 insertions(+), 22 deletions(-)

diff --git a/fdbclient/NativeAPI.actor.cpp b/fdbclient/NativeAPI.actor.cpp
index 978c708c643..22e92155aa5 100644
--- a/fdbclient/NativeAPI.actor.cpp
+++ b/fdbclient/NativeAPI.actor.cpp
@@ -4399,9 +4399,11 @@ Future<RangeResultFamily> getRange(Reference<TransactionState> trState,
 					output.readToBegin = readToBegin;
 					output.readThroughEnd = readThroughEnd;
 
-					if (BUGGIFY && limits.hasByteLimit() && output.size() > std::max(1, originalLimits.minRows)) {
+					if (BUGGIFY && limits.hasByteLimit() && output.size() > std::max(1, originalLimits.minRows) &&
+					    (!std::is_same<GetKeyValuesFamilyRequest, GetMappedKeyValuesRequest>::value)) {
 						// Copy instead of resizing because TSS maybe be using output's arena for comparison. This only
 						// happens in simulation so it's fine
+						// disable it on prefetch, because boundary entries serve as continuations
 						RangeResultFamily copy;
 						int newSize =
 						    deterministicRandom()->randomInt(std::max(1, originalLimits.minRows), output.size());
diff --git a/fdbclient/ServerKnobs.cpp b/fdbclient/ServerKnobs.cpp
index b74124ee919..72d3c7a2ec5 100644
--- a/fdbclient/ServerKnobs.cpp
+++ b/fdbclient/ServerKnobs.cpp
@@ -738,11 +738,13 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi
 	init( RANGESTREAM_LIMIT_BYTES,                               2e6 ); if( randomize && BUGGIFY ) RANGESTREAM_LIMIT_BYTES = 1;
 	init( CHANGEFEEDSTREAM_LIMIT_BYTES,                          1e6 ); if( randomize && BUGGIFY ) CHANGEFEEDSTREAM_LIMIT_BYTES = 1;
 	init( BLOBWORKERSTATUSSTREAM_LIMIT_BYTES,                    1e4 ); if( randomize && BUGGIFY ) BLOBWORKERSTATUSSTREAM_LIMIT_BYTES = 1;
-	init( ENABLE_CLEAR_RANGE_EAGER_READS,                       true ); if( randomize && BUGGIFY ) ENABLE_CLEAR_RANGE_EAGER_READS = deterministicRandom()->coinflip() ? false : true;
+	init( ENABLE_CLEAR_RANGE_EAGER_READS,                       true ); if( randomize && BUGGIFY ) ENABLE_CLEAR_RANGE_EAGER_READS = deterministicRandom()->coinflip();
 	init( CHECKPOINT_TRANSFER_BLOCK_BYTES,                      40e6 );
 	init( QUICK_GET_VALUE_FALLBACK,                             true );
 	init( QUICK_GET_KEY_VALUES_FALLBACK,                        true );
-	init( MAX_PARALLEL_QUICK_GET_VALUE,                           50 ); if ( randomize && BUGGIFY ) MAX_PARALLEL_QUICK_GET_VALUE = deterministicRandom()->randomInt(1, 100);
+	init( STRICTLY_ENFORCE_BYTE_LIMIT,                          false); if( randomize && BUGGIFY ) STRICTLY_ENFORCE_BYTE_LIMIT = deterministicRandom()->coinflip();
+	init( FRACTION_INDEX_BYTELIMIT_PREFETCH,                      0.2); if( randomize && BUGGIFY ) FRACTION_INDEX_BYTELIMIT_PREFETCH = 0.01 + deterministicRandom()->random01();
+	init( MAX_PARALLEL_QUICK_GET_VALUE,                           10 ); if ( randomize && BUGGIFY ) MAX_PARALLEL_QUICK_GET_VALUE = deterministicRandom()->randomInt(1, 100);
 	init( QUICK_GET_KEY_VALUES_LIMIT,                           2000 );
 	init( QUICK_GET_KEY_VALUES_LIMIT_BYTES,                      1e7 );
 
diff --git a/fdbclient/ServerKnobs.h b/fdbclient/ServerKnobs.h
index 26b7b787c73..35532512904 100644
--- a/fdbclient/ServerKnobs.h
+++ b/fdbclient/ServerKnobs.h
@@ -676,6 +676,8 @@ class ServerKnobs : public KnobsImpl<ServerKnobs> {
 	bool ENABLE_CLEAR_RANGE_EAGER_READS;
 	bool QUICK_GET_VALUE_FALLBACK;
 	bool QUICK_GET_KEY_VALUES_FALLBACK;
+	bool STRICTLY_ENFORCE_BYTE_LIMIT;
+	double FRACTION_INDEX_BYTELIMIT_PREFETCH;
 	int MAX_PARALLEL_QUICK_GET_VALUE;
 	int CHECKPOINT_TRANSFER_BLOCK_BYTES;
 	int QUICK_GET_KEY_VALUES_LIMIT;
diff --git a/fdbserver/storageserver.actor.cpp b/fdbserver/storageserver.actor.cpp
index 2dd73443302..4fb179a0508 100644
--- a/fdbserver/storageserver.actor.cpp
+++ b/fdbserver/storageserver.actor.cpp
@@ -3851,15 +3851,29 @@ ACTOR Future<Void> mapSubquery(StorageServer* data,
 	return Void();
 }
 
+int getMappedKeyValueSize(MappedKeyValueRef mappedKeyValue) {
+	auto& reqAndResult = mappedKeyValue.reqAndResult;
+	int bytes = 0;
+	if (std::holds_alternative<GetValueReqAndResultRef>(reqAndResult)) {
+		const auto& getValue = std::get<GetValueReqAndResultRef>(reqAndResult);
+		bytes = getValue.expectedSize();
+	} else if (std::holds_alternative<GetRangeReqAndResultRef>(reqAndResult)) {
+		const auto& getRange = std::get<GetRangeReqAndResultRef>(reqAndResult);
+		bytes = getRange.result.expectedSize();
+	} else {
+		throw internal_error();
+	}
+	return bytes;
+}
+
 ACTOR Future<GetMappedKeyValuesReply> mapKeyValues(StorageServer* data,
                                                    GetKeyValuesReply input,
                                                    StringRef mapper,
                                                    // To provide span context, tags, debug ID to underlying lookups.
                                                    GetMappedKeyValuesRequest* pOriginalReq,
-                                                   Optional<Key> tenantPrefix) {
+                                                   int* remainingLimitBytes) {
 	state GetMappedKeyValuesReply result;
 	result.version = input.version;
-	result.more = input.more;
 	result.cached = input.cached;
 	result.arena.dependsOn(input.arena);
 
@@ -3887,7 +3901,7 @@ ACTOR Future<GetMappedKeyValuesReply> mapKeyValues(StorageServer* data,
 	if (pOriginalReq->debugID.present())
 		g_traceBatch.addEvent(
 		    "TransactionDebug", pOriginalReq->debugID.get().first(), "storageserver.mapKeyValues.BeforeLoop");
-	for (; offset < sz; offset += SERVER_KNOBS->MAX_PARALLEL_QUICK_GET_VALUE) {
+	for (; offset<sz&& * remainingLimitBytes> 0; offset += SERVER_KNOBS->MAX_PARALLEL_QUICK_GET_VALUE) {
 		// Divide into batches of MAX_PARALLEL_QUICK_GET_VALUE subqueries
 		for (int i = 0; i + offset < sz && i < SERVER_KNOBS->MAX_PARALLEL_QUICK_GET_VALUE; i++) {
 			KeyValueRef* it = &input.data[i + offset];
@@ -3911,12 +3925,21 @@ ACTOR Future<GetMappedKeyValuesReply> mapKeyValues(StorageServer* data,
 			    "TransactionDebug", pOriginalReq->debugID.get().first(), "storageserver.mapKeyValues.AfterBatch");
 		subqueries.clear();
 		for (int i = 0; i + offset < sz && i < SERVER_KNOBS->MAX_PARALLEL_QUICK_GET_VALUE; i++) {
+			// since we always read the index, so always consider the index size
+			int indexSize = sizeof(KeyValueRef) + input.data[i + offset].expectedSize();
+			int size = indexSize + getMappedKeyValueSize(kvms[i]);
+			*remainingLimitBytes -= size;
 			result.data.push_back(result.arena, kvms[i]);
+			if (SERVER_KNOBS->STRICTLY_ENFORCE_BYTE_LIMIT && *remainingLimitBytes <= 0) {
+				break;
+			}
 		}
 	}
 	if (pOriginalReq->debugID.present())
 		g_traceBatch.addEvent(
 		    "TransactionDebug", pOriginalReq->debugID.get().first(), "storageserver.mapKeyValues.AfterAll");
+
+	result.more = input.more || result.data.size() < sz;
 	return result;
 }
 
@@ -4058,23 +4081,21 @@ ACTOR Future<Void> getMappedKeyValuesQ(StorageServer* data, GetMappedKeyValuesRe
 			req.reply.send(none);
 		} else {
 			state int remainingLimitBytes = req.limitBytes;
-
-			GetKeyValuesReply getKeyValuesReply = wait(readRange(data,
-			                                                     version,
-			                                                     KeyRangeRef(begin, end),
-			                                                     req.limit,
-			                                                     &remainingLimitBytes,
-			                                                     span.context,
-			                                                     type,
-			                                                     tenantPrefix));
+			// create a temporary byte limit for index fetching ONLY, this should be excessive
+			// because readRange is cheap when reading additional bytes
+			state int bytesForIndex =
+			    std::min(req.limitBytes, (int)(req.limitBytes * SERVER_KNOBS->FRACTION_INDEX_BYTELIMIT_PREFETCH));
+			GetKeyValuesReply getKeyValuesReply = wait(readRange(
+			    data, version, KeyRangeRef(begin, end), req.limit, &bytesForIndex, span.context, type, tenantPrefix));
 
 			state GetMappedKeyValuesReply r;
 			try {
 				// Map the scanned range to another list of keys and look up.
 				GetMappedKeyValuesReply _r =
-				    wait(mapKeyValues(data, getKeyValuesReply, req.mapper, &req, tenantPrefix));
+				    wait(mapKeyValues(data, getKeyValuesReply, req.mapper, &req, &remainingLimitBytes));
 				r = _r;
 			} catch (Error& e) {
+				// catch txn_too_old here if prefetch runs for too long, and returns it back to client
 				TraceEvent("MapError").error(e);
 				throw;
 			}
diff --git a/fdbserver/workloads/GetMappedRange.actor.cpp b/fdbserver/workloads/GetMappedRange.actor.cpp
index cb454b6ef6c..abbf2c248f5 100644
--- a/fdbserver/workloads/GetMappedRange.actor.cpp
+++ b/fdbserver/workloads/GetMappedRange.actor.cpp
@@ -38,6 +38,8 @@ const KeyRef prefix = "prefix"_sr;
 const KeyRef RECORD = "RECORD"_sr;
 const KeyRef INDEX = "INDEX"_sr;
 
+int recordSize;
+int indexSize;
 struct GetMappedRangeWorkload : ApiWorkload {
 	bool enabled;
 	Snapshot snapshot = Snapshot::False;
@@ -98,19 +100,32 @@ struct GetMappedRangeWorkload : ApiWorkload {
 		loop {
 			std::cout << "start fillInRecords n=" << n << std::endl;
 			// TODO: When n is large, split into multiple transactions.
+			recordSize = 0;
+			indexSize = 0;
 			try {
 				for (int i = 0; i < n; i++) {
 					if (self->SPLIT_RECORDS) {
 						for (int split = 0; split < SPLIT_SIZE; split++) {
 							tr.set(recordKey(i, split), recordValue(i, split));
+							if (i == 0) {
+								recordSize +=
+								    recordKey(i, split).size() + recordValue(i, split).size() + sizeof(KeyValueRef);
+							}
 						}
 					} else {
 						tr.set(recordKey(i), recordValue(i));
+						if (i == 0) {
+							recordSize += recordKey(i).size() + recordValue(i).size() + sizeof(KeyValueRef);
+						}
 					}
 					tr.set(indexEntryKey(i), EMPTY);
+					if (i == 0) {
+						indexSize += indexEntryKey(i).size() + sizeof(KeyValueRef);
+					}
 				}
 				wait(tr.commit());
-				std::cout << "finished fillInRecords with version " << tr.getCommittedVersion() << std::endl;
+				std::cout << "finished fillInRecords with version " << tr.getCommittedVersion() << " recordSize "
+				          << recordSize << " indexSize " << indexSize << std::endl;
 				break;
 			} catch (Error& e) {
 				std::cout << "failed fillInRecords, retry" << std::endl;
@@ -188,17 +203,23 @@ struct GetMappedRangeWorkload : ApiWorkload {
 	                                                          KeySelector endSelector,
 	                                                          Key mapper,
 	                                                          int limit,
+	                                                          int byteLimit,
 	                                                          int expectedBeginId,
 	                                                          GetMappedRangeWorkload* self) {
 
 		std::cout << "start scanMappedRangeWithLimits beginSelector:" << beginSelector.toString()
 		          << " endSelector:" << endSelector.toString() << " expectedBeginId:" << expectedBeginId
-		          << " limit:" << limit << std::endl;
+		          << " limit:" << limit << " byteLimit: " << byteLimit << "  recordSize: " << recordSize
+		          << " STRICTLY_ENFORCE_BYTE_LIMIT: " << SERVER_KNOBS->STRICTLY_ENFORCE_BYTE_LIMIT << std::endl;
 		loop {
 			state Reference<TransactionWrapper> tr = self->createTransaction();
 			try {
-				MappedRangeResult result = wait(tr->getMappedRange(
-				    beginSelector, endSelector, mapper, GetRangeLimits(limit), self->snapshot, Reverse::False));
+				MappedRangeResult result = wait(tr->getMappedRange(beginSelector,
+				                                                   endSelector,
+				                                                   mapper,
+				                                                   GetRangeLimits(limit, byteLimit),
+				                                                   self->snapshot,
+				                                                   Reverse::False));
 				//			showResult(result);
 				if (self->BAD_MAPPER) {
 					TraceEvent("GetMappedRangeWorkloadShouldNotReachable").detail("ResultSize", result.size());
@@ -242,16 +263,43 @@ struct GetMappedRangeWorkload : ApiWorkload {
 		Key endTuple = Tuple().append(prefix).append(INDEX).append(indexKey(endId)).getDataAsStandalone();
 		state KeySelector endSelector = KeySelector(firstGreaterOrEqual(endTuple));
 		state int limit = 100;
+		state int byteLimit = deterministicRandom()->randomInt(1, 9) * 10000;
 		state int expectedBeginId = beginId;
+		std::cout << "ByteLimit: " << byteLimit << " limit: " << limit
+		          << " FRACTION_INDEX_BYTELIMIT_PREFETCH: " << SERVER_KNOBS->FRACTION_INDEX_BYTELIMIT_PREFETCH
+		          << " MAX_PARALLEL_QUICK_GET_VALUE: " << SERVER_KNOBS->MAX_PARALLEL_QUICK_GET_VALUE << std::endl;
 		while (true) {
-			MappedRangeResult result = wait(
-			    self->scanMappedRangeWithLimits(cx, beginSelector, endSelector, mapper, limit, expectedBeginId, self));
+			MappedRangeResult result = wait(self->scanMappedRangeWithLimits(
+			    cx, beginSelector, endSelector, mapper, limit, byteLimit, expectedBeginId, self));
 			expectedBeginId += result.size();
 			if (result.more) {
 				if (result.empty()) {
 					// This is usually not expected.
 					std::cout << "not result but have more, try again" << std::endl;
 				} else {
+					int size = indexSize + recordSize;
+					int expectedCnt = limit;
+					int indexByteLimit = byteLimit * SERVER_KNOBS->FRACTION_INDEX_BYTELIMIT_PREFETCH;
+					int indexCountByteLimit = indexByteLimit / indexSize + (indexByteLimit % indexSize != 0);
+					int indexCount = std::min(limit, indexCountByteLimit);
+					std::cout << "indexCount:  " << indexCount << std::endl;
+					// result set cannot be larger than the number of index fetched
+					ASSERT(result.size() <= indexCount);
+
+					expectedCnt = std::min(expectedCnt, indexCount);
+					int boundByRecord;
+					if (SERVER_KNOBS->STRICTLY_ENFORCE_BYTE_LIMIT) {
+						// might have 1 additional entry over the limit
+						boundByRecord = byteLimit / size + (byteLimit % size != 0);
+					} else {
+						// might have 1 additional batch over the limit
+						int roundSize = size * SERVER_KNOBS->MAX_PARALLEL_QUICK_GET_VALUE;
+						int round = byteLimit / roundSize + (byteLimit % roundSize != 0);
+						boundByRecord = round * SERVER_KNOBS->MAX_PARALLEL_QUICK_GET_VALUE;
+					}
+					expectedCnt = std::min(expectedCnt, boundByRecord);
+					std::cout << "boundByRecord:  " << boundByRecord << std::endl;
+					ASSERT(result.size() == expectedCnt);
 					beginSelector = KeySelector(firstGreaterThan(result.back().key));
 				}
 			} else {
@@ -260,6 +308,7 @@ struct GetMappedRangeWorkload : ApiWorkload {
 			}
 		}
 		ASSERT(expectedBeginId == endId);
+
 		return Void();
 	}
 
@@ -394,7 +443,10 @@ struct GetMappedRangeWorkload : ApiWorkload {
 		Key mapper = getMapper(self);
 		// The scanned range cannot be too large to hit get_mapped_key_values_has_more. We have a unit validating the
 		// error is thrown when the range is large.
+		state bool originalStrictlyEnforeByteLimit = SERVER_KNOBS->STRICTLY_ENFORCE_BYTE_LIMIT;
+		(const_cast<ServerKnobs*> SERVER_KNOBS)->STRICTLY_ENFORCE_BYTE_LIMIT = deterministicRandom()->coinflip();
 		wait(self->scanMappedRange(cx, 10, 490, mapper, self));
+		(const_cast<ServerKnobs*> SERVER_KNOBS)->STRICTLY_ENFORCE_BYTE_LIMIT = originalStrictlyEnforeByteLimit;
 		return Void();
 	}
 

From fadcb0820c8a5901bbefe70825bd7a77ebc93081 Mon Sep 17 00:00:00 2001
From: hao fu <hfu5@apple.com>
Date: Mon, 14 Nov 2022 15:44:29 -0800
Subject: [PATCH 017/468] debug seg fault

---
 bindings/java/fdbJNI.cpp          | 14 ++++++++++++++
 fdbserver/storageserver.actor.cpp |  1 +
 2 files changed, 15 insertions(+)

diff --git a/bindings/java/fdbJNI.cpp b/bindings/java/fdbJNI.cpp
index f44d52e1690..c8b60f8afe9 100644
--- a/bindings/java/fdbJNI.cpp
+++ b/bindings/java/fdbJNI.cpp
@@ -492,11 +492,15 @@ class ExecuteOnLeave {
 };
 
 void cpBytesAndLengthInner(uint8_t*& pByte, jint*& pLength, const uint8_t* data, const int& length) {
+	fprintf(stderr, "Hfu5: cpBytesAndLengthInner start, length index %d, length %d, byte %p.\n", *pLength, length, pByte);
+	fprintf(stdout, "Hfu5: cpBytesAndLengthInner start, length index %d, length %d, byte %p.\n", *pLength, length, pByte);
 	*pLength = length;
 	pLength++;
 
 	memcpy(pByte, data, length);
 	pByte += length;
+	fprintf(stderr, "Hfu5: cpBytesAndLengthInner end, length index %d, length %d, byte %p.\n", *pLength, length, pByte);
+	fprintf(stdout, "Hfu5: cpBytesAndLengthInner end, length index %d, length %d, byte %p.\n", *pLength, length, pByte);
 }
 
 void cpBytesAndLength(uint8_t*& pByte, jint*& pLength, const FDBKey& key) {
@@ -534,12 +538,19 @@ JNIEXPORT jobject JNICALL Java_com_apple_foundationdb_FutureMappedResults_Future
 		int kvm_count = kvm.getRange.m_size;
 
 		const int totalLengths = 4 + kvm_count * 2;
+		fprintf(stderr, "Hfu5: index %d, totalLengths %d.\n", i, totalLengths);
+		fprintf(stdout, "Hfu5: index %d, totalLengths %d.\n", i, totalLengths);
 
 		int totalBytes = kvm.key.key_length + kvm.value.key_length + kvm.getRange.begin.key.key_length +
 		                 kvm.getRange.end.key.key_length;
+
+		fprintf(stderr, "Hfu5: index %d, totalBytes %d.\n", i, totalBytes);
+		fprintf(stdout, "Hfu5: index %d, totalBytes %d.\n", i, totalBytes);
 		for (int i = 0; i < kvm_count; i++) {
 			auto kv = kvm.getRange.data[i];
 			totalBytes += kv.key_length + kv.value_length;
+			fprintf(stdout, "Hfu5: index %d, totalBytes each %d, key: %d, v: %d.\n", i, totalBytes, kv.key_length, kv.value_length);
+			fprintf(stderr, "Hfu5: index %d, totalBytes each %d, key: %d, v: %d.\n", i, totalBytes, kv.key_length, kv.value_length);
 		}
 
 		jbyteArray bytesArray = jenv->NewByteArray(totalBytes);
@@ -587,6 +598,9 @@ JNIEXPORT jobject JNICALL Java_com_apple_foundationdb_FutureMappedResults_Future
 				}
 			}
 		}
+		fprintf(stderr, "Hfu5: Finish %d.\n", totalBytes);
+		fprintf(stdout, "Hfu5: Finish %d.\n", totalBytes);
+
 		// After native arrays are released
 		jobject mkv = jenv->CallStaticObjectMethod(
 		    mapped_key_value_class, mapped_key_value_from_bytes, (jbyteArray)bytesArray, (jintArray)lengthArray);
diff --git a/fdbserver/storageserver.actor.cpp b/fdbserver/storageserver.actor.cpp
index 4fb179a0508..516a9220c5c 100644
--- a/fdbserver/storageserver.actor.cpp
+++ b/fdbserver/storageserver.actor.cpp
@@ -3928,6 +3928,7 @@ ACTOR Future<GetMappedKeyValuesReply> mapKeyValues(StorageServer* data,
 			// since we always read the index, so always consider the index size
 			int indexSize = sizeof(KeyValueRef) + input.data[i + offset].expectedSize();
 			int size = indexSize + getMappedKeyValueSize(kvms[i]);
+			TraceEvent("Hfu5Add").detail("IndexSize", indexSize).detail("Index", i).detail("RecordSize", getMappedKeyValueSize(kvms[i]));
 			*remainingLimitBytes -= size;
 			result.data.push_back(result.arena, kvms[i]);
 			if (SERVER_KNOBS->STRICTLY_ENFORCE_BYTE_LIMIT && *remainingLimitBytes <= 0) {

From f6dc3bab9671421d13e2a4db2609d9d15b2b4cdc Mon Sep 17 00:00:00 2001
From: hao fu <hfu5@apple.com>
Date: Tue, 15 Nov 2022 11:30:35 -0800
Subject: [PATCH 018/468] Revert "debug seg fault"

This reverts commit fadcb0820c8a5901bbefe70825bd7a77ebc93081.
---
 bindings/java/fdbJNI.cpp          | 14 --------------
 fdbserver/storageserver.actor.cpp |  1 -
 2 files changed, 15 deletions(-)

diff --git a/bindings/java/fdbJNI.cpp b/bindings/java/fdbJNI.cpp
index c8b60f8afe9..f44d52e1690 100644
--- a/bindings/java/fdbJNI.cpp
+++ b/bindings/java/fdbJNI.cpp
@@ -492,15 +492,11 @@ class ExecuteOnLeave {
 };
 
 void cpBytesAndLengthInner(uint8_t*& pByte, jint*& pLength, const uint8_t* data, const int& length) {
-	fprintf(stderr, "Hfu5: cpBytesAndLengthInner start, length index %d, length %d, byte %p.\n", *pLength, length, pByte);
-	fprintf(stdout, "Hfu5: cpBytesAndLengthInner start, length index %d, length %d, byte %p.\n", *pLength, length, pByte);
 	*pLength = length;
 	pLength++;
 
 	memcpy(pByte, data, length);
 	pByte += length;
-	fprintf(stderr, "Hfu5: cpBytesAndLengthInner end, length index %d, length %d, byte %p.\n", *pLength, length, pByte);
-	fprintf(stdout, "Hfu5: cpBytesAndLengthInner end, length index %d, length %d, byte %p.\n", *pLength, length, pByte);
 }
 
 void cpBytesAndLength(uint8_t*& pByte, jint*& pLength, const FDBKey& key) {
@@ -538,19 +534,12 @@ JNIEXPORT jobject JNICALL Java_com_apple_foundationdb_FutureMappedResults_Future
 		int kvm_count = kvm.getRange.m_size;
 
 		const int totalLengths = 4 + kvm_count * 2;
-		fprintf(stderr, "Hfu5: index %d, totalLengths %d.\n", i, totalLengths);
-		fprintf(stdout, "Hfu5: index %d, totalLengths %d.\n", i, totalLengths);
 
 		int totalBytes = kvm.key.key_length + kvm.value.key_length + kvm.getRange.begin.key.key_length +
 		                 kvm.getRange.end.key.key_length;
-
-		fprintf(stderr, "Hfu5: index %d, totalBytes %d.\n", i, totalBytes);
-		fprintf(stdout, "Hfu5: index %d, totalBytes %d.\n", i, totalBytes);
 		for (int i = 0; i < kvm_count; i++) {
 			auto kv = kvm.getRange.data[i];
 			totalBytes += kv.key_length + kv.value_length;
-			fprintf(stdout, "Hfu5: index %d, totalBytes each %d, key: %d, v: %d.\n", i, totalBytes, kv.key_length, kv.value_length);
-			fprintf(stderr, "Hfu5: index %d, totalBytes each %d, key: %d, v: %d.\n", i, totalBytes, kv.key_length, kv.value_length);
 		}
 
 		jbyteArray bytesArray = jenv->NewByteArray(totalBytes);
@@ -598,9 +587,6 @@ JNIEXPORT jobject JNICALL Java_com_apple_foundationdb_FutureMappedResults_Future
 				}
 			}
 		}
-		fprintf(stderr, "Hfu5: Finish %d.\n", totalBytes);
-		fprintf(stdout, "Hfu5: Finish %d.\n", totalBytes);
-
 		// After native arrays are released
 		jobject mkv = jenv->CallStaticObjectMethod(
 		    mapped_key_value_class, mapped_key_value_from_bytes, (jbyteArray)bytesArray, (jintArray)lengthArray);
diff --git a/fdbserver/storageserver.actor.cpp b/fdbserver/storageserver.actor.cpp
index 516a9220c5c..4fb179a0508 100644
--- a/fdbserver/storageserver.actor.cpp
+++ b/fdbserver/storageserver.actor.cpp
@@ -3928,7 +3928,6 @@ ACTOR Future<GetMappedKeyValuesReply> mapKeyValues(StorageServer* data,
 			// since we always read the index, so always consider the index size
 			int indexSize = sizeof(KeyValueRef) + input.data[i + offset].expectedSize();
 			int size = indexSize + getMappedKeyValueSize(kvms[i]);
-			TraceEvent("Hfu5Add").detail("IndexSize", indexSize).detail("Index", i).detail("RecordSize", getMappedKeyValueSize(kvms[i]));
 			*remainingLimitBytes -= size;
 			result.data.push_back(result.arena, kvms[i]);
 			if (SERVER_KNOBS->STRICTLY_ENFORCE_BYTE_LIMIT && *remainingLimitBytes <= 0) {

From f9338d636ab58441edc090eba4c7899f52e68d07 Mon Sep 17 00:00:00 2001
From: Yao Xiao <87789492+yao-xiao-github@users.noreply.github.com>
Date: Wed, 16 Nov 2022 14:52:36 -0800
Subject: [PATCH 019/468] [release-7.1] Add SS read range bytes metrics.
 (#8697) (#8724)

* Add SS read range bytes metrics. (#8697)

* Fix build failure

* clang-fmt

* fmt
---
 fdbserver/StorageMetrics.actor.h  | 38 ++++++++++++++++---------------
 fdbserver/storageserver.actor.cpp | 13 +++++++++++
 2 files changed, 33 insertions(+), 18 deletions(-)

diff --git a/fdbserver/StorageMetrics.actor.h b/fdbserver/StorageMetrics.actor.h
index fa35fffdca1..c9df24b53b6 100644
--- a/fdbserver/StorageMetrics.actor.h
+++ b/fdbserver/StorageMetrics.actor.h
@@ -39,6 +39,8 @@ const StringRef TLOG_MSGS_PTREE_UPDATES_LATENCY_HISTOGRAM = LiteralStringRef("TL
 const StringRef STORAGE_UPDATES_DURABLE_LATENCY_HISTOGRAM = LiteralStringRef("StorageUpdatesDurableLatency");
 const StringRef STORAGE_COMMIT_LATENCY_HISTOGRAM = LiteralStringRef("StorageCommitLatency");
 const StringRef SS_DURABLE_VERSION_UPDATE_LATENCY_HISTOGRAM = LiteralStringRef("SSDurableVersionUpdateLatency");
+const StringRef SS_READ_RANGE_BYTES_RETURNED_HISTOGRAM = LiteralStringRef("SSReadRangeBytesReturned");
+const StringRef SS_READ_RANGE_BYTES_LIMIT_HISTOGRAM = LiteralStringRef("SSReadRangeBytesLimit");
 
 struct StorageMetricSample {
 	IndexedSet<Key, int64_t> sample;
@@ -93,24 +95,6 @@ struct StorageMetricSample {
 	}
 };
 
-TEST_CASE("/fdbserver/StorageMetricSample/simple") {
-	StorageMetricSample s(1000);
-	s.sample.insert(LiteralStringRef("Apple"), 1000);
-	s.sample.insert(LiteralStringRef("Banana"), 2000);
-	s.sample.insert(LiteralStringRef("Cat"), 1000);
-	s.sample.insert(LiteralStringRef("Cathode"), 1000);
-	s.sample.insert(LiteralStringRef("Dog"), 1000);
-
-	ASSERT(s.getEstimate(KeyRangeRef(LiteralStringRef("A"), LiteralStringRef("D"))) == 5000);
-	ASSERT(s.getEstimate(KeyRangeRef(LiteralStringRef("A"), LiteralStringRef("E"))) == 6000);
-	ASSERT(s.getEstimate(KeyRangeRef(LiteralStringRef("B"), LiteralStringRef("C"))) == 2000);
-
-	// ASSERT(s.splitEstimate(KeyRangeRef(LiteralStringRef("A"), LiteralStringRef("D")), 3500) ==
-	// LiteralStringRef("Cat"));
-
-	return Void();
-}
-
 struct TransientStorageMetricSample : StorageMetricSample {
 	Deque<std::pair<double, std::pair<Key, int64_t>>> queue;
 
@@ -615,6 +599,24 @@ struct StorageServerMetrics {
 	}
 };
 
+TEST_CASE("/fdbserver/StorageMetricSample/simple") {
+	StorageMetricSample s(1000);
+	s.sample.insert(LiteralStringRef("Apple"), 1000);
+	s.sample.insert(LiteralStringRef("Banana"), 2000);
+	s.sample.insert(LiteralStringRef("Cat"), 1000);
+	s.sample.insert(LiteralStringRef("Cathode"), 1000);
+	s.sample.insert(LiteralStringRef("Dog"), 1000);
+
+	ASSERT(s.getEstimate(KeyRangeRef(LiteralStringRef("A"), LiteralStringRef("D"))) == 5000);
+	ASSERT(s.getEstimate(KeyRangeRef(LiteralStringRef("A"), LiteralStringRef("E"))) == 6000);
+	ASSERT(s.getEstimate(KeyRangeRef(LiteralStringRef("B"), LiteralStringRef("C"))) == 2000);
+
+	// ASSERT(s.splitEstimate(KeyRangeRef(LiteralStringRef("A"), LiteralStringRef("D")), 3500) ==
+	// LiteralStringRef("Cat"));
+
+	return Void();
+}
+
 TEST_CASE("/fdbserver/StorageMetricSample/rangeSplitPoints/simple") {
 
 	int64_t sampleUnit = SERVER_KNOBS->BYTES_READ_UNITS_PER_SAMPLE;
diff --git a/fdbserver/storageserver.actor.cpp b/fdbserver/storageserver.actor.cpp
index 2dd73443302..1d83b17a4fc 100644
--- a/fdbserver/storageserver.actor.cpp
+++ b/fdbserver/storageserver.actor.cpp
@@ -566,6 +566,9 @@ struct StorageServer {
 	Reference<Histogram> storageUpdatesDurableLatencyHistogram;
 	Reference<Histogram> storageCommitLatencyHistogram;
 	Reference<Histogram> ssDurableVersionUpdateLatencyHistogram;
+	// Histograms of requests sent to KVS.
+	Reference<Histogram> readRangeBytesReturnedHistogram;
+	Reference<Histogram> readRangeBytesLimitHistogram;
 
 	// watch map operations
 	Reference<ServerWatchMetadata> getWatchMetadata(KeyRef key) const;
@@ -1088,6 +1091,12 @@ struct StorageServer {
 	    ssDurableVersionUpdateLatencyHistogram(Histogram::getHistogram(STORAGESERVER_HISTOGRAM_GROUP,
 	                                                                   SS_DURABLE_VERSION_UPDATE_LATENCY_HISTOGRAM,
 	                                                                   Histogram::Unit::microseconds)),
+	    readRangeBytesReturnedHistogram(Histogram::getHistogram(STORAGESERVER_HISTOGRAM_GROUP,
+	                                                            SS_READ_RANGE_BYTES_RETURNED_HISTOGRAM,
+	                                                            Histogram::Unit::bytes)),
+	    readRangeBytesLimitHistogram(Histogram::getHistogram(STORAGESERVER_HISTOGRAM_GROUP,
+	                                                         SS_READ_RANGE_BYTES_LIMIT_HISTOGRAM,
+	                                                         Histogram::Unit::bytes)),
 	    tag(invalidTag), poppedAllAfter(std::numeric_limits<Version>::max()), cpuUsage(0.0), diskUsage(0.0),
 	    storage(this, storage), shardChangeCounter(0), lastTLogVersion(0), lastVersionWithData(0), restoredVersion(0),
 	    prevVersion(0), rebootAfterDurableVersion(std::numeric_limits<Version>::max()),
@@ -3018,6 +3027,8 @@ ACTOR Future<GetKeyValuesReply> readRange(StorageServer* data,
 			RangeResult atStorageVersion =
 			    wait(data->storage.readRange(KeyRangeRef(readBegin, readEnd), limit, *pLimitBytes, type));
 			data->counters.kvScanBytes += atStorageVersion.logicalSize();
+			data->readRangeBytesReturnedHistogram->sample(atStorageVersion.logicalSize());
+			data->readRangeBytesLimitHistogram->sample(*pLimitBytes);
 
 			ASSERT(atStorageVersion.size() <= limit);
 			if (data->storageVersion() > version) {
@@ -3117,6 +3128,8 @@ ACTOR Future<GetKeyValuesReply> readRange(StorageServer* data,
 			RangeResult atStorageVersion =
 			    wait(data->storage.readRange(KeyRangeRef(readBegin, readEnd), limit, *pLimitBytes, type));
 			data->counters.kvScanBytes += atStorageVersion.logicalSize();
+			data->readRangeBytesReturnedHistogram->sample(atStorageVersion.logicalSize());
+			data->readRangeBytesLimitHistogram->sample(*pLimitBytes);
 
 			ASSERT(atStorageVersion.size() <= -limit);
 			if (data->storageVersion() > version) {

From 074e6f7d231bd05a2ecfd88d6b5d4670554a3c2d Mon Sep 17 00:00:00 2001
From: neethuhaneesha <nbingi@apple.com>
Date: Wed, 16 Nov 2022 15:08:39 -0800
Subject: [PATCH 020/468] Rocksdb suggest compact range checks

---
 fdbclient/ServerKnobs.cpp                |  1 +
 fdbclient/ServerKnobs.h                  |  2 +-
 fdbserver/KeyValueStoreRocksDB.actor.cpp | 23 ++++++++++++++---------
 3 files changed, 16 insertions(+), 10 deletions(-)

diff --git a/fdbclient/ServerKnobs.cpp b/fdbclient/ServerKnobs.cpp
index 72d3c7a2ec5..005a3648a06 100644
--- a/fdbclient/ServerKnobs.cpp
+++ b/fdbclient/ServerKnobs.cpp
@@ -413,6 +413,7 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi
 	init( ROCKSDB_SINGLEKEY_DELETES_BYTES_LIMIT,              200000 ); // 200KB
 	init( ROCKSDB_ENABLE_CLEAR_RANGE_EAGER_READS,               true ); if( randomize && BUGGIFY ) ROCKSDB_ENABLE_CLEAR_RANGE_EAGER_READS = deterministicRandom()->coinflip();
 	// ROCKSDB_STATS_LEVEL=1 indicates rocksdb::StatsLevel::kExceptHistogramOrTimers
+	// Refer StatsLevel: https://github.com/facebook/rocksdb/blob/main/include/rocksdb/statistics.h#L594
 	init( ROCKSDB_STATS_LEVEL,                                     1 ); if( randomize && BUGGIFY ) ROCKSDB_STATS_LEVEL = deterministicRandom()->randomInt(0, 6);
 	// Can commit will delay ROCKSDB_CAN_COMMIT_DELAY_ON_OVERLOAD seconds for
 	// ROCKSDB_CAN_COMMIT_DELAY_TIMES_ON_OVERLOAD times, if rocksdb overloaded.
diff --git a/fdbclient/ServerKnobs.h b/fdbclient/ServerKnobs.h
index 35532512904..6dff7a507a7 100644
--- a/fdbclient/ServerKnobs.h
+++ b/fdbclient/ServerKnobs.h
@@ -292,7 +292,7 @@ class ServerKnobs : public KnobsImpl<ServerKnobs> {
 
 	// KeyValueStoreRocksDB
 	bool ROCKSDB_LEVEL_COMPACTION_DYNAMIC_LEVEL_BYTES;
-	int ROCKSDB_SUGGEST_COMPACT_CLEAR_RANGE;
+	bool ROCKSDB_SUGGEST_COMPACT_CLEAR_RANGE;
 	int ROCKSDB_THREAD_PROMISE_PRIORITY;
 	int ROCKSDB_READER_THREAD_PRIORITY;
 	int ROCKSDB_WRITER_THREAD_PRIORITY;
diff --git a/fdbserver/KeyValueStoreRocksDB.actor.cpp b/fdbserver/KeyValueStoreRocksDB.actor.cpp
index ebab54bca49..e3c62e6ad74 100644
--- a/fdbserver/KeyValueStoreRocksDB.actor.cpp
+++ b/fdbserver/KeyValueStoreRocksDB.actor.cpp
@@ -899,6 +899,7 @@ ACTOR Future<Void> rocksDBMetricLogger(UID id,
 	};
 
 	// To control the rocksdb::StatsLevel, use ROCKSDB_STATS_LEVEL knob.
+	// Refer StatsLevel: https://github.com/facebook/rocksdb/blob/main/include/rocksdb/statistics.h#L594
 	state std::vector<std::pair<const char*, uint32_t>> histogramStats = {
 		{ "CompactionTime", rocksdb::COMPACTION_TIME }, // enabled if rocksdb::StatsLevel > kExceptTimers(2)
 		{ "CompactionCPUTime", rocksdb::COMPACTION_CPU_TIME }, // enabled if rocksdb::StatsLevel > kExceptTimers(2)
@@ -964,6 +965,7 @@ ACTOR Future<Void> rocksDBMetricLogger(UID id,
 		}
 
 		// None of the histogramStats are enabled unless the ROCKSDB_STATS_LEVEL > kExceptHistogramOrTimers(1)
+		// Refer StatsLevel: https://github.com/facebook/rocksdb/blob/main/include/rocksdb/statistics.h#L594
 		if (SERVER_KNOBS->ROCKSDB_STATS_LEVEL > rocksdb::kExceptHistogramOrTimers) {
 			for (auto& [name, histogram] : histogramStats) {
 				rocksdb::HistogramData histogram_data;
@@ -1233,15 +1235,18 @@ struct RocksDBKeyValueStore : IKeyValueStore {
 				    threadIndex, ROCKSDB_COMMIT_QUEUEWAIT_HISTOGRAM.toString(), commitBeginTime - a.startTime));
 			}
 			Standalone<VectorRef<KeyRangeRef>> deletes;
-			DeleteVisitor dv(deletes, deletes.arena());
-			rocksdb::Status s = a.batchToCommit->Iterate(&dv);
-			if (!s.ok()) {
-				logRocksDBError(id, s, "CommitDeleteVisitor");
-				a.done.sendError(statusToError(s));
-				return;
+			if (SERVER_KNOBS->ROCKSDB_SUGGEST_COMPACT_CLEAR_RANGE) {
+				DeleteVisitor dv(deletes, deletes.arena());
+				rocksdb::Status s = a.batchToCommit->Iterate(&dv);
+				if (!s.ok()) {
+					logRocksDBError(id, s, "CommitDeleteVisitor");
+					a.done.sendError(statusToError(s));
+					return;
+				}
+				// If there are any range deletes, we should have added them to be deleted.
+				ASSERT(!deletes.empty() || !a.batchToCommit->HasDeleteRange());
 			}
-			// If there are any range deletes, we should have added them to be deleted.
-			ASSERT(!deletes.empty() || !a.batchToCommit->HasDeleteRange());
+
 			rocksdb::WriteOptions options;
 			options.sync = !SERVER_KNOBS->ROCKSDB_UNSAFE_AUTO_FSYNC;
 			if (SERVER_KNOBS->ROCKSDB_DISABLE_WAL_EXPERIMENTAL) {
@@ -1255,7 +1260,7 @@ struct RocksDBKeyValueStore : IKeyValueStore {
 				// Request for batchToCommit bytes. If this request cannot be satisfied, the call is blocked.
 				rateLimiter->Request(a.batchToCommit->GetDataSize() /* bytes */, rocksdb::Env::IO_HIGH);
 			}
-			s = db->Write(options, a.batchToCommit.get());
+			rocksdb::Status s = db->Write(options, a.batchToCommit.get());
 			readIterPool->update();
 			double currTime = timer_monotonic();
 			sharedState->dbWriteLatency.addMeasurement(currTime - writeBeginTime);

From a7161277bfef0876a4db040ff6c06f0825f64836 Mon Sep 17 00:00:00 2001
From: neethuhaneesha <nbingi@apple.com>
Date: Thu, 17 Nov 2022 13:20:32 -0800
Subject: [PATCH 021/468] RocksDB 7.7.3 version upgrade

---
 cmake/CompileRocksDB.cmake               | 6 +++---
 fdbserver/KeyValueStoreRocksDB.actor.cpp | 9 +++------
 2 files changed, 6 insertions(+), 9 deletions(-)

diff --git a/cmake/CompileRocksDB.cmake b/cmake/CompileRocksDB.cmake
index 4634e57e7c5..bf38364458f 100644
--- a/cmake/CompileRocksDB.cmake
+++ b/cmake/CompileRocksDB.cmake
@@ -1,6 +1,6 @@
 # FindRocksDB
 
-find_package(RocksDB 6.27.3)
+find_package(RocksDB 7.7.3)
 
 include(ExternalProject)
 
@@ -37,8 +37,8 @@ if (RocksDB_FOUND)
       ${BINARY_DIR}/librocksdb.a)
 else()
   ExternalProject_Add(rocksdb
-    URL        https://github.com/facebook/rocksdb/archive/refs/tags/v6.27.3.tar.gz
-    URL_HASH   SHA256=ee29901749b9132692b26f0a6c1d693f47d1a9ed8e3771e60556afe80282bf58
+    URL        https://github.com/facebook/rocksdb/archive/refs/tags/v7.7.3.tar.gz
+    URL_HASH   SHA256=b8ac9784a342b2e314c821f6d701148912215666ac5e9bdbccd93cf3767cb611
     CMAKE_ARGS -DUSE_RTTI=1 -DPORTABLE=${PORTABLE_ROCKSDB}
                -DCMAKE_CXX_STANDARD=${CMAKE_CXX_STANDARD}
                -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}
diff --git a/fdbserver/KeyValueStoreRocksDB.actor.cpp b/fdbserver/KeyValueStoreRocksDB.actor.cpp
index e3c62e6ad74..c86ee8b9df1 100644
--- a/fdbserver/KeyValueStoreRocksDB.actor.cpp
+++ b/fdbserver/KeyValueStoreRocksDB.actor.cpp
@@ -68,12 +68,9 @@
 
 #ifdef SSD_ROCKSDB_EXPERIMENTAL
 
-// Enforcing rocksdb version to be 6.27.3 or greater.
-static_assert(ROCKSDB_MAJOR >= 6, "Unsupported rocksdb version. Update the rocksdb to 6.27.3 version");
-static_assert(ROCKSDB_MAJOR == 6 ? ROCKSDB_MINOR >= 27 : true,
-              "Unsupported rocksdb version. Update the rocksdb to 6.27.3 version");
-static_assert((ROCKSDB_MAJOR == 6 && ROCKSDB_MINOR == 27) ? ROCKSDB_PATCH >= 3 : true,
-              "Unsupported rocksdb version. Update the rocksdb to 6.27.3 version");
+// Enforcing rocksdb version to be 7.7.3.
+static_assert((ROCKSDB_MAJOR == 7 && ROCKSDB_MINOR == 7 && ROCKSDB_PATCH == 3),
+              "Unsupported rocksdb version. Update the rocksdb to 7.7.3 version");
 
 namespace {
 using rocksdb::BackgroundErrorReason;

From 5a129795fdf42afac2c544eaade18435a5680884 Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Fri, 18 Nov 2022 22:28:13 -0800
Subject: [PATCH 022/468] Fix backup worker assertion failure

The number of released bytes exceeds the number of acquired bytes in locks.
This is because the bytes counted towards release is calculated after a "wait",
when more bytes could be allocated.
---
 fdbserver/BackupWorker.actor.cpp | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/fdbserver/BackupWorker.actor.cpp b/fdbserver/BackupWorker.actor.cpp
index 789935bd2cd..0f45f29a33e 100644
--- a/fdbserver/BackupWorker.actor.cpp
+++ b/fdbserver/BackupWorker.actor.cpp
@@ -46,8 +46,8 @@ struct VersionedMessage {
 	Arena arena; // Keep a reference to the memory containing the message
 	size_t bytes; // arena's size when inserted, which can grow afterwards
 
-	VersionedMessage(LogMessageVersion v, StringRef m, const VectorRef<Tag>& t, const Arena& a)
-	  : version(v), message(m), tags(t), arena(a), bytes(a.getSize()) {}
+	VersionedMessage(LogMessageVersion v, StringRef m, const VectorRef<Tag>& t, const Arena& a, size_t n)
+	  : version(v), message(m), tags(t), arena(a), bytes(n) {}
 	Version getVersion() const { return version.version; }
 	uint32_t getSubVersion() const { return version.sub; }
 
@@ -924,15 +924,17 @@ ACTOR Future<Void> pullAsyncData(BackupData* self) {
 		// Note we aggressively peek (uncommitted) messages, but only committed
 		// messages/mutations will be flushed to disk/blob in uploadData().
 		while (r->hasMessage()) {
+			state size_t takeBytes = 0;
 			if (!prev.sameArena(r->arena())) {
 				TraceEvent(SevDebugMemory, "BackupWorkerMemory", self->myId)
 				    .detail("Take", r->arena().getSize())
 				    .detail("Current", self->lock->activePermits());
 
-				wait(self->lock->take(TaskPriority::DefaultYield, r->arena().getSize()));
+				takeBytes = r->arena().getSize(); // more bytes can be allocated after the wait.
+				wait(self->lock->take(TaskPriority::DefaultYield, takeBytes));
 				prev = r->arena();
 			}
-			self->messages.emplace_back(r->version(), r->getMessage(), r->getTags(), r->arena());
+			self->messages.emplace_back(r->version(), r->getMessage(), r->getTags(), r->arena(), takeBytes);
 			r->nextMessage();
 		}
 

From 5dbbbe99c2b84642c8ae3116de34d1bcf62bd537 Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Mon, 28 Nov 2022 10:31:55 -0800
Subject: [PATCH 023/468] Increase buggified lock bytes for backup workers

To fix simulation failures where the knob value is too small.
---
 fdbclient/ServerKnobs.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/fdbclient/ServerKnobs.cpp b/fdbclient/ServerKnobs.cpp
index 005a3648a06..b80e00eff78 100644
--- a/fdbclient/ServerKnobs.cpp
+++ b/fdbclient/ServerKnobs.cpp
@@ -524,7 +524,7 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi
 	init( BACKUP_TIMEOUT,                                        0.4 );
 	init( BACKUP_NOOP_POP_DELAY,                                 5.0 );
 	init( BACKUP_FILE_BLOCK_BYTES,                       1024 * 1024 );
-	init( BACKUP_LOCK_BYTES,                                     3e9 ); if(randomize && BUGGIFY) BACKUP_LOCK_BYTES = deterministicRandom()->randomInt(1024, 4096) * 15 * 1024;
+	init( BACKUP_LOCK_BYTES,                                     3e9 ); if(randomize && BUGGIFY) BACKUP_LOCK_BYTES = deterministicRandom()->randomInt(1024, 4096) * 30 * 1024;
 	init( BACKUP_UPLOAD_DELAY,                                  10.0 ); if(randomize && BUGGIFY) BACKUP_UPLOAD_DELAY = deterministicRandom()->random01() * 60;
 
 	//Cluster Controller

From 45e88e177dae6cf380101302e75c07fa691f22c0 Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Wed, 16 Nov 2022 15:36:55 -0800
Subject: [PATCH 024/468] Send error when LogRouterPeekPopped happens

Otherwise, the remote tlog won't get a response and the parallel peek requests
will never be cleared, blocking subsequent peeks. As a result, remote tlog will
no longer be able to pop the log router, which in turn can no longer peek tlogs.
The whole remote side will become blocked.
---
 fdbserver/LogRouter.actor.cpp | 19 ++++++++++++-------
 1 file changed, 12 insertions(+), 7 deletions(-)

diff --git a/fdbserver/LogRouter.actor.cpp b/fdbserver/LogRouter.actor.cpp
index c142da42157..3ea4ea4db96 100644
--- a/fdbserver/LogRouter.actor.cpp
+++ b/fdbserver/LogRouter.actor.cpp
@@ -538,14 +538,19 @@ Future<Void> logRouterPeekMessages(PromiseType replyPromise,
 				// kills logRouterPeekStream actor, otherwise that actor becomes stuck
 				throw operation_obsolete();
 			}
-			replyPromise.send(Never());
-			if (reqSequence.present()) {
-				auto& trackerData = self->peekTracker[peekId];
-				auto& sequenceData = trackerData.sequence_version[sequence + 1];
-				if (!sequenceData.isSet()) {
-					sequenceData.send(std::make_pair(reqBegin, reqOnlySpilled));
-				}
+			if (std::is_same<PromiseType, ReplyPromise<TLogPeekReply>>::value) {
+				replyPromise.sendError(operation_obsolete());
+			} else {
+				replyPromise.send(Never());
 			}
+
+			/*if (reqSequence.present()) {
+			    auto& trackerData = self->peekTracker[peekId];
+			    auto& sequenceData = trackerData.sequence_version[sequence + 1];
+			    if (!sequenceData.isSet()) {
+			        sequenceData.send(std::make_pair(reqBegin, reqOnlySpilled));
+			    }
+			}*/
 			return Void();
 		}
 

From 719927d96b6de02ab1005201099b6dcfa07860b8 Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Thu, 17 Nov 2022 11:36:36 -0800
Subject: [PATCH 025/468] Add more debug events

---
 fdbserver/LogRouter.actor.cpp           | 30 +++++++++----
 fdbserver/LogSystemPeekCursor.actor.cpp | 58 ++++++++++++++++++++-----
 fdbserver/TLogServer.actor.cpp          |  3 ++
 3 files changed, 72 insertions(+), 19 deletions(-)

diff --git a/fdbserver/LogRouter.actor.cpp b/fdbserver/LogRouter.actor.cpp
index 3ea4ea4db96..7bc31ec3fb6 100644
--- a/fdbserver/LogRouter.actor.cpp
+++ b/fdbserver/LogRouter.actor.cpp
@@ -33,7 +33,7 @@
 #include "flow/ActorCollection.h"
 #include "flow/Arena.h"
 #include "flow/Histogram.h"
-#include "flow/TDMetric.actor.h"
+#include "flow/Trace.h"
 #include "flow/network.h"
 
 #include "flow/actorcompiler.h" // This must be the last #include.
@@ -457,6 +457,14 @@ Future<Void> logRouterPeekMessages(PromiseType replyPromise,
 	state int sequence = -1;
 	state UID peekId;
 
+	DebugLogTraceEvent("LogRouterPeek0", self->dbgid)
+	    .detail("ReturnIfBlocked", reqReturnIfBlocked)
+	    .detail("Tag", reqTag.toString())
+	    .detail("Seq", reqSequence.present() ? reqSequence.get().second : -1)
+	    .detail("SeqCursor", reqSequence.present() ? reqSequence.get().first : UID())
+	    .detail("Ver", self->version.get())
+	    .detail("Begin", reqBegin);
+
 	if (reqSequence.present()) {
 		try {
 			peekId = reqSequence.get().first;
@@ -490,6 +498,13 @@ Future<Void> logRouterPeekMessages(PromiseType replyPromise,
 			reqOnlySpilled = prevPeekData.second;
 			wait(yield());
 		} catch (Error& e) {
+			DebugLogTraceEvent("LogRouterPeekError", self->dbgid)
+			    .error(e)
+			    .detail("Tag", reqTag.toString())
+			    .detail("Seq", reqSequence.present() ? reqSequence.get().second : -1)
+			    .detail("SeqCursor", reqSequence.present() ? reqSequence.get().first : UID())
+			    .detail("Begin", reqBegin);
+
 			if (e.code() == error_code_timed_out || e.code() == error_code_operation_obsolete) {
 				replyPromise.sendError(e);
 				return Void();
@@ -533,24 +548,22 @@ Future<Void> logRouterPeekMessages(PromiseType replyPromise,
 			TraceEvent(SevWarnAlways, "LogRouterPeekPopped", self->dbgid)
 			    .detail("Begin", reqBegin)
 			    .detail("Popped", poppedVer)
+			    .detail("Tag", reqTag.toString())
+			    .detail("Seq", reqSequence.present() ? reqSequence.get().second : -1)
+			    .detail("SeqCursor", reqSequence.present() ? reqSequence.get().first : UID())
 			    .detail("Start", self->startVersion);
 			if (std::is_same<PromiseType, Promise<TLogPeekReply>>::value) {
 				// kills logRouterPeekStream actor, otherwise that actor becomes stuck
 				throw operation_obsolete();
 			}
 			if (std::is_same<PromiseType, ReplyPromise<TLogPeekReply>>::value) {
+				// Send error to avoid a race condition that the peer is really retrying,
+				// otherwise, the peer could be blocked forever.
 				replyPromise.sendError(operation_obsolete());
 			} else {
 				replyPromise.send(Never());
 			}
 
-			/*if (reqSequence.present()) {
-			    auto& trackerData = self->peekTracker[peekId];
-			    auto& sequenceData = trackerData.sequence_version[sequence + 1];
-			    if (!sequenceData.isSet()) {
-			        sequenceData.send(std::make_pair(reqBegin, reqOnlySpilled));
-			    }
-			}*/
 			return Void();
 		}
 
@@ -688,6 +701,7 @@ ACTOR Future<Void> logRouterPop(LogRouterData* self, TLogPopRequest req) {
 	if (!tagData) {
 		tagData = self->createTagData(req.tag, req.to, req.durableKnownCommittedVersion);
 	} else if (req.to > tagData->popped) {
+		DebugLogTraceEvent("LogRouterPop", self->dbgid).detail("Tag", req.tag.toString()).detail("PopVersion", req.to);
 		tagData->popped = req.to;
 		tagData->durableKnownCommittedVersion = req.durableKnownCommittedVersion;
 		wait(tagData->eraseMessagesBefore(req.to, self, TaskPriority::TLogPop));
diff --git a/fdbserver/LogSystemPeekCursor.actor.cpp b/fdbserver/LogSystemPeekCursor.actor.cpp
index a17556e61cd..5968a476216 100644
--- a/fdbserver/LogSystemPeekCursor.actor.cpp
+++ b/fdbserver/LogSystemPeekCursor.actor.cpp
@@ -58,6 +58,8 @@ ILogSystem::ServerPeekCursor::ServerPeekCursor(Reference<AsyncVar<OptionalInterf
 	this->results.minKnownCommittedVersion = 0;
 	DisabledTraceEvent(SevDebug, "SPC_Starting", randomID)
 	    .detail("Tag", tag.toString())
+	    .detail("Parallel", parallelGetMore)
+	    .detail("Interf", interf && interf->get().present() ? interf->get().id() : UID())
 	    .detail("UsePeekStream", usePeekStream)
 	    .detail("Begin", begin)
 	    .detail("End", end);
@@ -107,7 +109,9 @@ bool ILogSystem::ServerPeekCursor::hasMessage() const {
 }
 
 void ILogSystem::ServerPeekCursor::nextMessage() {
-	//TraceEvent("SPC_NextMessage", randomID).detail("MessageVersion", messageVersion.toString());
+	DebugLogTraceEvent("SPC_NextMessage", randomID)
+	    .detail("Tag", tag.toString())
+	    .detail("MessageVersion", messageVersion.toString());
 	ASSERT(hasMsg);
 	if (rd.empty()) {
 		messageVersion.reset(std::min(results.end, end.version));
@@ -139,11 +143,13 @@ void ILogSystem::ServerPeekCursor::nextMessage() {
 	rd.rewind();
 	rd.readBytes(messageAndTags.getHeaderSize());
 	hasMsg = true;
-	//TraceEvent("SPC_NextMessageB", randomID).detail("MessageVersion", messageVersion.toString());
+	DebugLogTraceEvent("SPC_NextMessageB", randomID)
+	    .detail("Tag", tag.toString())
+	    .detail("MessageVersion", messageVersion.toString());
 }
 
 StringRef ILogSystem::ServerPeekCursor::getMessage() {
-	//TraceEvent("SPC_GetMessage", randomID);
+	DebugLogTraceEvent("SPC_GetMessage", randomID).detail("Tag", tag.toString());
 	StringRef message = messageAndTags.getMessageWithoutTags();
 	rd.readBytes(message.size()); // Consumes the message.
 	return message;
@@ -256,6 +262,14 @@ ACTOR Future<Void> serverPeekParallelGetMore(ILogSystem::ServerPeekCursor* self,
 	}
 
 	loop {
+		DebugLogTraceEvent("SPC_GetMoreP", self->randomID)
+		    .detail("Tag", self->tag.toString())
+		    .detail("Has", self->hasMessage())
+		    .detail("Begin", self->messageVersion.version)
+		    .detail("Parallel", self->parallelGetMore)
+		    .detail("Seq", self->sequence)
+		    .detail("Sizes", self->futureResults.size())
+		    .detail("Interf", self->interf->get().present() ? self->interf->get().id() : UID());
 		state Version expectedBegin = self->messageVersion.version;
 		try {
 			if (self->parallelGetMore || self->onlySpilled) {
@@ -290,7 +304,12 @@ ACTOR Future<Void> serverPeekParallelGetMore(ILogSystem::ServerPeekCursor* self,
 					expectedBegin = res.end;
 					self->futureResults.pop_front();
 					updateCursorWithReply(self, res);
-					//TraceEvent("SPC_GetMoreB", self->randomID).detail("Has", self->hasMessage()).detail("End", res.end).detail("Popped", res.popped.present() ? res.popped.get() : 0);
+					DebugLogTraceEvent("SPC_GetMoreReply", self->randomID)
+					    .detail("Has", self->hasMessage())
+					    .detail("Tag", self->tag.toString())
+					    .detail("End", res.end)
+					    .detail("Size", self->futureResults.size())
+					    .detail("Popped", res.popped.present() ? res.popped.get() : 0);
 					return Void();
 				}
 				when(wait(self->interfaceChanged)) {
@@ -302,11 +321,17 @@ ACTOR Future<Void> serverPeekParallelGetMore(ILogSystem::ServerPeekCursor* self,
 				}
 			}
 		} catch (Error& e) {
+			DebugLogTraceEvent("PeekCursorError", self->randomID)
+			    .error(e)
+			    .detail("Tag", self->tag.toString())
+			    .detail("Begin", self->messageVersion.version)
+			    .detail("Interf", self->interf->get().present() ? self->interf->get().id() : UID());
+
 			if (e.code() == error_code_end_of_stream) {
 				self->end.reset(self->messageVersion.version);
 				return Void();
 			} else if (e.code() == error_code_timed_out || e.code() == error_code_operation_obsolete) {
-				TraceEvent("PeekCursorTimedOut", self->randomID).error(e);
+				TraceEvent ev("PeekCursorTimedOut", self->randomID);
 				// We *should* never get timed_out(), as it means the TLog got stuck while handling a parallel peek,
 				// and thus we've likely just wasted 10min.
 				// timed_out() is sent by cleanupPeekTrackers as value PEEK_TRACKER_EXPIRATION_TIME
@@ -316,6 +341,11 @@ ACTOR Future<Void> serverPeekParallelGetMore(ILogSystem::ServerPeekCursor* self,
 				self->randomID = deterministicRandom()->randomUniqueID();
 				self->sequence = 0;
 				self->futureResults.clear();
+				ev.error(e)
+				    .detail("Tag", self->tag.toString())
+				    .detail("Begin", self->messageVersion.version)
+				    .detail("NewID", self->randomID)
+				    .detail("Interf", self->interf->get().present() ? self->interf->get().id() : UID());
 			} else {
 				throw e;
 			}
@@ -401,7 +431,11 @@ ACTOR Future<Void> serverPeekGetMore(ILogSystem::ServerPeekCursor* self, TaskPri
 				                        taskID))
 				                  : Never())) {
 					updateCursorWithReply(self, res);
-					//TraceEvent("SPC_GetMoreB", self->randomID).detail("Has", self->hasMessage()).detail("End", res.end).detail("Popped", res.popped.present() ? res.popped.get() : 0);
+					DebugLogTraceEvent("SPC_GetMoreB", self->randomID)
+					    .detail("Tag", self->tag.toString())
+					    .detail("Has", self->hasMessage())
+					    .detail("End", res.end)
+					    .detail("Popped", res.popped.present() ? res.popped.get() : 0);
 					return Void();
 				}
 				when(wait(self->interf->onChange())) { self->onlySpilled = false; }
@@ -417,11 +451,13 @@ ACTOR Future<Void> serverPeekGetMore(ILogSystem::ServerPeekCursor* self, TaskPri
 }
 
 Future<Void> ILogSystem::ServerPeekCursor::getMore(TaskPriority taskID) {
-	// TraceEvent("SPC_GetMore", randomID)
-	//     .detail("HasMessage", hasMessage())
-	//     .detail("More", !more.isValid() || more.isReady())
-	//     .detail("MessageVersion", messageVersion.toString())
-	//     .detail("End", end.toString());
+	DebugLogTraceEvent("SPC_GetMore", randomID)
+	    .detail("Tag", tag.toString())
+	    .detail("HasMessage", hasMessage())
+	    .detail("More", !more.isValid() || more.isReady())
+	    .detail("Parallel", parallelGetMore)
+	    .detail("MessageVersion", messageVersion.toString())
+	    .detail("End", end.toString());
 	if (hasMessage() && !parallelGetMore)
 		return Void();
 	if (!more.isValid() || more.isReady()) {
diff --git a/fdbserver/TLogServer.actor.cpp b/fdbserver/TLogServer.actor.cpp
index 4064f37c071..ee233a09830 100644
--- a/fdbserver/TLogServer.actor.cpp
+++ b/fdbserver/TLogServer.actor.cpp
@@ -2182,6 +2182,9 @@ ACTOR Future<Void> doQueueCommit(TLogData* self,
 	if (logData->logSystem->get() &&
 	    (!logData->isPrimary || logData->logRouterPoppedVersion < logData->logRouterPopToVersion)) {
 		logData->logRouterPoppedVersion = ver;
+		DebugLogTraceEvent("LogPop", self->dbgid)
+		    .detail("Tag", logData->remoteTag.toString())
+		    .detail("Version", knownCommittedVersion);
 		logData->logSystem->get()->pop(ver, logData->remoteTag, knownCommittedVersion, logData->locality);
 	}
 

From 8ec6ae15534eb0cbc7cf7fc1ebb26d3929ceb525 Mon Sep 17 00:00:00 2001
From: Meng Xu <meng_xu@apple.com>
Date: Fri, 2 Dec 2022 10:50:42 -0800
Subject: [PATCH 026/468] Add DebugTrace.h to 7.1 branch

Cherry-pick PR#8856 requires DebugTrace.h
due to the use of DebugLogTraceEvent function
---
 fdbserver/LogRouter.actor.cpp           |  1 +
 fdbserver/LogSystemPeekCursor.actor.cpp |  1 +
 fdbserver/TLogServer.actor.cpp          |  1 +
 flow/DebugTrace.h                       | 32 +++++++++++++++++++++++++
 4 files changed, 35 insertions(+)
 create mode 100644 flow/DebugTrace.h

diff --git a/fdbserver/LogRouter.actor.cpp b/fdbserver/LogRouter.actor.cpp
index 7bc31ec3fb6..eb84a1990f3 100644
--- a/fdbserver/LogRouter.actor.cpp
+++ b/fdbserver/LogRouter.actor.cpp
@@ -34,6 +34,7 @@
 #include "flow/Arena.h"
 #include "flow/Histogram.h"
 #include "flow/Trace.h"
+#include "flow/DebugTrace.h"
 #include "flow/network.h"
 
 #include "flow/actorcompiler.h" // This must be the last #include.
diff --git a/fdbserver/LogSystemPeekCursor.actor.cpp b/fdbserver/LogSystemPeekCursor.actor.cpp
index 5968a476216..456e462fec1 100644
--- a/fdbserver/LogSystemPeekCursor.actor.cpp
+++ b/fdbserver/LogSystemPeekCursor.actor.cpp
@@ -23,6 +23,7 @@
 #include "fdbserver/Knobs.h"
 #include "fdbserver/MutationTracking.h"
 #include "fdbrpc/ReplicationUtils.h"
+#include "flow/DebugTrace.h"
 #include "flow/actorcompiler.h" // has to be last include
 
 // create a peek stream for cursor when it's possible
diff --git a/fdbserver/TLogServer.actor.cpp b/fdbserver/TLogServer.actor.cpp
index ee233a09830..3d012f118e7 100644
--- a/fdbserver/TLogServer.actor.cpp
+++ b/fdbserver/TLogServer.actor.cpp
@@ -46,6 +46,7 @@
 #include "fdbserver/RecoveryState.h"
 #include "fdbserver/FDBExecHelper.actor.h"
 #include "flow/Histogram.h"
+#include "flow/DebugTrace.h"
 #include "flow/actorcompiler.h" // This must be the last #include.
 
 struct TLogQueueEntryRef {
diff --git a/flow/DebugTrace.h b/flow/DebugTrace.h
new file mode 100644
index 00000000000..bcc2670eae9
--- /dev/null
+++ b/flow/DebugTrace.h
@@ -0,0 +1,32 @@
+/*
+ * DebugTrace.h
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2022 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef FOUNDATIONDB_DEBUGTRACE_H
+#define FOUNDATIONDB_DEBUGTRACE_H
+
+#define DebugTraceEvent(enable, ...) enable&& TraceEvent(__VA_ARGS__)
+
+constexpr bool debugLogTraces = false;
+#define DebugLogTraceEvent(...) DebugTraceEvent(debugLogTraces, __VA_ARGS__)
+
+constexpr bool debugRelocationTraces = false;
+#define DebugRelocationTraceEvent(...) DebugTraceEvent(debugRelocationTraces, __VA_ARGS__)
+
+#endif // FOUNDATIONDB_DEBUGTRACE_H

From c764f8049235fc179fbfc53a3a2715932a024273 Mon Sep 17 00:00:00 2001
From: Huiyoung <bryant507@foxmail.com>
Date: Tue, 29 Nov 2022 15:40:03 +0800
Subject: [PATCH 027/468] Fix the bug of variable int32 overflow. 1.the content
 length from http response transformed using 'atoi' would rise int32 overflow.
 2.the offset's aligning would rise int32 overflow.

---
 fdbrpc/AsyncFileReadAhead.actor.h | 4 ++--
 fdbrpc/HTTP.actor.cpp             | 6 +++---
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/fdbrpc/AsyncFileReadAhead.actor.h b/fdbrpc/AsyncFileReadAhead.actor.h
index b26e2a5e585..0b120c03c0a 100644
--- a/fdbrpc/AsyncFileReadAhead.actor.h
+++ b/fdbrpc/AsyncFileReadAhead.actor.h
@@ -102,7 +102,7 @@ class AsyncFileReadAheadCache final : public IAsyncFile, public ReferenceCounted
 			// If not found, start the read.
 			if (i == f->m_blocks.end() || (i->second.isValid() && i->second.isError())) {
 				// printf("starting read of %s block %d\n", f->getFilename().c_str(), blockNum);
-				fblock = readBlock(f.getPtr(), f->m_block_size, f->m_block_size * blockNum);
+				fblock = readBlock(f.getPtr(), f->m_block_size, (int64_t)f->m_block_size * blockNum);
 				f->m_blocks[blockNum] = fblock;
 			} else
 				fblock = i->second;
@@ -121,7 +121,7 @@ class AsyncFileReadAheadCache final : public IAsyncFile, public ReferenceCounted
 			// Calculate the block-relative read range.  It's a given that the offset / length range touches this block
 			// so readStart will never be greater than blocksize (though it could be past the actual end of a short
 			// block).
-			int64_t blockStart = blockNum * f->m_block_size;
+			int64_t blockStart = (int64_t)blockNum * f->m_block_size;
 			int64_t readStart = std::max<int64_t>(0, offset - blockStart);
 			int64_t readEnd = std::min<int64_t>(f->m_block_size, offset + length - blockStart);
 			int rlen = readEnd - readStart;
diff --git a/fdbrpc/HTTP.actor.cpp b/fdbrpc/HTTP.actor.cpp
index b48d36904c4..e0ff76656f1 100644
--- a/fdbrpc/HTTP.actor.cpp
+++ b/fdbrpc/HTTP.actor.cpp
@@ -232,7 +232,7 @@ ACTOR Future<Void> read_http_response(Reference<HTTP::Response> r, Reference<ICo
 
 	auto i = r->headers.find("Content-Length");
 	if (i != r->headers.end())
-		r->contentLen = atoi(i->second.c_str());
+		r->contentLen = strtoll(i->second.c_str(), NULL, 10);
 	else
 		r->contentLen = -1; // Content length unknown
 
@@ -464,7 +464,7 @@ ACTOR Future<Reference<HTTP::Response>> doRequest(Reference<IConnection> conn,
 		}
 
 		if (FLOW_KNOBS->HTTP_VERBOSE_LEVEL > 0) {
-			printf("[%s] HTTP %scode=%d early=%d, time=%fs %s %s contentLen=%d [%d out, response content len %d]\n",
+			printf("[%s] HTTP %scode=%d early=%d, time=%fs %s %s contentLen=%d [%d out, response content len %lld]\n",
 			       conn->getDebugID().toString().c_str(),
 			       (err.present() ? format("*ERROR*=%s ", err.get().name()).c_str() : ""),
 			       r->code,
@@ -474,7 +474,7 @@ ACTOR Future<Reference<HTTP::Response>> doRequest(Reference<IConnection> conn,
 			       resource.c_str(),
 			       contentLen,
 			       total_sent,
-			       (int)r->contentLen);
+			       r->contentLen);
 		}
 		if (FLOW_KNOBS->HTTP_VERBOSE_LEVEL > 2) {
 			printf("[%s] HTTP RESPONSE:  %s %s\n%s\n",

From 55d2fd3e693c9d66751fa24583bbb0c742515f58 Mon Sep 17 00:00:00 2001
From: sfc-gh-tclinkenbeard <trevor.clinkenbeard@snowflake.com>
Date: Tue, 29 Nov 2022 15:50:45 -0800
Subject: [PATCH 028/468] Fix -Wformat warning

---
 fdbrpc/HTTP.actor.cpp | 28 ++++++++++++++++------------
 1 file changed, 16 insertions(+), 12 deletions(-)

diff --git a/fdbrpc/HTTP.actor.cpp b/fdbrpc/HTTP.actor.cpp
index e0ff76656f1..cd2ed8f8a04 100644
--- a/fdbrpc/HTTP.actor.cpp
+++ b/fdbrpc/HTTP.actor.cpp
@@ -20,10 +20,13 @@
 
 #include "fdbrpc/HTTP.h"
 
+#include <cctype>
+
+#include "contrib/fmt-8.1.1/include/fmt/format.h"
 #include "fdbrpc/md5/md5.h"
 #include "fdbrpc/libb64/encode.h"
 #include "flow/Knobs.h"
-#include <cctype>
+
 #include "flow/actorcompiler.h" // has to be last include
 
 namespace HTTP {
@@ -464,17 +467,18 @@ ACTOR Future<Reference<HTTP::Response>> doRequest(Reference<IConnection> conn,
 		}
 
 		if (FLOW_KNOBS->HTTP_VERBOSE_LEVEL > 0) {
-			printf("[%s] HTTP %scode=%d early=%d, time=%fs %s %s contentLen=%d [%d out, response content len %lld]\n",
-			       conn->getDebugID().toString().c_str(),
-			       (err.present() ? format("*ERROR*=%s ", err.get().name()).c_str() : ""),
-			       r->code,
-			       earlyResponse,
-			       elapsed,
-			       verb.c_str(),
-			       resource.c_str(),
-			       contentLen,
-			       total_sent,
-			       r->contentLen);
+			fmt::print("[{0}] HTTP {1}code={2} early={3}, time={4} {5} {6} contentLen={7} [{8} out, response content "
+			           "len {9}]\n",
+			           conn->getDebugID().toString(),
+			           (err.present() ? format("*ERROR*=%s ", err.get().name()).c_str() : ""),
+			           r->code,
+			           earlyResponse,
+			           elapsed,
+			           verb,
+			           resource,
+			           contentLen,
+			           total_sent,
+			           r->contentLen);
 		}
 		if (FLOW_KNOBS->HTTP_VERBOSE_LEVEL > 2) {
 			printf("[%s] HTTP RESPONSE:  %s %s\n%s\n",

From fc6a77f8d966eb28d5ceef710a98f1bfce945896 Mon Sep 17 00:00:00 2001
From: Zhe Wu <halfprice@users.noreply.github.com>
Date: Fri, 2 Dec 2022 10:55:26 -0800
Subject: [PATCH 029/468] Add determinstic in gray failure degraded server
 selection

---
 fdbserver/ClusterController.actor.h | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/fdbserver/ClusterController.actor.h b/fdbserver/ClusterController.actor.h
index f1ef75c3e2e..0c2d2375a5f 100644
--- a/fdbserver/ClusterController.actor.h
+++ b/fdbserver/ClusterController.actor.h
@@ -3021,18 +3021,23 @@ class ClusterControllerData {
 			}
 		}
 
+		auto deterministicDecendingOrder = [](const std::pair<int, NetworkAddress>& a,
+		                                      const std::pair<int, NetworkAddress>& b) -> bool {
+			return a.first > b.first || (a.first == b.first && a.second < b.second);
+		};
+
 		// Sort degraded peers based on the number of workers complaining about it.
 		std::vector<std::pair<int, NetworkAddress>> count2DegradedPeer;
 		for (const auto& [degradedPeer, complainers] : degradedLinkDst2Src) {
 			count2DegradedPeer.push_back({ complainers.size(), degradedPeer });
 		}
-		std::sort(count2DegradedPeer.begin(), count2DegradedPeer.end(), std::greater<>());
+		std::sort(count2DegradedPeer.begin(), count2DegradedPeer.end(), deterministicDecendingOrder);
 
 		std::vector<std::pair<int, NetworkAddress>> count2DisconnectedPeer;
 		for (const auto& [disconnectedPeer, complainers] : disconnectedLinkDst2Src) {
 			count2DisconnectedPeer.push_back({ complainers.size(), disconnectedPeer });
 		}
-		std::sort(count2DisconnectedPeer.begin(), count2DisconnectedPeer.end(), std::greater<>());
+		std::sort(count2DisconnectedPeer.begin(), count2DisconnectedPeer.end(), deterministicDecendingOrder);
 
 		// Go through all reported degraded peers by decreasing order of the number of complainers. For a particular
 		// degraded peer, if a complainer has already be considered as degraded, we skip the current examine degraded

From 5c644f20e3c5b752c3dc11bcbf620536ba497419 Mon Sep 17 00:00:00 2001
From: FoundationDB CI <foundationdb_ci@apple.com>
Date: Thu, 8 Dec 2022 17:34:28 +0000
Subject: [PATCH 030/468] format source code after switch to clang 15

---
 documentation/tutorial/tutorial.actor.cpp     | 16 +++--
 fdbbackup/backup.actor.cpp                    |  8 ++-
 fdbcli/fdbcli.actor.cpp                       |  4 +-
 fdbclient/FileBackupAgent.actor.cpp           |  2 +-
 fdbclient/ManagementAPI.actor.cpp             |  8 ++-
 fdbclient/MonitorLeader.actor.cpp             |  4 +-
 fdbclient/NativeAPI.actor.cpp                 | 52 ++++++++++++----
 fdbclient/PaxosConfigTransaction.actor.cpp    |  8 ++-
 fdbclient/ReadYourWrites.actor.cpp            | 32 +++++++---
 fdbclient/S3BlobStore.actor.cpp               |  8 ++-
 fdbclient/SpecialKeySpace.actor.cpp           |  4 +-
 .../json_spirit/json_spirit_reader_template.h |  2 +-
 fdbrpc/AsyncFileNonDurable.actor.h            | 16 +++--
 fdbrpc/FailureMonitor.actor.cpp               |  4 +-
 fdbrpc/FlowTransport.actor.cpp                | 16 +++--
 fdbrpc/LoadBalance.actor.cpp                  |  4 +-
 fdbrpc/dsltest.actor.cpp                      | 22 +++++--
 fdbrpc/genericactors.actor.h                  | 16 +++--
 fdbrpc/libcoroutine/Common.h                  |  4 +-
 fdbrpc/libcoroutine/Coro.h                    |  6 +-
 fdbrpc/libcoroutine/taskimpl.h                |  4 +-
 fdbserver/BackupWorker.actor.cpp              |  8 ++-
 fdbserver/BlobManager.actor.cpp               |  8 ++-
 fdbserver/BlobWorker.actor.cpp                | 40 +++++++++----
 fdbserver/ClusterController.actor.cpp         | 24 ++++++--
 fdbserver/ClusterRecovery.actor.cpp           | 20 +++++--
 fdbserver/ConfigDatabaseUnitTests.actor.cpp   |  4 +-
 fdbserver/ConfigNode.actor.cpp                |  8 ++-
 fdbserver/CoordinatedState.actor.cpp          |  8 ++-
 fdbserver/Coordination.actor.cpp              |  8 ++-
 fdbserver/DDTeamCollection.actor.cpp          | 20 +++++--
 fdbserver/DataDistributionQueue.actor.cpp     |  8 ++-
 fdbserver/DataDistributionTracker.actor.cpp   |  4 +-
 fdbserver/KeyValueStoreSQLite.actor.cpp       |  4 +-
 fdbserver/LeaderElection.actor.cpp            |  8 ++-
 fdbserver/LocalConfiguration.actor.cpp        |  8 ++-
 fdbserver/LogRouter.actor.cpp                 |  4 +-
 fdbserver/LogSystemDiskQueueAdapter.actor.cpp |  4 +-
 fdbserver/LogSystemPeekCursor.actor.cpp       |  4 +-
 fdbserver/OldTLogServer_4_6.actor.cpp         | 12 +++-
 fdbserver/OldTLogServer_6_0.actor.cpp         | 20 +++++--
 fdbserver/OldTLogServer_6_2.actor.cpp         | 20 +++++--
 fdbserver/QuietDatabase.actor.cpp             |  4 +-
 fdbserver/RemoteIKeyValueStore.actor.cpp      | 12 +++-
 fdbserver/Resolver.actor.cpp                  |  8 ++-
 fdbserver/RestoreLoader.actor.cpp             |  4 +-
 fdbserver/RestoreUtil.h                       |  4 +-
 fdbserver/SimEncryptVaultProxy.actor.cpp      |  8 ++-
 fdbserver/SkipList.cpp                        |  2 +-
 fdbserver/StorageCache.actor.cpp              | 60 ++++++++++++++-----
 fdbserver/TCInfo.actor.cpp                    |  4 +-
 fdbserver/TLogServer.actor.cpp                | 24 ++++++--
 fdbserver/VFSAsync.cpp                        |  4 +-
 fdbserver/VersionedBTree.actor.cpp            |  8 ++-
 fdbserver/WorkerInterface.actor.h             | 12 +++-
 fdbserver/art.h                               |  2 +-
 fdbserver/sqlite/hash.h                       |  2 +-
 fdbserver/sqlite/sqlite3.h                    |  6 +-
 fdbserver/storageserver.actor.cpp             |  8 ++-
 fdbserver/tester.actor.cpp                    |  4 +-
 fdbserver/worker.actor.cpp                    | 32 +++++++---
 fdbserver/workloads/AtomicOps.actor.cpp       |  2 +-
 .../workloads/ConsistencyCheck.actor.cpp      |  2 +-
 fdbserver/workloads/IndexScan.actor.cpp       |  4 +-
 fdbserver/workloads/KVStoreTest.actor.cpp     |  4 +-
 fdbserver/workloads/KillRegion.actor.cpp      |  4 +-
 fdbserver/workloads/MiniCycle.actor.cpp       |  4 +-
 fdbserver/workloads/UDPWorkload.actor.cpp     |  8 ++-
 flow/ActorCollection.actor.cpp                |  4 +-
 flow/Arena.h                                  | 10 ++--
 flow/Error.h                                  |  2 +-
 flow/FastAlloc.cpp                            | 10 ++--
 flow/FastAlloc.h                              | 10 ++--
 flow/FileTraceLogWriter.cpp                   |  6 +-
 flow/Hash3.c                                  |  2 +-
 flow/Net2.actor.cpp                           | 16 +++--
 flow/flat_buffers.h                           |  6 +-
 flow/flow.h                                   |  2 +-
 flow/genericactors.actor.cpp                  | 16 +++--
 flow/genericactors.actor.h                    | 60 ++++++++++++++-----
 80 files changed, 624 insertions(+), 240 deletions(-)

diff --git a/documentation/tutorial/tutorial.actor.cpp b/documentation/tutorial/tutorial.actor.cpp
index 67c542b6326..32f6c10eca6 100644
--- a/documentation/tutorial/tutorial.actor.cpp
+++ b/documentation/tutorial/tutorial.actor.cpp
@@ -63,7 +63,9 @@ ACTOR Future<Void> simpleTimer() {
 ACTOR Future<Void> someFuture(Future<int> ready) {
 	// loop choose {} works as well here - the braces are optional
 	loop choose {
-		when(wait(delay(0.5))) { std::cout << "Still waiting...\n"; }
+		when(wait(delay(0.5))) {
+			std::cout << "Still waiting...\n";
+		}
 		when(int r = wait(ready)) {
 			std::cout << format("Ready %d\n", r);
 			wait(delay(double(r)));
@@ -84,8 +86,12 @@ ACTOR Future<Void> promiseDemo() {
 
 ACTOR Future<Void> eventLoop(AsyncTrigger* trigger) {
 	loop choose {
-		when(wait(delay(0.5))) { std::cout << "Still waiting...\n"; }
-		when(wait(trigger->onTrigger())) { std::cout << "Triggered!\n"; }
+		when(wait(delay(0.5))) {
+			std::cout << "Still waiting...\n";
+		}
+		when(wait(trigger->onTrigger())) {
+			std::cout << "Triggered!\n";
+		}
 	}
 }
 
@@ -185,7 +191,9 @@ ACTOR Future<Void> echoServer() {
 				when(GetInterfaceRequest req = waitNext(echoServer.getInterface.getFuture())) {
 					req.reply.send(echoServer);
 				}
-				when(EchoRequest req = waitNext(echoServer.echo.getFuture())) { req.reply.send(req.message); }
+				when(EchoRequest req = waitNext(echoServer.echo.getFuture())) {
+					req.reply.send(req.message);
+				}
 				when(ReverseRequest req = waitNext(echoServer.reverse.getFuture())) {
 					req.reply.send(std::string(req.message.rbegin(), req.message.rend()));
 				}
diff --git a/fdbbackup/backup.actor.cpp b/fdbbackup/backup.actor.cpp
index 81db05f01d6..9d4bc2ab056 100644
--- a/fdbbackup/backup.actor.cpp
+++ b/fdbbackup/backup.actor.cpp
@@ -2636,7 +2636,9 @@ ACTOR Future<Void> expireBackupData(const char* name,
 						lastProgress = p;
 					}
 				}
-				when(wait(expire)) { break; }
+				when(wait(expire)) {
+					break;
+				}
 			}
 		}
 
@@ -2679,7 +2681,9 @@ ACTOR Future<Void> deleteBackupContainer(const char* name,
 
 		loop {
 			choose {
-				when(wait(done)) { break; }
+				when(wait(done)) {
+					break;
+				}
 				when(wait(delay(5))) {
 					if (numDeleted != lastUpdate) {
 						printf("\r%d...", numDeleted);
diff --git a/fdbcli/fdbcli.actor.cpp b/fdbcli/fdbcli.actor.cpp
index c132695f326..f918307cc7f 100644
--- a/fdbcli/fdbcli.actor.cpp
+++ b/fdbcli/fdbcli.actor.cpp
@@ -648,7 +648,9 @@ ACTOR template <class T>
 Future<T> makeInterruptable(Future<T> f) {
 	Future<Void> interrupt = LineNoise::onKeyboardInterrupt();
 	choose {
-		when(T t = wait(f)) { return t; }
+		when(T t = wait(f)) {
+			return t;
+		}
 		when(wait(interrupt)) {
 			f.cancel();
 			throw operation_cancelled();
diff --git a/fdbclient/FileBackupAgent.actor.cpp b/fdbclient/FileBackupAgent.actor.cpp
index e53042bdc7d..edf1ef5c1fc 100644
--- a/fdbclient/FileBackupAgent.actor.cpp
+++ b/fdbclient/FileBackupAgent.actor.cpp
@@ -47,7 +47,7 @@ FDB_DEFINE_BOOLEAN_PARAM(IncrementalBackupOnly);
 FDB_DEFINE_BOOLEAN_PARAM(OnlyApplyMutationLogs);
 
 #define SevFRTestInfo SevVerbose
-//#define SevFRTestInfo SevInfo
+// #define SevFRTestInfo SevInfo
 
 static std::string boolToYesOrNo(bool val) {
 	return val ? std::string("Yes") : std::string("No");
diff --git a/fdbclient/ManagementAPI.actor.cpp b/fdbclient/ManagementAPI.actor.cpp
index f627168e8bd..7487a6083aa 100644
--- a/fdbclient/ManagementAPI.actor.cpp
+++ b/fdbclient/ManagementAPI.actor.cpp
@@ -879,7 +879,9 @@ ACTOR Future<Optional<CoordinatorsResult>> changeQuorumChecker(Transaction* tr,
 
 	choose {
 		when(wait(waitForAll(leaderServers))) {}
-		when(wait(delay(5.0))) { return CoordinatorsResult::COORDINATOR_UNREACHABLE; }
+		when(wait(delay(5.0))) {
+			return CoordinatorsResult::COORDINATOR_UNREACHABLE;
+		}
 	}
 	tr->set(coordinatorsKey, conn->toString());
 	return Optional<CoordinatorsResult>();
@@ -976,7 +978,9 @@ ACTOR Future<CoordinatorsResult> changeQuorum(Database cx, Reference<IQuorumChan
 				                                           TaskPriority::CoordinationReply));
 			choose {
 				when(wait(waitForAll(leaderServers))) {}
-				when(wait(delay(5.0))) { return CoordinatorsResult::COORDINATOR_UNREACHABLE; }
+				when(wait(delay(5.0))) {
+					return CoordinatorsResult::COORDINATOR_UNREACHABLE;
+				}
 			}
 
 			tr.set(coordinatorsKey, conn.toString());
diff --git a/fdbclient/MonitorLeader.actor.cpp b/fdbclient/MonitorLeader.actor.cpp
index a23976fb625..22724dd43d5 100644
--- a/fdbclient/MonitorLeader.actor.cpp
+++ b/fdbclient/MonitorLeader.actor.cpp
@@ -644,7 +644,9 @@ ACTOR Future<Void> asyncDeserializeClusterInterface(Reference<AsyncVar<Value>> s
 	state Future<Void> deserializer = asyncDeserialize(serializedInfo, knownLeader);
 	loop {
 		choose {
-			when(wait(deserializer)) { UNSTOPPABLE_ASSERT(false); }
+			when(wait(deserializer)) {
+				UNSTOPPABLE_ASSERT(false);
+			}
 			when(wait(knownLeader->onChange())) {
 				if (knownLeader->get().present()) {
 					outKnownLeader->set(knownLeader->get().get().clientInterface);
diff --git a/fdbclient/NativeAPI.actor.cpp b/fdbclient/NativeAPI.actor.cpp
index 22e92155aa5..9ea0ca6eb6d 100644
--- a/fdbclient/NativeAPI.actor.cpp
+++ b/fdbclient/NativeAPI.actor.cpp
@@ -971,7 +971,9 @@ ACTOR static Future<Void> monitorClientDBInfoChange(DatabaseContext* cx,
 					proxiesChangeTrigger->trigger();
 				}
 			}
-			when(wait(actors.getResult())) { UNSTOPPABLE_ASSERT(false); }
+			when(wait(actors.getResult())) {
+				UNSTOPPABLE_ASSERT(false);
+			}
 		}
 	}
 }
@@ -3330,7 +3332,9 @@ ACTOR Future<Optional<Value>> getValue(Reference<TransactionState> trState,
 					    std::vector<Error>{ transaction_too_old(), future_version() });
 				}
 				choose {
-					when(wait(trState->cx->connectionFileChanged())) { throw transaction_too_old(); }
+					when(wait(trState->cx->connectionFileChanged())) {
+						throw transaction_too_old();
+					}
 					when(GetValueReply _reply = wait(loadBalance(
 					         trState->cx.getPtr(),
 					         locationInfo.locations,
@@ -3472,7 +3476,9 @@ ACTOR Future<Key> getKey(Reference<TransactionState> trState,
 			state GetKeyReply reply;
 			try {
 				choose {
-					when(wait(trState->cx->connectionFileChanged())) { throw transaction_too_old(); }
+					when(wait(trState->cx->connectionFileChanged())) {
+						throw transaction_too_old();
+					}
 					when(GetKeyReply _reply = wait(loadBalance(
 					         trState->cx.getPtr(),
 					         locationInfo.locations,
@@ -3634,7 +3640,9 @@ ACTOR Future<Version> watchValue(Database cx, Reference<const WatchParameters> p
 				                     TaskPriority::DefaultPromiseEndpoint))) {
 					resp = r;
 				}
-				when(wait(cx->connectionRecord ? cx->connectionRecord->onChange() : Never())) { wait(Never()); }
+				when(wait(cx->connectionRecord ? cx->connectionRecord->onChange() : Never())) {
+					wait(Never());
+				}
 			}
 			if (watchValueID.present()) {
 				g_traceBatch.addEvent("WatchValueDebug", watchValueID.get().first(), "NativeAPI.watchValue.After");
@@ -3934,7 +3942,9 @@ Future<RangeResultFamily> getExactRange(Reference<TransactionState> trState,
 				state GetKeyValuesFamilyReply rep;
 				try {
 					choose {
-						when(wait(trState->cx->connectionFileChanged())) { throw transaction_too_old(); }
+						when(wait(trState->cx->connectionFileChanged())) {
+							throw transaction_too_old();
+						}
 						when(GetKeyValuesFamilyReply _rep = wait(loadBalance(
 						         trState->cx.getPtr(),
 						         locations[shard].locations,
@@ -4825,7 +4835,9 @@ ACTOR Future<Void> getRangeStreamFragment(Reference<TransactionState> trState,
 								return Void();
 							}
 
-							when(GetKeyValuesStreamReply _rep = waitNext(replyStream.getFuture())) { rep = _rep; }
+							when(GetKeyValuesStreamReply _rep = waitNext(replyStream.getFuture())) {
+								rep = _rep;
+							}
 						}
 						++trState->cx->transactionPhysicalReadsCompleted;
 					} catch (Error& e) {
@@ -5279,7 +5291,9 @@ ACTOR Future<Void> watch(Reference<Watch> watch,
 				loop {
 					choose {
 						// NativeAPI watchValue future finishes or errors
-						when(wait(watch->watchFuture)) { break; }
+						when(wait(watch->watchFuture)) {
+							break;
+						}
 
 						when(wait(cx->connectionFileChanged())) {
 							TEST(true); // Recreated a watch after switch
@@ -6659,7 +6673,9 @@ ACTOR Future<GetReadVersionReply> getConsistentReadVersion(SpanID parentSpan,
 			state Future<Void> onProxiesChanged = cx->onProxiesChanged();
 
 			choose {
-				when(wait(onProxiesChanged)) { onProxiesChanged = cx->onProxiesChanged(); }
+				when(wait(onProxiesChanged)) {
+					onProxiesChanged = cx->onProxiesChanged();
+				}
 				when(GetReadVersionReply v =
 				         wait(basicLoadBalance(cx->getGrvProxies(UseProvisionalProxies(
 				                                   flags & GetReadVersionRequest::FLAG_USE_PROVISIONAL_PROXIES)),
@@ -7083,7 +7099,9 @@ ACTOR Future<ProtocolVersion> getClusterProtocolImpl(
 				needToConnect = false;
 			}
 			choose {
-				when(wait(coordinator->onChange())) { needToConnect = true; }
+				when(wait(coordinator->onChange())) {
+					needToConnect = true;
+				}
 
 				when(ProtocolVersion pv = wait(protocolVersion)) {
 					if (!expectedVersion.present() || expectedVersion.get() != pv) {
@@ -8211,8 +8229,12 @@ ACTOR Future<std::vector<CheckpointMetaData>> getCheckpointMetaData(Database cx,
 			}
 
 			choose {
-				when(wait(cx->connectionFileChanged())) { cx->invalidateCache(KeyRef(), keys); }
-				when(wait(waitForAll(futures))) { break; }
+				when(wait(cx->connectionFileChanged())) {
+					cx->invalidateCache(KeyRef(), keys);
+				}
+				when(wait(waitForAll(futures))) {
+					break;
+				}
 				when(wait(delay(timeout))) {
 					TraceEvent("GetCheckpointTimeout").detail("Range", keys).detail("Version", version);
 				}
@@ -8562,8 +8584,12 @@ ACTOR Future<Void> changeFeedWhenAtLatest(Reference<ChangeFeedData> self, Versio
 		// only allowed to use empty versions if you're caught up
 		Future<Void> waitEmptyVersion = (self->notAtLatest.get() == 0) ? changeFeedWaitLatest(self, version) : Never();
 		choose {
-			when(wait(waitEmptyVersion)) { break; }
-			when(wait(lastReturned)) { break; }
+			when(wait(waitEmptyVersion)) {
+				break;
+			}
+			when(wait(lastReturned)) {
+				break;
+			}
 			when(wait(self->refresh.getFuture())) {}
 			when(wait(self->notAtLatest.onChange())) {}
 		}
diff --git a/fdbclient/PaxosConfigTransaction.actor.cpp b/fdbclient/PaxosConfigTransaction.actor.cpp
index 4b7c19c05af..d20ced3ace3 100644
--- a/fdbclient/PaxosConfigTransaction.actor.cpp
+++ b/fdbclient/PaxosConfigTransaction.actor.cpp
@@ -194,8 +194,12 @@ class GetGenerationQuorum {
 			}
 			try {
 				choose {
-					when(ConfigGeneration generation = wait(self->result.getFuture())) { return generation; }
-					when(wait(self->actors.getResult())) { ASSERT(false); }
+					when(ConfigGeneration generation = wait(self->result.getFuture())) {
+						return generation;
+					}
+					when(wait(self->actors.getResult())) {
+						ASSERT(false);
+					}
 				}
 			} catch (Error& e) {
 				if (e.code() == error_code_failed_to_reach_quorum) {
diff --git a/fdbclient/ReadYourWrites.actor.cpp b/fdbclient/ReadYourWrites.actor.cpp
index c651adad32a..67fb6939534 100644
--- a/fdbclient/ReadYourWrites.actor.cpp
+++ b/fdbclient/ReadYourWrites.actor.cpp
@@ -351,16 +351,24 @@ class RYWImpl {
 	                                                                 Req req,
 	                                                                 Snapshot snapshot) {
 		choose {
-			when(typename Req::Result result = wait(readThrough(ryw, req, snapshot))) { return result; }
-			when(wait(ryw->resetPromise.getFuture())) { throw internal_error(); }
+			when(typename Req::Result result = wait(readThrough(ryw, req, snapshot))) {
+				return result;
+			}
+			when(wait(ryw->resetPromise.getFuture())) {
+				throw internal_error();
+			}
 		}
 	}
 	ACTOR template <class Req>
 	static Future<typename Req::Result> readWithConflictRangeSnapshot(ReadYourWritesTransaction* ryw, Req req) {
 		state SnapshotCache::iterator it(&ryw->cache, &ryw->writes);
 		choose {
-			when(typename Req::Result result = wait(read(ryw, req, &it))) { return result; }
-			when(wait(ryw->resetPromise.getFuture())) { throw internal_error(); }
+			when(typename Req::Result result = wait(read(ryw, req, &it))) {
+				return result;
+			}
+			when(wait(ryw->resetPromise.getFuture())) {
+				throw internal_error();
+			}
 		}
 	}
 	ACTOR template <class Req>
@@ -376,7 +384,9 @@ class RYWImpl {
 					addConflictRange(ryw, req, it.extractWriteMapIterator(), result);
 				return result;
 			}
-			when(wait(ryw->resetPromise.getFuture())) { throw internal_error(); }
+			when(wait(ryw->resetPromise.getFuture())) {
+				throw internal_error();
+			}
 		}
 	}
 	template <class Req>
@@ -1191,7 +1201,9 @@ class RYWImpl {
 				addConflictRangeAndMustUnmodified<backwards>(ryw, req, writes, result);
 				return result;
 			}
-			when(wait(ryw->resetPromise.getFuture())) { throw internal_error(); }
+			when(wait(ryw->resetPromise.getFuture())) {
+				throw internal_error();
+			}
 		}
 	}
 
@@ -1437,9 +1449,13 @@ class RYWImpl {
 
 	ACTOR static Future<Version> getReadVersion(ReadYourWritesTransaction* ryw) {
 		choose {
-			when(Version v = wait(ryw->tr.getReadVersion())) { return v; }
+			when(Version v = wait(ryw->tr.getReadVersion())) {
+				return v;
+			}
 
-			when(wait(ryw->resetPromise.getFuture())) { throw internal_error(); }
+			when(wait(ryw->resetPromise.getFuture())) {
+				throw internal_error();
+			}
 		}
 	}
 };
diff --git a/fdbclient/S3BlobStore.actor.cpp b/fdbclient/S3BlobStore.actor.cpp
index 80e3611690e..3dc1f551020 100644
--- a/fdbclient/S3BlobStore.actor.cpp
+++ b/fdbclient/S3BlobStore.actor.cpp
@@ -422,7 +422,9 @@ ACTOR Future<Void> deleteRecursively_impl(Reference<S3BlobStoreEndpoint> b,
 		loop {
 			choose {
 				// Throw if done throws, otherwise don't stop until end_of_stream
-				when(wait(done)) { done = Never(); }
+				when(wait(done)) {
+					done = Never();
+				}
 
 				when(S3BlobStoreEndpoint::ListResult list = waitNext(resultStream.getFuture())) {
 					for (auto& object : list.objects) {
@@ -1078,7 +1080,9 @@ ACTOR Future<S3BlobStoreEndpoint::ListResult> listObjects_impl(Reference<S3BlobS
 		loop {
 			choose {
 				// Throw if done throws, otherwise don't stop until end_of_stream
-				when(wait(done)) { done = Never(); }
+				when(wait(done)) {
+					done = Never();
+				}
 
 				when(S3BlobStoreEndpoint::ListResult info = waitNext(resultStream.getFuture())) {
 					results.commonPrefixes.insert(
diff --git a/fdbclient/SpecialKeySpace.actor.cpp b/fdbclient/SpecialKeySpace.actor.cpp
index 6d2d717e49b..f90ce4a29e5 100644
--- a/fdbclient/SpecialKeySpace.actor.cpp
+++ b/fdbclient/SpecialKeySpace.actor.cpp
@@ -320,7 +320,9 @@ ACTOR Future<RangeResult> SpecialKeySpace::checkRYWValid(SpecialKeySpace* sks,
 		         wait(SpecialKeySpace::getRangeAggregationActor(sks, ryw, begin, end, limits, reverse))) {
 			return result;
 		}
-		when(wait(ryw->resetFuture())) { throw internal_error(); }
+		when(wait(ryw->resetFuture())) {
+			throw internal_error();
+		}
 	}
 }
 
diff --git a/fdbclient/json_spirit/json_spirit_reader_template.h b/fdbclient/json_spirit/json_spirit_reader_template.h
index 207e2e8e74e..9005a09764b 100644
--- a/fdbclient/json_spirit/json_spirit_reader_template.h
+++ b/fdbclient/json_spirit/json_spirit_reader_template.h
@@ -13,7 +13,7 @@
 #include "json_spirit_value.h"
 #include "json_spirit_error_position.h"
 
-//#define BOOST_SPIRIT_THREADSAFE  // uncomment for multithreaded use, requires linking to boost.thread
+// #define BOOST_SPIRIT_THREADSAFE  // uncomment for multithreaded use, requires linking to boost.thread
 
 #include <boost/bind/bind.hpp>
 #include <boost/function.hpp>
diff --git a/fdbrpc/AsyncFileNonDurable.actor.h b/fdbrpc/AsyncFileNonDurable.actor.h
index 99a314ea300..9cd264853c2 100644
--- a/fdbrpc/AsyncFileNonDurable.actor.h
+++ b/fdbrpc/AsyncFileNonDurable.actor.h
@@ -52,7 +52,9 @@ Future<T> sendErrorOnShutdown(Future<T> in, bool assertOnCancel = false) {
 			when(wait(success(g_simulator.getCurrentProcess()->shutdownSignal.getFuture()))) {
 				throw io_error().asInjectedFault();
 			}
-			when(T rep = wait(in)) { return rep; }
+			when(T rep = wait(in)) {
+				return rep;
+			}
 		}
 	} catch (Error& e) {
 		ASSERT(e.code() != error_code_actor_cancelled || !assertOnCancel);
@@ -82,7 +84,9 @@ class AsyncFileDetachable final : public IAsyncFile, public ReferenceCounted<Asy
 			when(wait(success(g_simulator.getCurrentProcess()->shutdownSignal.getFuture()))) {
 				throw io_error().asInjectedFault();
 			}
-			when(Reference<IAsyncFile> f = wait(wrappedFile)) { return makeReference<AsyncFileDetachable>(f); }
+			when(Reference<IAsyncFile> f = wait(wrappedFile)) {
+				return makeReference<AsyncFileDetachable>(f);
+			}
 		}
 	}
 
@@ -507,7 +511,9 @@ class AsyncFileNonDurable final : public IAsyncFile, public ReferenceCounted<Asy
 		state bool saveDurable = true;
 		choose {
 			when(wait(delay(delayDuration))) {}
-			when(bool durable = wait(startSyncFuture)) { saveDurable = durable; }
+			when(bool durable = wait(startSyncFuture)) {
+				saveDurable = durable;
+			}
 		}
 
 		debugFileCheck("AsyncFileNonDurableWriteAfterWait", self->filename, dataCopy.begin(), offset, length);
@@ -684,7 +690,9 @@ class AsyncFileNonDurable final : public IAsyncFile, public ReferenceCounted<Asy
 		state bool saveDurable = true;
 		choose {
 			when(wait(delay(delayDuration))) {}
-			when(bool durable = wait(startSyncFuture)) { saveDurable = durable; }
+			when(bool durable = wait(startSyncFuture)) {
+				saveDurable = durable;
+			}
 		}
 
 		if (g_network->check_yield(TaskPriority::DefaultYield)) {
diff --git a/fdbrpc/FailureMonitor.actor.cpp b/fdbrpc/FailureMonitor.actor.cpp
index dcb4052cf50..372b3812a5c 100644
--- a/fdbrpc/FailureMonitor.actor.cpp
+++ b/fdbrpc/FailureMonitor.actor.cpp
@@ -53,7 +53,9 @@ ACTOR Future<Void> waitForContinuousFailure(IFailureMonitor* monitor,
 		choose {
 			when(wait(monitor->onStateEqual(endpoint, FailureStatus(false)))) {
 			} // SOMEDAY: Use onStateChanged() for efficiency
-			when(wait(delay(waitDelay))) { return Void(); }
+			when(wait(delay(waitDelay))) {
+				return Void();
+			}
 		}
 	}
 }
diff --git a/fdbrpc/FlowTransport.actor.cpp b/fdbrpc/FlowTransport.actor.cpp
index 123120f4c0e..60ae80f4195 100644
--- a/fdbrpc/FlowTransport.actor.cpp
+++ b/fdbrpc/FlowTransport.actor.cpp
@@ -504,7 +504,9 @@ ACTOR Future<Void> connectionMonitor(Reference<Peer> peer) {
 					}
 					break;
 				}
-				when(wait(peer->resetPing.onTrigger())) { break; }
+				when(wait(peer->resetPing.onTrigger())) {
+					break;
+				}
 			}
 		}
 	}
@@ -600,7 +602,9 @@ ACTOR Future<Void> connectionKeeper(Reference<Peer> self,
 
 					choose {
 						when(wait(self->dataToSend.onTrigger())) {}
-						when(wait(retryConnectF)) { break; }
+						when(wait(retryConnectF)) {
+							break;
+						}
 					}
 				}
 
@@ -649,7 +653,9 @@ ACTOR Future<Void> connectionKeeper(Reference<Peer> self,
 							self->prependConnectPacket();
 							reader = connectionReader(self->transport, conn, self, Promise<Reference<Peer>>());
 						}
-						when(wait(delay(FLOW_KNOBS->CONNECTION_MONITOR_TIMEOUT))) { throw connection_failed(); }
+						when(wait(delay(FLOW_KNOBS->CONNECTION_MONITOR_TIMEOUT))) {
+							throw connection_failed();
+						}
 					}
 				} catch (Error& e) {
 					++self->connectFailedCount;
@@ -1348,7 +1354,9 @@ ACTOR static Future<Void> connectionIncoming(TransportData* self, Reference<ICon
 				ASSERT(false);
 				return Void();
 			}
-			when(Reference<Peer> p = wait(onConnected.getFuture())) { p->onIncomingConnection(p, conn, reader); }
+			when(Reference<Peer> p = wait(onConnected.getFuture())) {
+				p->onIncomingConnection(p, conn, reader);
+			}
 			when(wait(delayJittered(FLOW_KNOBS->CONNECTION_MONITOR_TIMEOUT))) {
 				TEST(true); // Incoming connection timed out
 				throw timed_out();
diff --git a/fdbrpc/LoadBalance.actor.cpp b/fdbrpc/LoadBalance.actor.cpp
index 1d02e35cbe0..b668deb8ddf 100644
--- a/fdbrpc/LoadBalance.actor.cpp
+++ b/fdbrpc/LoadBalance.actor.cpp
@@ -50,7 +50,9 @@ ACTOR Future<Void> allAlternativesFailedDelay(Future<Void> okFuture) {
 
 	choose {
 		when(wait(okFuture)) {}
-		when(wait(::delayJittered(delay))) { throw all_alternatives_failed(); }
+		when(wait(::delayJittered(delay))) {
+			throw all_alternatives_failed();
+		}
 	}
 	return Void();
 }
diff --git a/fdbrpc/dsltest.actor.cpp b/fdbrpc/dsltest.actor.cpp
index fa333011453..34c06ba7f89 100644
--- a/fdbrpc/dsltest.actor.cpp
+++ b/fdbrpc/dsltest.actor.cpp
@@ -257,7 +257,9 @@ ACTOR template <int N, class X>
 ACTOR template <class A, class B>
 [[flow_allow_discard]] Future<Void> switchTest(FutureStream<A> as, Future<B> oneb) {
 	loop choose {
-		when(A a = waitNext(as)) { std::cout << "A " << a << std::endl; }
+		when(A a = waitNext(as)) {
+			std::cout << "A " << a << std::endl;
+		}
 		when(B b = wait(oneb)) {
 			std::cout << "B " << b << std::endl;
 			break;
@@ -283,7 +285,7 @@ class TestBuffer : public ReferenceCounted<TestBuffer> {
 #if !defined(__INTEL_COMPILER)
 	void operator delete(void* buf) {
 		std::cout << "Freeing buffer" << std::endl;
-		delete[](int*) buf;
+		delete[] (int*)buf;
 	}
 #endif
 
@@ -614,8 +616,12 @@ void returnCancelRaceTest() {
 
 ACTOR [[flow_allow_discard]] Future<int> chooseTest(Future<int> a, Future<int> b) {
 	choose {
-		when(int A = wait(a)) { return A; }
-		when(int B = wait(b)) { return B; }
+		when(int A = wait(a)) {
+			return A;
+		}
+		when(int B = wait(b)) {
+			return B;
+		}
 	}
 }
 
@@ -960,8 +966,12 @@ ACTOR [[flow_allow_discard]] Future<int> introAdd(Future<int> a, Future<int> b)
 
 ACTOR [[flow_allow_discard]] Future<int> introFirst(Future<int> a, Future<int> b) {
 	choose {
-		when(int x = wait(a)) { return x; }
-		when(int x = wait(b)) { return x; }
+		when(int x = wait(a)) {
+			return x;
+		}
+		when(int x = wait(b)) {
+			return x;
+		}
 	}
 }
 
diff --git a/fdbrpc/genericactors.actor.h b/fdbrpc/genericactors.actor.h
index 2fdf8c5712c..80f51d3a14c 100644
--- a/fdbrpc/genericactors.actor.h
+++ b/fdbrpc/genericactors.actor.h
@@ -193,7 +193,9 @@ ACTOR template <class T>
 Future<T> timeoutWarning(Future<T> what, double time, PromiseStream<Void> output) {
 	state Future<Void> end = delay(time);
 	loop choose {
-		when(T t = wait(what)) { return t; }
+		when(T t = wait(what)) {
+			return t;
+		}
 		when(wait(end)) {
 			output.send(Void());
 			end = delay(time);
@@ -331,7 +333,9 @@ void endStreamOnDisconnect(Future<Void> signal,
 	stream.setRequestStreamEndpoint(endpoint);
 	try {
 		choose {
-			when(wait(signal)) { stream.sendError(connection_failed()); }
+			when(wait(signal)) {
+				stream.sendError(connection_failed());
+			}
 			when(wait(peer.isValid() ? peer->disconnect.getFuture() : Never())) {
 				stream.sendError(connection_failed());
 			}
@@ -360,8 +364,12 @@ Future<ErrorOr<X>> waitValueOrSignal(Future<X> value,
 	loop {
 		try {
 			choose {
-				when(X x = wait(value)) { return x; }
-				when(wait(signal)) { return ErrorOr<X>(request_maybe_delivered()); }
+				when(X x = wait(value)) {
+					return x;
+				}
+				when(wait(signal)) {
+					return ErrorOr<X>(request_maybe_delivered());
+				}
 			}
 		} catch (Error& e) {
 			if (signal.isError()) {
diff --git a/fdbrpc/libcoroutine/Common.h b/fdbrpc/libcoroutine/Common.h
index 97fb4a71f49..70d7f825b1e 100644
--- a/fdbrpc/libcoroutine/Common.h
+++ b/fdbrpc/libcoroutine/Common.h
@@ -39,7 +39,7 @@ typedef long long int64_t;
 
 #if defined(WIN32) || defined(__WINS__) || defined(__MINGW32__) || defined(_MSC_VER)
 #define inline __inline
-//#define snprintf _snprintf
+// #define snprintf _snprintf
 #define usleep(x) Sleep(((x) + 999) / 1000)
 
 #define HAS_FIBERS 1
@@ -145,7 +145,7 @@ as errors in my dev settings */
 extern "C" {
 #endif
 
-//#define IO_CHECK_ALLOC ENABLED(NOT_IN_CLEAN)
+// #define IO_CHECK_ALLOC ENABLED(NOT_IN_CLEAN)
 
 #ifdef IO_CHECK_ALLOC
 BASEKIT_API size_t io_memsize(void* ptr);
diff --git a/fdbrpc/libcoroutine/Coro.h b/fdbrpc/libcoroutine/Coro.h
index f8d7aa72c86..a2b36fcf60f 100644
--- a/fdbrpc/libcoroutine/Coro.h
+++ b/fdbrpc/libcoroutine/Coro.h
@@ -13,8 +13,8 @@
 #define CORO_STACK_SIZE 8192
 #define CORO_STACK_SIZE_MIN 1024
 #else
-//#define CORO_DEFAULT_STACK_SIZE     (65536/2)
-//#define CORO_DEFAULT_STACK_SIZE  (65536*4)
+// #define CORO_DEFAULT_STACK_SIZE     (65536/2)
+// #define CORO_DEFAULT_STACK_SIZE  (65536*4)
 
 // 128k needed on PPC due to parser
 #define CORO_DEFAULT_STACK_SIZE (128 * 1024)
@@ -45,7 +45,7 @@
 #if defined(WIN32) && defined(HAS_FIBERS)
 #define USE_FIBERS
 #elif defined(HAS_UCONTEXT)
-//#elif defined(HAS_UCONTEXT) && !defined(__x86_64__)
+// #elif defined(HAS_UCONTEXT) && !defined(__x86_64__)
 #if !defined(USE_UCONTEXT)
 #define USE_UCONTEXT
 #endif
diff --git a/fdbrpc/libcoroutine/taskimpl.h b/fdbrpc/libcoroutine/taskimpl.h
index 76c6c3d44dd..dfe20f93efa 100644
--- a/fdbrpc/libcoroutine/taskimpl.h
+++ b/fdbrpc/libcoroutine/taskimpl.h
@@ -14,7 +14,7 @@
 #endif
 #endif
 
-//#define USE_UCONTEXT 1
+// #define USE_UCONTEXT 1
 
 #if defined(__OpenBSD__)
 #undef USE_UCONTEXT
@@ -45,7 +45,7 @@
 #endif
 #include <sys/utsname.h>
 #include <inttypes.h>
-//#include "task.h"
+// #include "task.h"
 
 #define nil ((void*)0)
 #define nelem(x) (sizeof(x) / sizeof((x)[0]))
diff --git a/fdbserver/BackupWorker.actor.cpp b/fdbserver/BackupWorker.actor.cpp
index 0f45f29a33e..476e5473f4d 100644
--- a/fdbserver/BackupWorker.actor.cpp
+++ b/fdbserver/BackupWorker.actor.cpp
@@ -909,7 +909,9 @@ ACTOR Future<Void> pullAsyncData(BackupData* self) {
 		}
 
 		loop choose {
-			when(wait(r ? r->getMore(TaskPriority::TLogCommit) : Never())) { break; }
+			when(wait(r ? r->getMore(TaskPriority::TLogCommit) : Never())) {
+				break;
+			}
 			when(wait(logSystemChange)) {
 				if (self->logSystem.get()) {
 					r = self->logSystem.get()->peekLogRouter(self->myId, tagAt, self->tag);
@@ -981,7 +983,9 @@ ACTOR Future<Void> monitorBackupKeyOrPullData(BackupData* self, bool keyPresent)
 			state Future<Version> committedVersion = self->getMinKnownCommittedVersion();
 
 			loop choose {
-				when(wait(success(present))) { break; }
+				when(wait(success(present))) {
+					break;
+				}
 				when(wait(success(committedVersion) || delay(SERVER_KNOBS->BACKUP_NOOP_POP_DELAY, self->cx->taskID))) {
 					if (committedVersion.isReady()) {
 						self->popVersion =
diff --git a/fdbserver/BlobManager.actor.cpp b/fdbserver/BlobManager.actor.cpp
index 23b32381d55..062f036e2c6 100644
--- a/fdbserver/BlobManager.actor.cpp
+++ b/fdbserver/BlobManager.actor.cpp
@@ -2090,7 +2090,9 @@ ACTOR Future<Void> blobWorkerRecruiter(
 
 	// wait until existing blob workers have been acknowledged so we don't break recruitment invariants
 	loop choose {
-		when(wait(self->startRecruiting.onTrigger())) { break; }
+		when(wait(self->startRecruiting.onTrigger())) {
+			break;
+		}
 	}
 
 	loop {
@@ -2127,7 +2129,9 @@ ACTOR Future<Void> blobWorkerRecruiter(
 				}
 
 				// when the CC changes, so does the request stream so we need to restart recruiting here
-				when(wait(recruitBlobWorker->onChange())) { fCandidateWorker = Future<RecruitBlobWorkerReply>(); }
+				when(wait(recruitBlobWorker->onChange())) {
+					fCandidateWorker = Future<RecruitBlobWorkerReply>();
+				}
 
 				// signal used to restart the loop and try to recruit the next blob worker
 				when(wait(self->restartRecruiting.onTrigger())) {}
diff --git a/fdbserver/BlobWorker.actor.cpp b/fdbserver/BlobWorker.actor.cpp
index 0dba051a8cd..aebca27a308 100644
--- a/fdbserver/BlobWorker.actor.cpp
+++ b/fdbserver/BlobWorker.actor.cpp
@@ -931,9 +931,13 @@ ACTOR Future<BlobFileIndex> checkSplitAndReSnapshot(Reference<BlobWorkerData> bw
 				// wait for manager stream to become ready, and send a message
 				loop {
 					choose {
-						when(wait(bwData->currentManagerStatusStream.get().onReady())) { break; }
+						when(wait(bwData->currentManagerStatusStream.get().onReady())) {
+							break;
+						}
 						when(wait(bwData->currentManagerStatusStream.onChange())) {}
-						when(wait(metadata->resumeSnapshot.getFuture())) { break; }
+						when(wait(metadata->resumeSnapshot.getFuture())) {
+							break;
+						}
 					}
 				}
 				if (metadata->resumeSnapshot.isSet()) {
@@ -962,7 +966,9 @@ ACTOR Future<BlobFileIndex> checkSplitAndReSnapshot(Reference<BlobWorkerData> bw
 		// change/no response
 		choose {
 			when(wait(bwData->currentManagerStatusStream.onChange())) {}
-			when(wait(metadata->resumeSnapshot.getFuture())) { break; }
+			when(wait(metadata->resumeSnapshot.getFuture())) {
+				break;
+			}
 			when(wait(delay(1.0))) {}
 		}
 
@@ -1226,7 +1232,9 @@ ACTOR Future<Void> waitOnCFVersion(Reference<GranuleMetadata> metadata, Version
 			                                 ? metadata->activeCFData.get()->whenAtLeast(waitVersion)
 			                                 : Never();
 			choose {
-				when(wait(atLeast)) { break; }
+				when(wait(atLeast)) {
+					break;
+				}
 				when(wait(metadata->activeCFData.onChange())) {}
 			}
 		} catch (Error& e) {
@@ -2250,7 +2258,9 @@ ACTOR Future<Void> doBlobGranuleFileRequest(Reference<BlobWorkerData> bwData, Bl
 
 			choose {
 				when(wait(metadata->readable.getFuture())) {}
-				when(wait(metadata->cancelled.getFuture())) { throw wrong_shard_server(); }
+				when(wait(metadata->cancelled.getFuture())) {
+					throw wrong_shard_server();
+				}
 			}
 
 			// in case both readable and cancelled are ready, check cancelled
@@ -2267,7 +2277,9 @@ ACTOR Future<Void> doBlobGranuleFileRequest(Reference<BlobWorkerData> bwData, Bl
 				if (metadata->historyLoaded.canBeSet()) {
 					choose {
 						when(wait(metadata->historyLoaded.getFuture())) {}
-						when(wait(metadata->cancelled.getFuture())) { throw wrong_shard_server(); }
+						when(wait(metadata->cancelled.getFuture())) {
+							throw wrong_shard_server();
+						}
 					}
 				}
 
@@ -2315,8 +2327,12 @@ ACTOR Future<Void> doBlobGranuleFileRequest(Reference<BlobWorkerData> bwData, Bl
 				}
 
 				choose {
-					when(GranuleFiles _f = wait(cur->files)) { chunkFiles = _f; }
-					when(wait(metadata->cancelled.getFuture())) { throw wrong_shard_server(); }
+					when(GranuleFiles _f = wait(cur->files)) {
+						chunkFiles = _f;
+					}
+					when(wait(metadata->cancelled.getFuture())) {
+						throw wrong_shard_server();
+					}
 				}
 
 				if (chunkFiles.snapshotFiles.empty()) {
@@ -2351,9 +2367,13 @@ ACTOR Future<Void> doBlobGranuleFileRequest(Reference<BlobWorkerData> bwData, Bl
 					// version on rollback
 					try {
 						choose {
-							when(wait(waitForVersionFuture)) { break; }
+							when(wait(waitForVersionFuture)) {
+								break;
+							}
 							when(wait(metadata->activeCFData.onChange())) {}
-							when(wait(metadata->cancelled.getFuture())) { throw wrong_shard_server(); }
+							when(wait(metadata->cancelled.getFuture())) {
+								throw wrong_shard_server();
+							}
 						}
 					} catch (Error& e) {
 						// We can get change feed cancelled from whenAtLeast. This means the change feed may retry, or
diff --git a/fdbserver/ClusterController.actor.cpp b/fdbserver/ClusterController.actor.cpp
index 3deee6c8593..2e799e348e4 100644
--- a/fdbserver/ClusterController.actor.cpp
+++ b/fdbserver/ClusterController.actor.cpp
@@ -286,7 +286,9 @@ ACTOR Future<Void> clusterWatchDatabase(ClusterControllerData* cluster,
 					req.reply.send(Void());
 					TraceEvent(SevDebug, "BackupWorkerDoneRequest", cluster->id).log();
 				}
-				when(wait(collection)) { throw internal_error(); }
+				when(wait(collection)) {
+					throw internal_error();
+				}
 			}
 			// failed master (better master exists) could happen while change-coordinators request processing is
 			// in-progress
@@ -344,7 +346,9 @@ ACTOR Future<Void> clusterGetServerInfo(ClusterControllerData::DBInfo* db,
 	while (db->serverInfo->get().id == knownServerInfoID) {
 		choose {
 			when(wait(yieldedFuture(db->serverInfo->onChange()))) {}
-			when(wait(delayJittered(300))) { break; } // The server might be long gone!
+			when(wait(delayJittered(300))) {
+				break;
+			} // The server might be long gone!
 		}
 	}
 	reply.send(db->serverInfo->get());
@@ -2350,7 +2354,9 @@ ACTOR Future<Void> monitorBlobManager(ClusterControllerData* self) {
 						self->db.clearInterf(ProcessClass::BlobManagerClass);
 						break;
 					}
-					when(wait(self->recruitBlobManager.onChange())) { break; }
+					when(wait(self->recruitBlobManager.onChange())) {
+						break;
+					}
 					when(wait(self->db.blobGranulesEnabled.onChange())) {
 						// if there is a blob manager present but blob granules are now disabled, stop the BM
 						if (!self->db.blobGranulesEnabled.get()) {
@@ -2377,7 +2383,9 @@ ACTOR Future<Void> dbInfoUpdater(ClusterControllerData* self) {
 	state Future<Void> updateDBInfo = self->updateDBInfo.onTrigger();
 	loop {
 		choose {
-			when(wait(updateDBInfo)) { wait(delay(SERVER_KNOBS->DBINFO_BATCH_DELAY) || dbInfoChange); }
+			when(wait(updateDBInfo)) {
+				wait(delay(SERVER_KNOBS->DBINFO_BATCH_DELAY) || dbInfoChange);
+			}
 			when(wait(dbInfoChange)) {}
 		}
 
@@ -2634,7 +2642,9 @@ ACTOR Future<Void> clusterControllerCore(ClusterControllerFullInterface interf,
 			TEST(true); // Leader replaced
 			return Void();
 		}
-		when(ReplyPromise<Void> ping = waitNext(interf.clientInterface.ping.getFuture())) { ping.send(Void()); }
+		when(ReplyPromise<Void> ping = waitNext(interf.clientInterface.ping.getFuture())) {
+			ping.send(Void());
+		}
 	}
 }
 
@@ -2671,7 +2681,9 @@ ACTOR Future<Void> clusterController(ServerCoordinators coordinators,
 						ASSERT(false);
 						throw internal_error();
 					}
-					when(wait(shouldReplace)) { break; }
+					when(wait(shouldReplace)) {
+						break;
+					}
 				}
 			}
 			if (!shouldReplace.isReady()) {
diff --git a/fdbserver/ClusterRecovery.actor.cpp b/fdbserver/ClusterRecovery.actor.cpp
index b604220390b..3b5f73b67f1 100644
--- a/fdbserver/ClusterRecovery.actor.cpp
+++ b/fdbserver/ClusterRecovery.actor.cpp
@@ -62,7 +62,9 @@ ACTOR Future<Void> recoveryTerminateOnConflict(UID dbgid,
 			}
 			return Void();
 		}
-		when(wait(switchedState)) { return Void(); }
+		when(wait(switchedState)) {
+			return Void();
+		}
 	}
 }
 
@@ -891,8 +893,12 @@ ACTOR Future<Standalone<CommitTransactionRef>> provisionalMaster(Reference<Clust
 		         waitNext(parent->provisionalCommitProxies[0].getKeyServersLocations.getFuture())) {
 			req.reply.send(Never());
 		}
-		when(wait(waitCommitProxyFailure)) { throw worker_removed(); }
-		when(wait(waitGrvProxyFailure)) { throw worker_removed(); }
+		when(wait(waitCommitProxyFailure)) {
+			throw worker_removed();
+		}
+		when(wait(waitGrvProxyFailure)) {
+			throw worker_removed();
+		}
 	}
 }
 
@@ -1495,8 +1501,12 @@ ACTOR Future<Void> clusterRecoveryCore(Reference<ClusterRecoveryData> self) {
 				break;
 			}
 			when(wait(oldLogSystems->onChange())) {}
-			when(wait(reg)) { throw internal_error(); }
-			when(wait(recoverAndEndEpoch)) { throw internal_error(); }
+			when(wait(reg)) {
+				throw internal_error();
+			}
+			when(wait(recoverAndEndEpoch)) {
+				throw internal_error();
+			}
 		}
 	}
 
diff --git a/fdbserver/ConfigDatabaseUnitTests.actor.cpp b/fdbserver/ConfigDatabaseUnitTests.actor.cpp
index dba90dcbbd8..be14a688595 100644
--- a/fdbserver/ConfigDatabaseUnitTests.actor.cpp
+++ b/fdbserver/ConfigDatabaseUnitTests.actor.cpp
@@ -675,7 +675,9 @@ Future<Void> testIgnore(UnitTestParameters params) {
 	wait(set(env, "class-B"_sr, "test_long"_sr, int64_t{ 1 }));
 	choose {
 		when(wait(delay(5))) {}
-		when(wait(check(env, &TestKnobs::TEST_LONG, Optional<int64_t>{ 1 }))) { ASSERT(false); }
+		when(wait(check(env, &TestKnobs::TEST_LONG, Optional<int64_t>{ 1 }))) {
+			ASSERT(false);
+		}
 	}
 	return Void();
 }
diff --git a/fdbserver/ConfigNode.actor.cpp b/fdbserver/ConfigNode.actor.cpp
index c76ee882cfd..9f842ff4e86 100644
--- a/fdbserver/ConfigNode.actor.cpp
+++ b/fdbserver/ConfigNode.actor.cpp
@@ -423,7 +423,9 @@ class ConfigNodeImpl {
 				when(ConfigTransactionGetKnobsRequest req = waitNext(cti->getKnobs.getFuture())) {
 					wait(getKnobs(self, req));
 				}
-				when(wait(self->kvStore->getError())) { ASSERT(false); }
+				when(wait(self->kvStore->getError())) {
+					ASSERT(false);
+				}
 			}
 		}
 	}
@@ -575,7 +577,9 @@ class ConfigNodeImpl {
 					++self->getCommittedVersionRequests;
 					wait(getCommittedVersion(self, req));
 				}
-				when(wait(self->kvStore->getError())) { ASSERT(false); }
+				when(wait(self->kvStore->getError())) {
+					ASSERT(false);
+				}
 			}
 		}
 	}
diff --git a/fdbserver/CoordinatedState.actor.cpp b/fdbserver/CoordinatedState.actor.cpp
index 72e75bf3be1..1af81799cca 100644
--- a/fdbserver/CoordinatedState.actor.cpp
+++ b/fdbserver/CoordinatedState.actor.cpp
@@ -301,7 +301,9 @@ struct MovableCoordinatedStateImpl {
 		ASSERT(self->lastValue.present() && self->lastCSValue.present());
 		TraceEvent("StartMove").detail("ConnectionString", nc.toString());
 		choose {
-			when(wait(creationTimeout)) { throw new_coordinators_timed_out(); }
+			when(wait(creationTimeout)) {
+				throw new_coordinators_timed_out();
+			}
 			when(Value ncInitialValue = wait(nccs.read())) {
 				ASSERT(!ncInitialValue.size()); // The new coordinators must be uninitialized!
 			}
@@ -309,7 +311,9 @@ struct MovableCoordinatedStateImpl {
 		TraceEvent("FinishedRead").detail("ConnectionString", nc.toString());
 
 		choose {
-			when(wait(creationTimeout)) { throw new_coordinators_timed_out(); }
+			when(wait(creationTimeout)) {
+				throw new_coordinators_timed_out();
+			}
 			when(wait(nccs.setExclusive(
 			    BinaryWriter::toValue(MovableValue(self->lastValue.get(),
 			                                       MovableValue::MovingFrom,
diff --git a/fdbserver/Coordination.actor.cpp b/fdbserver/Coordination.actor.cpp
index 2a208ffe29e..257b659fb80 100644
--- a/fdbserver/Coordination.actor.cpp
+++ b/fdbserver/Coordination.actor.cpp
@@ -47,7 +47,9 @@ class LivenessChecker {
 	ACTOR static Future<Void> checkStuck(LivenessChecker const* self) {
 		loop {
 			choose {
-				when(wait(delayUntil(self->lastTime.get() + self->threshold))) { return Void(); }
+				when(wait(delayUntil(self->lastTime.get() + self->threshold))) {
+					return Void();
+				}
 				when(wait(self->lastTime.onChange())) {}
 			}
 		}
@@ -275,7 +277,9 @@ ACTOR Future<Void> remoteMonitorLeader(int* clientCount,
 			when(wait(yieldedFuture(currentElectedLeaderOnChange))) {
 				currentElectedLeaderOnChange = currentElectedLeader->onChange();
 			}
-			when(wait(delayJittered(SERVER_KNOBS->CLIENT_REGISTER_INTERVAL))) { break; }
+			when(wait(delayJittered(SERVER_KNOBS->CLIENT_REGISTER_INTERVAL))) {
+				break;
+			}
 		}
 	}
 
diff --git a/fdbserver/DDTeamCollection.actor.cpp b/fdbserver/DDTeamCollection.actor.cpp
index d573f5c910f..6f016c48eca 100644
--- a/fdbserver/DDTeamCollection.actor.cpp
+++ b/fdbserver/DDTeamCollection.actor.cpp
@@ -127,7 +127,9 @@ class DDTeamCollectionImpl {
 
 		loop {
 			choose {
-				when(wait(self->buildTeams())) { return Void(); }
+				when(wait(self->buildTeams())) {
+					return Void();
+				}
 				when(wait(self->restartTeamBuilder.onTrigger())) {}
 			}
 		}
@@ -505,7 +507,9 @@ class DDTeamCollectionImpl {
 		while (self->pauseWiggle && !self->pauseWiggle->get() && self->waitUntilRecruited.get()) {
 			choose {
 				when(wait(self->waitUntilRecruited.onChange() || self->pauseWiggle->onChange())) {}
-				when(wait(delay(SERVER_KNOBS->PERPETUAL_WIGGLE_DELAY, g_network->getCurrentTask()))) { break; }
+				when(wait(delay(SERVER_KNOBS->PERPETUAL_WIGGLE_DELAY, g_network->getCurrentTask()))) {
+					break;
+				}
 			}
 		}
 
@@ -1339,7 +1343,9 @@ class DDTeamCollectionImpl {
 						    .detail("ConfigStoreType", self->configuration.storageServerStoreType)
 						    .detail("WrongStoreTypeRemoved", server->wrongStoreTypeToRemove.get());
 					}
-					when(wait(server->wakeUpTracker.getFuture())) { server->wakeUpTracker = Promise<Void>(); }
+					when(wait(server->wakeUpTracker.getFuture())) {
+						server->wakeUpTracker = Promise<Void>();
+					}
 					when(wait(storageMetadataTracker || storeTypeTracker)) {}
 					when(wait(server->ssVersionTooFarBehind.onChange())) {}
 					when(wait(self->disableFailingLaggingServers.onChange())) {}
@@ -2063,7 +2069,9 @@ class DDTeamCollectionImpl {
 							    .detail("ExtraHealthyTeamCount", extraTeamCount)
 							    .detail("HealthyTeamCount", self->healthyTeamCount);
 						}
-						when(wait(pauseChanged)) { continue; }
+						when(wait(pauseChanged)) {
+							continue;
+						}
 					}
 				}
 			}
@@ -2587,7 +2595,9 @@ class DDTeamCollectionImpl {
 							}
 						}
 					}
-					when(wait(recruitStorage->onChange())) { fCandidateWorker = Future<RecruitStorageReply>(); }
+					when(wait(recruitStorage->onChange())) {
+						fCandidateWorker = Future<RecruitStorageReply>();
+					}
 					when(wait(self->zeroHealthyTeams->onChange())) {
 						if (!pendingTSSCheck && self->zeroHealthyTeams->get() &&
 						    (self->isTssRecruiting || self->tss_info_by_pair.size() > 0)) {
diff --git a/fdbserver/DataDistributionQueue.actor.cpp b/fdbserver/DataDistributionQueue.actor.cpp
index 4a6ab30b555..00db7129e09 100644
--- a/fdbserver/DataDistributionQueue.actor.cpp
+++ b/fdbserver/DataDistributionQueue.actor.cpp
@@ -1795,7 +1795,9 @@ ACTOR Future<Void> dataDistributionQueue(Database cx,
 						debug_setCheckRelocationDuration(false);
 					}
 				}
-				when(KeyRange done = waitNext(rangesComplete.getFuture())) { keysToLaunchFrom = done; }
+				when(KeyRange done = waitNext(rangesComplete.getFuture())) {
+					keysToLaunchFrom = done;
+				}
 				when(wait(recordMetrics)) {
 					Promise<int64_t> req;
 					getAverageShardBytes.send(req);
@@ -1838,7 +1840,9 @@ ACTOR Future<Void> dataDistributionQueue(Database cx,
 				}
 				when(wait(self.error.getFuture())) {} // Propagate errors from dataDistributionRelocator
 				when(wait(waitForAll(balancingFutures))) {}
-				when(Promise<int> r = waitNext(getUnhealthyRelocationCount)) { r.send(self.unhealthyRelocations); }
+				when(Promise<int> r = waitNext(getUnhealthyRelocationCount)) {
+					r.send(self.unhealthyRelocations);
+				}
 			}
 		}
 	} catch (Error& e) {
diff --git a/fdbserver/DataDistributionTracker.actor.cpp b/fdbserver/DataDistributionTracker.actor.cpp
index e68f3df2a05..ebdace2df90 100644
--- a/fdbserver/DataDistributionTracker.actor.cpp
+++ b/fdbserver/DataDistributionTracker.actor.cpp
@@ -914,7 +914,9 @@ ACTOR Future<Void> fetchShardMetricsList_impl(DataDistributionTracker* self, Get
 ACTOR Future<Void> fetchShardMetricsList(DataDistributionTracker* self, GetMetricsListRequest req) {
 	choose {
 		when(wait(fetchShardMetricsList_impl(self, req))) {}
-		when(wait(delay(SERVER_KNOBS->DD_SHARD_METRICS_TIMEOUT))) { req.reply.sendError(timed_out()); }
+		when(wait(delay(SERVER_KNOBS->DD_SHARD_METRICS_TIMEOUT))) {
+			req.reply.sendError(timed_out());
+		}
 	}
 	return Void();
 }
diff --git a/fdbserver/KeyValueStoreSQLite.actor.cpp b/fdbserver/KeyValueStoreSQLite.actor.cpp
index 1c24ce44d13..9b5472dcef9 100644
--- a/fdbserver/KeyValueStoreSQLite.actor.cpp
+++ b/fdbserver/KeyValueStoreSQLite.actor.cpp
@@ -696,7 +696,7 @@ struct IntKeyCursor {
 				db.checkError("BtreeCloseCursor", sqlite3BtreeCloseCursor(cursor));
 			} catch (...) {
 			}
-			delete[](char*) cursor;
+			delete[] (char*)cursor;
 		}
 	}
 };
@@ -734,7 +734,7 @@ struct RawCursor {
 			} catch (...) {
 				TraceEvent(SevError, "RawCursorDestructionError").log();
 			}
-			delete[](char*) cursor;
+			delete[] (char*)cursor;
 		}
 	}
 	void moveFirst() {
diff --git a/fdbserver/LeaderElection.actor.cpp b/fdbserver/LeaderElection.actor.cpp
index 3c23827e710..e641f3d0f31 100644
--- a/fdbserver/LeaderElection.actor.cpp
+++ b/fdbserver/LeaderElection.actor.cpp
@@ -210,8 +210,12 @@ ACTOR Future<Void> tryBecomeLeaderInternal(ServerCoordinators coordinators,
 					TraceEvent("LeaderBadCandidateTimeout", myInfo.changeID).log();
 					break;
 				}
-				when(wait(candidacies)) { ASSERT(false); }
-				when(wait(asyncPriorityInfo->onChange())) { break; }
+				when(wait(candidacies)) {
+					ASSERT(false);
+				}
+				when(wait(asyncPriorityInfo->onChange())) {
+					break;
+				}
 			}
 		}
 
diff --git a/fdbserver/LocalConfiguration.actor.cpp b/fdbserver/LocalConfiguration.actor.cpp
index fbc3380f9e2..d31f037eba3 100644
--- a/fdbserver/LocalConfiguration.actor.cpp
+++ b/fdbserver/LocalConfiguration.actor.cpp
@@ -318,8 +318,12 @@ class LocalConfigurationImpl {
 	ACTOR static Future<Void> consume(LocalConfigurationImpl* self, ConfigBroadcastInterface broadcaster) {
 		loop {
 			choose {
-				when(wait(consumeInternal(self, broadcaster))) { ASSERT(false); }
-				when(wait(self->kvStore->getError())) { ASSERT(false); }
+				when(wait(consumeInternal(self, broadcaster))) {
+					ASSERT(false);
+				}
+				when(wait(self->kvStore->getError())) {
+					ASSERT(false);
+				}
 			}
 		}
 	}
diff --git a/fdbserver/LogRouter.actor.cpp b/fdbserver/LogRouter.actor.cpp
index eb84a1990f3..3bab655a178 100644
--- a/fdbserver/LogRouter.actor.cpp
+++ b/fdbserver/LogRouter.actor.cpp
@@ -794,7 +794,9 @@ ACTOR Future<Void> logRouter(TLogInterface interf,
 		    .detail("Locality", req.locality);
 		state Future<Void> core = logRouterCore(interf, req, db);
 		loop choose {
-			when(wait(core)) { return Void(); }
+			when(wait(core)) {
+				return Void();
+			}
 			when(wait(checkRemoved(db, req.recoveryCount, interf))) {}
 		}
 	} catch (Error& e) {
diff --git a/fdbserver/LogSystemDiskQueueAdapter.actor.cpp b/fdbserver/LogSystemDiskQueueAdapter.actor.cpp
index 029e551e7aa..180abdbe76a 100644
--- a/fdbserver/LogSystemDiskQueueAdapter.actor.cpp
+++ b/fdbserver/LogSystemDiskQueueAdapter.actor.cpp
@@ -43,7 +43,9 @@ class LogSystemDiskQueueAdapterImpl {
 			if (!self->cursor->hasMessage()) {
 				loop {
 					choose {
-						when(wait(self->cursor->getMore())) { break; }
+						when(wait(self->cursor->getMore())) {
+							break;
+						}
 						when(wait(self->localityChanged)) {
 							self->cursor = self->logSystem->peekTxs(
 							    UID(),
diff --git a/fdbserver/LogSystemPeekCursor.actor.cpp b/fdbserver/LogSystemPeekCursor.actor.cpp
index 456e462fec1..96f43df73e7 100644
--- a/fdbserver/LogSystemPeekCursor.actor.cpp
+++ b/fdbserver/LogSystemPeekCursor.actor.cpp
@@ -439,7 +439,9 @@ ACTOR Future<Void> serverPeekGetMore(ILogSystem::ServerPeekCursor* self, TaskPri
 					    .detail("Popped", res.popped.present() ? res.popped.get() : 0);
 					return Void();
 				}
-				when(wait(self->interf->onChange())) { self->onlySpilled = false; }
+				when(wait(self->interf->onChange())) {
+					self->onlySpilled = false;
+				}
 			}
 		}
 	} catch (Error& e) {
diff --git a/fdbserver/OldTLogServer_4_6.actor.cpp b/fdbserver/OldTLogServer_4_6.actor.cpp
index b523212f49c..9863f015233 100644
--- a/fdbserver/OldTLogServer_4_6.actor.cpp
+++ b/fdbserver/OldTLogServer_4_6.actor.cpp
@@ -771,7 +771,9 @@ ACTOR Future<Void> updateStorage(TLogData* self) {
 ACTOR Future<Void> updateStorageLoop(TLogData* self) {
 	wait(delay(0, TaskPriority::UpdateStorage));
 
-	loop { wait(updateStorage(self)); }
+	loop {
+		wait(updateStorage(self));
+	}
 }
 
 void commitMessages(Reference<LogData> self,
@@ -1593,13 +1595,17 @@ ACTOR Future<Void> restorePersistentState(TLogData* self, LocalityData locality)
 
 								choose {
 									when(wait(updateStorage(self))) {}
-									when(wait(allRemoved)) { throw worker_removed(); }
+									when(wait(allRemoved)) {
+										throw worker_removed();
+									}
 								}
 							}
 						}
 					}
 				}
-				when(wait(allRemoved)) { throw worker_removed(); }
+				when(wait(allRemoved)) {
+					throw worker_removed();
+				}
 			}
 		}
 	} catch (Error& e) {
diff --git a/fdbserver/OldTLogServer_6_0.actor.cpp b/fdbserver/OldTLogServer_6_0.actor.cpp
index 1502dbaa2b0..9c4f3ef611e 100644
--- a/fdbserver/OldTLogServer_6_0.actor.cpp
+++ b/fdbserver/OldTLogServer_6_0.actor.cpp
@@ -1020,7 +1020,9 @@ ACTOR Future<Void> updateStorage(TLogData* self) {
 ACTOR Future<Void> updateStorageLoop(TLogData* self) {
 	wait(delay(0, TaskPriority::UpdateStorage));
 
-	loop { wait(updateStorage(self)); }
+	loop {
+		wait(updateStorage(self));
+	}
 }
 
 void commitMessages(TLogData* self,
@@ -2078,7 +2080,9 @@ ACTOR Future<Void> pullAsyncData(TLogData* self,
 	while (!endVersion.present() || logData->version.get() < endVersion.get()) {
 		loop {
 			choose {
-				when(wait(r ? r->getMore(TaskPriority::TLogCommit) : Never())) { break; }
+				when(wait(r ? r->getMore(TaskPriority::TLogCommit) : Never())) {
+					break;
+				}
 				when(wait(dbInfoChange)) {
 					if (logData->logSystem->get()) {
 						r = logData->logSystem->get()->peek(logData->logId, tagAt, endVersion, tags, true);
@@ -2510,13 +2514,17 @@ ACTOR Future<Void> restorePersistentState(TLogData* self,
 
 								choose {
 									when(wait(updateStorage(self))) {}
-									when(wait(allRemoved)) { throw worker_removed(); }
+									when(wait(allRemoved)) {
+										throw worker_removed();
+									}
 								}
 							}
 						}
 					}
 				}
-				when(wait(allRemoved)) { throw worker_removed(); }
+				when(wait(allRemoved)) {
+					throw worker_removed();
+				}
 			}
 		}
 	} catch (Error& e) {
@@ -2836,7 +2844,9 @@ ACTOR Future<Void> tLog(IKeyValueStore* persistentData,
 						forwardPromise(req.reply, self.tlogCache.get(req.recruitmentID));
 					}
 				}
-				when(wait(error)) { throw internal_error(); }
+				when(wait(error)) {
+					throw internal_error();
+				}
 				when(wait(activeSharedChange)) {
 					if (activeSharedTLog->get() == tlogId) {
 						self.targetVolatileBytes = SERVER_KNOBS->TLOG_SPILL_THRESHOLD;
diff --git a/fdbserver/OldTLogServer_6_2.actor.cpp b/fdbserver/OldTLogServer_6_2.actor.cpp
index ce9b4e5c407..f591d95f5e1 100644
--- a/fdbserver/OldTLogServer_6_2.actor.cpp
+++ b/fdbserver/OldTLogServer_6_2.actor.cpp
@@ -1239,7 +1239,9 @@ ACTOR Future<Void> updateStorage(TLogData* self) {
 ACTOR Future<Void> updateStorageLoop(TLogData* self) {
 	wait(delay(0, TaskPriority::UpdateStorage));
 
-	loop { wait(updateStorage(self)); }
+	loop {
+		wait(updateStorage(self));
+	}
 }
 
 void commitMessages(TLogData* self,
@@ -2538,7 +2540,9 @@ ACTOR Future<Void> pullAsyncData(TLogData* self,
 	while (!endVersion.present() || logData->version.get() < endVersion.get()) {
 		loop {
 			choose {
-				when(wait(r ? r->getMore(TaskPriority::TLogCommit) : Never())) { break; }
+				when(wait(r ? r->getMore(TaskPriority::TLogCommit) : Never())) {
+					break;
+				}
 				when(wait(dbInfoChange)) {
 					if (logData->logSystem->get()) {
 						r = logData->logSystem->get()->peek(logData->logId, tagAt, endVersion, tags, parallelGetMore);
@@ -2988,7 +2992,9 @@ ACTOR Future<Void> restorePersistentState(TLogData* self,
 
 								choose {
 									when(wait(updateStorage(self))) {}
-									when(wait(allRemoved)) { throw worker_removed(); }
+									when(wait(allRemoved)) {
+										throw worker_removed();
+									}
 								}
 							}
 						} else {
@@ -2999,7 +3005,9 @@ ACTOR Future<Void> restorePersistentState(TLogData* self,
 						}
 					}
 				}
-				when(wait(allRemoved)) { throw worker_removed(); }
+				when(wait(allRemoved)) {
+					throw worker_removed();
+				}
 			}
 		}
 	} catch (Error& e) {
@@ -3325,7 +3333,9 @@ ACTOR Future<Void> tLog(IKeyValueStore* persistentData,
 						forwardPromise(req.reply, self.tlogCache.get(req.recruitmentID));
 					}
 				}
-				when(wait(error)) { throw internal_error(); }
+				when(wait(error)) {
+					throw internal_error();
+				}
 				when(wait(activeSharedTLog->onChange())) {
 					if (activeSharedTLog->get() == tlogId) {
 						self.targetVolatileBytes = SERVER_KNOBS->TLOG_SPILL_THRESHOLD;
diff --git a/fdbserver/QuietDatabase.actor.cpp b/fdbserver/QuietDatabase.actor.cpp
index 014e091b79c..615de7b523d 100644
--- a/fdbserver/QuietDatabase.actor.cpp
+++ b/fdbserver/QuietDatabase.actor.cpp
@@ -358,7 +358,9 @@ ACTOR Future<TraceEventFields> getStorageMetricsTimeout(UID storage, WorkerInter
 	    wi.eventLogRequest.getReply(EventLogRequest(StringRef(storage.toString() + "/StorageMetrics")));
 	state Future<Void> timeout = delay(1.0);
 	choose {
-		when(TraceEventFields res = wait(result)) { return res; }
+		when(TraceEventFields res = wait(result)) {
+			return res;
+		}
 		when(wait(timeout)) {
 			TraceEvent("QuietDatabaseFailure")
 			    .detail("Reason", "Could not fetch StorageMetrics")
diff --git a/fdbserver/RemoteIKeyValueStore.actor.cpp b/fdbserver/RemoteIKeyValueStore.actor.cpp
index bead82267ac..0e5bcc80d29 100644
--- a/fdbserver/RemoteIKeyValueStore.actor.cpp
+++ b/fdbserver/RemoteIKeyValueStore.actor.cpp
@@ -55,7 +55,9 @@ struct AfterReturn {
 ACTOR void sendCommitReply(IKVSCommitRequest commitReq, IKeyValueStore* kvStore, Future<Void> onClosed) {
 	try {
 		choose {
-			when(wait(onClosed)) { commitReq.reply.sendError(remote_kvs_cancelled()); }
+			when(wait(onClosed)) {
+				commitReq.reply.sendError(remote_kvs_cancelled());
+			}
 			when(wait(kvStore->commit(commitReq.sequential))) {
 				StorageBytes storageBytes = kvStore->getStorageBytes();
 				commitReq.reply.send(IKVSCommitReply(storageBytes));
@@ -102,8 +104,12 @@ ACTOR Future<Void> runIKVS(OpenKVStoreRequest openReq, IKVSInterface ikvsInterfa
 					actors.add(cancellableForwardPromise(getReq.reply,
 					                                     kvStore->readValue(getReq.key, getReq.type, getReq.debugID)));
 				}
-				when(IKVSSetRequest req = waitNext(ikvsInterface.set.getFuture())) { kvStore->set(req.keyValue); }
-				when(IKVSClearRequest req = waitNext(ikvsInterface.clear.getFuture())) { kvStore->clear(req.range); }
+				when(IKVSSetRequest req = waitNext(ikvsInterface.set.getFuture())) {
+					kvStore->set(req.keyValue);
+				}
+				when(IKVSClearRequest req = waitNext(ikvsInterface.clear.getFuture())) {
+					kvStore->clear(req.range);
+				}
 				when(IKVSCommitRequest commitReq = waitNext(ikvsInterface.commit.getFuture())) {
 					sendCommitReply(commitReq, kvStore, onClosed.getFuture());
 				}
diff --git a/fdbserver/Resolver.actor.cpp b/fdbserver/Resolver.actor.cpp
index 976d3a20701..751f1de5447 100644
--- a/fdbserver/Resolver.actor.cpp
+++ b/fdbserver/Resolver.actor.cpp
@@ -237,7 +237,9 @@ ACTOR Future<Void> resolveBatch(Reference<Resolver> self, ResolveTransactionBatc
 		}
 
 		choose {
-			when(wait(self->version.whenAtLeast(req.prevVersion))) { break; }
+			when(wait(self->version.whenAtLeast(req.prevVersion))) {
+				break;
+			}
 			when(wait(self->checkNeededVersion.onTrigger())) {}
 		}
 	}
@@ -718,7 +720,9 @@ ACTOR Future<Void> resolver(ResolverInterface resolver,
 	try {
 		state Future<Void> core = resolverCore(resolver, initReq, db);
 		loop choose {
-			when(wait(core)) { return Void(); }
+			when(wait(core)) {
+				return Void();
+			}
 			when(wait(checkRemoved(db, initReq.recoveryCount, resolver))) {}
 		}
 	} catch (Error& e) {
diff --git a/fdbserver/RestoreLoader.actor.cpp b/fdbserver/RestoreLoader.actor.cpp
index 1afabdcb950..2d87bacc551 100644
--- a/fdbserver/RestoreLoader.actor.cpp
+++ b/fdbserver/RestoreLoader.actor.cpp
@@ -296,7 +296,9 @@ ACTOR Future<Void> restoreLoaderCore(RestoreLoaderInterface loaderInterf,
 					TraceEvent("FastRestoreLoaderCoreExitRole", self->id());
 					break;
 				}
-				when(wait(error)) { TraceEvent("FastRestoreLoaderActorCollectionError", self->id()); }
+				when(wait(error)) {
+					TraceEvent("FastRestoreLoaderActorCollectionError", self->id());
+				}
 			}
 		} catch (Error& e) {
 			bool isError = e.code() != error_code_operation_cancelled; // == error_code_broken_promise
diff --git a/fdbserver/RestoreUtil.h b/fdbserver/RestoreUtil.h
index e198e39e01e..a21b51187fa 100644
--- a/fdbserver/RestoreUtil.h
+++ b/fdbserver/RestoreUtil.h
@@ -38,10 +38,10 @@
 #include <cstdarg>
 
 #define SevFRMutationInfo SevVerbose
-//#define SevFRMutationInfo SevInfo
+// #define SevFRMutationInfo SevInfo
 
 #define SevFRDebugInfo SevVerbose
-//#define SevFRDebugInfo SevInfo
+// #define SevFRDebugInfo SevInfo
 
 struct VersionedMutation {
 	MutationRef mutation;
diff --git a/fdbserver/SimEncryptVaultProxy.actor.cpp b/fdbserver/SimEncryptVaultProxy.actor.cpp
index 6f3e8cfea30..93da4433ada 100644
--- a/fdbserver/SimEncryptVaultProxy.actor.cpp
+++ b/fdbserver/SimEncryptVaultProxy.actor.cpp
@@ -146,8 +146,12 @@ TEST_CASE("fdbserver/SimEncryptVaultProxy") {
 	state uint32_t maxEncryptKeys = 64;
 
 	loop choose {
-		when(wait(simEncryptVaultProxyCore(inf, maxEncryptKeys))) { throw internal_error(); }
-		when(wait(testRunWorkload(inf, maxEncryptKeys))) { break; }
+		when(wait(simEncryptVaultProxyCore(inf, maxEncryptKeys))) {
+			throw internal_error();
+		}
+		when(wait(testRunWorkload(inf, maxEncryptKeys))) {
+			break;
+		}
 	}
 	return Void();
 }
\ No newline at end of file
diff --git a/fdbserver/SkipList.cpp b/fdbserver/SkipList.cpp
index 180dc13dfa7..517eca30a8b 100644
--- a/fdbserver/SkipList.cpp
+++ b/fdbserver/SkipList.cpp
@@ -295,7 +295,7 @@ class SkipList : NonCopyable {
 				FastAllocator<128>::release(this);
 				INSTRUMENT_RELEASE("SkipListNode128");
 			} else {
-				delete[](char*) this;
+				delete[] (char*)this;
 				INSTRUMENT_RELEASE("SkipListNodeLarge");
 			}
 		}
diff --git a/fdbserver/StorageCache.actor.cpp b/fdbserver/StorageCache.actor.cpp
index 8c71eca465c..ca2090ee8e1 100644
--- a/fdbserver/StorageCache.actor.cpp
+++ b/fdbserver/StorageCache.actor.cpp
@@ -452,7 +452,9 @@ ACTOR Future<Version> waitForVersionNoTooOld(StorageCacheData* data, Version ver
 	if (version <= data->version.get())
 		return version;
 	choose {
-		when(wait(data->version.whenAtLeast(version))) { return version; }
+		when(wait(data->version.whenAtLeast(version))) {
+			return version;
+		}
 		when(wait(delay(SERVER_KNOBS->FUTURE_VERSION_DELAY))) {
 			if (deterministicRandom()->random01() < 0.001)
 				TraceEvent(SevWarn, "CacheServerFutureVersion1000x", data->thisServerID)
@@ -1840,7 +1842,9 @@ ACTOR Future<Void> pullAsyncData(StorageCacheData* data) {
 
 	loop {
 		loop choose {
-			when(wait(cursor ? cursor->getMore(TaskPriority::TLogCommit) : Never())) { break; }
+			when(wait(cursor ? cursor->getMore(TaskPriority::TLogCommit) : Never())) {
+				break;
+			}
 			when(wait(dbInfoChange)) {
 				dbInfoChange = data->db->onChange();
 				if (data->db->get().recoveryState >= RecoveryState::ACCEPTING_COMMITS) {
@@ -2207,13 +2211,21 @@ ACTOR Future<Void> storageCacheServer(StorageServerInterface ssi,
 				// actors.add(self->readGuard(req , getValueQ));
 				actors.add(getValueQ(&self, req));
 			}
-			when(WatchValueRequest req = waitNext(ssi.watchValue.getFuture())) { ASSERT(false); }
-			when(GetKeyRequest req = waitNext(ssi.getKey.getFuture())) { actors.add(getKey(&self, req)); }
+			when(WatchValueRequest req = waitNext(ssi.watchValue.getFuture())) {
+				ASSERT(false);
+			}
+			when(GetKeyRequest req = waitNext(ssi.getKey.getFuture())) {
+				actors.add(getKey(&self, req));
+			}
 			when(GetKeyValuesRequest req = waitNext(ssi.getKeyValues.getFuture())) {
 				actors.add(getKeyValues(&self, req));
 			}
-			when(GetShardStateRequest req = waitNext(ssi.getShardState.getFuture())) { ASSERT(false); }
-			when(StorageQueuingMetricsRequest req = waitNext(ssi.getQueuingMetrics.getFuture())) { ASSERT(false); }
+			when(GetShardStateRequest req = waitNext(ssi.getShardState.getFuture())) {
+				ASSERT(false);
+			}
+			when(StorageQueuingMetricsRequest req = waitNext(ssi.getQueuingMetrics.getFuture())) {
+				ASSERT(false);
+			}
 			// when( ReplyPromise<Version> reply = waitNext(ssi.getVersion.getFuture()) ) {
 			//	ASSERT(false);
 			//}
@@ -2221,21 +2233,39 @@ ACTOR Future<Void> storageCacheServer(StorageServerInterface ssi,
 				ASSERT(false);
 			}
 
-			when(GetMappedKeyValuesRequest req = waitNext(ssi.getMappedKeyValues.getFuture())) { ASSERT(false); }
-			when(WaitMetricsRequest req = waitNext(ssi.waitMetrics.getFuture())) { ASSERT(false); }
-			when(SplitMetricsRequest req = waitNext(ssi.splitMetrics.getFuture())) { ASSERT(false); }
-			when(GetStorageMetricsRequest req = waitNext(ssi.getStorageMetrics.getFuture())) { ASSERT(false); }
-			when(ReadHotSubRangeRequest req = waitNext(ssi.getReadHotRanges.getFuture())) { ASSERT(false); }
-			when(SplitRangeRequest req = waitNext(ssi.getRangeSplitPoints.getFuture())) { ASSERT(false); }
-			when(GetKeyValuesStreamRequest req = waitNext(ssi.getKeyValuesStream.getFuture())) { ASSERT(false); }
-			when(ChangeFeedStreamRequest req = waitNext(ssi.changeFeedStream.getFuture())) { ASSERT(false); }
+			when(GetMappedKeyValuesRequest req = waitNext(ssi.getMappedKeyValues.getFuture())) {
+				ASSERT(false);
+			}
+			when(WaitMetricsRequest req = waitNext(ssi.waitMetrics.getFuture())) {
+				ASSERT(false);
+			}
+			when(SplitMetricsRequest req = waitNext(ssi.splitMetrics.getFuture())) {
+				ASSERT(false);
+			}
+			when(GetStorageMetricsRequest req = waitNext(ssi.getStorageMetrics.getFuture())) {
+				ASSERT(false);
+			}
+			when(ReadHotSubRangeRequest req = waitNext(ssi.getReadHotRanges.getFuture())) {
+				ASSERT(false);
+			}
+			when(SplitRangeRequest req = waitNext(ssi.getRangeSplitPoints.getFuture())) {
+				ASSERT(false);
+			}
+			when(GetKeyValuesStreamRequest req = waitNext(ssi.getKeyValuesStream.getFuture())) {
+				ASSERT(false);
+			}
+			when(ChangeFeedStreamRequest req = waitNext(ssi.changeFeedStream.getFuture())) {
+				ASSERT(false);
+			}
 			when(OverlappingChangeFeedsRequest req = waitNext(ssi.overlappingChangeFeeds.getFuture())) {
 				// Simulate endpoint not found so that the requester will try another endpoint
 				// This is a workaround to the fact that storage servers do not have an easy way to enforce this
 				// request goes only to other storage servers, and in simulation we manage to trigger this behavior
 				req.reply.sendError(broken_promise());
 			}
-			when(ChangeFeedPopRequest req = waitNext(ssi.changeFeedPop.getFuture())) { ASSERT(false); }
+			when(ChangeFeedPopRequest req = waitNext(ssi.changeFeedPop.getFuture())) {
+				ASSERT(false);
+			}
 			when(ChangeFeedVersionUpdateRequest req = waitNext(ssi.changeFeedVersionUpdate.getFuture())) {
 				ASSERT(false);
 			}
diff --git a/fdbserver/TCInfo.actor.cpp b/fdbserver/TCInfo.actor.cpp
index 67a4e0a96c2..341ee94f9d8 100644
--- a/fdbserver/TCInfo.actor.cpp
+++ b/fdbserver/TCInfo.actor.cpp
@@ -51,7 +51,9 @@ class TCServerInfoImpl {
 					interfaceChanged = server->onInterfaceChanged;
 					resetRequest = Void();
 				}
-				when(wait(serverRemoved)) { return Void(); }
+				when(wait(serverRemoved)) {
+					return Void();
+				}
 				when(wait(resetRequest)) { // To prevent a tight spin loop
 					if (IFailureMonitor::failureMonitor().getState(ssi.getStorageMetrics.getEndpoint()).isFailed()) {
 						resetRequest = IFailureMonitor::failureMonitor().onStateEqual(
diff --git a/fdbserver/TLogServer.actor.cpp b/fdbserver/TLogServer.actor.cpp
index 3d012f118e7..b31a266d824 100644
--- a/fdbserver/TLogServer.actor.cpp
+++ b/fdbserver/TLogServer.actor.cpp
@@ -1449,7 +1449,9 @@ ACTOR Future<Void> updateStorage(TLogData* self) {
 ACTOR Future<Void> updateStorageLoop(TLogData* self) {
 	wait(delay(0, TaskPriority::UpdateStorage));
 
-	loop { wait(updateStorage(self)); }
+	loop {
+		wait(updateStorage(self));
+	}
 }
 
 void commitMessages(TLogData* self,
@@ -1610,7 +1612,9 @@ ACTOR Future<Void> waitForMessagesForTag(Reference<LogData> self, Tag reqTag, Ve
 			// we want the caller to finish first, otherwise the data structure it is building might not be complete
 			wait(delay(0.0));
 		}
-		when(wait(delay(timeout))) { self->blockingPeekTimeouts += 1; }
+		when(wait(delay(timeout))) {
+			self->blockingPeekTimeouts += 1;
+		}
 	}
 	return Void();
 }
@@ -2827,7 +2831,9 @@ ACTOR Future<Void> pullAsyncData(TLogData* self,
 	while (!endVersion.present() || logData->version.get() < endVersion.get()) {
 		loop {
 			choose {
-				when(wait(r ? r->getMore(TaskPriority::TLogCommit) : Never())) { break; }
+				when(wait(r ? r->getMore(TaskPriority::TLogCommit) : Never())) {
+					break;
+				}
 				when(wait(dbInfoChange)) {
 					if (logData->logSystem->get()) {
 						r = logData->logSystem->get()->peek(logData->logId, tagAt, endVersion, tags, true);
@@ -3298,7 +3304,9 @@ ACTOR Future<Void> restorePersistentState(TLogData* self,
 
 								choose {
 									when(wait(updateStorage(self))) {}
-									when(wait(allRemoved)) { throw worker_removed(); }
+									when(wait(allRemoved)) {
+										throw worker_removed();
+									}
 								}
 							}
 						} else {
@@ -3309,7 +3317,9 @@ ACTOR Future<Void> restorePersistentState(TLogData* self,
 						}
 					}
 				}
-				when(wait(allRemoved)) { throw worker_removed(); }
+				when(wait(allRemoved)) {
+					throw worker_removed();
+				}
 			}
 		}
 	} catch (Error& e) {
@@ -3653,7 +3663,9 @@ ACTOR Future<Void> tLog(IKeyValueStore* persistentData,
 							forwardPromise(req.reply, self.tlogCache.get(req.recruitmentID));
 						}
 					}
-					when(wait(error)) { throw internal_error(); }
+					when(wait(error)) {
+						throw internal_error();
+					}
 					when(wait(activeSharedChange)) {
 						if (activeSharedTLog->get() == tlogId) {
 							TraceEvent("SharedTLogNowActive", self.dbgid).detail("NowActive", activeSharedTLog->get());
diff --git a/fdbserver/VFSAsync.cpp b/fdbserver/VFSAsync.cpp
index ac00876bd1d..3dd7eed68a7 100644
--- a/fdbserver/VFSAsync.cpp
+++ b/fdbserver/VFSAsync.cpp
@@ -168,7 +168,7 @@ static int asyncReadZeroCopy(sqlite3_file* pFile, void** data, int iAmt, sqlite_
 }
 static int asyncReleaseZeroCopy(sqlite3_file* pFile, void* data, int iAmt, sqlite_int64 iOfst) {
 	// printf("-asyncReleaseRef %p +%lld %d <= %p\n", pFile, iOfst, iAmt, data);
-	delete[](char*) data;
+	delete[] (char*)data;
 	return SQLITE_OK;
 }
 #endif
@@ -299,7 +299,7 @@ struct SharedMemoryInfo { // for a file
 	}
 	void cleanup() {
 		for (int i = 0; i < regions.size(); i++)
-			delete[](uint8_t*) regions[i];
+			delete[] (uint8_t*)regions[i];
 		table.erase(filename);
 	}
 
diff --git a/fdbserver/VersionedBTree.actor.cpp b/fdbserver/VersionedBTree.actor.cpp
index 63c8dec6ea0..c4f942e5275 100644
--- a/fdbserver/VersionedBTree.actor.cpp
+++ b/fdbserver/VersionedBTree.actor.cpp
@@ -2425,7 +2425,9 @@ class DWALPager final : public IPager2 {
 							self->remappedPages[r.originalPageID][r.version] = r.newPageID;
 						}
 					}
-					when(wait(remapRecoverActor)) { remapRecoverActor = Never(); }
+					when(wait(remapRecoverActor)) {
+						remapRecoverActor = Never();
+					}
 				}
 			} catch (Error& e) {
 				if (e.code() != error_code_end_of_stream) {
@@ -10387,7 +10389,9 @@ TEST_CASE(":/redwood/performance/extentQueue") {
 				if (entriesRead == m_extentQueue.numEntries)
 					break;
 			}
-			when(wait(queueRecoverActor)) { queueRecoverActor = Never(); }
+			when(wait(queueRecoverActor)) {
+				queueRecoverActor = Never();
+			}
 		}
 	} catch (Error& e) {
 		if (e.code() != error_code_end_of_stream) {
diff --git a/fdbserver/WorkerInterface.actor.h b/fdbserver/WorkerInterface.actor.h
index 47b9e7f926e..23b71a0e3cd 100644
--- a/fdbserver/WorkerInterface.actor.h
+++ b/fdbserver/WorkerInterface.actor.h
@@ -1209,7 +1209,9 @@ Future<T> ioTimeoutError(Future<T> what, double time) {
 	}
 	Future<Void> end = lowPriorityDelay(time);
 	choose {
-		when(T t = wait(what)) { return t; }
+		when(T t = wait(what)) {
+			return t;
+		}
 		when(wait(end)) {
 			Error err = io_timeout();
 			if (g_network->isSimulated() && !g_simulator.getCurrentProcess()->isReliable()) {
@@ -1237,7 +1239,9 @@ Future<T> ioDegradedOrTimeoutError(Future<T> what,
 	if (degradedTime < errTime) {
 		Future<Void> degradedEnd = lowPriorityDelay(degradedTime);
 		choose {
-			when(T t = wait(what)) { return t; }
+			when(T t = wait(what)) {
+				return t;
+			}
 			when(wait(degradedEnd)) {
 				TEST(true); // TLog degraded
 				TraceEvent(SevWarnAlways, "IoDegraded").log();
@@ -1248,7 +1252,9 @@ Future<T> ioDegradedOrTimeoutError(Future<T> what,
 
 	Future<Void> end = lowPriorityDelay(errTime - degradedTime);
 	choose {
-		when(T t = wait(what)) { return t; }
+		when(T t = wait(what)) {
+			return t;
+		}
 		when(wait(end)) {
 			Error err = io_timeout();
 			if (g_network->isSimulated() && !g_simulator.getCurrentProcess()->isReliable()) {
diff --git a/fdbserver/art.h b/fdbserver/art.h
index b318ac2547f..27dbbbcd50c 100644
--- a/fdbserver/art.h
+++ b/fdbserver/art.h
@@ -52,7 +52,7 @@ struct art_tree {
 #define ART_PREV -1
 #define ART_NEITHER 0
 
-	//#define ART_IS_LEAF(x) ( (*((ART_NODE_TYPE*)x) == ART_LEAF))
+	// #define ART_IS_LEAF(x) ( (*((ART_NODE_TYPE*)x) == ART_LEAF))
 	template <class T>
 	static inline bool ART_IS_LEAF(T const& x) {
 		return *((ART_NODE_TYPE*)x) == ART_LEAF;
diff --git a/fdbserver/sqlite/hash.h b/fdbserver/sqlite/hash.h
index df98d432c5c..937c4abc97c 100644
--- a/fdbserver/sqlite/hash.h
+++ b/fdbserver/sqlite/hash.h
@@ -47,7 +47,7 @@ struct Hash {
 	struct _ht { /* the hash table */
 		int count; /* Number of entries with this hash */
 		HashElem* chain; /* Pointer to first entry with this hash */
-	} * ht;
+	}* ht;
 };
 
 /* Each element in the hash table is an instance of the following
diff --git a/fdbserver/sqlite/sqlite3.h b/fdbserver/sqlite/sqlite3.h
index 7c66883070b..93f9b0f42bd 100644
--- a/fdbserver/sqlite/sqlite3.h
+++ b/fdbserver/sqlite/sqlite3.h
@@ -4623,17 +4623,17 @@ struct sqlite3_index_info {
 		unsigned char op; /* Constraint operator */
 		unsigned char usable; /* True if this constraint is usable */
 		int iTermOffset; /* Used internally - xBestIndex should ignore */
-	} * aConstraint; /* Table of WHERE clause constraints */
+	}* aConstraint; /* Table of WHERE clause constraints */
 	int nOrderBy; /* Number of terms in the ORDER BY clause */
 	struct sqlite3_index_orderby {
 		int iColumn; /* Column number */
 		unsigned char desc; /* True for DESC.  False for ASC. */
-	} * aOrderBy; /* The ORDER BY clause */
+	}* aOrderBy; /* The ORDER BY clause */
 	/* Outputs */
 	struct sqlite3_index_constraint_usage {
 		int argvIndex; /* if >0, constraint is part of argv to xFilter */
 		unsigned char omit; /* Do not code a test for this constraint */
-	} * aConstraintUsage;
+	}* aConstraintUsage;
 	int idxNum; /* Number used to identify the index */
 	char* idxStr; /* String, possibly obtained from sqlite3_malloc */
 	int needToFreeIdxStr; /* Free idxStr using sqlite3_free() if true */
diff --git a/fdbserver/storageserver.actor.cpp b/fdbserver/storageserver.actor.cpp
index ca78cbb4824..13d832c6be3 100644
--- a/fdbserver/storageserver.actor.cpp
+++ b/fdbserver/storageserver.actor.cpp
@@ -1567,7 +1567,9 @@ ACTOR Future<Version> waitForVersionNoTooOld(StorageServer* data, Version versio
 	if (version <= data->version.get())
 		return version;
 	choose {
-		when(wait(data->version.whenAtLeast(version))) { return version; }
+		when(wait(data->version.whenAtLeast(version))) {
+			return version;
+		}
 		when(wait(delay(SERVER_KNOBS->FUTURE_VERSION_DELAY))) {
 			if (deterministicRandom()->random01() < 0.001)
 				TraceEvent(SevWarn, "ShardServerFutureVersion1000x", data->thisServerID)
@@ -8464,7 +8466,9 @@ ACTOR Future<Void> waitMetrics(StorageServerMetrics* self, WaitMetricsRequest re
 
 						  }*/
 					}
-					when(wait(timeout)) { timedout = true; }
+					when(wait(timeout)) {
+						timedout = true;
+					}
 				}
 			} catch (Error& e) {
 				if (e.code() == error_code_actor_cancelled)
diff --git a/fdbserver/tester.actor.cpp b/fdbserver/tester.actor.cpp
index 10aefd4e96d..f60ae66b97b 100644
--- a/fdbserver/tester.actor.cpp
+++ b/fdbserver/tester.actor.cpp
@@ -1792,7 +1792,9 @@ ACTOR Future<Void> runTests(Reference<IClusterConnectionRecord> connRecord,
 	}
 
 	choose {
-		when(wait(tests)) { return Void(); }
+		when(wait(tests)) {
+			return Void();
+		}
 		when(wait(quorum(actors, 1))) {
 			ASSERT(false);
 			throw internal_error();
diff --git a/fdbserver/worker.actor.cpp b/fdbserver/worker.actor.cpp
index 491ca451ac8..7f7c39b6abb 100644
--- a/fdbserver/worker.actor.cpp
+++ b/fdbserver/worker.actor.cpp
@@ -640,14 +640,30 @@ ACTOR Future<Void> registrationClient(
 					TraceEvent(SevWarn, "WorkerRegisterTimeout").detail("WaitTime", now() - startTime);
 				}
 			}
-			when(wait(ccInterface->onChange())) { break; }
-			when(wait(ddInterf->onChange())) { break; }
-			when(wait(rkInterf->onChange())) { break; }
-			when(wait(bmInterf->onChange())) { break; }
-			when(wait(ekpInterf->onChange())) { break; }
-			when(wait(degraded->onChange())) { break; }
-			when(wait(FlowTransport::transport().onIncompatibleChanged())) { break; }
-			when(wait(issues->onChange())) { break; }
+			when(wait(ccInterface->onChange())) {
+				break;
+			}
+			when(wait(ddInterf->onChange())) {
+				break;
+			}
+			when(wait(rkInterf->onChange())) {
+				break;
+			}
+			when(wait(bmInterf->onChange())) {
+				break;
+			}
+			when(wait(ekpInterf->onChange())) {
+				break;
+			}
+			when(wait(degraded->onChange())) {
+				break;
+			}
+			when(wait(FlowTransport::transport().onIncompatibleChanged())) {
+				break;
+			}
+			when(wait(issues->onChange())) {
+				break;
+			}
 		}
 	}
 }
diff --git a/fdbserver/workloads/AtomicOps.actor.cpp b/fdbserver/workloads/AtomicOps.actor.cpp
index 2cc2f1020fc..25c44732273 100644
--- a/fdbserver/workloads/AtomicOps.actor.cpp
+++ b/fdbserver/workloads/AtomicOps.actor.cpp
@@ -26,7 +26,7 @@
 #include "fdbserver/workloads/workloads.actor.h"
 #include "flow/actorcompiler.h" // This must be the last #include.
 
-//#define SevAtomicOpDebug SevInfo
+// #define SevAtomicOpDebug SevInfo
 #define SevAtomicOpDebug SevVerbose
 
 struct AtomicOpsWorkload : TestWorkload {
diff --git a/fdbserver/workloads/ConsistencyCheck.actor.cpp b/fdbserver/workloads/ConsistencyCheck.actor.cpp
index 65a5592b450..537bac2b868 100644
--- a/fdbserver/workloads/ConsistencyCheck.actor.cpp
+++ b/fdbserver/workloads/ConsistencyCheck.actor.cpp
@@ -40,7 +40,7 @@
 
 #include "flow/actorcompiler.h" // This must be the last #include.
 
-//#define SevCCheckInfo SevVerbose
+// #define SevCCheckInfo SevVerbose
 #define SevCCheckInfo SevInfo
 
 struct ConsistencyCheckWorkload : TestWorkload {
diff --git a/fdbserver/workloads/IndexScan.actor.cpp b/fdbserver/workloads/IndexScan.actor.cpp
index ca844c980c4..62e41422b6f 100644
--- a/fdbserver/workloads/IndexScan.actor.cpp
+++ b/fdbserver/workloads/IndexScan.actor.cpp
@@ -93,7 +93,9 @@ struct IndexScanWorkload : KVWorkload {
 	ACTOR static Future<Void> serialScans(Database cx, IndexScanWorkload* self) {
 		state double start = now();
 		try {
-			loop { wait(scanDatabase(cx, self)); }
+			loop {
+				wait(scanDatabase(cx, self));
+			}
 		} catch (...) {
 			self->totalTimeFetching = now() - start;
 			throw;
diff --git a/fdbserver/workloads/KVStoreTest.actor.cpp b/fdbserver/workloads/KVStoreTest.actor.cpp
index 177c95da6d8..faecbc692c8 100644
--- a/fdbserver/workloads/KVStoreTest.actor.cpp
+++ b/fdbserver/workloads/KVStoreTest.actor.cpp
@@ -401,7 +401,9 @@ ACTOR Future<Void> testKVStore(KVStoreTestWorkload* workload) {
 	try {
 		choose {
 			when(wait(main)) {}
-			when(wait(test.store->getError())) { ASSERT(false); }
+			when(wait(test.store->getError())) {
+				ASSERT(false);
+			}
 		}
 	} catch (Error& e) {
 		err = e;
diff --git a/fdbserver/workloads/KillRegion.actor.cpp b/fdbserver/workloads/KillRegion.actor.cpp
index 08b3f629466..e1425899182 100644
--- a/fdbserver/workloads/KillRegion.actor.cpp
+++ b/fdbserver/workloads/KillRegion.actor.cpp
@@ -118,7 +118,9 @@ struct KillRegionWorkload : TestWorkload {
 				wait(success(ManagementAPI::changeConfig(
 				    cx.getReference(), g_simulator.disablePrimary + " repopulate_anti_quorum=1", true)));
 				choose {
-					when(wait(waitForStorageRecovered(self))) { break; }
+					when(wait(waitForStorageRecovered(self))) {
+						break;
+					}
 					when(wait(delay(300.0))) {}
 				}
 			}
diff --git a/fdbserver/workloads/MiniCycle.actor.cpp b/fdbserver/workloads/MiniCycle.actor.cpp
index b071902a8cf..82d93d5240d 100644
--- a/fdbserver/workloads/MiniCycle.actor.cpp
+++ b/fdbserver/workloads/MiniCycle.actor.cpp
@@ -90,7 +90,9 @@ struct MiniCycleWorkload : TestWorkload {
 					if (!ok)
 						return false;
 				}
-				when(wait(end)) { break; }
+				when(wait(end)) {
+					break;
+				}
 			}
 		}
 
diff --git a/fdbserver/workloads/UDPWorkload.actor.cpp b/fdbserver/workloads/UDPWorkload.actor.cpp
index d9ea607d938..97a59ff49ea 100644
--- a/fdbserver/workloads/UDPWorkload.actor.cpp
+++ b/fdbserver/workloads/UDPWorkload.actor.cpp
@@ -184,7 +184,9 @@ struct UDPWorkload : TestWorkload {
 					finished = delay(1.0);
 					done = Never();
 				}
-				when(wait(finished)) { return Void(); }
+				when(wait(finished)) {
+					return Void();
+				}
 			}
 		}
 	}
@@ -198,7 +200,9 @@ struct UDPWorkload : TestWorkload {
 		loop {
 			choose {
 				when(wait(delay(0.1))) {}
-				when(wait(actors.getResult())) { UNSTOPPABLE_ASSERT(false); }
+				when(wait(actors.getResult())) {
+					UNSTOPPABLE_ASSERT(false);
+				}
 			}
 			if (!socket.get().isValid() || deterministicRandom()->random01() < 0.05) {
 				peer = deterministicRandom()->randomChoice(*remotes);
diff --git a/flow/ActorCollection.actor.cpp b/flow/ActorCollection.actor.cpp
index e19419b09d4..3958fa24c33 100644
--- a/flow/ActorCollection.actor.cpp
+++ b/flow/ActorCollection.actor.cpp
@@ -61,7 +61,9 @@ ACTOR Future<Void> actorCollection(FutureStream<Future<Void>> addActor,
 			}
 			tag_streamHelper.erase(t);
 		}
-		when(Error e = waitNext(errors.getFuture())) { throw e; }
+		when(Error e = waitNext(errors.getFuture())) {
+			throw e;
+		}
 	}
 }
 
diff --git a/flow/Arena.h b/flow/Arena.h
index 5a3c66aec9e..fd7af76d56f 100644
--- a/flow/Arena.h
+++ b/flow/Arena.h
@@ -355,7 +355,7 @@ struct union_like_traits<Optional<T>> : std::true_type {
 	}
 };
 
-//#define STANDALONE_ALWAYS_COPY
+// #define STANDALONE_ALWAYS_COPY
 
 template <class T>
 class Standalone : private Arena, public T {
@@ -437,7 +437,7 @@ class StringRef {
 public:
 	constexpr static FileIdentifier file_identifier = 13300811;
 	StringRef() : data(0), length(0) {}
-	StringRef(Arena& p, const StringRef& toCopy) : data(new (p) uint8_t[toCopy.size()]), length(toCopy.size()) {
+	StringRef(Arena& p, const StringRef& toCopy) : data(new(p) uint8_t[toCopy.size()]), length(toCopy.size()) {
 		if (length > 0) {
 			memcpy((void*)data, toCopy.data, length);
 		}
@@ -448,7 +448,7 @@ class StringRef {
 		if (length)
 			memcpy((void*)data, &toCopy[0], length);
 	}
-	StringRef(Arena& p, const uint8_t* toCopy, int length) : data(new (p) uint8_t[length]), length(length) {
+	StringRef(Arena& p, const uint8_t* toCopy, int length) : data(new(p) uint8_t[length]), length(length) {
 		if (length > 0) {
 			memcpy((void*)data, toCopy, length);
 		}
@@ -939,7 +939,7 @@ class VectorRef : public ComposedIdentifier<T, 3>, public VectorRefPreserializer
 	// Arena constructor for non-Ref types, identified by !flow_ref
 	template <class T2 = T, VecSerStrategy S>
 	VectorRef(Arena& p, const VectorRef<T, S>& toCopy, typename std::enable_if<!flow_ref<T2>::value, int>::type = 0)
-	  : VPS(toCopy), data((T*)new (p) uint8_t[sizeof(T) * toCopy.size()]), m_size(toCopy.size()),
+	  : VPS(toCopy), data((T*)new(p) uint8_t[sizeof(T) * toCopy.size()]), m_size(toCopy.size()),
 	    m_capacity(toCopy.size()) {
 		if (m_size > 0) {
 			std::copy(toCopy.data, toCopy.data + m_size, data);
@@ -949,7 +949,7 @@ class VectorRef : public ComposedIdentifier<T, 3>, public VectorRefPreserializer
 	// Arena constructor for Ref types, which must have an Arena constructor
 	template <class T2 = T, VecSerStrategy S>
 	VectorRef(Arena& p, const VectorRef<T, S>& toCopy, typename std::enable_if<flow_ref<T2>::value, int>::type = 0)
-	  : VPS(), data((T*)new (p) uint8_t[sizeof(T) * toCopy.size()]), m_size(toCopy.size()), m_capacity(toCopy.size()) {
+	  : VPS(), data((T*)new(p) uint8_t[sizeof(T) * toCopy.size()]), m_size(toCopy.size()), m_capacity(toCopy.size()) {
 		for (int i = 0; i < m_size; i++) {
 			auto ptr = new (&data[i]) T(p, toCopy[i]);
 			VPS::add(*ptr);
diff --git a/flow/Error.h b/flow/Error.h
index 1ebb8620ef3..3cd72d039d8 100644
--- a/flow/Error.h
+++ b/flow/Error.h
@@ -107,7 +107,7 @@ extern Error internal_error_impl(const char* a_nm,
 #define internal_error_msg(msg) internal_error_impl(msg, __FILE__, __LINE__)
 
 extern bool isAssertDisabled(int line);
-//#define ASSERT( condition ) ((void)0)
+// #define ASSERT( condition ) ((void)0)
 #define ASSERT(condition)                                                                                              \
 	do {                                                                                                               \
 		if (!((condition) || isAssertDisabled(__LINE__))) {                                                            \
diff --git a/flow/FastAlloc.cpp b/flow/FastAlloc.cpp
index 262d1b017c7..03123a51e88 100644
--- a/flow/FastAlloc.cpp
+++ b/flow/FastAlloc.cpp
@@ -32,11 +32,11 @@
 #include <cstdint>
 #include <unordered_map>
 
-//#ifdef WIN32
-//#include <windows.h>
-//#undef min
-//#undef max
-//#endif
+// #ifdef WIN32
+// #include <windows.h>
+// #undef min
+// #undef max
+// #endif
 
 #ifdef __linux__
 #include <sys/mman.h>
diff --git a/flow/FastAlloc.h b/flow/FastAlloc.h
index 779bd94cd06..3992e590e89 100644
--- a/flow/FastAlloc.h
+++ b/flow/FastAlloc.h
@@ -28,11 +28,11 @@
 
 // ALLOC_INSTRUMENTATION_STDOUT enables non-sampled logging of all allocations and deallocations to stdout to be
 // processed by tools/alloc_instrumentation.py
-//#define ALLOC_INSTRUMENTATION_STDOUT ENABLED(NOT_IN_CLEAN)
+// #define ALLOC_INSTRUMENTATION_STDOUT ENABLED(NOT_IN_CLEAN)
 
-//#define ALLOC_INSTRUMENTATION ENABLED(NOT_IN_CLEAN)
-// The form "(1==1)" in this context is used to satisfy both clang and vc++ with a single syntax.  Clang rejects "1" and
-// vc++ rejects "true".
+// #define ALLOC_INSTRUMENTATION ENABLED(NOT_IN_CLEAN)
+//  The form "(1==1)" in this context is used to satisfy both clang and vc++ with a single syntax.  Clang rejects "1"
+//  and vc++ rejects "true".
 #define FASTALLOC_THREAD_SAFE (FLOW_THREAD_SAFE || (1 == 1))
 
 #if VALGRIND
@@ -265,7 +265,7 @@ inline void freeFast(int size, void* ptr) {
 		return FastAllocator<128>::release(ptr);
 	if (size <= 256)
 		return FastAllocator<256>::release(ptr);
-	delete[](uint8_t*) ptr;
+	delete[] (uint8_t*)ptr;
 }
 
 // Allocate a block of memory aligned to 4096 bytes. Size must be a multiple of
diff --git a/flow/FileTraceLogWriter.cpp b/flow/FileTraceLogWriter.cpp
index 998f915f779..b9617cb00eb 100644
--- a/flow/FileTraceLogWriter.cpp
+++ b/flow/FileTraceLogWriter.cpp
@@ -31,9 +31,9 @@
 #define TRACEFILE_FLAGS O_WRONLY | O_CREAT | O_EXCL | O_CLOEXEC
 #define TRACEFILE_MODE 0664
 #elif defined(_WIN32)
-//#include <windows.h>
-//#undef max
-//#undef min
+// #include <windows.h>
+// #undef max
+// #undef min
 #include <io.h>
 #include <stdio.h>
 #include <sys/stat.h>
diff --git a/flow/Hash3.c b/flow/Hash3.c
index 103a205ea62..1b694d81bc9 100644
--- a/flow/Hash3.c
+++ b/flow/Hash3.c
@@ -33,7 +33,7 @@ mixing with 12*3 instructions on 3 integers than you can with 3 instructions
 on 1 byte), but shoehorning those bytes into integers efficiently is messy.
 -------------------------------------------------------------------------------
 */
-//#define SELF_TEST ENABLED(NOT_IN_CLEAN)
+// #define SELF_TEST ENABLED(NOT_IN_CLEAN)
 
 #include <stdio.h> /* defines printf for tests */
 #include <time.h> /* defines time_t for timings in the test */
diff --git a/flow/Net2.actor.cpp b/flow/Net2.actor.cpp
index b503c4a45da..1f932421e99 100644
--- a/flow/Net2.actor.cpp
+++ b/flow/Net2.actor.cpp
@@ -953,8 +953,12 @@ class SSLConnection final : public IConnection, ReferenceCounted<SSLConnection>
 		doAcceptHandshake(self, connected);
 		try {
 			choose {
-				when(wait(connected.getFuture())) { return Void(); }
-				when(wait(delay(FLOW_KNOBS->CONNECTION_MONITOR_TIMEOUT))) { throw connection_failed(); }
+				when(wait(connected.getFuture())) {
+					return Void();
+				}
+				when(wait(delay(FLOW_KNOBS->CONNECTION_MONITOR_TIMEOUT))) {
+					throw connection_failed();
+				}
 			}
 		} catch (Error& e) {
 			if (e.code() != error_code_actor_cancelled) {
@@ -1009,8 +1013,12 @@ class SSLConnection final : public IConnection, ReferenceCounted<SSLConnection>
 		doConnectHandshake(self, connected);
 		try {
 			choose {
-				when(wait(connected.getFuture())) { return Void(); }
-				when(wait(delay(FLOW_KNOBS->CONNECTION_MONITOR_TIMEOUT))) { throw connection_failed(); }
+				when(wait(connected.getFuture())) {
+					return Void();
+				}
+				when(wait(delay(FLOW_KNOBS->CONNECTION_MONITOR_TIMEOUT))) {
+					throw connection_failed();
+				}
 			}
 		} catch (Error& e) {
 			// Either the connection failed, or was cancelled by the caller
diff --git a/flow/flat_buffers.h b/flow/flat_buffers.h
index 895c7a2969a..53a60b3efba 100644
--- a/flow/flat_buffers.h
+++ b/flow/flat_buffers.h
@@ -373,7 +373,7 @@ template <class T>
 constexpr bool is_vector_like = vector_like_traits<T>::value;
 
 template <class T>
-constexpr bool is_vector_of_union_like = is_vector_like<T>&& is_union_like<typename vector_like_traits<T>::value_type>;
+constexpr bool is_vector_of_union_like = is_vector_like<T> && is_union_like<typename vector_like_traits<T>::value_type>;
 
 template <class T>
 constexpr bool is_struct_like = struct_like_traits<T>::value;
@@ -470,8 +470,8 @@ template <class T>
 constexpr int fb_size = is_struct_like<T> ? struct_size(typename struct_like_traits<T>::types{}) : fb_scalar_size<T>;
 
 template <class T>
-constexpr int fb_align = is_struct_like<T> ? align_helper(typename struct_like_traits<T>::types{})
-                                           : AlignToPowerOfTwo(fb_scalar_size<T>);
+constexpr int fb_align =
+    is_struct_like<T> ? align_helper(typename struct_like_traits<T>::types{}) : AlignToPowerOfTwo(fb_scalar_size<T>);
 
 template <class T>
 struct _SizeOf {
diff --git a/flow/flow.h b/flow/flow.h
index a3c156c9d11..0cbba3bf452 100644
--- a/flow/flow.h
+++ b/flow/flow.h
@@ -1149,7 +1149,7 @@ auto const& getReplyPromiseStream(Request const& r) {
 // Neither of these implementations of REPLY_TYPE() works on both MSVC and g++, so...
 #ifdef __GNUG__
 #define REPLY_TYPE(RequestType) decltype(getReplyPromise(std::declval<RequestType>()).getFuture().getValue())
-//#define REPLY_TYPE(RequestType) decltype( getReplyFuture( std::declval<RequestType>() ).getValue() )
+// #define REPLY_TYPE(RequestType) decltype( getReplyFuture( std::declval<RequestType>() ).getValue() )
 #else
 template <class T>
 struct ReplyType {
diff --git a/flow/genericactors.actor.cpp b/flow/genericactors.actor.cpp
index 6bb1e3fd8d7..c90a8a02962 100644
--- a/flow/genericactors.actor.cpp
+++ b/flow/genericactors.actor.cpp
@@ -58,7 +58,9 @@ ACTOR Future<Void> timeoutWarningCollector(FutureStream<Void> input, double logD
 	state uint64_t counter = 0;
 	state Future<Void> end = delay(logDelay);
 	loop choose {
-		when(waitNext(input)) { counter++; }
+		when(waitNext(input)) {
+			counter++;
+		}
 		when(wait(end)) {
 			if (counter)
 				TraceEvent(SevWarn, context, id).detail("LateProcessCount", counter).detail("LoggingDelay", logDelay);
@@ -79,8 +81,12 @@ ACTOR Future<bool> quorumEqualsTrue(std::vector<Future<bool>> futures, int requi
 	}
 
 	choose {
-		when(wait(quorum(true_futures, required))) { return true; }
-		when(wait(quorum(false_futures, futures.size() - required + 1))) { return false; }
+		when(wait(quorum(true_futures, required))) {
+			return true;
+		}
+		when(wait(quorum(false_futures, futures.size() - required + 1))) {
+			return false;
+		}
 	}
 }
 
@@ -103,7 +109,9 @@ ACTOR Future<bool> shortCircuitAny(std::vector<Future<bool>> f) {
 			}
 			return false;
 		}
-		when(wait(waitForAny(sc))) { return true; }
+		when(wait(waitForAny(sc))) {
+			return true;
+		}
 	}
 }
 
diff --git a/flow/genericactors.actor.h b/flow/genericactors.actor.h
index ab3672cb421..81d4422fb05 100644
--- a/flow/genericactors.actor.h
+++ b/flow/genericactors.actor.h
@@ -187,8 +187,12 @@ ACTOR template <class T>
 Future<T> timeout(Future<T> what, double time, T timedoutValue, TaskPriority taskID = TaskPriority::DefaultDelay) {
 	Future<Void> end = delay(time, taskID);
 	choose {
-		when(T t = wait(what)) { return t; }
-		when(wait(end)) { return timedoutValue; }
+		when(T t = wait(what)) {
+			return t;
+		}
+		when(wait(end)) {
+			return timedoutValue;
+		}
 	}
 }
 
@@ -196,8 +200,12 @@ ACTOR template <class T>
 Future<Optional<T>> timeout(Future<T> what, double time) {
 	Future<Void> end = delay(time);
 	choose {
-		when(T t = wait(what)) { return t; }
-		when(wait(end)) { return Optional<T>(); }
+		when(T t = wait(what)) {
+			return t;
+		}
+		when(wait(end)) {
+			return Optional<T>();
+		}
 	}
 }
 
@@ -205,8 +213,12 @@ ACTOR template <class T>
 Future<T> timeoutError(Future<T> what, double time, TaskPriority taskID = TaskPriority::DefaultDelay) {
 	Future<Void> end = delay(time, taskID);
 	choose {
-		when(T t = wait(what)) { return t; }
-		when(wait(end)) { throw timed_out(); }
+		when(T t = wait(what)) {
+			return t;
+		}
+		when(wait(end)) {
+			throw timed_out();
+		}
 	}
 }
 
@@ -227,7 +239,9 @@ Future<T> delayed(Future<T> what, double time = 0.0, TaskPriority taskID = TaskP
 ACTOR template <class Func>
 Future<Void> recurring(Func what, double interval, TaskPriority taskID = TaskPriority::DefaultDelay) {
 	loop choose {
-		when(wait(delay(interval, taskID))) { what(); }
+		when(wait(delay(interval, taskID))) {
+			what();
+		}
 	}
 }
 
@@ -335,7 +349,9 @@ Future<Void> mapAsync(FutureStream<T> input, F actorFunc, PromiseStream<U> outpu
 	loop {
 		try {
 			choose {
-				when(T nextInput = waitNext(input)) { futures.push_back(actorFunc(nextInput)); }
+				when(T nextInput = waitNext(input)) {
+					futures.push_back(actorFunc(nextInput));
+				}
 				when(U nextOutput = wait(futures.size() == 0 ? Never() : futures.front())) {
 					output.send(nextOutput);
 					futures.pop_front();
@@ -452,7 +468,9 @@ Future<Void> asyncFilter(FutureStream<T> input, F actorPred, PromiseStream<T> ou
 	loop {
 		try {
 			choose {
-				when(T nextInput = waitNext(input)) { futures.emplace_back(nextInput, actorPred(nextInput)); }
+				when(T nextInput = waitNext(input)) {
+					futures.emplace_back(nextInput, actorPred(nextInput));
+				}
 				when(bool pass = wait(futures.size() == 0 ? Never() : futures.front().second)) {
 					if (pass)
 						output.send(futures.front().first);
@@ -746,7 +764,9 @@ class Debouncer : NonCopyable {
 			loop {
 				choose {
 					when(wait(self->input.onChange())) {}
-					when(wait(delay(bounceTime))) { break; }
+					when(wait(delay(bounceTime))) {
+						break;
+					}
 				}
 			}
 			self->output.setUnconditional(Void());
@@ -991,8 +1011,12 @@ Future<Void> smartQuorum(std::vector<Future<T>> results,
 		return Void();
 	wait(quorum(results, required));
 	choose {
-		when(wait(quorum(results, (int)results.size()))) { return Void(); }
-		when(wait(delay(extraSeconds, taskID))) { return Void(); }
+		when(wait(quorum(results, (int)results.size()))) {
+			return Void();
+		}
+		when(wait(delay(extraSeconds, taskID))) {
+			return Void();
+		}
 	}
 }
 
@@ -1175,8 +1199,12 @@ Future<Void> orYield(Future<Void> f);
 ACTOR template <class T>
 Future<T> chooseActor(Future<T> lhs, Future<T> rhs) {
 	choose {
-		when(T t = wait(lhs)) { return t; }
-		when(T t = wait(rhs)) { return t; }
+		when(T t = wait(lhs)) {
+			return t;
+		}
+		when(T t = wait(rhs)) {
+			return t;
+		}
 	}
 }
 
@@ -1271,7 +1299,9 @@ void tagAndForwardError(PromiseStream<T>* pOutput, Error value, Future<Void> sig
 ACTOR template <class T>
 Future<T> waitOrError(Future<T> f, Future<Void> errorSignal) {
 	choose {
-		when(T val = wait(f)) { return val; }
+		when(T val = wait(f)) {
+			return val;
+		}
 		when(wait(errorSignal)) {
 			ASSERT(false);
 			throw internal_error();

From af99d3bcf73c7da22bc46e26e0b8e874deb29b28 Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Fri, 9 Dec 2022 13:20:54 -0800
Subject: [PATCH 031/468] Fix clang 15 compiling errors

---
 fdbbackup/backup.actor.cpp                                | 2 +-
 fdbclient/NativeAPI.actor.cpp                             | 4 ----
 fdbrpc/ReplicationPolicy.cpp                              | 2 --
 fdbserver/DDTeamCollection.actor.cpp                      | 2 --
 fdbserver/DataDistributionQueue.actor.cpp                 | 2 +-
 fdbserver/KeyValueStoreSQLite.actor.cpp                   | 3 ---
 fdbserver/LogRouter.actor.cpp                             | 3 ---
 fdbserver/RestoreApplier.actor.cpp                        | 2 --
 fdbserver/SimulatedCluster.actor.cpp                      | 8 +-------
 fdbserver/VersionedBTree.actor.cpp                        | 2 +-
 .../workloads/BlobGranuleCorrectnessWorkload.actor.cpp    | 3 +--
 fdbserver/workloads/WatchAndWait.actor.cpp                | 2 --
 flow/Net2.actor.cpp                                       | 2 +-
 13 files changed, 6 insertions(+), 31 deletions(-)

diff --git a/fdbbackup/backup.actor.cpp b/fdbbackup/backup.actor.cpp
index 9d4bc2ab056..17ee95ac92b 100644
--- a/fdbbackup/backup.actor.cpp
+++ b/fdbbackup/backup.actor.cpp
@@ -3088,7 +3088,7 @@ static std::vector<std::vector<StringRef>> parseLine(std::string& line, bool& er
 
 static void addKeyRange(std::string optionValue, Standalone<VectorRef<KeyRangeRef>>& keyRanges) {
 	bool err = false, partial = false;
-	int tokenArray = 0;
+	[[maybe_unused]] int tokenArray = 0;
 
 	auto parsed = parseLine(optionValue, err, partial);
 
diff --git a/fdbclient/NativeAPI.actor.cpp b/fdbclient/NativeAPI.actor.cpp
index 9ea0ca6eb6d..8f4188a1e85 100644
--- a/fdbclient/NativeAPI.actor.cpp
+++ b/fdbclient/NativeAPI.actor.cpp
@@ -8517,15 +8517,11 @@ Version ChangeFeedData::getVersion() {
 // native api has consumed and processed, them, and then the fdb client has consumed all of the mutations.
 ACTOR Future<Void> changeFeedWaitLatest(Reference<ChangeFeedData> self, Version version) {
 	// wait on SS to have sent up through version
-	int desired = 0;
-	int waiting = 0;
 	std::vector<Future<Void>> allAtLeast;
 	for (auto& it : self->storageData) {
 		if (it->version.get() < version) {
-			waiting++;
 			if (version > it->desired.get()) {
 				it->desired.set(version);
-				desired++;
 			}
 			allAtLeast.push_back(it->version.whenAtLeast(version));
 		}
diff --git a/fdbrpc/ReplicationPolicy.cpp b/fdbrpc/ReplicationPolicy.cpp
index a31cd89190a..712afc08ceb 100644
--- a/fdbrpc/ReplicationPolicy.cpp
+++ b/fdbrpc/ReplicationPolicy.cpp
@@ -77,13 +77,11 @@ bool PolicyOne::selectReplicas(Reference<LocalitySet>& fromServers,
                                std::vector<LocalityEntry> const& alsoServers,
                                std::vector<LocalityEntry>& results) {
 	int totalUsed = 0;
-	int itemsUsed = 0;
 	if (alsoServers.size()) {
 		totalUsed++;
 	} else if (fromServers->size()) {
 		auto randomEntry = fromServers->random();
 		results.push_back(randomEntry);
-		itemsUsed++;
 		totalUsed++;
 	}
 	return (totalUsed > 0);
diff --git a/fdbserver/DDTeamCollection.actor.cpp b/fdbserver/DDTeamCollection.actor.cpp
index 6f016c48eca..cf0303bbefc 100644
--- a/fdbserver/DDTeamCollection.actor.cpp
+++ b/fdbserver/DDTeamCollection.actor.cpp
@@ -4034,7 +4034,6 @@ void DDTeamCollection::traceAllInfo(bool shouldPrint) const {
 
 void DDTeamCollection::rebuildMachineLocalityMap() {
 	machineLocalityMap.clear();
-	int numHealthyMachine = 0;
 	for (auto machine = machine_info.begin(); machine != machine_info.end(); ++machine) {
 		if (machine->second->serversOnMachine.empty()) {
 			TraceEvent(SevWarn, "RebuildMachineLocalityMapError")
@@ -4055,7 +4054,6 @@ void DDTeamCollection::rebuildMachineLocalityMap() {
 		}
 		const LocalityEntry& localityEntry = machineLocalityMap.add(locality, &representativeServer->getId());
 		machine->second->localityEntry = localityEntry;
-		++numHealthyMachine;
 	}
 }
 
diff --git a/fdbserver/DataDistributionQueue.actor.cpp b/fdbserver/DataDistributionQueue.actor.cpp
index 00db7129e09..081efd8a307 100644
--- a/fdbserver/DataDistributionQueue.actor.cpp
+++ b/fdbserver/DataDistributionQueue.actor.cpp
@@ -926,7 +926,7 @@ struct DDQueueData {
 	// canceled inflight relocateData. Launch the relocation for the rd.
 	void launchQueuedWork(std::set<RelocateData, std::greater<RelocateData>> combined,
 	                      const DDEnabledState* ddEnabledState) {
-		int startedHere = 0;
+		[[maybe_unused]] int startedHere = 0;
 		double startTime = now();
 		// kick off relocators from items in the queue as need be
 		std::set<RelocateData, std::greater<RelocateData>>::iterator it = combined.begin();
diff --git a/fdbserver/KeyValueStoreSQLite.actor.cpp b/fdbserver/KeyValueStoreSQLite.actor.cpp
index 9b5472dcef9..dd56e412433 100644
--- a/fdbserver/KeyValueStoreSQLite.actor.cpp
+++ b/fdbserver/KeyValueStoreSQLite.actor.cpp
@@ -1887,14 +1887,11 @@ class KeyValueStoreSQLite final : public IKeyValueStore {
 				readThreads[i].clear();
 		}
 		void checkFreePages() {
-			int iterations = 0;
-
 			int64_t freeListSize = freeListPages;
 			while (!freeTableEmpty && freeListSize < SERVER_KNOBS->CHECK_FREE_PAGE_AMOUNT) {
 				int deletedPages = cursor->lazyDelete(SERVER_KNOBS->CHECK_FREE_PAGE_AMOUNT);
 				freeTableEmpty = (deletedPages != SERVER_KNOBS->CHECK_FREE_PAGE_AMOUNT);
 				springCleaningStats.lazyDeletePages += deletedPages;
-				++iterations;
 
 				freeListSize = conn.freePages();
 			}
diff --git a/fdbserver/LogRouter.actor.cpp b/fdbserver/LogRouter.actor.cpp
index 3bab655a178..31b83de8969 100644
--- a/fdbserver/LogRouter.actor.cpp
+++ b/fdbserver/LogRouter.actor.cpp
@@ -66,11 +66,8 @@ struct LogRouterData {
 		                                       TaskPriority taskID) {
 			while (!self->version_messages.empty() && self->version_messages.front().first < before) {
 				Version version = self->version_messages.front().first;
-				int64_t messagesErased = 0;
 
 				while (!self->version_messages.empty() && self->version_messages.front().first == version) {
-					++messagesErased;
-
 					self->version_messages.pop_front();
 				}
 
diff --git a/fdbserver/RestoreApplier.actor.cpp b/fdbserver/RestoreApplier.actor.cpp
index 8049aba1338..278a3e67c59 100644
--- a/fdbserver/RestoreApplier.actor.cpp
+++ b/fdbserver/RestoreApplier.actor.cpp
@@ -284,12 +284,10 @@ ACTOR static Future<Void> getAndComputeStagingKeys(
 		    .detail("GetKeys", incompleteStagingKeys.size())
 		    .detail("DelayTime", delayTime);
 		ASSERT(!g_network->isSimulated());
-		int i = 0;
 		for (auto& key : incompleteStagingKeys) {
 			MutationRef m(MutationRef::SetValue, key.first, LiteralStringRef("0"));
 			key.second->second.add(m, LogMessageVersion(1));
 			key.second->second.precomputeResult("GetAndComputeStagingKeys", applierID, batchIndex);
-			i++;
 		}
 		return Void();
 	}
diff --git a/fdbserver/SimulatedCluster.actor.cpp b/fdbserver/SimulatedCluster.actor.cpp
index 558ee88442f..50d7bb1824e 100644
--- a/fdbserver/SimulatedCluster.actor.cpp
+++ b/fdbserver/SimulatedCluster.actor.cpp
@@ -2111,7 +2111,7 @@ void setupSimulatedSystem(std::vector<Future<Void>>* systemActors,
 	    .detail("ConfigString", startingConfigString);
 
 	bool requiresExtraDBMachines = testConfig.extraDB && g_simulator.extraDB->toString() != conn.toString();
-	int assignedMachines = 0, nonVersatileMachines = 0;
+	int assignedMachines = 0;
 	bool gradualMigrationPossible = true;
 	std::vector<ProcessClass::ClassType> processClassesSubSet = { ProcessClass::UnsetClass,
 		                                                          ProcessClass::StatelessClass };
@@ -2165,10 +2165,6 @@ void setupSimulatedSystem(std::vector<Future<Void>>* systemActors,
 				else
 					processClass = ProcessClass((ProcessClass::ClassType)deterministicRandom()->randomInt(0, 3),
 					                            ProcessClass::CommandLineSource); // Unset, Storage, or Transaction
-				if (processClass ==
-				    ProcessClass::StatelessClass) { // *can't* be assigned to other roles, even in an emergency
-					nonVersatileMachines++;
-				}
 				if (processClass == ProcessClass::UnsetClass || processClass == ProcessClass::StorageClass) {
 					possible_ss++;
 				}
@@ -2180,11 +2176,9 @@ void setupSimulatedSystem(std::vector<Future<Void>>* systemActors,
 			if (machine >= machines) {
 				if (storageCacheMachines > 0 && dc == 0) {
 					processClass = ProcessClass(ProcessClass::StorageCacheClass, ProcessClass::CommandLineSource);
-					nonVersatileMachines++;
 					storageCacheMachines--;
 				} else if (blobWorkerMachines > 0) { // add blob workers to every DC
 					processClass = ProcessClass(ProcessClass::BlobWorkerClass, ProcessClass::CommandLineSource);
-					nonVersatileMachines++;
 					blobWorkerMachines--;
 				}
 			}
diff --git a/fdbserver/VersionedBTree.actor.cpp b/fdbserver/VersionedBTree.actor.cpp
index c4f942e5275..c20f3f656d9 100644
--- a/fdbserver/VersionedBTree.actor.cpp
+++ b/fdbserver/VersionedBTree.actor.cpp
@@ -6533,7 +6533,7 @@ class VersionedBTree {
 		debug_print(addPrefix(context, update->toString()));
 
 		if (REDWOOD_DEBUG) {
-			int c = 0;
+			[[maybe_unused]] int c = 0;
 			auto i = mBegin;
 			while (1) {
 				debug_printf("%s Mutation %4d '%s':  %s\n",
diff --git a/fdbserver/workloads/BlobGranuleCorrectnessWorkload.actor.cpp b/fdbserver/workloads/BlobGranuleCorrectnessWorkload.actor.cpp
index d7ffd4e92c7..eeec9b5c069 100644
--- a/fdbserver/workloads/BlobGranuleCorrectnessWorkload.actor.cpp
+++ b/fdbserver/workloads/BlobGranuleCorrectnessWorkload.actor.cpp
@@ -610,8 +610,7 @@ struct BlobGranuleCorrectnessWorkload : TestWorkload {
 
 				int targetQueryBytes = (deterministicRandom()->randomInt(1, 20) * targetBytesReadPerQuery) / 10;
 				int estimatedQueryBytes = 0;
-				for (int i = 0; estimatedQueryBytes < targetQueryBytes && endKeyIt != threadData->keyData.end();
-				     i++, endKeyIt++) {
+				for (; estimatedQueryBytes < targetQueryBytes && endKeyIt != threadData->keyData.end(); endKeyIt++) {
 					// iterate forward until end or target keys have passed
 					estimatedQueryBytes += (1 + endKeyIt->second.writes.size() - endKeyIt->second.nextClearIdx) *
 					                       threadData->targetValLength;
diff --git a/fdbserver/workloads/WatchAndWait.actor.cpp b/fdbserver/workloads/WatchAndWait.actor.cpp
index 0d227923d36..b15e9090516 100644
--- a/fdbserver/workloads/WatchAndWait.actor.cpp
+++ b/fdbserver/workloads/WatchAndWait.actor.cpp
@@ -89,7 +89,6 @@ struct WatchAndWaitWorkload : TestWorkload {
 
 	ACTOR Future<Void> _start(Database cx, WatchAndWaitWorkload* self) {
 		state std::vector<Future<Void>> watches;
-		int watchCounter = 0;
 		uint64_t endNode = (self->nodeCount * (self->clientId + 1)) / self->clientCount;
 		uint64_t startNode = (self->nodeCount * self->clientId) / self->clientCount;
 		uint64_t NodesPerWatch = self->nodeCount / self->watchCount;
@@ -101,7 +100,6 @@ struct WatchAndWaitWorkload : TestWorkload {
 		    .detail("Npw", NodesPerWatch);
 		for (uint64_t i = startNode; i < endNode; i += NodesPerWatch) {
 			watches.push_back(self->watchAndWait(cx, self, i));
-			watchCounter++;
 		}
 		wait(delay(self->testDuration)); // || waitForAll( watches )
 		TraceEvent("WatchAndWaitEnd").detail("Duration", self->testDuration);
diff --git a/flow/Net2.actor.cpp b/flow/Net2.actor.cpp
index 1f932421e99..40433762374 100644
--- a/flow/Net2.actor.cpp
+++ b/flow/Net2.actor.cpp
@@ -1717,7 +1717,7 @@ void Net2::trackAtPriority(TaskPriority priority, double now) {
 }
 
 void Net2::processThreadReady() {
-	int numReady = 0;
+	[[maybe_unused]] int numReady = 0;
 	while (true) {
 		Optional<OrderedTask> t = threadReady.pop();
 		if (!t.present())

From 711fb7f9336451a90c7fa394ed156f77e4795122 Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Fri, 9 Dec 2022 13:59:16 -0800
Subject: [PATCH 032/468] Fix gcc 11 compiling errors

---
 fdbrpc/FlowTransport.actor.cpp | 2 +-
 flow/WriteOnlySet.actor.cpp    | 2 ++
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/fdbrpc/FlowTransport.actor.cpp b/fdbrpc/FlowTransport.actor.cpp
index 60ae80f4195..5e0a02a7c27 100644
--- a/fdbrpc/FlowTransport.actor.cpp
+++ b/fdbrpc/FlowTransport.actor.cpp
@@ -368,7 +368,7 @@ struct ConnectPacket {
 	uint16_t flags;
 	uint8_t canonicalRemoteIp6[16];
 
-	ConnectPacket() { memset(this, 0, sizeof(*this)); }
+	ConnectPacket() { memset((void*)this, 0, sizeof(*this)); }
 
 	IPAddress canonicalRemoteIp() const {
 		if (isIPv6()) {
diff --git a/flow/WriteOnlySet.actor.cpp b/flow/WriteOnlySet.actor.cpp
index 46077e4aa2c..e75547c2e08 100644
--- a/flow/WriteOnlySet.actor.cpp
+++ b/flow/WriteOnlySet.actor.cpp
@@ -25,6 +25,8 @@
 
 #include <chrono>
 #include <random>
+#include <thread>
+
 #include "flow/actorcompiler.h" // has to be last include
 
 #ifdef ENABLE_SAMPLING

From e362a1f84529ef5ff77eee32d14dadec3c17b234 Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Fri, 9 Dec 2022 14:43:48 -0800
Subject: [PATCH 033/468] Fix more warnings

---
 bindings/c/foundationdb/fdb_c.h | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/bindings/c/foundationdb/fdb_c.h b/bindings/c/foundationdb/fdb_c.h
index cbd90226e7b..1088e786dbc 100644
--- a/bindings/c/foundationdb/fdb_c.h
+++ b/bindings/c/foundationdb/fdb_c.h
@@ -75,12 +75,12 @@ DLLEXPORT WARN_UNUSED_RESULT fdb_error_t fdb_network_set_option(FDBNetworkOption
                                                                 int value_length);
 
 #if FDB_API_VERSION >= 14
-DLLEXPORT WARN_UNUSED_RESULT fdb_error_t fdb_setup_network();
+DLLEXPORT WARN_UNUSED_RESULT fdb_error_t fdb_setup_network(void);
 #endif
 
-DLLEXPORT WARN_UNUSED_RESULT fdb_error_t fdb_run_network();
+DLLEXPORT WARN_UNUSED_RESULT fdb_error_t fdb_run_network(void);
 
-DLLEXPORT WARN_UNUSED_RESULT fdb_error_t fdb_stop_network();
+DLLEXPORT WARN_UNUSED_RESULT fdb_error_t fdb_stop_network(void);
 
 DLLEXPORT WARN_UNUSED_RESULT fdb_error_t fdb_add_network_thread_completion_hook(void (*hook)(void*),
                                                                                 void* hook_parameter);
@@ -476,8 +476,8 @@ DLLEXPORT WARN_UNUSED_RESULT FDBResult* fdb_transaction_read_blob_granules(FDBTr
 
 DLLEXPORT WARN_UNUSED_RESULT fdb_error_t fdb_select_api_version_impl(int runtime_version, int header_version);
 
-DLLEXPORT int fdb_get_max_api_version();
-DLLEXPORT const char* fdb_get_client_version();
+DLLEXPORT int fdb_get_max_api_version(void);
+DLLEXPORT const char* fdb_get_client_version(void);
 
 /* LEGACY API VERSIONS */
 

From dae0e0eb7fc0edf2405b152c98f60263d6d6bc5d Mon Sep 17 00:00:00 2001
From: neethuhaneesha <nbingi@apple.com>
Date: Tue, 13 Dec 2022 09:09:35 -0800
Subject: [PATCH 034/468] Moving rocksdb read iterator destruction from commit
 path to actor. (#8971)

---
 fdbserver/KeyValueStoreRocksDB.actor.cpp | 37 +++++++++++++++++++-----
 1 file changed, 29 insertions(+), 8 deletions(-)

diff --git a/fdbserver/KeyValueStoreRocksDB.actor.cpp b/fdbserver/KeyValueStoreRocksDB.actor.cpp
index c86ee8b9df1..1deff5a888b 100644
--- a/fdbserver/KeyValueStoreRocksDB.actor.cpp
+++ b/fdbserver/KeyValueStoreRocksDB.actor.cpp
@@ -434,7 +434,7 @@ gets deleted as the ref count becomes 0.
 class ReadIteratorPool {
 public:
 	ReadIteratorPool(UID id, DB& db, CF& cf)
-	  : db(db), cf(cf), index(0), iteratorsReuseCount(0), readRangeOptions(getReadOptions()) {
+	  : db(db), cf(cf), index(0), deletedUptoIndex(0), iteratorsReuseCount(0), readRangeOptions(getReadOptions()) {
 		readRangeOptions.background_purge_on_iterator_cleanup = true;
 		readRangeOptions.auto_prefix_mode = (SERVER_KNOBS->ROCKSDB_PREFIX_LEN > 0);
 		TraceEvent("ReadIteratorPool", id)
@@ -454,8 +454,12 @@ class ReadIteratorPool {
 	void update() {
 		if (SERVER_KNOBS->ROCKSDB_READ_RANGE_REUSE_ITERATORS ||
 		    SERVER_KNOBS->ROCKSDB_READ_RANGE_REUSE_BOUNDED_ITERATORS) {
-			std::lock_guard<std::mutex> lock(mutex);
-			iteratorsMap.clear();
+			mutex.lock();
+			// The latest index might contain the current iterator which is getting created.
+			// But, that should be ok to avoid adding more code complexity.
+			deletedUptoIndex = index;
+			mutex.unlock();
+			deleteIteratorsPromise.send(Void());
 		}
 	}
 
@@ -464,7 +468,7 @@ class ReadIteratorPool {
 		if (SERVER_KNOBS->ROCKSDB_READ_RANGE_REUSE_ITERATORS) {
 			mutex.lock();
 			for (it = iteratorsMap.begin(); it != iteratorsMap.end(); it++) {
-				if (!it->second.inUse) {
+				if (!it->second.inUse && it->second.index > deletedUptoIndex) {
 					it->second.inUse = true;
 					iteratorsReuseCount++;
 					ReadIterator iter = it->second;
@@ -485,7 +489,8 @@ class ReadIteratorPool {
 			// TODO: Based on the datasize in the keyrange, decide whether to store the iterator for reuse.
 			mutex.lock();
 			for (it = iteratorsMap.begin(); it != iteratorsMap.end(); it++) {
-				if (!it->second.inUse && it->second.keyRange.contains(keyRange)) {
+				if (!it->second.inUse && it->second.index > deletedUptoIndex &&
+				    it->second.keyRange.contains(keyRange)) {
 					it->second.inUse = true;
 					iteratorsReuseCount++;
 					ReadIterator iter = it->second;
@@ -532,8 +537,10 @@ class ReadIteratorPool {
 	void refreshIterators() {
 		std::lock_guard<std::mutex> lock(mutex);
 		it = iteratorsMap.begin();
+		auto currTime = now();
 		while (it != iteratorsMap.end()) {
-			if (now() - it->second.creationTime > SERVER_KNOBS->ROCKSDB_READ_RANGE_ITERATOR_REFRESH_TIME) {
+			if ((it->second.index <= deletedUptoIndex) ||
+			    ((currTime - it->second.creationTime) > SERVER_KNOBS->ROCKSDB_READ_RANGE_ITERATOR_REFRESH_TIME)) {
 				it = iteratorsMap.erase(it);
 			} else {
 				it++;
@@ -545,6 +552,8 @@ class ReadIteratorPool {
 
 	uint64_t numTimesReadIteratorsReused() { return iteratorsReuseCount; }
 
+	FutureStream<Void> getDeleteIteratorsFutureStream() { return deleteIteratorsPromise.getFuture(); }
+
 private:
 	std::unordered_map<int, ReadIterator> iteratorsMap;
 	std::unordered_map<int, ReadIterator>::iterator it;
@@ -554,7 +563,9 @@ class ReadIteratorPool {
 	std::mutex mutex;
 	// incrementing counter for every new iterator creation, to uniquely identify the iterator in returnIterator().
 	uint64_t index;
+	uint64_t deletedUptoIndex;
 	uint64_t iteratorsReuseCount;
+	ThreadReturnPromiseStream<Void> deleteIteratorsPromise;
 };
 
 class PerfContextMetrics {
@@ -824,9 +835,19 @@ uint64_t PerfContextMetrics::getRocksdbPerfcontextMetric(int metric) {
 
 ACTOR Future<Void> refreshReadIteratorPool(std::shared_ptr<ReadIteratorPool> readIterPool) {
 	if (SERVER_KNOBS->ROCKSDB_READ_RANGE_REUSE_ITERATORS || SERVER_KNOBS->ROCKSDB_READ_RANGE_REUSE_BOUNDED_ITERATORS) {
+		state FutureStream<Void> deleteIteratorsFutureStream = readIterPool->getDeleteIteratorsFutureStream();
 		loop {
-			wait(delay(SERVER_KNOBS->ROCKSDB_READ_RANGE_ITERATOR_REFRESH_TIME));
-			readIterPool->refreshIterators();
+			choose {
+				when(wait(delay(SERVER_KNOBS->ROCKSDB_READ_RANGE_ITERATOR_REFRESH_TIME))) {
+					readIterPool->refreshIterators();
+				}
+				when(waitNext(deleteIteratorsFutureStream)) {
+					// Add a delay(0.0) to ensure the rest of the caller code runs before refreshing iterators,
+					// i.e., making the refreshIterators() call here asynchronous.
+					wait(delay(0.0));
+					readIterPool->refreshIterators();
+				}
+			}
 		}
 	}
 	return Void();

From 0f21fc6ccbde35f8b06610d9307f2bb5c733764f Mon Sep 17 00:00:00 2001
From: Sreenath Bodagala <82616783+sbodagala@users.noreply.github.com>
Date: Wed, 14 Dec 2022 13:24:47 -0500
Subject: [PATCH 035/468] Release 7.1: Cherry pick pull request #9033 (#9037)

* Merge pull request #9033 from sbodagala/main

* - Code formatting

Co-authored-by: Jingyu Zhou <jingyu_zhou@apple.com>
---
 fdbserver/fdbserver.actor.cpp | 4 ++--
 fdbserver/worker.actor.cpp    | 9 +++++++--
 flow/SystemMonitor.cpp        | 1 +
 flow/SystemMonitor.h          | 7 +++++--
 4 files changed, 15 insertions(+), 6 deletions(-)

diff --git a/fdbserver/fdbserver.actor.cpp b/fdbserver/fdbserver.actor.cpp
index 5e8c3229b0e..2cd200a509d 100644
--- a/fdbserver/fdbserver.actor.cpp
+++ b/fdbserver/fdbserver.actor.cpp
@@ -494,8 +494,8 @@ Future<Void> startSystemMonitor(std::string dataFolder,
                                 Optional<Standalone<StringRef>> dcId,
                                 Optional<Standalone<StringRef>> zoneId,
                                 Optional<Standalone<StringRef>> machineId) {
-	initializeSystemMonitorMachineState(
-	    SystemMonitorMachineState(dataFolder, dcId, zoneId, machineId, g_network->getLocalAddress().ip));
+	initializeSystemMonitorMachineState(SystemMonitorMachineState(
+	    dataFolder, dcId, zoneId, machineId, g_network->getLocalAddress().ip, FDB_VT_VERSION));
 
 	systemMonitor();
 	return recurring(&systemMonitor, SERVER_KNOBS->SYSTEM_MONITOR_FREQUENCY, TaskPriority::FlushTrace);
diff --git a/fdbserver/worker.actor.cpp b/fdbserver/worker.actor.cpp
index 7f7c39b6abb..9425ba1b65b 100644
--- a/fdbserver/worker.actor.cpp
+++ b/fdbserver/worker.actor.cpp
@@ -26,6 +26,7 @@
 #include "fdbclient/GlobalConfig.actor.h"
 #include "fdbclient/ProcessInterface.h"
 #include "fdbclient/StorageServerInterface.h"
+#include "fdbclient/versions.h"
 #include "fdbserver/Knobs.h"
 #include "flow/ActorCollection.h"
 #include "flow/ProtocolVersion.h"
@@ -1659,8 +1660,12 @@ ACTOR Future<Void> workerServer(Reference<IClusterConnectionRecord> connRecord,
 
 	filesClosed.add(stopping.getFuture());
 
-	initializeSystemMonitorMachineState(SystemMonitorMachineState(
-	    folder, locality.dcId(), locality.zoneId(), locality.machineId(), g_network->getLocalAddress().ip));
+	initializeSystemMonitorMachineState(SystemMonitorMachineState(folder,
+	                                                              locality.dcId(),
+	                                                              locality.zoneId(),
+	                                                              locality.machineId(),
+	                                                              g_network->getLocalAddress().ip,
+	                                                              FDB_VT_VERSION));
 
 	{
 		auto recruited = interf;
diff --git a/flow/SystemMonitor.cpp b/flow/SystemMonitor.cpp
index 6ff7decbd61..c4572c0b9dc 100644
--- a/flow/SystemMonitor.cpp
+++ b/flow/SystemMonitor.cpp
@@ -120,6 +120,7 @@ SystemStatistics customSystemMonitor(std::string const& eventName, StatisticsSta
 			    .detail("DCID", machineState.dcId)
 			    .detail("ZoneID", machineState.zoneId)
 			    .detail("MachineID", machineState.machineId)
+			    .detail("Version", machineState.fdbVersion)
 			    .detail("AIOSubmitCount", netData.countAIOSubmit - statState->networkState.countAIOSubmit)
 			    .detail("AIOCollectCount", netData.countAIOCollect - statState->networkState.countAIOCollect)
 			    .detail("AIOSubmitLag",
diff --git a/flow/SystemMonitor.h b/flow/SystemMonitor.h
index 528b659807e..41c764220d7 100644
--- a/flow/SystemMonitor.h
+++ b/flow/SystemMonitor.h
@@ -31,6 +31,7 @@ struct SystemMonitorMachineState {
 	Optional<Standalone<StringRef>> zoneId;
 	Optional<Standalone<StringRef>> machineId;
 	Optional<IPAddress> ip;
+	Optional<std::string> fdbVersion;
 
 	double monitorStartTime;
 
@@ -40,8 +41,10 @@ struct SystemMonitorMachineState {
 	                          Optional<Standalone<StringRef>> const& dcId,
 	                          Optional<Standalone<StringRef>> const& zoneId,
 	                          Optional<Standalone<StringRef>> const& machineId,
-	                          IPAddress const& ip)
-	  : folder(folder), dcId(dcId), zoneId(zoneId), machineId(machineId), ip(ip), monitorStartTime(0) {}
+	                          IPAddress const& ip,
+	                          std::string const& fdbVersion)
+	  : folder(folder), dcId(dcId), zoneId(zoneId), machineId(machineId), ip(ip), fdbVersion(fdbVersion),
+	    monitorStartTime(0) {}
 };
 
 void initializeSystemMonitorMachineState(SystemMonitorMachineState machineState);

From 250a67a2a5b404a70203400ef41027c5ec24e1c3 Mon Sep 17 00:00:00 2001
From: Meng Xu <meng_xu@apple.com>
Date: Thu, 10 Nov 2022 22:59:07 +0000
Subject: [PATCH 036/468] Fix:Exclusion stuck because DD cannot build new teams

Bug behavior:
When DD has zero healthy machine teams but more unhealthy machine teams
than the max machine teams DD plans to build, DD will stop building
new machine teams. Due to zero healthy machine team (and zero healthy
server team), DD cannot find a healthy destination team  to relocate data.
When data relocation stops, exclusion stops progressing and stuck.

Bug happens when we *shrink* a k-host cluster by
first adding k/2 new host;
then quickly excluding all old hosts.

Fix:
Let DD build temporary extra teams to relocate data.
The extra teams will be cleaned up later by DD's remove extra teams logic.

Simulation test:
There is no simulation test to cover cluster expansion scnenario.
To most closely simulate this behavior, we intentionally overbuild all possible
machine teams to trigger the condition that unhealthy teams is larger than
the maximum teams DD wants to build later.
---
 fdbclient/ServerKnobs.cpp            |  1 +
 fdbclient/ServerKnobs.h              |  1 +
 fdbserver/DDTeamCollection.actor.cpp | 84 ++++++++++++++++++++++------
 flow/flow.cpp                        | 20 +++++++
 flow/flow.h                          |  2 +
 5 files changed, 91 insertions(+), 17 deletions(-)

diff --git a/fdbclient/ServerKnobs.cpp b/fdbclient/ServerKnobs.cpp
index b80e00eff78..962e594ebeb 100644
--- a/fdbclient/ServerKnobs.cpp
+++ b/fdbclient/ServerKnobs.cpp
@@ -282,6 +282,7 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi
 	init( DD_TEAM_ZERO_SERVER_LEFT_LOG_DELAY,                    120 ); if( randomize && BUGGIFY ) DD_TEAM_ZERO_SERVER_LEFT_LOG_DELAY = 5;
 	init( DD_STORAGE_WIGGLE_PAUSE_THRESHOLD,                      10 ); if( randomize && BUGGIFY ) DD_STORAGE_WIGGLE_PAUSE_THRESHOLD = 1000;
 	init( DD_STORAGE_WIGGLE_STUCK_THRESHOLD,                      20 );
+	init( DD_BUILD_EXTRA_TEAMS_OVERRIDE,                          10 ); if( randomize && BUGGIFY ) DD_BUILD_EXTRA_TEAMS_OVERRIDE = 2;
 
 	// TeamRemover
 	init( TR_FLAG_DISABLE_MACHINE_TEAM_REMOVER,                false ); if( randomize && BUGGIFY ) TR_FLAG_DISABLE_MACHINE_TEAM_REMOVER = deterministicRandom()->random01() < 0.1 ? true : false; // false by default. disable the consistency check when it's true
diff --git a/fdbclient/ServerKnobs.h b/fdbclient/ServerKnobs.h
index 6dff7a507a7..1a323fc5e65 100644
--- a/fdbclient/ServerKnobs.h
+++ b/fdbclient/ServerKnobs.h
@@ -240,6 +240,7 @@ class ServerKnobs : public KnobsImpl<ServerKnobs> {
 
 	double DD_FAILURE_TIME;
 	double DD_ZERO_HEALTHY_TEAM_DELAY;
+	int DD_BUILD_EXTRA_TEAMS_OVERRIDE; // build extra teams to allow data movement to progress. must be larger than 0
 
 	// Run storage enginee on a child process on the same machine with storage process
 	bool REMOTE_KV_STORE;
diff --git a/fdbserver/DDTeamCollection.actor.cpp b/fdbserver/DDTeamCollection.actor.cpp
index cf0303bbefc..ad9f628f26c 100644
--- a/fdbserver/DDTeamCollection.actor.cpp
+++ b/fdbserver/DDTeamCollection.actor.cpp
@@ -21,6 +21,7 @@
 #include "fdbserver/DDTeamCollection.h"
 #include "flow/Trace.h"
 #include "flow/actorcompiler.h" // This must be the last #include.
+#include <climits>
 
 FDB_DEFINE_BOOLEAN_PARAM(IsPrimary);
 FDB_DEFINE_BOOLEAN_PARAM(IsInitialTeam);
@@ -554,6 +555,14 @@ class DDTeamCollectionImpl {
 			state int teamsToBuild;
 			teamsToBuild = std::max(0, std::min(desiredTeams - teamCount, maxTeams - totalTeamCount));
 
+			if (teamCount == 0 && teamsToBuild == 0 && SERVER_KNOBS->DD_BUILD_EXTRA_TEAMS_OVERRIDE > 0) {
+				// Use DD_BUILD_EXTRA_TEAMS_OVERRIDE > 0 as the feature flag: Set to 0 to disable it
+				TraceEvent(SevWarnAlways, "BuildServerTeamsHaveTooManyUnhealthyTeams")
+				    .detail("Hint", "Build teams may stuck and prevent DD from relocating data")
+				    .detail("BuildExtraServerTeamsOverride", SERVER_KNOBS->DD_BUILD_EXTRA_TEAMS_OVERRIDE);
+				teamsToBuild = SERVER_KNOBS->DD_BUILD_EXTRA_TEAMS_OVERRIDE;
+			}
+
 			TraceEvent("BuildTeamsBegin", self->distributorId)
 			    .detail("TeamsToBuild", teamsToBuild)
 			    .detail("DesiredTeams", desiredTeams)
@@ -562,7 +571,8 @@ class DDTeamCollectionImpl {
 			    .detail("PerpetualWigglingTeams", wigglingTeams)
 			    .detail("UniqueMachines", uniqueMachines)
 			    .detail("TeamSize", self->configuration.storageTeamSize)
-			    .detail("Servers", serverCount)
+			    .detail("Servers", self->server_info.size())
+			    .detail("HealthyServers", serverCount)
 			    .detail("CurrentTrackedServerTeams", self->teams.size())
 			    .detail("HealthyTeamCount", teamCount)
 			    .detail("TotalTeamCount", totalTeamCount)
@@ -619,6 +629,10 @@ class DDTeamCollectionImpl {
 			}
 		} else {
 			self->lastBuildTeamsFailed = true;
+			TraceEvent(SevWarnAlways, "BuildTeamsNotEnoughUniqueMachines", self->distributorId)
+			    .detail("Primary", self->primary)
+			    .detail("UniqueMachines", uniqueMachines)
+			    .detail("Replication", self->configuration.storageTeamSize);
 		}
 
 		self->evaluateTeamQuality();
@@ -3109,10 +3123,11 @@ class DDTeamCollectionImpl {
 				server = server_info.begin();
 				for (i = 0; i < server_info.size(); i++) {
 					const UID& uid = server->first;
-
 					TraceEvent e("ServerStatus", self->getDistributorId());
 					e.detail("ServerUID", uid)
+					    .detail("ServerInfoIndex", i)
 					    .detail("MachineIsValid", server_info[uid]->machine.isValid())
+					    .detail("IsMachineHealthy", self->isMachineHealthy(server_info[uid]->machine))
 					    .detail("MachineTeamSize",
 					            server_info[uid]->machine.isValid() ? server_info[uid]->machine->machineTeams.size()
 					                                                : -1)
@@ -3156,8 +3171,9 @@ class DDTeamCollectionImpl {
 				state bool isMachineHealthy = false;
 				for (i = 0; i < machine_info.size(); i++) {
 					Reference<TCMachineInfo> _machine = machine->second;
-					if (!_machine.isValid() || machine_info.find(_machine->machineID) == machine_info.end() ||
-					    _machine->serversOnMachine.empty()) {
+					bool machineIDFound = machine_info.find(_machine->machineID) != machine_info.end();
+					bool zeroHealthyServersOnMachine = true;
+					if (!_machine.isValid() || !machineIDFound || _machine->serversOnMachine.empty()) {
 						isMachineHealthy = false;
 					}
 
@@ -3169,6 +3185,7 @@ class DDTeamCollectionImpl {
 						auto it = server_status.find(server->getId());
 						if (it != server_status.end() && !it->second.isUnhealthy()) {
 							isMachineHealthy = true;
+							zeroHealthyServersOnMachine = false;
 						}
 					}
 
@@ -3176,6 +3193,9 @@ class DDTeamCollectionImpl {
 					TraceEvent("MachineInfo", self->getDistributorId())
 					    .detail("MachineInfoIndex", i)
 					    .detail("Healthy", isMachineHealthy)
+					    .detail("MachineIDFound", machineIDFound)
+					    .detail("ZeroServersOnMachine", _machine->serversOnMachine.empty())
+					    .detail("ZeroHealthyServersOnMachine", zeroHealthyServersOnMachine)
 					    .detail("MachineID", machine->first.contents().toString())
 					    .detail("MachineTeamOwned", machine->second->machineTeams.size())
 					    .detail("ServerNumOnMachine", machine->second->serversOnMachine.size())
@@ -3267,21 +3287,25 @@ void DDTeamCollection::traceServerInfo() const {
 	int i = 0;
 
 	TraceEvent("ServerInfo", distributorId).detail("Size", server_info.size());
-	for (auto& server : server_info) {
+	for (auto& [serverID, server] : server_info) {
 		TraceEvent("ServerInfo", distributorId)
 		    .detail("ServerInfoIndex", i++)
-		    .detail("ServerID", server.first.toString())
-		    .detail("ServerTeamOwned", server.second->getTeams().size())
-		    .detail("MachineID", server.second->machine->machineID.contents().toString())
-		    .detail("StoreType", server.second->getStoreType().toString())
-		    .detail("InDesiredDC", server.second->isInDesiredDC());
+		    .detail("ServerID", serverID.toString())
+		    .detail("ServerTeamOwned", server->getTeams().size())
+		    .detail("MachineID", server->machine->machineID.contents().toString())
+		    .detail("StoreType", server->getStoreType().toString())
+		    .detail("InDesiredDC", server->isInDesiredDC());
 	}
+	i = 0;
 	for (auto& server : server_info) {
-		const UID& uid = server.first;
+		const UID& serverID = server.first;
+		const ServerStatus& status = server_status.get(serverID);
 		TraceEvent("ServerStatus", distributorId)
-		    .detail("ServerID", uid)
-		    .detail("Healthy", !server_status.get(uid).isUnhealthy())
-		    .detail("MachineIsValid", get(server_info, uid)->machine.isValid())
+		    .detail("ServerInfoIndex", i++)
+		    .detail("ServerID", serverID)
+		    .detail("Healthy", !status.isUnhealthy())
+		    .detail("StatusString", status.toString())
+		    .detail("MachineIsValid", get(server_info, serverID)->machine.isValid())
 		    .detail("MachineTeamSize",
 		            get(server_info, uid)->machine.isValid() ? get(server_info, uid)->machine->machineTeams.size()
 		                                                     : -1);
@@ -4022,6 +4046,7 @@ void DDTeamCollection::traceAllInfo(bool shouldPrint) const {
 		}
 	}
 
+	// TODO: flush trace log to avoid trace buffer overflow when DD has too many servers and teams
 	TraceEvent("TraceAllInfo", distributorId).detail("Primary", primary);
 	traceConfigInfo();
 	traceServerInfo();
@@ -4190,10 +4215,13 @@ int DDTeamCollection::addBestMachineTeams(int machineTeamsToBuild) {
 			addMachineTeam(machines);
 			addedMachineTeams++;
 		} else {
-			traceAllInfo(true);
+			// When too many teams exist in simulation, traceAllInfo will buffer too many trace logs before
+			// trace has a chance to flush its buffer, which causes assertion failure.
+			traceAllInfo(g_network->isSimulated() ? false : true);
 			TraceEvent(SevWarn, "DataDistributionBuildTeams", distributorId)
 			    .detail("Primary", primary)
-			    .detail("Reason", "Unable to make desired machine Teams");
+			    .detail("Reason", "Unable to make desired machine Teams")
+			    .detail("Hint", "Check TraceAllInfo event");
 			lastBuildTeamsFailed = true;
 			break;
 		}
@@ -4482,13 +4510,34 @@ int DDTeamCollection::addTeamsBestOf(int teamsToBuild, int desiredTeams, int max
 	// machineTeamsToBuild mimics how the teamsToBuild is calculated in buildTeams()
 	int machineTeamsToBuild =
 	    std::max(0, std::min(desiredMachineTeams - healthyMachineTeamCount, maxMachineTeams - totalMachineTeamCount));
+	if (healthyMachineTeamCount == 0 && machineTeamsToBuild == 0 && SERVER_KNOBS->DD_BUILD_EXTRA_TEAMS_OVERRIDE > 0) {
+		// Use DD_BUILD_EXTRA_TEAMS_OVERRIDE > 0 as the feature flag: Set to 0 to disable it
+		TraceEvent(SevWarnAlways, "BuildMachineTeamsHaveTooManyUnhealthyMachineTeams")
+		    .detail("Hint", "Build teams may stuck and prevent DD from relocating data")
+		    .detail("BuildExtraMachineTeamsOverride", SERVER_KNOBS->DD_BUILD_EXTRA_TEAMS_OVERRIDE);
+		machineTeamsToBuild = SERVER_KNOBS->DD_BUILD_EXTRA_TEAMS_OVERRIDE;
+	}
+	if (g_network->isSimulated() && deterministicRandom()->random01() < 0.1) {
+		// Test when the system has lots of unhealthy machine teams, which may prevent TC from building new teams.
+		// The scenario creates a deadlock situation that DD cannot relocate data.
+		int totalMachineTeams = nChooseK(machine_info.size(), configuration.storageTeamSize);
+		TraceEvent("BuildMachineTeams")
+		    .detail("Primary", primary)
+		    .detail("CalculatedMachineTeamsToBuild", machineTeamsToBuild)
+		    .detail("OverwriteMachineTeamsToBuildForTesting", totalMachineTeams);
+
+		machineTeamsToBuild = totalMachineTeams;
+	}
 
 	{
 		TraceEvent te("BuildMachineTeams");
-		te.detail("TotalHealthyMachine", totalHealthyMachineCount)
+		te.detail("Primary", primary)
+		    .detail("TotalMachines", machine_info.size())
+		    .detail("TotalHealthyMachine", totalHealthyMachineCount)
 		    .detail("HealthyMachineTeamCount", healthyMachineTeamCount)
 		    .detail("DesiredMachineTeams", desiredMachineTeams)
 		    .detail("MaxMachineTeams", maxMachineTeams)
+		    .detail("TotalMachineTeams", totalMachineTeamCount)
 		    .detail("MachineTeamsToBuild", machineTeamsToBuild);
 		// Pre-build all machine teams until we have the desired number of machine teams
 		if (machineTeamsToBuild > 0 || notEnoughMachineTeamsForAMachine()) {
@@ -4791,6 +4840,7 @@ Reference<TCMachineInfo> DDTeamCollection::checkAndCreateMachine(Reference<TCSer
 		machineInfo->serversOnMachine.push_back(server);
 	}
 	server->machine = machineInfo;
+	ASSERT(machineInfo->machineID == machine_id); // invariant for TC to work
 
 	return machineInfo;
 }
diff --git a/flow/flow.cpp b/flow/flow.cpp
index 285de7e9045..40bf1c183f2 100644
--- a/flow/flow.cpp
+++ b/flow/flow.cpp
@@ -359,6 +359,26 @@ void enableBuggify(bool enabled, BuggifyType type) {
 	buggifyActivated[int(type)] = enabled;
 }
 
+int nChooseK(int n, int k) {
+	if (k == 0) {
+		return 1;
+	}
+	if (k > n / 2) {
+		return nChooseK(n, n - k);
+	}
+
+	long ret = 1;
+
+	// To avoid integer overflow, we do n/1 * (n-1)/2 * (n-2)/3 * (n-i+1)/i, where i = k
+	for (int i = 1; i <= k; ++i) {
+		ret *= n - i + 1;
+		ret /= i;
+	}
+	ASSERT(ret <= INT_MAX);
+
+	return ret;
+}
+
 namespace {
 // Simple message for flatbuffers unittests
 struct Int {
diff --git a/flow/flow.h b/flow/flow.h
index 0cbba3bf452..92289255016 100644
--- a/flow/flow.h
+++ b/flow/flow.h
@@ -105,6 +105,8 @@ extern StringRef strinc(StringRef const& str, Arena& arena);
 extern Standalone<StringRef> addVersionStampAtEnd(StringRef const& str);
 extern StringRef addVersionStampAtEnd(StringRef const& str, Arena& arena);
 
+int nChooseK(int n, int k);
+
 template <typename Iter>
 StringRef concatenate(Iter b, Iter const& e, Arena& arena) {
 	int rsize = 0;

From 89854c4f227ed3e9a1a2b4579ddea1337fb673ed Mon Sep 17 00:00:00 2001
From: Meng Xu <meng_xu@apple.com>
Date: Tue, 13 Dec 2022 15:52:45 -0800
Subject: [PATCH 037/468] Resolve review comment: No functional change

---
 fdbserver/DDTeamCollection.actor.cpp | 7 ++++---
 flow/flow.cpp                        | 1 +
 flow/flow.h                          | 1 +
 3 files changed, 6 insertions(+), 3 deletions(-)

diff --git a/fdbserver/DDTeamCollection.actor.cpp b/fdbserver/DDTeamCollection.actor.cpp
index ad9f628f26c..97c67492eea 100644
--- a/fdbserver/DDTeamCollection.actor.cpp
+++ b/fdbserver/DDTeamCollection.actor.cpp
@@ -3307,8 +3307,9 @@ void DDTeamCollection::traceServerInfo() const {
 		    .detail("StatusString", status.toString())
 		    .detail("MachineIsValid", get(server_info, serverID)->machine.isValid())
 		    .detail("MachineTeamSize",
-		            get(server_info, uid)->machine.isValid() ? get(server_info, uid)->machine->machineTeams.size()
-		                                                     : -1);
+		            get(server_info, serverID)->machine.isValid()
+		                ? get(server_info, serverID)->machine->machineTeams.size()
+		                : -1);
 	}
 }
 
@@ -4217,7 +4218,7 @@ int DDTeamCollection::addBestMachineTeams(int machineTeamsToBuild) {
 		} else {
 			// When too many teams exist in simulation, traceAllInfo will buffer too many trace logs before
 			// trace has a chance to flush its buffer, which causes assertion failure.
-			traceAllInfo(g_network->isSimulated() ? false : true);
+			traceAllInfo(!g_network->isSimulated());
 			TraceEvent(SevWarn, "DataDistributionBuildTeams", distributorId)
 			    .detail("Primary", primary)
 			    .detail("Reason", "Unable to make desired machine Teams")
diff --git a/flow/flow.cpp b/flow/flow.cpp
index 40bf1c183f2..26ea7afe8c8 100644
--- a/flow/flow.cpp
+++ b/flow/flow.cpp
@@ -360,6 +360,7 @@ void enableBuggify(bool enabled, BuggifyType type) {
 }
 
 int nChooseK(int n, int k) {
+	assert(n >= k && k >= 0);
 	if (k == 0) {
 		return 1;
 	}
diff --git a/flow/flow.h b/flow/flow.h
index 92289255016..d8a142d4142 100644
--- a/flow/flow.h
+++ b/flow/flow.h
@@ -105,6 +105,7 @@ extern StringRef strinc(StringRef const& str, Arena& arena);
 extern Standalone<StringRef> addVersionStampAtEnd(StringRef const& str);
 extern StringRef addVersionStampAtEnd(StringRef const& str, Arena& arena);
 
+// Return the number of combinations to choose k items out of n choices
 int nChooseK(int n, int k);
 
 template <typename Iter>

From 3012c71b875cff436692fb56acf28259833f2565 Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyu_zhou@apple.com>
Date: Wed, 21 Dec 2022 14:10:49 -0800
Subject: [PATCH 038/468] Add back samples for (non)empty peeks stats
 [release-7.1] (#9074)

* Add back samples for (non)empty peeks stats

These were lost, likely due to refactoring. Now TLogMetrics have meaningful
data like:

TLogMetrics ID=59ec9c67b4d07433 Elapsed=5 BytesInput=0 -1 17048 BytesDurable=47.4 225.405 17048 BlockingPeeks=0 -1 0 BlockingPeekTimeouts=0 -1 0 EmptyPeeks=1.6 2.79237 236 NonEmptyPeeks=0 -1 32 ...

* Use LATENCY_SAMPLE_SIZE
---
 fdbserver/TLogServer.actor.cpp | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)

diff --git a/fdbserver/TLogServer.actor.cpp b/fdbserver/TLogServer.actor.cpp
index b31a266d824..74ed6ff8e22 100644
--- a/fdbserver/TLogServer.actor.cpp
+++ b/fdbserver/TLogServer.actor.cpp
@@ -1626,6 +1626,7 @@ void peekMessagesFromMemory(Reference<LogData> self,
                             Version& endVersion) {
 	ASSERT(!messages.getLength());
 
+	int versionCount = 0;
 	auto& deque = getVersionMessages(self, tag);
 	//TraceEvent("TLogPeekMem", self->dbgid).detail("Tag", req.tag1).detail("PDS", self->persistentDataSequence).detail("PDDS", self->persistentDataDurableSequence).detail("Oldest", map1.empty() ? 0 : map1.begin()->key ).detail("OldestMsgCount", map1.empty() ? 0 : map1.begin()->value.size());
 
@@ -1656,6 +1657,23 @@ void peekMessagesFromMemory(Reference<LogData> self,
 		DEBUG_TAGS_AND_MESSAGE(
 		    "TLogPeek", currentVersion, StringRef((uint8_t*)data + offset, messages.getLength() - offset), self->logId)
 		    .detail("PeekTag", tag);
+		versionCount++;
+	}
+
+	if (versionCount == 0) {
+		++self->emptyPeeks;
+	} else {
+		++self->nonEmptyPeeks;
+
+		// TODO (version vector) check if this should be included in "status details" json
+		if (self->peekVersionCounts.find(tag) == self->peekVersionCounts.end()) {
+			UID ssID = deterministicRandom()->randomUniqueID();
+			std::string s = "PeekVersionCounts " + tag.toString();
+			self->peekVersionCounts.try_emplace(
+			    tag, s, ssID, SERVER_KNOBS->LATENCY_METRICS_LOGGING_INTERVAL, SERVER_KNOBS->LATENCY_SAMPLE_SIZE);
+		}
+		LatencySample& sample = self->peekVersionCounts.at(tag);
+		sample.addMeasurement(versionCount);
 	}
 }
 

From f5163b33525d4cae4f8667280dcd894e3ec00e75 Mon Sep 17 00:00:00 2001
From: Zhe Wu <halfprice@users.noreply.github.com>
Date: Wed, 21 Dec 2022 16:18:08 -0800
Subject: [PATCH 039/468] fix health monitor last logged time

---
 fdbserver/worker.actor.cpp | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/fdbserver/worker.actor.cpp b/fdbserver/worker.actor.cpp
index 9425ba1b65b..56eb27204b4 100644
--- a/fdbserver/worker.actor.cpp
+++ b/fdbserver/worker.actor.cpp
@@ -990,6 +990,9 @@ ACTOR Future<Void> healthMonitor(Reference<AsyncVar<Optional<ClusterControllerFu
 					}
 					bool degradedPeer = false;
 					bool disconnectedPeer = false;
+
+					// If peer->lastLoggedTime == 0, we just started monitor this peer and haven't logged it once yet.
+					double lastLoggedTime = peer->lastLoggedTime <= 0.0 ? peer->lastConnectTime : peer->lastLoggedTime;
 					if ((workerLocation == Primary && addressInDbAndPrimaryDc(address, dbInfo)) ||
 					    (workerLocation == Remote && addressInDbAndRemoteDc(address, dbInfo))) {
 						// Monitors intra DC latencies between servers that in the primary or remote DC's transaction
@@ -1007,7 +1010,7 @@ ACTOR Future<Void> healthMonitor(Reference<AsyncVar<Optional<ClusterControllerFu
 						if (disconnectedPeer || degradedPeer) {
 							TraceEvent("HealthMonitorDetectDegradedPeer")
 							    .detail("Peer", address)
-							    .detail("Elapsed", now() - peer->lastLoggedTime)
+							    .detail("Elapsed", now() - lastLoggedTime)
 							    .detail("Disconnected", disconnectedPeer)
 							    .detail("MinLatency", peer->pingLatencies.min())
 							    .detail("MaxLatency", peer->pingLatencies.max())
@@ -1038,7 +1041,7 @@ ACTOR Future<Void> healthMonitor(Reference<AsyncVar<Optional<ClusterControllerFu
 							TraceEvent("HealthMonitorDetectDegradedPeer")
 							    .detail("Peer", address)
 							    .detail("Satellite", true)
-							    .detail("Elapsed", now() - peer->lastLoggedTime)
+							    .detail("Elapsed", now() - lastLoggedTime)
 							    .detail("Disconnected", disconnectedPeer)
 							    .detail("MinLatency", peer->pingLatencies.min())
 							    .detail("MaxLatency", peer->pingLatencies.max())

From 4d9493e1124b1867095fef67d13a4dd23d81b22c Mon Sep 17 00:00:00 2001
From: Xiaoge Su <magichp@gmail.com>
Date: Thu, 5 Jan 2023 09:00:57 -0800
Subject: [PATCH 040/468] Backport RocksDB cmake file to 7.1 (#9093)

* Fix the RocksDB compile issue with clang

By default, RocksDB is using its own compile/link flags, no matter how
FDB flags are. This led to the issue that if FDB decides to use
clang/ldd/libc++, RocksDB will pick up the compiler/linker but still use
libstdc++, which is incompatible to libc++, causing Symobl Missing error
during the link stage.

With this patch, if FDB uses libc++, then the information is stored in
CMAKE_CXX_FLAGS and being forwarded to RocksDB. RocksDB will then use
libc++ and compatible with FDB.

* fixup! Fix the clang error in bindings/c
---
 bindings/c/CMakeLists.txt     |  6 ++-
 cmake/CompileRocksDB.cmake    | 89 +++++++++++++++++------------------
 cmake/ConfigureCompiler.cmake | 13 +++--
 3 files changed, 58 insertions(+), 50 deletions(-)

diff --git a/bindings/c/CMakeLists.txt b/bindings/c/CMakeLists.txt
index 676fe824e30..b1a187b994f 100644
--- a/bindings/c/CMakeLists.txt
+++ b/bindings/c/CMakeLists.txt
@@ -184,7 +184,11 @@ endif()
     # Make sure that fdb_c.h is compatible with c90
     add_executable(fdb_c90_test test/fdb_c90_test.c)
     set_property(TARGET fdb_c90_test PROPERTY C_STANDARD 90)
-    target_compile_options(fdb_c90_test PRIVATE -Wall -Wextra -Wpedantic -Werror)
+	if (CMAKE_CXX_COMPILER_ID STREQUAL "Clang")
+	  target_compile_options(fdb_c90_test PRIVATE -Wall -Wextra -Wpedantic -Wno-gnu-line-marker -Werror)
+	else ()
+	  target_compile_options(fdb_c90_test PRIVATE -Wall -Wextra -Wpedantic -Werror)
+	endif ()
     target_link_libraries(fdb_c90_test PRIVATE fdb_c)
   endif()
 
diff --git a/cmake/CompileRocksDB.cmake b/cmake/CompileRocksDB.cmake
index bf38364458f..d66ee4dec8e 100644
--- a/cmake/CompileRocksDB.cmake
+++ b/cmake/CompileRocksDB.cmake
@@ -4,71 +4,68 @@ find_package(RocksDB 7.7.3)
 
 include(ExternalProject)
 
-if (RocksDB_FOUND)
+set(RocksDB_CMAKE_ARGS
+  -DUSE_RTTI=1
+  -DPORTABLE=${PORTABLE_ROCKSDB}
+  -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER}
+  -DCMAKE_C_FLAGS=${CMAKE_C_FLAGS}
+  -DCMAKE_CXX_STANDARD=${CMAKE_CXX_STANDARD}
+  -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}
+  -DCMAKE_CXX_FLAGS=${CMAKE_CXX_FLAGS}
+  -DCMAKE_SHARED_LINKER_FLAGS=${CMAKE_SHARED_LINKER_FLAGS}
+  -DCMAKE_STATIC_LINKER_FLAGS=${CMAKE_STATIC_LINKER_FLAGS}
+  -DCMAKE_EXE_LINKER_FLAGS=${CMAKE_EXE_LINKER_FLAGS}
+  -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}
+  -DFAIL_ON_WARNINGS=OFF
+  -DWITH_GFLAGS=OFF
+  -DWITH_TESTS=OFF
+  -DWITH_TOOLS=OFF
+  -DWITH_CORE_TOOLS=OFF
+  -DWITH_BENCHMARK_TOOLS=OFF
+  -DWITH_BZ2=OFF
+  -DWITH_LZ4=ON
+  -DWITH_SNAPPY=OFF
+  -DWITH_ZLIB=OFF
+  -DWITH_ZSTD=OFF
+  -DWITH_LIBURING=${WITH_LIBURING}
+  -DWITH_TSAN=${USE_TSAN}
+  -DWITH_ASAN=${USE_ASAN}
+  -DWITH_UBSAN=${USE_UBSAN}
+  -DROCKSDB_BUILD_SHARED=OFF
+  -DCMAKE_POSITION_INDEPENDENT_CODE=True
+)
+
+if(ROCKSDB_FOUND)
   ExternalProject_Add(rocksdb
     SOURCE_DIR "${RocksDB_ROOT}"
     DOWNLOAD_COMMAND ""
-    CMAKE_ARGS -DUSE_RTTI=1 -DPORTABLE=${PORTABLE_ROCKSDB}
-               -DCMAKE_CXX_STANDARD=${CMAKE_CXX_STANDARD}
-               -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}
-               -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}
-               -DWITH_GFLAGS=OFF
-               -DWITH_TESTS=OFF
-               -DWITH_TOOLS=OFF
-               -DWITH_CORE_TOOLS=OFF
-               -DWITH_BENCHMARK_TOOLS=OFF
-               -DWITH_BZ2=OFF
-               -DWITH_LZ4=ON
-               -DWITH_SNAPPY=OFF
-               -DWITH_ZLIB=OFF
-               -DWITH_ZSTD=OFF
-               -DWITH_LIBURING=${WITH_LIBURING}
-               -DWITH_TSAN=${USE_TSAN}
-               -DWITH_ASAN=${USE_ASAN}
-               -DWITH_UBSAN=${USE_UBSAN}
-               -DROCKSDB_BUILD_SHARED=OFF
-               -DCMAKE_POSITION_INDEPENDENT_CODE=True
+    CMAKE_ARGS ${RocksDB_CMAKE_ARGS}
     BUILD_BYPRODUCTS <BINARY_DIR>/librocksdb.a
     INSTALL_COMMAND ""
+    LOG_CONFIGURE TRUE
+    LOG_BUILD TRUE
   )
 
   ExternalProject_Get_Property(rocksdb BINARY_DIR)
   set(ROCKSDB_LIBRARIES
-      ${BINARY_DIR}/librocksdb.a)
+    ${BINARY_DIR}/librocksdb.a)
 else()
   ExternalProject_Add(rocksdb
     URL        https://github.com/facebook/rocksdb/archive/refs/tags/v7.7.3.tar.gz
     URL_HASH   SHA256=b8ac9784a342b2e314c821f6d701148912215666ac5e9bdbccd93cf3767cb611
-    CMAKE_ARGS -DUSE_RTTI=1 -DPORTABLE=${PORTABLE_ROCKSDB}
-               -DCMAKE_CXX_STANDARD=${CMAKE_CXX_STANDARD}
-               -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}
-               -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}
-               -DWITH_GFLAGS=OFF
-               -DWITH_TESTS=OFF
-               -DWITH_TOOLS=OFF
-               -DWITH_CORE_TOOLS=OFF
-               -DWITH_BENCHMARK_TOOLS=OFF
-               -DWITH_BZ2=OFF
-               -DWITH_LZ4=ON
-               -DWITH_SNAPPY=OFF
-               -DWITH_ZLIB=OFF
-               -DWITH_ZSTD=OFF
-               -DWITH_LIBURING=${WITH_LIBURING}
-               -DWITH_TSAN=${USE_TSAN}
-               -DWITH_ASAN=${USE_ASAN}
-               -DWITH_UBSAN=${USE_UBSAN}
-               -DROCKSDB_BUILD_SHARED=OFF
-               -DCMAKE_POSITION_INDEPENDENT_CODE=True
+    CMAKE_ARGS ${RocksDB_CMAKE_ARGS}
     BUILD_BYPRODUCTS <BINARY_DIR>/librocksdb.a
     INSTALL_COMMAND ""
+    LOG_CONFIGURE TRUE
+    LOG_BUILD TRUE
   )
 
   ExternalProject_Get_Property(rocksdb BINARY_DIR)
   set(ROCKSDB_LIBRARIES
-      ${BINARY_DIR}/librocksdb.a)
+    ${BINARY_DIR}/librocksdb.a)
 
   ExternalProject_Get_Property(rocksdb SOURCE_DIR)
-  set (ROCKSDB_INCLUDE_DIR "${SOURCE_DIR}/include")
+  set(ROCKSDB_INCLUDE_DIR "${SOURCE_DIR}/include")
 
   set(ROCKSDB_FOUND TRUE)
 endif()
@@ -77,6 +74,6 @@ message(STATUS "Found RocksDB library: ${ROCKSDB_LIBRARIES}")
 message(STATUS "Found RocksDB includes: ${ROCKSDB_INCLUDE_DIR}")
 
 mark_as_advanced(
-    ROCKSDB_LIBRARIES
-    ROCKSDB_INCLUDE_DIR
+  ROCKSDB_LIBRARIES
+  ROCKSDB_INCLUDE_DIR
 )
diff --git a/cmake/ConfigureCompiler.cmake b/cmake/ConfigureCompiler.cmake
index 398dfe5fb61..e52d8665ee8 100644
--- a/cmake/ConfigureCompiler.cmake
+++ b/cmake/ConfigureCompiler.cmake
@@ -278,14 +278,21 @@ else()
   if (CLANG)
     add_compile_options()
     if (APPLE OR USE_LIBCXX)
-      add_compile_options($<$<COMPILE_LANGUAGE:CXX>:-stdlib=libc++>)
+      set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -stdlib=libc++")
       if (NOT APPLE)
         if (STATIC_LINK_LIBCXX)
-          add_link_options(-static-libgcc -nostdlib++  -Wl,-Bstatic -lc++ -lc++abi -Wl,-Bdynamic)
+          set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -static-libgcc -nostdlib++ -Wl,-Bstatic -lc++ -lc++abi -Wl,-Bdynamic")
+          set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -static-libgcc -nostdlib++ -Wl,-Bstatic -lc++ -lc++abi -Wl,-Bdynamic")
         endif()
-        add_link_options(-stdlib=libc++ -Wl,-build-id=sha1)
+        set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -stdlib=libc++ -Wl,-build-id=sha1")
+        set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -stdlib=libc++ -Wl,-build-id=sha1")
       endif()
     endif()
+    if (NOT APPLE AND NOT USE_LIBCXX)
+      message(STATUS "Linking libatomic")
+      set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -latomic")
+      set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -latomic")
+    endif()
     if (OPEN_FOR_IDE)
       add_compile_options(
         -Wno-unknown-attributes)

From 1eaf95a76a9e04b0f5304b50d063a58491543efd Mon Sep 17 00:00:00 2001
From: Aaron Molitor <amolitor@apple.com>
Date: Fri, 6 Jan 2023 15:12:40 -0600
Subject: [PATCH 041/468] add some rocksdb compile options that can be passed
 in at build time

---
 cmake/CompileRocksDB.cmake | 3 +++
 cmake/FDBComponents.cmake  | 3 +++
 2 files changed, 6 insertions(+)

diff --git a/cmake/CompileRocksDB.cmake b/cmake/CompileRocksDB.cmake
index d66ee4dec8e..62405d04df9 100644
--- a/cmake/CompileRocksDB.cmake
+++ b/cmake/CompileRocksDB.cmake
@@ -7,6 +7,9 @@ include(ExternalProject)
 set(RocksDB_CMAKE_ARGS
   -DUSE_RTTI=1
   -DPORTABLE=${PORTABLE_ROCKSDB}
+  -DFORCE_SSE42=${ROCKSDB_SSE42}
+  -DFORCE_AVX=${ROCKSDB_AVX}
+  -DFORCE_AVX2=${ROCKSDB_AVX2}
   -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER}
   -DCMAKE_C_FLAGS=${CMAKE_C_FLAGS}
   -DCMAKE_CXX_STANDARD=${CMAKE_CXX_STANDARD}
diff --git a/cmake/FDBComponents.cmake b/cmake/FDBComponents.cmake
index 60fbe44a0d7..8ce7c243a7c 100644
--- a/cmake/FDBComponents.cmake
+++ b/cmake/FDBComponents.cmake
@@ -163,6 +163,9 @@ endif()
 
 set(SSD_ROCKSDB_EXPERIMENTAL ON CACHE BOOL "Build with experimental RocksDB support")
 set(PORTABLE_ROCKSDB ON CACHE BOOL "Compile RocksDB in portable mode") # Set this to OFF to compile RocksDB with `-march=native`
+set(ROCKSDB_SSE42 OFF CACHE BOOL "Compile RocksDB with SSE42 enabled")
+set(ROCKSDB_AVX ${USE_AVX} CACHE BOOL "Compile RocksDB with AVX enabled")
+set(ROCKSDB_AVX2 OFF CACHE BOOL "Compile RocksDB with AVX2 enabled")
 set(WITH_LIBURING OFF CACHE BOOL "Build with liburing enabled") # Set this to ON to include liburing
 # RocksDB is currently enabled by default for GCC but does not build with the latest
 # Clang.

From ee7c9d4959c5737c6f018d10c9e9c4a81050e475 Mon Sep 17 00:00:00 2001
From: Zhe Wu <halfprice@users.noreply.github.com>
Date: Thu, 5 Jan 2023 22:23:02 -0800
Subject: [PATCH 042/468] Disconnection to satellite TLog should trigger
 recovery in gray failure detection

---
 fdbserver/ClusterController.actor.cpp |  4 +++-
 fdbserver/ClusterController.actor.h   | 16 ++++++++++++----
 2 files changed, 15 insertions(+), 5 deletions(-)

diff --git a/fdbserver/ClusterController.actor.cpp b/fdbserver/ClusterController.actor.cpp
index 2e799e348e4..0136192da7e 100644
--- a/fdbserver/ClusterController.actor.cpp
+++ b/fdbserver/ClusterController.actor.cpp
@@ -3141,8 +3141,10 @@ TEST_CASE("/fdbserver/clustercontroller/shouldTriggerRecoveryDueToDegradedServer
 	data.degradationInfo.degradedServers.insert(satelliteTlog);
 	ASSERT(!data.shouldTriggerRecoveryDueToDegradedServers());
 	data.degradationInfo.degradedServers.clear();
+
+	// Trigger recovery when satellite Tlog is disconnected.
 	data.degradationInfo.disconnectedServers.insert(satelliteTlog);
-	ASSERT(!data.shouldTriggerRecoveryDueToDegradedServers());
+	ASSERT(data.shouldTriggerRecoveryDueToDegradedServers());
 	data.degradationInfo.disconnectedServers.clear();
 
 	// No recovery when remote tlog is degraded.
diff --git a/fdbserver/ClusterController.actor.h b/fdbserver/ClusterController.actor.h
index 0c2d2375a5f..ef96581d7d0 100644
--- a/fdbserver/ClusterController.actor.h
+++ b/fdbserver/ClusterController.actor.h
@@ -3104,14 +3104,20 @@ class ClusterControllerData {
 	// Whether the transaction system (in primary DC if in HA setting) contains degraded servers.
 	bool transactionSystemContainsDegradedServers() {
 		const ServerDBInfo& dbi = db.serverInfo->get();
-		auto transactionWorkerInList = [&dbi](const std::unordered_set<NetworkAddress>& serverList) -> bool {
+		auto transactionWorkerInList = [&dbi](const std::unordered_set<NetworkAddress>& serverList,
+		                                      bool skipSatellite) -> bool {
 			for (const auto& server : serverList) {
 				if (dbi.master.addresses().contains(server)) {
 					return true;
 				}
 
 				for (const auto& logSet : dbi.logSystemConfig.tLogs) {
-					if (!logSet.isLocal || logSet.locality == tagLocalitySatellite) {
+					if (!logSet.isLocal) {
+						// We don't check server degradation for remote TLogs since it is not on the transaction system
+						// critical path.
+						continue;
+					}
+					if (skipSatellite && logSet.locality == tagLocalitySatellite) {
 						continue;
 					}
 					for (const auto& tlog : logSet.tLogs) {
@@ -3143,8 +3149,10 @@ class ClusterControllerData {
 			return false;
 		};
 
-		return transactionWorkerInList(degradationInfo.degradedServers) ||
-		       transactionWorkerInList(degradationInfo.disconnectedServers);
+		// Check if transaction system contains degraded/disconnected servers. For satellite, we only check for
+		// disconnection since the latency between prmary and satellite is across WAN and may not be very stable.
+		return transactionWorkerInList(degradationInfo.degradedServers, /*skipSatellite=*/true) ||
+		       transactionWorkerInList(degradationInfo.disconnectedServers, /*skipSatellite=*/false);
 	}
 
 	// Whether transaction system in the remote DC, e.g. log router and tlogs in the remote DC, contains degraded

From 1f1390c913c84507ef6f296d8db87ffa3d03f7f1 Mon Sep 17 00:00:00 2001
From: Markus Pilman <markus.pilman@snowflake.com>
Date: Tue, 20 Sep 2022 16:42:30 -0600
Subject: [PATCH 043/468] Upgrade sphinx and document test harness and code
 probes

---
 documentation/sphinx/conf.py                  |  4 +-
 documentation/sphinx/extensions/rubydomain.py | 38 ++++++------
 documentation/sphinx/requirements.txt         | 10 ++--
 documentation/sphinx/source/api-c.rst         | 28 ++++-----
 documentation/sphinx/source/api-python.rst    | 12 ----
 documentation/sphinx/source/data-modeling.rst |  1 -
 .../sphinx/source/developer-guide.rst         |  1 -
 documentation/sphinx/source/index.rst         |  1 +
 .../sphinx/source/internal-dev-tools.rst      | 58 +++++++++++++++++++
 9 files changed, 99 insertions(+), 54 deletions(-)
 create mode 100644 documentation/sphinx/source/internal-dev-tools.rst

diff --git a/documentation/sphinx/conf.py b/documentation/sphinx/conf.py
index 031c7d6f673..04ed43d87b6 100644
--- a/documentation/sphinx/conf.py
+++ b/documentation/sphinx/conf.py
@@ -69,7 +69,7 @@
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.
-language = None
+language = 'en'
 
 # There are two options for replacing |today|: either, you set today to some
 # non-false value, then it is used:
@@ -185,7 +185,7 @@
 htmlhelp_basename = 'FoundationDB'
 
 # Disable permalinks
-html_add_permalinks = ""
+html_permalinks = False
 
 
 # -- Options for LaTeX output --------------------------------------------------
diff --git a/documentation/sphinx/extensions/rubydomain.py b/documentation/sphinx/extensions/rubydomain.py
index 1e5fb0bce4c..e6c849cc413 100755
--- a/documentation/sphinx/extensions/rubydomain.py
+++ b/documentation/sphinx/extensions/rubydomain.py
@@ -42,7 +42,7 @@
 
 from sphinx import addnodes
 from sphinx.roles import XRefRole
-from sphinx.locale import l_, _
+from sphinx.locale import _
 from sphinx.domains import Domain, ObjType, Index
 from sphinx.directives import ObjectDescription
 from sphinx.util.nodes import make_refnode
@@ -83,18 +83,18 @@ class RubyObject(ObjectDescription):
     }
 
     doc_field_types = [
-        TypedField('parameter', label=l_('Parameters'),
+        TypedField('parameter', label=_('Parameters'),
                    names=('param', 'parameter', 'arg', 'argument'),
                    typerolename='obj', typenames=('paramtype', 'type')),
-        TypedField('variable', label=l_('Variables'), rolename='obj',
+        TypedField('variable', label=_('Variables'), rolename='obj',
                    names=('var', 'ivar', 'cvar'),
                    typerolename='obj', typenames=('vartype',)),
-        GroupedField('exceptions', label=l_('Raises'), rolename='exc',
+        GroupedField('exceptions', label=_('Raises'), rolename='exc',
                      names=('raises', 'raise', 'exception', 'except'),
                      can_collapse=True),
-        Field('returnvalue', label=l_('Returns'), has_arg=False,
+        Field('returnvalue', label=_('Returns'), has_arg=False,
               names=('returns', 'return')),
-        Field('returntype', label=l_('Return type'), has_arg=False,
+        Field('returntype', label=_('Return type'), has_arg=False,
               names=('rtype',)),
     ]
 
@@ -493,8 +493,8 @@ class RubyModuleIndex(Index):
     """
 
     name = 'modindex'
-    localname = l_('Ruby Module Index')
-    shortname = l_('modules')
+    localname = _('Ruby Module Index')
+    shortname = _('modules')
 
     def generate(self, docnames=None):
         content = {}
@@ -561,17 +561,17 @@ class RubyDomain(Domain):
     name = 'rb'
     label = 'Ruby'
     object_types = {
-        'function':        ObjType(l_('function'),         'func', 'obj'),
-        'global':          ObjType(l_('global variable'),  'global', 'obj'),
-        'method':          ObjType(l_('method'),           'meth', 'obj'),
-        'class':           ObjType(l_('class'),            'class', 'obj'),
-        'exception':       ObjType(l_('exception'),        'exc', 'obj'),
-        'classmethod':     ObjType(l_('class method'),     'meth', 'obj'),
-        'attr_reader':     ObjType(l_('attribute'),        'attr', 'obj'),
-        'attr_writer':     ObjType(l_('attribute'),        'attr', 'obj'),
-        'attr_accessor':   ObjType(l_('attribute'),        'attr', 'obj'),
-        'const':           ObjType(l_('const'),            'const', 'obj'),
-        'module':          ObjType(l_('module'),           'mod', 'obj'),
+        'function':        ObjType(_('function'),         'func', 'obj'),
+        'global':          ObjType(_('global variable'),  'global', 'obj'),
+        'method':          ObjType(_('method'),           'meth', 'obj'),
+        'class':           ObjType(_('class'),            'class', 'obj'),
+        'exception':       ObjType(_('exception'),        'exc', 'obj'),
+        'classmethod':     ObjType(_('class method'),     'meth', 'obj'),
+        'attr_reader':     ObjType(_('attribute'),        'attr', 'obj'),
+        'attr_writer':     ObjType(_('attribute'),        'attr', 'obj'),
+        'attr_accessor':   ObjType(_('attribute'),        'attr', 'obj'),
+        'const':           ObjType(_('const'),            'const', 'obj'),
+        'module':          ObjType(_('module'),           'mod', 'obj'),
     }
 
     directives = {
diff --git a/documentation/sphinx/requirements.txt b/documentation/sphinx/requirements.txt
index 06e23ea6d30..8e33b564f2a 100644
--- a/documentation/sphinx/requirements.txt
+++ b/documentation/sphinx/requirements.txt
@@ -1,6 +1,6 @@
 --index-url https://pypi.python.org/simple
-setuptools>=20.10.0,<=57.4.0
-sphinx==1.5.6
-sphinx-bootstrap-theme==0.4.8
-docutils==0.16
-Jinja2==3.0.3
+setuptools==65.3.0
+sphinx==5.1.1
+sphinx-bootstrap-theme==0.8.1
+docutils==0.19
+Jinja2==3.1.2
diff --git a/documentation/sphinx/source/api-c.rst b/documentation/sphinx/source/api-c.rst
index cccb9ee7d07..b896b7bf6d4 100644
--- a/documentation/sphinx/source/api-c.rst
+++ b/documentation/sphinx/source/api-c.rst
@@ -222,7 +222,7 @@ The FoundationDB client library performs most tasks on a singleton thread (which
 Future
 ======
 
-Most functions in the FoundationDB API are asynchronous, meaning that they may return to the caller before actually delivering their result. These functions always return :type:`FDBFuture*`. An :type:`FDBFuture` object represents a result value or error to be delivered at some future time. You can wait for a Future to be "ready" -- to have a value or error delivered -- by setting a callback function, or by blocking a thread, or by polling. Once a Future is ready, you can extract either an error code or a value of the appropriate type (the documentation for the original function will tell you which :func:`fdb_future_get_*()` function you should call).
+Most functions in the FoundationDB API are asynchronous, meaning that they may return to the caller before actually delivering their result. These functions always return ``FDBFuture*``. An :type:`FDBFuture` object represents a result value or error to be delivered at some future time. You can wait for a Future to be "ready" -- to have a value or error delivered -- by setting a callback function, or by blocking a thread, or by polling. Once a Future is ready, you can extract either an error code or a value of the appropriate type (the documentation for the original function will tell you which ``fdb_future_get_()`` function you should call).
 
 To use the API in a synchronous way, you would typically do something like this for each asynchronous call::
 
@@ -282,7 +282,7 @@ See :ref:`developer-guide-programming-with-futures` for further (language-indepe
 
 .. type:: FDBCallback
 
-   A pointer to a function which takes :type:`FDBFuture*` and ``void*`` and returns ``void``.
+   A pointer to a function which takes ``FDBFuture*`` and ``void*`` and returns ``void``.
 
 .. function:: void fdb_future_release_memory(FDBFuture* future)
 
@@ -298,13 +298,13 @@ See :ref:`developer-guide-programming-with-futures` for further (language-indepe
 
 .. function:: fdb_error_t fdb_future_get_int64(FDBFuture* future, int64_t* out)
 
-   Extracts a 64-bit integer from an :type:`FDBFuture*` into a caller-provided variable of type ``int64_t``. |future-warning|
+   Extracts a 64-bit integer from a pointer to :type:`FDBFuture` into a caller-provided variable of type ``int64_t``. |future-warning|
 
    |future-get-return1| |future-get-return2|.
 
 .. function:: fdb_error_t fdb_future_get_key_array( FDBFuture* f, FDBKey const** out_key_array, int* out_count)
 
-   Extracts an array of :type:`FDBKey` from an :type:`FDBFuture*` into a caller-provided variable of type ``FDBKey*``. The size of the array will also be extracted and passed back by a caller-provided variable of type ``int`` |future-warning|
+   Extracts an array of :type:`FDBKey` from an ``FDBFuture*`` into a caller-provided variable of type ``FDBKey*``. The size of the array will also be extracted and passed back by a caller-provided variable of type ``int`` |future-warning|
 
    |future-get-return1| |future-get-return2|.
 
@@ -547,13 +547,13 @@ Applications must provide error handling and an appropriate retry loop around th
 
 .. function:: void fdb_transaction_set_read_version(FDBTransaction* transaction, int64_t version)
 
-   Sets the snapshot read version used by a transaction. This is not needed in simple cases. If the given version is too old, subsequent reads will fail with error_code_transaction_too_old; if it is too new, subsequent reads may be delayed indefinitely and/or fail with error_code_future_version. If any of :func:`fdb_transaction_get_*()` have been called on this transaction already, the result is undefined.
+   Sets the snapshot read version used by a transaction. This is not needed in simple cases. If the given version is too old, subsequent reads will fail with error_code_transaction_too_old; if it is too new, subsequent reads may be delayed indefinitely and/or fail with error_code_future_version. If any of ``fdb_transaction_get_*()`` have been called on this transaction already, the result is undefined.
 
 .. function:: FDBFuture* fdb_transaction_get_read_version(FDBTransaction* transaction)
 
    |future-return0| the transaction snapshot read version. |future-return1| call :func:`fdb_future_get_int64()` to extract the version into an int64_t that you provide, |future-return2|
 
-   The transaction obtains a snapshot read version automatically at the time of the first call to :func:`fdb_transaction_get_*()` (including this one) and (unless causal consistency has been deliberately compromised by transaction options) is guaranteed to represent all transactions which were reported committed before that call.
+   The transaction obtains a snapshot read version automatically at the time of the first call to ``fdb_transaction_get_*()`` (including this one) and (unless causal consistency has been deliberately compromised by transaction options) is guaranteed to represent all transactions which were reported committed before that call.
 
 .. function:: FDBFuture* fdb_transaction_get(FDBTransaction* transaction, uint8_t const* key_name, int key_name_length, fdb_bool_t snapshot)
 
@@ -829,7 +829,7 @@ Applications must provide error handling and an appropriate retry loop around th
 
    |future-returnvoid|
 
-   Callers will usually want to retry a transaction if the commit or a prior :func:`fdb_transaction_get_*()` returns a retryable error (see :func:`fdb_transaction_on_error()`).
+   Callers will usually want to retry a transaction if the commit or a prior ``fdb_transaction_get_*()`` returns a retryable error (see :func:`fdb_transaction_on_error()`).
 
    |commit-unknown-result-blurb|
 
@@ -878,9 +878,9 @@ Applications must provide error handling and an appropriate retry loop around th
 
 .. function:: FDBFuture* fdb_transaction_on_error(FDBTransaction* transaction, fdb_error_t error)
 
-   Implements the recommended retry and backoff behavior for a transaction. This function knows which of the error codes generated by other :func:`fdb_transaction_*()` functions represent temporary error conditions and which represent application errors that should be handled by the application. It also implements an exponential backoff strategy to avoid swamping the database cluster with excessive retries when there is a high level of conflict between transactions.
+   Implements the recommended retry and backoff behavior for a transaction. This function knows which of the error codes generated by other ``fdb_transaction_*()`` functions represent temporary error conditions and which represent application errors that should be handled by the application. It also implements an exponential backoff strategy to avoid swamping the database cluster with excessive retries when there is a high level of conflict between transactions.
 
-   On receiving any type of error from an :func:`fdb_transaction_*()` function, the application should:
+   On receiving any type of error from an ``fdb_transaction_*()`` function, the application should:
 
    1. Call :func:`fdb_transaction_on_error()` with the returned :type:`fdb_error_t` code.
 
@@ -963,15 +963,15 @@ Key selectors
 
 In the FoundationDB C API, key selectors are not represented by a structure of any kind, but are instead expressed as sequential parameters to |get-key-func| and |get-range-func|. For convenience, the most common key selectors are available as C macros that expand to the appropriate parameters.
 
-.. function:: FDB_KEYSEL_LAST_LESS_THAN(key_name, key_name_length)
+.. type:: FDB_KEYSEL_LAST_LESS_THAN(key_name, key_name_length)
 
-.. function:: FDB_KEYSEL_LAST_LESS_OR_EQUAL(key_name, key_name_length)
+.. type:: FDB_KEYSEL_LAST_LESS_OR_EQUAL(key_name, key_name_length)
 
-.. function:: FDB_KEYSEL_FIRST_GREATER_THAN(key_name, key_name_length)
+.. type:: FDB_KEYSEL_FIRST_GREATER_THAN(key_name, key_name_length)
 
-.. function:: FDB_KEYSEL_FIRST_GREATER_OR_EQUAL(key_name, key_name_length)
+.. type:: FDB_KEYSEL_FIRST_GREATER_OR_EQUAL(key_name, key_name_length)
 
-To use one of these macros, simply replace the four parameters in the function with one of :func:`FDB_KEYSEL_*`::
+To use one of these macros, simply replace the four parameters in the function with one of ``FDB_KEYSEL_*``::
 
     future = fdb_transaction_get_key(transaction, "key", 3, 0, 2, 0);
 
diff --git a/documentation/sphinx/source/api-python.rst b/documentation/sphinx/source/api-python.rst
index 0f8c16d6bd2..7cd7a8e295f 100644
--- a/documentation/sphinx/source/api-python.rst
+++ b/documentation/sphinx/source/api-python.rst
@@ -194,10 +194,6 @@ After importing the ``fdb`` module and selecting an API version, you probably wa
 
        |option-tls-key-bytes|
     
-    .. method :: fdb.options.set_tls_verify_peers(verification_pattern)
-
-       |option-tls-verify-peers|
-    
     .. method :: fdb.options.set_tls_ca_bytes(ca_bundle)
 
        |option-tls-ca-bytes|
@@ -210,10 +206,6 @@ After importing the ``fdb`` module and selecting an API version, you probably wa
 
        |option-tls-password|
 
-    .. method :: fdb.options.set_disable_multi_version_client_api()
-
-       |option-disable-multi-version-client-api|
-
     .. method :: fdb.options.set_disable_local_client()
 
       |option-set-disable-local-client|
@@ -761,10 +753,6 @@ In each of the methods below, ``param`` should be a string appropriately packed
 Committing
 ----------
 
-.. decorator:: transactional()
-
-    The ``transactional`` decorator makes it easy to write transactional functions which accept a :class:`Database`, :class`Tenant`, or :class:`Transaction` as a parameter and automatically commit. See :func:`@fdb.transactional <transactional>` for explanation and examples.
-
 .. method :: Transaction.commit()
 
     Attempt to commit the changes made in the transaction to the database.  Returns a :class:`FutureVoid` representing the asynchronous result of the commit. You **must** call the :meth:`Future.wait()` method on the returned :class:`FutureVoid`, which will raise an exception if the commit failed.
diff --git a/documentation/sphinx/source/data-modeling.rst b/documentation/sphinx/source/data-modeling.rst
index 146b006809f..5972c5110a9 100644
--- a/documentation/sphinx/source/data-modeling.rst
+++ b/documentation/sphinx/source/data-modeling.rst
@@ -1,7 +1,6 @@
 .. default-domain:: py
 .. default-domain:: py
 .. highlight:: python
-.. module:: fdb
 
 .. Required substitutions for api-common.rst.inc
 
diff --git a/documentation/sphinx/source/developer-guide.rst b/documentation/sphinx/source/developer-guide.rst
index 3bf5ec30c0c..c889e4909be 100644
--- a/documentation/sphinx/source/developer-guide.rst
+++ b/documentation/sphinx/source/developer-guide.rst
@@ -1,7 +1,6 @@
 .. default-domain:: py
 .. default-domain:: py
 .. highlight:: python
-.. module:: fdb
 
 .. Required substitutions for api-common.rst.inc
 
diff --git a/documentation/sphinx/source/index.rst b/documentation/sphinx/source/index.rst
index 40c7a762797..167bebda43f 100644
--- a/documentation/sphinx/source/index.rst
+++ b/documentation/sphinx/source/index.rst
@@ -50,6 +50,7 @@ The latest changes are detailed in :ref:`release-notes`. The documentation has t
    :hidden:
 
    local-dev
+   internal-dev-tools
    why-foundationdb
    technical-overview
    client-design
diff --git a/documentation/sphinx/source/internal-dev-tools.rst b/documentation/sphinx/source/internal-dev-tools.rst
new file mode 100644
index 00000000000..796162dc5c7
--- /dev/null
+++ b/documentation/sphinx/source/internal-dev-tools.rst
@@ -0,0 +1,58 @@
+##################
+Internal Dev Tools
+##################
+
+Code Probes
+===========
+
+Code probes are a mechanism in FDB to prove that certain code-paths are being tested under the right conditions. They differ from code coverage in multiple ways (explained below).
+
+The general format of a code probe is:
+
+.. code-block:: C++
+
+   CODE_PROBE(<condition>, "Comment", [annotations...]);
+
+A simple example of a code probe could look as follows:
+
+.. code-block:: C++
+
+   CODE_PROBE(self->forceRecovery, "Resolver detects forced recovery", context::sim2);
+
+On a very high level, the above code will indicate that whenever this line is executed and ``self->forceRecovery`` is ``true``, we ran into some interesting case. In addition this probe is also annotated with ``context::sim2``. This indicates that we expect this code to be eventually hit in simulation.
+
+By default, FDB simply will write a trace-line when this code is hit and the condition is ``true``. If the code is never hit, the simulator will, at the end of the run, print the code probe but set the ``covered`` field to ``false``.
+
+We expect that ALL code probes will be hit in a nightly run. In the future we can potentially use this feature for other things (like instructing the simulator to do an extensive search starting when one of these probes is being hit).
+
+In addition to ``context`` annotations, users can also define and pass assertsions. For example:
+
+.. code-block:: C++
+
+   CODE_PROBE(condition, "Some comment", assert::simOnly);
+
+These will add an assertion to the code. In addition to that, the simulator will not print missed code probes that asserted that the probe won't be hit in simulation.
+
+Test Harness
+============
+
+TestHarness is our primary testing tool. It has multiple jobs:
+
+* *Running*: It can run a test in Joshua.
+* *Statictics*: It will chose a test to run based on previously spent CPU time for all tests. It does that by writing statistics about the test at the end of each run.
+* *Reporting*: After an ensemble has finished (or while it is running), ``TestHarness`` can be used to generate a report in ``xml`` or ``json``.
+
+Test Harness can be found in the FDB source repository under ``contrib/TestHarness2``. It has a weak dependency to `joshua <https://github.com/foundationDB/fdb-joshua>`_ (if Test Harness can find joshua it will report back about failed tests, otherwise it will just print out general statistics about the ensemble). Joshua will call Test Harness as follows:
+
+.. code-block:: shell
+
+   python3 -m test_harness.app -s ${JOSHUA_SEED} --old-binaries-path ${OLDBINDIR}
+
+Here the seed is a random number generated by joshua and ``OLDBINDIR`` is a directory path where the old fdb binaries can be found (this is needed for restart tests). If once wants to retry a test they can pass the previous joshua seed, a directory path that has *exactly* the same content as ``OLDBINARYDIR``, plus the reported statistics to the test harness app. This should then rerun the same code as before.
+
+In order to figure out what command line arguments ``test_harness.app`` (and ``test_harness.results``) accepts, one can check the contents of ``contrib/TestHarness2/test_harness/config.py``.
+
+Reporting
+---------
+
+After a joshua ensemble completed, ``test_harness.results`` can be used in order to get a report on the ensemble. This will include, by default, a list of all failed tests (similar to ``joshua tail --errors``, though in a more human readable file). For completed ensemble it will also print code probes that weren't hit often enough. An ensemble is considered to be successful if no simulation runs completed with an error AND all code probes have been hit sufficiently often.

From 056653389b8e766432977802731cb792c85372f4 Mon Sep 17 00:00:00 2001
From: Markus Pilman <markus.pilman@snowflake.com>
Date: Wed, 21 Sep 2022 08:28:01 -0600
Subject: [PATCH 044/468] Apply suggestions from code review

Co-authored-by: Trevor Clinkenbeard <trevor.clinkenbeard@snowflake.com>
Co-authored-by: Bharadwaj V.R <bharadwaj.vr@snowflake.com>
---
 documentation/sphinx/source/internal-dev-tools.rst | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/documentation/sphinx/source/internal-dev-tools.rst b/documentation/sphinx/source/internal-dev-tools.rst
index 796162dc5c7..2a1c5883758 100644
--- a/documentation/sphinx/source/internal-dev-tools.rst
+++ b/documentation/sphinx/source/internal-dev-tools.rst
@@ -17,15 +17,15 @@ A simple example of a code probe could look as follows:
 
 .. code-block:: C++
 
-   CODE_PROBE(self->forceRecovery, "Resolver detects forced recovery", context::sim2);
+   CODE_PROBE(self->forceRecovery, "Resolver detects forced recovery", probe::context::sim2);
 
-On a very high level, the above code will indicate that whenever this line is executed and ``self->forceRecovery`` is ``true``, we ran into some interesting case. In addition this probe is also annotated with ``context::sim2``. This indicates that we expect this code to be eventually hit in simulation.
+On a very high level, the above code will indicate that whenever this line is executed and ``self->forceRecovery`` is ``true``, we ran into some interesting case. In addition this probe is also annotated with ``probe::context::sim2``. This indicates that we expect this code to be eventually hit in simulation.
 
 By default, FDB simply will write a trace-line when this code is hit and the condition is ``true``. If the code is never hit, the simulator will, at the end of the run, print the code probe but set the ``covered`` field to ``false``.
 
 We expect that ALL code probes will be hit in a nightly run. In the future we can potentially use this feature for other things (like instructing the simulator to do an extensive search starting when one of these probes is being hit).
 
-In addition to ``context`` annotations, users can also define and pass assertsions. For example:
+In addition to ``context`` annotations, users can also define and pass assertions. For example:
 
 .. code-block:: C++
 
@@ -39,7 +39,7 @@ Test Harness
 TestHarness is our primary testing tool. It has multiple jobs:
 
 * *Running*: It can run a test in Joshua.
-* *Statictics*: It will chose a test to run based on previously spent CPU time for all tests. It does that by writing statistics about the test at the end of each run.
+* *Statistics*: It will choose a test to run based on previously spent CPU time for all tests. It does that by writing statistics about the test at the end of each run.
 * *Reporting*: After an ensemble has finished (or while it is running), ``TestHarness`` can be used to generate a report in ``xml`` or ``json``.
 
 Test Harness can be found in the FDB source repository under ``contrib/TestHarness2``. It has a weak dependency to `joshua <https://github.com/foundationDB/fdb-joshua>`_ (if Test Harness can find joshua it will report back about failed tests, otherwise it will just print out general statistics about the ensemble). Joshua will call Test Harness as follows:
@@ -48,7 +48,7 @@ Test Harness can be found in the FDB source repository under ``contrib/TestHarne
 
    python3 -m test_harness.app -s ${JOSHUA_SEED} --old-binaries-path ${OLDBINDIR}
 
-Here the seed is a random number generated by joshua and ``OLDBINDIR`` is a directory path where the old fdb binaries can be found (this is needed for restart tests). If once wants to retry a test they can pass the previous joshua seed, a directory path that has *exactly* the same content as ``OLDBINARYDIR``, plus the reported statistics to the test harness app. This should then rerun the same code as before.
+Here the seed is a random number generated by joshua and ``OLDBINDIR`` is a directory path where the old fdb binaries can be found (this is needed for restart tests). If one wants to retry a test they can pass the previous joshua seed, a directory path that has *exactly* the same content as ``OLDBINARYDIR``, plus the reported statistics to the test harness app. This should then re-run the same code as before.
 
 In order to figure out what command line arguments ``test_harness.app`` (and ``test_harness.results``) accepts, one can check the contents of ``contrib/TestHarness2/test_harness/config.py``.
 

From 72e1ca68ed02f673e6b2091a70dd1e580b64b4cb Mon Sep 17 00:00:00 2001
From: Markus Pilman <markus.pilman@snowflake.com>
Date: Wed, 21 Sep 2022 09:27:28 -0600
Subject: [PATCH 045/468] clarify how code probes are reported

---
 documentation/sphinx/source/internal-dev-tools.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/documentation/sphinx/source/internal-dev-tools.rst b/documentation/sphinx/source/internal-dev-tools.rst
index 2a1c5883758..659796b567f 100644
--- a/documentation/sphinx/source/internal-dev-tools.rst
+++ b/documentation/sphinx/source/internal-dev-tools.rst
@@ -21,7 +21,7 @@ A simple example of a code probe could look as follows:
 
 On a very high level, the above code will indicate that whenever this line is executed and ``self->forceRecovery`` is ``true``, we ran into some interesting case. In addition this probe is also annotated with ``probe::context::sim2``. This indicates that we expect this code to be eventually hit in simulation.
 
-By default, FDB simply will write a trace-line when this code is hit and the condition is ``true``. If the code is never hit, the simulator will, at the end of the run, print the code probe but set the ``covered`` field to ``false``.
+By default, FDB simply will write a trace-line when this code is hit and the condition is ``true``. If the code is never hit, the simulator will, at the end of the run, print the code probe but set the ``covered`` field to ``false``. This all happens in the context of a single simulation run (``fdbserver`` doesn't have a concept of ensembles). This information is written into the log file. ``TestHarness`` (see below) will then use this information to write code probe statistics to the ensemble in the Joshua cluster (if the test is run in Joshua).
 
 We expect that ALL code probes will be hit in a nightly run. In the future we can potentially use this feature for other things (like instructing the simulator to do an extensive search starting when one of these probes is being hit).
 

From 5218535f20a7af2a6a0d5f55347babe1a561b6f1 Mon Sep 17 00:00:00 2001
From: Markus Pilman <markus.pilman@snowflake.com>
Date: Wed, 21 Sep 2022 09:29:49 -0600
Subject: [PATCH 046/468] clarify statistics of TestHarness

---
 documentation/sphinx/source/internal-dev-tools.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/documentation/sphinx/source/internal-dev-tools.rst b/documentation/sphinx/source/internal-dev-tools.rst
index 659796b567f..ea809473126 100644
--- a/documentation/sphinx/source/internal-dev-tools.rst
+++ b/documentation/sphinx/source/internal-dev-tools.rst
@@ -39,7 +39,7 @@ Test Harness
 TestHarness is our primary testing tool. It has multiple jobs:
 
 * *Running*: It can run a test in Joshua.
-* *Statistics*: It will choose a test to run based on previously spent CPU time for all tests. It does that by writing statistics about the test at the end of each run.
+* *Statistics*: It will choose a test to run based on previous runs (within the same ensemble) spent CPU time for each test. It does that by writing statistics about the test at the end of each run.
 * *Reporting*: After an ensemble has finished (or while it is running), ``TestHarness`` can be used to generate a report in ``xml`` or ``json``.
 
 Test Harness can be found in the FDB source repository under ``contrib/TestHarness2``. It has a weak dependency to `joshua <https://github.com/foundationDB/fdb-joshua>`_ (if Test Harness can find joshua it will report back about failed tests, otherwise it will just print out general statistics about the ensemble). Joshua will call Test Harness as follows:

From ddd85efdebac0eae15cdd9fb5f357d3cc74e8afc Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Tue, 27 Dec 2022 17:17:08 +0000
Subject: [PATCH 047/468] Bump setuptools from 65.3.0 to 65.5.1 in
 /documentation/sphinx

Bumps [setuptools](https://github.com/pypa/setuptools) from 65.3.0 to 65.5.1.
- [Release notes](https://github.com/pypa/setuptools/releases)
- [Changelog](https://github.com/pypa/setuptools/blob/main/CHANGES.rst)
- [Commits](https://github.com/pypa/setuptools/compare/v65.3.0...v65.5.1)

---
updated-dependencies:
- dependency-name: setuptools
  dependency-type: direct:production
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 documentation/sphinx/requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/documentation/sphinx/requirements.txt b/documentation/sphinx/requirements.txt
index 8e33b564f2a..6742ef130ff 100644
--- a/documentation/sphinx/requirements.txt
+++ b/documentation/sphinx/requirements.txt
@@ -1,5 +1,5 @@
 --index-url https://pypi.python.org/simple
-setuptools==65.3.0
+setuptools==65.5.1
 sphinx==5.1.1
 sphinx-bootstrap-theme==0.8.1
 docutils==0.19

From 91f6252e23319fcc3fbd0ab63cc3bde9c3a9b441 Mon Sep 17 00:00:00 2001
From: Zhe Wu <halfprice@users.noreply.github.com>
Date: Tue, 10 Jan 2023 21:35:37 -0800
Subject: [PATCH 048/468] Add event for txn server initialization and a warning
 for TLog slow catching up

---
 fdbclient/ServerKnobs.cpp                   | 1 +
 fdbclient/ServerKnobs.h                     | 1 +
 fdbserver/ClusterRecovery.actor.cpp         | 3 +++
 fdbserver/TLogServer.actor.cpp              | 6 ++++++
 fdbserver/TagPartitionedLogSystem.actor.cpp | 2 ++
 5 files changed, 13 insertions(+)

diff --git a/fdbclient/ServerKnobs.cpp b/fdbclient/ServerKnobs.cpp
index 962e594ebeb..9ee0b47725f 100644
--- a/fdbclient/ServerKnobs.cpp
+++ b/fdbclient/ServerKnobs.cpp
@@ -117,6 +117,7 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi
 	init( PEEK_BATCHING_EMPTY_MSG,                             false ); if ( randomize && BUGGIFY ) PEEK_BATCHING_EMPTY_MSG = true;
 	init( PEEK_BATCHING_EMPTY_MSG_INTERVAL,                    0.001 ); if ( randomize && BUGGIFY ) PEEK_BATCHING_EMPTY_MSG_INTERVAL = 0.01;
 	init( POP_FROM_LOG_DELAY,                                      1 ); if ( randomize && BUGGIFY ) POP_FROM_LOG_DELAY = 0;
+	init( TLOG_PULL_ASYNC_DATA_WARNING_TIMEOUT_SECS,             120 );
 
 	// disk snapshot max timeout, to be put in TLog, storage and coordinator nodes
 	init( MAX_FORKED_PROCESS_OUTPUT,                            1024 );
diff --git a/fdbclient/ServerKnobs.h b/fdbclient/ServerKnobs.h
index 1a323fc5e65..a29dee282b6 100644
--- a/fdbclient/ServerKnobs.h
+++ b/fdbclient/ServerKnobs.h
@@ -116,6 +116,7 @@ class ServerKnobs : public KnobsImpl<ServerKnobs> {
 	bool PEEK_BATCHING_EMPTY_MSG;
 	double PEEK_BATCHING_EMPTY_MSG_INTERVAL;
 	double POP_FROM_LOG_DELAY;
+	double TLOG_PULL_ASYNC_DATA_WARNING_TIMEOUT_SECS;
 
 	// Data distribution queue
 	double HEALTH_POLL_TIME;
diff --git a/fdbserver/ClusterRecovery.actor.cpp b/fdbserver/ClusterRecovery.actor.cpp
index 3b5f73b67f1..4fa2c93d453 100644
--- a/fdbserver/ClusterRecovery.actor.cpp
+++ b/fdbserver/ClusterRecovery.actor.cpp
@@ -199,6 +199,7 @@ ACTOR Future<Void> newCommitProxies(Reference<ClusterRecoveryData> self, Recruit
 	}
 
 	std::vector<CommitProxyInterface> newRecruits = wait(getAll(initializationReplies));
+	TraceEvent("CommitProxyInitializationComplete", self->dbgid).log();
 	// It is required for the correctness of COMMIT_ON_FIRST_PROXY that self->commitProxies[0] is the firstCommitProxy.
 	self->commitProxies = newRecruits;
 
@@ -220,6 +221,7 @@ ACTOR Future<Void> newGrvProxies(Reference<ClusterRecoveryData> self, RecruitFro
 	}
 
 	std::vector<GrvProxyInterface> newRecruits = wait(getAll(initializationReplies));
+	TraceEvent("GrvProxyInitializationComplete", self->dbgid).log();
 	self->grvProxies = newRecruits;
 	return Void();
 }
@@ -240,6 +242,7 @@ ACTOR Future<Void> newResolvers(Reference<ClusterRecoveryData> self, RecruitFrom
 	}
 
 	std::vector<ResolverInterface> newRecruits = wait(getAll(initializationReplies));
+	TraceEvent("ResolverInitializationComplete", self->dbgid).log();
 	self->resolvers = newRecruits;
 
 	return Void();
diff --git a/fdbserver/TLogServer.actor.cpp b/fdbserver/TLogServer.actor.cpp
index 74ed6ff8e22..81c2e34d985 100644
--- a/fdbserver/TLogServer.actor.cpp
+++ b/fdbserver/TLogServer.actor.cpp
@@ -2837,6 +2837,7 @@ ACTOR Future<Void> pullAsyncData(TLogData* self,
 	state Reference<ILogSystem::IPeekCursor> r;
 	state Version tagAt = beginVersion;
 	state Version lastVer = 0;
+	state double startTime = now();
 
 	if (endVersion.present()) {
 		TraceEvent("TLogRestoreReplicationFactor", self->dbgid)
@@ -2860,6 +2861,11 @@ ACTOR Future<Void> pullAsyncData(TLogData* self,
 					}
 					dbInfoChange = logData->logSystem->onChange();
 				}
+				when(wait(delay(SERVER_KNOBS->TLOG_PULL_ASYNC_DATA_WARNING_TIMEOUT_SECS))) {
+					TraceEvent(SevWarn, "TLogPullAsyncDataSlow", logData->logId)
+					    .detail("Elapsed", now() - startTime)
+					    .detail("Version", logData->version.get());
+				}
 			}
 		}
 
diff --git a/fdbserver/TagPartitionedLogSystem.actor.cpp b/fdbserver/TagPartitionedLogSystem.actor.cpp
index bb132750fa4..72612356970 100644
--- a/fdbserver/TagPartitionedLogSystem.actor.cpp
+++ b/fdbserver/TagPartitionedLogSystem.actor.cpp
@@ -3049,6 +3049,7 @@ ACTOR Future<Reference<ILogSystem>> TagPartitionedLogSystem::newEpoch(
 		}
 
 		wait(waitForAll(satelliteInitializationReplies) || oldRouterRecruitment);
+		TraceEvent("PrimarySatelliteTLogInitializationComplete").log();
 
 		for (int i = 0; i < satelliteInitializationReplies.size(); i++) {
 			logSystem->tLogs[1]->logServers[i] = makeReference<AsyncVar<OptionalInterface<TLogInterface>>>(
@@ -3066,6 +3067,7 @@ ACTOR Future<Reference<ILogSystem>> TagPartitionedLogSystem::newEpoch(
 	}
 
 	wait(waitForAll(initializationReplies) || oldRouterRecruitment);
+	TraceEvent("PrimaryTLogInitializationComplete", logSystem->getDebugID()).log();
 
 	for (int i = 0; i < initializationReplies.size(); i++) {
 		logSystem->tLogs[0]->logServers[i] = makeReference<AsyncVar<OptionalInterface<TLogInterface>>>(

From 9268e1903258aa74713f9435f94b9f2c303079fb Mon Sep 17 00:00:00 2001
From: Zhe Wu <halfprice@users.noreply.github.com>
Date: Wed, 11 Jan 2023 19:36:00 -0800
Subject: [PATCH 049/468] Change TLog pull async data warning timeout

---
 fdbserver/TLogServer.actor.cpp | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/fdbserver/TLogServer.actor.cpp b/fdbserver/TLogServer.actor.cpp
index 81c2e34d985..757d81d67f9 100644
--- a/fdbserver/TLogServer.actor.cpp
+++ b/fdbserver/TLogServer.actor.cpp
@@ -2848,6 +2848,8 @@ ACTOR Future<Void> pullAsyncData(TLogData* self,
 	}
 
 	while (!endVersion.present() || logData->version.get() < endVersion.get()) {
+		// When we just processed some data, we reset the warning start time.
+		state double lastPullAsyncDataWarningTime = now();
 		loop {
 			choose {
 				when(wait(r ? r->getMore(TaskPriority::TLogCommit) : Never())) {
@@ -2861,10 +2863,12 @@ ACTOR Future<Void> pullAsyncData(TLogData* self,
 					}
 					dbInfoChange = logData->logSystem->onChange();
 				}
-				when(wait(delay(SERVER_KNOBS->TLOG_PULL_ASYNC_DATA_WARNING_TIMEOUT_SECS))) {
+				when(wait(delay(lastPullAsyncDataWarningTime + SERVER_KNOBS->TLOG_PULL_ASYNC_DATA_WARNING_TIMEOUT_SECS -
+				                now()))) {
 					TraceEvent(SevWarn, "TLogPullAsyncDataSlow", logData->logId)
 					    .detail("Elapsed", now() - startTime)
 					    .detail("Version", logData->version.get());
+					lastPullAsyncDataWarningTime = now();
 				}
 			}
 		}

From 89a4555727da4718e6ca4a635a69da4b4a14b44d Mon Sep 17 00:00:00 2001
From: neethuhaneesha <nbingi@apple.com>
Date: Wed, 18 Jan 2023 08:34:33 -0800
Subject: [PATCH 050/468] Adding rocksDB control compaction on deletion knobs.
 (#9165)

---
 fdbclient/ServerKnobs.cpp                |  6 ++++++
 fdbclient/ServerKnobs.h                  |  4 ++++
 fdbserver/KeyValueStoreRocksDB.actor.cpp | 19 ++++++++++++++++++-
 3 files changed, 28 insertions(+), 1 deletion(-)

diff --git a/fdbclient/ServerKnobs.cpp b/fdbclient/ServerKnobs.cpp
index 9ee0b47725f..76816e36ace 100644
--- a/fdbclient/ServerKnobs.cpp
+++ b/fdbclient/ServerKnobs.cpp
@@ -417,6 +417,12 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi
 	// ROCKSDB_STATS_LEVEL=1 indicates rocksdb::StatsLevel::kExceptHistogramOrTimers
 	// Refer StatsLevel: https://github.com/facebook/rocksdb/blob/main/include/rocksdb/statistics.h#L594
 	init( ROCKSDB_STATS_LEVEL,                                     1 ); if( randomize && BUGGIFY ) ROCKSDB_STATS_LEVEL = deterministicRandom()->randomInt(0, 6);
+	init( ROCKSDB_ENABLE_COMPACT_ON_DELETION,                   true );
+	// CDCF: CompactOnDeletionCollectorFactory. The below 3 are parameters of the CompactOnDeletionCollectorFactory
+	// which controls the compaction on deleted data.
+	init( ROCKSDB_CDCF_SILIDING_WINDOW_SIZE,                     128 );
+	init( ROCKSDB_CDCF_DELETION_TRIGGER,                           1 );
+	init( ROCKSDB_CDCF_DELETION_RATIO,                             0 );
 	// Can commit will delay ROCKSDB_CAN_COMMIT_DELAY_ON_OVERLOAD seconds for
 	// ROCKSDB_CAN_COMMIT_DELAY_TIMES_ON_OVERLOAD times, if rocksdb overloaded.
 	// Set ROCKSDB_CAN_COMMIT_DELAY_TIMES_ON_OVERLOAD to 0, to disable
diff --git a/fdbclient/ServerKnobs.h b/fdbclient/ServerKnobs.h
index a29dee282b6..9d660443562 100644
--- a/fdbclient/ServerKnobs.h
+++ b/fdbclient/ServerKnobs.h
@@ -338,6 +338,10 @@ class ServerKnobs : public KnobsImpl<ServerKnobs> {
 	bool ROCKSDB_SINGLEKEY_DELETES_ON_CLEARRANGE;
 	int64_t ROCKSDB_SINGLEKEY_DELETES_BYTES_LIMIT;
 	bool ROCKSDB_ENABLE_CLEAR_RANGE_EAGER_READS;
+	bool ROCKSDB_ENABLE_COMPACT_ON_DELETION;
+	int64_t ROCKSDB_CDCF_SILIDING_WINDOW_SIZE; // CDCF: CompactOnDeletionCollectorFactory
+	int64_t ROCKSDB_CDCF_DELETION_TRIGGER; // CDCF: CompactOnDeletionCollectorFactory
+	double ROCKSDB_CDCF_DELETION_RATIO; // CDCF: CompactOnDeletionCollectorFactory
 	int ROCKSDB_STATS_LEVEL;
 	int64_t ROCKSDB_COMPACTION_READAHEAD_SIZE;
 	int64_t ROCKSDB_BLOCK_SIZE;
diff --git a/fdbserver/KeyValueStoreRocksDB.actor.cpp b/fdbserver/KeyValueStoreRocksDB.actor.cpp
index 1deff5a888b..0b72f69b60d 100644
--- a/fdbserver/KeyValueStoreRocksDB.actor.cpp
+++ b/fdbserver/KeyValueStoreRocksDB.actor.cpp
@@ -312,7 +312,24 @@ rocksdb::ColumnFamilyOptions getCFOptions() {
 		options.hard_pending_compaction_bytes_limit = SERVER_KNOBS->ROCKSDB_HARD_PENDING_COMPACT_BYTES_LIMIT;
 	}
 	// Compact sstables when there's too much deleted stuff.
-	options.table_properties_collector_factories = { rocksdb::NewCompactOnDeletionCollectorFactory(128, 1) };
+	if (SERVER_KNOBS->ROCKSDB_ENABLE_COMPACT_ON_DELETION) {
+		// Creates a factory of a table property collector that marks a SST
+		// file as need-compaction when it observe at least "D" deletion
+		// entries in any "N" consecutive entries, or the ratio of tombstone
+		// entries >= deletion_ratio.
+
+		// @param sliding_window_size "N". Note that this number will be
+		//     round up to the smallest multiple of 128 that is no less
+		//     than the specified size.
+		// @param deletion_trigger "D".  Note that even when "N" is changed,
+		//     the specified number for "D" will not be changed.
+		// @param deletion_ratio, if <= 0 or > 1, disable triggering compaction
+		//     based on deletion ratio. Disabled by default.
+		options.table_properties_collector_factories = { rocksdb::NewCompactOnDeletionCollectorFactory(
+			SERVER_KNOBS->ROCKSDB_CDCF_SILIDING_WINDOW_SIZE,
+			SERVER_KNOBS->ROCKSDB_CDCF_DELETION_TRIGGER,
+			SERVER_KNOBS->ROCKSDB_CDCF_DELETION_RATIO) };
+	}
 
 	rocksdb::BlockBasedTableOptions bbOpts;
 	// TODO: Add a knob for the block cache size. (Default is 8 MB)

From 3df4a40dc1049326979cb120e5c2ab7899a97c23 Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyu_zhou@apple.com>
Date: Fri, 20 Jan 2023 08:41:48 -0800
Subject: [PATCH 051/468] Add 7.1.26, 7.1.27 release notes (#9186)

---
 .../release-notes/release-notes-710.rst       | 24 +++++++++++++++++++
 1 file changed, 24 insertions(+)

diff --git a/documentation/sphinx/source/release-notes/release-notes-710.rst b/documentation/sphinx/source/release-notes/release-notes-710.rst
index 2df927160c5..8296635c234 100644
--- a/documentation/sphinx/source/release-notes/release-notes-710.rst
+++ b/documentation/sphinx/source/release-notes/release-notes-710.rst
@@ -4,6 +4,30 @@
 Release Notes
 #############
 
+7.1.27
+======
+* Same as 7.1.26 release with AVX enabled.
+
+7.1.26
+======
+* Released with AVX disabled.
+* Added detection of disconnection to satellite TLog in gray failure detection. `(PR #9107) <https://github.com/apple/foundationdb/pull/9107>`_
+* Fixed (non)empty peeks stats in TLogMetrics. `(PR #9074) <https://github.com/apple/foundationdb/pull/9074>`_
+* Fixed a data distribution bug where exclusions can become stuck because DD cannot build new teams. `(PR #9035) <https://github.com/apple/foundationdb/pull/9035>`_
+* Added FoundationDB version to ProcessMetrics. `(PR #9037) <https://github.com/apple/foundationdb/pull/9037>`_
+* Removed RocksDB read iterator destruction from the commit path. `(PR #8971) <https://github.com/apple/foundationdb/pull/8971>`_
+* Added determinstic degraded server selection in gray failure detection. `(PR #9001) <https://github.com/apple/foundationdb/pull/9001>`_
+* Fixed an interger overflow bug that causes fetching backup files to fail. `(PR #8996) <https://github.com/apple/foundationdb/pull/8996>`_
+* Fixed a log router race condition that blocks remote tlogs forever. `(PR #8966) <https://github.com/apple/foundationdb/pull/8966>`_
+* Fixed a backup worker assertion failure. `(PR #8887) <https://github.com/apple/foundationdb/pull/8887>`_
+* Upgraded RocksDB to 7.7.3 version. `(PR #8880) <https://github.com/apple/foundationdb/pull/8880>`_
+* Added byte limit for index prefetch. `(PR #8802) <https://github.com/apple/foundationdb/pull/8802>`_
+* Added storage server read range bytes metrics. `(PR #8724) <https://github.com/apple/foundationdb/pull/8724>`_
+* Added counters for single key clear requests. `(PR #8792) <https://github.com/apple/foundationdb/pull/8792>`_
+* Added more RocksDB knobs. `(PR #8713) <https://github.com/apple/foundationdb/pull/8713>`_, `(PR #8862) <https://github.com/apple/foundationdb/pull/8862>`_, and `(PR #9165) <https://github.com/apple/foundationdb/pull/9165>`_
+* Added a new network option "retain_client_library_copies" to keep the client library copies. `(PR #8740) <https://github.com/apple/foundationdb/pull/8740>`_
+* Fixed a transaction_too_old error on storage servers when version vector is enabled. `(PR #8710) <https://github.com/apple/foundationdb/pull/8710>`_
+
 7.1.25
 ======
 * Same as 7.1.24 release with AVX enabled.

From 83b235df19a1834e30138631f5bc190f25e794b7 Mon Sep 17 00:00:00 2001
From: neethuhaneesha <nbingi@apple.com>
Date: Fri, 20 Jan 2023 16:38:41 -0800
Subject: [PATCH 052/468] Added metrics for read range operations.

---
 fdbserver/KeyValueStoreRocksDB.actor.cpp | 74 ++++++++++++++++--------
 fdbserver/StorageMetrics.actor.h         |  1 +
 fdbserver/storageserver.actor.cpp        | 55 +++++++++++++-----
 3 files changed, 90 insertions(+), 40 deletions(-)

diff --git a/fdbserver/KeyValueStoreRocksDB.actor.cpp b/fdbserver/KeyValueStoreRocksDB.actor.cpp
index 0b72f69b60d..29b260bd686 100644
--- a/fdbserver/KeyValueStoreRocksDB.actor.cpp
+++ b/fdbserver/KeyValueStoreRocksDB.actor.cpp
@@ -97,6 +97,8 @@ const StringRef ROCKSDB_READPREFIX_QUEUEWAIT_HISTOGRAM = LiteralStringRef("Rocks
 const StringRef ROCKSDB_READRANGE_NEWITERATOR_HISTOGRAM = LiteralStringRef("RocksDBReadRangeNewIterator");
 const StringRef ROCKSDB_READVALUE_GET_HISTOGRAM = LiteralStringRef("RocksDBReadValueGet");
 const StringRef ROCKSDB_READPREFIX_GET_HISTOGRAM = LiteralStringRef("RocksDBReadPrefixGet");
+const StringRef ROCKSDB_READ_RANGE_BYTES_RETURNED_HISTOGRAM = LiteralStringRef("RocksDBReadRangeBytesReturned");
+const StringRef ROCKSDB_READ_RANGE_KV_PAIRS_RETURNED_HISTOGRAM = LiteralStringRef("RocksDBReadRangeKVPairsReturned");
 
 std::shared_ptr<rocksdb::Cache> rocksdb_block_cache = nullptr;
 
@@ -408,12 +410,14 @@ struct Counters {
 	Counter deleteRangeReqs;
 	Counter convertedDeleteKeyReqs;
 	Counter convertedDeleteRangeReqs;
+	Counter rocksdbReadRangeQueries;
 
 	Counters()
 	  : cc("RocksDBThrottle"), immediateThrottle("ImmediateThrottle", cc), failedToAcquire("FailedToAcquire", cc),
 	    deleteKeyReqs("DeleteKeyRequests", cc), deleteRangeReqs("DeleteRangeRequests", cc),
 	    convertedDeleteKeyReqs("ConvertedDeleteKeyRequests", cc),
-	    convertedDeleteRangeReqs("ConvertedDeleteRangeRequests", cc) {}
+	    convertedDeleteRangeReqs("ConvertedDeleteRangeRequests", cc),
+	    rocksdbReadRangeQueries("RocksdbReadRangeQueries", cc) {}
 };
 
 struct ReadIterator {
@@ -1717,14 +1721,15 @@ struct RocksDBKeyValueStore : IKeyValueStore {
 			double startTime;
 			bool getHistograms;
 			ThreadReturnPromise<RangeResult> result;
-			ReadRangeAction(KeyRange keys, int rowLimit, int byteLimit)
-			  : keys(keys), rowLimit(rowLimit), byteLimit(byteLimit), startTime(timer_monotonic()),
-			    getHistograms(
-			        (deterministicRandom()->random01() < SERVER_KNOBS->ROCKSDB_HISTOGRAMS_SAMPLE_RATE) ? true : false),
+			Counters& counters;
+			ReadRangeAction(KeyRange keys, int rowLimit, int byteLimit, Counters& counters)
+			  : keys(keys), rowLimit(rowLimit), byteLimit(byteLimit), startTime(timer_monotonic()), counters(counters),
+			    getHistograms(deterministicRandom()->random01() < SERVER_KNOBS->ROCKSDB_HISTOGRAMS_SAMPLE_RATE),
 			    result(static_cast<TaskPriority>(SERVER_KNOBS->ROCKSDB_THREAD_PROMISE_PRIORITY)) {}
 			double getTimeEstimate() const override { return SERVER_KNOBS->READ_RANGE_TIME_ESTIMATE; }
 		};
 		void action(ReadRangeAction& a) {
+			++a.counters.rocksdbReadRangeQueries;
 			bool doPerfContextMetrics =
 			    SERVER_KNOBS->ROCKSDB_PERFCONTEXT_ENABLE &&
 			    (deterministicRandom()->random01() < SERVER_KNOBS->ROCKSDB_PERFCONTEXT_SAMPLE_RATE);
@@ -1828,6 +1833,13 @@ struct RocksDBKeyValueStore : IKeyValueStore {
 				result.readThrough = result[result.size() - 1].key;
 			}
 			a.result.send(result);
+			// Temporarily not sampling to understand the pattern of readRange results.
+			if (SERVER_KNOBS->ROCKSDB_HISTOGRAMS_SAMPLE_RATE > 0) {
+				metricPromiseStream.send(std::make_tuple(
+				    threadIndex, ROCKSDB_READ_RANGE_BYTES_RETURNED_HISTOGRAM.toString(), result.logicalSize()));
+				metricPromiseStream.send(std::make_tuple(
+				    threadIndex, ROCKSDB_READ_RANGE_KV_PAIRS_RETURNED_HISTOGRAM.toString(), result.size()));
+			}
 			const double endTime = timer_monotonic();
 			if (a.getHistograms) {
 				metricPromiseStream.send(std::make_tuple(
@@ -1948,45 +1960,57 @@ struct RocksDBKeyValueStore : IKeyValueStore {
 		    ROCKSDBSTORAGE_HISTOGRAM_GROUP, ROCKSDB_READVALUE_GET_HISTOGRAM, Histogram::Unit::microseconds);
 		state Reference<Histogram> readPrefixGetHistogram = Histogram::getHistogram(
 		    ROCKSDBSTORAGE_HISTOGRAM_GROUP, ROCKSDB_READPREFIX_GET_HISTOGRAM, Histogram::Unit::microseconds);
+		state Reference<Histogram> rocksdbReadRangeBytesReturnedHistogram = Histogram::getHistogram(
+		    ROCKSDBSTORAGE_HISTOGRAM_GROUP, ROCKSDB_READ_RANGE_BYTES_RETURNED_HISTOGRAM, Histogram::Unit::bytes);
+		state Reference<Histogram> rocksdbReadRangeKVPairsReturnedHistogram =
+		    Histogram::getHistogram(ROCKSDBSTORAGE_HISTOGRAM_GROUP,
+		                            ROCKSDB_READ_RANGE_KV_PAIRS_RETURNED_HISTOGRAM,
+		                            Histogram::Unit::countLinear);
 		loop {
 			choose {
 				when(std::tuple<int, std::string, double> measure = waitNext(metricFutureStream)) {
 					std::string metricName = std::get<1>(measure);
-					double latency = std::get<2>(measure);
+					double metricValue = std::get<2>(measure);
 					if (metricName == ROCKSDB_COMMIT_LATENCY_HISTOGRAM.toString()) {
-						commitLatencyHistogram->sampleSeconds(latency);
+						commitLatencyHistogram->sampleSeconds(metricValue);
 					} else if (metricName == ROCKSDB_COMMIT_ACTION_HISTOGRAM.toString()) {
-						commitActionHistogram->sampleSeconds(latency);
+						commitActionHistogram->sampleSeconds(metricValue);
 					} else if (metricName == ROCKSDB_COMMIT_QUEUEWAIT_HISTOGRAM.toString()) {
-						commitQueueWaitHistogram->sampleSeconds(latency);
+						commitQueueWaitHistogram->sampleSeconds(metricValue);
 					} else if (metricName == ROCKSDB_WRITE_HISTOGRAM.toString()) {
-						writeHistogram->sampleSeconds(latency);
+						writeHistogram->sampleSeconds(metricValue);
 					} else if (metricName == ROCKSDB_DELETE_COMPACTRANGE_HISTOGRAM.toString()) {
-						deleteCompactRangeHistogram->sampleSeconds(latency);
+						deleteCompactRangeHistogram->sampleSeconds(metricValue);
 					} else if (metricName == ROCKSDB_READRANGE_LATENCY_HISTOGRAM.toString()) {
-						readRangeLatencyHistogram->sampleSeconds(latency);
+						readRangeLatencyHistogram->sampleSeconds(metricValue);
 					} else if (metricName == ROCKSDB_READVALUE_LATENCY_HISTOGRAM.toString()) {
-						readValueLatencyHistogram->sampleSeconds(latency);
+						readValueLatencyHistogram->sampleSeconds(metricValue);
 					} else if (metricName == ROCKSDB_READPREFIX_LATENCY_HISTOGRAM.toString()) {
-						readPrefixLatencyHistogram->sampleSeconds(latency);
+						readPrefixLatencyHistogram->sampleSeconds(metricValue);
 					} else if (metricName == ROCKSDB_READRANGE_ACTION_HISTOGRAM.toString()) {
-						readRangeActionHistogram->sampleSeconds(latency);
+						readRangeActionHistogram->sampleSeconds(metricValue);
 					} else if (metricName == ROCKSDB_READVALUE_ACTION_HISTOGRAM.toString()) {
-						readValueActionHistogram->sampleSeconds(latency);
+						readValueActionHistogram->sampleSeconds(metricValue);
 					} else if (metricName == ROCKSDB_READPREFIX_ACTION_HISTOGRAM.toString()) {
-						readPrefixActionHistogram->sampleSeconds(latency);
+						readPrefixActionHistogram->sampleSeconds(metricValue);
 					} else if (metricName == ROCKSDB_READRANGE_QUEUEWAIT_HISTOGRAM.toString()) {
-						readRangeQueueWaitHistogram->sampleSeconds(latency);
+						readRangeQueueWaitHistogram->sampleSeconds(metricValue);
 					} else if (metricName == ROCKSDB_READVALUE_QUEUEWAIT_HISTOGRAM.toString()) {
-						readValueQueueWaitHistogram->sampleSeconds(latency);
+						readValueQueueWaitHistogram->sampleSeconds(metricValue);
 					} else if (metricName == ROCKSDB_READPREFIX_QUEUEWAIT_HISTOGRAM.toString()) {
-						readPrefixQueueWaitHistogram->sampleSeconds(latency);
+						readPrefixQueueWaitHistogram->sampleSeconds(metricValue);
 					} else if (metricName == ROCKSDB_READRANGE_NEWITERATOR_HISTOGRAM.toString()) {
-						readRangeNewIteratorHistogram->sampleSeconds(latency);
+						readRangeNewIteratorHistogram->sampleSeconds(metricValue);
 					} else if (metricName == ROCKSDB_READVALUE_GET_HISTOGRAM.toString()) {
-						readValueGetHistogram->sampleSeconds(latency);
+						readValueGetHistogram->sampleSeconds(metricValue);
 					} else if (metricName == ROCKSDB_READPREFIX_GET_HISTOGRAM.toString()) {
-						readPrefixGetHistogram->sampleSeconds(latency);
+						readPrefixGetHistogram->sampleSeconds(metricValue);
+					} else if (metricName == ROCKSDB_READ_RANGE_BYTES_RETURNED_HISTOGRAM.toString()) {
+						rocksdbReadRangeBytesReturnedHistogram->sample(metricValue);
+					} else if (metricName == ROCKSDB_READ_RANGE_KV_PAIRS_RETURNED_HISTOGRAM.toString()) {
+						rocksdbReadRangeKVPairsReturnedHistogram->sampleRecordCounter(metricValue);
+					} else {
+						UNREACHABLE();
 					}
 				}
 			}
@@ -2211,7 +2235,7 @@ struct RocksDBKeyValueStore : IKeyValueStore {
 	                              int byteLimit,
 	                              IKeyValueStore::ReadType type) override {
 		if (!shouldThrottle(type, keys.begin)) {
-			auto a = new Reader::ReadRangeAction(keys, rowLimit, byteLimit);
+			auto a = new Reader::ReadRangeAction(keys, rowLimit, byteLimit, counters);
 			auto res = a->result.getFuture();
 			readThreads->post(a);
 			return res;
@@ -2221,7 +2245,7 @@ struct RocksDBKeyValueStore : IKeyValueStore {
 		int maxWaiters = (type == IKeyValueStore::ReadType::FETCH) ? numFetchWaiters : numReadWaiters;
 
 		checkWaiters(semaphore, maxWaiters);
-		auto a = std::make_unique<Reader::ReadRangeAction>(keys, rowLimit, byteLimit);
+		auto a = std::make_unique<Reader::ReadRangeAction>(keys, rowLimit, byteLimit, counters);
 		return read(a.release(), &semaphore, readThreads.getPtr(), &counters.failedToAcquire);
 	}
 
diff --git a/fdbserver/StorageMetrics.actor.h b/fdbserver/StorageMetrics.actor.h
index c9df24b53b6..b30337fbcea 100644
--- a/fdbserver/StorageMetrics.actor.h
+++ b/fdbserver/StorageMetrics.actor.h
@@ -41,6 +41,7 @@ const StringRef STORAGE_COMMIT_LATENCY_HISTOGRAM = LiteralStringRef("StorageComm
 const StringRef SS_DURABLE_VERSION_UPDATE_LATENCY_HISTOGRAM = LiteralStringRef("SSDurableVersionUpdateLatency");
 const StringRef SS_READ_RANGE_BYTES_RETURNED_HISTOGRAM = LiteralStringRef("SSReadRangeBytesReturned");
 const StringRef SS_READ_RANGE_BYTES_LIMIT_HISTOGRAM = LiteralStringRef("SSReadRangeBytesLimit");
+const StringRef SS_READ_RANGE_KV_PAIRS_RETURNED_HISTOGRAM = LiteralStringRef("SSReadRangeKVPairsReturned");
 
 struct StorageMetricSample {
 	IndexedSet<Key, int64_t> sample;
diff --git a/fdbserver/storageserver.actor.cpp b/fdbserver/storageserver.actor.cpp
index 13d832c6be3..7a06891ec15 100644
--- a/fdbserver/storageserver.actor.cpp
+++ b/fdbserver/storageserver.actor.cpp
@@ -569,6 +569,7 @@ struct StorageServer {
 	// Histograms of requests sent to KVS.
 	Reference<Histogram> readRangeBytesReturnedHistogram;
 	Reference<Histogram> readRangeBytesLimitHistogram;
+	Reference<Histogram> readRangeKVPairsReturnedHistogram;
 
 	// watch map operations
 	Reference<ServerWatchMetadata> getWatchMetadata(KeyRef key) const;
@@ -880,9 +881,10 @@ struct StorageServer {
 
 	struct Counters {
 		CounterCollection cc;
-		Counter allQueries, getKeyQueries, getValueQueries, getRangeQueries, getMappedRangeQueries,
-		    getRangeStreamQueries, finishedQueries, lowPriorityQueries, rowsQueried, bytesQueried, watchQueries,
-		    emptyQueries, feedRowsQueried, feedBytesQueried, feedStreamQueries, feedVersionQueries;
+		Counter allQueries, systemKeyQueries, getKeyQueries, getValueQueries, getRangeQueries, getRangeSystemKeyQueries,
+		    getMappedRangeQueries, getRangeStreamQueries, finishedQueries, lowPriorityQueries, rowsQueried,
+		    bytesQueried, watchQueries, emptyQueries, feedRowsQueried, feedBytesQueried, feedStreamQueries,
+		    rejectedFeedStreamQueries, feedVersionQueries;
 
 		// Bytes of the mutations that have been added to the memory of the storage server. When the data is durable
 		// and cleared from the memory, we do not subtract it but add it to bytesDurable.
@@ -961,13 +963,15 @@ struct StorageServer {
 
 		Counters(StorageServer* self)
 		  : cc("StorageServer", self->thisServerID.toString()), allQueries("QueryQueue", cc),
-		    getKeyQueries("GetKeyQueries", cc), getValueQueries("GetValueQueries", cc),
-		    getRangeQueries("GetRangeQueries", cc), getMappedRangeQueries("GetMappedRangeQueries", cc),
-		    getRangeStreamQueries("GetRangeStreamQueries", cc), finishedQueries("FinishedQueries", cc),
-		    lowPriorityQueries("LowPriorityQueries", cc), rowsQueried("RowsQueried", cc),
-		    bytesQueried("BytesQueried", cc), watchQueries("WatchQueries", cc), emptyQueries("EmptyQueries", cc),
-		    feedRowsQueried("FeedRowsQueried", cc), feedBytesQueried("FeedBytesQueried", cc),
-		    feedStreamQueries("FeedStreamQueries", cc), feedVersionQueries("FeedVersionQueries", cc),
+		    systemKeyQueries("SystemKeyQueries", cc), getKeyQueries("GetKeyQueries", cc),
+		    getValueQueries("GetValueQueries", cc), getRangeQueries("GetRangeQueries", cc),
+		    getRangeSystemKeyQueries("GetRangeSystemKeyQueries", cc),
+		    getMappedRangeQueries("GetMappedRangeQueries", cc), getRangeStreamQueries("GetRangeStreamQueries", cc),
+		    finishedQueries("FinishedQueries", cc), lowPriorityQueries("LowPriorityQueries", cc),
+		    rowsQueried("RowsQueried", cc), bytesQueried("BytesQueried", cc), watchQueries("WatchQueries", cc),
+		    emptyQueries("EmptyQueries", cc), feedRowsQueried("FeedRowsQueried", cc),
+		    feedBytesQueried("FeedBytesQueried", cc), feedStreamQueries("FeedStreamQueries", cc),
+		    rejectedFeedStreamQueries("RejectedFeedStreamQueries", cc), feedVersionQueries("FeedVersionQueries", cc),
 		    bytesInput("BytesInput", cc), logicalBytesInput("LogicalBytesInput", cc),
 		    logicalBytesMoveInOverhead("LogicalBytesMoveInOverhead", cc),
 		    kvCommitLogicalBytes("KVCommitLogicalBytes", cc), kvClearRanges("KVClearRanges", cc),
@@ -1097,6 +1101,9 @@ struct StorageServer {
 	    readRangeBytesLimitHistogram(Histogram::getHistogram(STORAGESERVER_HISTOGRAM_GROUP,
 	                                                         SS_READ_RANGE_BYTES_LIMIT_HISTOGRAM,
 	                                                         Histogram::Unit::bytes)),
+	    readRangeKVPairsReturnedHistogram(Histogram::getHistogram(STORAGESERVER_HISTOGRAM_GROUP,
+	                                                              SS_READ_RANGE_KV_PAIRS_RETURNED_HISTOGRAM,
+	                                                              Histogram::Unit::countLinear)),
 	    tag(invalidTag), poppedAllAfter(std::numeric_limits<Version>::max()), cpuUsage(0.0), diskUsage(0.0),
 	    storage(this, storage), shardChangeCounter(0), lastTLogVersion(0), lastVersionWithData(0), restoredVersion(0),
 	    prevVersion(0), rebootAfterDurableVersion(std::numeric_limits<Version>::max()),
@@ -1617,6 +1624,9 @@ ACTOR Future<Void> getValueQ(StorageServer* data, GetValueRequest req) {
 		++data->counters.getValueQueries;
 		++data->counters.allQueries;
 		++data->readQueueSizeMetric;
+		if (req.key.startsWith(systemKeys.begin)) {
+			++data->counters.systemKeyQueries;
+		}
 		data->maxQueryQueue = std::max<int>(
 		    data->maxQueryQueue, data->counters.allQueries.getValue() - data->counters.finishedQueries.getValue());
 
@@ -2961,6 +2971,8 @@ ACTOR Future<GetKeyValuesReply> readRange(StorageServer* data,
 	state Key readBeginTemp;
 	state int vCount = 0;
 	state Span span("SS:readRange"_loc, parentSpan);
+	state int resultLogicalSize = 0;
+	state int logicalSize = 0;
 
 	// for caching the storage queue results during the first PTree traversal
 	state VectorRef<KeyValueRef> resultCache;
@@ -2998,7 +3010,6 @@ ACTOR Future<GetKeyValuesReply> readRange(StorageServer* data,
 			// Either way that's the correct result for lower_bound.
 			vCurrent = view.begin();
 		}
-
 		while (limit > 0 && *pLimitBytes > 0 && readBegin < range.end) {
 			ASSERT(!vCurrent || vCurrent.key() >= readBegin);
 			ASSERT(data->storageVersion() <= version);
@@ -3028,8 +3039,9 @@ ACTOR Future<GetKeyValuesReply> readRange(StorageServer* data,
 			readEnd = vCurrent ? std::min(vCurrent.key(), range.end) : range.end;
 			RangeResult atStorageVersion =
 			    wait(data->storage.readRange(KeyRangeRef(readBegin, readEnd), limit, *pLimitBytes, type));
-			data->counters.kvScanBytes += atStorageVersion.logicalSize();
-			data->readRangeBytesReturnedHistogram->sample(atStorageVersion.logicalSize());
+			logicalSize = atStorageVersion.logicalSize();
+			data->counters.kvScanBytes += logicalSize;
+			resultLogicalSize += logicalSize;
 			data->readRangeBytesLimitHistogram->sample(*pLimitBytes);
 
 			ASSERT(atStorageVersion.size() <= limit);
@@ -3129,8 +3141,9 @@ ACTOR Future<GetKeyValuesReply> readRange(StorageServer* data,
 			                     : range.begin;
 			RangeResult atStorageVersion =
 			    wait(data->storage.readRange(KeyRangeRef(readBegin, readEnd), limit, *pLimitBytes, type));
-			data->counters.kvScanBytes += atStorageVersion.logicalSize();
-			data->readRangeBytesReturnedHistogram->sample(atStorageVersion.logicalSize());
+			logicalSize = atStorageVersion.logicalSize();
+			data->counters.kvScanBytes += logicalSize;
+			resultLogicalSize += logicalSize;
 			data->readRangeBytesLimitHistogram->sample(*pLimitBytes);
 
 			ASSERT(atStorageVersion.size() <= -limit);
@@ -3181,6 +3194,8 @@ ACTOR Future<GetKeyValuesReply> readRange(StorageServer* data,
 			}
 		}
 	}
+	data->readRangeBytesReturnedHistogram->sample(resultLogicalSize);
+	data->readRangeKVPairsReturnedHistogram->sampleRecordCounter(result.data.size());
 
 	// all but the last item are less than *pLimitBytes
 	ASSERT(result.data.size() == 0 || *pLimitBytes + result.data.end()[-1].expectedSize() + sizeof(KeyValueRef) > 0);
@@ -3338,6 +3353,10 @@ ACTOR Future<Void> getKeyValuesQ(StorageServer* data, GetKeyValuesRequest req)
 	++data->counters.getRangeQueries;
 	++data->counters.allQueries;
 	++data->readQueueSizeMetric;
+	if (req.begin.getKey().startsWith(systemKeys.begin)) {
+		++data->counters.systemKeyQueries;
+		++data->counters.getRangeSystemKeyQueries;
+	}
 	data->maxQueryQueue = std::max<int>(
 	    data->maxQueryQueue, data->counters.allQueries.getValue() - data->counters.finishedQueries.getValue());
 
@@ -3978,6 +3997,9 @@ ACTOR Future<Void> getMappedKeyValuesQ(StorageServer* data, GetMappedKeyValuesRe
 	++data->counters.getMappedRangeQueries;
 	++data->counters.allQueries;
 	++data->readQueueSizeMetric;
+	if (req.begin.getKey().startsWith(systemKeys.begin)) {
+		++data->counters.systemKeyQueries;
+	}
 	data->maxQueryQueue = std::max<int>(
 	    data->maxQueryQueue, data->counters.allQueries.getValue() - data->counters.finishedQueries.getValue());
 
@@ -4195,6 +4217,9 @@ ACTOR Future<Void> getKeyValuesStreamQ(StorageServer* data, GetKeyValuesStreamRe
 	++data->counters.getRangeStreamQueries;
 	++data->counters.allQueries;
 	++data->readQueueSizeMetric;
+	if (req.begin.getKey().startsWith(systemKeys.begin)) {
+		++data->counters.systemKeyQueries;
+	}
 	data->maxQueryQueue = std::max<int>(
 	    data->maxQueryQueue, data->counters.allQueries.getValue() - data->counters.finishedQueries.getValue());
 

From 2a0ac990b9fbb2b02912ef167b36053c00307303 Mon Sep 17 00:00:00 2001
From: Zhe Wu <halfprice@users.noreply.github.com>
Date: Fri, 20 Jan 2023 18:35:24 -0800
Subject: [PATCH 053/468] Log PingLatency when there is no ping latency
 samples, but ping attempts

---
 fdbrpc/FlowTransport.actor.cpp | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/fdbrpc/FlowTransport.actor.cpp b/fdbrpc/FlowTransport.actor.cpp
index 5e0a02a7c27..a3ef374f140 100644
--- a/fdbrpc/FlowTransport.actor.cpp
+++ b/fdbrpc/FlowTransport.actor.cpp
@@ -300,7 +300,8 @@ ACTOR Future<Void> pingLatencyLogger(TransportData* self) {
 				peer->lastLoggedTime = peer->lastConnectTime;
 			}
 
-			if (peer && peer->pingLatencies.getPopulationSize() >= 10) {
+			if (peer && (peer->pingLatencies.getPopulationSize() >= 10 || peer->connectFailedCount > 0 ||
+			             peer->timeoutCount > 0)) {
 				TraceEvent("PingLatency")
 				    .detail("Elapsed", now() - peer->lastLoggedTime)
 				    .detail("PeerAddr", lastAddress)

From d59629e2f1717af6c0f2e7bbac84f21a79fa8440 Mon Sep 17 00:00:00 2001
From: neethuhaneesha <nbingi@apple.com>
Date: Wed, 25 Jan 2023 11:15:04 -0800
Subject: [PATCH 054/468] Changing histogram type. (#9227)

---
 fdbclient/ServerKnobs.cpp                |  2 +-
 fdbclient/ServerKnobs.h                  |  2 +-
 fdbserver/KeyValueStoreRocksDB.actor.cpp | 10 ++++------
 fdbserver/storageserver.actor.cpp        |  4 ++--
 4 files changed, 8 insertions(+), 10 deletions(-)

diff --git a/fdbclient/ServerKnobs.cpp b/fdbclient/ServerKnobs.cpp
index 76816e36ace..2ac23068864 100644
--- a/fdbclient/ServerKnobs.cpp
+++ b/fdbclient/ServerKnobs.cpp
@@ -420,7 +420,7 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi
 	init( ROCKSDB_ENABLE_COMPACT_ON_DELETION,                   true );
 	// CDCF: CompactOnDeletionCollectorFactory. The below 3 are parameters of the CompactOnDeletionCollectorFactory
 	// which controls the compaction on deleted data.
-	init( ROCKSDB_CDCF_SILIDING_WINDOW_SIZE,                     128 );
+	init( ROCKSDB_CDCF_SLIDING_WINDOW_SIZE,                      128 );
 	init( ROCKSDB_CDCF_DELETION_TRIGGER,                           1 );
 	init( ROCKSDB_CDCF_DELETION_RATIO,                             0 );
 	// Can commit will delay ROCKSDB_CAN_COMMIT_DELAY_ON_OVERLOAD seconds for
diff --git a/fdbclient/ServerKnobs.h b/fdbclient/ServerKnobs.h
index 9d660443562..0f967af8577 100644
--- a/fdbclient/ServerKnobs.h
+++ b/fdbclient/ServerKnobs.h
@@ -339,7 +339,7 @@ class ServerKnobs : public KnobsImpl<ServerKnobs> {
 	int64_t ROCKSDB_SINGLEKEY_DELETES_BYTES_LIMIT;
 	bool ROCKSDB_ENABLE_CLEAR_RANGE_EAGER_READS;
 	bool ROCKSDB_ENABLE_COMPACT_ON_DELETION;
-	int64_t ROCKSDB_CDCF_SILIDING_WINDOW_SIZE; // CDCF: CompactOnDeletionCollectorFactory
+	int64_t ROCKSDB_CDCF_SLIDING_WINDOW_SIZE; // CDCF: CompactOnDeletionCollectorFactory
 	int64_t ROCKSDB_CDCF_DELETION_TRIGGER; // CDCF: CompactOnDeletionCollectorFactory
 	double ROCKSDB_CDCF_DELETION_RATIO; // CDCF: CompactOnDeletionCollectorFactory
 	int ROCKSDB_STATS_LEVEL;
diff --git a/fdbserver/KeyValueStoreRocksDB.actor.cpp b/fdbserver/KeyValueStoreRocksDB.actor.cpp
index 29b260bd686..6da3467427b 100644
--- a/fdbserver/KeyValueStoreRocksDB.actor.cpp
+++ b/fdbserver/KeyValueStoreRocksDB.actor.cpp
@@ -328,7 +328,7 @@ rocksdb::ColumnFamilyOptions getCFOptions() {
 		// @param deletion_ratio, if <= 0 or > 1, disable triggering compaction
 		//     based on deletion ratio. Disabled by default.
 		options.table_properties_collector_factories = { rocksdb::NewCompactOnDeletionCollectorFactory(
-			SERVER_KNOBS->ROCKSDB_CDCF_SILIDING_WINDOW_SIZE,
+			SERVER_KNOBS->ROCKSDB_CDCF_SLIDING_WINDOW_SIZE,
 			SERVER_KNOBS->ROCKSDB_CDCF_DELETION_TRIGGER,
 			SERVER_KNOBS->ROCKSDB_CDCF_DELETION_RATIO) };
 	}
@@ -1962,10 +1962,8 @@ struct RocksDBKeyValueStore : IKeyValueStore {
 		    ROCKSDBSTORAGE_HISTOGRAM_GROUP, ROCKSDB_READPREFIX_GET_HISTOGRAM, Histogram::Unit::microseconds);
 		state Reference<Histogram> rocksdbReadRangeBytesReturnedHistogram = Histogram::getHistogram(
 		    ROCKSDBSTORAGE_HISTOGRAM_GROUP, ROCKSDB_READ_RANGE_BYTES_RETURNED_HISTOGRAM, Histogram::Unit::bytes);
-		state Reference<Histogram> rocksdbReadRangeKVPairsReturnedHistogram =
-		    Histogram::getHistogram(ROCKSDBSTORAGE_HISTOGRAM_GROUP,
-		                            ROCKSDB_READ_RANGE_KV_PAIRS_RETURNED_HISTOGRAM,
-		                            Histogram::Unit::countLinear);
+		state Reference<Histogram> rocksdbReadRangeKVPairsReturnedHistogram = Histogram::getHistogram(
+		    ROCKSDBSTORAGE_HISTOGRAM_GROUP, ROCKSDB_READ_RANGE_KV_PAIRS_RETURNED_HISTOGRAM, Histogram::Unit::bytes);
 		loop {
 			choose {
 				when(std::tuple<int, std::string, double> measure = waitNext(metricFutureStream)) {
@@ -2008,7 +2006,7 @@ struct RocksDBKeyValueStore : IKeyValueStore {
 					} else if (metricName == ROCKSDB_READ_RANGE_BYTES_RETURNED_HISTOGRAM.toString()) {
 						rocksdbReadRangeBytesReturnedHistogram->sample(metricValue);
 					} else if (metricName == ROCKSDB_READ_RANGE_KV_PAIRS_RETURNED_HISTOGRAM.toString()) {
-						rocksdbReadRangeKVPairsReturnedHistogram->sampleRecordCounter(metricValue);
+						rocksdbReadRangeKVPairsReturnedHistogram->sample(metricValue);
 					} else {
 						UNREACHABLE();
 					}
diff --git a/fdbserver/storageserver.actor.cpp b/fdbserver/storageserver.actor.cpp
index 7a06891ec15..9d9fdf8aa1c 100644
--- a/fdbserver/storageserver.actor.cpp
+++ b/fdbserver/storageserver.actor.cpp
@@ -1103,7 +1103,7 @@ struct StorageServer {
 	                                                         Histogram::Unit::bytes)),
 	    readRangeKVPairsReturnedHistogram(Histogram::getHistogram(STORAGESERVER_HISTOGRAM_GROUP,
 	                                                              SS_READ_RANGE_KV_PAIRS_RETURNED_HISTOGRAM,
-	                                                              Histogram::Unit::countLinear)),
+	                                                              Histogram::Unit::bytes)),
 	    tag(invalidTag), poppedAllAfter(std::numeric_limits<Version>::max()), cpuUsage(0.0), diskUsage(0.0),
 	    storage(this, storage), shardChangeCounter(0), lastTLogVersion(0), lastVersionWithData(0), restoredVersion(0),
 	    prevVersion(0), rebootAfterDurableVersion(std::numeric_limits<Version>::max()),
@@ -3195,7 +3195,7 @@ ACTOR Future<GetKeyValuesReply> readRange(StorageServer* data,
 		}
 	}
 	data->readRangeBytesReturnedHistogram->sample(resultLogicalSize);
-	data->readRangeKVPairsReturnedHistogram->sampleRecordCounter(result.data.size());
+	data->readRangeKVPairsReturnedHistogram->sample(result.data.size());
 
 	// all but the last item are less than *pLimitBytes
 	ASSERT(result.data.size() == 0 || *pLimitBytes + result.data.end()[-1].expectedSize() + sizeof(KeyValueRef) > 0);

From acd663e88a96b6eafbeaed223c972ae6c179c172 Mon Sep 17 00:00:00 2001
From: Sreenath Bodagala <82616783+sbodagala@users.noreply.github.com>
Date: Thu, 26 Jan 2023 23:13:18 -0500
Subject: [PATCH 055/468] Release 7.1: Cherry pick pull request #9225 (#9252)

* - Do not add fdbserver processes to the client list. (#9225)

Note: Server processes started getting reported as clients since 7.1.0
(not sure if this change in behavior was intentional or not), and this
breaks the operator upgrade logic.

* - Address a compilation error

* - Update release-notes.

* - Address a review comment/CI failure.

* - Address CI, related to release notes, failure.
---
 .../sphinx/source/release-notes/release-notes-710.rst    | 1 +
 fdbclient/CoordinationInterface.h                        | 4 +++-
 fdbclient/MonitorLeader.actor.cpp                        | 9 ++++++---
 fdbclient/MonitorLeader.h                                | 4 +++-
 fdbclient/NativeAPI.actor.cpp                            | 3 ++-
 fdbserver/Coordination.actor.cpp                         | 4 ++--
 6 files changed, 17 insertions(+), 8 deletions(-)

diff --git a/documentation/sphinx/source/release-notes/release-notes-710.rst b/documentation/sphinx/source/release-notes/release-notes-710.rst
index 8296635c234..c1261387d19 100644
--- a/documentation/sphinx/source/release-notes/release-notes-710.rst
+++ b/documentation/sphinx/source/release-notes/release-notes-710.rst
@@ -11,6 +11,7 @@ Release Notes
 7.1.26
 ======
 * Released with AVX disabled.
+* Fixed status json to not report fdbserver processes as clients. `(PR #9252) <https://github.com/apple/foundationdb/pull/9252>`_
 * Added detection of disconnection to satellite TLog in gray failure detection. `(PR #9107) <https://github.com/apple/foundationdb/pull/9107>`_
 * Fixed (non)empty peeks stats in TLogMetrics. `(PR #9074) <https://github.com/apple/foundationdb/pull/9074>`_
 * Fixed a data distribution bug where exclusions can become stuck because DD cannot build new teams. `(PR #9035) <https://github.com/apple/foundationdb/pull/9035>`_
diff --git a/fdbclient/CoordinationInterface.h b/fdbclient/CoordinationInterface.h
index eb0f99895b5..7897ded33e0 100644
--- a/fdbclient/CoordinationInterface.h
+++ b/fdbclient/CoordinationInterface.h
@@ -241,6 +241,7 @@ struct OpenDatabaseCoordRequest {
 	std::vector<Hostname> hostnames;
 	std::vector<NetworkAddress> coordinators;
 	ReplyPromise<CachedSerialization<struct ClientDBInfo>> reply;
+	bool internal{ true };
 
 	template <class Ar>
 	void serialize(Ar& ar) {
@@ -252,7 +253,8 @@ struct OpenDatabaseCoordRequest {
 		           clusterKey,
 		           coordinators,
 		           reply,
-		           hostnames);
+		           hostnames,
+		           internal);
 	}
 };
 
diff --git a/fdbclient/MonitorLeader.actor.cpp b/fdbclient/MonitorLeader.actor.cpp
index 22724dd43d5..67b1412ec09 100644
--- a/fdbclient/MonitorLeader.actor.cpp
+++ b/fdbclient/MonitorLeader.actor.cpp
@@ -862,7 +862,8 @@ ACTOR Future<MonitorLeaderInfo> monitorProxiesOneGeneration(
     Reference<AsyncVar<Optional<ClientLeaderRegInterface>>> coordinator,
     MonitorLeaderInfo info,
     Reference<ReferencedObject<Standalone<VectorRef<ClientVersionRef>>>> supportedVersions,
-    Key traceLogGroup) {
+    Key traceLogGroup,
+    IsInternal internal) {
 	state ClusterConnectionString cs = info.intermediateConnRecord->getConnectionString();
 	state int coordinatorsSize = cs.hostnames.size() + cs.coords.size();
 	state int index = 0;
@@ -894,6 +895,7 @@ ACTOR Future<MonitorLeaderInfo> monitorProxiesOneGeneration(
 		req.knownClientInfoID = clientInfo->get().id;
 		req.supportedVersions = supportedVersions->get();
 		req.traceLogGroup = traceLogGroup;
+		req.internal = internal;
 
 		state ClusterConnectionString storedConnectionString;
 		if (connRecord) {
@@ -982,12 +984,13 @@ ACTOR Future<Void> monitorProxies(
     Reference<AsyncVar<ClientDBInfo>> clientInfo,
     Reference<AsyncVar<Optional<ClientLeaderRegInterface>>> coordinator,
     Reference<ReferencedObject<Standalone<VectorRef<ClientVersionRef>>>> supportedVersions,
-    Key traceLogGroup) {
+    Key traceLogGroup,
+    IsInternal internal) {
 	state MonitorLeaderInfo info(connRecord->get());
 	loop {
 		choose {
 			when(MonitorLeaderInfo _info = wait(monitorProxiesOneGeneration(
-			         connRecord->get(), clientInfo, coordinator, info, supportedVersions, traceLogGroup))) {
+			         connRecord->get(), clientInfo, coordinator, info, supportedVersions, traceLogGroup, internal))) {
 				info = _info;
 			}
 			when(wait(connRecord->onChange())) {
diff --git a/fdbclient/MonitorLeader.h b/fdbclient/MonitorLeader.h
index 8819b5702b4..86f2545fe9c 100644
--- a/fdbclient/MonitorLeader.h
+++ b/fdbclient/MonitorLeader.h
@@ -26,6 +26,7 @@
 #include "fdbclient/CoordinationInterface.h"
 #include "fdbclient/ClusterInterface.h"
 #include "fdbclient/CommitProxyInterface.h"
+#include "fdbclient/ClientBooleanParams.h"
 
 #define CLUSTER_FILE_ENV_VAR_NAME "FDB_CLUSTER_FILE"
 
@@ -85,7 +86,8 @@ Future<Void> monitorProxies(
     Reference<AsyncVar<ClientDBInfo>> const& clientInfo,
     Reference<AsyncVar<Optional<ClientLeaderRegInterface>>> const& coordinator,
     Reference<ReferencedObject<Standalone<VectorRef<ClientVersionRef>>>> const& supportedVersions,
-    Key const& traceLogGroup);
+    Key const& traceLogGroup,
+    IsInternal const& internal);
 
 void shrinkProxyList(ClientDBInfo& ni,
                      std::vector<UID>& lastCommitProxyUIDs,
diff --git a/fdbclient/NativeAPI.actor.cpp b/fdbclient/NativeAPI.actor.cpp
index 8f4188a1e85..cd6865028bd 100644
--- a/fdbclient/NativeAPI.actor.cpp
+++ b/fdbclient/NativeAPI.actor.cpp
@@ -2254,7 +2254,8 @@ Database Database::createDatabase(Reference<IClusterConnectionRecord> connRecord
 	                                                clientInfo,
 	                                                coordinator,
 	                                                networkOptions.supportedVersions,
-	                                                StringRef(networkOptions.traceLogGroup));
+	                                                StringRef(networkOptions.traceLogGroup),
+	                                                internal);
 
 	DatabaseContext* db;
 	if (preallocatedDb) {
diff --git a/fdbserver/Coordination.actor.cpp b/fdbserver/Coordination.actor.cpp
index 257b659fb80..2e611c6f371 100644
--- a/fdbserver/Coordination.actor.cpp
+++ b/fdbserver/Coordination.actor.cpp
@@ -221,7 +221,7 @@ ACTOR Future<Void> openDatabase(ClientData* db,
 	++(*clientCount);
 	hasConnectedClients->set(true);
 
-	if (req.supportedVersions.size() > 0) {
+	if (req.supportedVersions.size() > 0 && !req.internal) {
 		db->clientStatusInfoMap[req.reply.getEndpoint().getPrimaryAddress()] =
 		    ClientStatusInfo(req.traceLogGroup, req.supportedVersions, req.issues);
 	}
@@ -247,7 +247,7 @@ ACTOR Future<Void> openDatabase(ClientData* db,
 		}
 	}
 
-	if (req.supportedVersions.size() > 0) {
+	if (req.supportedVersions.size() > 0 && !req.internal) {
 		db->clientStatusInfoMap.erase(req.reply.getEndpoint().getPrimaryAddress());
 	}
 

From 2d0427521d8ff14bfd617dccb6c875e64e238ee4 Mon Sep 17 00:00:00 2001
From: FoundationDB CI <foundationdb_ci@apple.com>
Date: Fri, 27 Jan 2023 23:42:45 +0000
Subject: [PATCH 056/468] disable AVX for 7.1.26 release

---
 cmake/ConfigureCompiler.cmake | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cmake/ConfigureCompiler.cmake b/cmake/ConfigureCompiler.cmake
index e52d8665ee8..7e50469ba6a 100644
--- a/cmake/ConfigureCompiler.cmake
+++ b/cmake/ConfigureCompiler.cmake
@@ -253,7 +253,7 @@ else()
       set(USE_AVX512F OFF)
     endif()
   endif()
-  set(USE_AVX ON CACHE BOOL "Enable AVX instructions")
+  set(USE_AVX OFF CACHE BOOL "Enable AVX instructions")
   if (USE_AVX)
     if (CMAKE_HOST_SYSTEM_PROCESSOR MATCHES "^x86")
       add_compile_options(-mavx)

From dc4bd7ef6fcda345276073599ed6a2b28b089e90 Mon Sep 17 00:00:00 2001
From: FoundationDB CI <foundationdb_ci@apple.com>
Date: Fri, 27 Jan 2023 23:43:06 +0000
Subject: [PATCH 057/468] enable AVX and update version for 7.1.27 release

---
 CMakeLists.txt                | 2 +-
 cmake/ConfigureCompiler.cmake | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 2e7e4cda729..abe42749768 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -23,7 +23,7 @@ else()
 endif()
 
 project(foundationdb
-  VERSION 7.1.26
+  VERSION 7.1.27
   DESCRIPTION "FoundationDB is a scalable, fault-tolerant, ordered key-value store with full ACID transactions."
   HOMEPAGE_URL "http://www.foundationdb.org/"
   LANGUAGES C CXX ASM)
diff --git a/cmake/ConfigureCompiler.cmake b/cmake/ConfigureCompiler.cmake
index 7e50469ba6a..e52d8665ee8 100644
--- a/cmake/ConfigureCompiler.cmake
+++ b/cmake/ConfigureCompiler.cmake
@@ -253,7 +253,7 @@ else()
       set(USE_AVX512F OFF)
     endif()
   endif()
-  set(USE_AVX OFF CACHE BOOL "Enable AVX instructions")
+  set(USE_AVX ON CACHE BOOL "Enable AVX instructions")
   if (USE_AVX)
     if (CMAKE_HOST_SYSTEM_PROCESSOR MATCHES "^x86")
       add_compile_options(-mavx)

From 788f36d4e0f880b15354328d515838e230d698d2 Mon Sep 17 00:00:00 2001
From: FoundationDB CI <foundationdb_ci@apple.com>
Date: Tue, 31 Jan 2023 05:15:38 +0000
Subject: [PATCH 058/468] update version after 7.1.27 release

---
 CMakeLists.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index abe42749768..5528c6a6b6e 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -23,7 +23,7 @@ else()
 endif()
 
 project(foundationdb
-  VERSION 7.1.27
+  VERSION 7.1.28
   DESCRIPTION "FoundationDB is a scalable, fault-tolerant, ordered key-value store with full ACID transactions."
   HOMEPAGE_URL "http://www.foundationdb.org/"
   LANGUAGES C CXX ASM)

From 8b3a5abdfe61ffafc562977fd3fc37e62028b153 Mon Sep 17 00:00:00 2001
From: Dan Adkins <dan.adkins@snowflake.com>
Date: Thu, 19 Jan 2023 17:36:06 -0800
Subject: [PATCH 059/468] Increase buggified lock bytes for backup workers to
 at least 256 MB.

We are still encountered simulation failures where the backup worker
is waiting on the lock and an assertion fails.
---
 fdbclient/ServerKnobs.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/fdbclient/ServerKnobs.cpp b/fdbclient/ServerKnobs.cpp
index 2ac23068864..90ba68695a6 100644
--- a/fdbclient/ServerKnobs.cpp
+++ b/fdbclient/ServerKnobs.cpp
@@ -532,7 +532,7 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi
 	init( BACKUP_TIMEOUT,                                        0.4 );
 	init( BACKUP_NOOP_POP_DELAY,                                 5.0 );
 	init( BACKUP_FILE_BLOCK_BYTES,                       1024 * 1024 );
-	init( BACKUP_LOCK_BYTES,                                     3e9 ); if(randomize && BUGGIFY) BACKUP_LOCK_BYTES = deterministicRandom()->randomInt(1024, 4096) * 30 * 1024;
+	init( BACKUP_LOCK_BYTES,                                     3e9 ); if(randomize && BUGGIFY) BACKUP_LOCK_BYTES = deterministicRandom()->randomInt(1024, 4096) * 256 * 1024;
 	init( BACKUP_UPLOAD_DELAY,                                  10.0 ); if(randomize && BUGGIFY) BACKUP_UPLOAD_DELAY = deterministicRandom()->random01() * 60;
 
 	//Cluster Controller

From 668e142b54f39d73a43bf7735a52fc08a2eb0e4b Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Wed, 1 Feb 2023 09:51:16 -0800
Subject: [PATCH 060/468] Reduce logging level for verbose events

From one of nightly failure due to too many log lines, these are top 3:

  60100 FastRestoreLoaderDispatchRequests
  79655 FastRestoreGetVersionSize
  93888 FastRestoreSplitMutation
---
 fdbserver/RestoreController.actor.h | 4 ++--
 fdbserver/RestoreLoader.actor.cpp   | 6 +++---
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/fdbserver/RestoreController.actor.h b/fdbserver/RestoreController.actor.h
index 77aa5e6494a..bcb6597bacb 100644
--- a/fdbserver/RestoreController.actor.h
+++ b/fdbserver/RestoreController.actor.h
@@ -238,7 +238,7 @@ struct RestoreControllerData : RestoreRoleData, public ReferenceCounted<RestoreC
 	                                                                   int rangeIdx,
 	                                                                   const std::vector<RestoreFileFR>& logFiles) {
 		double size = 0;
-		TraceEvent(SevDebug, "FastRestoreGetVersionSize")
+		TraceEvent(SevVerbose, "FastRestoreGetVersionSize")
 		    .detail("PreviousVersion", prevVersion)
 		    .detail("NextVersion", nextVersion)
 		    .detail("RangeFiles", rangeFiles.size())
@@ -246,7 +246,7 @@ struct RestoreControllerData : RestoreRoleData, public ReferenceCounted<RestoreC
 		    .detail("LogFiles", logFiles.size());
 		ASSERT(prevVersion <= nextVersion);
 		while (rangeIdx < rangeFiles.size()) {
-			TraceEvent(SevDebug, "FastRestoreGetVersionSize").detail("RangeFile", rangeFiles[rangeIdx].toString());
+			TraceEvent(SevVerbose, "FastRestoreGetVersionSize").detail("RangeFile", rangeFiles[rangeIdx].toString());
 			if (rangeFiles[rangeIdx].version < nextVersion) {
 				ASSERT(rangeFiles[rangeIdx].version >= prevVersion);
 				size += rangeFiles[rangeIdx].fileSize;
diff --git a/fdbserver/RestoreLoader.actor.cpp b/fdbserver/RestoreLoader.actor.cpp
index 2d87bacc551..ff33173f878 100644
--- a/fdbserver/RestoreLoader.actor.cpp
+++ b/fdbserver/RestoreLoader.actor.cpp
@@ -91,7 +91,7 @@ ACTOR Future<Void> dispatchRequests(Reference<RestoreLoaderData> self) {
 		state int sendLoadParams = 0;
 		state int lastLoadReqs = 0;
 		loop {
-			TraceEvent(SevDebug, "FastRestoreLoaderDispatchRequests", self->id())
+			TraceEvent(SevVerbose, "FastRestoreLoaderDispatchRequests", self->id())
 			    .detail("SendingQueue", self->sendingQueue.size())
 			    .detail("LoadingQueue", self->loadingQueue.size())
 			    .detail("SendingLoadParamQueue", self->sendLoadParamQueue.size())
@@ -216,7 +216,7 @@ ACTOR Future<Void> dispatchRequests(Reference<RestoreLoaderData> self) {
 			updateProcessStats(self);
 
 			if (self->loadingQueue.empty() && self->sendingQueue.empty() && self->sendLoadParamQueue.empty()) {
-				TraceEvent(SevDebug, "FastRestoreLoaderDispatchRequestsWaitOnRequests", self->id())
+				TraceEvent(SevVerbose, "FastRestoreLoaderDispatchRequestsWaitOnRequests", self->id())
 				    .detail("HasPendingRequests", self->hasPendingRequests->get());
 				self->hasPendingRequests->set(false);
 				wait(self->hasPendingRequests->onChange()); // CAREFUL:Improper req release may cause restore stuck here
@@ -996,7 +996,7 @@ void splitMutation(const KeyRangeMap<UID>& krMap,
                    VectorRef<MutationRef>& mvector,
                    Arena& nodeIDs_arena,
                    VectorRef<UID>& nodeIDs) {
-	TraceEvent(SevDebug, "FastRestoreSplitMutation").detail("Mutation", m);
+	TraceEvent(SevVerbose, "FastRestoreSplitMutation").detail("Mutation", m);
 	ASSERT(mvector.empty());
 	ASSERT(nodeIDs.empty());
 	auto r = krMap.intersectingRanges(KeyRangeRef(m.param1, m.param2));

From 1f5da87059ec8fc0ca489db12baea8153840c328 Mon Sep 17 00:00:00 2001
From: Dan Lambright <dlambrig@gmail.com>
Date: Thu, 2 Feb 2023 11:01:56 -0500
Subject: [PATCH 061/468] Fix typo in fdb.options

---
 fdbclient/vexillographer/fdb.options | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/fdbclient/vexillographer/fdb.options b/fdbclient/vexillographer/fdb.options
index bb9c5d2cff2..98abe24e48e 100644
--- a/fdbclient/vexillographer/fdb.options
+++ b/fdbclient/vexillographer/fdb.options
@@ -30,8 +30,6 @@ description is not currently required but encouraged.
 
 <Options>
   <Scope name="NetworkOption">
-    <Option name="retain_client_library_copies" code="67"
-            description="Retain temporary external client library copies that are created for enabling multi-threading." />
     <Option name="local_address"  code="10" 
             paramType="String" paramDescription="IP:PORT" 
             description="Deprecated"/>
@@ -117,6 +115,8 @@ description is not currently required but encouraged.
     <Option name="client_threads_per_version" code="65"
             paramType="Int" paramDescription="Number of client threads to be spawned.  Each cluster will be serviced by a single client thread."
             description="Spawns multiple worker threads for each version of the client that is loaded.  Setting this to a number greater than one implies disable_local_client." />
+    <Option name="retain_client_library_copies" code="67"
+            description="Retain temporary external client library copies that are created for enabling multi-threading." />
     <Option name="disable_client_statistics_logging" code="70"
             description="Disables logging of client statistics, such as sampled transaction activity." />
     <Option name="enable_slow_task_profiling" code="71"

From ff2531dd9749f9f45ca71ecc2d252548b7ecdb89 Mon Sep 17 00:00:00 2001
From: Dan Lambright <dlambrig@gmail.com>
Date: Fri, 3 Feb 2023 11:56:38 -0500
Subject: [PATCH 062/468] update  bindings/go/src/fdb/generated.go

---
 bindings/go/src/fdb/generated.go | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/bindings/go/src/fdb/generated.go b/bindings/go/src/fdb/generated.go
index f8bb231980c..61a5b2517d0 100644
--- a/bindings/go/src/fdb/generated.go
+++ b/bindings/go/src/fdb/generated.go
@@ -39,11 +39,6 @@ func int64ToBytes(i int64) []byte {
 	return buf
 }
 
-// Retain temporary external client library copies that are created for enabling multi-threading.
-func (o NetworkOptions) SetRetainClientLibraryCopies() error {
-	return o.setOpt(67, nil)
-}
-
 // Deprecated
 //
 // Parameter: IP:PORT
@@ -244,6 +239,11 @@ func (o NetworkOptions) SetClientThreadsPerVersion(param int64) error {
 	return o.setOpt(65, int64ToBytes(param))
 }
 
+// Retain temporary external client library copies that are created for enabling multi-threading.
+func (o NetworkOptions) SetRetainClientLibraryCopies() error {
+	return o.setOpt(67, nil)
+}
+
 // Disables logging of client statistics, such as sampled transaction activity.
 func (o NetworkOptions) SetDisableClientStatisticsLogging() error {
 	return o.setOpt(70, nil)

From b0f78ef49d2195d51922d4ab908b6a3bca49afe8 Mon Sep 17 00:00:00 2001
From: Hao Fu <77984096+hfu94@users.noreply.github.com>
Date: Mon, 13 Feb 2023 11:04:42 -0800
Subject: [PATCH 063/468] Fix getMappedRange metrics(release-7.1) (#9331)

* Fix getMappedRange metrics

Metrics related to getMappedRange API are counted twice,
having a set of new metrics specifically for getMappedRange solves
the issue.

* Fix clang init order issue
---
 fdbserver/storageserver.actor.cpp | 47 +++++++++++++------------------
 1 file changed, 20 insertions(+), 27 deletions(-)

diff --git a/fdbserver/storageserver.actor.cpp b/fdbserver/storageserver.actor.cpp
index 9d9fdf8aa1c..7e141468302 100644
--- a/fdbserver/storageserver.actor.cpp
+++ b/fdbserver/storageserver.actor.cpp
@@ -882,9 +882,13 @@ struct StorageServer {
 	struct Counters {
 		CounterCollection cc;
 		Counter allQueries, systemKeyQueries, getKeyQueries, getValueQueries, getRangeQueries, getRangeSystemKeyQueries,
-		    getMappedRangeQueries, getRangeStreamQueries, finishedQueries, lowPriorityQueries, rowsQueried,
-		    bytesQueried, watchQueries, emptyQueries, feedRowsQueried, feedBytesQueried, feedStreamQueries,
-		    rejectedFeedStreamQueries, feedVersionQueries;
+		    getRangeStreamQueries, finishedQueries, lowPriorityQueries, rowsQueried, bytesQueried, watchQueries,
+		    emptyQueries, feedRowsQueried, feedBytesQueried, feedStreamQueries, rejectedFeedStreamQueries,
+		    feedVersionQueries;
+
+		// counters related to getMappedRange queries
+		Counter getMappedRangeBytesQueried, finishedGetMappedRangeSecondaryQueries, getMappedRangeQueries,
+		    finishedGetMappedRangeQueries;
 
 		// Bytes of the mutations that have been added to the memory of the storage server. When the data is durable
 		// and cleared from the memory, we do not subtract it but add it to bytesDurable.
@@ -966,14 +970,16 @@ struct StorageServer {
 		    systemKeyQueries("SystemKeyQueries", cc), getKeyQueries("GetKeyQueries", cc),
 		    getValueQueries("GetValueQueries", cc), getRangeQueries("GetRangeQueries", cc),
 		    getRangeSystemKeyQueries("GetRangeSystemKeyQueries", cc),
-		    getMappedRangeQueries("GetMappedRangeQueries", cc), getRangeStreamQueries("GetRangeStreamQueries", cc),
-		    finishedQueries("FinishedQueries", cc), lowPriorityQueries("LowPriorityQueries", cc),
-		    rowsQueried("RowsQueried", cc), bytesQueried("BytesQueried", cc), watchQueries("WatchQueries", cc),
-		    emptyQueries("EmptyQueries", cc), feedRowsQueried("FeedRowsQueried", cc),
-		    feedBytesQueried("FeedBytesQueried", cc), feedStreamQueries("FeedStreamQueries", cc),
-		    rejectedFeedStreamQueries("RejectedFeedStreamQueries", cc), feedVersionQueries("FeedVersionQueries", cc),
-		    bytesInput("BytesInput", cc), logicalBytesInput("LogicalBytesInput", cc),
-		    logicalBytesMoveInOverhead("LogicalBytesMoveInOverhead", cc),
+		    getRangeStreamQueries("GetRangeStreamQueries", cc), finishedQueries("FinishedQueries", cc),
+		    lowPriorityQueries("LowPriorityQueries", cc), rowsQueried("RowsQueried", cc),
+		    bytesQueried("BytesQueried", cc), watchQueries("WatchQueries", cc), emptyQueries("EmptyQueries", cc),
+		    feedRowsQueried("FeedRowsQueried", cc), feedBytesQueried("FeedBytesQueried", cc),
+		    feedStreamQueries("FeedStreamQueries", cc), rejectedFeedStreamQueries("RejectedFeedStreamQueries", cc),
+		    feedVersionQueries("FeedVersionQueries", cc), getMappedRangeBytesQueried("GetMappedRangeBytesQueried", cc),
+		    finishedGetMappedRangeSecondaryQueries("FinishedGetMappedRangeSecondaryQueries", cc),
+		    getMappedRangeQueries("GetMappedRangeQueries", cc),
+		    finishedGetMappedRangeQueries("FinishedGetMappedRangeQueries", cc), bytesInput("BytesInput", cc),
+		    logicalBytesInput("LogicalBytesInput", cc), logicalBytesMoveInOverhead("LogicalBytesMoveInOverhead", cc),
 		    kvCommitLogicalBytes("KVCommitLogicalBytes", cc), kvClearRanges("KVClearRanges", cc),
 		    kvClearSingleKey("KVClearSingleKey", cc), kvSystemClearRanges("KVSystemClearRanges", cc),
 		    bytesDurable("BytesDurable", cc), bytesFetched("BytesFetched", cc), mutationBytes("MutationBytes", cc),
@@ -4150,25 +4156,12 @@ ACTOR Future<Void> getMappedKeyValuesQ(StorageServer* data, GetMappedKeyValuesRe
 				//                ASSERT(r.data.size() <= std::abs(req.limit));
 			}
 
-			// For performance concerns, the cost of a range read is billed to the start key and end key of the range.
-			int64_t totalByteSize = 0;
-			for (int i = 0; i < r.data.size(); i++) {
-				totalByteSize += r.data[i].expectedSize();
-			}
-			if (totalByteSize > 0 && SERVER_KNOBS->READ_SAMPLING_ENABLED) {
-				int64_t bytesReadPerKSecond = std::max(totalByteSize, SERVER_KNOBS->EMPTY_READ_PENALTY) / 2;
-				data->metrics.notifyBytesReadPerKSecond(addPrefix(r.data[0].key, tenantPrefix, req.arena),
-				                                        bytesReadPerKSecond);
-				data->metrics.notifyBytesReadPerKSecond(
-				    addPrefix(r.data[r.data.size() - 1].key, tenantPrefix, req.arena), bytesReadPerKSecond);
-			}
-
 			r.penalty = data->getPenalty();
 			req.reply.send(r);
 
 			resultSize = req.limitBytes - remainingLimitBytes;
-			data->counters.bytesQueried += resultSize;
-			data->counters.rowsQueried += r.data.size();
+			data->counters.getMappedRangeBytesQueried += resultSize;
+			data->counters.finishedGetMappedRangeSecondaryQueries += r.data.size();
 			if (r.data.size() == 0) {
 				++data->counters.emptyQueries;
 			}
@@ -4180,7 +4173,7 @@ ACTOR Future<Void> getMappedKeyValuesQ(StorageServer* data, GetMappedKeyValuesRe
 	}
 
 	data->transactionTagCounter.addRequest(req.tags, resultSize);
-	++data->counters.finishedQueries;
+	++data->counters.finishedGetMappedRangeQueries;
 	--data->readQueueSizeMetric;
 
 	double duration = g_network->timer() - req.requestTime();

From f99915ac3e048b6a0dd4e803518e4eef581d1e9e Mon Sep 17 00:00:00 2001
From: neethuhaneesha <nbingi@apple.com>
Date: Wed, 15 Feb 2023 13:03:58 -0800
Subject: [PATCH 064/468] Enable rocksdb in simulation in 7.1. Exclude FuzzApi
 and HighContention tests temporarily for rocksdb. (#9374)

---
 fdbserver/SimulatedCluster.actor.cpp          | 4 +---
 tests/fast/FuzzApiCorrectness.toml            | 2 ++
 tests/fast/FuzzApiCorrectnessClean.toml       | 2 ++
 tests/rare/HighContentionPrefixAllocator.toml | 4 ++++
 4 files changed, 9 insertions(+), 3 deletions(-)

diff --git a/fdbserver/SimulatedCluster.actor.cpp b/fdbserver/SimulatedCluster.actor.cpp
index 50d7bb1824e..3d70a8f42af 100644
--- a/fdbserver/SimulatedCluster.actor.cpp
+++ b/fdbserver/SimulatedCluster.actor.cpp
@@ -1369,9 +1369,7 @@ void SimulationConfig::setDatacenters(const TestConfig& testConfig) {
 
 // Sets storage engine based on testConfig details
 void SimulationConfig::setStorageEngine(const TestConfig& testConfig) {
-	// Using [0, 4) to disable the RocksDB storage engine.
-	// TODO: Figure out what is broken with the RocksDB engine in simulation.
-	int storage_engine_type = deterministicRandom()->randomInt(0, 4);
+	int storage_engine_type = deterministicRandom()->randomInt(0, 5);
 	if (testConfig.storageEngineType.present()) {
 		storage_engine_type = testConfig.storageEngineType.get();
 	} else {
diff --git a/tests/fast/FuzzApiCorrectness.toml b/tests/fast/FuzzApiCorrectness.toml
index 36b8e6812f8..bbd44582d43 100644
--- a/tests/fast/FuzzApiCorrectness.toml
+++ b/tests/fast/FuzzApiCorrectness.toml
@@ -2,6 +2,8 @@
 StderrSeverity = 30
 allowDisablingTenants = false
 allowDefaultTenant = false
+# FIXME: re-enable rocks at some point
+storageEngineExcludeTypes = [4]
 
 [[test]]
 testTitle = 'FuzzApiCorrectness'
diff --git a/tests/fast/FuzzApiCorrectnessClean.toml b/tests/fast/FuzzApiCorrectnessClean.toml
index deddfacce31..1f808768cd5 100644
--- a/tests/fast/FuzzApiCorrectnessClean.toml
+++ b/tests/fast/FuzzApiCorrectnessClean.toml
@@ -2,6 +2,8 @@
 StderrSeverity = 30
 allowDisablingTenants = false
 allowDefaultTenant = false
+# FIXME: re-enable rocks at some point
+storageEngineExcludeTypes = [4]
 
 [[test]]
 testTitle = 'FuzzApiCorrectness'
diff --git a/tests/rare/HighContentionPrefixAllocator.toml b/tests/rare/HighContentionPrefixAllocator.toml
index d7bf6edbd96..d66c82018d5 100644
--- a/tests/rare/HighContentionPrefixAllocator.toml
+++ b/tests/rare/HighContentionPrefixAllocator.toml
@@ -1,3 +1,7 @@
+[configuration]
+# FIXME: re-enable rocks at some point
+storageEngineExcludeTypes = [4]
+
 [[test]]
 testTitle = 'HighContentionPrefixAllocator'
 

From 55c13437de9a9130e8b73f221b09df7ae2960d13 Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Thu, 16 Feb 2023 09:39:35 -0800
Subject: [PATCH 065/468] Fix IDE build and warnings

---
 fdbclient/GenericManagementAPI.actor.h | 1 -
 fdbclient/SpecialKeySpace.actor.cpp    | 2 +-
 fdbrpc/LoadBalance.actor.h             | 8 ++++----
 flow/actorcompiler.h                   | 6 +++---
 4 files changed, 8 insertions(+), 9 deletions(-)

diff --git a/fdbclient/GenericManagementAPI.actor.h b/fdbclient/GenericManagementAPI.actor.h
index 8ea27b33cf4..87e11b79a99 100644
--- a/fdbclient/GenericManagementAPI.actor.h
+++ b/fdbclient/GenericManagementAPI.actor.h
@@ -291,7 +291,6 @@ Future<ConfigurationResult> changeConfig(Reference<DB> db, std::map<std::string,
 	state Key versionKey = BinaryWriter::toValue(deterministicRandom()->randomUniqueID(), Unversioned());
 	state bool oldReplicationUsesDcId = false;
 	state bool warnPPWGradual = false;
-	state bool warnChangeStorageNoMigrate = false;
 	state bool warnRocksDBIsExperimental = false;
 	loop {
 		try {
diff --git a/fdbclient/SpecialKeySpace.actor.cpp b/fdbclient/SpecialKeySpace.actor.cpp
index f90ce4a29e5..451febfbeeb 100644
--- a/fdbclient/SpecialKeySpace.actor.cpp
+++ b/fdbclient/SpecialKeySpace.actor.cpp
@@ -975,7 +975,7 @@ ACTOR Future<bool> checkExclusion(Database db,
 	state int ssTotalCount = 0;
 	state int ssExcludedCount = 0;
 
-	state std::unordered_set<std::string> diskLocalities();
+	state std::unordered_set<std::string> diskLocalities;
 	state int64_t totalKvStoreFreeBytes = 0;
 	state int64_t totalKvStoreUsedBytes = 0;
 	state int64_t totalKvStoreUsedBytesNonExcluded = 0;
diff --git a/fdbrpc/LoadBalance.actor.h b/fdbrpc/LoadBalance.actor.h
index b25a1ea73e4..4d0aad7a249 100644
--- a/fdbrpc/LoadBalance.actor.h
+++ b/fdbrpc/LoadBalance.actor.h
@@ -94,10 +94,12 @@ Future<Void> tssComparison(Req req,
 	// we want to record ss/tss errors to metrics
 	state int srcErrorCode = error_code_success;
 	state int tssErrorCode = error_code_success;
+	state ErrorOr<Resp> src;
+	state Optional<ErrorOr<Resp>> tss;
 
 	loop {
 		choose {
-			when(state ErrorOr<Resp> src = wait(fSource)) {
+			when(wait(store(src, fSource))) {
 				srcEndTime = now();
 				fSource = Never();
 				finished++;
@@ -105,7 +107,7 @@ Future<Void> tssComparison(Req req,
 					break;
 				}
 			}
-			when(state Optional<ErrorOr<Resp>> tss = wait(fTssWithTimeout)) {
+			when(wait(store(tss, fTssWithTimeout))) {
 				tssEndTime = now();
 				fTssWithTimeout = Never();
 				finished++;
@@ -659,7 +661,6 @@ Future<REPLY_TYPE(Request)> loadBalance(
 				    .detail("Attempts", numAttempts);
 			}
 			secondRequestData.startRequest(backoff, triedAllOptions, stream, request, model, alternatives, channel);
-			state bool firstFinished = false;
 
 			loop choose {
 				when(ErrorOr<REPLY_TYPE(Request)> result =
@@ -669,7 +670,6 @@ Future<REPLY_TYPE(Request)> loadBalance(
 					}
 
 					firstRequestEndpoint = Optional<uint64_t>();
-					firstFinished = true;
 				}
 				when(ErrorOr<REPLY_TYPE(Request)> result = wait(secondRequestData.response)) {
 					if (secondRequestData.checkAndProcessResult(atMostOnce)) {
diff --git a/flow/actorcompiler.h b/flow/actorcompiler.h
index 3a60aa35887..0c526f14d90 100644
--- a/flow/actorcompiler.h
+++ b/flow/actorcompiler.h
@@ -38,10 +38,10 @@ T waitNext(const FutureStream<T>&) = delete;
 #ifndef POST_ACTOR_COMPILER
 
 template <typename T>
-struct Future;
-struct Never;
+class Future;
+class Never;
 template <typename T>
-struct FutureStream;
+class FutureStream;
 
 // These are for intellisense to do proper type inferring, etc. They are no included at build time.
 #ifndef NO_INTELLISENSE

From 5657e3c803481611cba7357cbfecc1709fb6adde Mon Sep 17 00:00:00 2001
From: neethuhaneesha <nbingi@apple.com>
Date: Fri, 17 Feb 2023 15:33:06 -0800
Subject: [PATCH 066/468] Rocksdb knob changes. (#9393)

---
 fdbclient/ServerKnobs.cpp | 25 ++++++++++++++-----------
 1 file changed, 14 insertions(+), 11 deletions(-)

diff --git a/fdbclient/ServerKnobs.cpp b/fdbclient/ServerKnobs.cpp
index 90ba68695a6..eaee631267b 100644
--- a/fdbclient/ServerKnobs.cpp
+++ b/fdbclient/ServerKnobs.cpp
@@ -363,11 +363,11 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi
 
 	// KeyValueStoreRocksDB
 	init( ROCKSDB_LEVEL_COMPACTION_DYNAMIC_LEVEL_BYTES,         true ); if( randomize && BUGGIFY )  ROCKSDB_LEVEL_COMPACTION_DYNAMIC_LEVEL_BYTES = false;
- 	init( ROCKSDB_SUGGEST_COMPACT_CLEAR_RANGE,                  true ); if( randomize && BUGGIFY )  ROCKSDB_SUGGEST_COMPACT_CLEAR_RANGE = false;
+	init( ROCKSDB_SUGGEST_COMPACT_CLEAR_RANGE,                 false );
 	init( ROCKSDB_THREAD_PROMISE_PRIORITY,                      7500 );
 	init( ROCKSDB_READER_THREAD_PRIORITY,                          0 );
  	init( ROCKSDB_WRITER_THREAD_PRIORITY,                          0 );
-	init( ROCKSDB_BACKGROUND_PARALLELISM,                          4 );
+	init( ROCKSDB_BACKGROUND_PARALLELISM,                          2 );
 	init( ROCKSDB_READ_PARALLELISM,                                4 );
 	// Use a smaller memtable in simulation to avoid OOMs.
 	int64_t memtableBytes = isSimulated ? 32 * 1024 : 512 * 1024 * 1024;
@@ -377,12 +377,15 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi
 	init( ROCKSDB_PERIODIC_COMPACTION_SECONDS,                     0 );
 	init( ROCKSDB_PREFIX_LEN,                                      0 );
 	// If rocksdb block cache size is 0, the default 8MB is used.
-	int64_t blockCacheSize = isSimulated ? 0 : 1024 * 1024 * 1024 /* 1GB */;
+	int64_t blockCacheSize = isSimulated ? 16 * 1024 * 1024 : 2147483648 /* 2GB */;
 	init( ROCKSDB_BLOCK_CACHE_SIZE,                   blockCacheSize );
 	init( ROCKSDB_METRICS_DELAY,                                60.0 );
-	init( ROCKSDB_READ_VALUE_TIMEOUT,                            5.0 );
-	init( ROCKSDB_READ_VALUE_PREFIX_TIMEOUT,                     5.0 );
-	init( ROCKSDB_READ_RANGE_TIMEOUT,                            5.0 );
+	// ROCKSDB_READ_VALUE_TIMEOUT, ROCKSDB_READ_VALUE_PREFIX_TIMEOUT, ROCKSDB_READ_RANGE_TIMEOUT knobs:
+	// In simulation, increasing the read operation timeouts to 5 minutes, as some of the tests have
+	// very high load and single read thread cannot process all the load within the timeouts.
+	init( ROCKSDB_READ_VALUE_TIMEOUT,      isSimulated ? 300.0 : 5.0 );
+	init( ROCKSDB_READ_VALUE_PREFIX_TIMEOUT, isSimulated ? 300.0 : 5.0 );
+	init( ROCKSDB_READ_RANGE_TIMEOUT,      isSimulated ? 300.0 : 5.0 );
 	init( ROCKSDB_READ_QUEUE_WAIT,                               1.0 );
 	init( ROCKSDB_READ_QUEUE_HARD_MAX,                          1000 );
 	init( ROCKSDB_READ_QUEUE_SOFT_MAX,                           500 );
@@ -402,7 +405,7 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi
 
 	init( ROCKSDB_PERFCONTEXT_ENABLE,                          false ); if( randomize && BUGGIFY ) ROCKSDB_PERFCONTEXT_ENABLE = deterministicRandom()->coinflip();
 	init( ROCKSDB_PERFCONTEXT_SAMPLE_RATE,                    0.0001 );
-	init( ROCKSDB_MAX_SUBCOMPACTIONS,                              2 );
+	init( ROCKSDB_MAX_SUBCOMPACTIONS,                              0 );
 	init( ROCKSDB_SOFT_PENDING_COMPACT_BYTES_LIMIT,      64000000000 ); // 64GB, Rocksdb option, Writes will slow down.
 	init( ROCKSDB_HARD_PENDING_COMPACT_BYTES_LIMIT,     100000000000 ); // 100GB, Rocksdb option, Writes will stall.
 	init( ROCKSDB_CAN_COMMIT_COMPACT_BYTES_LIMIT,        50000000000 ); // 50GB, Commit waits.
@@ -411,13 +414,13 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi
 	init( ROCKSDB_DISABLE_WAL_EXPERIMENTAL,                    false );
 	// If ROCKSDB_SINGLEKEY_DELETES_ON_CLEARRANGE is enabled, disable ROCKSDB_ENABLE_CLEAR_RANGE_EAGER_READS knob.
 	// These knobs have contrary functionality.
-	init( ROCKSDB_SINGLEKEY_DELETES_ON_CLEARRANGE,             false ); if( randomize && BUGGIFY ) ROCKSDB_SINGLEKEY_DELETES_ON_CLEARRANGE = deterministicRandom()->coinflip();
-	init( ROCKSDB_SINGLEKEY_DELETES_BYTES_LIMIT,              200000 ); // 200KB
-	init( ROCKSDB_ENABLE_CLEAR_RANGE_EAGER_READS,               true ); if( randomize && BUGGIFY ) ROCKSDB_ENABLE_CLEAR_RANGE_EAGER_READS = deterministicRandom()->coinflip();
+	init( ROCKSDB_SINGLEKEY_DELETES_ON_CLEARRANGE,              true );
+	init( ROCKSDB_SINGLEKEY_DELETES_BYTES_LIMIT,               10000 ); // 10KB
+	init( ROCKSDB_ENABLE_CLEAR_RANGE_EAGER_READS,              false );
 	// ROCKSDB_STATS_LEVEL=1 indicates rocksdb::StatsLevel::kExceptHistogramOrTimers
 	// Refer StatsLevel: https://github.com/facebook/rocksdb/blob/main/include/rocksdb/statistics.h#L594
 	init( ROCKSDB_STATS_LEVEL,                                     1 ); if( randomize && BUGGIFY ) ROCKSDB_STATS_LEVEL = deterministicRandom()->randomInt(0, 6);
-	init( ROCKSDB_ENABLE_COMPACT_ON_DELETION,                   true );
+	init( ROCKSDB_ENABLE_COMPACT_ON_DELETION,                  false );
 	// CDCF: CompactOnDeletionCollectorFactory. The below 3 are parameters of the CompactOnDeletionCollectorFactory
 	// which controls the compaction on deleted data.
 	init( ROCKSDB_CDCF_SLIDING_WINDOW_SIZE,                      128 );

From 950c04797a34791c8c7535e3e62c5011babf166b Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Wed, 22 Feb 2023 11:07:27 -0800
Subject: [PATCH 067/468] Fix compiler warnings

---
 fdbrpc/sim2.actor.cpp                | 2 +-
 fdbserver/ResolutionBalancer.actor.h | 1 +
 fdbserver/TLogServer.actor.cpp       | 1 -
 3 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/fdbrpc/sim2.actor.cpp b/fdbrpc/sim2.actor.cpp
index 958fa25e5e7..f23e377b176 100644
--- a/fdbrpc/sim2.actor.cpp
+++ b/fdbrpc/sim2.actor.cpp
@@ -2511,7 +2511,7 @@ Future<Void> waitUntilDiskReady(Reference<DiskParameters> diskParameters, int64_
 
 	if (diskParameters->nextOperation < now())
 		diskParameters->nextOperation = now();
-	diskParameters->nextOperation += (1.0 / diskParameters->iops) + (size / diskParameters->bandwidth);
+	diskParameters->nextOperation += (1.0 / diskParameters->iops) + (1.0 * size / diskParameters->bandwidth);
 
 	double randomLatency;
 	if (sync) {
diff --git a/fdbserver/ResolutionBalancer.actor.h b/fdbserver/ResolutionBalancer.actor.h
index 263195f0e54..2e969a7d263 100644
--- a/fdbserver/ResolutionBalancer.actor.h
+++ b/fdbserver/ResolutionBalancer.actor.h
@@ -33,6 +33,7 @@
 #include "flow/Arena.h"
 #include "flow/IRandom.h"
 #include "flow/genericactors.actor.h"
+#include "flow/actorcompiler.h" // This must be the last #include
 
 struct ResolutionBalancer {
 	AsyncVar<Standalone<VectorRef<ResolverMoveRef>>> resolverChanges;
diff --git a/fdbserver/TLogServer.actor.cpp b/fdbserver/TLogServer.actor.cpp
index 757d81d67f9..a4193ea90f1 100644
--- a/fdbserver/TLogServer.actor.cpp
+++ b/fdbserver/TLogServer.actor.cpp
@@ -2887,7 +2887,6 @@ ACTOR Future<Void> pullAsyncData(TLogData* self,
 
 		state Version ver = 0;
 		state std::vector<TagsAndMessage> messages;
-		state bool pullingRecoveryData = endVersion.present() && endVersion.get() == logData->recoveredAt;
 		loop {
 			state bool foundMessage = r->hasMessage();
 			if (!foundMessage || r->version().version != ver) {

From 8ffacc7aa57af7f08ae211e53424db38172f4798 Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Thu, 16 Feb 2023 15:34:43 -0800
Subject: [PATCH 068/468] Add exclude to fdbcli's configure command

Right now this only allows one server address being excluded. This is useful
when the database is unavailable but we want the recruitment to skip some
particular processes.

Manually tested the concept works with a loopback cluster.
---
 fdbcli/ConfigureCommand.actor.cpp   | 6 +++++-
 fdbclient/DatabaseConfiguration.cpp | 2 ++
 fdbclient/ManagementAPI.actor.cpp   | 8 ++++++++
 3 files changed, 15 insertions(+), 1 deletion(-)

diff --git a/fdbcli/ConfigureCommand.actor.cpp b/fdbcli/ConfigureCommand.actor.cpp
index f6209f2f410..17b8d8c2e8b 100644
--- a/fdbcli/ConfigureCommand.actor.cpp
+++ b/fdbcli/ConfigureCommand.actor.cpp
@@ -275,7 +275,8 @@ CommandFactory configureFactory(
         "commit_proxies=<COMMIT_PROXIES>|grv_proxies=<GRV_PROXIES>|logs=<LOGS>|resolvers=<RESOLVERS>>*|"
         "count=<TSS_COUNT>|perpetual_storage_wiggle=<WIGGLE_SPEED>|perpetual_storage_wiggle_locality="
         "<<LOCALITY_KEY>:<LOCALITY_VALUE>|0>|storage_migration_type={disabled|gradual|aggressive}"
-        "|tenant_mode={disabled|optional_experimental|required_experimental}|blob_granules_enabled={0|1}",
+        "|tenant_mode={disabled|optional_experimental|required_experimental}|blob_granules_enabled={0|1}"
+        "|exclude=<ADDRESS...>",
         "change the database configuration",
         "The `new' option, if present, initializes a new database with the given configuration rather than changing "
         "the configuration of an existing one. When used, both a redundancy mode and a storage engine must be "
@@ -309,6 +310,9 @@ CommandFactory configureFactory(
         "tenant_mode=<disabled|optional_experimental|required_experimental>: Sets the tenant mode for the cluster. If "
         "optional, then transactions can be run with or without specifying tenants. If required, all data must be "
         "accessed using tenants.\n\n"
+        "exclude=<ADDRESS...>: Sets the addresses in the format of IP:port pair to be excluded during recruitment. "
+        "Note this should be only used the database is unavailable because of the faulty processes that are blocking "
+        "the recovery from completion. The number of addresses should be less than the replica factor.\n\n"
 
         "See the FoundationDB Administration Guide for more information."));
 
diff --git a/fdbclient/DatabaseConfiguration.cpp b/fdbclient/DatabaseConfiguration.cpp
index 657129f86e2..bd9554018fe 100644
--- a/fdbclient/DatabaseConfiguration.cpp
+++ b/fdbclient/DatabaseConfiguration.cpp
@@ -639,6 +639,8 @@ bool DatabaseConfiguration::setInternal(KeyRef key, ValueRef value) {
 	} else if (ck == LiteralStringRef("blob_granules_enabled")) {
 		parse((&type), value);
 		blobGranulesEnabled = (type != 0);
+	} else if (ck.startsWith("excluded/"_sr)) {
+		// excluded servers: don't keep the state internally
 	} else {
 		return false;
 	}
diff --git a/fdbclient/ManagementAPI.actor.cpp b/fdbclient/ManagementAPI.actor.cpp
index 7487a6083aa..94c74a657c8 100644
--- a/fdbclient/ManagementAPI.actor.cpp
+++ b/fdbclient/ManagementAPI.actor.cpp
@@ -200,6 +200,14 @@ std::map<std::string, std::string> configForToken(std::string const& mode) {
 			}
 			out[p + key] = format("%d", tenantMode);
 		}
+		if (key == "exclude") {
+			AddressExclusion addr = AddressExclusion::parse(value);
+			if (!addr.isValid()) {
+				printf("Error: invalid address format: %s\n", value.c_str());
+				return out;
+			}
+			out[encodeExcludedServersKey(addr)] = "";
+		}
 		return out;
 	}
 

From 550be07d2674deebd1c35efa48e2b77e22871a1f Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Thu, 16 Feb 2023 16:17:46 -0800
Subject: [PATCH 069/468] Allow a comma separated list of excluded addresses

---
 fdbcli/ConfigureCommand.actor.cpp |  7 ++++---
 fdbclient/ManagementAPI.actor.cpp | 18 +++++++++++++-----
 2 files changed, 17 insertions(+), 8 deletions(-)

diff --git a/fdbcli/ConfigureCommand.actor.cpp b/fdbcli/ConfigureCommand.actor.cpp
index 17b8d8c2e8b..16e98db58f3 100644
--- a/fdbcli/ConfigureCommand.actor.cpp
+++ b/fdbcli/ConfigureCommand.actor.cpp
@@ -310,9 +310,10 @@ CommandFactory configureFactory(
         "tenant_mode=<disabled|optional_experimental|required_experimental>: Sets the tenant mode for the cluster. If "
         "optional, then transactions can be run with or without specifying tenants. If required, all data must be "
         "accessed using tenants.\n\n"
-        "exclude=<ADDRESS...>: Sets the addresses in the format of IP:port pair to be excluded during recruitment. "
-        "Note this should be only used the database is unavailable because of the faulty processes that are blocking "
-        "the recovery from completion. The number of addresses should be less than the replica factor.\n\n"
+        "exclude=<ADDRESS...>: Sets the addresses in the format of IP1:port1,IP2:port2 pairs to be excluded during "
+        "recruitment. Note this should be only used when the database is unavailable because of the faulty processes "
+        "that are blocking the recovery from completion. The number of addresses should be less than the replication "
+        "factor to avoid data loss.\n\n"
 
         "See the FoundationDB Administration Guide for more information."));
 
diff --git a/fdbclient/ManagementAPI.actor.cpp b/fdbclient/ManagementAPI.actor.cpp
index 94c74a657c8..d1ba8ba7900 100644
--- a/fdbclient/ManagementAPI.actor.cpp
+++ b/fdbclient/ManagementAPI.actor.cpp
@@ -201,12 +201,20 @@ std::map<std::string, std::string> configForToken(std::string const& mode) {
 			out[p + key] = format("%d", tenantMode);
 		}
 		if (key == "exclude") {
-			AddressExclusion addr = AddressExclusion::parse(value);
-			if (!addr.isValid()) {
-				printf("Error: invalid address format: %s\n", value.c_str());
-				return out;
+			int p = 0;
+			while (p < value.size()) {
+				int end = value.find_first_of(',', p);
+				if (end == value.npos)
+					end = value.size();
+				auto addrRef = StringRef(value).substr(p, end - p);
+				AddressExclusion addr = AddressExclusion::parse(addrRef);
+				if (addr.isValid()) {
+					out[encodeExcludedServersKey(addr)] = "";
+				} else {
+					printf("Error: invalid address format: %s\n", addrRef.toString().c_str());
+				}
+				p = end + 1;
 			}
-			out[encodeExcludedServersKey(addr)] = "";
 		}
 		return out;
 	}

From e6a64e0604c8db88ad1216080f71c1c100aafecc Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Mon, 20 Feb 2023 17:06:55 -0800
Subject: [PATCH 070/468] Add ClogTlog workload

---
 fdbrpc/sim2.actor.cpp                        |  14 ++
 fdbserver/CMakeLists.txt                     |   1 +
 fdbserver/ClusterRecovery.actor.cpp          |   1 +
 fdbserver/workloads/ClogTlog.actor.cpp       | 179 +++++++++++++++++++
 fdbserver/workloads/RandomClogging.actor.cpp |   1 -
 tests/CMakeLists.txt                         |   1 +
 tests/ClogTlog.toml                          |  21 +++
 7 files changed, 217 insertions(+), 1 deletion(-)
 create mode 100644 fdbserver/workloads/ClogTlog.actor.cpp
 create mode 100644 tests/ClogTlog.toml

diff --git a/fdbrpc/sim2.actor.cpp b/fdbrpc/sim2.actor.cpp
index f23e377b176..0371f55360e 100644
--- a/fdbrpc/sim2.actor.cpp
+++ b/fdbrpc/sim2.actor.cpp
@@ -21,6 +21,7 @@
 #include <cinttypes>
 #include <memory>
 #include <string>
+#include <utility>
 
 #include "contrib/fmt-8.1.1/include/fmt/format.h"
 #include "fdbrpc/simulator.h"
@@ -139,6 +140,10 @@ struct SimClogging {
 		if (!g_simulator.speedUpSimulation && !stableConnection && clogPairUntil.count(pair))
 			t = std::max(t, clogPairUntil[pair]);
 
+		auto p = std::make_pair(from, to);
+		if (!g_simulator.speedUpSimulation && !stableConnection && clogProcessPairUntil.count(p))
+			t = std::max(t, clogProcessPairUntil[p]);
+
 		if (!g_simulator.speedUpSimulation && !stableConnection && clogRecvUntil.count(to.ip))
 			t = std::max(t, clogRecvUntil[to.ip]);
 
@@ -149,6 +154,14 @@ struct SimClogging {
 		auto& u = clogPairUntil[std::make_pair(from, to)];
 		u = std::max(u, now() + t);
 	}
+
+	// Clog a pair of processes until a time. This is more fine-grained than
+	// the IPAddress based one.
+	void clogPairFor(const NetworkAddress& from, const NetworkAddress& to, double t) {
+		auto& u = clogProcessPairUntil[std::make_pair(from, to)];
+		u = std::max(u, now() + t);
+	}
+
 	void clogSendFor(const IPAddress& from, double t) {
 		auto& u = clogSendUntil[from];
 		u = std::max(u, now() + t);
@@ -168,6 +181,7 @@ struct SimClogging {
 	std::map<IPAddress, double> clogSendUntil, clogRecvUntil;
 	std::map<std::pair<IPAddress, IPAddress>, double> clogPairUntil;
 	std::map<std::pair<IPAddress, IPAddress>, double> clogPairLatency;
+	std::map<std::pair<NetworkAddress, NetworkAddress>, double> clogProcessPairUntil;
 	double halfLatency() const {
 		double a = deterministicRandom()->random01();
 		const double pFast = 0.999;
diff --git a/fdbserver/CMakeLists.txt b/fdbserver/CMakeLists.txt
index 3ffe1febbd2..f9c2735bb52 100644
--- a/fdbserver/CMakeLists.txt
+++ b/fdbserver/CMakeLists.txt
@@ -199,6 +199,7 @@ set(FDBSERVER_SRCS
   workloads/ClearSingleRange.actor.cpp
   workloads/ClientTransactionProfileCorrectness.actor.cpp
   workloads/ClogSingleConnection.actor.cpp
+  workloads/ClogTlog.actor.cpp
   workloads/CommitBugCheck.actor.cpp
   workloads/ConfigIncrement.actor.cpp
   workloads/ConfigureDatabase.actor.cpp
diff --git a/fdbserver/ClusterRecovery.actor.cpp b/fdbserver/ClusterRecovery.actor.cpp
index 4fa2c93d453..d185cd1ab38 100644
--- a/fdbserver/ClusterRecovery.actor.cpp
+++ b/fdbserver/ClusterRecovery.actor.cpp
@@ -935,6 +935,7 @@ ACTOR Future<std::vector<Standalone<CommitTransactionRef>>> recruitEverything(
 	} else
 		TraceEvent(getRecoveryEventName(ClusterRecoveryEventType::CLUSTER_RECOVERY_STATE_EVENT_NAME).c_str(),
 		           self->dbgid)
+		    .setMaxFieldLength(-1)
 		    .detail("StatusCode", RecoveryStatus::recruiting_transaction_servers)
 		    .detail("Status", RecoveryStatus::names[RecoveryStatus::recruiting_transaction_servers])
 		    .detail("Conf", self->configuration.toString())
diff --git a/fdbserver/workloads/ClogTlog.actor.cpp b/fdbserver/workloads/ClogTlog.actor.cpp
new file mode 100644
index 00000000000..cb844c8ad3c
--- /dev/null
+++ b/fdbserver/workloads/ClogTlog.actor.cpp
@@ -0,0 +1,179 @@
+/*
+ * ClogTlog.actor.cpp
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2023 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "fdbclient/NativeAPI.actor.h"
+#include "fdbserver/ServerDBInfo.actor.h"
+#include "fdbserver/TesterInterface.actor.h"
+#include "fdbserver/workloads/workloads.actor.h"
+#include "fdbrpc/simulator.h"
+#include "flow/Error.h"
+#include "flow/Trace.h"
+#include "flow/flow.h"
+#include "flow/network.h"
+#include "flow/actorcompiler.h" // This must be the last #include.
+
+struct ClogTlogWorkload : TestWorkload {
+	bool enabled;
+	double testDuration;
+	bool processClassUpdated = false;
+	Optional<IPAddress> tlog;
+
+	ClogTlogWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
+		enabled = !clientId; // only do this on the "first" client
+		testDuration = getOption(options, LiteralStringRef("testDuration"), 100.0);
+	}
+
+	std::string description() const override {
+		if (&g_simulator == g_network)
+			return "ClogTlog";
+		else
+			return "NoRC";
+	}
+
+	Future<Void> setup(Database const& cx) override { return Void(); }
+	Future<Void> start(Database const& cx) override {
+		if (&g_simulator == g_network && enabled)
+			return timeout(reportErrors(clogClient(this, cx), "ClogTlogError"), testDuration, Void());
+		else
+			return Void();
+	}
+	Future<bool> check(Database const& cx) override { return true; }
+	void getMetrics(std::vector<PerfMetric>& m) override {}
+
+	ACTOR void doClog(ISimulator::ProcessInfo* machine, double t, double delay = 0.0) {
+		wait(::delay(delay));
+		g_simulator.clogInterface(machine->address.ip, t);
+	}
+
+	// Change satellite tlogs process class to logs. This is needed because
+	// simulator starts with "unset" class for all satellite processes
+	ACTOR static Future<Void> updateSatelliteProcessClass(ClogTlogWorkload* self, Database cx) {
+		if (self->processClassUpdated)
+			return Void();
+		state Reference<ReadYourWritesTransaction> tx = makeReference<ReadYourWritesTransaction>(cx);
+		loop {
+			try {
+				tx->setOption(FDBTransactionOptions::RAW_ACCESS);
+				tx->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES);
+
+				for (int i = 0; i < self->dbInfo->get().logSystemConfig.tLogs.size(); i++) {
+					const auto& tlogset = self->dbInfo->get().logSystemConfig.tLogs[i];
+					if (!tlogset.isLocal)
+						continue;
+					for (const auto& log : tlogset.tLogs) {
+						const NetworkAddress& addr = log.interf().address();
+						Key key =
+						    Key("process/class_type/" + formatIpPort(addr.ip, addr.port))
+						        .withPrefix(
+						            SpecialKeySpace::getModuleRange(SpecialKeySpace::MODULE::CONFIGURATION).begin);
+						tx->set(key, "log"_sr);
+					}
+				}
+
+				wait(tx->commit());
+				self->processClassUpdated = true;
+				return Void();
+			} catch (Error& e) {
+				if (e.code() == error_code_actor_cancelled)
+					throw;
+				TraceEvent("ClogTlogSetProcessClassError").error(e);
+				wait(tx->onError(e));
+			}
+		}
+	}
+
+	// Clog a random tlog with all proxies so that this triggers a recovery
+	// and the recovery will become stuck.
+	ACTOR static Future<Void> clogTlog(ClogTlogWorkload* self, Database cx, double seconds) {
+		ASSERT(self->dbInfo->get().recoveryState >= RecoveryState::ACCEPTING_COMMITS);
+		wait(updateSatelliteProcessClass(self, cx));
+
+		IPAddress cc = self->dbInfo->get().clusterInterface.address().ip;
+		std::vector<IPAddress> proxies;
+		for (const auto& proxy : self->dbInfo->get().client.commitProxies) {
+			const auto& ip = proxy.address().ip;
+			if (cc == ip) {
+				TraceEvent("ClogTlogSkipCC").detail("IP", ip);
+			} else {
+				proxies.push_back(ip);
+			}
+		}
+		ASSERT(proxies.size() > 0);
+
+		// Find all primary tlogs
+		std::vector<IPAddress> logs;
+		bool found = false;
+		for (int i = 0; i < self->dbInfo->get().logSystemConfig.tLogs.size(); i++) {
+			const auto& tlogset = self->dbInfo->get().logSystemConfig.tLogs[i];
+			if (!tlogset.isLocal)
+				continue;
+			for (const auto& log : tlogset.tLogs) {
+				const NetworkAddress& addr = log.interf().address();
+				if (cc == addr.ip) {
+					TraceEvent("ClogTlogSkipCC").detail("IP", addr.ip);
+				} else {
+					if (self->tlog.present() && self->tlog.get() == addr.ip) {
+						found = true;
+					}
+					logs.push_back(addr.ip);
+				}
+				// logs.push_back(addr.ip);
+			}
+		}
+		ASSERT(logs.size() > 0);
+
+		// Clog all proxies and one satellite tlogs
+		self->tlog = found ? self->tlog : logs[0];
+		for (const auto& proxy : proxies) {
+			g_simulator.clogPair(proxy, self->tlog.get(), seconds);
+			/*for (const auto& log : logs) {
+			    g_simulator.clogPair(proxy, log, seconds);
+			}*/
+		}
+		return Void();
+	}
+
+	ACTOR Future<Void> clogClient(ClogTlogWorkload* self, Database cx) {
+		double startTime = now();
+		state double workloadEnd = now() + self->testDuration;
+		TraceEvent("ClogTlog").detail("StartTime", startTime).detail("EndTime", workloadEnd);
+		loop {
+			while (self->dbInfo->get().recoveryState < RecoveryState::ACCEPTING_COMMITS) {
+				wait(self->dbInfo->onChange());
+			}
+
+			wait(clogTlog(self, cx, workloadEnd - now()));
+			loop choose {
+				when(wait(self->dbInfo->onChange())) {
+					if (self->dbInfo->get().recoveryState < RecoveryState::ACCEPTING_COMMITS) {
+						break;
+					}
+				}
+				when(wait(delayUntil(workloadEnd))) {
+					// Clog a random tlog and all proxies
+					TraceEvent("ClogDone").detail("RecoveryState", self->dbInfo->get().recoveryState);
+					return Void();
+				}
+			}
+		}
+	}
+};
+
+WorkloadFactory<ClogTlogWorkload> ClogTlogWorkloadFactory("ClogTlog");
diff --git a/fdbserver/workloads/RandomClogging.actor.cpp b/fdbserver/workloads/RandomClogging.actor.cpp
index 7afa89e93b4..78bb553c63f 100644
--- a/fdbserver/workloads/RandomClogging.actor.cpp
+++ b/fdbserver/workloads/RandomClogging.actor.cpp
@@ -106,7 +106,6 @@ struct RandomCloggingWorkload : TestWorkload {
 			for (int i = 0; i < 10; i++)
 				self->clogRandomPair(t);
 
-			std::vector<Future<Void>> cloggers;
 			for (int i = 0; i < swizzled.size(); i++)
 				self->doClog(swizzled[i], ends[i] - starts[i], starts[i]);
 		}
diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt
index 3dc6d17595b..f8ddb6ab79f 100644
--- a/tests/CMakeLists.txt
+++ b/tests/CMakeLists.txt
@@ -53,6 +53,7 @@ if(WITH_PYTHON)
   add_fdb_test(TEST_FILES BGServerCommonUnit.toml)
   add_fdb_test(TEST_FILES BlobGranuleFileUnit.toml)
   add_fdb_test(TEST_FILES BlobManagerUnit.toml)
+  add_fdb_test(TEST_FILES ClogTlog.toml IGNORE)
   add_fdb_test(TEST_FILES ConsistencyCheck.txt IGNORE)
   add_fdb_test(TEST_FILES DDMetricsExclude.txt IGNORE)
   add_fdb_test(TEST_FILES DataDistributionMetrics.txt IGNORE)
diff --git a/tests/ClogTlog.toml b/tests/ClogTlog.toml
new file mode 100644
index 00000000000..e6228a141c0
--- /dev/null
+++ b/tests/ClogTlog.toml
@@ -0,0 +1,21 @@
+[configuration]
+generateFearless = false
+processesPerMachine = 1
+machineCount = 20
+commitProxyCount = 4
+config = 'triple'
+desiredTLogCount = 6
+
+[[test]]
+testTitle = 'ClogTlog'
+
+    [[test.workload]]
+    testName = 'Cycle'
+    nodeCount = 30000
+    transactionsPerSecond = 2500.0
+    testDuration = 120.0
+    expectedRate = 0
+
+    [[test.workload]]
+    testName = 'ClogTlog'
+    testDuration = 1000.0

From 5b345b265a212bc407dfe27cd46296a9c096d8a2 Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Tue, 21 Feb 2023 09:00:17 -0800
Subject: [PATCH 071/468] Update clogTlog workload to be single region

---
 fdbserver/workloads/ClogTlog.actor.cpp | 46 +++++++++++++++++---------
 tests/ClogTlog.toml                    |  4 +++
 2 files changed, 34 insertions(+), 16 deletions(-)

diff --git a/fdbserver/workloads/ClogTlog.actor.cpp b/fdbserver/workloads/ClogTlog.actor.cpp
index cb844c8ad3c..465a16e3a63 100644
--- a/fdbserver/workloads/ClogTlog.actor.cpp
+++ b/fdbserver/workloads/ClogTlog.actor.cpp
@@ -19,6 +19,7 @@
  */
 
 #include "fdbclient/NativeAPI.actor.h"
+#include "fdbrpc/Locality.h"
 #include "fdbserver/ServerDBInfo.actor.h"
 #include "fdbserver/TesterInterface.actor.h"
 #include "fdbserver/workloads/workloads.actor.h"
@@ -33,7 +34,7 @@ struct ClogTlogWorkload : TestWorkload {
 	bool enabled;
 	double testDuration;
 	bool processClassUpdated = false;
-	Optional<IPAddress> tlog;
+	Optional<IPAddress> tlog; // the tlog to be clogged with all other processes except the CC
 
 	ClogTlogWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
 		enabled = !clientId; // only do this on the "first" client
@@ -102,20 +103,24 @@ struct ClogTlogWorkload : TestWorkload {
 	// Clog a random tlog with all proxies so that this triggers a recovery
 	// and the recovery will become stuck.
 	ACTOR static Future<Void> clogTlog(ClogTlogWorkload* self, Database cx, double seconds) {
-		ASSERT(self->dbInfo->get().recoveryState >= RecoveryState::ACCEPTING_COMMITS);
-		wait(updateSatelliteProcessClass(self, cx));
+		ASSERT(self->dbInfo->get().recoveryState >= RecoveryState::RECOVERY_TRANSACTION);
+		// wait(updateSatelliteProcessClass(self, cx));
 
 		IPAddress cc = self->dbInfo->get().clusterInterface.address().ip;
-		std::vector<IPAddress> proxies;
-		for (const auto& proxy : self->dbInfo->get().client.commitProxies) {
-			const auto& ip = proxy.address().ip;
+		std::vector<IPAddress> ips;
+		std::set<IPAddress> candidates;
+		for (const auto& process : g_simulator.getAllProcesses()) {
+			const auto& ip = process->address.ip;
 			if (cc == ip) {
 				TraceEvent("ClogTlogSkipCC").detail("IP", ip);
 			} else {
-				proxies.push_back(ip);
+				if (process->startingClass == ProcessClass::LogClass) {
+					candidates.insert(ip);
+				}
+				ips.push_back(ip);
 			}
 		}
-		ASSERT(proxies.size() > 0);
+		ASSERT(ips.size() > 0);
 
 		// Find all primary tlogs
 		std::vector<IPAddress> logs;
@@ -139,13 +144,22 @@ struct ClogTlogWorkload : TestWorkload {
 		}
 		ASSERT(logs.size() > 0);
 
-		// Clog all proxies and one satellite tlogs
-		self->tlog = found ? self->tlog : logs[0];
-		for (const auto& proxy : proxies) {
+		// Clog all proxies and one tlogs
+		if (!found) {
+			// Choose a tlog, preferring the one of "log" class
+			self->tlog.reset();
+			for (const auto& log : logs) {
+				if (candidates.find(log) != candidates.end()) {
+					self->tlog = log;
+					break;
+				}
+			}
+			if (!self->tlog.present()) {
+				self->tlog = logs[0];
+			}
+		}
+		for (const auto& proxy : ips) {
 			g_simulator.clogPair(proxy, self->tlog.get(), seconds);
-			/*for (const auto& log : logs) {
-			    g_simulator.clogPair(proxy, log, seconds);
-			}*/
 		}
 		return Void();
 	}
@@ -155,14 +169,14 @@ struct ClogTlogWorkload : TestWorkload {
 		state double workloadEnd = now() + self->testDuration;
 		TraceEvent("ClogTlog").detail("StartTime", startTime).detail("EndTime", workloadEnd);
 		loop {
-			while (self->dbInfo->get().recoveryState < RecoveryState::ACCEPTING_COMMITS) {
+			while (self->dbInfo->get().recoveryState < RecoveryState::RECOVERY_TRANSACTION) {
 				wait(self->dbInfo->onChange());
 			}
 
 			wait(clogTlog(self, cx, workloadEnd - now()));
 			loop choose {
 				when(wait(self->dbInfo->onChange())) {
-					if (self->dbInfo->get().recoveryState < RecoveryState::ACCEPTING_COMMITS) {
+					if (self->dbInfo->get().recoveryState < RecoveryState::RECOVERY_TRANSACTION) {
 						break;
 					}
 				}
diff --git a/tests/ClogTlog.toml b/tests/ClogTlog.toml
index e6228a141c0..104e3da653d 100644
--- a/tests/ClogTlog.toml
+++ b/tests/ClogTlog.toml
@@ -6,6 +6,10 @@ commitProxyCount = 4
 config = 'triple'
 desiredTLogCount = 6
 
+[[knobs]]
+enable_worker_health_monitor = true
+cc_enable_worker_health_monitor = true
+
 [[test]]
 testTitle = 'ClogTlog'
 

From 314a4d1ee103676b8cf1ce45e285f0c0d8c9187f Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Wed, 22 Feb 2023 16:43:21 -0800
Subject: [PATCH 072/468] Exclude failed tlog if recovery stuck more than 30s

Because the tlog is clogged, recovery can stuck in initializing_transaction_servers.
This exclude allows the recovery to complete.
---
 fdbserver/workloads/ClogTlog.actor.cpp | 99 ++++++++++++++------------
 tests/ClogTlog.toml                    |  2 +-
 2 files changed, 54 insertions(+), 47 deletions(-)

diff --git a/fdbserver/workloads/ClogTlog.actor.cpp b/fdbserver/workloads/ClogTlog.actor.cpp
index 465a16e3a63..1b9a6a865f7 100644
--- a/fdbserver/workloads/ClogTlog.actor.cpp
+++ b/fdbserver/workloads/ClogTlog.actor.cpp
@@ -18,8 +18,11 @@
  * limitations under the License.
  */
 
+#include "fdbclient/GenericManagementAPI.actor.h"
+#include "fdbclient/ManagementAPI.actor.h"
 #include "fdbclient/NativeAPI.actor.h"
 #include "fdbrpc/Locality.h"
+#include "fdbserver/RecoveryState.h"
 #include "fdbserver/ServerDBInfo.actor.h"
 #include "fdbserver/TesterInterface.actor.h"
 #include "fdbserver/workloads/workloads.actor.h"
@@ -34,7 +37,7 @@ struct ClogTlogWorkload : TestWorkload {
 	bool enabled;
 	double testDuration;
 	bool processClassUpdated = false;
-	Optional<IPAddress> tlog; // the tlog to be clogged with all other processes except the CC
+	Optional<NetworkAddress> tlog; // the tlog to be clogged with all other processes except the CC
 
 	ClogTlogWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
 		enabled = !clientId; // only do this on the "first" client
@@ -63,48 +66,10 @@ struct ClogTlogWorkload : TestWorkload {
 		g_simulator.clogInterface(machine->address.ip, t);
 	}
 
-	// Change satellite tlogs process class to logs. This is needed because
-	// simulator starts with "unset" class for all satellite processes
-	ACTOR static Future<Void> updateSatelliteProcessClass(ClogTlogWorkload* self, Database cx) {
-		if (self->processClassUpdated)
-			return Void();
-		state Reference<ReadYourWritesTransaction> tx = makeReference<ReadYourWritesTransaction>(cx);
-		loop {
-			try {
-				tx->setOption(FDBTransactionOptions::RAW_ACCESS);
-				tx->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES);
-
-				for (int i = 0; i < self->dbInfo->get().logSystemConfig.tLogs.size(); i++) {
-					const auto& tlogset = self->dbInfo->get().logSystemConfig.tLogs[i];
-					if (!tlogset.isLocal)
-						continue;
-					for (const auto& log : tlogset.tLogs) {
-						const NetworkAddress& addr = log.interf().address();
-						Key key =
-						    Key("process/class_type/" + formatIpPort(addr.ip, addr.port))
-						        .withPrefix(
-						            SpecialKeySpace::getModuleRange(SpecialKeySpace::MODULE::CONFIGURATION).begin);
-						tx->set(key, "log"_sr);
-					}
-				}
-
-				wait(tx->commit());
-				self->processClassUpdated = true;
-				return Void();
-			} catch (Error& e) {
-				if (e.code() == error_code_actor_cancelled)
-					throw;
-				TraceEvent("ClogTlogSetProcessClassError").error(e);
-				wait(tx->onError(e));
-			}
-		}
-	}
-
 	// Clog a random tlog with all proxies so that this triggers a recovery
 	// and the recovery will become stuck.
 	ACTOR static Future<Void> clogTlog(ClogTlogWorkload* self, Database cx, double seconds) {
 		ASSERT(self->dbInfo->get().recoveryState >= RecoveryState::RECOVERY_TRANSACTION);
-		// wait(updateSatelliteProcessClass(self, cx));
 
 		IPAddress cc = self->dbInfo->get().clusterInterface.address().ip;
 		std::vector<IPAddress> ips;
@@ -123,7 +88,7 @@ struct ClogTlogWorkload : TestWorkload {
 		ASSERT(ips.size() > 0);
 
 		// Find all primary tlogs
-		std::vector<IPAddress> logs;
+		std::vector<NetworkAddress> logs;
 		bool found = false;
 		for (int i = 0; i < self->dbInfo->get().logSystemConfig.tLogs.size(); i++) {
 			const auto& tlogset = self->dbInfo->get().logSystemConfig.tLogs[i];
@@ -134,10 +99,10 @@ struct ClogTlogWorkload : TestWorkload {
 				if (cc == addr.ip) {
 					TraceEvent("ClogTlogSkipCC").detail("IP", addr.ip);
 				} else {
-					if (self->tlog.present() && self->tlog.get() == addr.ip) {
+					if (self->tlog.present() && self->tlog.get() == addr) {
 						found = true;
 					}
-					logs.push_back(addr.ip);
+					logs.push_back(addr);
 				}
 				// logs.push_back(addr.ip);
 			}
@@ -149,7 +114,7 @@ struct ClogTlogWorkload : TestWorkload {
 			// Choose a tlog, preferring the one of "log" class
 			self->tlog.reset();
 			for (const auto& log : logs) {
-				if (candidates.find(log) != candidates.end()) {
+				if (candidates.find(log.ip) != candidates.end()) {
 					self->tlog = log;
 					break;
 				}
@@ -159,15 +124,50 @@ struct ClogTlogWorkload : TestWorkload {
 			}
 		}
 		for (const auto& proxy : ips) {
-			g_simulator.clogPair(proxy, self->tlog.get(), seconds);
+			if (proxy != self->tlog.get().ip) {
+				g_simulator.clogPair(proxy, self->tlog.get().ip, seconds);
+				g_simulator.clogPair(self->tlog.get().ip, proxy, seconds);
+			}
 		}
 		return Void();
 	}
 
+	ACTOR static Future<Void> excludeFailedLog(ClogTlogWorkload* self, Database cx) {
+		state Future<Void> timeout = delay(30);
+
+		loop choose {
+			when(wait(self->dbInfo->onChange())) {
+				if (self->dbInfo->get().recoveryState >= RecoveryState::ACCEPTING_COMMITS) {
+					return Void();
+				}
+				timeout = delay(30);
+			}
+			when(wait(timeout)) {
+				// recovery state hasn't changed in 10s, exclude the failed tlog
+				std::vector<StringRef> tokens;
+				state Optional<ConfigureAutoResult> conf;
+
+				// tokens.push_back("configure"_sr);
+				TraceEvent("ExcludeFailedLog").detail("TLog", self->tlog.get());
+				tokens.push_back(StringRef("exclude=" + self->tlog.get().toString()));
+				ConfigurationResult r =
+				    wait(ManagementAPI::changeConfig(cx.getReference(), tokens, conf, /*force=*/true));
+				TraceEvent("ExcludeFailedLog")
+				    .detail("Result", r)
+				    .detail("ConfigIsValid", conf.present() && conf.get().isValid());
+				return Void();
+			}
+		}
+	}
+
 	ACTOR Future<Void> clogClient(ClogTlogWorkload* self, Database cx) {
+		// Let cycle workload issue some transactions.
+		wait(delay(20.0));
+
 		double startTime = now();
 		state double workloadEnd = now() + self->testDuration;
 		TraceEvent("ClogTlog").detail("StartTime", startTime).detail("EndTime", workloadEnd);
+		state Future<Void> excludeLog;
 		loop {
 			while (self->dbInfo->get().recoveryState < RecoveryState::RECOVERY_TRANSACTION) {
 				wait(self->dbInfo->onChange());
@@ -179,13 +179,20 @@ struct ClogTlogWorkload : TestWorkload {
 					if (self->dbInfo->get().recoveryState < RecoveryState::RECOVERY_TRANSACTION) {
 						break;
 					}
+					if (self->dbInfo->get().recoveryState == RecoveryState::FULLY_RECOVERED) {
+						TraceEvent("ClogDoneFullyRecovered");
+						return Void();
+					}
 				}
 				when(wait(delayUntil(workloadEnd))) {
-					// Clog a random tlog and all proxies
-					TraceEvent("ClogDone").detail("RecoveryState", self->dbInfo->get().recoveryState);
+					// Expect to reach fully recovered state before workload ends
+					TraceEvent(SevError, "ClogTlogFailure").detail("RecoveryState", self->dbInfo->get().recoveryState);
 					return Void();
 				}
 			}
+			if (self->dbInfo->get().recoveryState <= RecoveryState::ALL_LOGS_RECRUITED) {
+				excludeLog = excludeFailedLog(self, cx);
+			}
 		}
 	}
 };
diff --git a/tests/ClogTlog.toml b/tests/ClogTlog.toml
index 104e3da653d..c92455d34f6 100644
--- a/tests/ClogTlog.toml
+++ b/tests/ClogTlog.toml
@@ -22,4 +22,4 @@ testTitle = 'ClogTlog'
 
     [[test.workload]]
     testName = 'ClogTlog'
-    testDuration = 1000.0
+    testDuration = 3000.0

From 51375c53c8dffa7ae79293d329bd5f5fa76eb5c6 Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Thu, 23 Feb 2023 12:37:15 -0800
Subject: [PATCH 073/468] Change to only clog once for a particular tlog

If we repeat clogging, different tlogs may be excluded, which can cause the
recovery to stuck.
---
 fdbrpc/sim2.actor.cpp                  |  19 ++++
 fdbrpc/simulator.h                     |   1 +
 fdbserver/workloads/ClogTlog.actor.cpp | 129 ++++++++++++-------------
 3 files changed, 80 insertions(+), 69 deletions(-)

diff --git a/fdbrpc/sim2.actor.cpp b/fdbrpc/sim2.actor.cpp
index 0371f55360e..5aa2597b893 100644
--- a/fdbrpc/sim2.actor.cpp
+++ b/fdbrpc/sim2.actor.cpp
@@ -155,6 +155,19 @@ struct SimClogging {
 		u = std::max(u, now() + t);
 	}
 
+	void unclogPair(const IPAddress& from, const IPAddress& to) {
+		auto pair = std::make_pair(from, to);
+		auto it = clogPairUntil.find(pair);
+		if (it != clogPairUntil.end()) {
+			clogPairUntil.erase(it);
+		}
+
+		auto i = clogPairLatency.find(std::make_pair(from, to));
+		if (i == clogPairLatency.end()) {
+			clogPairLatency.erase(i);
+		}
+	}
+
 	// Clog a pair of processes until a time. This is more fine-grained than
 	// the IPAddress based one.
 	void clogPairFor(const NetworkAddress& from, const NetworkAddress& to, double t) {
@@ -2064,6 +2077,12 @@ class Sim2 final : public ISimulator, public INetworkConnections {
 		TraceEvent("CloggingPair").detail("From", from).detail("To", to).detail("Seconds", seconds);
 		g_clogging.clogPairFor(from, to, seconds);
 	}
+
+	void unclogPair(const IPAddress& from, const IPAddress& to) override {
+		TraceEvent("UncloggingPair").detail("From", from).detail("To", to);
+		g_clogging.unclogPair(from, to);
+	}
+
 	std::vector<ProcessInfo*> getAllProcesses() const override {
 		std::vector<ProcessInfo*> processes;
 		for (auto& c : machines) {
diff --git a/fdbrpc/simulator.h b/fdbrpc/simulator.h
index 62a8f7c6792..557e6fcdf7b 100644
--- a/fdbrpc/simulator.h
+++ b/fdbrpc/simulator.h
@@ -400,6 +400,7 @@ class ISimulator : public INetwork {
 
 	virtual void clogInterface(const IPAddress& ip, double seconds, ClogMode mode = ClogDefault) = 0;
 	virtual void clogPair(const IPAddress& from, const IPAddress& to, double seconds) = 0;
+	virtual void unclogPair(const IPAddress& from, const IPAddress& to) = 0;
 	virtual std::vector<ProcessInfo*> getAllProcesses() const = 0;
 	virtual ProcessInfo* getProcessByAddress(NetworkAddress const& address) = 0;
 	virtual MachineInfo* getMachineByNetworkAddress(NetworkAddress const& address) = 0;
diff --git a/fdbserver/workloads/ClogTlog.actor.cpp b/fdbserver/workloads/ClogTlog.actor.cpp
index 1b9a6a865f7..c5a87af6fc2 100644
--- a/fdbserver/workloads/ClogTlog.actor.cpp
+++ b/fdbserver/workloads/ClogTlog.actor.cpp
@@ -36,8 +36,9 @@
 struct ClogTlogWorkload : TestWorkload {
 	bool enabled;
 	double testDuration;
-	bool processClassUpdated = false;
+	bool clogged = false;
 	Optional<NetworkAddress> tlog; // the tlog to be clogged with all other processes except the CC
+	std::vector<std::pair<IPAddress, IPAddress>> cloggedPairs;
 
 	ClogTlogWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
 		enabled = !clientId; // only do this on the "first" client
@@ -66,70 +67,57 @@ struct ClogTlogWorkload : TestWorkload {
 		g_simulator.clogInterface(machine->address.ip, t);
 	}
 
-	// Clog a random tlog with all proxies so that this triggers a recovery
-	// and the recovery will become stuck.
-	ACTOR static Future<Void> clogTlog(ClogTlogWorkload* self, Database cx, double seconds) {
-		ASSERT(self->dbInfo->get().recoveryState >= RecoveryState::RECOVERY_TRANSACTION);
+	// Clog a random tlog with all other processes so that this triggers a recovery
+	// and the recovery may become stuck if the clogged tlog is recruited again.
+	void clogTlog(double seconds) {
+		ASSERT(dbInfo->get().recoveryState >= RecoveryState::RECOVERY_TRANSACTION);
 
-		IPAddress cc = self->dbInfo->get().clusterInterface.address().ip;
-		std::vector<IPAddress> ips;
-		std::set<IPAddress> candidates;
+		IPAddress cc = dbInfo->get().clusterInterface.address().ip;
+		std::vector<IPAddress> ips; // all FDB process IPs
 		for (const auto& process : g_simulator.getAllProcesses()) {
 			const auto& ip = process->address.ip;
-			if (cc == ip) {
-				TraceEvent("ClogTlogSkipCC").detail("IP", ip);
-			} else {
-				if (process->startingClass == ProcessClass::LogClass) {
-					candidates.insert(ip);
-				}
+			if (process->startingClass != ProcessClass::TesterClass) {
 				ips.push_back(ip);
 			}
 		}
 		ASSERT(ips.size() > 0);
 
 		// Find all primary tlogs
-		std::vector<NetworkAddress> logs;
-		bool found = false;
-		for (int i = 0; i < self->dbInfo->get().logSystemConfig.tLogs.size(); i++) {
-			const auto& tlogset = self->dbInfo->get().logSystemConfig.tLogs[i];
+		std::vector<NetworkAddress> logs; // all primary logs except CC
+		for (int i = 0; i < dbInfo->get().logSystemConfig.tLogs.size(); i++) {
+			const auto& tlogset = dbInfo->get().logSystemConfig.tLogs[i];
 			if (!tlogset.isLocal)
 				continue;
 			for (const auto& log : tlogset.tLogs) {
 				const NetworkAddress& addr = log.interf().address();
-				if (cc == addr.ip) {
-					TraceEvent("ClogTlogSkipCC").detail("IP", addr.ip);
-				} else {
-					if (self->tlog.present() && self->tlog.get() == addr) {
-						found = true;
+				if (cc != addr.ip) {
+					if (!tlog.present()) {
+						tlog = addr;
 					}
 					logs.push_back(addr);
 				}
-				// logs.push_back(addr.ip);
 			}
 		}
-		ASSERT(logs.size() > 0);
-
-		// Clog all proxies and one tlogs
-		if (!found) {
-			// Choose a tlog, preferring the one of "log" class
-			self->tlog.reset();
-			for (const auto& log : logs) {
-				if (candidates.find(log.ip) != candidates.end()) {
-					self->tlog = log;
-					break;
-				}
-			}
-			if (!self->tlog.present()) {
-				self->tlog = logs[0];
+		ASSERT(logs.size() > 0 && tlog.present());
+
+		// clog pairs
+		for (const auto& ip : ips) {
+			if (ip != tlog.get().ip && ip != cc) {
+				g_simulator.clogPair(ip, tlog.get().ip, seconds);
+				g_simulator.clogPair(tlog.get().ip, ip, seconds);
+				cloggedPairs.emplace_back(ip, tlog.get().ip);
+				cloggedPairs.emplace_back(tlog.get().ip, ip);
 			}
 		}
-		for (const auto& proxy : ips) {
-			if (proxy != self->tlog.get().ip) {
-				g_simulator.clogPair(proxy, self->tlog.get().ip, seconds);
-				g_simulator.clogPair(self->tlog.get().ip, proxy, seconds);
-			}
+		clogged = true;
+	}
+
+	void unclogAll() {
+		// unclog previously clogged connections
+		for (const auto& pair : cloggedPairs) {
+			g_simulator.unclogPair(pair.first, pair.second);
 		}
-		return Void();
+		cloggedPairs.clear();
 	}
 
 	ACTOR static Future<Void> excludeFailedLog(ClogTlogWorkload* self, Database cx) {
@@ -143,13 +131,15 @@ struct ClogTlogWorkload : TestWorkload {
 				timeout = delay(30);
 			}
 			when(wait(timeout)) {
-				// recovery state hasn't changed in 10s, exclude the failed tlog
+				// recovery state hasn't changed in 30s, exclude the failed tlog
 				std::vector<StringRef> tokens;
 				state Optional<ConfigureAutoResult> conf;
 
-				// tokens.push_back("configure"_sr);
-				TraceEvent("ExcludeFailedLog").detail("TLog", self->tlog.get());
-				tokens.push_back(StringRef("exclude=" + self->tlog.get().toString()));
+				TEST(true); // Exclude failed tlog
+				TraceEvent("ExcludeFailedLog")
+				    .detail("TLog", self->tlog.get())
+				    .detail("RecoveryState", self->dbInfo->get().recoveryState);
+				tokens.push_back(StringRef("exclude=" + formatIpPort(self->tlog.get().ip, self->tlog.get().port)));
 				ConfigurationResult r =
 				    wait(ManagementAPI::changeConfig(cx.getReference(), tokens, conf, /*force=*/true));
 				TraceEvent("ExcludeFailedLog")
@@ -164,34 +154,35 @@ struct ClogTlogWorkload : TestWorkload {
 		// Let cycle workload issue some transactions.
 		wait(delay(20.0));
 
+		while (self->dbInfo->get().recoveryState < RecoveryState::FULLY_RECOVERED) {
+			wait(self->dbInfo->onChange());
+		}
+
 		double startTime = now();
 		state double workloadEnd = now() + self->testDuration;
 		TraceEvent("ClogTlog").detail("StartTime", startTime).detail("EndTime", workloadEnd);
-		state Future<Void> excludeLog;
-		loop {
-			while (self->dbInfo->get().recoveryState < RecoveryState::RECOVERY_TRANSACTION) {
-				wait(self->dbInfo->onChange());
-			}
 
-			wait(clogTlog(self, cx, workloadEnd - now()));
-			loop choose {
-				when(wait(self->dbInfo->onChange())) {
-					if (self->dbInfo->get().recoveryState < RecoveryState::RECOVERY_TRANSACTION) {
-						break;
-					}
-					if (self->dbInfo->get().recoveryState == RecoveryState::FULLY_RECOVERED) {
-						TraceEvent("ClogDoneFullyRecovered");
-						return Void();
-					}
-				}
-				when(wait(delayUntil(workloadEnd))) {
-					// Expect to reach fully recovered state before workload ends
-					TraceEvent(SevError, "ClogTlogFailure").detail("RecoveryState", self->dbInfo->get().recoveryState);
+		// Clog and wait for recovery to happen
+		self->clogTlog(workloadEnd - now());
+		while (self->dbInfo->get().recoveryState == RecoveryState::FULLY_RECOVERED) {
+			wait(self->dbInfo->onChange());
+		}
+
+		// start exclusion and wait for fully recovery
+		state Future<Void> excludeLog = excludeFailedLog(self, cx);
+		state Future<Void> onChange = self->dbInfo->onChange();
+		loop choose {
+			when(wait(onChange)) {
+				if (self->dbInfo->get().recoveryState == RecoveryState::FULLY_RECOVERED) {
+					TraceEvent("ClogDoneFullyRecovered");
 					return Void();
 				}
+				onChange = self->dbInfo->onChange();
 			}
-			if (self->dbInfo->get().recoveryState <= RecoveryState::ALL_LOGS_RECRUITED) {
-				excludeLog = excludeFailedLog(self, cx);
+			when(wait(delayUntil(workloadEnd))) {
+				// Expect to reach fully recovered state before workload ends
+				TraceEvent(SevError, "ClogTlogFailure").detail("RecoveryState", self->dbInfo->get().recoveryState);
+				return Void();
 			}
 		}
 	}

From ac58b84cae0441988be062d6d888a6994cb2ab0b Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Thu, 23 Feb 2023 14:23:08 -0800
Subject: [PATCH 074/468] Move ClogTlog.toml to rare

---
 tests/CMakeLists.txt           | 2 +-
 tests/{ => rare}/ClogTlog.toml | 0
 2 files changed, 1 insertion(+), 1 deletion(-)
 rename tests/{ => rare}/ClogTlog.toml (100%)

diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt
index f8ddb6ab79f..86c6423bbcd 100644
--- a/tests/CMakeLists.txt
+++ b/tests/CMakeLists.txt
@@ -53,7 +53,6 @@ if(WITH_PYTHON)
   add_fdb_test(TEST_FILES BGServerCommonUnit.toml)
   add_fdb_test(TEST_FILES BlobGranuleFileUnit.toml)
   add_fdb_test(TEST_FILES BlobManagerUnit.toml)
-  add_fdb_test(TEST_FILES ClogTlog.toml IGNORE)
   add_fdb_test(TEST_FILES ConsistencyCheck.txt IGNORE)
   add_fdb_test(TEST_FILES DDMetricsExclude.txt IGNORE)
   add_fdb_test(TEST_FILES DataDistributionMetrics.txt IGNORE)
@@ -192,6 +191,7 @@ if(WITH_PYTHON)
     add_fdb_test(TEST_FILES fast/PhysicalShardMove.toml IGNORE)
   endif()
   add_fdb_test(TEST_FILES rare/CheckRelocation.toml)
+  add_fdb_test(TEST_FILES rare/ClogTlog.toml)
   add_fdb_test(TEST_FILES rare/ClogUnclog.toml)
   add_fdb_test(TEST_FILES rare/CloggedCycleWithKills.toml)
   add_fdb_test(TEST_FILES rare/ConfigIncrement.toml)
diff --git a/tests/ClogTlog.toml b/tests/rare/ClogTlog.toml
similarity index 100%
rename from tests/ClogTlog.toml
rename to tests/rare/ClogTlog.toml

From 58685729d23226b62f8c016f8e54609d21fb39cd Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Thu, 23 Feb 2023 15:40:15 -0800
Subject: [PATCH 075/468] Fix rare test failures

Unclog after DB is recovered, otherwise another recovery may become stuck again.
---
 fdbrpc/sim2.actor.cpp                  | 2 +-
 fdbserver/workloads/ClogTlog.actor.cpp | 1 +
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/fdbrpc/sim2.actor.cpp b/fdbrpc/sim2.actor.cpp
index 5aa2597b893..5db0cacd40a 100644
--- a/fdbrpc/sim2.actor.cpp
+++ b/fdbrpc/sim2.actor.cpp
@@ -163,7 +163,7 @@ struct SimClogging {
 		}
 
 		auto i = clogPairLatency.find(std::make_pair(from, to));
-		if (i == clogPairLatency.end()) {
+		if (i != clogPairLatency.end()) {
 			clogPairLatency.erase(i);
 		}
 	}
diff --git a/fdbserver/workloads/ClogTlog.actor.cpp b/fdbserver/workloads/ClogTlog.actor.cpp
index c5a87af6fc2..cc3c8a77204 100644
--- a/fdbserver/workloads/ClogTlog.actor.cpp
+++ b/fdbserver/workloads/ClogTlog.actor.cpp
@@ -175,6 +175,7 @@ struct ClogTlogWorkload : TestWorkload {
 			when(wait(onChange)) {
 				if (self->dbInfo->get().recoveryState == RecoveryState::FULLY_RECOVERED) {
 					TraceEvent("ClogDoneFullyRecovered");
+					self->unclogAll();
 					return Void();
 				}
 				onChange = self->dbInfo->onChange();

From bb3bb7894afddc2c00d3456faad2a94a7b422792 Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Fri, 24 Feb 2023 10:47:32 -0800
Subject: [PATCH 076/468] Address review comments

---
 fdbclient/ManagementAPI.actor.cpp |  3 ++-
 fdbrpc/sim2.actor.cpp             | 11 ++---------
 2 files changed, 4 insertions(+), 10 deletions(-)

diff --git a/fdbclient/ManagementAPI.actor.cpp b/fdbclient/ManagementAPI.actor.cpp
index d1ba8ba7900..ff06b3e18ac 100644
--- a/fdbclient/ManagementAPI.actor.cpp
+++ b/fdbclient/ManagementAPI.actor.cpp
@@ -204,8 +204,9 @@ std::map<std::string, std::string> configForToken(std::string const& mode) {
 			int p = 0;
 			while (p < value.size()) {
 				int end = value.find_first_of(',', p);
-				if (end == value.npos)
+				if (end == value.npos) {
 					end = value.size();
+				}
 				auto addrRef = StringRef(value).substr(p, end - p);
 				AddressExclusion addr = AddressExclusion::parse(addrRef);
 				if (addr.isValid()) {
diff --git a/fdbrpc/sim2.actor.cpp b/fdbrpc/sim2.actor.cpp
index 5db0cacd40a..4abcba8d2d4 100644
--- a/fdbrpc/sim2.actor.cpp
+++ b/fdbrpc/sim2.actor.cpp
@@ -157,15 +157,8 @@ struct SimClogging {
 
 	void unclogPair(const IPAddress& from, const IPAddress& to) {
 		auto pair = std::make_pair(from, to);
-		auto it = clogPairUntil.find(pair);
-		if (it != clogPairUntil.end()) {
-			clogPairUntil.erase(it);
-		}
-
-		auto i = clogPairLatency.find(std::make_pair(from, to));
-		if (i != clogPairLatency.end()) {
-			clogPairLatency.erase(i);
-		}
+		clogPairUntil.erase(pair);
+		clogPairLatency.erase(pair);
 	}
 
 	// Clog a pair of processes until a time. This is more fine-grained than

From e545dc30944c2afec53e63b68cbf652108bec340 Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Fri, 24 Feb 2023 22:05:44 -0800
Subject: [PATCH 077/468] Allow fdbdecode to read filters from a file

---
 fdbbackup/FileConverter.h       |   2 +
 fdbbackup/FileDecoder.actor.cpp | 100 +++++++++++++++++++++++++-------
 2 files changed, 82 insertions(+), 20 deletions(-)

diff --git a/fdbbackup/FileConverter.h b/fdbbackup/FileConverter.h
index 0aa1d105a6c..87e8cb830b4 100644
--- a/fdbbackup/FileConverter.h
+++ b/fdbbackup/FileConverter.h
@@ -43,6 +43,7 @@ enum {
 	OPT_BUILD_FLAGS,
 	OPT_LIST_ONLY,
 	OPT_KEY_PREFIX,
+	OPT_FILTERS,
 	OPT_HEX_KEY_PREFIX,
 	OPT_BEGIN_VERSION_FILTER,
 	OPT_END_VERSION_FILTER,
@@ -71,6 +72,7 @@ CSimpleOpt::SOption gConverterOptions[] = { { OPT_CONTAINER, "-r", SO_REQ_SEP },
 	                                        { OPT_BUILD_FLAGS, "--build-flags", SO_NONE },
 	                                        { OPT_LIST_ONLY, "--list-only", SO_NONE },
 	                                        { OPT_KEY_PREFIX, "-k", SO_REQ_SEP },
+	                                        { OPT_FILTERS, "--filters", SO_REQ_SEP },
 	                                        { OPT_HEX_KEY_PREFIX, "--hex-prefix", SO_REQ_SEP },
 	                                        { OPT_BEGIN_VERSION_FILTER, "--begin-version-filter", SO_REQ_SEP },
 	                                        { OPT_END_VERSION_FILTER, "--end-version-filter", SO_REQ_SEP },
diff --git a/fdbbackup/FileDecoder.actor.cpp b/fdbbackup/FileDecoder.actor.cpp
index 71f6932598c..ccebe20c272 100644
--- a/fdbbackup/FileDecoder.actor.cpp
+++ b/fdbbackup/FileDecoder.actor.cpp
@@ -20,12 +20,15 @@
 
 #include <algorithm>
 #include <cstdlib>
+#include <fstream>
 #include <iostream>
 #include <limits>
 #include <memory>
+#include <streambuf>
 #include <string>
 #include <vector>
 #include <fcntl.h>
+
 #ifdef _WIN32
 #include <io.h>
 #endif
@@ -82,8 +85,11 @@ void printDecodeUsage() {
 	             "  --build-flags  Print build information and exit.\n"
 	             "  --list-only    Print file list and exit.\n"
 	             "  -k KEY_PREFIX  Use the prefix for filtering mutations\n"
+	             "  --filters PREFIX_FILTER_FILE\n"
+	             "                 A file containing a list of prefix filters in HEX format separated by \";\",\n"
+	             "                 e.g., \"\\x05\\x01;\\x15\\x2b\"\n"
 	             "  --hex-prefix   HEX_PREFIX\n"
-	             "                 The prefix specified in HEX format, e.g., \"\\\\x05\\\\x01\".\n"
+	             "                 The prefix specified in HEX format, e.g., --hex-prefix \"\\\\x05\\\\x01\".\n"
 	             "  --begin-version-filter BEGIN_VERSION\n"
 	             "                 The version range's begin version (inclusive) for filtering.\n"
 	             "  --end-version-filter END_VERSION\n"
@@ -108,7 +114,7 @@ struct DecodeParams {
 	BackupTLSConfig tlsConfig;
 	bool list_only = false;
 	bool save_file_locally = false;
-	std::string prefix; // Key prefix for filtering
+	std::vector<std::string> prefixes; // Key prefixes for filtering
 	Version beginVersionFilter = 0;
 	Version endVersionFilter = std::numeric_limits<Version>::max();
 
@@ -120,6 +126,25 @@ struct DecodeParams {
 		return !(begin >= endVersionFilter || end <= beginVersionFilter);
 	}
 
+	bool matchFilters(MutationRef m) {
+		for (const auto& prefix : prefixes) {
+			if (isSingleKeyMutation((MutationRef::Type)m.type)) {
+				if (m.param1.startsWith(StringRef(prefix))) {
+					return true;
+				}
+			} else if (m.type == MutationRef::ClearRange) {
+				KeyRange range(KeyRangeRef(m.param1, m.param2));
+				KeyRange range2 = prefixRange(StringRef(prefix));
+				if (range.intersects(range2)) {
+					return true;
+				}
+			} else {
+				ASSERT(false);
+			}
+		}
+		return false;
+	}
+
 	std::string toString() {
 		std::string s;
 		s.append("ContainerURL: ");
@@ -148,8 +173,8 @@ struct DecodeParams {
 		if (endVersionFilter < std::numeric_limits<Version>::max()) {
 			s.append(", endVersionFilter: ").append(std::to_string(endVersionFilter));
 		}
-		if (!prefix.empty()) {
-			s.append(", KeyPrefix: ").append(printable(KeyRef(prefix)));
+		if (!prefixes.empty()) {
+			s.append(", KeyPrefixes: ").append(printable(describe(prefixes)));
 		}
 		for (const auto& [knob, value] : knobs) {
 			s.append(", KNOB-").append(knob).append(" = ").append(value);
@@ -167,8 +192,8 @@ struct DecodeParams {
 };
 
 // Decode an ASCII string, e.g., "\x15\x1b\x19\x04\xaf\x0c\x28\x0a",
-// into the binary string.
-std::string decode_hex_string(std::string line) {
+// into the binary string. Set "err" to true if the format is invalid.
+std::string decode_hex_string(std::string line, bool& err) {
 	size_t i = 0;
 	std::string ret;
 
@@ -177,6 +202,7 @@ std::string decode_hex_string(std::string line) {
 		case '\\':
 			if (i + 2 > line.length()) {
 				std::cerr << "Invalid hex string at: " << i << "\n";
+				err = true;
 				return ret;
 			}
 			switch (line[i + 1]) {
@@ -190,6 +216,7 @@ std::string decode_hex_string(std::string line) {
 			case 'x':
 				if (i + 4 > line.length()) {
 					std::cerr << "Invalid hex string at: " << i << "\n";
+					err = true;
 					return ret;
 				}
 				char* pEnd;
@@ -205,6 +232,7 @@ std::string decode_hex_string(std::string line) {
 				break;
 			default:
 				std::cerr << "Invalid hex string at: " << i << "\n";
+				err = true;
 				return ret;
 			}
 		default:
@@ -215,7 +243,43 @@ std::string decode_hex_string(std::string line) {
 	return line.substr(0, i);
 }
 
+// Parses and returns a ";" separated HEX encoded strings.
+// Sets "err" to true if there is any parsing error.
+std::vector<std::string> parsePrefixesLine(const std::string& line, bool& err) {
+	std::vector<std::string> results;
+	err = false;
+
+	int p = 0;
+	while (p < line.size()) {
+		int end = line.find_first_of(',', p);
+		if (end == line.npos) {
+			end = line.size();
+		}
+		auto prefix = decode_hex_string(line.substr(p, end - p), err);
+		if (err) {
+			return results;
+		}
+		results.push_back(prefix);
+		p = end + 1;
+	}
+	return results;
+}
+
+std::vector<std::string> parsePrefixFile(const std::string& filename, bool& err) {
+	std::ifstream t(filename);
+	std::string line;
+
+	t.seekg(0, std::ios::end);
+	line.reserve(t.tellg());
+	t.seekg(0, std::ios::beg);
+
+	line.assign((std::istreambuf_iterator<char>(t)), std::istreambuf_iterator<char>());
+	return parsePrefixesLine(line, err);
+}
+
 int parseDecodeCommandLine(DecodeParams* param, CSimpleOpt* args) {
+	bool err = false;
+
 	while (args->Next()) {
 		auto lastError = args->LastError();
 		switch (lastError) {
@@ -241,11 +305,15 @@ int parseDecodeCommandLine(DecodeParams* param, CSimpleOpt* args) {
 			break;
 
 		case OPT_KEY_PREFIX:
-			param->prefix = args->OptionArg();
+			param->prefixes.push_back(args->OptionArg());
+			break;
+
+		case OPT_FILTERS:
+			param->prefixes = parsePrefixFile(args->OptionArg(), err);
 			break;
 
 		case OPT_HEX_KEY_PREFIX:
-			param->prefix = decode_hex_string(args->OptionArg());
+			param->prefixes.push_back(decode_hex_string(args->OptionArg(), err));
 			break;
 
 		case OPT_BEGIN_VERSION_FILTER:
@@ -539,23 +607,15 @@ ACTOR Future<Void> process_file(Reference<IBackupContainer> container, LogFile f
 		int sub = 0;
 		for (const auto& m : vms.mutations) {
 			sub++; // sub sequence number starts at 1
-			bool print = params.prefix.empty(); // no filtering
+			bool print = params.prefixes.empty(); // no filtering
 
 			if (!print) {
-				if (isSingleKeyMutation((MutationRef::Type)m.type)) {
-					print = m.param1.startsWith(StringRef(params.prefix));
-				} else if (m.type == MutationRef::ClearRange) {
-					KeyRange range(KeyRangeRef(m.param1, m.param2));
-					KeyRange range2 = prefixRange(StringRef(params.prefix));
-					print = range.intersects(range2);
-				} else {
-					ASSERT(false);
-				}
+				print = params.matchFilters(m);
 			}
 			if (print) {
-				TraceEvent(format("Mutation_%llu_%d", vms.version, sub).c_str(), uid)
+				TraceEvent(SevVerbose, format("Mutation_%llu_%d", vms.version, sub).c_str(), uid)
 				    .detail("Version", vms.version)
-				    .setMaxFieldLength(10000)
+				    .setMaxFieldLength(-1)
 				    .detail("M", m.toString());
 				std::cout << vms.version << " " << m.toString() << "\n";
 			}

From f72693db2ea84947a855a1d752de511e7fc5d2ad Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Fri, 24 Feb 2023 22:19:13 -0800
Subject: [PATCH 078/468] Fix filter delimiter and print sub versions

---
 fdbbackup/FileDecoder.actor.cpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/fdbbackup/FileDecoder.actor.cpp b/fdbbackup/FileDecoder.actor.cpp
index ccebe20c272..c236229d666 100644
--- a/fdbbackup/FileDecoder.actor.cpp
+++ b/fdbbackup/FileDecoder.actor.cpp
@@ -251,7 +251,7 @@ std::vector<std::string> parsePrefixesLine(const std::string& line, bool& err) {
 
 	int p = 0;
 	while (p < line.size()) {
-		int end = line.find_first_of(',', p);
+		int end = line.find_first_of(';', p);
 		if (end == line.npos) {
 			end = line.size();
 		}
@@ -617,7 +617,7 @@ ACTOR Future<Void> process_file(Reference<IBackupContainer> container, LogFile f
 				    .detail("Version", vms.version)
 				    .setMaxFieldLength(-1)
 				    .detail("M", m.toString());
-				std::cout << vms.version << " " << m.toString() << "\n";
+				std::cout << vms.version << "." << sub << " " << m.toString() << "\n";
 			}
 		}
 	}

From 745e86e93ead09849bfa1f8d883912e4a591f5ca Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Sat, 25 Feb 2023 09:35:26 -0800
Subject: [PATCH 079/468] Use KeyRangeMap for better matching performance

---
 fdbbackup/FileDecoder.actor.cpp | 110 ++++++++++++++++++++++----------
 1 file changed, 78 insertions(+), 32 deletions(-)

diff --git a/fdbbackup/FileDecoder.actor.cpp b/fdbbackup/FileDecoder.actor.cpp
index c236229d666..a1cd2b4150a 100644
--- a/fdbbackup/FileDecoder.actor.cpp
+++ b/fdbbackup/FileDecoder.actor.cpp
@@ -41,9 +41,12 @@
 #include "fdbclient/CommitTransaction.h"
 #include "fdbclient/FDBTypes.h"
 #include "fdbclient/IKnobCollection.h"
+#include "fdbclient/KeyRangeMap.h"
 #include "fdbclient/Knobs.h"
 #include "fdbclient/MutationList.h"
+#include "fdbclient/SystemData.h"
 #include "flow/ArgParseUtil.h"
+#include "flow/FastRef.h"
 #include "flow/IRandom.h"
 #include "flow/Platform.h"
 #include "flow/Trace.h"
@@ -105,7 +108,7 @@ void printBuildInformation() {
 	std::cout << jsonBuildInformation() << "\n";
 }
 
-struct DecodeParams {
+struct DecodeParams : public ReferenceCounted<DecodeParams> {
 	std::string container_url;
 	Optional<std::string> proxy;
 	std::string fileFilter; // only files match the filter will be decoded
@@ -115,6 +118,8 @@ struct DecodeParams {
 	bool list_only = false;
 	bool save_file_locally = false;
 	std::vector<std::string> prefixes; // Key prefixes for filtering
+	// more efficient data structure for intersection queries than "prefixes"
+	KeyRangeMap<int> rangeMap;
 	Version beginVersionFilter = 0;
 	Version endVersionFilter = std::numeric_limits<Version>::max();
 
@@ -126,22 +131,53 @@ struct DecodeParams {
 		return !(begin >= endVersionFilter || end <= beginVersionFilter);
 	}
 
+	void updateRangeMap() {
+		for (const auto& prefix : prefixes) {
+			rangeMap.insert(prefixRange(StringRef(prefix)), 1);
+		}
+		rangeMap.coalesce(allKeys);
+	}
+
 	bool matchFilters(MutationRef m) {
+		bool match = false;
+		if (isSingleKeyMutation((MutationRef::Type)m.type)) {
+			auto ranges = rangeMap.intersectingRanges(singleKeyRange(m.param1));
+			for ([[maybe_unused]] const auto r : ranges) {
+				if (r.cvalue() == 1) {
+					match = true;
+					break;
+				}
+			}
+		} else if (m.type == MutationRef::ClearRange) {
+			auto ranges = rangeMap.intersectingRanges(KeyRangeRef(m.param1, m.param2));
+			for ([[maybe_unused]] const auto r : ranges) {
+				if (r.cvalue() == 1) {
+					match = true;
+					break;
+				}
+			}
+		} else {
+			ASSERT(false);
+		}
+
 		for (const auto& prefix : prefixes) {
 			if (isSingleKeyMutation((MutationRef::Type)m.type)) {
 				if (m.param1.startsWith(StringRef(prefix))) {
+					ASSERT(match);
 					return true;
 				}
 			} else if (m.type == MutationRef::ClearRange) {
 				KeyRange range(KeyRangeRef(m.param1, m.param2));
 				KeyRange range2 = prefixRange(StringRef(prefix));
 				if (range.intersects(range2)) {
-					return true;
+				ASSERT(match);
+				return true;
 				}
 			} else {
 				ASSERT(false);
 			}
 		}
+		ASSERT(!match);
 		return false;
 	}
 
@@ -193,6 +229,8 @@ struct DecodeParams {
 
 // Decode an ASCII string, e.g., "\x15\x1b\x19\x04\xaf\x0c\x28\x0a",
 // into the binary string. Set "err" to true if the format is invalid.
+// Note ',' '\' '," ';' are escaped by '\'. Normal characters can be
+// unencoded into HEX, but not recommended.
 std::string decode_hex_string(std::string line, bool& err) {
 	size_t i = 0;
 	std::string ret;
@@ -243,7 +281,8 @@ std::string decode_hex_string(std::string line, bool& err) {
 	return line.substr(0, i);
 }
 
-// Parses and returns a ";" separated HEX encoded strings.
+// Parses and returns a ";" separated HEX encoded strings. So the ";" in
+// the string should be escaped as "\;".
 // Sets "err" to true if there is any parsing error.
 std::vector<std::string> parsePrefixesLine(const std::string& line, bool& err) {
 	std::vector<std::string> results;
@@ -277,7 +316,7 @@ std::vector<std::string> parsePrefixFile(const std::string& filename, bool& err)
 	return parsePrefixesLine(line, err);
 }
 
-int parseDecodeCommandLine(DecodeParams* param, CSimpleOpt* args) {
+int parseDecodeCommandLine(Reference<DecodeParams> param, CSimpleOpt* args) {
 	bool err = false;
 
 	while (args->Next()) {
@@ -310,6 +349,9 @@ int parseDecodeCommandLine(DecodeParams* param, CSimpleOpt* args) {
 
 		case OPT_FILTERS:
 			param->prefixes = parsePrefixFile(args->OptionArg(), err);
+			if (err) {
+				throw std::runtime_error("ERROR:" + std::string(args->OptionArg()) + "contains invalid prefix(es)");
+			}
 			break;
 
 		case OPT_HEX_KEY_PREFIX:
@@ -413,11 +455,11 @@ void printLogFiles(std::string msg, const std::vector<LogFile>& files) {
 	std::cout << std::endl;
 }
 
-std::vector<LogFile> getRelevantLogFiles(const std::vector<LogFile>& files, const DecodeParams& params) {
+std::vector<LogFile> getRelevantLogFiles(const std::vector<LogFile>& files, const Reference<DecodeParams> params) {
 	std::vector<LogFile> filtered;
 	for (const auto& file : files) {
-		if (file.fileName.find(params.fileFilter) != std::string::npos &&
-		    params.overlap(file.beginVersion, file.endVersion + 1)) {
+		if (file.fileName.find(params->fileFilter) != std::string::npos &&
+		    params->overlap(file.beginVersion, file.endVersion + 1)) {
 			filtered.push_back(file);
 		}
 	}
@@ -585,13 +627,16 @@ class DecodeProgress {
 	int lfd = -1; // local file descriptor
 };
 
-ACTOR Future<Void> process_file(Reference<IBackupContainer> container, LogFile file, UID uid, DecodeParams params) {
+ACTOR Future<Void> process_file(Reference<IBackupContainer> container,
+                                LogFile file,
+                                UID uid,
+                                Reference<DecodeParams> params) {
 	if (file.fileSize == 0) {
 		TraceEvent("SkipEmptyFile", uid).detail("Name", file.fileName);
 		return Void();
 	}
 
-	state DecodeProgress progress(file, params.save_file_locally);
+	state DecodeProgress progress(file, params->save_file_locally);
 	wait(progress.openFile(container));
 	while (true) {
 		auto batch = progress.getNextBatch();
@@ -599,7 +644,7 @@ ACTOR Future<Void> process_file(Reference<IBackupContainer> container, LogFile f
 			break;
 
 		const VersionedMutations& vms = batch.get();
-		if (vms.version < params.beginVersionFilter || vms.version >= params.endVersionFilter) {
+		if (vms.version < params->beginVersionFilter || vms.version >= params->endVersionFilter) {
 			TraceEvent("SkipVersion").detail("Version", vms.version);
 			continue;
 		}
@@ -607,10 +652,10 @@ ACTOR Future<Void> process_file(Reference<IBackupContainer> container, LogFile f
 		int sub = 0;
 		for (const auto& m : vms.mutations) {
 			sub++; // sub sequence number starts at 1
-			bool print = params.prefixes.empty(); // no filtering
+			bool print = params->prefixes.empty(); // no filtering
 
 			if (!print) {
-				print = params.matchFilters(m);
+				print = params->matchFilters(m);
 			}
 			if (print) {
 				TraceEvent(SevVerbose, format("Mutation_%llu_%d", vms.version, sub).c_str(), uid)
@@ -625,9 +670,9 @@ ACTOR Future<Void> process_file(Reference<IBackupContainer> container, LogFile f
 	return Void();
 }
 
-ACTOR Future<Void> decode_logs(DecodeParams params) {
+ACTOR Future<Void> decode_logs(Reference<DecodeParams> params) {
 	state Reference<IBackupContainer> container =
-	    IBackupContainer::openContainer(params.container_url, params.proxy, {});
+	    IBackupContainer::openContainer(params->container_url, params->proxy, {});
 	state UID uid = deterministicRandom()->randomUniqueID();
 	state BackupFileList listing = wait(container->dumpFileList());
 	// remove partitioned logs
@@ -639,8 +684,8 @@ ACTOR Future<Void> decode_logs(DecodeParams params) {
 	                                  }),
 	                   listing.logs.end());
 	std::sort(listing.logs.begin(), listing.logs.end());
-	TraceEvent("Container", uid).detail("URL", params.container_url).detail("Logs", listing.logs.size());
-	TraceEvent("DecodeParam", uid).setMaxFieldLength(100000).detail("Value", params.toString());
+	TraceEvent("Container", uid).detail("URL", params->container_url).detail("Logs", listing.logs.size());
+	TraceEvent("DecodeParam", uid).setMaxFieldLength(100000).detail("Value", params->toString());
 
 	BackupDescription desc = wait(container->describeBackup());
 	std::cout << "\n" << desc.toString() << "\n";
@@ -648,7 +693,7 @@ ACTOR Future<Void> decode_logs(DecodeParams params) {
 	state std::vector<LogFile> logs = getRelevantLogFiles(listing.logs, params);
 	printLogFiles("Relevant files are: ", logs);
 
-	if (params.list_only)
+	if (params->list_only)
 		return Void();
 
 	state int idx = 0;
@@ -667,31 +712,32 @@ int main(int argc, char** argv) {
 	try {
 		std::unique_ptr<CSimpleOpt> args(
 		    new CSimpleOpt(argc, argv, file_converter::gConverterOptions, SO_O_EXACT | SO_O_HYPHEN_TO_UNDERSCORE));
-		file_converter::DecodeParams param;
-		int status = file_converter::parseDecodeCommandLine(&param, args.get());
-		std::cout << "Params: " << param.toString() << "\n";
+		auto param = makeReference<file_converter::DecodeParams>();
+		int status = file_converter::parseDecodeCommandLine(param, args.get());
+		std::cout << "Params: " << param->toString() << "\n";
+		param->updateRangeMap();
 		if (status != FDB_EXIT_SUCCESS) {
 			file_converter::printDecodeUsage();
 			return status;
 		}
 
-		if (param.log_enabled) {
-			if (param.log_dir.empty()) {
+		if (param->log_enabled) {
+			if (param->log_dir.empty()) {
 				setNetworkOption(FDBNetworkOptions::TRACE_ENABLE);
 			} else {
-				setNetworkOption(FDBNetworkOptions::TRACE_ENABLE, StringRef(param.log_dir));
+				setNetworkOption(FDBNetworkOptions::TRACE_ENABLE, StringRef(param->log_dir));
 			}
-			if (!param.trace_format.empty()) {
-				setNetworkOption(FDBNetworkOptions::TRACE_FORMAT, StringRef(param.trace_format));
+			if (!param->trace_format.empty()) {
+				setNetworkOption(FDBNetworkOptions::TRACE_FORMAT, StringRef(param->trace_format));
 			} else {
 				setNetworkOption(FDBNetworkOptions::TRACE_FORMAT, "json"_sr);
 			}
-			if (!param.trace_log_group.empty()) {
-				setNetworkOption(FDBNetworkOptions::TRACE_LOG_GROUP, StringRef(param.trace_log_group));
+			if (!param->trace_log_group.empty()) {
+				setNetworkOption(FDBNetworkOptions::TRACE_LOG_GROUP, StringRef(param->trace_log_group));
 			}
 		}
 
-		if (!param.tlsConfig.setupTLS()) {
+		if (!param->tlsConfig.setupTLS()) {
 			TraceEvent(SevError, "TLSError").log();
 			throw tls_error();
 		}
@@ -699,15 +745,15 @@ int main(int argc, char** argv) {
 		platformInit();
 		Error::init();
 
-		StringRef url(param.container_url);
+		StringRef url(param->container_url);
 		setupNetwork(0, UseMetrics::True);
 
 		// Must be called after setupNetwork() to be effective
-		param.updateKnobs();
+		param->updateKnobs();
 
 		TraceEvent::setNetworkThread();
-		openTraceFile(NetworkAddress(), 10 << 20, 500 << 20, param.log_dir, "decode", param.trace_log_group);
-		param.tlsConfig.setupBlobCredentials();
+		openTraceFile(NetworkAddress(), 10 << 20, 500 << 20, param->log_dir, "decode", param->trace_log_group);
+		param->tlsConfig.setupBlobCredentials();
 
 		auto f = stopAfter(decode_logs(param));
 

From 6b70e5f34033d57b39bb4da2fcbdf564e4ec4568 Mon Sep 17 00:00:00 2001
From: Vishesh Yadav <vishesh_yadav@apple.com>
Date: Sat, 25 Feb 2023 14:15:34 -0800
Subject: [PATCH 080/468] fdbdecode: read backup range files

---
 fdbbackup/FileDecoder.actor.cpp | 149 +++++++++++++++++++++++++++++---
 1 file changed, 139 insertions(+), 10 deletions(-)

diff --git a/fdbbackup/FileDecoder.actor.cpp b/fdbbackup/FileDecoder.actor.cpp
index a1cd2b4150a..50b8e562743 100644
--- a/fdbbackup/FileDecoder.actor.cpp
+++ b/fdbbackup/FileDecoder.actor.cpp
@@ -131,6 +131,8 @@ struct DecodeParams : public ReferenceCounted<DecodeParams> {
 		return !(begin >= endVersionFilter || end <= beginVersionFilter);
 	}
 
+	bool overlap(Version version) const { return version >= beginVersionFilter && version < endVersionFilter; }
+
 	void updateRangeMap() {
 		for (const auto& prefix : prefixes) {
 			rangeMap.insert(prefixRange(StringRef(prefix)), 1);
@@ -170,8 +172,8 @@ struct DecodeParams : public ReferenceCounted<DecodeParams> {
 				KeyRange range(KeyRangeRef(m.param1, m.param2));
 				KeyRange range2 = prefixRange(StringRef(prefix));
 				if (range.intersects(range2)) {
-				ASSERT(match);
-				return true;
+					ASSERT(match);
+					return true;
 				}
 			} else {
 				ASSERT(false);
@@ -447,8 +449,9 @@ int parseDecodeCommandLine(Reference<DecodeParams> param, CSimpleOpt* args) {
 	return FDB_EXIT_SUCCESS;
 }
 
-void printLogFiles(std::string msg, const std::vector<LogFile>& files) {
-	std::cout << msg << " " << files.size() << " log files\n";
+template <class BackupFile>
+void printLogFiles(std::string msg, const std::vector<BackupFile>& files) {
+	std::cout << msg << " " << files.size() << " total\n";
 	for (const auto& file : files) {
 		std::cout << file.toString() << "\n";
 	}
@@ -466,6 +469,17 @@ std::vector<LogFile> getRelevantLogFiles(const std::vector<LogFile>& files, cons
 	return filtered;
 }
 
+std::vector<RangeFile> getRelevantRangeFiles(const std::vector<RangeFile>& files,
+                                             const Reference<DecodeParams> params) {
+	std::vector<RangeFile> filtered;
+	for (const auto& file : files) {
+		if (file.fileName.find(params->fileFilter) != std::string::npos && params->overlap(file.version)) {
+			filtered.push_back(file);
+		}
+	}
+	return filtered;
+}
+
 struct VersionedMutations {
 	Version version;
 	std::vector<MutationRef> mutations;
@@ -627,6 +641,107 @@ class DecodeProgress {
 	int lfd = -1; // local file descriptor
 };
 
+class DecodeRangeProgress {
+public:
+	std::vector<Standalone<VectorRef<KeyValueRef>>> blocks;
+
+	DecodeRangeProgress() = default;
+	DecodeRangeProgress(const RangeFile& file, bool save) : file(file), save(save) {}
+	~DecodeRangeProgress() {
+		if (lfd != -1) {
+			close(lfd);
+		}
+	}
+
+	// Open and loads file into memory
+	Future<Void> openFile(Reference<IBackupContainer> container) { return openFileImpl(this, container); }
+
+	ACTOR static Future<Void> openFileImpl(DecodeRangeProgress* self, Reference<IBackupContainer> container) {
+		Reference<IAsyncFile> fd = wait(container->readFile(self->file.fileName));
+		self->fd = fd;
+		if (self->save) {
+			std::string dir = self->file.fileName;
+			std::size_t found = self->file.fileName.find_last_of('/');
+			if (found != std::string::npos) {
+				std::string path = self->file.fileName.substr(0, found);
+				if (!directoryExists(path)) {
+					platform::createDirectory(path);
+				}
+			}
+		self->lfd = open(self->file.fileName.c_str(), O_WRONLY | O_CREAT | O_TRUNC);
+			if (self->lfd == -1) {
+				TraceEvent(SevError, "OpenLocalFileFailed").detail("File", self->file.fileName);
+				throw platform_error();
+			}
+		}
+		while (!self->eof) {
+			wait(readAndDecodeFile(self));
+		}
+		return Void();
+	}
+
+	// Reads a file block, decodes it into key/value pairs, and stores these pairs.
+	ACTOR static Future<Void> readAndDecodeFile(DecodeRangeProgress* self) {
+		try {
+			state int64_t len = std::min<int64_t>(self->file.blockSize, self->file.fileSize - self->offset);
+			if (len == 0) {
+				self->eof = true;
+				return Void();
+			}
+
+			state Standalone<VectorRef<KeyValueRef>> chunks =
+			    wait(fileBackup::decodeRangeFileBlock(self->fd, self->offset, len));
+			self->blocks.push_back(chunks);
+			// TODO (Vishesh): Implement self->save
+
+			TraceEvent("ReadFile")
+			    .detail("Name", self->file.fileName)
+			    .detail("Len", len)
+			    .detail("Offset", self->offset);
+			self->offset += len;
+
+		} catch (Error& e) {
+			TraceEvent(SevWarn, "CorruptRangeFileBlock")
+			    .error(e)
+			    .detail("Filename", self->file.fileName)
+			    .detail("BlockOffset", self->offset)
+			    .detail("BlockLen", self->file.blockSize);
+			throw;
+		}
+
+		return Void();
+	}
+
+	RangeFile file;
+	Reference<IAsyncFile> fd;
+	int64_t offset = 0;
+	bool save = false;
+	bool eof = false;
+	int lfd = -1; // local file descriptor
+};
+
+ACTOR Future<Void> process_range_file(Reference<IBackupContainer> container,
+                                      RangeFile file,
+                                      UID uid,
+                                      Reference<DecodeParams> params) {
+
+	if (file.fileSize == 0) {
+		TraceEvent("SkipEmptyFile", uid).detail("Name", file.fileName);
+		return Void();
+	}
+
+	state DecodeRangeProgress progress(file, params->save_file_locally);
+	wait(progress.openFile(container));
+
+	for (auto& block : progress.blocks) {
+		for (const auto& kv : block) {
+			std::cout << "Key = " << printable(kv.key) << "  Value = " << printable(kv.value) << std::endl;
+		}
+	}
+
+	return Void();
+}
+
 ACTOR Future<Void> process_file(Reference<IBackupContainer> container,
                                 LogFile file,
                                 UID uid,
@@ -690,19 +805,33 @@ ACTOR Future<Void> decode_logs(Reference<DecodeParams> params) {
 	BackupDescription desc = wait(container->describeBackup());
 	std::cout << "\n" << desc.toString() << "\n";
 
-	state std::vector<LogFile> logs = getRelevantLogFiles(listing.logs, params);
-	printLogFiles("Relevant files are: ", logs);
+	state std::vector<LogFile> logFiles = getRelevantLogFiles(listing.logs, params);
+	printLogFiles("Relevant log files are: ", logFiles);
+
+	state std::vector<RangeFile> rangeFiles = getRelevantRangeFiles(listing.ranges, params);
+	printLogFiles("Releavant range files are: ", rangeFiles);
 
 	if (params->list_only)
 		return Void();
 
+	// Decode log files.
 	state int idx = 0;
-	while (idx < logs.size()) {
-		TraceEvent("ProcessFile").detail("Name", logs[idx].fileName).detail("I", idx);
-		wait(process_file(container, logs[idx], uid, params));
+	while (idx < logFiles.size()) {
+		TraceEvent("ProcessFile").detail("Name", logFiles[idx].fileName).detail("I", idx);
+		wait(process_file(container, logFiles[idx], uid, params));
+		idx++;
+	}
+	TraceEvent("DecodeLogsDone", uid).log();
+
+	// Decode range files.
+	idx = 0;
+	while (idx < rangeFiles.size()) {
+		TraceEvent("ProcessFile").detail("Name", rangeFiles[idx].fileName).detail("I", idx);
+		wait(process_range_file(container, rangeFiles[idx], uid, params));
 		idx++;
 	}
-	TraceEvent("DecodeDone", uid).log();
+	TraceEvent("DecodeRangeFileDone", uid).log();
+
 	return Void();
 }
 

From 2be2644ce2cc27bcb80c2c319572d0613260adae Mon Sep 17 00:00:00 2001
From: Vishesh Yadav <vishesh_yadav@apple.com>
Date: Sat, 25 Feb 2023 15:14:00 -0800
Subject: [PATCH 081/468] add filtering

---
 fdbbackup/FileDecoder.actor.cpp | 40 ++++++++++++++++++++++++++++++---
 1 file changed, 37 insertions(+), 3 deletions(-)

diff --git a/fdbbackup/FileDecoder.actor.cpp b/fdbbackup/FileDecoder.actor.cpp
index 50b8e562743..04a83ea09dd 100644
--- a/fdbbackup/FileDecoder.actor.cpp
+++ b/fdbbackup/FileDecoder.actor.cpp
@@ -183,6 +183,26 @@ struct DecodeParams : public ReferenceCounted<DecodeParams> {
 		return false;
 	}
 
+	bool matchFilters(KeyValueRef kv) {
+		bool match = false;
+		auto ranges = rangeMap.intersectingRanges(singleKeyRange(kv.key));
+		for ([[maybe_unused]] const auto r : ranges) {
+			if (r.cvalue() == 1) {
+				match = true;
+				break;
+			}
+		}
+
+		for (const auto& prefix : prefixes) {
+			if (kv.key.startsWith(StringRef(prefix))) {
+				ASSERT(match);
+				return true;
+			}
+		}
+
+		return match;
+	}
+
 	std::string toString() {
 		std::string s;
 		s.append("ContainerURL: ");
@@ -668,7 +688,8 @@ class DecodeRangeProgress {
 					platform::createDirectory(path);
 				}
 			}
-		self->lfd = open(self->file.fileName.c_str(), O_WRONLY | O_CREAT | O_TRUNC);
+
+			self->lfd = open(self->file.fileName.c_str(), O_WRONLY | O_CREAT | O_TRUNC);
 			if (self->lfd == -1) {
 				TraceEvent(SevError, "OpenLocalFileFailed").detail("File", self->file.fileName);
 				throw platform_error();
@@ -692,7 +713,6 @@ class DecodeRangeProgress {
 			state Standalone<VectorRef<KeyValueRef>> chunks =
 			    wait(fileBackup::decodeRangeFileBlock(self->fd, self->offset, len));
 			self->blocks.push_back(chunks);
-			// TODO (Vishesh): Implement self->save
 
 			TraceEvent("ReadFile")
 			    .detail("Name", self->file.fileName)
@@ -735,7 +755,21 @@ ACTOR Future<Void> process_range_file(Reference<IBackupContainer> container,
 
 	for (auto& block : progress.blocks) {
 		for (const auto& kv : block) {
-			std::cout << "Key = " << printable(kv.key) << "  Value = " << printable(kv.value) << std::endl;
+			bool print = params->prefixes.empty(); // no filtering
+
+			if (!print) {
+				print = params->matchFilters(kv);
+			}
+
+			if (print) {
+				TraceEvent(SevVerbose, format("KVPair_%llu", file.version).c_str(), uid)
+				    .detail("Version", file.version)
+				    .setMaxFieldLength(-1)
+				    .detail("Key", kv.key)
+				    .detail("Value", kv.value);
+				std::cout << file.version << " Key = " << printable(kv.key) << "  Value = " << printable(kv.value)
+				          << std::endl;
+			}
 		}
 	}
 

From c9f13aa1468ffa3b24d684927600ea47947dcb82 Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Fri, 24 Feb 2023 22:05:44 -0800
Subject: [PATCH 082/468] Allow fdbdecode to read filters from a file

---
 fdbbackup/FileConverter.h       |   2 +
 fdbbackup/FileDecoder.actor.cpp | 100 +++++++++++++++++++++++++-------
 2 files changed, 82 insertions(+), 20 deletions(-)

diff --git a/fdbbackup/FileConverter.h b/fdbbackup/FileConverter.h
index 0aa1d105a6c..87e8cb830b4 100644
--- a/fdbbackup/FileConverter.h
+++ b/fdbbackup/FileConverter.h
@@ -43,6 +43,7 @@ enum {
 	OPT_BUILD_FLAGS,
 	OPT_LIST_ONLY,
 	OPT_KEY_PREFIX,
+	OPT_FILTERS,
 	OPT_HEX_KEY_PREFIX,
 	OPT_BEGIN_VERSION_FILTER,
 	OPT_END_VERSION_FILTER,
@@ -71,6 +72,7 @@ CSimpleOpt::SOption gConverterOptions[] = { { OPT_CONTAINER, "-r", SO_REQ_SEP },
 	                                        { OPT_BUILD_FLAGS, "--build-flags", SO_NONE },
 	                                        { OPT_LIST_ONLY, "--list-only", SO_NONE },
 	                                        { OPT_KEY_PREFIX, "-k", SO_REQ_SEP },
+	                                        { OPT_FILTERS, "--filters", SO_REQ_SEP },
 	                                        { OPT_HEX_KEY_PREFIX, "--hex-prefix", SO_REQ_SEP },
 	                                        { OPT_BEGIN_VERSION_FILTER, "--begin-version-filter", SO_REQ_SEP },
 	                                        { OPT_END_VERSION_FILTER, "--end-version-filter", SO_REQ_SEP },
diff --git a/fdbbackup/FileDecoder.actor.cpp b/fdbbackup/FileDecoder.actor.cpp
index 71f6932598c..ccebe20c272 100644
--- a/fdbbackup/FileDecoder.actor.cpp
+++ b/fdbbackup/FileDecoder.actor.cpp
@@ -20,12 +20,15 @@
 
 #include <algorithm>
 #include <cstdlib>
+#include <fstream>
 #include <iostream>
 #include <limits>
 #include <memory>
+#include <streambuf>
 #include <string>
 #include <vector>
 #include <fcntl.h>
+
 #ifdef _WIN32
 #include <io.h>
 #endif
@@ -82,8 +85,11 @@ void printDecodeUsage() {
 	             "  --build-flags  Print build information and exit.\n"
 	             "  --list-only    Print file list and exit.\n"
 	             "  -k KEY_PREFIX  Use the prefix for filtering mutations\n"
+	             "  --filters PREFIX_FILTER_FILE\n"
+	             "                 A file containing a list of prefix filters in HEX format separated by \";\",\n"
+	             "                 e.g., \"\\x05\\x01;\\x15\\x2b\"\n"
 	             "  --hex-prefix   HEX_PREFIX\n"
-	             "                 The prefix specified in HEX format, e.g., \"\\\\x05\\\\x01\".\n"
+	             "                 The prefix specified in HEX format, e.g., --hex-prefix \"\\\\x05\\\\x01\".\n"
 	             "  --begin-version-filter BEGIN_VERSION\n"
 	             "                 The version range's begin version (inclusive) for filtering.\n"
 	             "  --end-version-filter END_VERSION\n"
@@ -108,7 +114,7 @@ struct DecodeParams {
 	BackupTLSConfig tlsConfig;
 	bool list_only = false;
 	bool save_file_locally = false;
-	std::string prefix; // Key prefix for filtering
+	std::vector<std::string> prefixes; // Key prefixes for filtering
 	Version beginVersionFilter = 0;
 	Version endVersionFilter = std::numeric_limits<Version>::max();
 
@@ -120,6 +126,25 @@ struct DecodeParams {
 		return !(begin >= endVersionFilter || end <= beginVersionFilter);
 	}
 
+	bool matchFilters(MutationRef m) {
+		for (const auto& prefix : prefixes) {
+			if (isSingleKeyMutation((MutationRef::Type)m.type)) {
+				if (m.param1.startsWith(StringRef(prefix))) {
+					return true;
+				}
+			} else if (m.type == MutationRef::ClearRange) {
+				KeyRange range(KeyRangeRef(m.param1, m.param2));
+				KeyRange range2 = prefixRange(StringRef(prefix));
+				if (range.intersects(range2)) {
+					return true;
+				}
+			} else {
+				ASSERT(false);
+			}
+		}
+		return false;
+	}
+
 	std::string toString() {
 		std::string s;
 		s.append("ContainerURL: ");
@@ -148,8 +173,8 @@ struct DecodeParams {
 		if (endVersionFilter < std::numeric_limits<Version>::max()) {
 			s.append(", endVersionFilter: ").append(std::to_string(endVersionFilter));
 		}
-		if (!prefix.empty()) {
-			s.append(", KeyPrefix: ").append(printable(KeyRef(prefix)));
+		if (!prefixes.empty()) {
+			s.append(", KeyPrefixes: ").append(printable(describe(prefixes)));
 		}
 		for (const auto& [knob, value] : knobs) {
 			s.append(", KNOB-").append(knob).append(" = ").append(value);
@@ -167,8 +192,8 @@ struct DecodeParams {
 };
 
 // Decode an ASCII string, e.g., "\x15\x1b\x19\x04\xaf\x0c\x28\x0a",
-// into the binary string.
-std::string decode_hex_string(std::string line) {
+// into the binary string. Set "err" to true if the format is invalid.
+std::string decode_hex_string(std::string line, bool& err) {
 	size_t i = 0;
 	std::string ret;
 
@@ -177,6 +202,7 @@ std::string decode_hex_string(std::string line) {
 		case '\\':
 			if (i + 2 > line.length()) {
 				std::cerr << "Invalid hex string at: " << i << "\n";
+				err = true;
 				return ret;
 			}
 			switch (line[i + 1]) {
@@ -190,6 +216,7 @@ std::string decode_hex_string(std::string line) {
 			case 'x':
 				if (i + 4 > line.length()) {
 					std::cerr << "Invalid hex string at: " << i << "\n";
+					err = true;
 					return ret;
 				}
 				char* pEnd;
@@ -205,6 +232,7 @@ std::string decode_hex_string(std::string line) {
 				break;
 			default:
 				std::cerr << "Invalid hex string at: " << i << "\n";
+				err = true;
 				return ret;
 			}
 		default:
@@ -215,7 +243,43 @@ std::string decode_hex_string(std::string line) {
 	return line.substr(0, i);
 }
 
+// Parses and returns a ";" separated HEX encoded strings.
+// Sets "err" to true if there is any parsing error.
+std::vector<std::string> parsePrefixesLine(const std::string& line, bool& err) {
+	std::vector<std::string> results;
+	err = false;
+
+	int p = 0;
+	while (p < line.size()) {
+		int end = line.find_first_of(',', p);
+		if (end == line.npos) {
+			end = line.size();
+		}
+		auto prefix = decode_hex_string(line.substr(p, end - p), err);
+		if (err) {
+			return results;
+		}
+		results.push_back(prefix);
+		p = end + 1;
+	}
+	return results;
+}
+
+std::vector<std::string> parsePrefixFile(const std::string& filename, bool& err) {
+	std::ifstream t(filename);
+	std::string line;
+
+	t.seekg(0, std::ios::end);
+	line.reserve(t.tellg());
+	t.seekg(0, std::ios::beg);
+
+	line.assign((std::istreambuf_iterator<char>(t)), std::istreambuf_iterator<char>());
+	return parsePrefixesLine(line, err);
+}
+
 int parseDecodeCommandLine(DecodeParams* param, CSimpleOpt* args) {
+	bool err = false;
+
 	while (args->Next()) {
 		auto lastError = args->LastError();
 		switch (lastError) {
@@ -241,11 +305,15 @@ int parseDecodeCommandLine(DecodeParams* param, CSimpleOpt* args) {
 			break;
 
 		case OPT_KEY_PREFIX:
-			param->prefix = args->OptionArg();
+			param->prefixes.push_back(args->OptionArg());
+			break;
+
+		case OPT_FILTERS:
+			param->prefixes = parsePrefixFile(args->OptionArg(), err);
 			break;
 
 		case OPT_HEX_KEY_PREFIX:
-			param->prefix = decode_hex_string(args->OptionArg());
+			param->prefixes.push_back(decode_hex_string(args->OptionArg(), err));
 			break;
 
 		case OPT_BEGIN_VERSION_FILTER:
@@ -539,23 +607,15 @@ ACTOR Future<Void> process_file(Reference<IBackupContainer> container, LogFile f
 		int sub = 0;
 		for (const auto& m : vms.mutations) {
 			sub++; // sub sequence number starts at 1
-			bool print = params.prefix.empty(); // no filtering
+			bool print = params.prefixes.empty(); // no filtering
 
 			if (!print) {
-				if (isSingleKeyMutation((MutationRef::Type)m.type)) {
-					print = m.param1.startsWith(StringRef(params.prefix));
-				} else if (m.type == MutationRef::ClearRange) {
-					KeyRange range(KeyRangeRef(m.param1, m.param2));
-					KeyRange range2 = prefixRange(StringRef(params.prefix));
-					print = range.intersects(range2);
-				} else {
-					ASSERT(false);
-				}
+				print = params.matchFilters(m);
 			}
 			if (print) {
-				TraceEvent(format("Mutation_%llu_%d", vms.version, sub).c_str(), uid)
+				TraceEvent(SevVerbose, format("Mutation_%llu_%d", vms.version, sub).c_str(), uid)
 				    .detail("Version", vms.version)
-				    .setMaxFieldLength(10000)
+				    .setMaxFieldLength(-1)
 				    .detail("M", m.toString());
 				std::cout << vms.version << " " << m.toString() << "\n";
 			}

From f41f08d2ac0e52448f1682abd25c14ec795284ba Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Fri, 24 Feb 2023 22:19:13 -0800
Subject: [PATCH 083/468] Fix filter delimiter and print sub versions

---
 fdbbackup/FileDecoder.actor.cpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/fdbbackup/FileDecoder.actor.cpp b/fdbbackup/FileDecoder.actor.cpp
index ccebe20c272..c236229d666 100644
--- a/fdbbackup/FileDecoder.actor.cpp
+++ b/fdbbackup/FileDecoder.actor.cpp
@@ -251,7 +251,7 @@ std::vector<std::string> parsePrefixesLine(const std::string& line, bool& err) {
 
 	int p = 0;
 	while (p < line.size()) {
-		int end = line.find_first_of(',', p);
+		int end = line.find_first_of(';', p);
 		if (end == line.npos) {
 			end = line.size();
 		}
@@ -617,7 +617,7 @@ ACTOR Future<Void> process_file(Reference<IBackupContainer> container, LogFile f
 				    .detail("Version", vms.version)
 				    .setMaxFieldLength(-1)
 				    .detail("M", m.toString());
-				std::cout << vms.version << " " << m.toString() << "\n";
+				std::cout << vms.version << "." << sub << " " << m.toString() << "\n";
 			}
 		}
 	}

From 1626d6338e0bd13b3946b6225fcf81226c348a34 Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Sat, 25 Feb 2023 09:35:26 -0800
Subject: [PATCH 084/468] Use KeyRangeMap for better matching performance

---
 fdbbackup/FileDecoder.actor.cpp | 110 ++++++++++++++++++++++----------
 1 file changed, 78 insertions(+), 32 deletions(-)

diff --git a/fdbbackup/FileDecoder.actor.cpp b/fdbbackup/FileDecoder.actor.cpp
index c236229d666..a1cd2b4150a 100644
--- a/fdbbackup/FileDecoder.actor.cpp
+++ b/fdbbackup/FileDecoder.actor.cpp
@@ -41,9 +41,12 @@
 #include "fdbclient/CommitTransaction.h"
 #include "fdbclient/FDBTypes.h"
 #include "fdbclient/IKnobCollection.h"
+#include "fdbclient/KeyRangeMap.h"
 #include "fdbclient/Knobs.h"
 #include "fdbclient/MutationList.h"
+#include "fdbclient/SystemData.h"
 #include "flow/ArgParseUtil.h"
+#include "flow/FastRef.h"
 #include "flow/IRandom.h"
 #include "flow/Platform.h"
 #include "flow/Trace.h"
@@ -105,7 +108,7 @@ void printBuildInformation() {
 	std::cout << jsonBuildInformation() << "\n";
 }
 
-struct DecodeParams {
+struct DecodeParams : public ReferenceCounted<DecodeParams> {
 	std::string container_url;
 	Optional<std::string> proxy;
 	std::string fileFilter; // only files match the filter will be decoded
@@ -115,6 +118,8 @@ struct DecodeParams {
 	bool list_only = false;
 	bool save_file_locally = false;
 	std::vector<std::string> prefixes; // Key prefixes for filtering
+	// more efficient data structure for intersection queries than "prefixes"
+	KeyRangeMap<int> rangeMap;
 	Version beginVersionFilter = 0;
 	Version endVersionFilter = std::numeric_limits<Version>::max();
 
@@ -126,22 +131,53 @@ struct DecodeParams {
 		return !(begin >= endVersionFilter || end <= beginVersionFilter);
 	}
 
+	void updateRangeMap() {
+		for (const auto& prefix : prefixes) {
+			rangeMap.insert(prefixRange(StringRef(prefix)), 1);
+		}
+		rangeMap.coalesce(allKeys);
+	}
+
 	bool matchFilters(MutationRef m) {
+		bool match = false;
+		if (isSingleKeyMutation((MutationRef::Type)m.type)) {
+			auto ranges = rangeMap.intersectingRanges(singleKeyRange(m.param1));
+			for ([[maybe_unused]] const auto r : ranges) {
+				if (r.cvalue() == 1) {
+					match = true;
+					break;
+				}
+			}
+		} else if (m.type == MutationRef::ClearRange) {
+			auto ranges = rangeMap.intersectingRanges(KeyRangeRef(m.param1, m.param2));
+			for ([[maybe_unused]] const auto r : ranges) {
+				if (r.cvalue() == 1) {
+					match = true;
+					break;
+				}
+			}
+		} else {
+			ASSERT(false);
+		}
+
 		for (const auto& prefix : prefixes) {
 			if (isSingleKeyMutation((MutationRef::Type)m.type)) {
 				if (m.param1.startsWith(StringRef(prefix))) {
+					ASSERT(match);
 					return true;
 				}
 			} else if (m.type == MutationRef::ClearRange) {
 				KeyRange range(KeyRangeRef(m.param1, m.param2));
 				KeyRange range2 = prefixRange(StringRef(prefix));
 				if (range.intersects(range2)) {
-					return true;
+				ASSERT(match);
+				return true;
 				}
 			} else {
 				ASSERT(false);
 			}
 		}
+		ASSERT(!match);
 		return false;
 	}
 
@@ -193,6 +229,8 @@ struct DecodeParams {
 
 // Decode an ASCII string, e.g., "\x15\x1b\x19\x04\xaf\x0c\x28\x0a",
 // into the binary string. Set "err" to true if the format is invalid.
+// Note ',' '\' '," ';' are escaped by '\'. Normal characters can be
+// unencoded into HEX, but not recommended.
 std::string decode_hex_string(std::string line, bool& err) {
 	size_t i = 0;
 	std::string ret;
@@ -243,7 +281,8 @@ std::string decode_hex_string(std::string line, bool& err) {
 	return line.substr(0, i);
 }
 
-// Parses and returns a ";" separated HEX encoded strings.
+// Parses and returns a ";" separated HEX encoded strings. So the ";" in
+// the string should be escaped as "\;".
 // Sets "err" to true if there is any parsing error.
 std::vector<std::string> parsePrefixesLine(const std::string& line, bool& err) {
 	std::vector<std::string> results;
@@ -277,7 +316,7 @@ std::vector<std::string> parsePrefixFile(const std::string& filename, bool& err)
 	return parsePrefixesLine(line, err);
 }
 
-int parseDecodeCommandLine(DecodeParams* param, CSimpleOpt* args) {
+int parseDecodeCommandLine(Reference<DecodeParams> param, CSimpleOpt* args) {
 	bool err = false;
 
 	while (args->Next()) {
@@ -310,6 +349,9 @@ int parseDecodeCommandLine(DecodeParams* param, CSimpleOpt* args) {
 
 		case OPT_FILTERS:
 			param->prefixes = parsePrefixFile(args->OptionArg(), err);
+			if (err) {
+				throw std::runtime_error("ERROR:" + std::string(args->OptionArg()) + "contains invalid prefix(es)");
+			}
 			break;
 
 		case OPT_HEX_KEY_PREFIX:
@@ -413,11 +455,11 @@ void printLogFiles(std::string msg, const std::vector<LogFile>& files) {
 	std::cout << std::endl;
 }
 
-std::vector<LogFile> getRelevantLogFiles(const std::vector<LogFile>& files, const DecodeParams& params) {
+std::vector<LogFile> getRelevantLogFiles(const std::vector<LogFile>& files, const Reference<DecodeParams> params) {
 	std::vector<LogFile> filtered;
 	for (const auto& file : files) {
-		if (file.fileName.find(params.fileFilter) != std::string::npos &&
-		    params.overlap(file.beginVersion, file.endVersion + 1)) {
+		if (file.fileName.find(params->fileFilter) != std::string::npos &&
+		    params->overlap(file.beginVersion, file.endVersion + 1)) {
 			filtered.push_back(file);
 		}
 	}
@@ -585,13 +627,16 @@ class DecodeProgress {
 	int lfd = -1; // local file descriptor
 };
 
-ACTOR Future<Void> process_file(Reference<IBackupContainer> container, LogFile file, UID uid, DecodeParams params) {
+ACTOR Future<Void> process_file(Reference<IBackupContainer> container,
+                                LogFile file,
+                                UID uid,
+                                Reference<DecodeParams> params) {
 	if (file.fileSize == 0) {
 		TraceEvent("SkipEmptyFile", uid).detail("Name", file.fileName);
 		return Void();
 	}
 
-	state DecodeProgress progress(file, params.save_file_locally);
+	state DecodeProgress progress(file, params->save_file_locally);
 	wait(progress.openFile(container));
 	while (true) {
 		auto batch = progress.getNextBatch();
@@ -599,7 +644,7 @@ ACTOR Future<Void> process_file(Reference<IBackupContainer> container, LogFile f
 			break;
 
 		const VersionedMutations& vms = batch.get();
-		if (vms.version < params.beginVersionFilter || vms.version >= params.endVersionFilter) {
+		if (vms.version < params->beginVersionFilter || vms.version >= params->endVersionFilter) {
 			TraceEvent("SkipVersion").detail("Version", vms.version);
 			continue;
 		}
@@ -607,10 +652,10 @@ ACTOR Future<Void> process_file(Reference<IBackupContainer> container, LogFile f
 		int sub = 0;
 		for (const auto& m : vms.mutations) {
 			sub++; // sub sequence number starts at 1
-			bool print = params.prefixes.empty(); // no filtering
+			bool print = params->prefixes.empty(); // no filtering
 
 			if (!print) {
-				print = params.matchFilters(m);
+				print = params->matchFilters(m);
 			}
 			if (print) {
 				TraceEvent(SevVerbose, format("Mutation_%llu_%d", vms.version, sub).c_str(), uid)
@@ -625,9 +670,9 @@ ACTOR Future<Void> process_file(Reference<IBackupContainer> container, LogFile f
 	return Void();
 }
 
-ACTOR Future<Void> decode_logs(DecodeParams params) {
+ACTOR Future<Void> decode_logs(Reference<DecodeParams> params) {
 	state Reference<IBackupContainer> container =
-	    IBackupContainer::openContainer(params.container_url, params.proxy, {});
+	    IBackupContainer::openContainer(params->container_url, params->proxy, {});
 	state UID uid = deterministicRandom()->randomUniqueID();
 	state BackupFileList listing = wait(container->dumpFileList());
 	// remove partitioned logs
@@ -639,8 +684,8 @@ ACTOR Future<Void> decode_logs(DecodeParams params) {
 	                                  }),
 	                   listing.logs.end());
 	std::sort(listing.logs.begin(), listing.logs.end());
-	TraceEvent("Container", uid).detail("URL", params.container_url).detail("Logs", listing.logs.size());
-	TraceEvent("DecodeParam", uid).setMaxFieldLength(100000).detail("Value", params.toString());
+	TraceEvent("Container", uid).detail("URL", params->container_url).detail("Logs", listing.logs.size());
+	TraceEvent("DecodeParam", uid).setMaxFieldLength(100000).detail("Value", params->toString());
 
 	BackupDescription desc = wait(container->describeBackup());
 	std::cout << "\n" << desc.toString() << "\n";
@@ -648,7 +693,7 @@ ACTOR Future<Void> decode_logs(DecodeParams params) {
 	state std::vector<LogFile> logs = getRelevantLogFiles(listing.logs, params);
 	printLogFiles("Relevant files are: ", logs);
 
-	if (params.list_only)
+	if (params->list_only)
 		return Void();
 
 	state int idx = 0;
@@ -667,31 +712,32 @@ int main(int argc, char** argv) {
 	try {
 		std::unique_ptr<CSimpleOpt> args(
 		    new CSimpleOpt(argc, argv, file_converter::gConverterOptions, SO_O_EXACT | SO_O_HYPHEN_TO_UNDERSCORE));
-		file_converter::DecodeParams param;
-		int status = file_converter::parseDecodeCommandLine(&param, args.get());
-		std::cout << "Params: " << param.toString() << "\n";
+		auto param = makeReference<file_converter::DecodeParams>();
+		int status = file_converter::parseDecodeCommandLine(param, args.get());
+		std::cout << "Params: " << param->toString() << "\n";
+		param->updateRangeMap();
 		if (status != FDB_EXIT_SUCCESS) {
 			file_converter::printDecodeUsage();
 			return status;
 		}
 
-		if (param.log_enabled) {
-			if (param.log_dir.empty()) {
+		if (param->log_enabled) {
+			if (param->log_dir.empty()) {
 				setNetworkOption(FDBNetworkOptions::TRACE_ENABLE);
 			} else {
-				setNetworkOption(FDBNetworkOptions::TRACE_ENABLE, StringRef(param.log_dir));
+				setNetworkOption(FDBNetworkOptions::TRACE_ENABLE, StringRef(param->log_dir));
 			}
-			if (!param.trace_format.empty()) {
-				setNetworkOption(FDBNetworkOptions::TRACE_FORMAT, StringRef(param.trace_format));
+			if (!param->trace_format.empty()) {
+				setNetworkOption(FDBNetworkOptions::TRACE_FORMAT, StringRef(param->trace_format));
 			} else {
 				setNetworkOption(FDBNetworkOptions::TRACE_FORMAT, "json"_sr);
 			}
-			if (!param.trace_log_group.empty()) {
-				setNetworkOption(FDBNetworkOptions::TRACE_LOG_GROUP, StringRef(param.trace_log_group));
+			if (!param->trace_log_group.empty()) {
+				setNetworkOption(FDBNetworkOptions::TRACE_LOG_GROUP, StringRef(param->trace_log_group));
 			}
 		}
 
-		if (!param.tlsConfig.setupTLS()) {
+		if (!param->tlsConfig.setupTLS()) {
 			TraceEvent(SevError, "TLSError").log();
 			throw tls_error();
 		}
@@ -699,15 +745,15 @@ int main(int argc, char** argv) {
 		platformInit();
 		Error::init();
 
-		StringRef url(param.container_url);
+		StringRef url(param->container_url);
 		setupNetwork(0, UseMetrics::True);
 
 		// Must be called after setupNetwork() to be effective
-		param.updateKnobs();
+		param->updateKnobs();
 
 		TraceEvent::setNetworkThread();
-		openTraceFile(NetworkAddress(), 10 << 20, 500 << 20, param.log_dir, "decode", param.trace_log_group);
-		param.tlsConfig.setupBlobCredentials();
+		openTraceFile(NetworkAddress(), 10 << 20, 500 << 20, param->log_dir, "decode", param->trace_log_group);
+		param->tlsConfig.setupBlobCredentials();
 
 		auto f = stopAfter(decode_logs(param));
 

From 41a92e958178453ebccd9d49ba26994f6d6c74f8 Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Sat, 25 Feb 2023 13:54:42 -0800
Subject: [PATCH 085/468] Disable filter validate by default

---
 fdbbackup/FileConverter.h       |  2 ++
 fdbbackup/FileDecoder.actor.cpp | 21 ++++++++++++++++-----
 2 files changed, 18 insertions(+), 5 deletions(-)

diff --git a/fdbbackup/FileConverter.h b/fdbbackup/FileConverter.h
index 87e8cb830b4..c6e3e2f8190 100644
--- a/fdbbackup/FileConverter.h
+++ b/fdbbackup/FileConverter.h
@@ -42,6 +42,7 @@ enum {
 	OPT_INPUT_FILE,
 	OPT_BUILD_FLAGS,
 	OPT_LIST_ONLY,
+	OPT_VALIDATE_FILTERS,
 	OPT_KEY_PREFIX,
 	OPT_FILTERS,
 	OPT_HEX_KEY_PREFIX,
@@ -71,6 +72,7 @@ CSimpleOpt::SOption gConverterOptions[] = { { OPT_CONTAINER, "-r", SO_REQ_SEP },
 #endif
 	                                        { OPT_BUILD_FLAGS, "--build-flags", SO_NONE },
 	                                        { OPT_LIST_ONLY, "--list-only", SO_NONE },
+	                                        { OPT_VALIDATE_FILTERS, "--validate-filters", SO_NONE },
 	                                        { OPT_KEY_PREFIX, "-k", SO_REQ_SEP },
 	                                        { OPT_FILTERS, "--filters", SO_REQ_SEP },
 	                                        { OPT_HEX_KEY_PREFIX, "--hex-prefix", SO_REQ_SEP },
diff --git a/fdbbackup/FileDecoder.actor.cpp b/fdbbackup/FileDecoder.actor.cpp
index a1cd2b4150a..7c88bec0b3f 100644
--- a/fdbbackup/FileDecoder.actor.cpp
+++ b/fdbbackup/FileDecoder.actor.cpp
@@ -87,7 +87,8 @@ void printDecodeUsage() {
 #endif
 	             "  --build-flags  Print build information and exit.\n"
 	             "  --list-only    Print file list and exit.\n"
-	             "  -k KEY_PREFIX  Use the prefix for filtering mutations\n"
+	             "  --validate-filters Validate the default RangeMap filtering logic with a slower one.\n"
+	             "  -k KEY_PREFIX  Use a single prefix for filtering mutations.\n"
 	             "  --filters PREFIX_FILTER_FILE\n"
 	             "                 A file containing a list of prefix filters in HEX format separated by \";\",\n"
 	             "                 e.g., \"\\x05\\x01;\\x15\\x2b\"\n"
@@ -117,6 +118,7 @@ struct DecodeParams : public ReferenceCounted<DecodeParams> {
 	BackupTLSConfig tlsConfig;
 	bool list_only = false;
 	bool save_file_locally = false;
+	bool validate_filters = false;
 	std::vector<std::string> prefixes; // Key prefixes for filtering
 	// more efficient data structure for intersection queries than "prefixes"
 	KeyRangeMap<int> rangeMap;
@@ -142,7 +144,7 @@ struct DecodeParams : public ReferenceCounted<DecodeParams> {
 		bool match = false;
 		if (isSingleKeyMutation((MutationRef::Type)m.type)) {
 			auto ranges = rangeMap.intersectingRanges(singleKeyRange(m.param1));
-			for ([[maybe_unused]] const auto r : ranges) {
+			for (const auto& r : ranges) {
 				if (r.cvalue() == 1) {
 					match = true;
 					break;
@@ -150,7 +152,7 @@ struct DecodeParams : public ReferenceCounted<DecodeParams> {
 			}
 		} else if (m.type == MutationRef::ClearRange) {
 			auto ranges = rangeMap.intersectingRanges(KeyRangeRef(m.param1, m.param2));
-			for ([[maybe_unused]] const auto r : ranges) {
+			for (const auto& r : ranges) {
 				if (r.cvalue() == 1) {
 					match = true;
 					break;
@@ -159,7 +161,11 @@ struct DecodeParams : public ReferenceCounted<DecodeParams> {
 		} else {
 			ASSERT(false);
 		}
+		if (!validate_filters) {
+			return match;
+		}
 
+		// If we choose to validate the filters, go through filters one by one
 		for (const auto& prefix : prefixes) {
 			if (isSingleKeyMutation((MutationRef::Type)m.type)) {
 				if (m.param1.startsWith(StringRef(prefix))) {
@@ -170,8 +176,8 @@ struct DecodeParams : public ReferenceCounted<DecodeParams> {
 				KeyRange range(KeyRangeRef(m.param1, m.param2));
 				KeyRange range2 = prefixRange(StringRef(prefix));
 				if (range.intersects(range2)) {
-				ASSERT(match);
-				return true;
+					ASSERT(match);
+					return true;
 				}
 			} else {
 				ASSERT(false);
@@ -203,6 +209,7 @@ struct DecodeParams : public ReferenceCounted<DecodeParams> {
 			}
 		}
 		s.append(", list_only: ").append(list_only ? "true" : "false");
+		s.append(", validate_filters: ").append(validate_filters ? "true" : "false");
 		if (beginVersionFilter != 0) {
 			s.append(", beginVersionFilter: ").append(std::to_string(beginVersionFilter));
 		}
@@ -343,6 +350,10 @@ int parseDecodeCommandLine(Reference<DecodeParams> param, CSimpleOpt* args) {
 			param->list_only = true;
 			break;
 
+		case OPT_VALIDATE_FILTERS:
+			param->validate_filters = true;
+			break;
+
 		case OPT_KEY_PREFIX:
 			param->prefixes.push_back(args->OptionArg());
 			break;

From 2c8f9da932633330662d3b6403ac2ba48237a908 Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Sat, 25 Feb 2023 16:43:42 -0800
Subject: [PATCH 086/468] Use RangeMap for backup agent filtering

This is more efficient than going through ranges one by one.
---
 fdbclient/BackupContainer.h         |  2 +-
 fdbclient/FileBackupAgent.actor.cpp | 29 ++++++++++++++++++++---------
 2 files changed, 21 insertions(+), 10 deletions(-)

diff --git a/fdbclient/BackupContainer.h b/fdbclient/BackupContainer.h
index 36c9ff7cfab..ff7ba3e8910 100644
--- a/fdbclient/BackupContainer.h
+++ b/fdbclient/BackupContainer.h
@@ -335,7 +335,7 @@ struct AccumulatedMutations {
 	// Returns true if a complete chunk contains any MutationRefs which intersect with any
 	// range in ranges.
 	// It is undefined behavior to run this if isComplete() does not return true.
-	bool matchesAnyRange(const std::vector<KeyRange>& ranges) const;
+	bool matchesAnyRange(const KeyRangeMap<int>& rangeMap) const;
 
 	std::vector<KeyValueRef> kvs;
 	std::string serializedMutations;
diff --git a/fdbclient/FileBackupAgent.actor.cpp b/fdbclient/FileBackupAgent.actor.cpp
index edf1ef5c1fc..558fdbf6d35 100644
--- a/fdbclient/FileBackupAgent.actor.cpp
+++ b/fdbclient/FileBackupAgent.actor.cpp
@@ -3331,19 +3331,25 @@ bool AccumulatedMutations::isComplete() const {
 // Returns true if a complete chunk contains any MutationRefs which intersect with any
 // range in ranges.
 // It is undefined behavior to run this if isComplete() does not return true.
-bool AccumulatedMutations::matchesAnyRange(const std::vector<KeyRange>& ranges) const {
+bool AccumulatedMutations::matchesAnyRange(const KeyRangeMap<int>& rangeMap) const {
 	std::vector<MutationRef> mutations = decodeMutationLogValue(serializedMutations);
 	for (auto& m : mutations) {
-		for (auto& r : ranges) {
-			if (m.type == MutationRef::ClearRange) {
-				if (r.intersects(KeyRangeRef(m.param1, m.param2))) {
+		if (isSingleKeyMutation((MutationRef::Type)m.type)) {
+			auto ranges = rangeMap.intersectingRanges(singleKeyRange(m.param1));
+			for (const auto& r : ranges) {
+				if (r.cvalue() == 1) {
 					return true;
 				}
-			} else {
-				if (r.contains(m.param1)) {
+			}
+		} else if (m.type == MutationRef::ClearRange) {
+			auto ranges = rangeMap.intersectingRanges(KeyRangeRef(m.param1, m.param2));
+			for (const auto& r : ranges) {
+				if (r.cvalue() == 1) {
 					return true;
 				}
 			}
+		} else {
+			ASSERT(false);
 		}
 	}
 
@@ -3352,7 +3358,7 @@ bool AccumulatedMutations::matchesAnyRange(const std::vector<KeyRange>& ranges)
 
 // Returns a vector of filtered KV refs from data which are either part of incomplete mutation groups OR complete
 // and have data relevant to one of the KV ranges in ranges
-std::vector<KeyValueRef> filterLogMutationKVPairs(VectorRef<KeyValueRef> data, const std::vector<KeyRange>& ranges) {
+std::vector<KeyValueRef> filterLogMutationKVPairs(VectorRef<KeyValueRef> data, const KeyRangeMap<int>& rangeMap) {
 	std::unordered_map<Version, AccumulatedMutations> mutationBlocksByVersion;
 
 	for (auto& kv : data) {
@@ -3366,7 +3372,7 @@ std::vector<KeyValueRef> filterLogMutationKVPairs(VectorRef<KeyValueRef> data, c
 		AccumulatedMutations& m = vb.second;
 
 		// If the mutations are incomplete or match one of the ranges, include in results.
-		if (!m.isComplete() || m.matchesAnyRange(ranges)) {
+		if (!m.isComplete() || m.matchesAnyRange(rangeMap)) {
 			output.insert(output.end(), m.kvs.begin(), m.kvs.end());
 		}
 	}
@@ -3424,6 +3430,11 @@ struct RestoreLogDataTaskFunc : RestoreFileTaskFuncBase {
 				wait(tr->onError(e));
 			}
 		}
+		state KeyRangeMap<int> rangeMap;
+		for (const auto& range : ranges) {
+			rangeMap.insert(range, 1);
+		}
+		rangeMap.coalesce(allKeys);
 
 		state Key mutationLogPrefix = restore.mutationLogPrefix();
 		state Reference<IAsyncFile> inFile = wait(bc->readFile(logFile.fileName));
@@ -3432,7 +3443,7 @@ struct RestoreLogDataTaskFunc : RestoreFileTaskFuncBase {
 
 		// Filter the KV pairs extracted from the log file block to remove any records known to not be needed for this
 		// restore based on the restore range set.
-		state std::vector<KeyValueRef> dataFiltered = filterLogMutationKVPairs(dataOriginal, ranges);
+		state std::vector<KeyValueRef> dataFiltered = filterLogMutationKVPairs(dataOriginal, rangeMap);
 
 		state int start = 0;
 		state int end = dataFiltered.size();

From 7eb805d76d9291b714a3421120c55a263a0b5b64 Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Sat, 25 Feb 2023 21:39:00 -0800
Subject: [PATCH 087/468] Refactor code

---
 fdbbackup/FileDecoder.actor.cpp     | 32 +++---------------
 fdbclient/BackupContainer.h         | 29 +++++++++++++++-
 fdbclient/FileBackupAgent.actor.cpp | 51 +++++++++++++++--------------
 3 files changed, 59 insertions(+), 53 deletions(-)

diff --git a/fdbbackup/FileDecoder.actor.cpp b/fdbbackup/FileDecoder.actor.cpp
index 7c88bec0b3f..7f37af2d3e5 100644
--- a/fdbbackup/FileDecoder.actor.cpp
+++ b/fdbbackup/FileDecoder.actor.cpp
@@ -121,7 +121,7 @@ struct DecodeParams : public ReferenceCounted<DecodeParams> {
 	bool validate_filters = false;
 	std::vector<std::string> prefixes; // Key prefixes for filtering
 	// more efficient data structure for intersection queries than "prefixes"
-	KeyRangeMap<int> rangeMap;
+	fileBackup::RangeMapFilters filters;
 	Version beginVersionFilter = 0;
 	Version endVersionFilter = std::numeric_limits<Version>::max();
 
@@ -133,34 +133,10 @@ struct DecodeParams : public ReferenceCounted<DecodeParams> {
 		return !(begin >= endVersionFilter || end <= beginVersionFilter);
 	}
 
-	void updateRangeMap() {
-		for (const auto& prefix : prefixes) {
-			rangeMap.insert(prefixRange(StringRef(prefix)), 1);
-		}
-		rangeMap.coalesce(allKeys);
-	}
+	void updateRangeMap() { filters.updateFilters(prefixes); }
 
-	bool matchFilters(MutationRef m) {
-		bool match = false;
-		if (isSingleKeyMutation((MutationRef::Type)m.type)) {
-			auto ranges = rangeMap.intersectingRanges(singleKeyRange(m.param1));
-			for (const auto& r : ranges) {
-				if (r.cvalue() == 1) {
-					match = true;
-					break;
-				}
-			}
-		} else if (m.type == MutationRef::ClearRange) {
-			auto ranges = rangeMap.intersectingRanges(KeyRangeRef(m.param1, m.param2));
-			for (const auto& r : ranges) {
-				if (r.cvalue() == 1) {
-					match = true;
-					break;
-				}
-			}
-		} else {
-			ASSERT(false);
-		}
+	bool matchFilters(const MutationRef& m) const {
+		bool match = filters.match(m);
 		if (!validate_filters) {
 			return match;
 		}
diff --git a/fdbclient/BackupContainer.h b/fdbclient/BackupContainer.h
index ff7ba3e8910..2410a3d2161 100644
--- a/fdbclient/BackupContainer.h
+++ b/fdbclient/BackupContainer.h
@@ -315,6 +315,33 @@ class IBackupContainer {
 };
 
 namespace fileBackup {
+// Use RangeMap to store a list of ranges for efficient query if a mutation
+// matches to any of the range.
+class RangeMapFilters {
+public:
+	RangeMapFilters() = default;
+
+	explicit RangeMapFilters(const std::vector<KeyRange>& ranges) {
+		for (const auto& range : ranges) {
+			rangeMap.insert(range, 1);
+		}
+		rangeMap.coalesce(allKeys);
+	}
+
+	void updateFilters(std::vector<std::string>& prefixes) {
+		for (const auto& prefix : prefixes) {
+			rangeMap.insert(prefixRange(StringRef(prefix)), 1);
+		}
+		rangeMap.coalesce(allKeys);
+	}
+
+	// Returns if the mutation matches any filter ranges.
+	bool match(const MutationRef& m) const;
+
+private:
+	KeyRangeMap<int> rangeMap;
+};
+
 // Accumulates mutation log value chunks, as both a vector of chunks and as a combined chunk,
 // in chunk order, and can check the chunk set for completion or intersection with a set
 // of ranges.
@@ -335,7 +362,7 @@ struct AccumulatedMutations {
 	// Returns true if a complete chunk contains any MutationRefs which intersect with any
 	// range in ranges.
 	// It is undefined behavior to run this if isComplete() does not return true.
-	bool matchesAnyRange(const KeyRangeMap<int>& rangeMap) const;
+	bool matchesAnyRange(const RangeMapFilters& rangeMap) const;
 
 	std::vector<KeyValueRef> kvs;
 	std::string serializedMutations;
diff --git a/fdbclient/FileBackupAgent.actor.cpp b/fdbclient/FileBackupAgent.actor.cpp
index 558fdbf6d35..ff58db85253 100644
--- a/fdbclient/FileBackupAgent.actor.cpp
+++ b/fdbclient/FileBackupAgent.actor.cpp
@@ -3331,34 +3331,41 @@ bool AccumulatedMutations::isComplete() const {
 // Returns true if a complete chunk contains any MutationRefs which intersect with any
 // range in ranges.
 // It is undefined behavior to run this if isComplete() does not return true.
-bool AccumulatedMutations::matchesAnyRange(const KeyRangeMap<int>& rangeMap) const {
+bool AccumulatedMutations::matchesAnyRange(const RangeMapFilters& filters) const {
 	std::vector<MutationRef> mutations = decodeMutationLogValue(serializedMutations);
 	for (auto& m : mutations) {
-		if (isSingleKeyMutation((MutationRef::Type)m.type)) {
-			auto ranges = rangeMap.intersectingRanges(singleKeyRange(m.param1));
-			for (const auto& r : ranges) {
-				if (r.cvalue() == 1) {
-					return true;
-				}
+		if (filters.match(m)) {
+			return true;
+		}
+	}
+
+	return false;
+}
+
+bool RangeMapFilters::match(const MutationRef& m) const {
+	if (isSingleKeyMutation((MutationRef::Type)m.type)) {
+		auto ranges = rangeMap.intersectingRanges(singleKeyRange(m.param1));
+		for (const auto& r : ranges) {
+			if (r.cvalue() == 1) {
+				return true;
 			}
-		} else if (m.type == MutationRef::ClearRange) {
-			auto ranges = rangeMap.intersectingRanges(KeyRangeRef(m.param1, m.param2));
-			for (const auto& r : ranges) {
-				if (r.cvalue() == 1) {
-					return true;
-				}
+		}
+	} else if (m.type == MutationRef::ClearRange) {
+		auto ranges = rangeMap.intersectingRanges(KeyRangeRef(m.param1, m.param2));
+		for (const auto& r : ranges) {
+			if (r.cvalue() == 1) {
+				return true;
 			}
-		} else {
-			ASSERT(false);
 		}
+	} else {
+		ASSERT(false);
 	}
-
 	return false;
 }
 
 // Returns a vector of filtered KV refs from data which are either part of incomplete mutation groups OR complete
 // and have data relevant to one of the KV ranges in ranges
-std::vector<KeyValueRef> filterLogMutationKVPairs(VectorRef<KeyValueRef> data, const KeyRangeMap<int>& rangeMap) {
+std::vector<KeyValueRef> filterLogMutationKVPairs(VectorRef<KeyValueRef> data, const RangeMapFilters& filters) {
 	std::unordered_map<Version, AccumulatedMutations> mutationBlocksByVersion;
 
 	for (auto& kv : data) {
@@ -3372,7 +3379,7 @@ std::vector<KeyValueRef> filterLogMutationKVPairs(VectorRef<KeyValueRef> data, c
 		AccumulatedMutations& m = vb.second;
 
 		// If the mutations are incomplete or match one of the ranges, include in results.
-		if (!m.isComplete() || m.matchesAnyRange(rangeMap)) {
+		if (!m.isComplete() || m.matchesAnyRange(filters)) {
 			output.insert(output.end(), m.kvs.begin(), m.kvs.end());
 		}
 	}
@@ -3430,11 +3437,6 @@ struct RestoreLogDataTaskFunc : RestoreFileTaskFuncBase {
 				wait(tr->onError(e));
 			}
 		}
-		state KeyRangeMap<int> rangeMap;
-		for (const auto& range : ranges) {
-			rangeMap.insert(range, 1);
-		}
-		rangeMap.coalesce(allKeys);
 
 		state Key mutationLogPrefix = restore.mutationLogPrefix();
 		state Reference<IAsyncFile> inFile = wait(bc->readFile(logFile.fileName));
@@ -3443,7 +3445,8 @@ struct RestoreLogDataTaskFunc : RestoreFileTaskFuncBase {
 
 		// Filter the KV pairs extracted from the log file block to remove any records known to not be needed for this
 		// restore based on the restore range set.
-		state std::vector<KeyValueRef> dataFiltered = filterLogMutationKVPairs(dataOriginal, rangeMap);
+		RangeMapFilters filters(ranges);
+		state std::vector<KeyValueRef> dataFiltered = filterLogMutationKVPairs(dataOriginal, filters);
 
 		state int start = 0;
 		state int end = dataFiltered.size();

From 91bc83a9a67d81eb99961056137921322bbef86f Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Sun, 26 Feb 2023 12:00:58 -0800
Subject: [PATCH 088/468] Allow fdbbackup, fdbrestore to read keyranges from a
 file

---
 fdbbackup/FileDecoder.actor.cpp | 11 +----------
 fdbbackup/backup.actor.cpp      | 19 +++++++++++++++++++
 2 files changed, 20 insertions(+), 10 deletions(-)

diff --git a/fdbbackup/FileDecoder.actor.cpp b/fdbbackup/FileDecoder.actor.cpp
index 7f37af2d3e5..5b8dafbf1d6 100644
--- a/fdbbackup/FileDecoder.actor.cpp
+++ b/fdbbackup/FileDecoder.actor.cpp
@@ -20,11 +20,9 @@
 
 #include <algorithm>
 #include <cstdlib>
-#include <fstream>
 #include <iostream>
 #include <limits>
 #include <memory>
-#include <streambuf>
 #include <string>
 #include <vector>
 #include <fcntl.h>
@@ -288,14 +286,7 @@ std::vector<std::string> parsePrefixesLine(const std::string& line, bool& err) {
 }
 
 std::vector<std::string> parsePrefixFile(const std::string& filename, bool& err) {
-	std::ifstream t(filename);
-	std::string line;
-
-	t.seekg(0, std::ios::end);
-	line.reserve(t.tellg());
-	t.seekg(0, std::ios::beg);
-
-	line.assign((std::istreambuf_iterator<char>(t)), std::istreambuf_iterator<char>());
+	std::string line = readFileBytes(filename, 64 * 1024 * 1024);
 	return parsePrefixesLine(line, err);
 }
 
diff --git a/fdbbackup/backup.actor.cpp b/fdbbackup/backup.actor.cpp
index 17ee95ac92b..af7efba988a 100644
--- a/fdbbackup/backup.actor.cpp
+++ b/fdbbackup/backup.actor.cpp
@@ -134,6 +134,7 @@ enum {
 	OPT_PROXY,
 	OPT_TAGNAME,
 	OPT_BACKUPKEYS,
+	OPT_BACKUPKEYS_FILE,
 	OPT_WAITFORDONE,
 	OPT_BACKUPKEYS_FILTER,
 	OPT_INCREMENTALONLY,
@@ -248,6 +249,7 @@ CSimpleOpt::SOption g_rgBackupStartOptions[] = {
 	{ OPT_TAGNAME, "-t", SO_REQ_SEP },
 	{ OPT_TAGNAME, "--tagname", SO_REQ_SEP },
 	{ OPT_BACKUPKEYS, "-k", SO_REQ_SEP },
+	{ OPT_BACKUPKEYS_FILE, "--keys-file", SO_REQ_SEP },
 	{ OPT_BACKUPKEYS, "--keys", SO_REQ_SEP },
 	{ OPT_DRYRUN, "-n", SO_NONE },
 	{ OPT_DRYRUN, "--dryrun", SO_NONE },
@@ -721,6 +723,7 @@ CSimpleOpt::SOption g_rgRestoreOptions[] = {
 	{ OPT_TAGNAME, "-t", SO_REQ_SEP },
 	{ OPT_TAGNAME, "--tagname", SO_REQ_SEP },
 	{ OPT_BACKUPKEYS, "-k", SO_REQ_SEP },
+	{ OPT_BACKUPKEYS_FILE, "--keys-file", SO_REQ_SEP },
 	{ OPT_BACKUPKEYS, "--keys", SO_REQ_SEP },
 	{ OPT_WAITFORDONE, "-w", SO_NONE },
 	{ OPT_WAITFORDONE, "--waitfordone", SO_NONE },
@@ -798,6 +801,7 @@ CSimpleOpt::SOption g_rgDBStartOptions[] = {
 	{ OPT_TAGNAME, "-t", SO_REQ_SEP },
 	{ OPT_TAGNAME, "--tagname", SO_REQ_SEP },
 	{ OPT_BACKUPKEYS, "-k", SO_REQ_SEP },
+	{ OPT_BACKUPKEYS_FILE, "--keys-file", SO_REQ_SEP },
 	{ OPT_BACKUPKEYS, "--keys", SO_REQ_SEP },
 	{ OPT_TRACE, "--log", SO_NONE },
 	{ OPT_TRACE_DIR, "--logdir", SO_REQ_SEP },
@@ -1125,6 +1129,8 @@ static void printBackupUsage(bool devhelp) {
 	printf("  -e ERRORLIMIT  The maximum number of errors printed by status (default is 10).\n");
 	printf("  -k KEYS        List of key ranges to backup or to filter the backup in query operations.\n"
 	       "                 If not specified, the entire database will be backed up or no filter will be applied.\n");
+	printf("  --keys-file FILE\n"
+	       "                 Same as -k option, except keys are specified in the input file.\n");
 	printf("  --partitioned-log-experimental  Starts with new type of backup system using partitioned logs.\n");
 	printf("  -n, --dryrun   For backup start or restore start, performs a trial run with no actual changes made.\n");
 	printf("  --log          Enables trace file logging for the CLI session.\n"
@@ -1198,6 +1204,8 @@ static void printRestoreUsage(bool devhelp) {
 	printf("  -w, --waitfordone\n");
 	printf("                 Wait for the restore to complete before exiting.  Prints progress updates.\n");
 	printf("  -k KEYS        List of key ranges from the backup to restore.\n");
+	printf("  --keys-file FILE\n"
+	       "                 Same as -k option, except keys are specified in the input file.\n");
 	printf("  --remove-prefix PREFIX\n");
 	printf("                 Prefix to remove from the restored keys.\n");
 	printf("  --add-prefix PREFIX\n");
@@ -1321,6 +1329,8 @@ static void printDBBackupUsage(bool devhelp) {
 	printf("  -e ERRORLIMIT  The maximum number of errors printed by status (default is 10).\n");
 	printf("  -k KEYS        List of key ranges to backup.\n"
 	       "                 If not specified, the entire database will be backed up.\n");
+	printf("  --keys-file FILE\n"
+	       "                 Same as -k option, except keys are specified in the input file.\n");
 	printf("  --cleanup      Abort will attempt to stop mutation logging on the source cluster.\n");
 	printf("  --dstonly      Abort will not make any changes on the source cluster.\n");
 #ifndef TLS_DISABLED
@@ -3645,6 +3655,15 @@ int main(int argc, char* argv[]) {
 					return FDB_EXIT_ERROR;
 				}
 				break;
+			case OPT_BACKUPKEYS_FILE:
+				try {
+					std::string line = readFileBytes(args->OptionArg(), 64 * 1024 * 1024);
+					addKeyRange(line, backupKeys);
+				} catch (Error&) {
+					printHelpTeaser(argv[0]);
+					return FDB_EXIT_ERROR;
+				}
+				break;
 			case OPT_BACKUPKEYS_FILTER:
 				try {
 					addKeyRange(args->OptionArg(), backupKeysFilter);

From 794f37138f6dbc3cfd73b1f8725dd664a4176076 Mon Sep 17 00:00:00 2001
From: Vishesh Yadav <vishesh_yadav@apple.com>
Date: Sun, 26 Feb 2023 12:24:15 -0800
Subject: [PATCH 089/468] Use the RangeMapFilters

---
 fdbbackup/FileDecoder.actor.cpp     | 17 +++++++++++++++++
 fdbclient/BackupContainer.h         |  3 +++
 fdbclient/FileBackupAgent.actor.cpp | 12 ++++++++++++
 3 files changed, 32 insertions(+)

diff --git a/fdbbackup/FileDecoder.actor.cpp b/fdbbackup/FileDecoder.actor.cpp
index ee6935c8c2f..a1465198418 100644
--- a/fdbbackup/FileDecoder.actor.cpp
+++ b/fdbbackup/FileDecoder.actor.cpp
@@ -165,6 +165,23 @@ struct DecodeParams : public ReferenceCounted<DecodeParams> {
 		return false;
 	}
 
+	bool matchFilters(KeyValueRef kv) {
+ 		bool match = filters.match(kv);
+
+		if (!validate_filters) {
+			return match;
+		}
+
+ 		for (const auto& prefix : prefixes) {
+ 			if (kv.key.startsWith(StringRef(prefix))) {
+ 				ASSERT(match);
+ 				return true;
+ 			}
+ 		}
+
+ 		return match;
+ 	}
+
 	std::string toString() {
 		std::string s;
 		s.append("ContainerURL: ");
diff --git a/fdbclient/BackupContainer.h b/fdbclient/BackupContainer.h
index 2410a3d2161..825c5df0f29 100644
--- a/fdbclient/BackupContainer.h
+++ b/fdbclient/BackupContainer.h
@@ -338,6 +338,9 @@ class RangeMapFilters {
 	// Returns if the mutation matches any filter ranges.
 	bool match(const MutationRef& m) const;
 
+	// Returns if the key-value pair matches any filter ranges.
+	bool match(const KeyValueRef& kv) const;
+
 private:
 	KeyRangeMap<int> rangeMap;
 };
diff --git a/fdbclient/FileBackupAgent.actor.cpp b/fdbclient/FileBackupAgent.actor.cpp
index ff58db85253..6fec4b91e01 100644
--- a/fdbclient/FileBackupAgent.actor.cpp
+++ b/fdbclient/FileBackupAgent.actor.cpp
@@ -3363,6 +3363,18 @@ bool RangeMapFilters::match(const MutationRef& m) const {
 	return false;
 }
 
+bool RangeMapFilters::match(const KeyValueRef& kv) const {
+	auto ranges = rangeMap.intersectingRanges(singleKeyRange(kv.key));
+	for (const auto& r : ranges) {
+		if (r.cvalue() == 1) {
+			return true;
+		}
+	}
+
+	return false;
+}
+
+
 // Returns a vector of filtered KV refs from data which are either part of incomplete mutation groups OR complete
 // and have data relevant to one of the KV ranges in ranges
 std::vector<KeyValueRef> filterLogMutationKVPairs(VectorRef<KeyValueRef> data, const RangeMapFilters& filters) {

From c71243a5a55d1f4ddc14ebb1f9bf17794898c54f Mon Sep 17 00:00:00 2001
From: Vishesh Yadav <vishesh_yadav@apple.com>
Date: Sun, 26 Feb 2023 12:47:26 -0800
Subject: [PATCH 090/468] add command line option

---
 fdbbackup/FileConverter.h       |  3 ++
 fdbbackup/FileDecoder.actor.cpp | 79 ++++++++++++++++++++++-----------
 2 files changed, 57 insertions(+), 25 deletions(-)

diff --git a/fdbbackup/FileConverter.h b/fdbbackup/FileConverter.h
index c6e3e2f8190..e564836c9b0 100644
--- a/fdbbackup/FileConverter.h
+++ b/fdbbackup/FileConverter.h
@@ -31,6 +31,7 @@ namespace file_converter {
 // File format convertion constants
 enum {
 	OPT_CONTAINER,
+	OPT_FILE_TYPE,
 	OPT_BEGIN_VERSION,
 	OPT_BLOB_CREDENTIALS,
 	OPT_CRASHONERROR,
@@ -55,6 +56,8 @@ enum {
 
 CSimpleOpt::SOption gConverterOptions[] = { { OPT_CONTAINER, "-r", SO_REQ_SEP },
 	                                        { OPT_CONTAINER, "--container", SO_REQ_SEP },
+											{ OPT_FILE_TYPE, "-t", SO_REQ_SEP },
+											{ OPT_FILE_TYPE, "-file-type", SO_REQ_SEP },
 	                                        { OPT_BEGIN_VERSION, "-b", SO_REQ_SEP },
 	                                        { OPT_BEGIN_VERSION, "--begin", SO_REQ_SEP },
 	                                        { OPT_CRASHONERROR, "--crash", SO_NONE },
diff --git a/fdbbackup/FileDecoder.actor.cpp b/fdbbackup/FileDecoder.actor.cpp
index a1465198418..8b354a59f74 100644
--- a/fdbbackup/FileDecoder.actor.cpp
+++ b/fdbbackup/FileDecoder.actor.cpp
@@ -82,6 +82,8 @@ void printDecodeUsage() {
 	             "  --blob-credentials FILE\n"
 	             "                 File containing blob credentials in JSON format.\n"
 	             "                 The same credential format/file fdbbackup uses.\n"
+	             "  -t, --file-type [log|range|both]\n"
+	             "                 Specifies the backup file type to decode.\n"
 #ifndef TLS_DISABLED
 	    TLS_HELP
 #endif
@@ -117,6 +119,8 @@ struct DecodeParams : public ReferenceCounted<DecodeParams> {
 	std::string log_dir, trace_format, trace_log_group;
 	BackupTLSConfig tlsConfig;
 	bool list_only = false;
+	bool decode_logs = true;
+	bool decode_range = true;
 	bool save_file_locally = false;
 	bool validate_filters = false;
 	std::vector<std::string> prefixes; // Key prefixes for filtering
@@ -165,22 +169,22 @@ struct DecodeParams : public ReferenceCounted<DecodeParams> {
 		return false;
 	}
 
-	bool matchFilters(KeyValueRef kv) {
- 		bool match = filters.match(kv);
+	bool matchFilters(KeyValueRef kv) const {
+		bool match = filters.match(kv);
 
 		if (!validate_filters) {
 			return match;
 		}
 
- 		for (const auto& prefix : prefixes) {
- 			if (kv.key.startsWith(StringRef(prefix))) {
- 				ASSERT(match);
- 				return true;
- 			}
- 		}
+		for (const auto& prefix : prefixes) {
+			if (kv.key.startsWith(StringRef(prefix))) {
+				ASSERT(match);
+				return true;
+			}
+		}
 
- 		return match;
- 	}
+		return match;
+	}
 
 	std::string toString() {
 		std::string s;
@@ -334,6 +338,20 @@ int parseDecodeCommandLine(Reference<DecodeParams> param, CSimpleOpt* args) {
 			param->container_url = args->OptionArg();
 			break;
 
+		case OPT_FILE_TYPE: {
+			auto ftype = std::string(args->OptionArg());
+			if (ftype == "log") {
+				param->decode_range = false;
+			} else if (ftype == "range") {
+				param->decode_logs = false;
+			} else if (ftype != "both" && ftype != "") {
+				err = true;
+				std::cerr << "ERROR: Unrecognized backup file type option: " << args->OptionArg() << "\n";
+				return FDB_EXIT_ERROR;
+			}
+			break;
+		}
+
 		case OPT_LIST_ONLY:
 			param->list_only = true;
 			break;
@@ -816,32 +834,43 @@ ACTOR Future<Void> decode_logs(Reference<DecodeParams> params) {
 	BackupDescription desc = wait(container->describeBackup());
 	std::cout << "\n" << desc.toString() << "\n";
 
-	state std::vector<LogFile> logFiles = getRelevantLogFiles(listing.logs, params);
-	printLogFiles("Relevant log files are: ", logFiles);
+	state std::vector<LogFile> logFiles;
+	state std::vector<RangeFile> rangeFiles;
 
-	state std::vector<RangeFile> rangeFiles = getRelevantRangeFiles(listing.ranges, params);
-	printLogFiles("Releavant range files are: ", rangeFiles);
+	if (params->decode_logs) {
+		logFiles = getRelevantLogFiles(listing.logs, params);
+		printLogFiles("Relevant log files are: ", logFiles);
+	}
+
+	if (params->decode_range) {
+		rangeFiles = getRelevantRangeFiles(listing.ranges, params);
+		printLogFiles("Releavant range files are: ", rangeFiles);
+	}
 
 	if (params->list_only)
 		return Void();
 
 	// Decode log files.
 	state int idx = 0;
-	while (idx < logFiles.size()) {
-		TraceEvent("ProcessFile").detail("Name", logFiles[idx].fileName).detail("I", idx);
-		wait(process_file(container, logFiles[idx], uid, params));
-		idx++;
+	if (params->decode_logs) {
+		while (idx < logFiles.size()) {
+			TraceEvent("ProcessFile").detail("Name", logFiles[idx].fileName).detail("I", idx);
+			wait(process_file(container, logFiles[idx], uid, params));
+			idx++;
+		}
+		TraceEvent("DecodeLogsDone", uid).log();
 	}
-	TraceEvent("DecodeLogsDone", uid).log();
 
 	// Decode range files.
-	idx = 0;
-	while (idx < rangeFiles.size()) {
-		TraceEvent("ProcessFile").detail("Name", rangeFiles[idx].fileName).detail("I", idx);
-		wait(process_range_file(container, rangeFiles[idx], uid, params));
-		idx++;
+	if (params->decode_range) {
+		idx = 0;
+		while (idx < rangeFiles.size()) {
+			TraceEvent("ProcessFile").detail("Name", rangeFiles[idx].fileName).detail("I", idx);
+			wait(process_range_file(container, rangeFiles[idx], uid, params));
+			idx++;
+		}
+		TraceEvent("DecodeRangeFileDone", uid).log();
 	}
-	TraceEvent("DecodeRangeFileDone", uid).log();
 
 	return Void();
 }

From e9002835bcc17bb2b8cd98f5af2c364955c19070 Mon Sep 17 00:00:00 2001
From: Vishesh Yadav <vishesh_yadav@apple.com>
Date: Sun, 26 Feb 2023 12:56:34 -0800
Subject: [PATCH 091/468] Clang-format

---
 fdbbackup/FileConverter.h           | 4 ++--
 fdbclient/FileBackupAgent.actor.cpp | 1 -
 2 files changed, 2 insertions(+), 3 deletions(-)

diff --git a/fdbbackup/FileConverter.h b/fdbbackup/FileConverter.h
index e564836c9b0..2bb72561261 100644
--- a/fdbbackup/FileConverter.h
+++ b/fdbbackup/FileConverter.h
@@ -56,8 +56,8 @@ enum {
 
 CSimpleOpt::SOption gConverterOptions[] = { { OPT_CONTAINER, "-r", SO_REQ_SEP },
 	                                        { OPT_CONTAINER, "--container", SO_REQ_SEP },
-											{ OPT_FILE_TYPE, "-t", SO_REQ_SEP },
-											{ OPT_FILE_TYPE, "-file-type", SO_REQ_SEP },
+	                                        { OPT_FILE_TYPE, "-t", SO_REQ_SEP },
+	                                        { OPT_FILE_TYPE, "-file-type", SO_REQ_SEP },
 	                                        { OPT_BEGIN_VERSION, "-b", SO_REQ_SEP },
 	                                        { OPT_BEGIN_VERSION, "--begin", SO_REQ_SEP },
 	                                        { OPT_CRASHONERROR, "--crash", SO_NONE },
diff --git a/fdbclient/FileBackupAgent.actor.cpp b/fdbclient/FileBackupAgent.actor.cpp
index 6fec4b91e01..f2ac8110ee9 100644
--- a/fdbclient/FileBackupAgent.actor.cpp
+++ b/fdbclient/FileBackupAgent.actor.cpp
@@ -3374,7 +3374,6 @@ bool RangeMapFilters::match(const KeyValueRef& kv) const {
 	return false;
 }
 
-
 // Returns a vector of filtered KV refs from data which are either part of incomplete mutation groups OR complete
 // and have data relevant to one of the KV ranges in ranges
 std::vector<KeyValueRef> filterLogMutationKVPairs(VectorRef<KeyValueRef> data, const RangeMapFilters& filters) {

From 481d4eda3777241fd7345e93a2811bd93dbe257b Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyu_zhou@apple.com>
Date: Mon, 27 Feb 2023 08:53:59 -0800
Subject: [PATCH 092/468] Fix -t flag bug for fdbdecode (#9489)

---
 fdbbackup/FileConverter.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/fdbbackup/FileConverter.h b/fdbbackup/FileConverter.h
index 2bb72561261..dfb23d03fa1 100644
--- a/fdbbackup/FileConverter.h
+++ b/fdbbackup/FileConverter.h
@@ -57,7 +57,7 @@ enum {
 CSimpleOpt::SOption gConverterOptions[] = { { OPT_CONTAINER, "-r", SO_REQ_SEP },
 	                                        { OPT_CONTAINER, "--container", SO_REQ_SEP },
 	                                        { OPT_FILE_TYPE, "-t", SO_REQ_SEP },
-	                                        { OPT_FILE_TYPE, "-file-type", SO_REQ_SEP },
+	                                        { OPT_FILE_TYPE, "--file-type", SO_REQ_SEP },
 	                                        { OPT_BEGIN_VERSION, "-b", SO_REQ_SEP },
 	                                        { OPT_BEGIN_VERSION, "--begin", SO_REQ_SEP },
 	                                        { OPT_CRASHONERROR, "--crash", SO_NONE },

From 6593fa7ee61c0a385ef0e59f7068ffb88c5a93d0 Mon Sep 17 00:00:00 2001
From: Zhe Wu <halfprice@users.noreply.github.com>
Date: Sat, 25 Feb 2023 16:44:45 -0800
Subject: [PATCH 093/468] Fix fdbbackup query returning earliest version

---
 fdbbackup/backup.actor.cpp                    | 5 +++--
 fdbclient/BackupContainerFileSystem.actor.cpp | 5 ++++-
 fdbclient/FDBTypes.h                          | 7 ++++++-
 3 files changed, 13 insertions(+), 4 deletions(-)

diff --git a/fdbbackup/backup.actor.cpp b/fdbbackup/backup.actor.cpp
index 17ee95ac92b..700969d1909 100644
--- a/fdbbackup/backup.actor.cpp
+++ b/fdbbackup/backup.actor.cpp
@@ -1090,7 +1090,7 @@ static void printBackupUsage(bool devhelp) {
 	       "the backup.\n");
 	printf("  -qrv --query-restore-version VERSION\n"
 	       "                 For query operations, set target version for restoring a backup. Set -1 for maximum\n"
-	       "                 restorable version (default) and -2 for minimum restorable version.\n");
+	       "                 restorable version (default) and -3 for minimum restorable version.\n");
 	printf(
 	    "  --query-restore-timestamp DATETIME\n"
 	    "                 For query operations, instead of a numeric version, use this to specify a timestamp in %s\n",
@@ -2799,9 +2799,10 @@ ACTOR Future<Void> queryBackup(const char* name,
 				    result,
 				    errorMessage = format("the backup for the specified key ranges is not restorable to any version"));
 			}
+			TraceEvent("BackupQueryResolveMaxRestoreVersion").detail("Version", restoreVersion);
 		}
 
-		if (restoreVersion < 0 && restoreVersion != latestVersion) {
+		if (restoreVersion < 0 && restoreVersion != earliestVersion) {
 			reportBackupQueryError(operationId,
 			                       result,
 			                       errorMessage =
diff --git a/fdbclient/BackupContainerFileSystem.actor.cpp b/fdbclient/BackupContainerFileSystem.actor.cpp
index a4778ecc104..99a40a1a96f 100644
--- a/fdbclient/BackupContainerFileSystem.actor.cpp
+++ b/fdbclient/BackupContainerFileSystem.actor.cpp
@@ -925,6 +925,9 @@ class BackupContainerFileSystemImpl {
 		// Find the most recent keyrange snapshot through which we can restore filtered key ranges into targetVersion.
 		state std::vector<KeyspaceSnapshotFile> snapshots = wait(bc->listKeyspaceSnapshots());
 		state int i = snapshots.size() - 1;
+		if (targetVersion == earliestVersion) {
+			i = 0;
+		}
 		for (; i >= 0; i--) {
 			// The smallest version of filtered range files >= snapshot beginVersion > targetVersion
 			if (targetVersion >= 0 && snapshots[i].beginVersion > targetVersion) {
@@ -965,7 +968,7 @@ class BackupContainerFileSystemImpl {
 				}
 			}
 			// 'latestVersion' represents using the minimum restorable version in a snapshot.
-			restorable.targetVersion = targetVersion == latestVersion ? maxKeyRangeVersion : targetVersion;
+			restorable.targetVersion = targetVersion == earliestVersion ? maxKeyRangeVersion : targetVersion;
 			// Any version < maxKeyRangeVersion is not restorable.
 			if (restorable.targetVersion < maxKeyRangeVersion)
 				continue;
diff --git a/fdbclient/FDBTypes.h b/fdbclient/FDBTypes.h
index 447124e3e1f..c494762743e 100644
--- a/fdbclient/FDBTypes.h
+++ b/fdbclient/FDBTypes.h
@@ -514,7 +514,12 @@ using KeySelector = Standalone<struct KeySelectorRef>;
 using RangeResult = Standalone<struct RangeResultRef>;
 using MappedRangeResult = Standalone<struct MappedRangeResultRef>;
 
-enum { invalidVersion = -1, latestVersion = -2, MAX_VERSION = std::numeric_limits<int64_t>::max() };
+enum {
+	invalidVersion = -1,
+	latestVersion = -2,
+	earliestVersion = -3,
+	MAX_VERSION = std::numeric_limits<int64_t>::max()
+};
 
 inline Key keyAfter(const KeyRef& key) {
 	if (key == LiteralStringRef("\xff\xff"))

From dcdee8c58463fb01f45fd87262e0dc6fa92f8de9 Mon Sep 17 00:00:00 2001
From: Zhe Wu <halfprice@users.noreply.github.com>
Date: Sat, 25 Feb 2023 17:38:27 -0800
Subject: [PATCH 094/468] Query backup size from a specific snapshot

---
 fdbbackup/backup.actor.cpp  | 101 +++++++++++++++++++++++++++++++-----
 fdbclient/BackupContainer.h |   2 +-
 2 files changed, 89 insertions(+), 14 deletions(-)

diff --git a/fdbbackup/backup.actor.cpp b/fdbbackup/backup.actor.cpp
index 700969d1909..560e31638bd 100644
--- a/fdbbackup/backup.actor.cpp
+++ b/fdbbackup/backup.actor.cpp
@@ -146,6 +146,7 @@ enum {
 	// Restore constants
 	OPT_RESTORECONTAINER,
 	OPT_RESTORE_VERSION,
+	OPT_RESTORE_SNAPSHOT_VERSION,
 	OPT_RESTORE_TIMESTAMP,
 	OPT_PREFIX_ADD,
 	OPT_PREFIX_REMOVE,
@@ -679,6 +680,7 @@ CSimpleOpt::SOption g_rgBackupQueryOptions[] = {
 	{ OPT_PROXY, "--proxy", SO_REQ_SEP },
 	{ OPT_RESTORE_VERSION, "-qrv", SO_REQ_SEP },
 	{ OPT_RESTORE_VERSION, "--query-restore-version", SO_REQ_SEP },
+	{ OPT_RESTORE_SNAPSHOT_VERSION, "--query-restore-snapshot-version", SO_REQ_SEP },
 	{ OPT_BACKUPKEYS_FILTER, "-k", SO_REQ_SEP },
 	{ OPT_BACKUPKEYS_FILTER, "--keys", SO_REQ_SEP },
 	{ OPT_TRACE, "--log", SO_NONE },
@@ -1088,6 +1090,9 @@ static void printBackupUsage(bool devhelp) {
 	       "containing no data at or after a\n"
 	       "                 version approximately NUM_DAYS days worth of versions prior to the latest log version in "
 	       "the backup.\n");
+	printf("  --query-restore-snapshot-version VERSION\n"
+	       "                 For query operations, set snapshot version for restoring a backup. Set -1 for maximum\n"
+	       "                 restorable version (default) and -3 for minimum restorable version.\n");
 	printf("  -qrv --query-restore-version VERSION\n"
 	       "                 For query operations, set target version for restoring a backup. Set -1 for maximum\n"
 	       "                 restorable version (default) and -3 for minimum restorable version.\n");
@@ -2741,6 +2746,7 @@ ACTOR Future<Void> queryBackup(const char* name,
                                Optional<std::string> proxy,
                                Standalone<VectorRef<KeyRangeRef>> keyRangesFilter,
                                Version restoreVersion,
+                               Version snapshotVersion,
                                std::string originalClusterFile,
                                std::string restoreTimestamp,
                                Verbose verbose) {
@@ -2755,6 +2761,7 @@ ACTOR Future<Void> queryBackup(const char* name,
 	    .detail("DestinationContainer", destinationContainer)
 	    .detail("KeyRangesFilter", printable(keyRangesFilter))
 	    .detail("SpecifiedRestoreVersion", restoreVersion)
+	    .detail("SpecifiedSnapshotVersion", snapshotVersion)
 	    .detail("RestoreTimestamp", restoreTimestamp)
 	    .detail("BackupClusterFile", originalClusterFile);
 
@@ -2809,12 +2816,67 @@ ACTOR Future<Void> queryBackup(const char* name,
 			                           format("the specified restorable version %lld is not valid", restoreVersion));
 			return Void();
 		}
-		Optional<RestorableFileSet> fileSet = wait(bc->getRestoreSet(restoreVersion, keyRangesFilter));
+
+		state int64_t totalRangeFilesSize = 0;
+		state int64_t totalLogFilesSize = 0;
+		state JsonBuilderArray rangeFilesJson;
+		state JsonBuilderArray logFilesJson;
+		if (snapshotVersion != invalidVersion) {
+			Optional<RestorableFileSet> fileSet = wait(bc->getRestoreSet(snapshotVersion, keyRangesFilter));
+			if (fileSet.present()) {
+				result["snapshot_version"] = fileSet.get().targetVersion;
+				result["restore_version"] = fileSet.get().targetVersion;  // In case there isn't any more log files.
+				for (const auto& rangeFile : fileSet.get().ranges) {
+					JsonBuilderObject object;
+					object["file_name"] = rangeFile.fileName;
+					object["file_size"] = rangeFile.fileSize;
+					object["version"] = rangeFile.version;
+					object["key_range"] = fileSet.get().keyRanges.count(rangeFile.fileName) == 0
+					                          ? "none"
+					                          : fileSet.get().keyRanges.at(rangeFile.fileName).toString();
+					rangeFilesJson.push_back(object);
+					totalRangeFilesSize += rangeFile.fileSize;
+				}
+				for (const auto& log : fileSet.get().logs) {
+					JsonBuilderObject object;
+					object["file_name"] = log.fileName;
+					object["file_size"] = log.fileSize;
+					object["begin_version"] = log.beginVersion;
+					object["end_version"] = log.endVersion;
+					logFilesJson.push_back(object);
+					totalLogFilesSize += log.fileSize;
+				}
+
+				snapshotVersion = fileSet.get().targetVersion;
+
+				TraceEvent("BackupQueryReceivedRestorableFilesSetFromSnapshot")
+					.detail("SnapshotVersion", snapshotVersion)
+				    .detail("DestinationContainer", destinationContainer)
+				    .detail("KeyRangesFilter", printable(keyRangesFilter))
+				    .detail("ActualRestoreVersion", fileSet.get().targetVersion)
+				    .detail("NumRangeFiles", fileSet.get().ranges.size())
+				    .detail("NumLogFiles", fileSet.get().logs.size())
+				    .detail("RangeFilesBytes", totalRangeFilesSize)
+				    .detail("LogFilesBytes", totalLogFilesSize);
+			} else {
+				reportBackupQueryError(
+				    operationId,
+				    result,
+				    format("no restorable files set found for specified key ranges from snapshotVersion %lld",
+				           snapshotVersion));
+				return Void();
+			}
+		}
+
+		state Optional<RestorableFileSet> fileSet;
+		if (snapshotVersion == invalidVersion) {
+			wait(store(fileSet, bc->getRestoreSet(restoreVersion, keyRangesFilter)));
+		} else {
+			// We only need to know all the log files from snapshotVersion to restoreVersion.
+			wait(store(fileSet, bc->getRestoreSet(restoreVersion, keyRangesFilter, /*logOnly=*/true, snapshotVersion)));
+		}
 		if (fileSet.present()) {
-			int64_t totalRangeFilesSize = 0, totalLogFilesSize = 0;
 			result["restore_version"] = fileSet.get().targetVersion;
-			JsonBuilderArray rangeFilesJson;
-			JsonBuilderArray logFilesJson;
 			for (const auto& rangeFile : fileSet.get().ranges) {
 				JsonBuilderObject object;
 				object["file_name"] = rangeFile.fileName;
@@ -2836,14 +2898,6 @@ ACTOR Future<Void> queryBackup(const char* name,
 				totalLogFilesSize += log.fileSize;
 			}
 
-			result["total_range_files_size"] = totalRangeFilesSize;
-			result["total_log_files_size"] = totalLogFilesSize;
-
-			if (verbose) {
-				result["ranges"] = rangeFilesJson;
-				result["logs"] = logFilesJson;
-			}
-
 			TraceEvent("BackupQueryReceivedRestorableFilesSet")
 			    .detail("DestinationContainer", destinationContainer)
 			    .detail("KeyRangesFilter", printable(keyRangesFilter))
@@ -2852,7 +2906,7 @@ ACTOR Future<Void> queryBackup(const char* name,
 			    .detail("NumLogFiles", fileSet.get().logs.size())
 			    .detail("RangeFilesBytes", totalRangeFilesSize)
 			    .detail("LogFilesBytes", totalLogFilesSize);
-		} else {
+		} else if (snapshotVersion == invalidVersion) {
 			reportBackupQueryError(operationId, result, "no restorable files set found for specified key ranges");
 			return Void();
 		}
@@ -2862,6 +2916,14 @@ ACTOR Future<Void> queryBackup(const char* name,
 		return Void();
 	}
 
+	result["total_range_files_size"] = totalRangeFilesSize;
+	result["total_log_files_size"] = totalLogFilesSize;
+
+	if (verbose) {
+		result["ranges"] = rangeFilesJson;
+		result["logs"] = logFilesJson;
+	}
+
 	printf("%s\n", result.getJson().c_str());
 	return Void();
 }
@@ -3424,6 +3486,7 @@ int main(int argc, char* argv[]) {
 		int maxErrors = 20;
 		Version beginVersion = invalidVersion;
 		Version restoreVersion = invalidVersion;
+		Version snapshotVersion = invalidVersion;
 		std::string restoreTimestamp;
 		WaitForComplete waitForDone{ false };
 		StopWhenDone stopWhenDone{ true };
@@ -3757,6 +3820,17 @@ int main(int argc, char* argv[]) {
 				restoreVersion = ver;
 				break;
 			}
+			case OPT_RESTORE_SNAPSHOT_VERSION: {
+				const char* a = args->OptionArg();
+				long long ver = 0;
+				if (!sscanf(a, "%lld", &ver)) {
+					fprintf(stderr, "ERROR: Could not parse database version `%s'\n", a);
+					printHelpTeaser(argv[0]);
+					return FDB_EXIT_ERROR;
+				}
+				snapshotVersion = ver;
+				break;
+			}
 			case OPT_RESTORE_INCONSISTENT_SNAPSHOT_ONLY: {
 				inconsistentSnapshotOnly.set(true);
 				break;
@@ -4141,6 +4215,7 @@ int main(int argc, char* argv[]) {
 				                          proxy,
 				                          backupKeysFilter,
 				                          restoreVersion,
+				                          snapshotVersion,
 				                          restoreClusterFileOrig,
 				                          restoreTimestamp,
 				                          Verbose{ !quietDisplay }));
diff --git a/fdbclient/BackupContainer.h b/fdbclient/BackupContainer.h
index 36c9ff7cfab..f61cfa4e965 100644
--- a/fdbclient/BackupContainer.h
+++ b/fdbclient/BackupContainer.h
@@ -285,7 +285,7 @@ class IBackupContainer {
 	                                            Version end = std::numeric_limits<Version>::max()) = 0;
 
 	// Get exactly the files necessary to restore the key space filtered by the specified key ranges to targetVersion.
-	// If targetVersion is 'latestVersion', use the minimum restorable version in a snapshot.
+	// If targetVersion is 'earliestVersion', use the minimum restorable version in a snapshot.
 	// If logsOnly is set, only use log files in [beginVersion, targetVervions) in restore set.
 	// Returns non-present if restoring to the given version is not possible.
 	virtual Future<Optional<RestorableFileSet>> getRestoreSet(Version targetVersion,

From 20c663bf53b1888dd24d3a4364fdbd6acc456742 Mon Sep 17 00:00:00 2001
From: Zhe Wu <halfprice@users.noreply.github.com>
Date: Sat, 25 Feb 2023 18:13:55 -0800
Subject: [PATCH 095/468] clean format

---
 fdbbackup/backup.actor.cpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/fdbbackup/backup.actor.cpp b/fdbbackup/backup.actor.cpp
index 560e31638bd..968e6410cac 100644
--- a/fdbbackup/backup.actor.cpp
+++ b/fdbbackup/backup.actor.cpp
@@ -2825,7 +2825,7 @@ ACTOR Future<Void> queryBackup(const char* name,
 			Optional<RestorableFileSet> fileSet = wait(bc->getRestoreSet(snapshotVersion, keyRangesFilter));
 			if (fileSet.present()) {
 				result["snapshot_version"] = fileSet.get().targetVersion;
-				result["restore_version"] = fileSet.get().targetVersion;  // In case there isn't any more log files.
+				result["restore_version"] = fileSet.get().targetVersion; // In case there isn't any more log files.
 				for (const auto& rangeFile : fileSet.get().ranges) {
 					JsonBuilderObject object;
 					object["file_name"] = rangeFile.fileName;
@@ -2850,7 +2850,7 @@ ACTOR Future<Void> queryBackup(const char* name,
 				snapshotVersion = fileSet.get().targetVersion;
 
 				TraceEvent("BackupQueryReceivedRestorableFilesSetFromSnapshot")
-					.detail("SnapshotVersion", snapshotVersion)
+				    .detail("SnapshotVersion", snapshotVersion)
 				    .detail("DestinationContainer", destinationContainer)
 				    .detail("KeyRangesFilter", printable(keyRangesFilter))
 				    .detail("ActualRestoreVersion", fileSet.get().targetVersion)

From 62c3bf2e0c3793a6d46136cb9eb4c464310da6b1 Mon Sep 17 00:00:00 2001
From: Zhe Wu <halfprice@users.noreply.github.com>
Date: Sun, 26 Feb 2023 12:17:07 -0800
Subject: [PATCH 096/468] Explicitly using min and max restorable version from
 backup description in query command in stead of going throw snapshots

---
 fdbbackup/backup.actor.cpp                    | 48 ++++++++++++-------
 fdbclient/BackupContainer.h                   |  2 +-
 fdbclient/BackupContainerFileSystem.actor.cpp |  6 +--
 3 files changed, 35 insertions(+), 21 deletions(-)

diff --git a/fdbbackup/backup.actor.cpp b/fdbbackup/backup.actor.cpp
index 968e6410cac..7feeeb8e578 100644
--- a/fdbbackup/backup.actor.cpp
+++ b/fdbbackup/backup.actor.cpp
@@ -2738,6 +2738,24 @@ static void reportBackupQueryError(UID operationId, JsonBuilderObject& result, s
 	TraceEvent("BackupQueryFailure").detail("OperationId", operationId).detail("Reason", errorMessage);
 }
 
+std::pair<Version, Version> getMaxMinRestorableVersions(const BackupDescription& desc, bool mayOnlyApplyMutationLog) {
+	Version maxRestorableVersion = invalidVersion;
+	Version minRestorableVersion = invalidVersion;
+	if (desc.maxRestorableVersion.present()) {
+		maxRestorableVersion = desc.maxRestorableVersion.get();
+	} else if (mayOnlyApplyMutationLog && desc.contiguousLogEnd.present()) {
+		maxRestorableVersion = desc.contiguousLogEnd.get();
+	}
+
+	if (desc.minRestorableVersion.present()) {
+		minRestorableVersion = desc.minRestorableVersion.get();
+	} else if (mayOnlyApplyMutationLog && desc.minLogBegin.present()) {
+		minRestorableVersion = desc.minLogBegin.get();
+	}
+
+	return std::make_pair(maxRestorableVersion, minRestorableVersion);
+}
+
 // If restoreVersion is invalidVersion or latestVersion, use the maximum or minimum restorable version respectively for
 // selected key ranges. If restoreTimestamp is specified, any specified restoreVersion will be overriden to the version
 // resolved to that timestamp.
@@ -2793,23 +2811,21 @@ ACTOR Future<Void> queryBackup(const char* name,
 
 	try {
 		state Reference<IBackupContainer> bc = openBackupContainer(name, destinationContainer, proxy, {});
+		BackupDescription desc = wait(bc->describeBackup());
+		auto [maxRestorableVersion, minRestorableVersion] = getMaxMinRestorableVersions(desc, !keyRangesFilter.empty());
 		if (restoreVersion == invalidVersion) {
-			BackupDescription desc = wait(bc->describeBackup());
-			if (desc.maxRestorableVersion.present()) {
-				restoreVersion = desc.maxRestorableVersion.get();
-				// Use continuous log end version for the maximum restorable version for the key ranges.
-			} else if (keyRangesFilter.size() && desc.contiguousLogEnd.present()) {
-				restoreVersion = desc.contiguousLogEnd.get();
-			} else {
-				reportBackupQueryError(
-				    operationId,
-				    result,
-				    errorMessage = format("the backup for the specified key ranges is not restorable to any version"));
-			}
-			TraceEvent("BackupQueryResolveMaxRestoreVersion").detail("Version", restoreVersion);
+			restoreVersion = maxRestorableVersion;
 		}
-
-		if (restoreVersion < 0 && restoreVersion != earliestVersion) {
+		if (restoreVersion == earliestVersion) {
+			restoreVersion = minRestorableVersion;
+		}
+		if (snapshotVersion == earliestVersion) {
+			snapshotVersion = minRestorableVersion;
+		}
+		TraceEvent("BackupQueryResolveRestoreVersion")
+		    .detail("RestoreVersion", restoreVersion)
+		    .detail("SnapshotVersion", snapshotVersion);
+		if (restoreVersion < 0) {
 			reportBackupQueryError(operationId,
 			                       result,
 			                       errorMessage =
@@ -2825,7 +2841,6 @@ ACTOR Future<Void> queryBackup(const char* name,
 			Optional<RestorableFileSet> fileSet = wait(bc->getRestoreSet(snapshotVersion, keyRangesFilter));
 			if (fileSet.present()) {
 				result["snapshot_version"] = fileSet.get().targetVersion;
-				result["restore_version"] = fileSet.get().targetVersion; // In case there isn't any more log files.
 				for (const auto& rangeFile : fileSet.get().ranges) {
 					JsonBuilderObject object;
 					object["file_name"] = rangeFile.fileName;
@@ -2870,6 +2885,7 @@ ACTOR Future<Void> queryBackup(const char* name,
 
 		state Optional<RestorableFileSet> fileSet;
 		if (snapshotVersion == invalidVersion) {
+			// Using the latest snapshot.
 			wait(store(fileSet, bc->getRestoreSet(restoreVersion, keyRangesFilter)));
 		} else {
 			// We only need to know all the log files from snapshotVersion to restoreVersion.
diff --git a/fdbclient/BackupContainer.h b/fdbclient/BackupContainer.h
index f61cfa4e965..36c9ff7cfab 100644
--- a/fdbclient/BackupContainer.h
+++ b/fdbclient/BackupContainer.h
@@ -285,7 +285,7 @@ class IBackupContainer {
 	                                            Version end = std::numeric_limits<Version>::max()) = 0;
 
 	// Get exactly the files necessary to restore the key space filtered by the specified key ranges to targetVersion.
-	// If targetVersion is 'earliestVersion', use the minimum restorable version in a snapshot.
+	// If targetVersion is 'latestVersion', use the minimum restorable version in a snapshot.
 	// If logsOnly is set, only use log files in [beginVersion, targetVervions) in restore set.
 	// Returns non-present if restoring to the given version is not possible.
 	virtual Future<Optional<RestorableFileSet>> getRestoreSet(Version targetVersion,
diff --git a/fdbclient/BackupContainerFileSystem.actor.cpp b/fdbclient/BackupContainerFileSystem.actor.cpp
index 99a40a1a96f..634cba1c15e 100644
--- a/fdbclient/BackupContainerFileSystem.actor.cpp
+++ b/fdbclient/BackupContainerFileSystem.actor.cpp
@@ -912,6 +912,7 @@ class BackupContainerFileSystemImpl {
 
 		if (logsOnly) {
 			state RestorableFileSet restorableSet;
+			restorableSet.targetVersion = targetVersion;
 			state std::vector<LogFile> logFiles;
 			Version begin = beginVersion == invalidVersion ? 0 : beginVersion;
 			wait(store(logFiles, bc->listLogFiles(begin, targetVersion, false)));
@@ -925,9 +926,6 @@ class BackupContainerFileSystemImpl {
 		// Find the most recent keyrange snapshot through which we can restore filtered key ranges into targetVersion.
 		state std::vector<KeyspaceSnapshotFile> snapshots = wait(bc->listKeyspaceSnapshots());
 		state int i = snapshots.size() - 1;
-		if (targetVersion == earliestVersion) {
-			i = 0;
-		}
 		for (; i >= 0; i--) {
 			// The smallest version of filtered range files >= snapshot beginVersion > targetVersion
 			if (targetVersion >= 0 && snapshots[i].beginVersion > targetVersion) {
@@ -968,7 +966,7 @@ class BackupContainerFileSystemImpl {
 				}
 			}
 			// 'latestVersion' represents using the minimum restorable version in a snapshot.
-			restorable.targetVersion = targetVersion == earliestVersion ? maxKeyRangeVersion : targetVersion;
+			restorable.targetVersion = targetVersion == latestVersion ? maxKeyRangeVersion : targetVersion;
 			// Any version < maxKeyRangeVersion is not restorable.
 			if (restorable.targetVersion < maxKeyRangeVersion)
 				continue;

From f281be0798155cc71e64ede47493ef94cb76daae Mon Sep 17 00:00:00 2001
From: Zhe Wu <halfprice@users.noreply.github.com>
Date: Sun, 26 Feb 2023 12:38:52 -0800
Subject: [PATCH 097/468] fix clang build error

---
 fdbbackup/backup.actor.cpp | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/fdbbackup/backup.actor.cpp b/fdbbackup/backup.actor.cpp
index 7feeeb8e578..012400f9d5c 100644
--- a/fdbbackup/backup.actor.cpp
+++ b/fdbbackup/backup.actor.cpp
@@ -2809,6 +2809,10 @@ ACTOR Future<Void> queryBackup(const char* name,
 		restoreVersion = v;
 	}
 
+	state int64_t totalRangeFilesSize = 0;
+	state int64_t totalLogFilesSize = 0;
+	state JsonBuilderArray rangeFilesJson;
+	state JsonBuilderArray logFilesJson;
 	try {
 		state Reference<IBackupContainer> bc = openBackupContainer(name, destinationContainer, proxy, {});
 		BackupDescription desc = wait(bc->describeBackup());
@@ -2833,10 +2837,6 @@ ACTOR Future<Void> queryBackup(const char* name,
 			return Void();
 		}
 
-		state int64_t totalRangeFilesSize = 0;
-		state int64_t totalLogFilesSize = 0;
-		state JsonBuilderArray rangeFilesJson;
-		state JsonBuilderArray logFilesJson;
 		if (snapshotVersion != invalidVersion) {
 			Optional<RestorableFileSet> fileSet = wait(bc->getRestoreSet(snapshotVersion, keyRangesFilter));
 			if (fileSet.present()) {

From e86adab40162278f56dfafaf8c899fb73e6f09be Mon Sep 17 00:00:00 2001
From: Zhe Wu <halfprice@users.noreply.github.com>
Date: Mon, 27 Feb 2023 10:00:49 -0800
Subject: [PATCH 098/468] Add more comments in fdbbackup query command, and
 address comments

---
 fdbbackup/backup.actor.cpp | 24 ++++++++++++++----------
 1 file changed, 14 insertions(+), 10 deletions(-)

diff --git a/fdbbackup/backup.actor.cpp b/fdbbackup/backup.actor.cpp
index 012400f9d5c..2ed11f8d76a 100644
--- a/fdbbackup/backup.actor.cpp
+++ b/fdbbackup/backup.actor.cpp
@@ -1091,8 +1091,9 @@ static void printBackupUsage(bool devhelp) {
 	       "                 version approximately NUM_DAYS days worth of versions prior to the latest log version in "
 	       "the backup.\n");
 	printf("  --query-restore-snapshot-version VERSION\n"
-	       "                 For query operations, set snapshot version for restoring a backup. Set -1 for maximum\n"
-	       "                 restorable version (default) and -3 for minimum restorable version.\n");
+	       "                 For query operations, set the snapshot version, inclusive, used to restore a backup.\n"
+	       "                 Set -1 to use the latest valid snapshot,\n"
+	       "                 Set -3 to use the oldest valid snapshot.\n");
 	printf("  -qrv --query-restore-version VERSION\n"
 	       "                 For query operations, set target version for restoring a backup. Set -1 for maximum\n"
 	       "                 restorable version (default) and -3 for minimum restorable version.\n");
@@ -2816,6 +2817,8 @@ ACTOR Future<Void> queryBackup(const char* name,
 	try {
 		state Reference<IBackupContainer> bc = openBackupContainer(name, destinationContainer, proxy, {});
 		BackupDescription desc = wait(bc->describeBackup());
+		// Use continuous log end version for the maximum restorable version for the key ranges when a restorable
+		// version doesn't exist.
 		auto [maxRestorableVersion, minRestorableVersion] = getMaxMinRestorableVersions(desc, !keyRangesFilter.empty());
 		if (restoreVersion == invalidVersion) {
 			restoreVersion = maxRestorableVersion;
@@ -2837,8 +2840,11 @@ ACTOR Future<Void> queryBackup(const char* name,
 			return Void();
 		}
 
+		state Optional<RestorableFileSet> fileSet;
 		if (snapshotVersion != invalidVersion) {
-			Optional<RestorableFileSet> fileSet = wait(bc->getRestoreSet(snapshotVersion, keyRangesFilter));
+			// When a snapshot version is specified, we will first get a restore set using the latest snapshot file to
+			// restore to the snapshot version. After snapshot version, we will only use mutation logs to restore.
+			wait(store(fileSet, bc->getRestoreSet(snapshotVersion, keyRangesFilter)));
 			if (fileSet.present()) {
 				result["snapshot_version"] = fileSet.get().targetVersion;
 				for (const auto& rangeFile : fileSet.get().ranges) {
@@ -2881,16 +2887,14 @@ ACTOR Future<Void> queryBackup(const char* name,
 				           snapshotVersion));
 				return Void();
 			}
-		}
 
-		state Optional<RestorableFileSet> fileSet;
-		if (snapshotVersion == invalidVersion) {
-			// Using the latest snapshot.
-			wait(store(fileSet, bc->getRestoreSet(restoreVersion, keyRangesFilter)));
-		} else {
-			// We only need to know all the log files from snapshotVersion to restoreVersion.
+			// We only need to know all the mutation logs from `snapshotVersion` to `restoreVersion`.
 			wait(store(fileSet, bc->getRestoreSet(restoreVersion, keyRangesFilter, /*logOnly=*/true, snapshotVersion)));
+		} else {
+			// When a snapshot version is not specified, we use the latest snapshot to restore to the `restoreVersion`.
+			wait(store(fileSet, bc->getRestoreSet(restoreVersion, keyRangesFilter)));
 		}
+
 		if (fileSet.present()) {
 			result["restore_version"] = fileSet.get().targetVersion;
 			for (const auto& rangeFile : fileSet.get().ranges) {

From de6496fc15eafe4805187c98093654a32c5ba720 Mon Sep 17 00:00:00 2001
From: Andrew Noyes <andrew.noyes@snowflake.com>
Date: Wed, 18 Jan 2023 09:40:32 -0800
Subject: [PATCH 099/468] Change PTreeImpl::insert to overwrite existing
 entries (#9138)

* Change PTreeImpl::insert to overwrite existing entries

Maintaining partial persistence of course.

We can theoretically also avoid creating a new node if the insert version of the node comparing equal to `x` is the latestVersion. There isn't a generic way to tell from the ptree though since insertAt is a concept that only exists within VersionedMap. Either way, avoiding the `contains` call and the tree rotations is already a big improvement.

The old node should only be reachable from old roots, and so it should get cleaned up as part of forgetVersions in the storage server.

* Update fdbclient/include/fdbclient/VersionedMap.h
---
 fdbclient/VersionedMap.h | 21 +++++++++++----------
 fdbclient/WriteMap.h     |  2 ++
 2 files changed, 13 insertions(+), 10 deletions(-)

diff --git a/fdbclient/VersionedMap.h b/fdbclient/VersionedMap.h
index ad5d9f32f0f..993361812e1 100644
--- a/fdbclient/VersionedMap.h
+++ b/fdbclient/VersionedMap.h
@@ -284,12 +284,17 @@ void insert(Reference<PTree<T>>& p, Version at, const T& x) {
 	if (!p) {
 		p = makeReference<PTree<T>>(x, at);
 	} else {
-		bool direction = !(x < p->data);
-		Reference<PTree<T>> child = p->child(direction, at);
-		insert(child, at, x);
-		p = update(p, direction, child, at);
-		if (p->child(direction, at)->priority > p->priority)
-			rotate(p, at, !direction);
+		int c = ::compare(x, p->data);
+		if (c == 0) {
+			p = makeReference<PTree<T>>(p->priority, x, p->left(at), p->right(at), at);
+		} else {
+			const bool direction = !(c < 0);
+			Reference<PTree<T>> child = p->child(direction, at);
+			insert(child, at, x);
+			p = update(p, direction, child, at);
+			if (p->child(direction, at)->priority > p->priority)
+				rotate(p, at, !direction);
+		}
 	}
 }
 
@@ -731,10 +736,6 @@ class VersionedMap : NonCopyable {
 	// insert() and erase() invalidate atLatest() and all iterators into it
 	void insert(const K& k, const T& t) { insert(k, t, latestVersion); }
 	void insert(const K& k, const T& t, Version insertAt) {
-		if (PTreeImpl::contains(roots.back().second, latestVersion, k))
-			PTreeImpl::remove(roots.back().second,
-			                  latestVersion,
-			                  k); // FIXME: Make PTreeImpl::insert do this automatically  (see also WriteMap.h FIXME)
 		PTreeImpl::insert(
 		    roots.back().second, latestVersion, MapPair<K, std::pair<T, Version>>(k, std::make_pair(t, insertAt)));
 	}
diff --git a/fdbclient/WriteMap.h b/fdbclient/WriteMap.h
index 33fb6aee373..b0529b82c77 100644
--- a/fdbclient/WriteMap.h
+++ b/fdbclient/WriteMap.h
@@ -134,6 +134,8 @@ struct WriteMapEntry {
 
 	int compare(ExtStringRef const& r) const { return -r.compare(key); }
 
+	int compare(WriteMapEntry const& r) const { return key.compare(r.key); }
+
 	std::string toString() const { return printable(key); }
 };
 

From 1ae5b1fd5cba8510ee9390b7f872bd9931f35d1e Mon Sep 17 00:00:00 2001
From: Andrew Noyes <andrew.noyes@snowflake.com>
Date: Thu, 26 Jan 2023 17:26:05 -0800
Subject: [PATCH 100/468] Avoid repeated search in
 VersionedMap::erase(iterator) (#9143)

---
 fdbclient/VersionedMap.h | 40 +++++++++++++++++++++++++++++++++++++---
 1 file changed, 37 insertions(+), 3 deletions(-)

diff --git a/fdbclient/VersionedMap.h b/fdbclient/VersionedMap.h
index 993361812e1..b1b8ddf2aa6 100644
--- a/fdbclient/VersionedMap.h
+++ b/fdbclient/VersionedMap.h
@@ -350,6 +350,41 @@ void removeRoot(Reference<PTree<T>>& p, Version at) {
 	}
 }
 
+// changes p to point to a PTree with finger removed. p must be the root of the
+// tree associated with finger.
+//
+// Invalidates finger.
+template <class T>
+void removeFinger(Reference<PTree<T>>& p, Version at, PTreeFinger<T> finger) {
+	ASSERT_GT(finger.size(), 0);
+	// Start at the end of the finger, remove, and propagate copies up along the
+	// search path (finger) as needed.
+	auto node = Reference<PTree<T>>::addRef(const_cast<PTree<T>*>(finger.back()));
+	auto* before = node.getPtr();
+	removeRoot(node, at);
+	for (;;) {
+		if (before == node.getPtr()) {
+			// Done propagating copies
+			return;
+		}
+		if (finger.size() == 1) {
+			// Check we passed the correct root for this finger
+			ASSERT(p.getPtr() == before);
+			// Propagate copy to root
+			p = node;
+			return;
+		}
+		finger.pop_back();
+		auto parent = Reference<PTree<T>>::addRef(const_cast<PTree<T>*>(finger.back()));
+		bool isLeftChild = parent->left(at).getPtr() == before;
+		bool isRightChild = parent->right(at).getPtr() == before;
+		ASSERT(isLeftChild || isRightChild); // Corrupt finger?
+		// Prepare for next iteration
+		before = parent.getPtr();
+		node = update(parent, isRightChild, node, at);
+	}
+}
+
 // changes p to point to a PTree with x removed
 template <class T, class X>
 void remove(Reference<PTree<T>>& p, Version at, const X& x) {
@@ -744,9 +779,8 @@ class VersionedMap : NonCopyable {
 		PTreeImpl::remove(roots.back().second, latestVersion, key);
 	}
 	void erase(iterator const& item) { // iterator must be in latest version!
-		// SOMEDAY: Optimize to use item.finger and avoid repeated search
-		K key = item.key();
-		erase(key);
+		ASSERT_EQ(item.at, latestVersion);
+		PTreeImpl::removeFinger(roots.back().second, latestVersion, item.finger);
 	}
 
 	void printDetail() { PTreeImpl::printTreeDetails(roots.back().second, 0); }

From 57375125f73cb5bbfe8633479ea1b19251f46ede Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Mon, 27 Feb 2023 17:49:55 -0800
Subject: [PATCH 101/468] Use KeyspaceSnapshotFile to filter range files

---
 fdbbackup/FileDecoder.actor.cpp     | 48 ++++++++++++++++++++++++++++-
 fdbclient/BackupContainer.h         |  3 ++
 fdbclient/FileBackupAgent.actor.cpp | 21 ++++++-------
 3 files changed, 59 insertions(+), 13 deletions(-)

diff --git a/fdbbackup/FileDecoder.actor.cpp b/fdbbackup/FileDecoder.actor.cpp
index 8b354a59f74..065de6636a2 100644
--- a/fdbbackup/FileDecoder.actor.cpp
+++ b/fdbbackup/FileDecoder.actor.cpp
@@ -38,6 +38,7 @@
 #include "fdbbackup/FileConverter.h"
 #include "fdbclient/BackupAgent.actor.h"
 #include "fdbclient/BackupContainer.h"
+#include "fdbclient/BackupContainerFileSystem.h"
 #include "fdbclient/CommitTransaction.h"
 #include "fdbclient/FDBTypes.h"
 #include "fdbclient/IKnobCollection.h"
@@ -169,6 +170,21 @@ struct DecodeParams : public ReferenceCounted<DecodeParams> {
 		return false;
 	}
 
+	bool matchFilters(const KeyRange& range) const {
+		bool match = filters.match(range);
+		if (!validate_filters) {
+			return match;
+		}
+
+		for (const auto& prefix : prefixes) {
+			if (range.intersects(prefixRange(StringRef(prefix)))) {
+				ASSERT(match);
+				return true;
+			}
+		}
+		return false;
+	}
+
 	bool matchFilters(KeyValueRef kv) const {
 		bool match = filters.match(kv);
 
@@ -814,6 +830,34 @@ ACTOR Future<Void> process_file(Reference<IBackupContainer> container,
 	return Void();
 }
 
+// Use the snapshot metadata to quickly identify relevant range files and
+// then filter by versions.
+ACTOR Future<std::vector<RangeFile>> getRangeFiles(Reference<IBackupContainer> bc, Reference<DecodeParams> params) {
+	state std::vector<KeyspaceSnapshotFile> snapshots =
+	    wait((dynamic_cast<BackupContainerFileSystem*>(bc.getPtr()))->listKeyspaceSnapshots());
+	state std::vector<RangeFile> files;
+
+	state int i = 0;
+	for (; i < snapshots.size(); i++) {
+		try {
+			std::pair<std::vector<RangeFile>, std::map<std::string, KeyRange>> results =
+			    wait((dynamic_cast<BackupContainerFileSystem*>(bc.getPtr()))->readKeyspaceSnapshot(snapshots[i]));
+			for (const auto& rangeFile : results.first) {
+				const auto& keyRange = results.second.at(rangeFile.fileName);
+				if (params->matchFilters(keyRange)) {
+					files.push_back(rangeFile);
+				}
+			}
+		} catch (Error& e) {
+			TraceEvent("ReadKeyspaceSnapshotError").error(e).detail("I", i);
+			if (e.code() != error_code_restore_missing_data) {
+				throw;
+			}
+		}
+	}
+	return getRelevantRangeFiles(files, params);
+}
+
 ACTOR Future<Void> decode_logs(Reference<DecodeParams> params) {
 	state Reference<IBackupContainer> container =
 	    IBackupContainer::openContainer(params->container_url, params->proxy, {});
@@ -843,7 +887,9 @@ ACTOR Future<Void> decode_logs(Reference<DecodeParams> params) {
 	}
 
 	if (params->decode_range) {
-		rangeFiles = getRelevantRangeFiles(listing.ranges, params);
+		// rangeFiles = getRelevantRangeFiles(filteredRangeFiles, params);
+		std::vector<RangeFile> files = wait(getRangeFiles(container, params));
+		rangeFiles = files;
 		printLogFiles("Releavant range files are: ", rangeFiles);
 	}
 
diff --git a/fdbclient/BackupContainer.h b/fdbclient/BackupContainer.h
index 825c5df0f29..f6523bd141c 100644
--- a/fdbclient/BackupContainer.h
+++ b/fdbclient/BackupContainer.h
@@ -341,6 +341,9 @@ class RangeMapFilters {
 	// Returns if the key-value pair matches any filter ranges.
 	bool match(const KeyValueRef& kv) const;
 
+	// Returns if the range intersects with any filter ranges.
+	bool match(const KeyRangeRef& range) const;
+
 private:
 	KeyRangeMap<int> rangeMap;
 };
diff --git a/fdbclient/FileBackupAgent.actor.cpp b/fdbclient/FileBackupAgent.actor.cpp
index f2ac8110ee9..9b9de3a2990 100644
--- a/fdbclient/FileBackupAgent.actor.cpp
+++ b/fdbclient/FileBackupAgent.actor.cpp
@@ -3344,18 +3344,12 @@ bool AccumulatedMutations::matchesAnyRange(const RangeMapFilters& filters) const
 
 bool RangeMapFilters::match(const MutationRef& m) const {
 	if (isSingleKeyMutation((MutationRef::Type)m.type)) {
-		auto ranges = rangeMap.intersectingRanges(singleKeyRange(m.param1));
-		for (const auto& r : ranges) {
-			if (r.cvalue() == 1) {
-				return true;
-			}
+		if (match(singleKeyRange(m.param1))) {
+			return true;
 		}
 	} else if (m.type == MutationRef::ClearRange) {
-		auto ranges = rangeMap.intersectingRanges(KeyRangeRef(m.param1, m.param2));
-		for (const auto& r : ranges) {
-			if (r.cvalue() == 1) {
-				return true;
-			}
+		if (match(KeyRangeRef(m.param1, m.param2))) {
+			return true;
 		}
 	} else {
 		ASSERT(false);
@@ -3364,13 +3358,16 @@ bool RangeMapFilters::match(const MutationRef& m) const {
 }
 
 bool RangeMapFilters::match(const KeyValueRef& kv) const {
-	auto ranges = rangeMap.intersectingRanges(singleKeyRange(kv.key));
+	return match(singleKeyRange(kv.key));
+}
+
+bool RangeMapFilters::match(const KeyRangeRef& range) const {
+	auto ranges = rangeMap.intersectingRanges(range);
 	for (const auto& r : ranges) {
 		if (r.cvalue() == 1) {
 			return true;
 		}
 	}
-
 	return false;
 }
 

From c12e0f13b91cf4927abce5fa99e6e6f5a66896bd Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Tue, 28 Feb 2023 09:24:41 -0800
Subject: [PATCH 102/468] Change mutation and KV logging to SevInfo

Set max length as well to avoid TraceEventOverflow.
---
 fdbbackup/FileDecoder.actor.cpp | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/fdbbackup/FileDecoder.actor.cpp b/fdbbackup/FileDecoder.actor.cpp
index 065de6636a2..18bb1467fc2 100644
--- a/fdbbackup/FileDecoder.actor.cpp
+++ b/fdbbackup/FileDecoder.actor.cpp
@@ -773,9 +773,9 @@ ACTOR Future<Void> process_range_file(Reference<IBackupContainer> container,
 			}
 
 			if (print) {
-				TraceEvent(SevVerbose, format("KVPair_%llu", file.version).c_str(), uid)
+				TraceEvent(format("KVPair_%llu", file.version).c_str(), uid)
 				    .detail("Version", file.version)
-				    .setMaxFieldLength(-1)
+				    .setMaxFieldLength(1000)
 				    .detail("Key", kv.key)
 				    .detail("Value", kv.value);
 				std::cout << file.version << " Key = " << printable(kv.key) << "  Value = " << printable(kv.value)
@@ -818,9 +818,9 @@ ACTOR Future<Void> process_file(Reference<IBackupContainer> container,
 				print = params->matchFilters(m);
 			}
 			if (print) {
-				TraceEvent(SevVerbose, format("Mutation_%llu_%d", vms.version, sub).c_str(), uid)
+				TraceEvent(format("Mutation_%llu_%d", vms.version, sub).c_str(), uid)
 				    .detail("Version", vms.version)
-				    .setMaxFieldLength(-1)
+				    .setMaxFieldLength(1000)
 				    .detail("M", m.toString());
 				std::cout << vms.version << "." << sub << " " << m.toString() << "\n";
 			}

From 23702f1d5a2d5b504b3278f41e0b5c570a24803e Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Tue, 28 Feb 2023 13:38:21 -0800
Subject: [PATCH 103/468] Output in HEX format for easy regex matching

---
 fdbbackup/FileDecoder.actor.cpp | 18 ++++++++++++++----
 1 file changed, 14 insertions(+), 4 deletions(-)

diff --git a/fdbbackup/FileDecoder.actor.cpp b/fdbbackup/FileDecoder.actor.cpp
index 18bb1467fc2..350551d429e 100644
--- a/fdbbackup/FileDecoder.actor.cpp
+++ b/fdbbackup/FileDecoder.actor.cpp
@@ -751,6 +751,16 @@ class DecodeRangeProgress {
 	int lfd = -1; // local file descriptor
 };
 
+// convert a StringRef to Hex string
+std::string hexStringRef(const StringRef& s) {
+	std::string result;
+	result.reserve(s.size() * 2);
+	for (int i = 0; i < s.size(); i++) {
+		result.append(format("%02x", s[i]));
+	}
+	return result;
+}
+
 ACTOR Future<Void> process_range_file(Reference<IBackupContainer> container,
                                       RangeFile file,
                                       UID uid,
@@ -776,9 +786,8 @@ ACTOR Future<Void> process_range_file(Reference<IBackupContainer> container,
 				TraceEvent(format("KVPair_%llu", file.version).c_str(), uid)
 				    .detail("Version", file.version)
 				    .setMaxFieldLength(1000)
-				    .detail("Key", kv.key)
-				    .detail("Value", kv.value);
-				std::cout << file.version << " Key = " << printable(kv.key) << "  Value = " << printable(kv.value)
+				    .detail("KV", kv);
+				std::cout << file.version << " key: " << hexStringRef(kv.key) << "  value: " << hexStringRef(kv.value)
 				          << std::endl;
 			}
 		}
@@ -822,7 +831,8 @@ ACTOR Future<Void> process_file(Reference<IBackupContainer> container,
 				    .detail("Version", vms.version)
 				    .setMaxFieldLength(1000)
 				    .detail("M", m.toString());
-				std::cout << vms.version << "." << sub << " " << m.toString() << "\n";
+				std::cout << vms.version << "." << sub << " " << typeString[(int)m.type]
+				          << " param1: " << hexStringRef(m.param1) << " param2: " << hexStringRef(m.param2) << "\n";
 			}
 		}
 	}

From 8559e5ce458bfcf2600b390b901a20b74b495520 Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Thu, 2 Mar 2023 20:51:29 -0800
Subject: [PATCH 104/468] Refactor decoder to read file as a whole once

To reduce the number of network requests.
---
 fdbbackup/FileDecoder.actor.cpp     | 140 ++++++++++++-------------
 fdbclient/BackupAgent.actor.h       |   4 +
 fdbclient/FileBackupAgent.actor.cpp | 154 ++++++++++++++--------------
 3 files changed, 147 insertions(+), 151 deletions(-)

diff --git a/fdbbackup/FileDecoder.actor.cpp b/fdbbackup/FileDecoder.actor.cpp
index 350551d429e..45ef299edc3 100644
--- a/fdbbackup/FileDecoder.actor.cpp
+++ b/fdbbackup/FileDecoder.actor.cpp
@@ -579,6 +579,12 @@ class DecodeProgress {
 	ACTOR static Future<Void> openFileImpl(DecodeProgress* self, Reference<IBackupContainer> container) {
 		Reference<IAsyncFile> fd = wait(container->readFile(self->file.fileName));
 		self->fd = fd;
+		state Standalone<StringRef> buf = makeString(self->file.fileSize);
+		int rLen = wait(self->fd->read(mutateString(buf), self->file.fileSize, 0));
+		if (rLen != self->file.fileSize) {
+			throw restore_bad_read();
+		}
+
 		if (self->save) {
 			std::string dir = self->file.fileName;
 			std::size_t found = self->file.fileName.find_last_of('/');
@@ -593,10 +599,17 @@ class DecodeProgress {
 				TraceEvent(SevError, "OpenLocalFileFailed").detail("File", self->file.fileName);
 				throw platform_error();
 			}
+			int wlen = write(self->lfd, buf.begin(), self->file.fileSize);
+			if (wlen != self->file.fileSize) {
+				TraceEvent(SevError, "WriteLocalFileFailed")
+				    .detail("File", self->file.fileName)
+				    .detail("Len", self->file.fileSize);
+				throw platform_error();
+			}
+			TraceEvent("WriteLocalFile").detail("Name", self->file.fileName).detail("Len", self->file.fileSize);
 		}
-		while (!self->eof) {
-			wait(readAndDecodeFile(self));
-		}
+
+		self->decodeFile(buf);
 		return Void();
 	}
 
@@ -608,58 +621,28 @@ class DecodeProgress {
 		}
 	}
 
-	// Reads a file block, decodes it into key/value pairs, and stores these pairs.
-	ACTOR static Future<Void> readAndDecodeFile(DecodeProgress* self) {
+	// Reads a file a file content in the buffer, decodes it into key/value pairs, and stores these pairs.
+	void decodeFile(const Standalone<StringRef>& buf) {
 		try {
-			state int64_t len = std::min<int64_t>(self->file.blockSize, self->file.fileSize - self->offset);
-			if (len == 0) {
-				self->eof = true;
-				return Void();
-			}
-
-			// Decode a file block into log_key and log_value chunks
-			state Standalone<VectorRef<KeyValueRef>> chunks =
-			    wait(fileBackup::decodeMutationLogFileBlock(self->fd, self->offset, len));
-			self->blocks.push_back(chunks);
-
-			if (self->save) {
-				ASSERT(self->lfd != -1);
-
-				// Read the chunck one more time
-				state Standalone<StringRef> buf = makeString(len);
-				int rLen = wait(self->fd->read(mutateString(buf), len, self->offset));
-				if (rLen != len)
-					throw restore_bad_read();
-
-				int wlen = write(self->lfd, buf.begin(), len);
-				if (wlen != len) {
-					TraceEvent(SevError, "WriteLocalFileFailed")
-					    .detail("File", self->file.fileName)
-					    .detail("Offset", self->offset)
-					    .detail("Len", len)
-					    .detail("Wrote", wlen);
-					throw platform_error();
+			loop {
+				int64_t len = std::min<int64_t>(file.blockSize, file.fileSize - offset);
+				if (len == 0) {
+					return;
 				}
-				TraceEvent("WriteLocalFile")
-				    .detail("Name", self->file.fileName)
-				    .detail("Len", len)
-				    .detail("Offset", self->offset);
-			}
-
-			TraceEvent("ReadFile")
-			    .detail("Name", self->file.fileName)
-			    .detail("Len", len)
-			    .detail("Offset", self->offset);
-			self->addBlockKVPairs(chunks);
-			self->offset += len;
 
-			return Void();
+				// Decode a file block into log_key and log_value chunks
+				Standalone<VectorRef<KeyValueRef>> chunks =
+				    fileBackup::decodeMutationLogFileBlock(buf.substr(offset, len));
+				blocks.push_back(chunks);
+				addBlockKVPairs(chunks);
+				offset += len;
+			}
 		} catch (Error& e) {
 			TraceEvent(SevWarn, "CorruptLogFileBlock")
 			    .error(e)
-			    .detail("Filename", self->file.fileName)
-			    .detail("BlockOffset", self->offset)
-			    .detail("BlockLen", self->file.blockSize);
+			    .detail("Filename", file.fileName)
+			    .detail("BlockOffset", offset)
+			    .detail("BlockLen", file.blockSize);
 			throw;
 		}
 	}
@@ -688,8 +671,16 @@ class DecodeRangeProgress {
 	Future<Void> openFile(Reference<IBackupContainer> container) { return openFileImpl(this, container); }
 
 	ACTOR static Future<Void> openFileImpl(DecodeRangeProgress* self, Reference<IBackupContainer> container) {
+		TraceEvent("ReadFile").detail("Name", self->file.fileName).detail("Len", self->file.fileSize);
+
 		Reference<IAsyncFile> fd = wait(container->readFile(self->file.fileName));
 		self->fd = fd;
+		state Standalone<StringRef> buf = makeString(self->file.fileSize);
+		int rLen = wait(self->fd->read(mutateString(buf), self->file.fileSize, 0));
+		if (rLen != self->file.fileSize) {
+			throw restore_bad_read();
+		}
+
 		if (self->save) {
 			std::string dir = self->file.fileName;
 			std::size_t found = self->file.fileName.find_last_of('/');
@@ -705,49 +696,48 @@ class DecodeRangeProgress {
 				TraceEvent(SevError, "OpenLocalFileFailed").detail("File", self->file.fileName);
 				throw platform_error();
 			}
+			int wlen = write(self->lfd, buf.begin(), self->file.fileSize);
+			if (wlen != self->file.fileSize) {
+				TraceEvent(SevError, "WriteLocalFileFailed")
+				    .detail("File", self->file.fileName)
+				    .detail("Len", self->file.fileSize);
+				throw platform_error();
+			}
+			TraceEvent("WriteLocalFile").detail("Name", self->file.fileName).detail("Len", self->file.fileSize);
 		}
-		while (!self->eof) {
-			wait(readAndDecodeFile(self));
-		}
+
+		self->decodeFile(buf);
 		return Void();
 	}
 
-	// Reads a file block, decodes it into key/value pairs, and stores these pairs.
-	ACTOR static Future<Void> readAndDecodeFile(DecodeRangeProgress* self) {
+	// Reads a file content in the buffer, decodes it into key/value pairs, and stores these pairs.
+	void decodeFile(const Standalone<StringRef>& buf) {
 		try {
-			state int64_t len = std::min<int64_t>(self->file.blockSize, self->file.fileSize - self->offset);
-			if (len == 0) {
-				self->eof = true;
-				return Void();
-			}
-
-			state Standalone<VectorRef<KeyValueRef>> chunks =
-			    wait(fileBackup::decodeRangeFileBlock(self->fd, self->offset, len));
-			self->blocks.push_back(chunks);
-
-			TraceEvent("ReadFile")
-			    .detail("Name", self->file.fileName)
-			    .detail("Len", len)
-			    .detail("Offset", self->offset);
-			self->offset += len;
+			loop {
+				// process one block at a time
+				int64_t len = std::min<int64_t>(file.blockSize, file.fileSize - offset);
+				if (len == 0) {
+					return;
+				}
 
+				Standalone<VectorRef<KeyValueRef>> chunks = fileBackup::decodeRangeFileBlock(buf.substr(offset, len));
+				blocks.push_back(chunks);
+				offset += len;
+			}
 		} catch (Error& e) {
 			TraceEvent(SevWarn, "CorruptRangeFileBlock")
 			    .error(e)
-			    .detail("Filename", self->file.fileName)
-			    .detail("BlockOffset", self->offset)
-			    .detail("BlockLen", self->file.blockSize);
+			    .detail("Filename", file.fileName)
+			    .detail("BlockOffset", offset)
+			    .detail("BlockLen", file.blockSize);
 			throw;
 		}
-
-		return Void();
 	}
 
 	RangeFile file;
 	Reference<IAsyncFile> fd;
 	int64_t offset = 0;
 	bool save = false;
-	bool eof = false;
 	int lfd = -1; // local file descriptor
 };
 
diff --git a/fdbclient/BackupAgent.actor.h b/fdbclient/BackupAgent.actor.h
index 44fa7921d3b..43cd8ee811d 100644
--- a/fdbclient/BackupAgent.actor.h
+++ b/fdbclient/BackupAgent.actor.h
@@ -1002,10 +1002,14 @@ struct StringRefReader {
 };
 
 namespace fileBackup {
+Standalone<VectorRef<KeyValueRef>> decodeRangeFileBlock(const Standalone<StringRef>& buf);
+
 ACTOR Future<Standalone<VectorRef<KeyValueRef>>> decodeRangeFileBlock(Reference<IAsyncFile> file,
                                                                       int64_t offset,
                                                                       int len);
 
+Standalone<VectorRef<KeyValueRef>> decodeMutationLogFileBlock(const Standalone<StringRef>& buf);
+
 // Reads a mutation log block from file and parses into batch mutation blocks for further parsing.
 ACTOR Future<Standalone<VectorRef<KeyValueRef>>> decodeMutationLogFileBlock(Reference<IAsyncFile> file,
                                                                             int64_t offset,
diff --git a/fdbclient/FileBackupAgent.actor.cpp b/fdbclient/FileBackupAgent.actor.cpp
index 9b9de3a2990..482baaac604 100644
--- a/fdbclient/FileBackupAgent.actor.cpp
+++ b/fdbclient/FileBackupAgent.actor.cpp
@@ -584,66 +584,67 @@ struct RangeFileWriter {
 	Key lastValue;
 };
 
-ACTOR Future<Standalone<VectorRef<KeyValueRef>>> decodeRangeFileBlock(Reference<IAsyncFile> file,
-                                                                      int64_t offset,
-                                                                      int len) {
-	state Standalone<StringRef> buf = makeString(len);
-	int rLen = wait(uncancellable(holdWhile(buf, file->read(mutateString(buf), len, offset))));
-	if (rLen != len)
-		throw restore_bad_read();
+Standalone<VectorRef<KeyValueRef>> decodeRangeFileBlock(const Standalone<StringRef>& buf) {
+	Standalone<VectorRef<KeyValueRef>> results({}, buf.arena());
+	StringRefReader reader(buf, restore_corrupted_data());
 
-	simulateBlobFailure();
+	// Read header, currently only decoding BACKUP_AGENT_SNAPSHOT_FILE_VERSION
+	if (reader.consume<int32_t>() != BACKUP_AGENT_SNAPSHOT_FILE_VERSION)
+		throw restore_unsupported_file_version();
 
-	Standalone<VectorRef<KeyValueRef>> results({}, buf.arena());
-	state StringRefReader reader(buf, restore_corrupted_data());
+	// Read begin key, if this fails then block was invalid.
+	uint32_t kLen = reader.consumeNetworkUInt32();
+	const uint8_t* k = reader.consume(kLen);
+	results.push_back(results.arena(), KeyValueRef(KeyRef(k, kLen), ValueRef()));
 
-	try {
-		// Read header, currently only decoding BACKUP_AGENT_SNAPSHOT_FILE_VERSION
-		if (reader.consume<int32_t>() != BACKUP_AGENT_SNAPSHOT_FILE_VERSION)
-			throw restore_unsupported_file_version();
+	// Read kv pairs and end key
+	while (1) {
+		// Read a key.
+		kLen = reader.consumeNetworkUInt32();
+		k = reader.consume(kLen);
 
-		// Read begin key, if this fails then block was invalid.
-		uint32_t kLen = reader.consumeNetworkUInt32();
-		const uint8_t* k = reader.consume(kLen);
-		results.push_back(results.arena(), KeyValueRef(KeyRef(k, kLen), ValueRef()));
+		// If eof reached or first value len byte is 0xFF then a valid block end was reached.
+		if (reader.eof() || *reader.rptr == 0xFF) {
+			results.push_back(results.arena(), KeyValueRef(KeyRef(k, kLen), ValueRef()));
+			break;
+		}
 
-		// Read kv pairs and end key
-		while (1) {
-			// Read a key.
-			kLen = reader.consumeNetworkUInt32();
-			k = reader.consume(kLen);
+		// Read a value, which must exist or the block is invalid
+		uint32_t vLen = reader.consumeNetworkUInt32();
+		const uint8_t* v = reader.consume(vLen);
+		results.push_back(results.arena(), KeyValueRef(KeyRef(k, kLen), ValueRef(v, vLen)));
 
-			// If eof reached or first value len byte is 0xFF then a valid block end was reached.
-			if (reader.eof() || *reader.rptr == 0xFF) {
-				results.push_back(results.arena(), KeyValueRef(KeyRef(k, kLen), ValueRef()));
-				break;
-			}
+		// If eof reached or first byte of next key len is 0xFF then a valid block end was reached.
+		if (reader.eof() || *reader.rptr == 0xFF)
+			break;
+	}
 
-			// Read a value, which must exist or the block is invalid
-			uint32_t vLen = reader.consumeNetworkUInt32();
-			const uint8_t* v = reader.consume(vLen);
-			results.push_back(results.arena(), KeyValueRef(KeyRef(k, kLen), ValueRef(v, vLen)));
+	// Make sure any remaining bytes in the block are 0xFF
+	for (auto b : reader.remainder())
+		if (b != 0xFF)
+			throw restore_corrupted_data_padding();
 
-			// If eof reached or first byte of next key len is 0xFF then a valid block end was reached.
-			if (reader.eof() || *reader.rptr == 0xFF)
-				break;
-		}
+	return results;
+}
 
-		// Make sure any remaining bytes in the block are 0xFF
-		for (auto b : reader.remainder())
-			if (b != 0xFF)
-				throw restore_corrupted_data_padding();
+ACTOR Future<Standalone<VectorRef<KeyValueRef>>> decodeRangeFileBlock(Reference<IAsyncFile> file,
+                                                                      int64_t offset,
+                                                                      int len) {
+	state Standalone<StringRef> buf = makeString(len);
+	int rLen = wait(uncancellable(holdWhile(buf, file->read(mutateString(buf), len, offset))));
+	if (rLen != len)
+		throw restore_bad_read();
 
-		return results;
+	simulateBlobFailure();
 
+	try {
+		return decodeRangeFileBlock(buf);
 	} catch (Error& e) {
 		TraceEvent(SevWarn, "FileRestoreDecodeRangeFileBlockFailed")
 		    .error(e)
 		    .detail("Filename", file->getFilename())
 		    .detail("BlockOffset", offset)
-		    .detail("BlockLen", len)
-		    .detail("ErrorRelativeOffset", reader.rptr - buf.begin())
-		    .detail("ErrorAbsoluteOffset", reader.rptr - buf.begin() + offset);
+		    .detail("BlockLen", len);
 		throw;
 	}
 }
@@ -692,6 +693,37 @@ struct LogFileWriter {
 	int64_t blockEnd;
 };
 
+Standalone<VectorRef<KeyValueRef>> decodeMutationLogFileBlock(const Standalone<StringRef>& buf) {
+	Standalone<VectorRef<KeyValueRef>> results({}, buf.arena());
+	StringRefReader reader(buf, restore_corrupted_data());
+
+	// Read header, currently only decoding version BACKUP_AGENT_MLOG_VERSION
+	if (reader.consume<int32_t>() != BACKUP_AGENT_MLOG_VERSION)
+		throw restore_unsupported_file_version();
+
+	// Read k/v pairs.  Block ends either at end of last value exactly or with 0xFF as first key len byte.
+	while (1) {
+		// If eof reached or first key len bytes is 0xFF then end of block was reached.
+		if (reader.eof() || *reader.rptr == 0xFF)
+			break;
+
+		// Read key and value.  If anything throws then there is a problem.
+		uint32_t kLen = reader.consumeNetworkUInt32();
+		const uint8_t* k = reader.consume(kLen);
+		uint32_t vLen = reader.consumeNetworkUInt32();
+		const uint8_t* v = reader.consume(vLen);
+
+		results.push_back(results.arena(), KeyValueRef(KeyRef(k, kLen), ValueRef(v, vLen)));
+	}
+
+	// Make sure any remaining bytes in the block are 0xFF
+	for (auto b : reader.remainder())
+		if (b != 0xFF)
+			throw restore_corrupted_data_padding();
+
+	return results;
+}
+
 ACTOR Future<Standalone<VectorRef<KeyValueRef>>> decodeMutationLogFileBlock(Reference<IAsyncFile> file,
                                                                             int64_t offset,
                                                                             int len) {
@@ -700,44 +732,14 @@ ACTOR Future<Standalone<VectorRef<KeyValueRef>>> decodeMutationLogFileBlock(Refe
 	if (rLen != len)
 		throw restore_bad_read();
 
-	Standalone<VectorRef<KeyValueRef>> results({}, buf.arena());
-	state StringRefReader reader(buf, restore_corrupted_data());
-
 	try {
-		// Read header, currently only decoding version BACKUP_AGENT_MLOG_VERSION
-		if (reader.consume<int32_t>() != BACKUP_AGENT_MLOG_VERSION)
-			throw restore_unsupported_file_version();
-
-		// Read k/v pairs.  Block ends either at end of last value exactly or with 0xFF as first key len byte.
-		while (1) {
-			// If eof reached or first key len bytes is 0xFF then end of block was reached.
-			if (reader.eof() || *reader.rptr == 0xFF)
-				break;
-
-			// Read key and value.  If anything throws then there is a problem.
-			uint32_t kLen = reader.consumeNetworkUInt32();
-			const uint8_t* k = reader.consume(kLen);
-			uint32_t vLen = reader.consumeNetworkUInt32();
-			const uint8_t* v = reader.consume(vLen);
-
-			results.push_back(results.arena(), KeyValueRef(KeyRef(k, kLen), ValueRef(v, vLen)));
-		}
-
-		// Make sure any remaining bytes in the block are 0xFF
-		for (auto b : reader.remainder())
-			if (b != 0xFF)
-				throw restore_corrupted_data_padding();
-
-		return results;
-
+		return decodeMutationLogFileBlock(buf);
 	} catch (Error& e) {
 		TraceEvent(SevWarn, "FileRestoreCorruptLogFileBlock")
 		    .error(e)
 		    .detail("Filename", file->getFilename())
 		    .detail("BlockOffset", offset)
-		    .detail("BlockLen", len)
-		    .detail("ErrorRelativeOffset", reader.rptr - buf.begin())
-		    .detail("ErrorAbsoluteOffset", reader.rptr - buf.begin() + offset);
+		    .detail("BlockLen", len);
 		throw;
 	}
 }

From b59b4abf27b1f91df5aa9935aad338318ef3da91 Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Fri, 3 Mar 2023 09:01:43 -0800
Subject: [PATCH 105/468] Add more trace events

---
 fdbbackup/FileDecoder.actor.cpp | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/fdbbackup/FileDecoder.actor.cpp b/fdbbackup/FileDecoder.actor.cpp
index 45ef299edc3..b1e60c1aa88 100644
--- a/fdbbackup/FileDecoder.actor.cpp
+++ b/fdbbackup/FileDecoder.actor.cpp
@@ -783,6 +783,7 @@ ACTOR Future<Void> process_range_file(Reference<IBackupContainer> container,
 		}
 	}
 
+	TraceEvent("ProcessRangeFileDone", uid).detail("File", file.fileName);
 	return Void();
 }
 
@@ -893,6 +894,8 @@ ACTOR Future<Void> decode_logs(Reference<DecodeParams> params) {
 		printLogFiles("Releavant range files are: ", rangeFiles);
 	}
 
+	TraceEvent("TotalFiles", uid).detail("LogFiles", logFiles.size()).detail("RangeFiles", rangeFiles.size());
+
 	if (params->list_only)
 		return Void();
 

From 02eb4fc1edc5e0e35023c4b14fa16c657a3f431b Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyu_zhou@apple.com>
Date: Thu, 9 Mar 2023 17:39:44 -0800
Subject: [PATCH 106/468] Allow log router to detect slow peeks and to switch
 DC for peeking [release-7.1] (#9640)

* Add DcLag tests and workload

* Add disableSimSpeedup to clog network longer

* Ignore the DcLag test

* Refactor LogRouter's pullAsyncData

* Switch DC if log router peek becomes stuck

Trying to a different DC if this happens.

* Enable DcLag test

* Require at least 2 regions and having satellites

* Simplify DcLag code

* Limit connection failures to be within tests

In particular, disable connection failures when initializing the database
during the startup phase, i.e., before running with test specs.

* Revert disableSimSpeedup

* Fix conflicts after cherrypick

* More fixes after cherrypick

* Refactor to address comments

* Use a constant for connectionFailuresDisableDuration

* Fix ClogTlog workload valgrind error

* Address comments

* Reduce running time for DcLag

The switch can happen quicker than the workload detection time, so need to
adjust detection time lower than LOG_ROUTER_PEEK_SWITCH_DC_TIME.
---
 fdbclient/ServerKnobs.cpp                   |   1 +
 fdbclient/ServerKnobs.h                     |   1 +
 fdbrpc/CMakeLists.txt                       |   1 -
 fdbrpc/genericactors.actor.cpp              |  35 ----
 fdbrpc/genericactors.actor.h                |   2 -
 fdbrpc/sim2.actor.cpp                       |  25 ++-
 fdbrpc/simulator.h                          |   8 +
 fdbserver/BackupWorker.actor.cpp            |   3 +-
 fdbserver/CMakeLists.txt                    |   1 +
 fdbserver/ClusterRecovery.actor.cpp         |   4 +-
 fdbserver/LogRouter.actor.cpp               |  87 +++++----
 fdbserver/LogSystem.h                       |   2 +-
 fdbserver/QuietDatabase.h                   |   5 +
 fdbserver/TagPartitionedLogSystem.actor.cpp |  14 +-
 fdbserver/TagPartitionedLogSystem.actor.h   |   2 +-
 fdbserver/tester.actor.cpp                  |  35 +++-
 fdbserver/workloads/ClogTlog.actor.cpp      |  12 +-
 fdbserver/workloads/DcLag.actor.cpp         | 186 ++++++++++++++++++++
 tests/CMakeLists.txt                        |   1 +
 tests/rare/DcLag.toml                       |  19 ++
 20 files changed, 341 insertions(+), 103 deletions(-)
 delete mode 100644 fdbrpc/genericactors.actor.cpp
 create mode 100644 fdbserver/workloads/DcLag.actor.cpp
 create mode 100644 tests/rare/DcLag.toml

diff --git a/fdbclient/ServerKnobs.cpp b/fdbclient/ServerKnobs.cpp
index eaee631267b..c9358ae66f5 100644
--- a/fdbclient/ServerKnobs.cpp
+++ b/fdbclient/ServerKnobs.cpp
@@ -82,6 +82,7 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi
 	init( DESIRED_GET_MORE_DELAY,                              0.005 );
 	init( CONCURRENT_LOG_ROUTER_READS,                             5 ); if( randomize && BUGGIFY ) CONCURRENT_LOG_ROUTER_READS = 1;
 	init( LOG_ROUTER_PEEK_FROM_SATELLITES_PREFERRED,               1 ); if( randomize && BUGGIFY ) LOG_ROUTER_PEEK_FROM_SATELLITES_PREFERRED = 0;
+	init( LOG_ROUTER_PEEK_SWITCH_DC_TIME,                       60.0 );
 	init( DISK_QUEUE_ADAPTER_MIN_SWITCH_TIME,                    1.0 );
 	init( DISK_QUEUE_ADAPTER_MAX_SWITCH_TIME,                    5.0 );
 	init( TLOG_SPILL_REFERENCE_MAX_PEEK_MEMORY_BYTES,            2e9 ); if ( randomize && BUGGIFY ) TLOG_SPILL_REFERENCE_MAX_PEEK_MEMORY_BYTES = 2e6;
diff --git a/fdbclient/ServerKnobs.h b/fdbclient/ServerKnobs.h
index 0f967af8577..33fdcaf4f37 100644
--- a/fdbclient/ServerKnobs.h
+++ b/fdbclient/ServerKnobs.h
@@ -86,6 +86,7 @@ class ServerKnobs : public KnobsImpl<ServerKnobs> {
 	double DESIRED_GET_MORE_DELAY;
 	int CONCURRENT_LOG_ROUTER_READS;
 	int LOG_ROUTER_PEEK_FROM_SATELLITES_PREFERRED; // 0==peek from primary, non-zero==peek from satellites
+	double LOG_ROUTER_PEEK_SWITCH_DC_TIME;
 	double DISK_QUEUE_ADAPTER_MIN_SWITCH_TIME;
 	double DISK_QUEUE_ADAPTER_MAX_SWITCH_TIME;
 	int64_t TLOG_SPILL_REFERENCE_MAX_PEEK_MEMORY_BYTES;
diff --git a/fdbrpc/CMakeLists.txt b/fdbrpc/CMakeLists.txt
index 2da35b99086..a65169dc9ad 100644
--- a/fdbrpc/CMakeLists.txt
+++ b/fdbrpc/CMakeLists.txt
@@ -13,7 +13,6 @@ set(FDBRPC_SRCS
   FlowProcess.actor.h
   FlowTransport.actor.cpp
   genericactors.actor.h
-  genericactors.actor.cpp
   HealthMonitor.actor.cpp
   HTTP.actor.cpp
   IAsyncFile.actor.cpp
diff --git a/fdbrpc/genericactors.actor.cpp b/fdbrpc/genericactors.actor.cpp
deleted file mode 100644
index 8eb9ba49b1c..00000000000
--- a/fdbrpc/genericactors.actor.cpp
+++ /dev/null
@@ -1,35 +0,0 @@
-/*
- * genericactors.actor.cpp
- *
- * This source file is part of the FoundationDB open source project
- *
- * Copyright 2013-2022 Apple Inc. and the FoundationDB project authors
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include "flow/flow.h"
-#include "fdbrpc/genericactors.actor.h" // Gets genericactors.actor.g.h indirectly
-#include "flow/network.h"
-#include "fdbrpc/simulator.h"
-#include "flow/actorcompiler.h"
-
-ACTOR Future<Void> disableConnectionFailuresAfter(double time, std::string context) {
-	if (g_network->isSimulated()) {
-		wait(delayUntil(time));
-		g_simulator.connectionFailuresDisableDuration = 1e6;
-		g_simulator.speedUpSimulation = true;
-		TraceEvent(SevWarnAlways, ("DisableConnectionFailures_" + context).c_str());
-	}
-	return Void();
-}
diff --git a/fdbrpc/genericactors.actor.h b/fdbrpc/genericactors.actor.h
index 80f51d3a14c..0b6848e23df 100644
--- a/fdbrpc/genericactors.actor.h
+++ b/fdbrpc/genericactors.actor.h
@@ -418,8 +418,6 @@ Future<X> reportEndpointFailure(Future<X> value, Endpoint endpoint) {
 	}
 }
 
-Future<Void> disableConnectionFailuresAfter(double const& time, std::string const& context);
-
 #include "flow/unactorcompiler.h"
 
 #endif
diff --git a/fdbrpc/sim2.actor.cpp b/fdbrpc/sim2.actor.cpp
index 4abcba8d2d4..e63a2568286 100644
--- a/fdbrpc/sim2.actor.cpp
+++ b/fdbrpc/sim2.actor.cpp
@@ -42,7 +42,6 @@
 #include "fdbrpc/AsyncFileChaos.actor.h"
 #include "flow/crc32c.h"
 #include "fdbrpc/TraceFileIO.h"
-#include "flow/FaultInjection.h"
 #include "flow/flow.h"
 #include "flow/genericactors.actor.h"
 #include "flow/network.h"
@@ -51,6 +50,7 @@
 #include "fdbrpc/Replication.h"
 #include "fdbrpc/ReplicationUtils.h"
 #include "fdbrpc/AsyncFileWriteChecker.h"
+#include "fdbrpc/genericactors.actor.h"
 #include "flow/FaultInjection.h"
 #include "flow/actorcompiler.h" // This must be the last #include.
 
@@ -522,9 +522,7 @@ class SimpleFile : public IAsyncFile, public ReferenceCounted<SimpleFile> {
 		}
 
 		if (openCount == 2000) {
-			TraceEvent(SevWarnAlways, "DisableConnectionFailures_TooManyFiles").log();
-			g_simulator.speedUpSimulation = true;
-			g_simulator.connectionFailuresDisableDuration = 1e6;
+			disableConnectionFailures("TooManyFiles");
 		}
 
 		// Filesystems on average these days seem to start to have limits of around 255 characters for a
@@ -2463,7 +2461,8 @@ Future<Reference<IUDPSocket>> Sim2::createUDPSocket(bool isV6) {
 void startNewSimulator(bool printSimTime) {
 	ASSERT(!g_network);
 	g_network = g_pSimulator = new Sim2(printSimTime);
-	g_simulator.connectionFailuresDisableDuration = deterministicRandom()->random01() < 0.5 ? 0 : 1e6;
+	g_simulator.connectionFailuresDisableDuration =
+	    deterministicRandom()->coinflip() ? 0 : DISABLE_CONNECTION_FAILURE_FOREVER;
 }
 
 ACTOR void doReboot(ISimulator::ProcessInfo* p, ISimulator::KillType kt) {
@@ -2548,6 +2547,22 @@ Future<Void> waitUntilDiskReady(Reference<DiskParameters> diskParameters, int64_
 	return delayUntil(diskParameters->nextOperation + randomLatency);
 }
 
+void enableConnectionFailures(std::string const& context) {
+	if (g_network->isSimulated()) {
+		g_simulator.connectionFailuresDisableDuration = 0;
+		g_simulator.speedUpSimulation = false;
+		TraceEvent(SevWarnAlways, ("EnableConnectionFailures_" + context).c_str());
+	}
+}
+
+void disableConnectionFailures(std::string const& context) {
+	if (g_network->isSimulated()) {
+		g_simulator.connectionFailuresDisableDuration = DISABLE_CONNECTION_FAILURE_FOREVER;
+		g_simulator.speedUpSimulation = true;
+		TraceEvent(SevWarnAlways, ("DisableConnectionFailures_" + context).c_str());
+	}
+}
+
 #if defined(_WIN32)
 
 /* Opening with FILE_SHARE_DELETE lets simulation actually work on windows - previously renames were always failing.
diff --git a/fdbrpc/simulator.h b/fdbrpc/simulator.h
index 557e6fcdf7b..deceec32ebc 100644
--- a/fdbrpc/simulator.h
+++ b/fdbrpc/simulator.h
@@ -36,6 +36,8 @@
 
 enum ClogMode { ClogDefault, ClogAll, ClogSend, ClogReceive };
 
+constexpr double DISABLE_CONNECTION_FAILURE_FOREVER = 1e6;
+
 class ISimulator : public INetwork {
 public:
 	// Order matters!
@@ -511,6 +513,12 @@ struct DiskParameters : ReferenceCounted<DiskParameters> {
 // Simulates delays for performing operations on disk
 extern Future<Void> waitUntilDiskReady(Reference<DiskParameters> parameters, int64_t size, bool sync = false);
 
+// Enables connection failures, i.e., clogging, in simulation
+void enableConnectionFailures(std::string const& context);
+
+// Disables connection failures, i.e., clogging, in simulation
+void disableConnectionFailures(std::string const& context);
+
 class Sim2FileSystem : public IAsyncFileSystem {
 public:
 	// Opens a file for asynchronous I/O
diff --git a/fdbserver/BackupWorker.actor.cpp b/fdbserver/BackupWorker.actor.cpp
index 476e5473f4d..1da32a52ab4 100644
--- a/fdbserver/BackupWorker.actor.cpp
+++ b/fdbserver/BackupWorker.actor.cpp
@@ -914,7 +914,8 @@ ACTOR Future<Void> pullAsyncData(BackupData* self) {
 			}
 			when(wait(logSystemChange)) {
 				if (self->logSystem.get()) {
-					r = self->logSystem.get()->peekLogRouter(self->myId, tagAt, self->tag);
+					r = self->logSystem.get()->peekLogRouter(
+					    self->myId, tagAt, self->tag, SERVER_KNOBS->LOG_ROUTER_PEEK_FROM_SATELLITES_PREFERRED);
 				} else {
 					r = Reference<ILogSystem::IPeekCursor>();
 				}
diff --git a/fdbserver/CMakeLists.txt b/fdbserver/CMakeLists.txt
index f9c2735bb52..fdbe48ec414 100644
--- a/fdbserver/CMakeLists.txt
+++ b/fdbserver/CMakeLists.txt
@@ -209,6 +209,7 @@ set(FDBSERVER_SRCS
   workloads/Cycle.actor.cpp
   workloads/DataDistributionMetrics.actor.cpp
   workloads/DataLossRecovery.actor.cpp
+  workloads/DcLag.actor.cpp
   workloads/DDBalance.actor.cpp
   workloads/DDMetrics.actor.cpp
   workloads/DDMetricsExclude.actor.cpp
diff --git a/fdbserver/ClusterRecovery.actor.cpp b/fdbserver/ClusterRecovery.actor.cpp
index d185cd1ab38..027d4ad2c07 100644
--- a/fdbserver/ClusterRecovery.actor.cpp
+++ b/fdbserver/ClusterRecovery.actor.cpp
@@ -1452,9 +1452,7 @@ ACTOR Future<Void> clusterRecoveryCore(Reference<ClusterRecoveryData> self) {
 			           (self->cstate.myDBState.oldTLogData.size() - CLIENT_KNOBS->RECOVERY_DELAY_START_GENERATION)));
 		}
 		if (g_network->isSimulated() && self->cstate.myDBState.oldTLogData.size() > CLIENT_KNOBS->MAX_GENERATIONS_SIM) {
-			g_simulator.connectionFailuresDisableDuration = 1e6;
-			g_simulator.speedUpSimulation = true;
-			TraceEvent(SevWarnAlways, "DisableConnectionFailures_TooManyGenerations").log();
+			disableConnectionFailures("TooManyGenerations");
 		}
 	}
 
diff --git a/fdbserver/LogRouter.actor.cpp b/fdbserver/LogRouter.actor.cpp
index 31b83de8969..e0a8e3c36f4 100644
--- a/fdbserver/LogRouter.actor.cpp
+++ b/fdbserver/LogRouter.actor.cpp
@@ -84,6 +84,7 @@ struct LogRouterData {
 
 	const UID dbgid;
 	Reference<AsyncVar<Reference<ILogSystem>>> logSystem;
+	Future<Void> logSystemChanged = Void();
 	Optional<UID> primaryPeekLocation;
 	NotifiedVersion version; // The largest version at which the log router has peeked mutations
 	                         // from satellite tLog or primary tLogs.
@@ -304,48 +305,68 @@ ACTOR Future<Void> waitForVersion(LogRouterData* self, Version ver) {
 	return Void();
 }
 
+ACTOR Future<Reference<ILogSystem::IPeekCursor>> getPeekCursorData(LogRouterData* self,
+                                                                   Reference<ILogSystem::IPeekCursor> r,
+                                                                   Version startVersion) {
+	state Reference<ILogSystem::IPeekCursor> result = r;
+	state bool useSatellite = SERVER_KNOBS->LOG_ROUTER_PEEK_FROM_SATELLITES_PREFERRED;
+	loop {
+		Future<Void> getMoreF = Never();
+		if (result) {
+			getMoreF = result->getMore(TaskPriority::TLogCommit);
+			++self->getMoreCount;
+			if (!getMoreF.isReady()) {
+				++self->getMoreBlockedCount;
+			}
+		}
+		state double startTime = now();
+		choose {
+			when(wait(getMoreF)) {
+				double peekTime = now() - startTime;
+				self->peekLatencyDist->sampleSeconds(peekTime);
+				self->getMoreTime += peekTime;
+				self->maxGetMoreTime = std::max(self->maxGetMoreTime, peekTime);
+				return result;
+			}
+			when(wait(self->logSystemChanged)) {
+				if (self->logSystem->get()) {
+					result =
+					    self->logSystem->get()->peekLogRouter(self->dbgid, startVersion, self->routerTag, useSatellite);
+					self->primaryPeekLocation = result->getPrimaryPeekLocation();
+					TraceEvent("LogRouterPeekLocation", self->dbgid)
+					    .detail("LogID", result->getPrimaryPeekLocation())
+					    .trackLatest(self->eventCacheHolder->trackingKey);
+				} else {
+					result = Reference<ILogSystem::IPeekCursor>();
+				}
+				self->logSystemChanged = self->logSystem->onChange();
+			}
+			when(wait(result ? delay(SERVER_KNOBS->LOG_ROUTER_PEEK_SWITCH_DC_TIME) : Never())) {
+				// Peek has become stuck for a while, trying switching between primary DC and satellite
+				TraceEvent(SevWarnAlways, "LogRouterSlowPeek", self->dbgid).detail("NextTrySatellite", !useSatellite);
+				useSatellite = !useSatellite;
+				result =
+				    self->logSystem->get()->peekLogRouter(self->dbgid, startVersion, self->routerTag, useSatellite);
+				self->primaryPeekLocation = result->getPrimaryPeekLocation();
+				TraceEvent("LogRouterPeekLocation", self->dbgid)
+				    .detail("LogID", result->getPrimaryPeekLocation())
+				    .trackLatest(self->eventCacheHolder->trackingKey);
+			}
+		}
+	}
+}
+
 // Log router (LR) asynchronously pull data from satellite tLogs (preferred) or primary tLogs at tag (self->routerTag)
 // for the version range from the LR's current version (exclusive) to its epoch's end version or recovery version.
 ACTOR Future<Void> pullAsyncData(LogRouterData* self) {
-	state Future<Void> dbInfoChange = Void();
 	state Reference<ILogSystem::IPeekCursor> r;
 	state Version tagAt = self->version.get() + 1;
 	state Version lastVer = 0;
 	state std::vector<int> tags; // an optimization to avoid reallocating vector memory in every loop
 
 	loop {
-		loop {
-			Future<Void> getMoreF = Never();
-			if (r) {
-				getMoreF = r->getMore(TaskPriority::TLogCommit);
-				++self->getMoreCount;
-				if (!getMoreF.isReady()) {
-					++self->getMoreBlockedCount;
-				}
-			}
-			state double startTime = now();
-			choose {
-				when(wait(getMoreF)) {
-					double peekTime = now() - startTime;
-					self->peekLatencyDist->sampleSeconds(peekTime);
-					self->getMoreTime += peekTime;
-					self->maxGetMoreTime = std::max(self->maxGetMoreTime, peekTime);
-					break;
-				}
-				when(wait(dbInfoChange)) { // FIXME: does this actually happen?
-					if (self->logSystem->get()) {
-						r = self->logSystem->get()->peekLogRouter(self->dbgid, tagAt, self->routerTag);
-						self->primaryPeekLocation = r->getPrimaryPeekLocation();
-						TraceEvent("LogRouterPeekLocation", self->dbgid)
-						    .detail("LogID", r->getPrimaryPeekLocation())
-						    .trackLatest(self->eventCacheHolder->trackingKey);
-					} else {
-						r = Reference<ILogSystem::IPeekCursor>();
-					}
-					dbInfoChange = self->logSystem->onChange();
-				}
-			}
-		}
+		Reference<ILogSystem::IPeekCursor> _r = wait(getPeekCursorData(self, r, tagAt));
+		r = _r;
 
 		self->minKnownCommittedVersion = std::max(self->minKnownCommittedVersion, r->getMinKnownCommittedVersion());
 
diff --git a/fdbserver/LogSystem.h b/fdbserver/LogSystem.h
index 6178a6ad934..43783610c71 100644
--- a/fdbserver/LogSystem.h
+++ b/fdbserver/LogSystem.h
@@ -554,7 +554,7 @@ struct ILogSystem {
 	// Same contract as peek(), but blocks until the preferred log server(s) for the given tag are available (and is
 	// correspondingly less expensive)
 
-	virtual Reference<IPeekCursor> peekLogRouter(UID dbgid, Version begin, Tag tag) = 0;
+	virtual Reference<IPeekCursor> peekLogRouter(UID dbgid, Version begin, Tag tag, bool useSatellite) = 0;
 	// Same contract as peek(), but can only peek from the logs elected in the same generation.
 	// If the preferred log server is down, a different log from the same generation will merge results locally before
 	// sending them to the log router.
diff --git a/fdbserver/QuietDatabase.h b/fdbserver/QuietDatabase.h
index 5265608b4c5..3dade8d1910 100644
--- a/fdbserver/QuietDatabase.h
+++ b/fdbserver/QuietDatabase.h
@@ -47,6 +47,11 @@ Future<Void> repairDeadDatacenter(Database const& cx,
                                   Reference<AsyncVar<ServerDBInfo> const> const& dbInfo,
                                   std::string const& context);
 
+Future<Void> reconfigureAfter(Database const& cx,
+                              double const& time,
+                              Reference<AsyncVar<ServerDBInfo> const> const& dbInfo,
+                              std::string const& context);
+
 // Returns list of worker interfaces for available storage servers and the number of unavailable
 // storage servers
 Future<std::pair<std::vector<WorkerInterface>, int>>
diff --git a/fdbserver/TagPartitionedLogSystem.actor.cpp b/fdbserver/TagPartitionedLogSystem.actor.cpp
index 72612356970..9b538455908 100644
--- a/fdbserver/TagPartitionedLogSystem.actor.cpp
+++ b/fdbserver/TagPartitionedLogSystem.actor.cpp
@@ -1212,7 +1212,10 @@ Reference<ILogSystem::IPeekCursor> TagPartitionedLogSystem::peekSingle(UID dbgid
 	}
 }
 
-Reference<ILogSystem::IPeekCursor> TagPartitionedLogSystem::peekLogRouter(UID dbgid, Version begin, Tag tag) {
+Reference<ILogSystem::IPeekCursor> TagPartitionedLogSystem::peekLogRouter(UID dbgid,
+                                                                          Version begin,
+                                                                          Tag tag,
+                                                                          bool useSatellite) {
 	bool found = false;
 	for (const auto& log : tLogs) {
 		found = log->hasLogRouter(dbgid) || log->hasBackupWorker(dbgid);
@@ -1240,8 +1243,7 @@ Reference<ILogSystem::IPeekCursor> TagPartitionedLogSystem::peekLogRouter(UID db
 				}
 			}
 			int bestSet = bestPrimarySet;
-			if (SERVER_KNOBS->LOG_ROUTER_PEEK_FROM_SATELLITES_PREFERRED && bestSatelliteSet != -1 &&
-			    tLogs[bestSatelliteSet]->tLogVersion >= TLogVersion::V4) {
+			if (useSatellite && bestSatelliteSet != -1 && tLogs[bestSatelliteSet]->tLogVersion >= TLogVersion::V4) {
 				bestSet = bestSatelliteSet;
 			}
 
@@ -1266,8 +1268,7 @@ Reference<ILogSystem::IPeekCursor> TagPartitionedLogSystem::peekLogRouter(UID db
 				}
 			}
 			int bestSet = bestPrimarySet;
-			if (SERVER_KNOBS->LOG_ROUTER_PEEK_FROM_SATELLITES_PREFERRED && bestSatelliteSet != -1 &&
-			    tLogs[bestSatelliteSet]->tLogVersion >= TLogVersion::V4) {
+			if (useSatellite && bestSatelliteSet != -1 && tLogs[bestSatelliteSet]->tLogVersion >= TLogVersion::V4) {
 				bestSet = bestSatelliteSet;
 			}
 			const auto& log = tLogs[bestSet];
@@ -1307,8 +1308,7 @@ Reference<ILogSystem::IPeekCursor> TagPartitionedLogSystem::peekLogRouter(UID db
 				}
 			}
 			int bestSet = bestPrimarySet;
-			if (SERVER_KNOBS->LOG_ROUTER_PEEK_FROM_SATELLITES_PREFERRED && bestSatelliteSet != -1 &&
-			    old.tLogs[bestSatelliteSet]->tLogVersion >= TLogVersion::V4) {
+			if (useSatellite && bestSatelliteSet != -1 && old.tLogs[bestSatelliteSet]->tLogVersion >= TLogVersion::V4) {
 				bestSet = bestSatelliteSet;
 			}
 
diff --git a/fdbserver/TagPartitionedLogSystem.actor.h b/fdbserver/TagPartitionedLogSystem.actor.h
index df2bf57e42a..c8f026a5ca1 100644
--- a/fdbserver/TagPartitionedLogSystem.actor.h
+++ b/fdbserver/TagPartitionedLogSystem.actor.h
@@ -233,7 +233,7 @@ struct TagPartitionedLogSystem final : ILogSystem, ReferenceCounted<TagPartition
 	// Specifically, the epoch is determined by looking up "dbgid" in tlog sets of generations.
 	// The returned cursor can peek data at the "tag" from the given "begin" version to that epoch's end version or
 	// the recovery version for the latest old epoch. For the current epoch, the cursor has no end version.
-	Reference<IPeekCursor> peekLogRouter(UID dbgid, Version begin, Tag tag) final;
+	Reference<IPeekCursor> peekLogRouter(UID dbgid, Version begin, Tag tag, bool useSatellite) final;
 
 	Version getKnownCommittedVersion() final;
 
diff --git a/fdbserver/tester.actor.cpp b/fdbserver/tester.actor.cpp
index f60ae66b97b..56128e54324 100644
--- a/fdbserver/tester.actor.cpp
+++ b/fdbserver/tester.actor.cpp
@@ -919,9 +919,9 @@ ACTOR Future<Void> checkConsistency(Database cx,
 
 	state double connectionFailures;
 	if (g_network->isSimulated()) {
+		// NOTE: the value will be reset after consistency check
 		connectionFailures = g_simulator.connectionFailuresDisableDuration;
-		g_simulator.connectionFailuresDisableDuration = 1e6;
-		g_simulator.speedUpSimulation = true;
+		disableConnectionFailures("ConsistencyCheck");
 	}
 
 	Standalone<VectorRef<KeyValueRef>> options;
@@ -978,15 +978,23 @@ ACTOR Future<bool> runTest(Database cx,
                            Reference<AsyncVar<ServerDBInfo>> dbInfo,
                            Optional<TenantName> defaultTenant) {
 	state DistributedTestResults testResults;
+	state double savedDisableDuration = 0;
 
 	try {
 		Future<DistributedTestResults> fTestResults = runWorkload(cx, testers, spec, defaultTenant);
+		if (g_network->isSimulated() && spec.simConnectionFailuresDisableDuration > 0) {
+			savedDisableDuration = g_simulator.connectionFailuresDisableDuration;
+			g_simulator.connectionFailuresDisableDuration = spec.simConnectionFailuresDisableDuration;
+		}
 		if (spec.timeout > 0) {
 			fTestResults = timeoutError(fTestResults, spec.timeout);
 		}
 		DistributedTestResults _testResults = wait(fTestResults);
 		testResults = _testResults;
 		logMetrics(testResults.metrics);
+		if (g_network->isSimulated() && savedDisableDuration > 0) {
+			g_simulator.connectionFailuresDisableDuration = savedDisableDuration;
+		}
 	} catch (Error& e) {
 		if (e.code() == error_code_timed_out) {
 			TraceEvent(SevError, "TestFailure")
@@ -1197,8 +1205,6 @@ std::map<std::string, std::function<void(const std::string& value, TestSpec* spe
 	      sscanf(value.c_str(), "%lf", &connectionFailuresDisableDuration);
 	      ASSERT(connectionFailuresDisableDuration >= 0);
 	      spec->simConnectionFailuresDisableDuration = connectionFailuresDisableDuration;
-	      if (g_network->isSimulated())
-		      g_simulator.connectionFailuresDisableDuration = spec->simConnectionFailuresDisableDuration;
 	      TraceEvent("TestParserTest")
 	          .detail("ParsedSimConnectionFailuresDisableDuration", spec->simConnectionFailuresDisableDuration);
 	  } },
@@ -1447,6 +1453,17 @@ ACTOR Future<Void> monitorServerDBInfo(Reference<AsyncVar<Optional<ClusterContro
 	}
 }
 
+// Disables connection failures after the given time seconds
+ACTOR Future<Void> disableConnectionFailuresAfter(double seconds, std::string context) {
+	if (g_network->isSimulated()) {
+		TraceEvent(SevWarnAlways, ("ScheduleDisableConnectionFailures_" + context).c_str())
+		    .detail("At", now() + seconds);
+		wait(delay(seconds));
+		disableConnectionFailures(context);
+	}
+	return Void();
+}
+
 /**
  * \brief Test orchestrator: sends test specification to testers in the right order and collects the results.
  *
@@ -1522,7 +1539,7 @@ ACTOR Future<Void> runTests(Reference<AsyncVar<Optional<struct ClusterController
 		cx->defaultTenant = defaultTenant;
 	}
 
-	state Future<Void> disabler = disableConnectionFailuresAfter(FLOW_KNOBS->SIM_SPEEDUP_AFTER_SECONDS, "Tester");
+	disableConnectionFailures("Tester");
 
 	// Change the configuration (and/or create the database) if necessary
 	printf("startingConfiguration:%s start\n", startingConfiguration.toString().c_str());
@@ -1573,6 +1590,14 @@ ACTOR Future<Void> runTests(Reference<AsyncVar<Optional<struct ClusterController
 		}
 	}
 
+	enableConnectionFailures("Tester");
+	state Future<Void> disabler = disableConnectionFailuresAfter(FLOW_KNOBS->SIM_SPEEDUP_AFTER_SECONDS, "Tester");
+	state Future<Void> repairDataCenter;
+	if (useDB) {
+		Future<Void> reconfigure = reconfigureAfter(cx, FLOW_KNOBS->SIM_SPEEDUP_AFTER_SECONDS, dbInfo, "Tester");
+		repairDataCenter = reconfigure;
+	}
+
 	TraceEvent("TestsExpectedToPass").detail("Count", tests.size());
 	state int idx = 0;
 	state std::unique_ptr<KnobProtectiveGroup> knobProtectiveGroup;
diff --git a/fdbserver/workloads/ClogTlog.actor.cpp b/fdbserver/workloads/ClogTlog.actor.cpp
index cc3c8a77204..7e8ebe01daf 100644
--- a/fdbserver/workloads/ClogTlog.actor.cpp
+++ b/fdbserver/workloads/ClogTlog.actor.cpp
@@ -132,19 +132,13 @@ struct ClogTlogWorkload : TestWorkload {
 			}
 			when(wait(timeout)) {
 				// recovery state hasn't changed in 30s, exclude the failed tlog
-				std::vector<StringRef> tokens;
-				state Optional<ConfigureAutoResult> conf;
-
 				TEST(true); // Exclude failed tlog
 				TraceEvent("ExcludeFailedLog")
 				    .detail("TLog", self->tlog.get())
 				    .detail("RecoveryState", self->dbInfo->get().recoveryState);
-				tokens.push_back(StringRef("exclude=" + formatIpPort(self->tlog.get().ip, self->tlog.get().port)));
-				ConfigurationResult r =
-				    wait(ManagementAPI::changeConfig(cx.getReference(), tokens, conf, /*force=*/true));
-				TraceEvent("ExcludeFailedLog")
-				    .detail("Result", r)
-				    .detail("ConfigIsValid", conf.present() && conf.get().isValid());
+				std::string modes = "exclude=" + formatIpPort(self->tlog.get().ip, self->tlog.get().port);
+				ConfigurationResult r = wait(ManagementAPI::changeConfig(cx.getReference(), modes, /*force=*/true));
+				TraceEvent("ExcludeFailedLog").detail("Result", r);
 				return Void();
 			}
 		}
diff --git a/fdbserver/workloads/DcLag.actor.cpp b/fdbserver/workloads/DcLag.actor.cpp
new file mode 100644
index 00000000000..b77a69c8919
--- /dev/null
+++ b/fdbserver/workloads/DcLag.actor.cpp
@@ -0,0 +1,186 @@
+/*
+ * DcLag.actor.cpp
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2023 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "fdbclient/NativeAPI.actor.h"
+#include "fdbclient/StatusClient.h"
+#include "fdbrpc/Locality.h"
+#include "fdbserver/Knobs.h"
+#include "fdbserver/RecoveryState.h"
+#include "fdbserver/ServerDBInfo.actor.h"
+#include "fdbserver/TesterInterface.actor.h"
+#include "fdbserver/workloads/workloads.actor.h"
+#include "fdbrpc/simulator.h"
+#include "flow/Error.h"
+#include "flow/Trace.h"
+#include "flow/flow.h"
+#include "flow/network.h"
+#include "flow/actorcompiler.h" // This must be the last #include.
+
+// This workload tests a gray failure scenario: a satellite TLog is have network issue
+// for sending packets to the remote data center's log routers. This will cause these
+// log routers to fail to progress, and causing data center lag, i.e., lag between the
+// primary and remote DC's tlags. With changes to log routers, they can detect that the
+// peek is taking a long time (> LOG_ROUTER_PEEK_SWITCH_DC_TIME) and swith to use
+// another DC to get data, thus recoverying from the data center lag.
+struct DcLagWorkload : TestWorkload {
+	static constexpr auto NAME = "DcLag";
+	bool enabled;
+	double testDuration;
+	double startDelay;
+	std::vector<std::pair<IPAddress, IPAddress>> cloggedPairs;
+
+	DcLagWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
+		enabled = !clientId; // only do this on the "first" client
+		testDuration = getOption(options, "testDuration"_sr, 1000.0);
+		startDelay = getOption(options, "startDelay"_sr, 10.0);
+	}
+
+	std::string description() const override { return "DcLagWorkload"; }
+
+	Future<Void> setup(Database const& cx) override { return Void(); }
+	Future<Void> start(Database const& cx) override {
+		if (g_network->isSimulated() && enabled)
+			return timeout(reportErrors(clogClient(this, cx), "DcLagError"), testDuration, Void());
+		else
+			return Void();
+	}
+	Future<bool> check(Database const& cx) override { return true; }
+	void getMetrics(std::vector<PerfMetric>& m) override {}
+
+	// Clog a satellite tlog with all remote processes so that this triggers high
+	// data center lag.
+	bool clogTlog(double seconds) {
+		ASSERT(dbInfo->get().recoveryState >= RecoveryState::RECOVERY_TRANSACTION);
+
+		std::vector<IPAddress> ips; // all remote process IPs
+		for (const auto& process : g_simulator.getAllProcesses()) {
+			const auto& ip = process->address.ip;
+			if (process->locality.dcId().present() && process->locality.dcId() == g_simulator.remoteDcId) {
+				ips.push_back(ip);
+			}
+		}
+		ASSERT(ips.size() > 0);
+
+		// Find all primary satellite tlogs
+		std::vector<NetworkAddress> logs; // all primary satellite logs
+		for (const auto& tlogset : dbInfo->get().logSystemConfig.tLogs) {
+			if (!tlogset.isLocal || tlogset.locality != tagLocalitySatellite)
+				continue;
+			for (const auto& log : tlogset.tLogs) {
+				const NetworkAddress& addr = log.interf().address();
+				logs.push_back(addr);
+			}
+		}
+		if (logs.empty()) {
+			// No satellite tlog found: fearless config may not have satellites
+			return false;
+		}
+
+		// clog pairs
+		auto tlog = logs[0].ip;
+		for (const auto& ip : ips) {
+			if (tlog != ip) {
+				// Clog TLogReply messages, but allow peek/pop requests
+				// g_simulator.clogPair(ip, tlog, seconds);
+				g_simulator.clogPair(tlog, ip, seconds);
+				cloggedPairs.emplace_back(tlog, ip);
+			}
+		}
+		return true;
+	}
+
+	void unclogAll() {
+		// unclog previously clogged connections
+		for (const auto& pair : cloggedPairs) {
+			g_simulator.unclogPair(pair.first, pair.second);
+		}
+		cloggedPairs.clear();
+	}
+
+	ACTOR static Future<Optional<double>> fetchDatacenterLag(DcLagWorkload* self, Database cx) {
+		StatusObject result = wait(StatusClient::statusFetcher(cx));
+		StatusObjectReader statusObj(result);
+		StatusObjectReader statusObjCluster;
+		if (!statusObj.get("cluster", statusObjCluster)) {
+			TraceEvent("DcLagNoCluster");
+			return Optional<double>();
+		}
+
+		StatusObjectReader dcLag;
+		if (!statusObjCluster.get("datacenter_lag", dcLag)) {
+			TraceEvent("DcLagNoLagData");
+			return Optional<double>();
+		}
+
+		Version versions = 0;
+		double seconds = 0;
+		if (!dcLag.get("versions", versions)) {
+			TraceEvent("DcLagNoVersions");
+			return Optional<double>();
+		}
+		if (!dcLag.get("seconds", seconds)) {
+			TraceEvent("DcLagNoSeconds");
+			return Optional<double>();
+		}
+		TraceEvent("DcLag").detail("Versions", versions).detail("Seconds", seconds);
+		return seconds;
+	}
+
+	ACTOR Future<Void> clogClient(DcLagWorkload* self, Database cx) {
+		wait(delay(self->startDelay));
+
+		while (self->dbInfo->get().recoveryState < RecoveryState::FULLY_RECOVERED) {
+			wait(self->dbInfo->onChange());
+		}
+
+		double startTime = now();
+		double workloadEnd = now() + self->testDuration;
+		TraceEvent("DcLag").detail("StartTime", startTime).detail("EndTime", workloadEnd);
+
+		// Clog and wait for recovery to happen
+		if (!self->clogTlog(self->testDuration)) {
+			return Void(); // skip the test if no satellite found
+		}
+
+		state Future<Optional<double>> status = Never();
+		state bool lagged = false;
+		loop choose {
+			when(wait(delay(5.0))) {
+				// Fetch DC lag every 5s
+				status = fetchDatacenterLag(self, cx);
+			}
+			when(Optional<double> lag = wait(status)) {
+				if (lag.present() && lag.get() >= SERVER_KNOBS->LOG_ROUTER_PEEK_SWITCH_DC_TIME - 10.0) {
+					// Detect DC Lag happened before Log router switch DC reactions
+					lagged = true;
+					TraceEvent("DcLagDetected");
+				}
+				if (lagged && lag.present() && lag.get() < 5.0) {
+					TraceEvent("DcLagRecovered");
+					self->unclogAll();
+					return Void();
+				}
+				status = Never();
+			}
+		}
+	}
+};
+
+WorkloadFactory<DcLagWorkload> DcLagWorkloadFactory("DcLag");
\ No newline at end of file
diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt
index 86c6423bbcd..e3ba968e392 100644
--- a/tests/CMakeLists.txt
+++ b/tests/CMakeLists.txt
@@ -200,6 +200,7 @@ if(WITH_PYTHON)
   add_fdb_test(TEST_FILES rare/ConflictRangeRYOWCheck.toml)
   add_fdb_test(TEST_FILES rare/CycleRollbackClogged.toml)
   add_fdb_test(TEST_FILES rare/CycleWithKills.toml)
+  add_fdb_test(TEST_FILES rare/DcLag.toml)
   add_fdb_test(TEST_FILES rare/FuzzTest.toml)
   add_fdb_test(TEST_FILES rare/HighContentionPrefixAllocator.toml)
   add_fdb_test(TEST_FILES rare/InventoryTestHeavyWrites.toml)
diff --git a/tests/rare/DcLag.toml b/tests/rare/DcLag.toml
new file mode 100644
index 00000000000..30fecd8583d
--- /dev/null
+++ b/tests/rare/DcLag.toml
@@ -0,0 +1,19 @@
+[configuration]
+generateFearless = true
+processesPerMachine = 1
+machineCount = 20
+minimumRegions = 2
+
+[[test]]
+testTitle = 'DcLag'
+
+    [[test.workload]]
+    testName = 'Cycle'
+    nodeCount = 3000
+    transactionsPerSecond = 250.0
+    testDuration = 200.0
+    expectedRate = 0
+
+    [[test.workload]]
+    testName = 'DcLag'
+    testDuration = 1000.0

From 13a98ab8bbc78b003b0cd1b4471cbc024c7049c5 Mon Sep 17 00:00:00 2001
From: Lukas Joswiak <lukas.joswiak@snowflake.com>
Date: Fri, 2 Dec 2022 13:38:43 -0800
Subject: [PATCH 107/468] Fix issue where the versions on seed storage servers
 decreased

Seed storage servers are recruited as the intial set of storage servers
when a database is first created. They function a little bit differently
than normal, and do not set an initial version like storages normally do
when they get recruited (typically equal to the recovery version).

Version correction is a feature where versions advance in sync with the
clock, and are equal across FDB clusters. To allow different FDB
clusters to have matching versions, they must share the same base
version. This defaults to the Unix epoch, and clusters with the version
epoch enabled will have a current version equal to the number of
microseconds since the Unix epoch.

When the version epoch is enabled on a cluster, it causes a one time
jump from the clusters current version to the version based on the
epoch. After a recovery, the recovery version sent to storages should
have advanced by a significant amount.

The recovery path contained a `BUGGIFY` to randomly advance the recovery
version in simulation, testing the version epoch being enabled.
However, it was also advancing the version during an initial recovery,
when the seed storage servers are recruited. If a set of storage
servers were recruited as seed servers, but another recovery occurred
before the bootstrap process was complete, the randomly selected version
increase could be smaller during the second recovery than during the
first. This could cause the initial set of seed servers to think they
should be at a version larger than what the cluuster was actually at.

The fix contained in this commit is to only cause a random version jump
when the recovery is occuring on an existing database, and not when it
is recruiting seed storages.

This commit fixes an issue found in simulation, reproducible with:

Commit: 93dc4bfeb97a700bafa4b34bc18d38a248e47b35
Test: fast/DataLossRecovery.toml
Seed: 3101495991
Buggify: on
Compiler: clang
---
 fdbclient/VersionedMap.h            |  2 +-
 fdbserver/ClusterRecovery.actor.cpp | 10 +++++++---
 2 files changed, 8 insertions(+), 4 deletions(-)

diff --git a/fdbclient/VersionedMap.h b/fdbclient/VersionedMap.h
index b1b8ddf2aa6..2b47c96fc8d 100644
--- a/fdbclient/VersionedMap.h
+++ b/fdbclient/VersionedMap.h
@@ -724,7 +724,7 @@ class VersionedMap : NonCopyable {
 	}
 
 	Future<Void> forgetVersionsBeforeAsync(Version newOldestVersion, TaskPriority taskID = TaskPriority::DefaultYield) {
-		ASSERT(newOldestVersion <= latestVersion);
+		ASSERT_LE(newOldestVersion, latestVersion);
 		auto r = upper_bound(roots.begin(), roots.end(), newOldestVersion, rootsComparator());
 		auto upper = r;
 		--r;
diff --git a/fdbserver/ClusterRecovery.actor.cpp b/fdbserver/ClusterRecovery.actor.cpp
index 027d4ad2c07..da676149639 100644
--- a/fdbserver/ClusterRecovery.actor.cpp
+++ b/fdbserver/ClusterRecovery.actor.cpp
@@ -1093,10 +1093,14 @@ ACTOR Future<Void> readTransactionSystemState(Reference<ClusterRecoveryData> sel
 
 		if (self->recoveryTransactionVersion < minRequiredCommitVersion)
 			self->recoveryTransactionVersion = minRequiredCommitVersion;
-	}
 
-	if (BUGGIFY) {
-		self->recoveryTransactionVersion += deterministicRandom()->randomInt64(0, 10000000);
+		// Test randomly increasing the recovery version by a large number.
+		// When the version epoch is enabled, versions stay in sync with time.
+		// An offline cluster could see a large version jump when it comes back
+		// online, so test this behavior in simulation.
+		if (BUGGIFY) {
+			self->recoveryTransactionVersion += deterministicRandom()->randomInt64(0, 10000000);
+		}
 	}
 
 	TraceEvent(getRecoveryEventName(ClusterRecoveryEventType::CLUSTER_RECOVERY_RECOVERING_EVENT_NAME).c_str(),

From dd379cc541af5fd62beab7fe85438d72167d82b5 Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Sun, 12 Mar 2023 21:32:43 -0700
Subject: [PATCH 108/468] Added 7.1.28 and 7.1.29 release notes

---
 .../source/release-notes/release-notes-710.rst     | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/documentation/sphinx/source/release-notes/release-notes-710.rst b/documentation/sphinx/source/release-notes/release-notes-710.rst
index c1261387d19..b2922f80818 100644
--- a/documentation/sphinx/source/release-notes/release-notes-710.rst
+++ b/documentation/sphinx/source/release-notes/release-notes-710.rst
@@ -4,6 +4,20 @@
 Release Notes
 #############
 
+7.1.29
+======
+* Same as 7.1.28 release with AVX enabled.
+
+7.1.28
+======
+* Released with AVX disabled.
+* Changed log router to detect slow peeks and to automatically switch DC for peeking. `(PR #9640) <https://github.com/apple/foundationdb/pull/9640>`_
+* Added multiple prefix filter support for fdbdecode. `(PR #9483) <https://github.com/apple/foundationdb/pull/9483>`_, `(PR #9489) <https://github.com/apple/foundationdb/pull/9489>`_, `(PR #9511) <https://github.com/apple/foundationdb/pull/9511>`_, and `(PR #9560) <https://github.com/apple/foundationdb/pull/9560>`_
+* Enhanced fdbbackup query command to estimate data processing from a specific snapshot to a target version. `(PR #9506) <https://github.com/apple/foundationdb/pull/9506>`_
+* Improved PTree insertion and erase performance for storage servers. `(PR #9508) <https://github.com/apple/foundationdb/pull/9508>`_
+* Added exclude to fdbcli's configure command to prevent faulty TLogs from affecting recovery. `(PR #9404) <https://github.com/apple/foundationdb/pull/9404>`_
+* Fixed getMappedRange metrics. `(PR #9331) <https://github.com/apple/foundationdb/pull/9331>`_
+
 7.1.27
 ======
 * Same as 7.1.26 release with AVX enabled.

From c7b524cd48ec6b82e5b9fe52cf6350e98b49d2e5 Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Wed, 8 Mar 2023 09:20:33 -0800
Subject: [PATCH 109/468] Reduce running time for ClogTlog

When the ClogTlog is running, we may already pass the 450s, i.e., SIM_SPEEDUP_AFTER_SECONDS,
and clogging is no longer effective. If that's the case, we want to finish the test quickly.
---
 tests/rare/ClogTlog.toml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/rare/ClogTlog.toml b/tests/rare/ClogTlog.toml
index c92455d34f6..d5b5dc8096a 100644
--- a/tests/rare/ClogTlog.toml
+++ b/tests/rare/ClogTlog.toml
@@ -17,9 +17,9 @@ testTitle = 'ClogTlog'
     testName = 'Cycle'
     nodeCount = 30000
     transactionsPerSecond = 2500.0
-    testDuration = 120.0
+    testDuration = 60.0
     expectedRate = 0
 
     [[test.workload]]
     testName = 'ClogTlog'
-    testDuration = 3000.0
+    testDuration = 150.0

From c52b18feb350cf88306dfc15186351c5d9ebf81e Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Mon, 13 Mar 2023 14:58:33 -0700
Subject: [PATCH 110/468] Remove profile code from SpecialKeySpace workload

This part of code has problems with GlobalConfig and is buggy.
---
 .../SpecialKeySpaceCorrectness.actor.cpp      | 136 ------------------
 1 file changed, 136 deletions(-)

diff --git a/fdbserver/workloads/SpecialKeySpaceCorrectness.actor.cpp b/fdbserver/workloads/SpecialKeySpaceCorrectness.actor.cpp
index 05b509bc252..84f36de5f7f 100644
--- a/fdbserver/workloads/SpecialKeySpaceCorrectness.actor.cpp
+++ b/fdbserver/workloads/SpecialKeySpaceCorrectness.actor.cpp
@@ -1203,142 +1203,6 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload {
 		} catch (Error& e) {
 			wait(tx->onError(e));
 		}
-		// profile client get
-		loop {
-			try {
-				tx->setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
-				// client_txn_sample_rate
-				state Optional<Value> txnSampleRate =
-				    wait(tx->get(LiteralStringRef("client_txn_sample_rate")
-				                     .withPrefix(SpecialKeySpace::getManagementApiCommandPrefix("profile"))));
-				ASSERT(txnSampleRate.present());
-				Optional<Value> txnSampleRateKey = wait(tx->get(GlobalConfig::prefixedKey(fdbClientInfoTxnSampleRate)));
-				if (txnSampleRateKey.present()) {
-					const double sampleRateDbl =
-					    BinaryReader::fromStringRef<double>(txnSampleRateKey.get(), Unversioned());
-					if (!std::isinf(sampleRateDbl)) {
-						ASSERT(txnSampleRate.get().toString() == boost::lexical_cast<std::string>(sampleRateDbl));
-					} else {
-						ASSERT(txnSampleRate.get().toString() == "default");
-					}
-				} else {
-					ASSERT(txnSampleRate.get().toString() == "default");
-				}
-				// client_txn_size_limit
-				state Optional<Value> txnSizeLimit =
-				    wait(tx->get(LiteralStringRef("client_txn_size_limit")
-				                     .withPrefix(SpecialKeySpace::getManagementApiCommandPrefix("profile"))));
-				ASSERT(txnSizeLimit.present());
-				Optional<Value> txnSizeLimitKey = wait(tx->get(GlobalConfig::prefixedKey(fdbClientInfoTxnSizeLimit)));
-				if (txnSizeLimitKey.present()) {
-					const int64_t sizeLimit =
-					    BinaryReader::fromStringRef<int64_t>(txnSizeLimitKey.get(), Unversioned());
-					if (sizeLimit != -1) {
-						ASSERT(txnSizeLimit.get().toString() == boost::lexical_cast<std::string>(sizeLimit));
-					} else {
-						ASSERT(txnSizeLimit.get().toString() == "default");
-					}
-				} else {
-					ASSERT(txnSizeLimit.get().toString() == "default");
-				}
-				tx->reset();
-				break;
-			} catch (Error& e) {
-				TraceEvent(SevDebug, "ProfileClientGet").error(e);
-				wait(tx->onError(e));
-			}
-		}
-		{
-			state double r_sample_rate = deterministicRandom()->random01();
-			state int64_t r_size_limit = deterministicRandom()->randomInt64(1e3, 1e6);
-			// update the sample rate and size limit
-			loop {
-				try {
-					tx->setOption(FDBTransactionOptions::RAW_ACCESS);
-					tx->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES);
-					tx->set(LiteralStringRef("client_txn_sample_rate")
-					            .withPrefix(SpecialKeySpace::getManagementApiCommandPrefix("profile")),
-					        Value(boost::lexical_cast<std::string>(r_sample_rate)));
-					tx->set(LiteralStringRef("client_txn_size_limit")
-					            .withPrefix(SpecialKeySpace::getManagementApiCommandPrefix("profile")),
-					        Value(boost::lexical_cast<std::string>(r_size_limit)));
-					wait(tx->commit());
-					tx->reset();
-					break;
-				} catch (Error& e) {
-					wait(tx->onError(e));
-				}
-			}
-			// commit successfully, verify the system key changed
-			loop {
-				try {
-					state Optional<Value> sampleRate =
-					    wait(tx->get(GlobalConfig::prefixedKey(fdbClientInfoTxnSampleRate)));
-					if (!sampleRate.present()) {
-						// Wait for GlobalConfig synchronization
-						wait(delayJittered(1.0));
-						continue;
-					}
-					ASSERT(sampleRate.present());
-					ASSERT_LT(r_sample_rate - boost::lexical_cast<double>(sampleRate.get().toString()), 0.001);
-					Optional<Value> sizeLimit = wait(tx->get(GlobalConfig::prefixedKey(fdbClientInfoTxnSizeLimit)));
-					ASSERT(sizeLimit.present());
-					ASSERT(r_size_limit == boost::lexical_cast<int64_t>(sizeLimit.get().toString()));
-					tx->reset();
-					break;
-				} catch (Error& e) {
-					wait(tx->onError(e));
-				}
-			}
-			// Change back to default
-			loop {
-				try {
-					tx->setOption(FDBTransactionOptions::RAW_ACCESS);
-					tx->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES);
-					tx->set(LiteralStringRef("client_txn_sample_rate")
-					            .withPrefix(SpecialKeySpace::getManagementApiCommandPrefix("profile")),
-					        LiteralStringRef("default"));
-					tx->set(LiteralStringRef("client_txn_size_limit")
-					            .withPrefix(SpecialKeySpace::getManagementApiCommandPrefix("profile")),
-					        LiteralStringRef("default"));
-					wait(tx->commit());
-					tx->reset();
-					break;
-				} catch (Error& e) {
-					wait(tx->onError(e));
-				}
-			}
-			// Test invalid values
-			loop {
-				try {
-					tx->setOption(FDBTransactionOptions::RAW_ACCESS);
-					tx->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES);
-					tx->set((deterministicRandom()->coinflip() ? LiteralStringRef("client_txn_sample_rate")
-					                                           : LiteralStringRef("client_txn_size_limit"))
-					            .withPrefix(SpecialKeySpace::getManagementApiCommandPrefix("profile")),
-					        LiteralStringRef("invalid_value"));
-					wait(tx->commit());
-					ASSERT(false);
-				} catch (Error& e) {
-					if (e.code() == error_code_special_keys_api_failure) {
-						Optional<Value> errorMsg =
-						    wait(tx->get(SpecialKeySpace::getModuleRange(SpecialKeySpace::MODULE::ERRORMSG).begin));
-						ASSERT(errorMsg.present());
-						std::string errorStr;
-						auto valueObj = readJSONStrictly(errorMsg.get().toString()).get_obj();
-						auto schema = readJSONStrictly(JSONSchemas::managementApiErrorSchema.toString()).get_obj();
-						// special_key_space_management_api_error_msg schema validation
-						ASSERT(schemaMatch(schema, valueObj, errorStr, SevError, true));
-						ASSERT(valueObj["command"].get_str() == "profile" && !valueObj["retriable"].get_bool());
-						tx->reset();
-						break;
-					} else {
-						wait(tx->onError(e));
-					}
-					wait(delay(FLOW_KNOBS->PREVENT_FAST_SPIN_DELAY));
-				}
-			}
-		}
 		// data_distribution & maintenance get
 		loop {
 			try {

From a60d512ebbdcb516284ec8ca56fb9546531bf81f Mon Sep 17 00:00:00 2001
From: FoundationDB CI <foundationdb_ci@apple.com>
Date: Tue, 14 Mar 2023 19:59:10 +0000
Subject: [PATCH 111/468] disable AVX for 7.1.28 release

---
 cmake/ConfigureCompiler.cmake | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cmake/ConfigureCompiler.cmake b/cmake/ConfigureCompiler.cmake
index e52d8665ee8..7e50469ba6a 100644
--- a/cmake/ConfigureCompiler.cmake
+++ b/cmake/ConfigureCompiler.cmake
@@ -253,7 +253,7 @@ else()
       set(USE_AVX512F OFF)
     endif()
   endif()
-  set(USE_AVX ON CACHE BOOL "Enable AVX instructions")
+  set(USE_AVX OFF CACHE BOOL "Enable AVX instructions")
   if (USE_AVX)
     if (CMAKE_HOST_SYSTEM_PROCESSOR MATCHES "^x86")
       add_compile_options(-mavx)

From 1b2517abce552441e3d0ed8836d1cc3f40e61a2a Mon Sep 17 00:00:00 2001
From: FoundationDB CI <foundationdb_ci@apple.com>
Date: Tue, 14 Mar 2023 19:59:33 +0000
Subject: [PATCH 112/468] enable AVX and update version for 7.1.29 release

---
 CMakeLists.txt                | 2 +-
 cmake/ConfigureCompiler.cmake | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 5528c6a6b6e..945997d94be 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -23,7 +23,7 @@ else()
 endif()
 
 project(foundationdb
-  VERSION 7.1.28
+  VERSION 7.1.29
   DESCRIPTION "FoundationDB is a scalable, fault-tolerant, ordered key-value store with full ACID transactions."
   HOMEPAGE_URL "http://www.foundationdb.org/"
   LANGUAGES C CXX ASM)
diff --git a/cmake/ConfigureCompiler.cmake b/cmake/ConfigureCompiler.cmake
index 7e50469ba6a..e52d8665ee8 100644
--- a/cmake/ConfigureCompiler.cmake
+++ b/cmake/ConfigureCompiler.cmake
@@ -253,7 +253,7 @@ else()
       set(USE_AVX512F OFF)
     endif()
   endif()
-  set(USE_AVX OFF CACHE BOOL "Enable AVX instructions")
+  set(USE_AVX ON CACHE BOOL "Enable AVX instructions")
   if (USE_AVX)
     if (CMAKE_HOST_SYSTEM_PROCESSOR MATCHES "^x86")
       add_compile_options(-mavx)

From c3ad9b52a424dd6f7b71f6a4a411df6a67cc3f53 Mon Sep 17 00:00:00 2001
From: FoundationDB CI <foundationdb_ci@apple.com>
Date: Wed, 15 Mar 2023 15:34:08 +0000
Subject: [PATCH 113/468] update version after 7.1.29 release

---
 CMakeLists.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 945997d94be..7659c210c06 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -23,7 +23,7 @@ else()
 endif()
 
 project(foundationdb
-  VERSION 7.1.29
+  VERSION 7.1.30
   DESCRIPTION "FoundationDB is a scalable, fault-tolerant, ordered key-value store with full ACID transactions."
   HOMEPAGE_URL "http://www.foundationdb.org/"
   LANGUAGES C CXX ASM)

From 5815d89f1e80d9c69373dbbcc63c9964ac0b4ca6 Mon Sep 17 00:00:00 2001
From: Zhe Wu <halfprice@users.noreply.github.com>
Date: Tue, 21 Mar 2023 21:40:12 -0700
Subject: [PATCH 114/468] Update info trigger new DB info update immediately

---
 fdbserver/ClusterController.actor.cpp | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/fdbserver/ClusterController.actor.cpp b/fdbserver/ClusterController.actor.cpp
index 0136192da7e..cd6b78e7600 100644
--- a/fdbserver/ClusterController.actor.cpp
+++ b/fdbserver/ClusterController.actor.cpp
@@ -2389,7 +2389,7 @@ ACTOR Future<Void> dbInfoUpdater(ClusterControllerData* self) {
 			when(wait(dbInfoChange)) {}
 		}
 
-		UpdateServerDBInfoRequest req;
+		state UpdateServerDBInfoRequest req;
 		if (dbInfoChange.isReady()) {
 			for (auto& it : self->id_worker) {
 				req.broadcastInfo.push_back(it.second.details.interf.updateServerDBInfo.getEndpoint());
@@ -2411,7 +2411,9 @@ ACTOR Future<Void> dbInfoUpdater(ClusterControllerData* self) {
 		req.serializedDbInfo =
 		    BinaryWriter::toValue(self->db.serverInfo->get(), AssumeVersion(g_network->protocolVersion()));
 
-		TraceEvent("DBInfoStartBroadcast", self->id).log();
+		TraceEvent("DBInfoStartBroadcast", self->id)
+		    .detail("MasterLifetime", self->db.serverInfo->get().masterLifetime.toString());
+		;
 		choose {
 			when(std::vector<Endpoint> notUpdated =
 			         wait(broadcastDBInfoRequest(req, SERVER_KNOBS->DBINFO_SEND_AMOUNT, Optional<Endpoint>(), false))) {
@@ -2422,6 +2424,11 @@ ACTOR Future<Void> dbInfoUpdater(ClusterControllerData* self) {
 				}
 			}
 			when(wait(dbInfoChange)) {}
+			when(wait(updateDBInfo)) {
+				// The current round of broadcast hasn't finished yet. So we need to include all the current broadcast
+				// endpoints in the new round as well.
+				self->updateDBInfoEndpoints.insert(req.broadcastInfo.begin(), req.broadcastInfo.end());
+			}
 		}
 	}
 }

From 205f58b30cc0c3238b96b5da34180d153648563e Mon Sep 17 00:00:00 2001
From: Johannes Scheuermann <johscheuer@users.noreply.github.com>
Date: Thu, 23 Mar 2023 19:24:21 +0000
Subject: [PATCH 115/468] Backport exclusion fix #9468 (#9789)

* Don't block the exclusion of stateless processes by the free capacity check

* Fix syntax

* Make use of precomputed exclude check

* Format code

* Only consider newly excluded processes

* Format code and update comment
---
 fdbclient/SpecialKeySpace.actor.cpp | 20 ++++++++++++++++++--
 1 file changed, 18 insertions(+), 2 deletions(-)

diff --git a/fdbclient/SpecialKeySpace.actor.cpp b/fdbclient/SpecialKeySpace.actor.cpp
index 451febfbeeb..1f86beb3281 100644
--- a/fdbclient/SpecialKeySpace.actor.cpp
+++ b/fdbclient/SpecialKeySpace.actor.cpp
@@ -979,6 +979,9 @@ ACTOR Future<bool> checkExclusion(Database db,
 	state int64_t totalKvStoreFreeBytes = 0;
 	state int64_t totalKvStoreUsedBytes = 0;
 	state int64_t totalKvStoreUsedBytesNonExcluded = 0;
+	// Keep track if we exclude any storage process with the provided adddresses
+	state bool excludedAddressesContainsStorageRole = false;
+
 	try {
 		for (auto proc : processesMap.obj()) {
 			StatusObjectReader process(proc.second);
@@ -988,8 +991,8 @@ ACTOR Future<bool> checkExclusion(Database db,
 				return false;
 			}
 			NetworkAddress addr = NetworkAddress::parse(addrStr);
-			bool excluded =
-			    (process.has("excluded") && process.last().get_bool()) || addressExcluded(*exclusions, addr);
+			bool includedInExclusion = addressExcluded(*exclusions, addr);
+			bool excluded = (process.has("excluded") && process.last().get_bool()) || includedInExclusion;
 
 			StatusObjectReader localityObj;
 			std::string disk_id;
@@ -1002,6 +1005,13 @@ ACTOR Future<bool> checkExclusion(Database db,
 				if (role["role"].get_str() == "storage") {
 					ssTotalCount++;
 
+					// Check if we are excluding a process that serves the storage role. We only have to check the free
+					// capacity if we are excluding at least one process that serves the storage role.
+					if (!excludedAddressesContainsStorageRole && includedInExclusion) {
+						excludedAddressesContainsStorageRole = true;
+						break;
+					}
+
 					int64_t used_bytes;
 					if (!role.get("kvstore_used_bytes", used_bytes)) {
 						*msg = ManagementAPIError::toJsonString(
@@ -1041,6 +1051,12 @@ ACTOR Future<bool> checkExclusion(Database db,
 		return false;
 	}
 
+	// If the exclusion command only contains processes that serve a non storage role we can skip the free capacity
+	// check in order to not block those exclusions.
+	if (!excludedAddressesContainsStorageRole) {
+		return true;
+	}
+
 	double finalFreeRatio = 1 - (totalKvStoreUsedBytes / (totalKvStoreUsedBytesNonExcluded + totalKvStoreFreeBytes));
 	if (ssExcludedCount == ssTotalCount || finalFreeRatio <= 0.1) {
 		std::string temp = "ERROR: This exclude may cause the total free space in the cluster to drop below 10%.\n"

From a121a936b31f7560c7951185e7e1b54dcdc35d8a Mon Sep 17 00:00:00 2001
From: Hao Fu <77984096+hfu94@users.noreply.github.com>
Date: Thu, 23 Mar 2023 12:28:33 -0700
Subject: [PATCH 116/468]  Fix finishedQueries metric, add metrics reporting in
 GetMappedRange test [release-7.1] (#9785)

* Fix finishedQueries metric, add metrics reporting in GetMappedRange test

* refactor to make format work

* resolve comments

* Fix more comments

* Fix bugs and change running time of test
---
 fdbserver/storageserver.actor.cpp            |  1 +
 fdbserver/workloads/GetMappedRange.actor.cpp | 99 +++++++++++++++++++-
 tests/fast/GetMappedRange.toml               |  1 +
 3 files changed, 97 insertions(+), 4 deletions(-)

diff --git a/fdbserver/storageserver.actor.cpp b/fdbserver/storageserver.actor.cpp
index 7e141468302..3a3868e25cf 100644
--- a/fdbserver/storageserver.actor.cpp
+++ b/fdbserver/storageserver.actor.cpp
@@ -4173,6 +4173,7 @@ ACTOR Future<Void> getMappedKeyValuesQ(StorageServer* data, GetMappedKeyValuesRe
 	}
 
 	data->transactionTagCounter.addRequest(req.tags, resultSize);
+	++data->counters.finishedQueries;
 	++data->counters.finishedGetMappedRangeQueries;
 	--data->readQueueSizeMetric;
 
diff --git a/fdbserver/workloads/GetMappedRange.actor.cpp b/fdbserver/workloads/GetMappedRange.actor.cpp
index abbf2c248f5..6c77f58c2c5 100644
--- a/fdbserver/workloads/GetMappedRange.actor.cpp
+++ b/fdbserver/workloads/GetMappedRange.actor.cpp
@@ -23,6 +23,7 @@
 #include <algorithm>
 #include "fdbrpc/simulator.h"
 #include "fdbclient/MutationLogReader.actor.h"
+#include "fdbclient/StatusClient.h"
 #include "fdbclient/Tuple.h"
 #include "fdbserver/workloads/ApiWorkload.h"
 #include "fdbserver/workloads/workloads.actor.h"
@@ -49,9 +50,11 @@ struct GetMappedRangeWorkload : ApiWorkload {
 	//	const bool SPLIT_RECORDS = deterministicRandom()->random01() < 0.5;
 	const bool SPLIT_RECORDS = true;
 	const static int SPLIT_SIZE = 3;
+	int checkStorageQueueSeconds;
 
 	GetMappedRangeWorkload(WorkloadContext const& wcx) : ApiWorkload(wcx) {
 		enabled = !clientId; // only do this on the "first" client
+		checkStorageQueueSeconds = getOption(options, "checkStorageQueueSeconds"_sr, 30.0);
 	}
 
 	std::string description() const override { return "GetMappedRange"; }
@@ -248,8 +251,12 @@ struct GetMappedRangeWorkload : ApiWorkload {
 				     e.code() == error_code_quick_get_key_values_miss)) {
 					TraceEvent("GetMappedRangeWorkloadExpectedErrorDetected").error(e);
 					return MappedRangeResult();
+				} else if (e.code() == error_code_proxy_memory_limit_exceeded) {
+					// requests have overwhelmed commit proxy, rest a bit
+					wait(delay(FLOW_KNOBS->PREVENT_FAST_SPIN_DELAY));
+					continue;
 				} else {
-					std::cout << "error " << e.what() << std::endl;
+					std::cout << "error " << e.what() << "  code is " << e.code() << std::endl;
 					wait(tr->onError(e));
 				}
 				std::cout << "failed scanMappedRangeWithLimits" << std::endl;
@@ -257,6 +264,30 @@ struct GetMappedRangeWorkload : ApiWorkload {
 		}
 	}
 
+	// if sendFirstRequestIndefinitely is true, then this method would send the first request indefinitly
+	// it is in order to test the metric
+	ACTOR Future<Void> submitSmallRequestIndefinitely(Database cx,
+	                                                  int beginId,
+	                                                  int endId,
+	                                                  Key mapper,
+	                                                  GetMappedRangeWorkload* self) {
+		Key beginTuple = Tuple().append(prefix).append(INDEX).append(indexKey(beginId)).getDataAsStandalone();
+		state KeySelector beginSelector = KeySelector(firstGreaterOrEqual(beginTuple));
+		Key endTuple = Tuple().append(prefix).append(INDEX).append(indexKey(endId)).getDataAsStandalone();
+		state KeySelector endSelector = KeySelector(firstGreaterOrEqual(endTuple));
+		state int limit = 1;
+		state int byteLimit = 10000;
+		while (true) {
+			MappedRangeResult result = wait(self->scanMappedRangeWithLimits(
+			    cx, beginSelector, endSelector, mapper, limit, byteLimit, beginId, self));
+			if (result.empty()) {
+				TraceEvent("EmptyResult");
+			}
+			// to avoid requests make proxy memory overwhelmed
+			wait(delay(FLOW_KNOBS->PREVENT_FAST_SPIN_DELAY));
+		}
+	}
+
 	ACTOR Future<Void> scanMappedRange(Database cx, int beginId, int endId, Key mapper, GetMappedRangeWorkload* self) {
 		Key beginTuple = Tuple().append(prefix).append(INDEX).append(indexKey(beginId)).getDataAsStandalone();
 		state KeySelector beginSelector = KeySelector(firstGreaterOrEqual(beginTuple));
@@ -282,7 +313,6 @@ struct GetMappedRangeWorkload : ApiWorkload {
 					int indexByteLimit = byteLimit * SERVER_KNOBS->FRACTION_INDEX_BYTELIMIT_PREFETCH;
 					int indexCountByteLimit = indexByteLimit / indexSize + (indexByteLimit % indexSize != 0);
 					int indexCount = std::min(limit, indexCountByteLimit);
-					std::cout << "indexCount:  " << indexCount << std::endl;
 					// result set cannot be larger than the number of index fetched
 					ASSERT(result.size() <= indexCount);
 
@@ -298,7 +328,6 @@ struct GetMappedRangeWorkload : ApiWorkload {
 						boundByRecord = round * SERVER_KNOBS->MAX_PARALLEL_QUICK_GET_VALUE;
 					}
 					expectedCnt = std::min(expectedCnt, boundByRecord);
-					std::cout << "boundByRecord:  " << boundByRecord << std::endl;
 					ASSERT(result.size() == expectedCnt);
 					beginSelector = KeySelector(firstGreaterThan(result.back().key));
 				}
@@ -389,6 +418,46 @@ struct GetMappedRangeWorkload : ApiWorkload {
 		}
 	}
 
+	// checking the max storage queue length is bounded
+	ACTOR static Future<Void> reportMetric(GetMappedRangeWorkload* self, Database cx) {
+		loop {
+			StatusObject result = wait(StatusClient::statusFetcher(cx));
+			StatusObjectReader statusObj(result);
+			state StatusObjectReader statusObjCluster;
+			state StatusObjectReader processesMap;
+			state long queryQueueMax = 0;
+			state int waitInterval = 2;
+			if (!statusObj.get("cluster", statusObjCluster)) {
+				TraceEvent("NoCluster");
+				wait(delay(waitInterval));
+				continue;
+			}
+
+			if (!statusObjCluster.get("processes", processesMap)) {
+				TraceEvent("NoProcesses");
+				wait(delay(waitInterval));
+				continue;
+			}
+			for (auto proc : processesMap.obj()) {
+				StatusObjectReader process(proc.second);
+				if (process.has("roles")) {
+					StatusArray rolesArray = proc.second.get_obj()["roles"].get_array();
+					for (StatusObjectReader role : rolesArray) {
+						if (role["role"].get_str() == "storage") {
+							role.get("query_queue_max", queryQueueMax);
+							TEST(queryQueueMax > 0); // SS query queue is non-empty
+							ASSERT(queryQueueMax < 100); // if the queue size metric is wrong, it is likely above 100
+							TraceEvent(SevDebug, "QueryQueueMax").detail("Value", queryQueueMax);
+						}
+					}
+				} else {
+					TraceEvent("NoRoles");
+				}
+			}
+			wait(delay(waitInterval));
+		}
+	}
+
 	// If the same transaction writes to the read set (the scanned ranges) before reading, it should throw read your
 	// write exception.
 	ACTOR Future<Void> testRYW(GetMappedRangeWorkload* self) {
@@ -413,6 +482,27 @@ struct GetMappedRangeWorkload : ApiWorkload {
 		}
 	}
 
+	ACTOR static Future<Void> testMetric(Database cx,
+	                                     GetMappedRangeWorkload* self,
+	                                     int beginId,
+	                                     int endId,
+	                                     Key mapper,
+	                                     int seconds) {
+		loop choose {
+			when(wait(reportMetric(self, cx))) {
+				TraceEvent(SevError, "Error: ReportMetric has ended");
+				return Void();
+			}
+			when(wait(self->submitSmallRequestIndefinitely(cx, 10, 490, mapper, self))) {
+				TraceEvent(SevError, "Error: submitSmallRequestIndefinitely has ended");
+				return Void();
+			}
+			when(wait(delay(seconds))) {
+				return Void();
+			}
+		}
+	}
+
 	ACTOR Future<Void> _start(Database cx, GetMappedRangeWorkload* self) {
 		TraceEvent("GetMappedRangeWorkloadConfig").detail("BadMapper", self->BAD_MAPPER);
 
@@ -440,12 +530,13 @@ struct GetMappedRangeWorkload : ApiWorkload {
 		std::cout << "Test configuration: transactionType:" << self->transactionType << " snapshot:" << self->snapshot
 		          << "bad_mapper:" << self->BAD_MAPPER << std::endl;
 
-		Key mapper = getMapper(self);
+		state Key mapper = getMapper(self);
 		// The scanned range cannot be too large to hit get_mapped_key_values_has_more. We have a unit validating the
 		// error is thrown when the range is large.
 		state bool originalStrictlyEnforeByteLimit = SERVER_KNOBS->STRICTLY_ENFORCE_BYTE_LIMIT;
 		(const_cast<ServerKnobs*> SERVER_KNOBS)->STRICTLY_ENFORCE_BYTE_LIMIT = deterministicRandom()->coinflip();
 		wait(self->scanMappedRange(cx, 10, 490, mapper, self));
+		wait(testMetric(cx, self, 10, 490, mapper, self->checkStorageQueueSeconds));
 		(const_cast<ServerKnobs*> SERVER_KNOBS)->STRICTLY_ENFORCE_BYTE_LIMIT = originalStrictlyEnforeByteLimit;
 		return Void();
 	}
diff --git a/tests/fast/GetMappedRange.toml b/tests/fast/GetMappedRange.toml
index 2dd4672592a..93b2b8a26b1 100644
--- a/tests/fast/GetMappedRange.toml
+++ b/tests/fast/GetMappedRange.toml
@@ -4,3 +4,4 @@ useDB = true
 
     [[test.workload]]
     testName = 'GetMappedRange'
+    checkStorageQueueSeconds=60
\ No newline at end of file

From 4537535eb5bae1abe3c85886f3993eb76ab08606 Mon Sep 17 00:00:00 2001
From: neethuhaneesha <nbingi@apple.com>
Date: Thu, 23 Mar 2023 12:39:53 -0700
Subject: [PATCH 117/468] Adding rocksdb bloom filter knobs. (#9770)

---
 fdbclient/ServerKnobs.cpp                |  4 ++++
 fdbclient/ServerKnobs.h                  |  4 ++++
 fdbserver/KeyValueStoreRocksDB.actor.cpp | 21 ++++++++++++++++-----
 3 files changed, 24 insertions(+), 5 deletions(-)

diff --git a/fdbclient/ServerKnobs.cpp b/fdbclient/ServerKnobs.cpp
index c9358ae66f5..18e24741605 100644
--- a/fdbclient/ServerKnobs.cpp
+++ b/fdbclient/ServerKnobs.cpp
@@ -377,6 +377,10 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi
 	init( ROCKSDB_UNSAFE_AUTO_FSYNC,                           false );
 	init( ROCKSDB_PERIODIC_COMPACTION_SECONDS,                     0 );
 	init( ROCKSDB_PREFIX_LEN,                                      0 );
+	init( ROCKSDB_MEMTABLE_PREFIX_BLOOM_SIZE_RATIO,              0.1 );
+	init( ROCKSDB_BLOOM_BITS_PER_KEY,                             10 );
+	init( ROCKSDB_BLOOM_WHOLE_KEY_FILTERING,                   false );
+	init( ROCKSDB_MAX_AUTO_READAHEAD_SIZE,                         0 );
 	// If rocksdb block cache size is 0, the default 8MB is used.
 	int64_t blockCacheSize = isSimulated ? 16 * 1024 * 1024 : 2147483648 /* 2GB */;
 	init( ROCKSDB_BLOCK_CACHE_SIZE,                   blockCacheSize );
diff --git a/fdbclient/ServerKnobs.h b/fdbclient/ServerKnobs.h
index 33fdcaf4f37..529e306dc5b 100644
--- a/fdbclient/ServerKnobs.h
+++ b/fdbclient/ServerKnobs.h
@@ -306,6 +306,10 @@ class ServerKnobs : public KnobsImpl<ServerKnobs> {
 	bool ROCKSDB_UNSAFE_AUTO_FSYNC;
 	int64_t ROCKSDB_PERIODIC_COMPACTION_SECONDS;
 	int ROCKSDB_PREFIX_LEN;
+	double ROCKSDB_MEMTABLE_PREFIX_BLOOM_SIZE_RATIO;
+	int ROCKSDB_BLOOM_BITS_PER_KEY;
+	bool ROCKSDB_BLOOM_WHOLE_KEY_FILTERING;
+	int ROCKSDB_MAX_AUTO_READAHEAD_SIZE;
 	int64_t ROCKSDB_BLOCK_CACHE_SIZE;
 	double ROCKSDB_METRICS_DELAY;
 	double ROCKSDB_READ_VALUE_TIMEOUT;
diff --git a/fdbserver/KeyValueStoreRocksDB.actor.cpp b/fdbserver/KeyValueStoreRocksDB.actor.cpp
index 6da3467427b..663681043cf 100644
--- a/fdbserver/KeyValueStoreRocksDB.actor.cpp
+++ b/fdbserver/KeyValueStoreRocksDB.actor.cpp
@@ -340,8 +340,7 @@ rocksdb::ColumnFamilyOptions getCFOptions() {
 		options.prefix_extractor.reset(rocksdb::NewFixedPrefixTransform(SERVER_KNOBS->ROCKSDB_PREFIX_LEN));
 
 		// Also turn on bloom filters in the memtable.
-		// TODO: Make a knob for this as well.
-		options.memtable_prefix_bloom_size_ratio = 0.1;
+		options.memtable_prefix_bloom_size_ratio = SERVER_KNOBS->ROCKSDB_MEMTABLE_PREFIX_BLOOM_SIZE_RATIO;
 
 		// 5 -- Can be read by RocksDB's versions since 6.6.0. Full and partitioned
 		// filters use a generally faster and more accurate Bloom filter
@@ -352,11 +351,11 @@ rocksdb::ColumnFamilyOptions getCFOptions() {
 		// Create and apply a bloom filter using the 10 bits
 		// which should yield a ~1% false positive rate:
 		// https://github.com/facebook/rocksdb/wiki/RocksDB-Bloom-Filter#full-filters-new-format
-		bbOpts.filter_policy.reset(rocksdb::NewBloomFilterPolicy(10));
+		bbOpts.filter_policy.reset(rocksdb::NewBloomFilterPolicy(SERVER_KNOBS->ROCKSDB_BLOOM_BITS_PER_KEY));
 
 		// The whole key blooms are only used for point lookups.
 		// https://github.com/facebook/rocksdb/wiki/RocksDB-Bloom-Filter#prefix-vs-whole-key
-		bbOpts.whole_key_filtering = false;
+		bbOpts.whole_key_filtering = SERVER_KNOBS->ROCKSDB_BLOOM_WHOLE_KEY_FILTERING;
 	}
 
 	if (SERVER_KNOBS->ROCKSDB_BLOCK_CACHE_SIZE > 0) {
@@ -369,6 +368,12 @@ rocksdb::ColumnFamilyOptions getCFOptions() {
 		bbOpts.block_size = SERVER_KNOBS->ROCKSDB_BLOCK_SIZE;
 	}
 
+	// The readahead size starts with 8KB and is exponentially increased on each additional sequential IO,
+	// up to a max of BlockBasedTableOptions.max_auto_readahead_size (default 256 KB)
+	if (SERVER_KNOBS->ROCKSDB_MAX_AUTO_READAHEAD_SIZE > 0) {
+		bbOpts.max_auto_readahead_size = SERVER_KNOBS->ROCKSDB_MAX_AUTO_READAHEAD_SIZE;
+	}
+
 	options.table_factory.reset(rocksdb::NewBlockBasedTableFactory(bbOpts));
 
 	return options;
@@ -411,13 +416,14 @@ struct Counters {
 	Counter convertedDeleteKeyReqs;
 	Counter convertedDeleteRangeReqs;
 	Counter rocksdbReadRangeQueries;
+	Counter commitDelayed;
 
 	Counters()
 	  : cc("RocksDBThrottle"), immediateThrottle("ImmediateThrottle", cc), failedToAcquire("FailedToAcquire", cc),
 	    deleteKeyReqs("DeleteKeyRequests", cc), deleteRangeReqs("DeleteRangeRequests", cc),
 	    convertedDeleteKeyReqs("ConvertedDeleteKeyRequests", cc),
 	    convertedDeleteRangeReqs("ConvertedDeleteRangeRequests", cc),
-	    rocksdbReadRangeQueries("RocksdbReadRangeQueries", cc) {}
+	    rocksdbReadRangeQueries("RocksdbReadRangeQueries", cc), commitDelayed("CommitDelayed", cc) {}
 };
 
 struct ReadIterator {
@@ -935,6 +941,8 @@ ACTOR Future<Void> rocksDBMetricLogger(UID id,
 		{ "RowCacheHit", rocksdb::ROW_CACHE_HIT, 0 },
 		{ "RowCacheMiss", rocksdb::ROW_CACHE_MISS, 0 },
 		{ "CountIterSkippedKeys", rocksdb::NUMBER_ITER_SKIP, 0 },
+		{ "NoIteratorCreated", rocksdb::NO_ITERATOR_CREATED, 0 },
+		{ "NoIteratorDeleted", rocksdb::NO_ITERATOR_DELETED, 0 },
 	};
 
 	// To control the rocksdb::StatsLevel, use ROCKSDB_STATS_LEVEL knob.
@@ -2119,9 +2127,12 @@ struct RocksDBKeyValueStore : IKeyValueStore {
 		                                   &estPendCompactBytes);
 		while (count && estPendCompactBytes > SERVER_KNOBS->ROCKSDB_CAN_COMMIT_COMPACT_BYTES_LIMIT) {
 			wait(delay(SERVER_KNOBS->ROCKSDB_CAN_COMMIT_DELAY_ON_OVERLOAD));
+			++self->counters.commitDelayed;
 			count--;
 			self->db->GetAggregatedIntProperty(rocksdb::DB::Properties::kEstimatePendingCompactionBytes,
 			                                   &estPendCompactBytes);
+			if (deterministicRandom()->random01() < 0.001)
+				TraceEvent(SevWarn, "RocksDBCommitsDelayed1000x", self->id);
 		}
 
 		return Void();

From 0bfab1f2e1b99533946f652d116ff0060ce0eaf9 Mon Sep 17 00:00:00 2001
From: Zhe Wu <halfprice@users.noreply.github.com>
Date: Mon, 27 Mar 2023 15:51:42 -0700
Subject: [PATCH 118/468] [Release 7.1] Do not update exclude/failed system
 metadata in excludeServers if the input list is already excluded/failed
 (#9809)

* Add a check in excludeServer function that if the exclusion list already exists, don't need to issue new writes.

* Update documentation
---
 fdbclient/ManagementAPI.actor.cpp             | 72 +++++++++++++------
 fdbclient/ManagementAPI.actor.h               | 14 ++--
 fdbclient/SpecialKeySpace.actor.cpp           |  4 +-
 .../workloads/DataLossRecovery.actor.cpp      |  2 +-
 4 files changed, 65 insertions(+), 27 deletions(-)

diff --git a/fdbclient/ManagementAPI.actor.cpp b/fdbclient/ManagementAPI.actor.cpp
index ff06b3e18ac..938afc9118a 100644
--- a/fdbclient/ManagementAPI.actor.cpp
+++ b/fdbclient/ManagementAPI.actor.cpp
@@ -1124,7 +1124,7 @@ struct AutoQuorumChange final : IQuorumChange {
 			desiredCount = redundancy * 2 - 1;
 		}
 
-		std::vector<AddressExclusion> excl = wait(getExcludedServers(tr));
+		std::vector<AddressExclusion> excl = wait(getAllExcludedServers(tr));
 		state std::set<AddressExclusion> excluded(excl.begin(), excl.end());
 
 		std::vector<ProcessData> _workers = wait(getWorkers(tr));
@@ -1273,23 +1273,37 @@ Reference<IQuorumChange> autoQuorumChange(int desired) {
 	return Reference<IQuorumChange>(new AutoQuorumChange(desired));
 }
 
-void excludeServers(Transaction& tr, std::vector<AddressExclusion>& servers, bool failed) {
-	tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
-	tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
-	tr.setOption(FDBTransactionOptions::LOCK_AWARE);
-	tr.setOption(FDBTransactionOptions::USE_PROVISIONAL_PROXIES);
-	std::string excludeVersionKey = deterministicRandom()->randomUniqueID().toString();
-	auto serversVersionKey = failed ? failedServersVersionKey : excludedServersVersionKey;
-	tr.addReadConflictRange(singleKeyRange(serversVersionKey)); // To conflict with parallel includeServers
-	tr.set(serversVersionKey, excludeVersionKey);
+ACTOR Future<Void> excludeServers(Transaction* tr, std::vector<AddressExclusion> servers, bool failed) {
+	tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
+	tr->setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
+	tr->setOption(FDBTransactionOptions::LOCK_AWARE);
+	tr->setOption(FDBTransactionOptions::USE_PROVISIONAL_PROXIES);
+	std::vector<AddressExclusion> excl = wait(failed ? getExcludedFailedServerList(tr) : getExcludedServerList(tr));
+	std::set<AddressExclusion> exclusions(excl.begin(), excl.end());
+	bool containNewExclusion = false;
 	for (auto& s : servers) {
+		if (exclusions.find(s) != exclusions.end()) {
+			continue;
+		}
+		containNewExclusion = true;
 		if (failed) {
-			tr.set(encodeFailedServersKey(s), StringRef());
+			tr->set(encodeFailedServersKey(s), StringRef());
 		} else {
-			tr.set(encodeExcludedServersKey(s), StringRef());
+			tr->set(encodeExcludedServersKey(s), StringRef());
 		}
 	}
-	TraceEvent("ExcludeServersCommit").detail("Servers", describe(servers)).detail("ExcludeFailed", failed);
+
+	if (containNewExclusion) {
+		std::string excludeVersionKey = deterministicRandom()->randomUniqueID().toString();
+		auto serversVersionKey = failed ? failedServersVersionKey : excludedServersVersionKey;
+		tr->addReadConflictRange(singleKeyRange(serversVersionKey)); // To conflict with parallel includeServers
+		tr->set(serversVersionKey, excludeVersionKey);
+	}
+	TraceEvent("ExcludeServersCommit")
+	    .detail("Servers", describe(servers))
+	    .detail("ExcludeFailed", failed)
+	    .detail("ExclusionUpdated", containNewExclusion);
+	return Void();
 }
 
 ACTOR Future<Void> excludeServers(Database cx, std::vector<AddressExclusion> servers, bool failed) {
@@ -1321,7 +1335,7 @@ ACTOR Future<Void> excludeServers(Database cx, std::vector<AddressExclusion> ser
 		state Transaction tr(cx);
 		loop {
 			try {
-				excludeServers(tr, servers, failed);
+				wait(excludeServers(&tr, servers, failed));
 				wait(tr.commit());
 				return Void();
 			} catch (Error& e) {
@@ -1333,6 +1347,7 @@ ACTOR Future<Void> excludeServers(Database cx, std::vector<AddressExclusion> ser
 }
 
 // excludes localities by setting the keys in api version below 7.0
+// TODO(zhewu): update this function that if the locality is already excluded, don't need to update the system metadata.
 void excludeLocalities(Transaction& tr, std::unordered_set<std::string> localities, bool failed) {
 	tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 	tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
@@ -1617,11 +1632,9 @@ ACTOR Future<Void> setClass(Database cx, AddressExclusion server, ProcessClass p
 	}
 }
 
-ACTOR Future<std::vector<AddressExclusion>> getExcludedServers(Transaction* tr) {
+ACTOR Future<std::vector<AddressExclusion>> getExcludedServerList(Transaction* tr) {
 	state RangeResult r = wait(tr->getRange(excludedServersKeys, CLIENT_KNOBS->TOO_MANY));
 	ASSERT(!r.more && r.size() < CLIENT_KNOBS->TOO_MANY);
-	state RangeResult r2 = wait(tr->getRange(failedServersKeys, CLIENT_KNOBS->TOO_MANY));
-	ASSERT(!r2.more && r2.size() < CLIENT_KNOBS->TOO_MANY);
 
 	std::vector<AddressExclusion> exclusions;
 	for (auto i = r.begin(); i != r.end(); ++i) {
@@ -1629,7 +1642,16 @@ ACTOR Future<std::vector<AddressExclusion>> getExcludedServers(Transaction* tr)
 		if (a.isValid())
 			exclusions.push_back(a);
 	}
-	for (auto i = r2.begin(); i != r2.end(); ++i) {
+	uniquify(exclusions);
+	return exclusions;
+}
+
+ACTOR Future<std::vector<AddressExclusion>> getExcludedFailedServerList(Transaction* tr) {
+	state RangeResult r = wait(tr->getRange(failedServersKeys, CLIENT_KNOBS->TOO_MANY));
+	ASSERT(!r.more && r.size() < CLIENT_KNOBS->TOO_MANY);
+
+	std::vector<AddressExclusion> exclusions;
+	for (auto i = r.begin(); i != r.end(); ++i) {
 		auto a = decodeFailedServersKey(i->key);
 		if (a.isValid())
 			exclusions.push_back(a);
@@ -1638,14 +1660,24 @@ ACTOR Future<std::vector<AddressExclusion>> getExcludedServers(Transaction* tr)
 	return exclusions;
 }
 
-ACTOR Future<std::vector<AddressExclusion>> getExcludedServers(Database cx) {
+ACTOR Future<std::vector<AddressExclusion>> getAllExcludedServers(Transaction* tr) {
+	state std::vector<AddressExclusion> exclusions;
+	std::vector<AddressExclusion> excludedServers = wait(getExcludedServerList(tr));
+	exclusions.insert(exclusions.end(), excludedServers.begin(), excludedServers.end());
+	std::vector<AddressExclusion> excludedFailed = wait(getExcludedFailedServerList(tr));
+	exclusions.insert(exclusions.end(), excludedFailed.begin(), excludedFailed.end());
+	uniquify(exclusions);
+	return exclusions;
+}
+
+ACTOR Future<std::vector<AddressExclusion>> getAllExcludedServers(Database cx) {
 	state Transaction tr(cx);
 	loop {
 		try {
 			tr.setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
 			tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE); // necessary?
 			tr.setOption(FDBTransactionOptions::LOCK_AWARE);
-			std::vector<AddressExclusion> exclusions = wait(getExcludedServers(&tr));
+			std::vector<AddressExclusion> exclusions = wait(getAllExcludedServers(&tr));
 			return exclusions;
 		} catch (Error& e) {
 			wait(tr.onError(e));
diff --git a/fdbclient/ManagementAPI.actor.h b/fdbclient/ManagementAPI.actor.h
index 5ddaf2ffbd3..ef6227825f6 100644
--- a/fdbclient/ManagementAPI.actor.h
+++ b/fdbclient/ManagementAPI.actor.h
@@ -65,7 +65,7 @@ Reference<IQuorumChange> nameQuorumChange(std::string const& name, Reference<IQu
 // necessarily waiting for the servers to be evacuated.  A NetworkAddress with a port of 0 means all servers on the
 // given IP.
 ACTOR Future<Void> excludeServers(Database cx, std::vector<AddressExclusion> servers, bool failed = false);
-void excludeServers(Transaction& tr, std::vector<AddressExclusion>& servers, bool failed = false);
+ACTOR Future<Void> excludeServers(Transaction* tr, std::vector<AddressExclusion> servers, bool failed = false);
 
 // Exclude the servers matching the given set of localities from use as state servers.  Returns as soon as the change
 // is durable, without necessarily waiting for the servers to be evacuated.
@@ -86,9 +86,15 @@ ACTOR Future<Void> includeLocalities(Database cx,
 // the given IP.
 ACTOR Future<Void> setClass(Database cx, AddressExclusion server, ProcessClass processClass);
 
-// Get the current list of excluded servers
-ACTOR Future<std::vector<AddressExclusion>> getExcludedServers(Database cx);
-ACTOR Future<std::vector<AddressExclusion>> getExcludedServers(Transaction* tr);
+// Get the current list of excluded servers including both "exclude" and "failed".
+ACTOR Future<std::vector<AddressExclusion>> getAllExcludedServers(Database cx);
+ACTOR Future<std::vector<AddressExclusion>> getAllExcludedServers(Transaction* tr);
+
+// Get the current list of excluded servers.
+ACTOR Future<std::vector<AddressExclusion>> getExcludedServerList(Transaction* tr);
+
+// Get the current list of failed servers.
+ACTOR Future<std::vector<AddressExclusion>> getExcludedFailedServerList(Transaction* tr);
 
 // Get the current list of excluded localities
 ACTOR Future<std::vector<std::string>> getExcludedLocalities(Database cx);
diff --git a/fdbclient/SpecialKeySpace.actor.cpp b/fdbclient/SpecialKeySpace.actor.cpp
index 1f86beb3281..dd23ba0da6e 100644
--- a/fdbclient/SpecialKeySpace.actor.cpp
+++ b/fdbclient/SpecialKeySpace.actor.cpp
@@ -1122,7 +1122,7 @@ ACTOR Future<Optional<std::string>> excludeCommitActor(ReadYourWritesTransaction
 		if (!safe)
 			return result;
 	}
-	excludeServers(ryw->getTransaction(), addresses, failed);
+	wait(excludeServers(&(ryw->getTransaction()), addresses, failed));
 	includeServers(ryw);
 
 	return result;
@@ -1167,7 +1167,7 @@ ACTOR Future<RangeResult> ExclusionInProgressActor(ReadYourWritesTransaction* ry
 	tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE); // necessary?
 	tr.setOption(FDBTransactionOptions::LOCK_AWARE);
 
-	state std::vector<AddressExclusion> excl = wait((getExcludedServers(&tr)));
+	state std::vector<AddressExclusion> excl = wait((getAllExcludedServers(&tr)));
 	state std::set<AddressExclusion> exclusions(excl.begin(), excl.end());
 	state std::set<NetworkAddress> inProgressExclusion;
 	// Just getting a consistent read version proves that a set of tlogs satisfying the exclusions has completed
diff --git a/fdbserver/workloads/DataLossRecovery.actor.cpp b/fdbserver/workloads/DataLossRecovery.actor.cpp
index dce83880804..dcb34b4435b 100644
--- a/fdbserver/workloads/DataLossRecovery.actor.cpp
+++ b/fdbserver/workloads/DataLossRecovery.actor.cpp
@@ -152,7 +152,7 @@ struct DataLossRecoveryWorkload : TestWorkload {
 		servers.push_back(AddressExclusion(addr.ip, addr.port));
 		loop {
 			try {
-				excludeServers(tr, servers, true);
+				wait(excludeServers(&tr, servers, true));
 				wait(tr.commit());
 				break;
 			} catch (Error& e) {

From 50816eb642920b49fa8769fbcaf9f0ff1cdbd67e Mon Sep 17 00:00:00 2001
From: Hao Fu <77984096+hfu94@users.noreply.github.com>
Date: Mon, 27 Mar 2023 20:32:11 -0700
Subject: [PATCH 119/468] Parameterized queue length in GetMappedRange test
 (#9808)

Also retry when operation_cancelled happens
---
 fdbserver/workloads/GetMappedRange.actor.cpp | 13 ++++++++-----
 tests/fast/GetMappedRange.toml               |  3 ++-
 2 files changed, 10 insertions(+), 6 deletions(-)

diff --git a/fdbserver/workloads/GetMappedRange.actor.cpp b/fdbserver/workloads/GetMappedRange.actor.cpp
index 6c77f58c2c5..b735b3748d7 100644
--- a/fdbserver/workloads/GetMappedRange.actor.cpp
+++ b/fdbserver/workloads/GetMappedRange.actor.cpp
@@ -50,11 +50,13 @@ struct GetMappedRangeWorkload : ApiWorkload {
 	//	const bool SPLIT_RECORDS = deterministicRandom()->random01() < 0.5;
 	const bool SPLIT_RECORDS = true;
 	const static int SPLIT_SIZE = 3;
-	int checkStorageQueueSeconds;
+	double checkStorageQueueSeconds;
+	double queueMaxLength;
 
 	GetMappedRangeWorkload(WorkloadContext const& wcx) : ApiWorkload(wcx) {
 		enabled = !clientId; // only do this on the "first" client
-		checkStorageQueueSeconds = getOption(options, "checkStorageQueueSeconds"_sr, 30.0);
+		checkStorageQueueSeconds = getOption(options, "checkStorageQueueSeconds"_sr, 60.0);
+		queueMaxLength = getOption(options, "queueMaxLength"_sr, 200);
 	}
 
 	std::string description() const override { return "GetMappedRange"; }
@@ -251,7 +253,8 @@ struct GetMappedRangeWorkload : ApiWorkload {
 				     e.code() == error_code_quick_get_key_values_miss)) {
 					TraceEvent("GetMappedRangeWorkloadExpectedErrorDetected").error(e);
 					return MappedRangeResult();
-				} else if (e.code() == error_code_proxy_memory_limit_exceeded) {
+				} else if (e.code() == error_code_proxy_memory_limit_exceeded ||
+				           e.code() == error_code_operation_cancelled) {
 					// requests have overwhelmed commit proxy, rest a bit
 					wait(delay(FLOW_KNOBS->PREVENT_FAST_SPIN_DELAY));
 					continue;
@@ -425,7 +428,7 @@ struct GetMappedRangeWorkload : ApiWorkload {
 			StatusObjectReader statusObj(result);
 			state StatusObjectReader statusObjCluster;
 			state StatusObjectReader processesMap;
-			state long queryQueueMax = 0;
+			state int64_t queryQueueMax = 0;
 			state int waitInterval = 2;
 			if (!statusObj.get("cluster", statusObjCluster)) {
 				TraceEvent("NoCluster");
@@ -446,8 +449,8 @@ struct GetMappedRangeWorkload : ApiWorkload {
 						if (role["role"].get_str() == "storage") {
 							role.get("query_queue_max", queryQueueMax);
 							TEST(queryQueueMax > 0); // SS query queue is non-empty
-							ASSERT(queryQueueMax < 100); // if the queue size metric is wrong, it is likely above 100
 							TraceEvent(SevDebug, "QueryQueueMax").detail("Value", queryQueueMax);
+							ASSERT(queryQueueMax < self->queueMaxLength);
 						}
 					}
 				} else {
diff --git a/tests/fast/GetMappedRange.toml b/tests/fast/GetMappedRange.toml
index 93b2b8a26b1..36067edd063 100644
--- a/tests/fast/GetMappedRange.toml
+++ b/tests/fast/GetMappedRange.toml
@@ -4,4 +4,5 @@ useDB = true
 
     [[test.workload]]
     testName = 'GetMappedRange'
-    checkStorageQueueSeconds=60
\ No newline at end of file
+    checkStorageQueueSeconds=60
+    queueMaxLength=200
\ No newline at end of file

From 3e641a8e9f565e7ca6775e67779aec5511d57f16 Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyu_zhou@apple.com>
Date: Wed, 29 Mar 2023 09:00:21 -0700
Subject: [PATCH 120/468] Add 7.1.30, 7.1.31 release notes (#9822)

---
 .../source/release-notes/release-notes-710.rst       | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/documentation/sphinx/source/release-notes/release-notes-710.rst b/documentation/sphinx/source/release-notes/release-notes-710.rst
index b2922f80818..e08c124910a 100644
--- a/documentation/sphinx/source/release-notes/release-notes-710.rst
+++ b/documentation/sphinx/source/release-notes/release-notes-710.rst
@@ -4,6 +4,18 @@
 Release Notes
 #############
 
+7.1.31
+======
+* Same as 7.1.30 release with AVX enabled.
+
+7.1.30
+======
+* Released with AVX disabled.
+* Fixed storage server finishedQueries metric when using getMappedRange. `(PR #9785) <https://github.com/apple/foundationdb/pull/9785>`_
+* Fixed unnecessary transaction system recovery when excluding the servers that are already excluded/failed. `(PR #9809) <https://github.com/apple/foundationdb/pull/9809>`_
+* Fixed the exclusion of stateless processes by skipping the free capacity check. `(PR #9789) <https://github.com/apple/foundationdb/pull/9789>`_
+* Fixed an issue where the new worker cannot get ServerDBInfo update. `(PR #9778) <https://github.com/apple/foundationdb/pull/9778>`_
+
 7.1.29
 ======
 * Same as 7.1.28 release with AVX enabled.

From 21b1cdff1d27e7cb1bee73a2c29d597995ab1858 Mon Sep 17 00:00:00 2001
From: Johannes Scheuermann <johscheuer@users.noreply.github.com>
Date: Sun, 2 Apr 2023 18:14:59 +0100
Subject: [PATCH 121/468] Don't stop iterating over all storage processes in
 exclusion check (#9869)

---
 fdbclient/SpecialKeySpace.actor.cpp | 1 -
 1 file changed, 1 deletion(-)

diff --git a/fdbclient/SpecialKeySpace.actor.cpp b/fdbclient/SpecialKeySpace.actor.cpp
index dd23ba0da6e..1e2c0beb5c9 100644
--- a/fdbclient/SpecialKeySpace.actor.cpp
+++ b/fdbclient/SpecialKeySpace.actor.cpp
@@ -1009,7 +1009,6 @@ ACTOR Future<bool> checkExclusion(Database db,
 					// capacity if we are excluding at least one process that serves the storage role.
 					if (!excludedAddressesContainsStorageRole && includedInExclusion) {
 						excludedAddressesContainsStorageRole = true;
-						break;
 					}
 
 					int64_t used_bytes;

From d4ffd358d676869e144aff38565406291138021d Mon Sep 17 00:00:00 2001
From: Zhe Wu <halfprice@users.noreply.github.com>
Date: Sun, 2 Apr 2023 10:15:35 -0700
Subject: [PATCH 122/468] checkSafeExclusion should always create new
 ExclusionSafetyCheckRequest (#9871)

---
 fdbclient/NativeAPI.actor.cpp | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/fdbclient/NativeAPI.actor.cpp b/fdbclient/NativeAPI.actor.cpp
index cd6865028bd..9bbd0e70d0d 100644
--- a/fdbclient/NativeAPI.actor.cpp
+++ b/fdbclient/NativeAPI.actor.cpp
@@ -8264,7 +8264,6 @@ ACTOR Future<bool> checkSafeExclusions(Database cx, std::vector<AddressExclusion
 	TraceEvent("ExclusionSafetyCheckBegin")
 	    .detail("NumExclusion", exclusions.size())
 	    .detail("Exclusions", describe(exclusions));
-	state ExclusionSafetyCheckRequest req(exclusions);
 	state bool ddCheck;
 	try {
 		loop {
@@ -8273,7 +8272,7 @@ ACTOR Future<bool> checkSafeExclusions(Database cx, std::vector<AddressExclusion
 				when(ExclusionSafetyCheckReply _ddCheck =
 				         wait(basicLoadBalance(cx->getCommitProxies(UseProvisionalProxies::False),
 				                               &CommitProxyInterface::exclusionSafetyCheckReq,
-				                               req,
+				                               ExclusionSafetyCheckRequest(exclusions),
 				                               cx->taskID))) {
 					ddCheck = _ddCheck.safe;
 					break;

From 48087478044c92bd59c9fcb5ff7f935045ed2bf9 Mon Sep 17 00:00:00 2001
From: neethuhaneesha <nbingi@apple.com>
Date: Mon, 3 Apr 2023 12:49:25 -0700
Subject: [PATCH 123/468] RocksDB 7.10.2 version upgrade (#9829)

---
 cmake/CompileRocksDB.cmake                    |   6 +-
 .../release-notes/release-notes-710.rst       |   2 +
 fdbserver/KeyValueStoreRocksDB.actor.cpp      |  40 +++---
 fdbserver/RocksDBCheckpointUtils.actor.h      | 127 +++++++++++-------
 4 files changed, 106 insertions(+), 69 deletions(-)

diff --git a/cmake/CompileRocksDB.cmake b/cmake/CompileRocksDB.cmake
index 62405d04df9..b1f5d8e8e35 100644
--- a/cmake/CompileRocksDB.cmake
+++ b/cmake/CompileRocksDB.cmake
@@ -1,6 +1,6 @@
 # FindRocksDB
 
-find_package(RocksDB 7.7.3)
+find_package(RocksDB 7.10.2)
 
 include(ExternalProject)
 
@@ -54,8 +54,8 @@ if(ROCKSDB_FOUND)
     ${BINARY_DIR}/librocksdb.a)
 else()
   ExternalProject_Add(rocksdb
-    URL        https://github.com/facebook/rocksdb/archive/refs/tags/v7.7.3.tar.gz
-    URL_HASH   SHA256=b8ac9784a342b2e314c821f6d701148912215666ac5e9bdbccd93cf3767cb611
+    URL https://github.com/facebook/rocksdb/archive/refs/tags/v7.10.2.tar.gz
+    URL_HASH SHA256=4619ae7308cd3d11cdd36f0bfad3fb03a1ad399ca333f192b77b6b95b08e2f78
     CMAKE_ARGS ${RocksDB_CMAKE_ARGS}
     BUILD_BYPRODUCTS <BINARY_DIR>/librocksdb.a
     INSTALL_COMMAND ""
diff --git a/documentation/sphinx/source/release-notes/release-notes-710.rst b/documentation/sphinx/source/release-notes/release-notes-710.rst
index e08c124910a..621fce54b6d 100644
--- a/documentation/sphinx/source/release-notes/release-notes-710.rst
+++ b/documentation/sphinx/source/release-notes/release-notes-710.rst
@@ -15,6 +15,8 @@ Release Notes
 * Fixed unnecessary transaction system recovery when excluding the servers that are already excluded/failed. `(PR #9809) <https://github.com/apple/foundationdb/pull/9809>`_
 * Fixed the exclusion of stateless processes by skipping the free capacity check. `(PR #9789) <https://github.com/apple/foundationdb/pull/9789>`_
 * Fixed an issue where the new worker cannot get ServerDBInfo update. `(PR #9778) <https://github.com/apple/foundationdb/pull/9778>`_
+* Added RocksDB bloom filter knobs. `(PR #9770) <https://github.com/apple/foundationdb/pull/9770>`_
+* Upgraded RocksDB to version 7.10.2. `(PR #9829) <https://github.com/apple/foundationdb/pull/9829>`_
 
 7.1.29
 ======
diff --git a/fdbserver/KeyValueStoreRocksDB.actor.cpp b/fdbserver/KeyValueStoreRocksDB.actor.cpp
index 663681043cf..067726e7e86 100644
--- a/fdbserver/KeyValueStoreRocksDB.actor.cpp
+++ b/fdbserver/KeyValueStoreRocksDB.actor.cpp
@@ -68,9 +68,9 @@
 
 #ifdef SSD_ROCKSDB_EXPERIMENTAL
 
-// Enforcing rocksdb version to be 7.7.3.
-static_assert((ROCKSDB_MAJOR == 7 && ROCKSDB_MINOR == 7 && ROCKSDB_PATCH == 3),
-              "Unsupported rocksdb version. Update the rocksdb to 7.7.3 version");
+// Enforcing rocksdb version to be 7.10.2
+static_assert((ROCKSDB_MAJOR == 7 && ROCKSDB_MINOR == 10 && ROCKSDB_PATCH == 2),
+              "Unsupported rocksdb version. Update the rocksdb to 7.10.2 version");
 
 namespace {
 using rocksdb::BackgroundErrorReason;
@@ -226,10 +226,14 @@ rocksdb::ExportImportFilesMetaData getMetaData(const CheckpointMetaData& checkpo
 
 	for (const LiveFileMetaData& fileMetaData : rocksCF.sstFiles) {
 		rocksdb::LiveFileMetaData liveFileMetaData;
-		liveFileMetaData.size = fileMetaData.size;
-		liveFileMetaData.name = fileMetaData.name;
+		liveFileMetaData.relative_filename = fileMetaData.relative_filename;
+		liveFileMetaData.directory = fileMetaData.directory;
 		liveFileMetaData.file_number = fileMetaData.file_number;
-		liveFileMetaData.db_path = fileMetaData.db_path;
+		liveFileMetaData.file_type = static_cast<rocksdb::FileType>(fileMetaData.file_type);
+		liveFileMetaData.size = fileMetaData.size;
+		liveFileMetaData.temperature = static_cast<rocksdb::Temperature>(fileMetaData.temperature);
+		liveFileMetaData.file_checksum = fileMetaData.file_checksum;
+		liveFileMetaData.file_checksum_func_name = fileMetaData.file_checksum_func_name;
 		liveFileMetaData.smallest_seqno = fileMetaData.smallest_seqno;
 		liveFileMetaData.largest_seqno = fileMetaData.largest_seqno;
 		liveFileMetaData.smallestkey = fileMetaData.smallestkey;
@@ -238,12 +242,12 @@ rocksdb::ExportImportFilesMetaData getMetaData(const CheckpointMetaData& checkpo
 		liveFileMetaData.being_compacted = fileMetaData.being_compacted;
 		liveFileMetaData.num_entries = fileMetaData.num_entries;
 		liveFileMetaData.num_deletions = fileMetaData.num_deletions;
-		liveFileMetaData.temperature = static_cast<rocksdb::Temperature>(fileMetaData.temperature);
 		liveFileMetaData.oldest_blob_file_number = fileMetaData.oldest_blob_file_number;
 		liveFileMetaData.oldest_ancester_time = fileMetaData.oldest_ancester_time;
 		liveFileMetaData.file_creation_time = fileMetaData.file_creation_time;
-		liveFileMetaData.file_checksum = fileMetaData.file_checksum;
-		liveFileMetaData.file_checksum_func_name = fileMetaData.file_checksum_func_name;
+		liveFileMetaData.epoch_number = fileMetaData.epoch_number;
+		liveFileMetaData.name = fileMetaData.name;
+		liveFileMetaData.db_path = fileMetaData.db_path;
 		liveFileMetaData.column_family_name = fileMetaData.column_family_name;
 		liveFileMetaData.level = fileMetaData.level;
 		metaData.files.push_back(liveFileMetaData);
@@ -257,10 +261,14 @@ void populateMetaData(CheckpointMetaData* checkpoint, const rocksdb::ExportImpor
 	rocksCF.dbComparatorName = metaData.db_comparator_name;
 	for (const rocksdb::LiveFileMetaData& fileMetaData : metaData.files) {
 		LiveFileMetaData liveFileMetaData;
-		liveFileMetaData.size = fileMetaData.size;
-		liveFileMetaData.name = fileMetaData.name;
+		liveFileMetaData.relative_filename = fileMetaData.relative_filename;
+		liveFileMetaData.directory = fileMetaData.directory;
 		liveFileMetaData.file_number = fileMetaData.file_number;
-		liveFileMetaData.db_path = fileMetaData.db_path;
+		liveFileMetaData.file_type = static_cast<int>(fileMetaData.file_type);
+		liveFileMetaData.size = fileMetaData.size;
+		liveFileMetaData.temperature = static_cast<uint8_t>(fileMetaData.temperature);
+		liveFileMetaData.file_checksum = fileMetaData.file_checksum;
+		liveFileMetaData.file_checksum_func_name = fileMetaData.file_checksum_func_name;
 		liveFileMetaData.smallest_seqno = fileMetaData.smallest_seqno;
 		liveFileMetaData.largest_seqno = fileMetaData.largest_seqno;
 		liveFileMetaData.smallestkey = fileMetaData.smallestkey;
@@ -269,12 +277,12 @@ void populateMetaData(CheckpointMetaData* checkpoint, const rocksdb::ExportImpor
 		liveFileMetaData.being_compacted = fileMetaData.being_compacted;
 		liveFileMetaData.num_entries = fileMetaData.num_entries;
 		liveFileMetaData.num_deletions = fileMetaData.num_deletions;
-		liveFileMetaData.temperature = static_cast<uint8_t>(fileMetaData.temperature);
 		liveFileMetaData.oldest_blob_file_number = fileMetaData.oldest_blob_file_number;
 		liveFileMetaData.oldest_ancester_time = fileMetaData.oldest_ancester_time;
 		liveFileMetaData.file_creation_time = fileMetaData.file_creation_time;
-		liveFileMetaData.file_checksum = fileMetaData.file_checksum;
-		liveFileMetaData.file_checksum_func_name = fileMetaData.file_checksum_func_name;
+		liveFileMetaData.epoch_number = fileMetaData.epoch_number;
+		liveFileMetaData.name = fileMetaData.name;
+		liveFileMetaData.db_path = fileMetaData.db_path;
 		liveFileMetaData.column_family_name = fileMetaData.column_family_name;
 		liveFileMetaData.level = fileMetaData.level;
 		rocksCF.sstFiles.push_back(liveFileMetaData);
@@ -2472,7 +2480,7 @@ TEST_CASE("noSim/fdbserver/KeyValueStoreRocksDB/CheckpointRestore") {
 TEST_CASE("noSim/fdbserver/KeyValueStoreRocksDB/RocksDBTypes") {
 	// If the following assertion fails, update SstFileMetaData and LiveFileMetaData in RocksDBCheckpointUtils.actor.h
 	// to be the same as rocksdb::SstFileMetaData and rocksdb::LiveFileMetaData.
-	ASSERT_EQ(sizeof(rocksdb::LiveFileMetaData), 184);
+	ASSERT_EQ(sizeof(rocksdb::LiveFileMetaData), 192);
 	ASSERT_EQ(sizeof(rocksdb::ExportImportFilesMetaData), 32);
 	return Void();
 }
diff --git a/fdbserver/RocksDBCheckpointUtils.actor.h b/fdbserver/RocksDBCheckpointUtils.actor.h
index 3d0c157bf31..23979f22e08 100644
--- a/fdbserver/RocksDBCheckpointUtils.actor.h
+++ b/fdbserver/RocksDBCheckpointUtils.actor.h
@@ -35,41 +35,65 @@
 struct SstFileMetaData {
 	constexpr static FileIdentifier file_identifier = 3804347;
 	SstFileMetaData()
-	  : size(0), file_number(0), smallest_seqno(0), largest_seqno(0), num_reads_sampled(0), being_compacted(false),
-	    num_entries(0), num_deletions(0), temperature(0), oldest_blob_file_number(0), oldest_ancester_time(0),
-	    file_creation_time(0) {}
+	  : file_number(0), file_type(2), size(0), temperature(0), smallest_seqno(0), largest_seqno(0),
+	    num_reads_sampled(0), being_compacted(false), num_entries(0), num_deletions(0), oldest_blob_file_number(0),
+	    oldest_ancester_time(0), file_creation_time(0), epoch_number(0) {}
 
-	SstFileMetaData(const std::string& _file_name,
+	SstFileMetaData(const std::string& _relative_filename,
+	                const std::string& _directory,
 	                uint64_t _file_number,
-	                const std::string& _path,
-	                size_t _size,
+	                int _file_type,
+	                uint64_t _size,
+	                int _temperature,
+	                std::string& _file_checksum,
+	                std::string& _file_checksum_func_name,
 	                uint64_t _smallest_seqno,
 	                uint64_t _largest_seqno,
 	                const std::string& _smallestkey,
 	                const std::string& _largestkey,
 	                uint64_t _num_reads_sampled,
 	                bool _being_compacted,
-	                int _temperature,
+	                uint64_t _num_entries,
+	                uint64_t _num_deletions,
 	                uint64_t _oldest_blob_file_number,
 	                uint64_t _oldest_ancester_time,
 	                uint64_t _file_creation_time,
-	                std::string& _file_checksum,
-	                std::string& _file_checksum_func_name)
-	  : size(_size), name(_file_name), file_number(_file_number), db_path(_path), smallest_seqno(_smallest_seqno),
+	                uint64_t _epoch_number,
+	                const std::string& _name,
+	                const std::string& _db_path)
+	  : relative_filename(_relative_filename), directory(_directory), file_number(_file_number), file_type(_file_type),
+	    size(_size), temperature(_temperature), file_checksum(_file_checksum),
+	    file_checksum_func_name(_file_checksum_func_name), smallest_seqno(_smallest_seqno),
 	    largest_seqno(_largest_seqno), smallestkey(_smallestkey), largestkey(_largestkey),
-	    num_reads_sampled(_num_reads_sampled), being_compacted(_being_compacted), num_entries(0), num_deletions(0),
-	    temperature(_temperature), oldest_blob_file_number(_oldest_blob_file_number),
+	    num_reads_sampled(_num_reads_sampled), being_compacted(_being_compacted), num_entries(_num_entries),
+	    num_deletions(_num_deletions), oldest_blob_file_number(_oldest_blob_file_number),
 	    oldest_ancester_time(_oldest_ancester_time), file_creation_time(_file_creation_time),
-	    file_checksum(_file_checksum), file_checksum_func_name(_file_checksum_func_name) {}
-
-	// File size in bytes.
-	size_t size;
-	// The name of the file.
-	std::string name;
-	// The id of the file.
+	    epoch_number(_epoch_number), name(_name), db_path(_db_path) {}
+
+	// The name of the file within its directory (e.g. "123456.sst")
+	std::string relative_filename;
+	// The directory containing the file, without a trailing '/'. This could be
+	// a DB path, wal_dir, etc.
+	std::string directory;
+	// The id of the file within a single DB. Set to 0 if the file does not have
+	// a number (e.g. CURRENT)
 	uint64_t file_number;
-	// The full path where the file locates.
-	std::string db_path;
+	// The type of the file as part of a DB.
+	int file_type;
+	// File size in bytes. See also `trim_to_size`.
+	uint64_t size;
+	// This feature is experimental and subject to change.
+	int temperature;
+	// The checksum of a SST file, the value is decided by the file content and
+	// the checksum algorithm used for this SST file. The checksum function is
+	// identified by the file_checksum_func_name. If the checksum function is
+	// not specified, file_checksum is "0" by default.
+	std::string file_checksum;
+	// The name of the checksum function used to generate the file checksum
+	// value. If file checksum is not enabled (e.g., sst_file_checksum_func is
+	// null), file_checksum_func_name is UnknownFileChecksumFuncName, which is
+	// "Unknown".
+	std::string file_checksum_func_name;
 
 	uint64_t smallest_seqno; // Smallest sequence number in file.
 	uint64_t largest_seqno; // Largest sequence number in file.
@@ -77,13 +101,8 @@ struct SstFileMetaData {
 	std::string largestkey; // Largest user defined key in the file.
 	uint64_t num_reads_sampled; // How many times the file is read.
 	bool being_compacted; // true if the file is currently being compacted.
-
 	uint64_t num_entries;
 	uint64_t num_deletions;
-
-	// This feature is experimental and subject to change.
-	int temperature;
-
 	uint64_t oldest_blob_file_number; // The id of the oldest blob file
 	                                  // referenced by the file.
 	// An SST file may be generated by compactions whose input files may
@@ -97,26 +116,30 @@ struct SstFileMetaData {
 	// Timestamp when the SST file is created, provided by
 	// SystemClock::GetCurrentTime(). 0 if the information is not available.
 	uint64_t file_creation_time;
-
-	// The checksum of a SST file, the value is decided by the file content and
-	// the checksum algorithm used for this SST file. The checksum function is
-	// identified by the file_checksum_func_name. If the checksum function is
-	// not specified, file_checksum is "0" by default.
-	std::string file_checksum;
-
-	// The name of the checksum function used to generate the file checksum
-	// value. If file checksum is not enabled (e.g., sst_file_checksum_func is
-	// null), file_checksum_func_name is UnknownFileChecksumFuncName, which is
-	// "Unknown".
-	std::string file_checksum_func_name;
+	// The order of a file being flushed or ingested/imported.
+	// Compaction output file will be assigned with the minimum `epoch_number`
+	// among input files'.
+	// For L0, larger `epoch_number` indicates newer L0 file.
+	// 0 if the information is not available.
+	uint64_t epoch_number;
+	// DEPRECATED: The name of the file within its directory with a
+	// leading slash (e.g. "/123456.sst"). Use relative_filename from base struct
+	// instead.
+	std::string name;
+	// DEPRECATED: replaced by `directory` in base struct
+	std::string db_path;
 
 	template <class Ar>
 	void serialize(Ar& ar) {
 		serializer(ar,
-		           size,
-		           name,
+		           relative_filename,
+		           directory,
 		           file_number,
-		           db_path,
+		           file_type,
+		           size,
+		           temperature,
+		           file_checksum,
+		           file_checksum_func_name,
 		           smallest_seqno,
 		           largest_seqno,
 		           smallestkey,
@@ -125,12 +148,12 @@ struct SstFileMetaData {
 		           being_compacted,
 		           num_entries,
 		           num_deletions,
-		           temperature,
 		           oldest_blob_file_number,
 		           oldest_ancester_time,
 		           file_creation_time,
-		           file_checksum,
-		           file_checksum_func_name);
+		           epoch_number,
+		           name,
+		           db_path);
 	}
 };
 
@@ -145,10 +168,14 @@ struct LiveFileMetaData : public SstFileMetaData {
 	template <class Ar>
 	void serialize(Ar& ar) {
 		serializer(ar,
-		           SstFileMetaData::size,
-		           SstFileMetaData::name,
+		           SstFileMetaData::relative_filename,
+		           SstFileMetaData::directory,
 		           SstFileMetaData::file_number,
-		           SstFileMetaData::db_path,
+		           SstFileMetaData::file_type,
+		           SstFileMetaData::size,
+		           SstFileMetaData::temperature,
+		           SstFileMetaData::file_checksum,
+		           SstFileMetaData::file_checksum_func_name,
 		           SstFileMetaData::smallest_seqno,
 		           SstFileMetaData::largest_seqno,
 		           SstFileMetaData::smallestkey,
@@ -157,12 +184,12 @@ struct LiveFileMetaData : public SstFileMetaData {
 		           SstFileMetaData::being_compacted,
 		           SstFileMetaData::num_entries,
 		           SstFileMetaData::num_deletions,
-		           SstFileMetaData::temperature,
 		           SstFileMetaData::oldest_blob_file_number,
 		           SstFileMetaData::oldest_ancester_time,
 		           SstFileMetaData::file_creation_time,
-		           SstFileMetaData::file_checksum,
-		           SstFileMetaData::file_checksum_func_name,
+		           SstFileMetaData::epoch_number,
+		           SstFileMetaData::name,
+		           SstFileMetaData::db_path,
 		           column_family_name,
 		           level,
 		           fetched);

From 6af1ee6d1ec856703e60e9283ab68c3d4b8768c2 Mon Sep 17 00:00:00 2001
From: neethuhaneesha <nbingi@apple.com>
Date: Fri, 31 Mar 2023 08:14:30 -0700
Subject: [PATCH 124/468] Changing single key deletions to delete based on
 number of deletes instead of bytelimit.

---
 fdbclient/ServerKnobs.cpp                     |  2 ++
 fdbclient/ServerKnobs.h                       |  2 ++
 fdbserver/IKeyValueStore.h                    |  4 +---
 .../KeyValueStoreCompressTestData.actor.cpp   |  6 +-----
 fdbserver/KeyValueStoreMemory.actor.cpp       |  2 +-
 fdbserver/KeyValueStoreRocksDB.actor.cpp      | 21 ++++++++++++-------
 fdbserver/KeyValueStoreSQLite.actor.cpp       |  6 ++----
 fdbserver/RemoteIKeyValueStore.actor.h        |  4 +---
 fdbserver/VersionedBTree.actor.cpp            |  4 +---
 fdbserver/storageserver.actor.cpp             |  8 +++----
 10 files changed, 29 insertions(+), 30 deletions(-)

diff --git a/fdbclient/ServerKnobs.cpp b/fdbclient/ServerKnobs.cpp
index 18e24741605..20d16833100 100644
--- a/fdbclient/ServerKnobs.cpp
+++ b/fdbclient/ServerKnobs.cpp
@@ -421,7 +421,9 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi
 	// These knobs have contrary functionality.
 	init( ROCKSDB_SINGLEKEY_DELETES_ON_CLEARRANGE,              true );
 	init( ROCKSDB_SINGLEKEY_DELETES_BYTES_LIMIT,               10000 ); // 10KB
+	init( ROCKSDB_SINGLEKEY_DELETES_MAX,                         200 ); // Max rocksdb::delete calls in a transaction
 	init( ROCKSDB_ENABLE_CLEAR_RANGE_EAGER_READS,              false );
+	init( ROCKSDB_FORCE_DELETERANGE_FOR_CLEARRANGE,            false );
 	// ROCKSDB_STATS_LEVEL=1 indicates rocksdb::StatsLevel::kExceptHistogramOrTimers
 	// Refer StatsLevel: https://github.com/facebook/rocksdb/blob/main/include/rocksdb/statistics.h#L594
 	init( ROCKSDB_STATS_LEVEL,                                     1 ); if( randomize && BUGGIFY ) ROCKSDB_STATS_LEVEL = deterministicRandom()->randomInt(0, 6);
diff --git a/fdbclient/ServerKnobs.h b/fdbclient/ServerKnobs.h
index 529e306dc5b..7295ac7466e 100644
--- a/fdbclient/ServerKnobs.h
+++ b/fdbclient/ServerKnobs.h
@@ -342,7 +342,9 @@ class ServerKnobs : public KnobsImpl<ServerKnobs> {
 	bool ROCKSDB_DISABLE_WAL_EXPERIMENTAL;
 	bool ROCKSDB_SINGLEKEY_DELETES_ON_CLEARRANGE;
 	int64_t ROCKSDB_SINGLEKEY_DELETES_BYTES_LIMIT;
+	int ROCKSDB_SINGLEKEY_DELETES_MAX;
 	bool ROCKSDB_ENABLE_CLEAR_RANGE_EAGER_READS;
+	bool ROCKSDB_FORCE_DELETERANGE_FOR_CLEARRANGE;
 	bool ROCKSDB_ENABLE_COMPACT_ON_DELETION;
 	int64_t ROCKSDB_CDCF_SLIDING_WINDOW_SIZE; // CDCF: CompactOnDeletionCollectorFactory
 	int64_t ROCKSDB_CDCF_DELETION_TRIGGER; // CDCF: CompactOnDeletionCollectorFactory
diff --git a/fdbserver/IKeyValueStore.h b/fdbserver/IKeyValueStore.h
index 9c0f3afaa91..74b93b9620c 100644
--- a/fdbserver/IKeyValueStore.h
+++ b/fdbserver/IKeyValueStore.h
@@ -62,9 +62,7 @@ class IKeyValueStore : public IClosable {
 public:
 	virtual KeyValueStoreType getType() const = 0;
 	virtual void set(KeyValueRef keyValue, const Arena* arena = nullptr) = 0;
-	virtual void clear(KeyRangeRef range,
-	                   const StorageServerMetrics* storageMetrics = nullptr,
-	                   const Arena* arena = nullptr) = 0;
+	virtual void clear(KeyRangeRef range, const Arena* arena = nullptr) = 0;
 	virtual Future<Void> canCommit() { return Void(); }
 	virtual Future<Void> commit(
 	    bool sequential = false) = 0; // returns when prior sets and clears are (atomically) durable
diff --git a/fdbserver/KeyValueStoreCompressTestData.actor.cpp b/fdbserver/KeyValueStoreCompressTestData.actor.cpp
index 38ddd785e65..3121232b581 100644
--- a/fdbserver/KeyValueStoreCompressTestData.actor.cpp
+++ b/fdbserver/KeyValueStoreCompressTestData.actor.cpp
@@ -53,11 +53,7 @@ struct KeyValueStoreCompressTestData final : IKeyValueStore {
 	void set(KeyValueRef keyValue, const Arena* arena = nullptr) override {
 		store->set(KeyValueRef(keyValue.key, pack(keyValue.value)), arena);
 	}
-	void clear(KeyRangeRef range,
-	           const StorageServerMetrics* storageMetrics = nullptr,
-	           const Arena* arena = nullptr) override {
-		store->clear(range, storageMetrics, arena);
-	}
+	void clear(KeyRangeRef range, const Arena* arena = nullptr) override { store->clear(range, arena); }
 	Future<Void> commit(bool sequential = false) override { return store->commit(sequential); }
 
 	Future<Optional<Value>> readValue(KeyRef key, IKeyValueStore::ReadType, Optional<UID> debugID) override {
diff --git a/fdbserver/KeyValueStoreMemory.actor.cpp b/fdbserver/KeyValueStoreMemory.actor.cpp
index 616aaed8585..4ded4f67492 100644
--- a/fdbserver/KeyValueStoreMemory.actor.cpp
+++ b/fdbserver/KeyValueStoreMemory.actor.cpp
@@ -124,7 +124,7 @@ class KeyValueStoreMemory final : public IKeyValueStore, NonCopyable {
 		}
 	}
 
-	void clear(KeyRangeRef range, const StorageServerMetrics* storageMetrics, const Arena* arena) override {
+	void clear(KeyRangeRef range, const Arena* arena) override {
 		// A commit that occurs with no available space returns Never, so we can throw out all modifications
 		if (getAvailableSize() <= 0)
 			return;
diff --git a/fdbserver/KeyValueStoreRocksDB.actor.cpp b/fdbserver/KeyValueStoreRocksDB.actor.cpp
index 067726e7e86..1a320ef68ab 100644
--- a/fdbserver/KeyValueStoreRocksDB.actor.cpp
+++ b/fdbserver/KeyValueStoreRocksDB.actor.cpp
@@ -1884,6 +1884,8 @@ struct RocksDBKeyValueStore : IKeyValueStore {
 	Future<Void> openFuture;
 	std::unique_ptr<rocksdb::WriteBatch> writeBatch;
 	std::set<Key> keysSet;
+	// maximum number of single key deletes in a commit, if ROCKSDB_SINGLEKEY_DELETES_ON_CLEARRANGE is enabled.
+	int maxDeletes;
 	Optional<Future<Void>> metrics;
 	FlowLock readSemaphore;
 	int numReadWaiters;
@@ -2072,6 +2074,7 @@ struct RocksDBKeyValueStore : IKeyValueStore {
 		if (writeBatch == nullptr) {
 			writeBatch.reset(new rocksdb::WriteBatch());
 			keysSet.clear();
+			maxDeletes = SERVER_KNOBS->ROCKSDB_SINGLEKEY_DELETES_MAX;
 		}
 		ASSERT(defaultFdbCF != nullptr);
 		writeBatch->Put(defaultFdbCF, toSlice(kv.key), toSlice(kv.value));
@@ -2080,23 +2083,24 @@ struct RocksDBKeyValueStore : IKeyValueStore {
 		}
 	}
 
-	void clear(KeyRangeRef keyRange, const StorageServerMetrics* storageMetrics, const Arena*) override {
+	void clear(KeyRangeRef keyRange, const Arena*) override {
 		if (writeBatch == nullptr) {
 			writeBatch.reset(new rocksdb::WriteBatch());
 			keysSet.clear();
+			maxDeletes = SERVER_KNOBS->ROCKSDB_SINGLEKEY_DELETES_MAX;
 		}
 
 		ASSERT(defaultFdbCF != nullptr);
 		// Number of deletes to rocksdb = counters.deleteKeyReqs + convertedDeleteKeyReqs;
 		// Number of deleteRanges to rocksdb = counters.deleteRangeReqs - counters.convertedDeleteRangeReqs;
-		if (keyRange.singleKeyRange()) {
+		if (keyRange.singleKeyRange() && !SERVER_KNOBS->ROCKSDB_FORCE_DELETERANGE_FOR_CLEARRANGE) {
 			writeBatch->Delete(defaultFdbCF, toSlice(keyRange.begin));
 			++counters.deleteKeyReqs;
+			--maxDeletes;
 		} else {
 			++counters.deleteRangeReqs;
-			if (SERVER_KNOBS->ROCKSDB_SINGLEKEY_DELETES_ON_CLEARRANGE && storageMetrics != nullptr &&
-			    storageMetrics->byteSample.getEstimate(keyRange) <
-			        SERVER_KNOBS->ROCKSDB_SINGLEKEY_DELETES_BYTES_LIMIT) {
+			if (SERVER_KNOBS->ROCKSDB_SINGLEKEY_DELETES_ON_CLEARRANGE &&
+			    !SERVER_KNOBS->ROCKSDB_FORCE_DELETERANGE_FOR_CLEARRANGE && maxDeletes > 0) {
 				++counters.convertedDeleteRangeReqs;
 				rocksdb::ReadOptions options = getReadOptions();
 				auto beginSlice = toSlice(keyRange.begin);
@@ -2105,12 +2109,13 @@ struct RocksDBKeyValueStore : IKeyValueStore {
 				options.iterate_upper_bound = &endSlice;
 				auto cursor = std::unique_ptr<rocksdb::Iterator>(db->NewIterator(options, defaultFdbCF));
 				cursor->Seek(toSlice(keyRange.begin));
-				while (cursor->Valid() && toStringRef(cursor->key()) < keyRange.end) {
+				while (cursor->Valid() && toStringRef(cursor->key()) < keyRange.end && maxDeletes > 0) {
 					writeBatch->Delete(defaultFdbCF, cursor->key());
 					++counters.convertedDeleteKeyReqs;
+					--maxDeletes;
 					cursor->Next();
 				}
-				if (!cursor->status().ok()) {
+				if (!cursor->status().ok() || maxDeletes <= 0) {
 					// if readrange iteration fails, then do a deleteRange.
 					writeBatch->DeleteRange(defaultFdbCF, toSlice(keyRange.begin), toSlice(keyRange.end));
 				} else {
@@ -2118,6 +2123,7 @@ struct RocksDBKeyValueStore : IKeyValueStore {
 					while (it != keysSet.end() && *it < keyRange.end) {
 						writeBatch->Delete(defaultFdbCF, toSlice(*it));
 						++counters.convertedDeleteKeyReqs;
+						--maxDeletes;
 						it++;
 					}
 				}
@@ -2156,6 +2162,7 @@ struct RocksDBKeyValueStore : IKeyValueStore {
 		auto a = new Writer::CommitAction();
 		a->batchToCommit = std::move(writeBatch);
 		keysSet.clear();
+		maxDeletes = SERVER_KNOBS->ROCKSDB_SINGLEKEY_DELETES_MAX;
 		auto res = a->done.getFuture();
 		writeThread->post(a);
 		return res;
diff --git a/fdbserver/KeyValueStoreSQLite.actor.cpp b/fdbserver/KeyValueStoreSQLite.actor.cpp
index dd56e412433..435a9b2eb5d 100644
--- a/fdbserver/KeyValueStoreSQLite.actor.cpp
+++ b/fdbserver/KeyValueStoreSQLite.actor.cpp
@@ -1586,9 +1586,7 @@ class KeyValueStoreSQLite final : public IKeyValueStore {
 	StorageBytes getStorageBytes() const override;
 
 	void set(KeyValueRef keyValue, const Arena* arena = nullptr) override;
-	void clear(KeyRangeRef range,
-	           const StorageServerMetrics* storageMetrics = nullptr,
-	           const Arena* arena = nullptr) override;
+	void clear(KeyRangeRef range, const Arena* arena = nullptr) override;
 	Future<Void> commit(bool sequential = false) override;
 
 	Future<Optional<Value>> readValue(KeyRef key, IKeyValueStore::ReadType, Optional<UID> debugID) override;
@@ -2204,7 +2202,7 @@ void KeyValueStoreSQLite::set(KeyValueRef keyValue, const Arena* arena) {
 	++writesRequested;
 	writeThread->post(new Writer::SetAction(keyValue));
 }
-void KeyValueStoreSQLite::clear(KeyRangeRef range, const StorageServerMetrics* storageMetrics, const Arena* arena) {
+void KeyValueStoreSQLite::clear(KeyRangeRef range, const Arena* arena) {
 	++writesRequested;
 	writeThread->post(new Writer::ClearAction(range));
 }
diff --git a/fdbserver/RemoteIKeyValueStore.actor.h b/fdbserver/RemoteIKeyValueStore.actor.h
index de7cd362649..0465e4afbb2 100644
--- a/fdbserver/RemoteIKeyValueStore.actor.h
+++ b/fdbserver/RemoteIKeyValueStore.actor.h
@@ -392,9 +392,7 @@ struct RemoteIKeyValueStore : public IKeyValueStore {
 	void set(KeyValueRef keyValue, const Arena* arena = nullptr) override {
 		interf.set.send(IKVSSetRequest{ keyValue, ReplyPromise<Void>() });
 	}
-	void clear(KeyRangeRef range,
-	           const StorageServerMetrics* storageMetrics = nullptr,
-	           const Arena* arena = nullptr) override {
+	void clear(KeyRangeRef range, const Arena* arena = nullptr) override {
 		interf.clear.send(IKVSClearRequest{ range, ReplyPromise<Void>() });
 	}
 
diff --git a/fdbserver/VersionedBTree.actor.cpp b/fdbserver/VersionedBTree.actor.cpp
index c20f3f656d9..cf8211ed015 100644
--- a/fdbserver/VersionedBTree.actor.cpp
+++ b/fdbserver/VersionedBTree.actor.cpp
@@ -7822,9 +7822,7 @@ class KeyValueStoreRedwood : public IKeyValueStore {
 
 	Future<Void> getError() const override { return delayed(m_error.getFuture()); };
 
-	void clear(KeyRangeRef range,
-	           const StorageServerMetrics* storageMetrics = nullptr,
-	           const Arena* arena = 0) override {
+	void clear(KeyRangeRef range, const Arena* arena = 0) override {
 		debug_printf("CLEAR %s\n", printable(range).c_str());
 		m_tree->clear(range);
 	}
diff --git a/fdbserver/storageserver.actor.cpp b/fdbserver/storageserver.actor.cpp
index 3a3868e25cf..fed740f79bd 100644
--- a/fdbserver/storageserver.actor.cpp
+++ b/fdbserver/storageserver.actor.cpp
@@ -7870,7 +7870,7 @@ void setAssignedStatus(StorageServer* self, KeyRangeRef keys, bool nowAssigned)
 }
 
 void StorageServerDisk::clearRange(KeyRangeRef keys) {
-	storage->clear(keys, &data->metrics);
+	storage->clear(keys);
 	++(*kvClearRanges);
 	if (keys.singleKeyRange()) {
 		++(*kvClearSingleKey);
@@ -7887,7 +7887,7 @@ void StorageServerDisk::writeMutation(MutationRef mutation) {
 		storage->set(KeyValueRef(mutation.param1, mutation.param2));
 		*kvCommitLogicalBytes += mutation.expectedSize();
 	} else if (mutation.type == MutationRef::ClearRange) {
-		storage->clear(KeyRangeRef(mutation.param1, mutation.param2), &data->metrics);
+		storage->clear(KeyRangeRef(mutation.param1, mutation.param2));
 		++(*kvClearRanges);
 		if (KeyRangeRef(mutation.param1, mutation.param2).singleKeyRange()) {
 			++(*kvClearSingleKey);
@@ -7905,7 +7905,7 @@ void StorageServerDisk::writeMutations(const VectorRef<MutationRef>& mutations,
 			storage->set(KeyValueRef(m.param1, m.param2));
 			*kvCommitLogicalBytes += m.expectedSize();
 		} else if (m.type == MutationRef::ClearRange) {
-			storage->clear(KeyRangeRef(m.param1, m.param2), &data->metrics);
+			storage->clear(KeyRangeRef(m.param1, m.param2));
 			++(*kvClearRanges);
 			if (KeyRangeRef(m.param1, m.param2).singleKeyRange()) {
 				++(*kvClearSingleKey);
@@ -8318,7 +8318,7 @@ ACTOR Future<bool> restoreDurableState(StorageServer* data, IKeyValueStore* stor
 				++data->counters.kvSystemClearRanges;
 				// TODO(alexmiller): Figure out how to selectively enable spammy data distribution events.
 				// DEBUG_KEY_RANGE("clearInvalidVersion", invalidVersion, clearRange);
-				storage->clear(clearRange, &data->metrics);
+				storage->clear(clearRange);
 				++data->counters.kvSystemClearRanges;
 				data->byteSampleApplyClear(clearRange, invalidVersion);
 			}

From f4ee02a3ac6c67096fc5e2859809184962e1a7b6 Mon Sep 17 00:00:00 2001
From: Zhe Wu <halfprice@users.noreply.github.com>
Date: Mon, 3 Apr 2023 20:16:06 -0700
Subject: [PATCH 125/468] Implement check if locality is already excluded in
 exclude locality command (#9878)

---
 fdbclient/ManagementAPI.actor.cpp   | 70 ++++++++++++++++++++---------
 fdbclient/ManagementAPI.actor.h     | 12 +++--
 fdbclient/SpecialKeySpace.actor.cpp |  2 +-
 3 files changed, 59 insertions(+), 25 deletions(-)

diff --git a/fdbclient/ManagementAPI.actor.cpp b/fdbclient/ManagementAPI.actor.cpp
index 938afc9118a..dd0ce41d7be 100644
--- a/fdbclient/ManagementAPI.actor.cpp
+++ b/fdbclient/ManagementAPI.actor.cpp
@@ -1347,24 +1347,36 @@ ACTOR Future<Void> excludeServers(Database cx, std::vector<AddressExclusion> ser
 }
 
 // excludes localities by setting the keys in api version below 7.0
-// TODO(zhewu): update this function that if the locality is already excluded, don't need to update the system metadata.
-void excludeLocalities(Transaction& tr, std::unordered_set<std::string> localities, bool failed) {
-	tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
-	tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
-	tr.setOption(FDBTransactionOptions::LOCK_AWARE);
-	tr.setOption(FDBTransactionOptions::USE_PROVISIONAL_PROXIES);
-	std::string excludeVersionKey = deterministicRandom()->randomUniqueID().toString();
-	auto localityVersionKey = failed ? failedLocalityVersionKey : excludedLocalityVersionKey;
-	tr.addReadConflictRange(singleKeyRange(localityVersionKey)); // To conflict with parallel includeLocalities
-	tr.set(localityVersionKey, excludeVersionKey);
+ACTOR Future<Void> excludeLocalities(Transaction* tr, std::unordered_set<std::string> localities, bool failed) {
+	tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
+	tr->setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
+	tr->setOption(FDBTransactionOptions::LOCK_AWARE);
+	tr->setOption(FDBTransactionOptions::USE_PROVISIONAL_PROXIES);
+	std::vector<std::string> excl = wait(failed ? getExcludedFailedLocalityList(tr) : getExcludedLocalityList(tr));
+	std::set<std::string> exclusion(excl.begin(), excl.end());
+	bool containNewExclusion = false;
 	for (const auto& l : localities) {
+		if (exclusion.find(l) != exclusion.end()) {
+			continue;
+		}
+		containNewExclusion = true;
 		if (failed) {
-			tr.set(encodeFailedLocalityKey(l), StringRef());
+			tr->set(encodeFailedLocalityKey(l), StringRef());
 		} else {
-			tr.set(encodeExcludedLocalityKey(l), StringRef());
+			tr->set(encodeExcludedLocalityKey(l), StringRef());
 		}
 	}
-	TraceEvent("ExcludeLocalitiesCommit").detail("Localities", describe(localities)).detail("ExcludeFailed", failed);
+	if (containNewExclusion) {
+		std::string excludeVersionKey = deterministicRandom()->randomUniqueID().toString();
+		auto localityVersionKey = failed ? failedLocalityVersionKey : excludedLocalityVersionKey;
+		tr->addReadConflictRange(singleKeyRange(localityVersionKey)); // To conflict with parallel includeLocalities
+		tr->set(localityVersionKey, excludeVersionKey);
+	}
+	TraceEvent("ExcludeLocalitiesCommit")
+	    .detail("Localities", describe(localities))
+	    .detail("ExcludeFailed", failed)
+	    .detail("ExclusionUpdated", containNewExclusion);
+	return Void();
 }
 
 // Exclude the servers matching the given set of localities from use as state servers.
@@ -1399,7 +1411,7 @@ ACTOR Future<Void> excludeLocalities(Database cx, std::unordered_set<std::string
 		state Transaction tr(cx);
 		loop {
 			try {
-				excludeLocalities(tr, localities, failed);
+				wait(excludeLocalities(&tr, localities, failed));
 				wait(tr.commit());
 				return Void();
 			} catch (Error& e) {
@@ -1685,19 +1697,25 @@ ACTOR Future<std::vector<AddressExclusion>> getAllExcludedServers(Database cx) {
 	}
 }
 
-// Get the current list of excluded localities by reading the keys.
-ACTOR Future<std::vector<std::string>> getExcludedLocalities(Transaction* tr) {
+ACTOR Future<std::vector<std::string>> getExcludedLocalityList(Transaction* tr) {
 	state RangeResult r = wait(tr->getRange(excludedLocalityKeys, CLIENT_KNOBS->TOO_MANY));
 	ASSERT(!r.more && r.size() < CLIENT_KNOBS->TOO_MANY);
-	state RangeResult r2 = wait(tr->getRange(failedLocalityKeys, CLIENT_KNOBS->TOO_MANY));
-	ASSERT(!r2.more && r2.size() < CLIENT_KNOBS->TOO_MANY);
 
 	std::vector<std::string> excludedLocalities;
 	for (const auto& i : r) {
 		auto a = decodeExcludedLocalityKey(i.key);
 		excludedLocalities.push_back(a);
 	}
-	for (const auto& i : r2) {
+	uniquify(excludedLocalities);
+	return excludedLocalities;
+}
+
+ACTOR Future<std::vector<std::string>> getExcludedFailedLocalityList(Transaction* tr) {
+	state RangeResult r = wait(tr->getRange(failedLocalityKeys, CLIENT_KNOBS->TOO_MANY));
+	ASSERT(!r.more && r.size() < CLIENT_KNOBS->TOO_MANY);
+
+	std::vector<std::string> excludedLocalities;
+	for (const auto& i : r) {
 		auto a = decodeFailedLocalityKey(i.key);
 		excludedLocalities.push_back(a);
 	}
@@ -1705,15 +1723,25 @@ ACTOR Future<std::vector<std::string>> getExcludedLocalities(Transaction* tr) {
 	return excludedLocalities;
 }
 
+ACTOR Future<std::vector<std::string>> getAllExcludedLocalities(Transaction* tr) {
+	state std::vector<std::string> exclusions;
+	std::vector<std::string> excludedLocalities = wait(getExcludedLocalityList(tr));
+	exclusions.insert(exclusions.end(), excludedLocalities.begin(), excludedLocalities.end());
+	std::vector<std::string> failedLocalities = wait(getExcludedFailedLocalityList(tr));
+	exclusions.insert(exclusions.end(), failedLocalities.begin(), failedLocalities.end());
+	uniquify(exclusions);
+	return exclusions;
+}
+
 // Get the list of excluded localities by reading the keys.
-ACTOR Future<std::vector<std::string>> getExcludedLocalities(Database cx) {
+ACTOR Future<std::vector<std::string>> getAllExcludedLocalities(Database cx) {
 	state Transaction tr(cx);
 	loop {
 		try {
 			tr.setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
 			tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
 			tr.setOption(FDBTransactionOptions::LOCK_AWARE);
-			std::vector<std::string> exclusions = wait(getExcludedLocalities(&tr));
+			std::vector<std::string> exclusions = wait(getAllExcludedLocalities(&tr));
 			return exclusions;
 		} catch (Error& e) {
 			wait(tr.onError(e));
diff --git a/fdbclient/ManagementAPI.actor.h b/fdbclient/ManagementAPI.actor.h
index ef6227825f6..f9d204bdb39 100644
--- a/fdbclient/ManagementAPI.actor.h
+++ b/fdbclient/ManagementAPI.actor.h
@@ -70,7 +70,7 @@ ACTOR Future<Void> excludeServers(Transaction* tr, std::vector<AddressExclusion>
 // Exclude the servers matching the given set of localities from use as state servers.  Returns as soon as the change
 // is durable, without necessarily waiting for the servers to be evacuated.
 ACTOR Future<Void> excludeLocalities(Database cx, std::unordered_set<std::string> localities, bool failed = false);
-void excludeLocalities(Transaction& tr, std::unordered_set<std::string> localities, bool failed = false);
+ACTOR Future<Void> excludeLocalities(Transaction* tr, std::unordered_set<std::string> localities, bool failed = false);
 
 // Remove the given servers from the exclusion list.  A NetworkAddress with a port of 0 means all servers on the given
 // IP.  A NetworkAddress() means all servers (don't exclude anything)
@@ -97,8 +97,14 @@ ACTOR Future<std::vector<AddressExclusion>> getExcludedServerList(Transaction* t
 ACTOR Future<std::vector<AddressExclusion>> getExcludedFailedServerList(Transaction* tr);
 
 // Get the current list of excluded localities
-ACTOR Future<std::vector<std::string>> getExcludedLocalities(Database cx);
-ACTOR Future<std::vector<std::string>> getExcludedLocalities(Transaction* tr);
+ACTOR Future<std::vector<std::string>> getAllExcludedLocalities(Database cx);
+ACTOR Future<std::vector<std::string>> getAllExcludedLocalities(Transaction* tr);
+
+// Get the current list of excluded localities.
+ACTOR Future<std::vector<std::string>> getExcludedLocalityList(Transaction* tr);
+
+// Get the current list of failed localities.
+ACTOR Future<std::vector<std::string>> getExcludedFailedLocalityList(Transaction* tr);
 
 std::set<AddressExclusion> getAddressesByLocality(const std::vector<ProcessData>& workers, const std::string& locality);
 
diff --git a/fdbclient/SpecialKeySpace.actor.cpp b/fdbclient/SpecialKeySpace.actor.cpp
index 1e2c0beb5c9..1f0970b668d 100644
--- a/fdbclient/SpecialKeySpace.actor.cpp
+++ b/fdbclient/SpecialKeySpace.actor.cpp
@@ -2699,7 +2699,7 @@ ACTOR Future<Optional<std::string>> excludeLocalityCommitActor(ReadYourWritesTra
 			return result;
 	}
 
-	excludeLocalities(ryw->getTransaction(), localities, failed);
+	wait(excludeLocalities(&ryw->getTransaction(), localities, failed));
 	includeLocalities(ryw);
 
 	return result;

From 451a8ae424fcb3acd056f682409ea5dd605e106e Mon Sep 17 00:00:00 2001
From: Sreenath Bodagala <82616783+sbodagala@users.noreply.github.com>
Date: Tue, 4 Apr 2023 00:43:19 -0400
Subject: [PATCH 126/468] Merge pull request #9814 from sbodagala/main (#9883)

FdbServer not able to join cluster

Co-authored-by: Jingyu Zhou <jingyu_zhou@apple.com>
---
 fdbserver/LeaderElection.actor.cpp | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/fdbserver/LeaderElection.actor.cpp b/fdbserver/LeaderElection.actor.cpp
index e641f3d0f31..d010f7d3d19 100644
--- a/fdbserver/LeaderElection.actor.cpp
+++ b/fdbserver/LeaderElection.actor.cpp
@@ -158,7 +158,13 @@ ACTOR Future<Void> tryBecomeLeaderInternal(ServerCoordinators coordinators,
 				// These coordinators are forwarded to another set.  But before we change our own cluster file, we need
 				// to make sure that a majority of coordinators know that. SOMEDAY: Wait briefly to see if other
 				// coordinators will tell us they already know, to save communication?
-				wait(changeLeaderCoordinators(coordinators, leader.get().first.serializedInfo));
+				// NOTE: If a majority of coordinators (in the current connection string) have failed then we can
+				// end up waiting here indefinitely. Try to make progress in that scenario by proceeding with the
+				// connection string that we have received. Not a great solution, but can help in certain scenarios.
+				choose {
+					when(wait(changeLeaderCoordinators(coordinators, leader.get().first.serializedInfo))) {}
+					when(wait(delay(20))) {}
+				}
 
 				if (!hasConnected) {
 					TraceEvent(SevWarnAlways, "IncorrectClusterFileContentsAtConnection")

From d0cb8cdf27c8cce083647be422c0491559d807d4 Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Mon, 3 Apr 2023 14:53:24 -0700
Subject: [PATCH 127/468] Update 7.1.30 release notes

---
 .../sphinx/source/release-notes/release-notes-710.rst       | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/documentation/sphinx/source/release-notes/release-notes-710.rst b/documentation/sphinx/source/release-notes/release-notes-710.rst
index 621fce54b6d..cd56473903b 100644
--- a/documentation/sphinx/source/release-notes/release-notes-710.rst
+++ b/documentation/sphinx/source/release-notes/release-notes-710.rst
@@ -12,11 +12,13 @@ Release Notes
 ======
 * Released with AVX disabled.
 * Fixed storage server finishedQueries metric when using getMappedRange. `(PR #9785) <https://github.com/apple/foundationdb/pull/9785>`_
-* Fixed unnecessary transaction system recovery when excluding the servers that are already excluded/failed. `(PR #9809) <https://github.com/apple/foundationdb/pull/9809>`_
-* Fixed the exclusion of stateless processes by skipping the free capacity check. `(PR #9789) <https://github.com/apple/foundationdb/pull/9789>`_
+* Fixed unnecessary transaction system recovery when excluding the servers that are already excluded/failed. `(PR #9809) <https://github.com/apple/foundationdb/pull/9809>`_ and `(PR #9878) <https://github.com/apple/foundationdb/pull/9878>`_
+* Fixed the exclusion of stateless processes by skipping the free capacity check. `(PR #9789) <https://github.com/apple/foundationdb/pull/9789>`_ and `(PR #9769) <https://github.com/apple/foundationdb/pull/9769>`_
 * Fixed an issue where the new worker cannot get ServerDBInfo update. `(PR #9778) <https://github.com/apple/foundationdb/pull/9778>`_
 * Added RocksDB bloom filter knobs. `(PR #9770) <https://github.com/apple/foundationdb/pull/9770>`_
 * Upgraded RocksDB to version 7.10.2. `(PR #9829) <https://github.com/apple/foundationdb/pull/9829>`_
+* Fixed an issue where ExclusionSafetyCheckRequest could be blocked forever. `(PR #9871) <https://github.com/apple/foundationdb/pull/9871>`_
+* Fixed fdbserver not able to join the cluster if the majority of coordinators in its connection string have changed. `(PR #9883) <https://github.com/apple/foundationdb/pull/9883>`_
 
 7.1.29
 ======

From f3ae0c7b9788b81fcd89c530b77fc2803e13c474 Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Mon, 3 Apr 2023 21:06:52 -0700
Subject: [PATCH 128/468] Remove printable() from TSS trace events

---
 fdbclient/StorageServerInterface.cpp | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/fdbclient/StorageServerInterface.cpp b/fdbclient/StorageServerInterface.cpp
index b7efbfbc836..71097c939e3 100644
--- a/fdbclient/StorageServerInterface.cpp
+++ b/fdbclient/StorageServerInterface.cpp
@@ -48,7 +48,7 @@ void TSS_traceMismatch(TraceEvent& event,
                        const GetValueRequest& req,
                        const GetValueReply& src,
                        const GetValueReply& tss) {
-	event.detail("Key", req.key.printable())
+	event.detail("Key", req.key)
 	    .detail("Tenant", req.tenantInfo.name)
 	    .detail("Version", req.version)
 	    .detail("SSReply", src.value.present() ? traceChecksumValue(src.value.get()) : "missing")
@@ -167,10 +167,10 @@ static void traceKeyValuesDiff(TraceEvent& event,
 		if (i >= ssKV.size() || i >= tssKV.size() || ssKV[i] != tssKV[i]) {
 			event.detail("MismatchIndex", i);
 			if (i >= ssKV.size() || i >= tssKV.size() || ssKV[i].key != tssKV[i].key) {
-				event.detail("MismatchSSKey", i < ssKV.size() ? ssKV[i].key.printable() : "missing");
-				event.detail("MismatchTSSKey", i < tssKV.size() ? tssKV[i].key.printable() : "missing");
+				event.detail("MismatchSSKey", i < ssKV.size() ? ssKV[i].key : "missing"_sr);
+				event.detail("MismatchTSSKey", i < tssKV.size() ? tssKV[i].key : "missing"_sr);
 			} else {
-				event.detail("MismatchKey", ssKV[i].key.printable());
+				event.detail("MismatchKey", ssKV[i].key);
 				event.detail("MismatchSSValue", traceChecksumValue(ssKV[i].value));
 				event.detail("MismatchTSSValue", traceChecksumValue(tssKV[i].value));
 			}

From af31ee23bbbce05c53618ad73832e84b90bebe9d Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Tue, 4 Apr 2023 09:31:04 -0700
Subject: [PATCH 129/468] Fix releast notes

---
 documentation/sphinx/source/release-notes/release-notes-710.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/documentation/sphinx/source/release-notes/release-notes-710.rst b/documentation/sphinx/source/release-notes/release-notes-710.rst
index cd56473903b..d12024b27f6 100644
--- a/documentation/sphinx/source/release-notes/release-notes-710.rst
+++ b/documentation/sphinx/source/release-notes/release-notes-710.rst
@@ -18,7 +18,7 @@ Release Notes
 * Added RocksDB bloom filter knobs. `(PR #9770) <https://github.com/apple/foundationdb/pull/9770>`_
 * Upgraded RocksDB to version 7.10.2. `(PR #9829) <https://github.com/apple/foundationdb/pull/9829>`_
 * Fixed an issue where ExclusionSafetyCheckRequest could be blocked forever. `(PR #9871) <https://github.com/apple/foundationdb/pull/9871>`_
-* Fixed fdbserver not able to join the cluster if the majority of coordinators in its connection string have changed. `(PR #9883) <https://github.com/apple/foundationdb/pull/9883>`_
+* Fixed fdbserver not able to join the cluster if the majority of coordinators in its connection string have failed. `(PR #9883) <https://github.com/apple/foundationdb/pull/9883>`_
 
 7.1.29
 ======

From c6e4a3dc872fc733c696a61e39ae217af29211e8 Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyu_zhou@apple.com>
Date: Wed, 5 Apr 2023 13:54:45 -0700
Subject: [PATCH 130/468] Fixed stuck data movement when a server is removed
 [release-7.1] (#9904)

* Fixed stuck data movement when a server is removed

When a server is removed, dataDistributionRelocator doesn't remove the work for
the destination storage workers. As a result, it can no long move shard into
any of the healthy workers in the destination team.

* Avoid double complete the work
---
 .../source/release-notes/release-notes-710.rst |  1 +
 fdbserver/DataDistributionQueue.actor.cpp      | 18 ++++++++++++++----
 2 files changed, 15 insertions(+), 4 deletions(-)

diff --git a/documentation/sphinx/source/release-notes/release-notes-710.rst b/documentation/sphinx/source/release-notes/release-notes-710.rst
index d12024b27f6..eb711318fb5 100644
--- a/documentation/sphinx/source/release-notes/release-notes-710.rst
+++ b/documentation/sphinx/source/release-notes/release-notes-710.rst
@@ -19,6 +19,7 @@ Release Notes
 * Upgraded RocksDB to version 7.10.2. `(PR #9829) <https://github.com/apple/foundationdb/pull/9829>`_
 * Fixed an issue where ExclusionSafetyCheckRequest could be blocked forever. `(PR #9871) <https://github.com/apple/foundationdb/pull/9871>`_
 * Fixed fdbserver not able to join the cluster if the majority of coordinators in its connection string have failed. `(PR #9883) <https://github.com/apple/foundationdb/pull/9883>`_
+* Fixed stuck data movement when moving to removed a storage server. `(PR #9904) <https://github.com/apple/foundationdb/pull/9904>`_
 
 7.1.29
 ======
diff --git a/fdbserver/DataDistributionQueue.actor.cpp b/fdbserver/DataDistributionQueue.actor.cpp
index 081efd8a307..016f40d1fcf 100644
--- a/fdbserver/DataDistributionQueue.actor.cpp
+++ b/fdbserver/DataDistributionQueue.actor.cpp
@@ -386,15 +386,19 @@ void launchDest(RelocateData& relocation,
 	}
 }
 
+void completeDest(const RelocateData& relocation, std::map<UID, Busyness>& destBusymap) {
+	int destWorkFactor = getDestWorkFactor();
+	for (UID id : relocation.completeDests) {
+		destBusymap[id].removeWork(relocation.priority, destWorkFactor);
+	}
+}
+
 void complete(RelocateData const& relocation, std::map<UID, Busyness>& busymap, std::map<UID, Busyness>& destBusymap) {
 	ASSERT(relocation.workFactor > 0);
 	for (int i = 0; i < relocation.src.size(); i++)
 		busymap[relocation.src[i]].removeWork(relocation.priority, relocation.workFactor);
 
-	int destWorkFactor = getDestWorkFactor();
-	for (UID id : relocation.completeDests) {
-		destBusymap[id].removeWork(relocation.priority, destWorkFactor);
-	}
+	completeDest(relocation, destBusymap);
 }
 
 ACTOR Future<Void> dataDistributionRelocator(struct DDQueueData* self,
@@ -1388,6 +1392,12 @@ ACTOR Future<Void> dataDistributionRelocator(DDQueueData* self, RelocateData rd,
 			} else {
 				TEST(true); // move to removed server
 				healthyDestinations.addDataInFlightToTeam(-metrics.bytes);
+				if (!signalledTransferComplete) {
+					// Update destination busyness map to allow further movement to them.
+					// signalling transferComplete calls completeDest() in complete(), so
+					// doing so here would double-complete the work.
+					completeDest(rd, self->destBusymap);
+				}
 				rd.completeDests.clear();
 				wait(delay(SERVER_KNOBS->RETRY_RELOCATESHARD_DELAY, TaskPriority::DataDistributionLaunch));
 			}

From bfa0673dd4535fd12fbf408d9214f885f1d1f271 Mon Sep 17 00:00:00 2001
From: FoundationDB CI <foundationdb_ci@apple.com>
Date: Thu, 6 Apr 2023 17:09:11 +0000
Subject: [PATCH 131/468] disable AVX for 7.1.30 release

---
 cmake/ConfigureCompiler.cmake | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cmake/ConfigureCompiler.cmake b/cmake/ConfigureCompiler.cmake
index e52d8665ee8..7e50469ba6a 100644
--- a/cmake/ConfigureCompiler.cmake
+++ b/cmake/ConfigureCompiler.cmake
@@ -253,7 +253,7 @@ else()
       set(USE_AVX512F OFF)
     endif()
   endif()
-  set(USE_AVX ON CACHE BOOL "Enable AVX instructions")
+  set(USE_AVX OFF CACHE BOOL "Enable AVX instructions")
   if (USE_AVX)
     if (CMAKE_HOST_SYSTEM_PROCESSOR MATCHES "^x86")
       add_compile_options(-mavx)

From 94e42d5c64f55d3acf46a81bd6d46f424652fe90 Mon Sep 17 00:00:00 2001
From: FoundationDB CI <foundationdb_ci@apple.com>
Date: Thu, 6 Apr 2023 17:09:25 +0000
Subject: [PATCH 132/468] enable AVX and update version for 7.1.31 release

---
 CMakeLists.txt                | 2 +-
 cmake/ConfigureCompiler.cmake | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 7659c210c06..f069909c01d 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -23,7 +23,7 @@ else()
 endif()
 
 project(foundationdb
-  VERSION 7.1.30
+  VERSION 7.1.31
   DESCRIPTION "FoundationDB is a scalable, fault-tolerant, ordered key-value store with full ACID transactions."
   HOMEPAGE_URL "http://www.foundationdb.org/"
   LANGUAGES C CXX ASM)
diff --git a/cmake/ConfigureCompiler.cmake b/cmake/ConfigureCompiler.cmake
index 7e50469ba6a..e52d8665ee8 100644
--- a/cmake/ConfigureCompiler.cmake
+++ b/cmake/ConfigureCompiler.cmake
@@ -253,7 +253,7 @@ else()
       set(USE_AVX512F OFF)
     endif()
   endif()
-  set(USE_AVX OFF CACHE BOOL "Enable AVX instructions")
+  set(USE_AVX ON CACHE BOOL "Enable AVX instructions")
   if (USE_AVX)
     if (CMAKE_HOST_SYSTEM_PROCESSOR MATCHES "^x86")
       add_compile_options(-mavx)

From 98b2fb9221ee3d0c8b718c027e486ab832c9eab0 Mon Sep 17 00:00:00 2001
From: FoundationDB CI <foundationdb_ci@apple.com>
Date: Fri, 7 Apr 2023 16:55:40 +0000
Subject: [PATCH 133/468] update version after 7.1.31 release

---
 CMakeLists.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index f069909c01d..02e3fed12ca 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -23,7 +23,7 @@ else()
 endif()
 
 project(foundationdb
-  VERSION 7.1.31
+  VERSION 7.1.32
   DESCRIPTION "FoundationDB is a scalable, fault-tolerant, ordered key-value store with full ACID transactions."
   HOMEPAGE_URL "http://www.foundationdb.org/"
   LANGUAGES C CXX ASM)

From ef78ed382f3a7345d7b5ea069cee892d7b5ce463 Mon Sep 17 00:00:00 2001
From: Zhe Wu <halfprice@users.noreply.github.com>
Date: Mon, 3 Apr 2023 21:53:42 -0700
Subject: [PATCH 134/468] Gray failure detection account for disconnected
 remote log routers

---
 fdbserver/ClusterController.actor.h |  18 +++--
 fdbserver/worker.actor.cpp          | 106 ++++++++++++++++++++++++++--
 2 files changed, 111 insertions(+), 13 deletions(-)

diff --git a/fdbserver/ClusterController.actor.h b/fdbserver/ClusterController.actor.h
index ef96581d7d0..175e45e6935 100644
--- a/fdbserver/ClusterController.actor.h
+++ b/fdbserver/ClusterController.actor.h
@@ -3105,26 +3105,32 @@ class ClusterControllerData {
 	bool transactionSystemContainsDegradedServers() {
 		const ServerDBInfo& dbi = db.serverInfo->get();
 		auto transactionWorkerInList = [&dbi](const std::unordered_set<NetworkAddress>& serverList,
-		                                      bool skipSatellite) -> bool {
+		                                      bool skipSatellite, bool skipRemote) -> bool {
 			for (const auto& server : serverList) {
 				if (dbi.master.addresses().contains(server)) {
 					return true;
 				}
 
 				for (const auto& logSet : dbi.logSystemConfig.tLogs) {
-					if (!logSet.isLocal) {
-						// We don't check server degradation for remote TLogs since it is not on the transaction system
-						// critical path.
+					if (skipRemote && !logSet.isLocal) {
 						continue;
 					}
+
 					if (skipSatellite && logSet.locality == tagLocalitySatellite) {
 						continue;
 					}
+
 					for (const auto& tlog : logSet.tLogs) {
 						if (tlog.present() && tlog.interf().addresses().contains(server)) {
 							return true;
 						}
 					}
+
+					for (const auto& logRouter : logSet.logRouters) {
+						if (logRouter.present() && logRouter.interf().addresses().contains(server)) {
+							return true;
+						}
+					}
 				}
 
 				for (const auto& proxy : dbi.client.grvProxies) {
@@ -3151,8 +3157,8 @@ class ClusterControllerData {
 
 		// Check if transaction system contains degraded/disconnected servers. For satellite, we only check for
 		// disconnection since the latency between prmary and satellite is across WAN and may not be very stable.
-		return transactionWorkerInList(degradationInfo.degradedServers, /*skipSatellite=*/true) ||
-		       transactionWorkerInList(degradationInfo.disconnectedServers, /*skipSatellite=*/false);
+		return transactionWorkerInList(degradationInfo.degradedServers, /*skipSatellite=*/true, /*skipRemote=*/true) ||
+		       transactionWorkerInList(degradationInfo.disconnectedServers, /*skipSatellite=*/false, /*skipRemote=*/false);
 	}
 
 	// Whether transaction system in the remote DC, e.g. log router and tlogs in the remote DC, contains degraded
diff --git a/fdbserver/worker.actor.cpp b/fdbserver/worker.actor.cpp
index 56eb27204b4..ce4d6a03178 100644
--- a/fdbserver/worker.actor.cpp
+++ b/fdbserver/worker.actor.cpp
@@ -955,6 +955,68 @@ TEST_CASE("/fdbserver/worker/addressInDbAndRemoteDc") {
 
 } // namespace
 
+bool addressIsRemoteLogRouter(const NetworkAddress& address, Reference<AsyncVar<ServerDBInfo> const> dbInfo) {
+	const auto& dbi = dbInfo->get();
+
+	for (const auto& logSet : dbi.logSystemConfig.tLogs) {
+		if (logSet.isLocal || logSet.locality == tagLocalitySatellite) {
+			continue;
+		}
+
+		for (const auto& logRouter : logSet.logRouters) {
+			if (logRouter.present() && logRouter.interf().addresses().contains(address)) {
+				return true;
+			}
+		}
+	}
+
+	return false;
+}
+
+namespace {
+
+TEST_CASE("/fdbserver/worker/addressIsRemoteLogRouter") {
+	// Setup a ServerDBInfo for test.
+	ServerDBInfo testDbInfo;
+	LocalityData testLocal;
+	testLocal.set("dcid"_sr, StringRef(std::to_string(1)));
+	testDbInfo.master.locality = testLocal;
+
+	// First, create an empty TLogInterface, and check that it shouldn't be considered as remote log router.
+	testDbInfo.logSystemConfig.tLogs.push_back(TLogSet());
+	testDbInfo.logSystemConfig.tLogs.back().isLocal = true;
+	testDbInfo.logSystemConfig.tLogs.back().logRouters.push_back(OptionalInterface<TLogInterface>());
+	ASSERT(!addressIsRemoteLogRouter(g_network->getLocalAddress(), makeReference<AsyncVar<ServerDBInfo>>(testDbInfo)));
+
+	TLogInterface localLogRouter(testLocal);
+	localLogRouter.initEndpoints();
+	testDbInfo.logSystemConfig.tLogs.back().logRouters.push_back(OptionalInterface(localLogRouter));
+	ASSERT(!addressIsRemoteLogRouter(g_network->getLocalAddress(), makeReference<AsyncVar<ServerDBInfo>>(testDbInfo)));
+
+	// Create a remote TLog, and it shouldn't be considered as remote log router.
+	LocalityData fakeRemote;
+	fakeRemote.set("dcid"_sr, StringRef(std::to_string(2)));
+	TLogInterface remoteTlog(fakeRemote);
+	remoteTlog.initEndpoints();
+
+	testDbInfo.logSystemConfig.tLogs.push_back(TLogSet());
+	testDbInfo.logSystemConfig.tLogs.back().isLocal = false;
+	testDbInfo.logSystemConfig.tLogs.back().tLogs.push_back(OptionalInterface(remoteTlog));
+	ASSERT(!addressIsRemoteLogRouter(g_network->getLocalAddress(), makeReference<AsyncVar<ServerDBInfo>>(testDbInfo)));
+
+	// Create a remote log router, and it should be considered as remote log router.
+	NetworkAddress logRouterAddress(IPAddress(0x26262626), 1);
+	TLogInterface remoteLogRouter(fakeRemote);
+	remoteLogRouter.initEndpoints();
+	remoteLogRouter.peekMessages = RequestStream<struct TLogPeekRequest>(Endpoint({ logRouterAddress }, UID(1, 2)));
+	testDbInfo.logSystemConfig.tLogs.back().logRouters.push_back(OptionalInterface(remoteLogRouter));
+	ASSERT(addressIsRemoteLogRouter(logRouterAddress, makeReference<AsyncVar<ServerDBInfo>>(testDbInfo)));
+
+	return Void();
+}
+
+} // namespace
+
 // The actor that actively monitors the health of local and peer servers, and reports anomaly to the cluster controller.
 ACTOR Future<Void> healthMonitor(Reference<AsyncVar<Optional<ClusterControllerFullInterface>> const> ccInterface,
                                  WorkerInterface interf,
@@ -965,14 +1027,20 @@ ACTOR Future<Void> healthMonitor(Reference<AsyncVar<Optional<ClusterControllerFu
 		if (dbInfo->get().recoveryState >= RecoveryState::ACCEPTING_COMMITS && ccInterface->get().present()) {
 			nextHealthCheckDelay = delay(SERVER_KNOBS->WORKER_HEALTH_MONITOR_INTERVAL);
 			const auto& allPeers = FlowTransport::transport().getAllPeers();
+
+			// Check remote log router connectivity only when remote TLogs are recruited and in use.
+			bool checkRemoteLogRouterConnectivity = dbInfo->get().recoveryState == RecoveryState::ALL_LOGS_RECRUITED ||
+			                                        dbInfo->get().recoveryState == RecoveryState::FULLY_RECOVERED;
 			UpdateWorkerHealthRequest req;
 
-			enum WorkerLocation { None, Primary, Remote };
+			enum WorkerLocation { None, Primary, Satellite, Remote };
 			WorkerLocation workerLocation = None;
 			if (addressesInDbAndPrimaryDc(interf.addresses(), dbInfo)) {
 				workerLocation = Primary;
 			} else if (addressesInDbAndRemoteDc(interf.addresses(), dbInfo)) {
 				workerLocation = Remote;
+			} else if (addressesInDbAndPrimarySatelliteDc(interf.addresses(), dbInfo)) {
+				workerLocation = Satellite;
 			}
 
 			if (workerLocation != None) {
@@ -981,11 +1049,10 @@ ACTOR Future<Void> healthMonitor(Reference<AsyncVar<Optional<ClusterControllerFu
 					    peer->pingLatencies.getPopulationSize() < SERVER_KNOBS->PEER_LATENCY_CHECK_MIN_POPULATION) {
 						// Ignore peers that don't have enough samples.
 						// TODO(zhewu): Currently, FlowTransport latency monitor clears ping latency samples on a
-						// regular
-						//              basis, which may affect the measurement count. Currently,
-						//              WORKER_HEALTH_MONITOR_INTERVAL is much smaller than the ping clearance interval,
-						//              so it may be ok. If this ends to be a problem, we need to consider keep track of
-						//              last ping latencies logged.
+						// regular basis, which may affect the measurement count. Currently,
+						// WORKER_HEALTH_MONITOR_INTERVAL is much smaller than the ping clearance interval, so it may be
+						// ok. If this ends to be a problem, we need to consider keep track of last ping latencies
+						// logged.
 						continue;
 					}
 					bool degradedPeer = false;
@@ -1056,6 +1123,28 @@ ACTOR Future<Void> healthMonitor(Reference<AsyncVar<Optional<ClusterControllerFu
 							    .detail("PingTimeoutCount", peer->timeoutCount)
 							    .detail("ConnectionFailureCount", peer->connectFailedCount);
 						}
+					} else if (checkRemoteLogRouterConnectivity &&
+					           (workerLocation == Primary || workerLocation == Satellite) &&
+					           addressIsRemoteLogRouter(address, dbInfo)) {
+						// Monitor remote log router's connectivity to the primary DCs' transaction system. We ignore
+						// latency based degradation between primary region and remote region due to that remote region
+						// may be distant from primary region.
+						if (peer->connectFailedCount >= SERVER_KNOBS->PEER_DEGRADATION_CONNECTION_FAILURE_COUNT) {
+							TraceEvent("HealthMonitorDetectDegradedPeer")
+							    .detail("WorkerLocation", workerLocation)
+							    .detail("Peer", address)
+							    .detail("RemoteLogRouter", true)
+							    .detail("Elapsed", now() - lastLoggedTime)
+							    .detail("Disconnected", true)
+							    .detail("MinLatency", peer->pingLatencies.min())
+							    .detail("MaxLatency", peer->pingLatencies.max())
+							    .detail("MeanLatency", peer->pingLatencies.mean())
+							    .detail("MedianLatency", peer->pingLatencies.median())
+							    .detail("PingCount", peer->pingLatencies.getPopulationSize())
+							    .detail("PingTimeoutCount", peer->timeoutCount)
+							    .detail("ConnectionFailureCount", peer->connectFailedCount);
+							disconnectedPeer = true;
+						}
 					}
 
 					if (disconnectedPeer) {
@@ -1079,7 +1168,10 @@ ACTOR Future<Void> healthMonitor(Reference<AsyncVar<Optional<ClusterControllerFu
 
 						if ((workerLocation == Primary && addressInDbAndPrimaryDc(address, dbInfo)) ||
 						    (workerLocation == Remote && addressInDbAndRemoteDc(address, dbInfo)) ||
-						    (workerLocation == Primary && addressInDbAndPrimarySatelliteDc(address, dbInfo))) {
+						    (workerLocation == Primary && addressInDbAndPrimarySatelliteDc(address, dbInfo)) ||
+						    (checkRemoteLogRouterConnectivity &&
+						     (workerLocation == Primary || workerLocation == Satellite) &&
+						     addressIsRemoteLogRouter(address, dbInfo))) {
 							TraceEvent("HealthMonitorDetectRecentClosedPeer").suppressFor(30).detail("Peer", address);
 							req.disconnectedPeers.push_back(address);
 						}

From 99df8947928e6c55257b994f385ed0a852a767c6 Mon Sep 17 00:00:00 2001
From: Zhe Wu <halfprice@users.noreply.github.com>
Date: Mon, 3 Apr 2023 22:13:29 -0700
Subject: [PATCH 135/468] Adding test to test that disconnected remote log
 router would trigger a recovery

---
 fdbserver/ClusterController.actor.cpp | 12 ++++++-----
 fdbserver/ClusterController.actor.h   | 31 ++++++++++++++++-----------
 2 files changed, 25 insertions(+), 18 deletions(-)

diff --git a/fdbserver/ClusterController.actor.cpp b/fdbserver/ClusterController.actor.cpp
index cd6b78e7600..f0f601c2eaa 100644
--- a/fdbserver/ClusterController.actor.cpp
+++ b/fdbserver/ClusterController.actor.cpp
@@ -3088,14 +3088,11 @@ TEST_CASE("/fdbserver/clustercontroller/shouldTriggerRecoveryDueToDegradedServer
 
 	TLogInterface localTLogInterf;
 	localTLogInterf.peekMessages = RequestStream<struct TLogPeekRequest>(Endpoint({ tlog }, testUID));
-	TLogInterface localLogRouterInterf;
-	localLogRouterInterf.peekMessages = RequestStream<struct TLogPeekRequest>(Endpoint({ logRouter }, testUID));
 	BackupInterface backupInterf;
 	backupInterf.waitFailure = RequestStream<ReplyPromise<Void>>(Endpoint({ backup }, testUID));
 	TLogSet localTLogSet;
 	localTLogSet.isLocal = true;
 	localTLogSet.tLogs.push_back(OptionalInterface(localTLogInterf));
-	localTLogSet.logRouters.push_back(OptionalInterface(localLogRouterInterf));
 	localTLogSet.backupWorkers.push_back(OptionalInterface(backupInterf));
 	testDbInfo.logSystemConfig.tLogs.push_back(localTLogSet);
 
@@ -3109,9 +3106,12 @@ TEST_CASE("/fdbserver/clustercontroller/shouldTriggerRecoveryDueToDegradedServer
 
 	TLogInterface remoteTLogInterf;
 	remoteTLogInterf.peekMessages = RequestStream<struct TLogPeekRequest>(Endpoint({ remoteTlog }, testUID));
+	TLogInterface remoteLogRouterInterf;
+	remoteLogRouterInterf.peekMessages = RequestStream<struct TLogPeekRequest>(Endpoint({ logRouter }, testUID));
 	TLogSet remoteTLogSet;
 	remoteTLogSet.isLocal = false;
 	remoteTLogSet.tLogs.push_back(OptionalInterface(remoteTLogInterf));
+	remoteTLogSet.logRouters.push_back(OptionalInterface(remoteLogRouterInterf));
 	testDbInfo.logSystemConfig.tLogs.push_back(remoteTLogSet);
 
 	GrvProxyInterface proxyInterf;
@@ -3162,12 +3162,14 @@ TEST_CASE("/fdbserver/clustercontroller/shouldTriggerRecoveryDueToDegradedServer
 	ASSERT(!data.shouldTriggerRecoveryDueToDegradedServers());
 	data.degradationInfo.disconnectedServers.clear();
 
-	// No recovery when log router is degraded.
+	// No recovery when remote log router is degraded.
 	data.degradationInfo.degradedServers.insert(logRouter);
 	ASSERT(!data.shouldTriggerRecoveryDueToDegradedServers());
 	data.degradationInfo.degradedServers.clear();
+
+	// Trigger recovery when remote log router is disconnected.
 	data.degradationInfo.disconnectedServers.insert(logRouter);
-	ASSERT(!data.shouldTriggerRecoveryDueToDegradedServers());
+	ASSERT(data.shouldTriggerRecoveryDueToDegradedServers());
 	data.degradationInfo.disconnectedServers.clear();
 
 	// No recovery when backup worker is degraded.
diff --git a/fdbserver/ClusterController.actor.h b/fdbserver/ClusterController.actor.h
index 175e45e6935..070b6802933 100644
--- a/fdbserver/ClusterController.actor.h
+++ b/fdbserver/ClusterController.actor.h
@@ -3104,8 +3104,8 @@ class ClusterControllerData {
 	// Whether the transaction system (in primary DC if in HA setting) contains degraded servers.
 	bool transactionSystemContainsDegradedServers() {
 		const ServerDBInfo& dbi = db.serverInfo->get();
-		auto transactionWorkerInList = [&dbi](const std::unordered_set<NetworkAddress>& serverList,
-		                                      bool skipSatellite, bool skipRemote) -> bool {
+		auto transactionWorkerInList =
+		    [&dbi](const std::unordered_set<NetworkAddress>& serverList, bool skipSatellite, bool skipRemote) -> bool {
 			for (const auto& server : serverList) {
 				if (dbi.master.addresses().contains(server)) {
 					return true;
@@ -3120,15 +3120,18 @@ class ClusterControllerData {
 						continue;
 					}
 
-					for (const auto& tlog : logSet.tLogs) {
-						if (tlog.present() && tlog.interf().addresses().contains(server)) {
-							return true;
+					if (!logSet.isLocal) {
+						// Only check log routers in the remote region.
+						for (const auto& logRouter : logSet.logRouters) {
+							if (logRouter.present() && logRouter.interf().addresses().contains(server)) {
+								return true;
+							}
 						}
-					}
-
-					for (const auto& logRouter : logSet.logRouters) {
-						if (logRouter.present() && logRouter.interf().addresses().contains(server)) {
-							return true;
+					} else {
+						for (const auto& tlog : logSet.tLogs) {
+							if (tlog.present() && tlog.interf().addresses().contains(server)) {
+								return true;
+							}
 						}
 					}
 				}
@@ -3155,10 +3158,12 @@ class ClusterControllerData {
 			return false;
 		};
 
-		// Check if transaction system contains degraded/disconnected servers. For satellite, we only check for
-		// disconnection since the latency between prmary and satellite is across WAN and may not be very stable.
+		// Check if transaction system contains degraded/disconnected servers. For satellite and remote regions, we only
+		// check for disconnection since the latency between prmary and satellite is across WAN and may not be very
+		// stable.
 		return transactionWorkerInList(degradationInfo.degradedServers, /*skipSatellite=*/true, /*skipRemote=*/true) ||
-		       transactionWorkerInList(degradationInfo.disconnectedServers, /*skipSatellite=*/false, /*skipRemote=*/false);
+		       transactionWorkerInList(
+		           degradationInfo.disconnectedServers, /*skipSatellite=*/false, /*skipRemote=*/false);
 	}
 
 	// Whether transaction system in the remote DC, e.g. log router and tlogs in the remote DC, contains degraded

From 53cbce785774d091813d760a31bdd5c7e8420dec Mon Sep 17 00:00:00 2001
From: Zhe Wu <halfprice@users.noreply.github.com>
Date: Mon, 3 Apr 2023 22:31:22 -0700
Subject: [PATCH 136/468] Adding a knob to control log router triggered
 recovery

---
 fdbclient/ServerKnobs.cpp           | 1 +
 fdbclient/ServerKnobs.h             | 2 ++
 fdbserver/ClusterController.actor.h | 6 +++---
 3 files changed, 6 insertions(+), 3 deletions(-)

diff --git a/fdbclient/ServerKnobs.cpp b/fdbclient/ServerKnobs.cpp
index 20d16833100..9d8861b33c1 100644
--- a/fdbclient/ServerKnobs.cpp
+++ b/fdbclient/ServerKnobs.cpp
@@ -592,6 +592,7 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi
 	init( CC_ENABLE_ENTIRE_SATELLITE_MONITORING,               false );
 	init( CC_SATELLITE_DEGRADATION_MIN_COMPLAINER,                 3 );
 	init( CC_SATELLITE_DEGRADATION_MIN_BAD_SERVER,                 3 );
+	init( CC_ENABLE_REMOTE_LOG_ROUTER_MONITORING,               true );
 	init( CC_THROTTLE_SINGLETON_RERECRUIT_INTERVAL,              0.5 );
 
 	init( INCOMPATIBLE_PEERS_LOGGING_INTERVAL,                   600 ); if( randomize && BUGGIFY ) INCOMPATIBLE_PEERS_LOGGING_INTERVAL = 60.0;
diff --git a/fdbclient/ServerKnobs.h b/fdbclient/ServerKnobs.h
index 7295ac7466e..81f8b4d2040 100644
--- a/fdbclient/ServerKnobs.h
+++ b/fdbclient/ServerKnobs.h
@@ -510,6 +510,8 @@ class ServerKnobs : public KnobsImpl<ServerKnobs> {
 	                                             // be determined as degraded worker.
 	int CC_SATELLITE_DEGRADATION_MIN_BAD_SERVER; // The minimum amount of degraded server in satellite DC to be
 	                                             // determined as degraded satellite.
+	bool CC_ENABLE_REMOTE_LOG_ROUTER_MONITORING; // When enabled, gray failure tries to detect whether the remote log
+	                                             // router is degraded and may use trigger recovery to recover from it.
 	double CC_THROTTLE_SINGLETON_RERECRUIT_INTERVAL; // The interval to prevent re-recruiting the same singleton if a
 	                                                 // recruiting fight between two cluster controllers occurs.
 
diff --git a/fdbserver/ClusterController.actor.h b/fdbserver/ClusterController.actor.h
index 070b6802933..7252b5cdef1 100644
--- a/fdbserver/ClusterController.actor.h
+++ b/fdbserver/ClusterController.actor.h
@@ -3112,11 +3112,11 @@ class ClusterControllerData {
 				}
 
 				for (const auto& logSet : dbi.logSystemConfig.tLogs) {
-					if (skipRemote && !logSet.isLocal) {
+					if (skipSatellite && logSet.locality == tagLocalitySatellite) {
 						continue;
 					}
 
-					if (skipSatellite && logSet.locality == tagLocalitySatellite) {
+					if (skipRemote && !logSet.isLocal) {
 						continue;
 					}
 
@@ -3163,7 +3163,7 @@ class ClusterControllerData {
 		// stable.
 		return transactionWorkerInList(degradationInfo.degradedServers, /*skipSatellite=*/true, /*skipRemote=*/true) ||
 		       transactionWorkerInList(
-		           degradationInfo.disconnectedServers, /*skipSatellite=*/false, /*skipRemote=*/false);
+		           degradationInfo.disconnectedServers, /*skipSatellite=*/false, /*skipRemote=*/!SERVER_KNOBS->CC_ENABLE_REMOTE_LOG_ROUTER_MONITORING);
 	}
 
 	// Whether transaction system in the remote DC, e.g. log router and tlogs in the remote DC, contains degraded

From e87ed1358302ff364646a7f30ce1adb585b7b586 Mon Sep 17 00:00:00 2001
From: Zhe Wu <halfprice@users.noreply.github.com>
Date: Mon, 3 Apr 2023 22:44:32 -0700
Subject: [PATCH 137/468] Apply format

---
 fdbserver/ClusterController.actor.h | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/fdbserver/ClusterController.actor.h b/fdbserver/ClusterController.actor.h
index 7252b5cdef1..265211ce283 100644
--- a/fdbserver/ClusterController.actor.h
+++ b/fdbserver/ClusterController.actor.h
@@ -3162,8 +3162,9 @@ class ClusterControllerData {
 		// check for disconnection since the latency between prmary and satellite is across WAN and may not be very
 		// stable.
 		return transactionWorkerInList(degradationInfo.degradedServers, /*skipSatellite=*/true, /*skipRemote=*/true) ||
-		       transactionWorkerInList(
-		           degradationInfo.disconnectedServers, /*skipSatellite=*/false, /*skipRemote=*/!SERVER_KNOBS->CC_ENABLE_REMOTE_LOG_ROUTER_MONITORING);
+		       transactionWorkerInList(degradationInfo.disconnectedServers,
+		                               /*skipSatellite=*/false,
+		                               /*skipRemote=*/!SERVER_KNOBS->CC_ENABLE_REMOTE_LOG_ROUTER_MONITORING);
 	}
 
 	// Whether transaction system in the remote DC, e.g. log router and tlogs in the remote DC, contains degraded

From 78c8bdc84444f172fc2729499e39e129aa6fd9c4 Mon Sep 17 00:00:00 2001
From: Zhe Wu <halfprice@users.noreply.github.com>
Date: Thu, 6 Apr 2023 11:36:49 -0700
Subject: [PATCH 138/468] Update log router check and add some documents

---
 fdbserver/worker.actor.cpp | 13 ++++++-------
 1 file changed, 6 insertions(+), 7 deletions(-)

diff --git a/fdbserver/worker.actor.cpp b/fdbserver/worker.actor.cpp
index ce4d6a03178..e95eee465ca 100644
--- a/fdbserver/worker.actor.cpp
+++ b/fdbserver/worker.actor.cpp
@@ -959,13 +959,11 @@ bool addressIsRemoteLogRouter(const NetworkAddress& address, Reference<AsyncVar<
 	const auto& dbi = dbInfo->get();
 
 	for (const auto& logSet : dbi.logSystemConfig.tLogs) {
-		if (logSet.isLocal || logSet.locality == tagLocalitySatellite) {
-			continue;
-		}
-
-		for (const auto& logRouter : logSet.logRouters) {
-			if (logRouter.present() && logRouter.interf().addresses().contains(address)) {
-				return true;
+		if (!logSet.isLocal) {
+			for (const auto& logRouter : logSet.logRouters) {
+				if (logRouter.present() && logRouter.interf().addresses().contains(address)) {
+					return true;
+				}
 			}
 		}
 	}
@@ -988,6 +986,7 @@ TEST_CASE("/fdbserver/worker/addressIsRemoteLogRouter") {
 	testDbInfo.logSystemConfig.tLogs.back().logRouters.push_back(OptionalInterface<TLogInterface>());
 	ASSERT(!addressIsRemoteLogRouter(g_network->getLocalAddress(), makeReference<AsyncVar<ServerDBInfo>>(testDbInfo)));
 
+	// Create a local log router, and it shouldn't be considered as remote log router.
 	TLogInterface localLogRouter(testLocal);
 	localLogRouter.initEndpoints();
 	testDbInfo.logSystemConfig.tLogs.back().logRouters.push_back(OptionalInterface(localLogRouter));

From 34b042a2f120398a3444724db6c43eba10cb9865 Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyu_zhou@apple.com>
Date: Fri, 21 Apr 2023 15:42:30 -0700
Subject: [PATCH 139/468] Double TLOG_MAX_CREATE_DURATION in simulation
 (#10027)

There were cases this would cause failure in simulation, where the commit time
just exceeds 10s.
---
 fdbclient/ServerKnobs.cpp         | 2 +-
 fdbserver/WorkerInterface.actor.h | 4 +++-
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/fdbclient/ServerKnobs.cpp b/fdbclient/ServerKnobs.cpp
index 9d8861b33c1..a05116692dc 100644
--- a/fdbclient/ServerKnobs.cpp
+++ b/fdbclient/ServerKnobs.cpp
@@ -95,7 +95,7 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi
 	init( MAX_CACHE_VERSIONS,                                   10e6 );
 	init( TLOG_IGNORE_POP_AUTO_ENABLE_DELAY,                   300.0 );
 	init( TXS_POPPED_MAX_DELAY,                                  1.0 ); if ( randomize && BUGGIFY ) TXS_POPPED_MAX_DELAY = deterministicRandom()->random01();
-	init( TLOG_MAX_CREATE_DURATION,                             10.0 );
+	init( TLOG_MAX_CREATE_DURATION,                             10.0 ); if ( isSimulated ) TLOG_MAX_CREATE_DURATION = 20.0;
 	init( PEEK_LOGGING_AMOUNT,                                     5 );
 	init( PEEK_LOGGING_DELAY,                                    5.0 );
 	init( PEEK_RESET_INTERVAL,                                 300.0 ); if ( randomize && BUGGIFY ) PEEK_RESET_INTERVAL = 20.0;
diff --git a/fdbserver/WorkerInterface.actor.h b/fdbserver/WorkerInterface.actor.h
index 23b71a0e3cd..f5cb0ac414b 100644
--- a/fdbserver/WorkerInterface.actor.h
+++ b/fdbserver/WorkerInterface.actor.h
@@ -1204,6 +1204,8 @@ ACTOR template <class T>
 Future<T> ioTimeoutError(Future<T> what, double time) {
 	// Before simulation is sped up, IO operations can take a very long time so limit timeouts
 	// to not end until at least time after simulation is sped up.
+	state double orig = now();
+	state std::string trace = platform::get_backtrace();
 	if (g_network->isSimulated() && !g_simulator.speedUpSimulation) {
 		time += std::max(0.0, FLOW_KNOBS->SIM_SPEEDUP_AFTER_SECONDS - now());
 	}
@@ -1217,7 +1219,7 @@ Future<T> ioTimeoutError(Future<T> what, double time) {
 			if (g_network->isSimulated() && !g_simulator.getCurrentProcess()->isReliable()) {
 				err = err.asInjectedFault();
 			}
-			TraceEvent(SevError, "IoTimeoutError").error(err);
+			TraceEvent(SevError, "IoTimeoutError").error(err).detail("OrigTime", orig).detail("OrigTrace", trace);
 			throw err;
 		}
 	}

From 1b48898915309d8976ed73c3193d089f76291268 Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyu_zhou@apple.com>
Date: Tue, 25 Apr 2023 12:55:52 -0700
Subject: [PATCH 140/468] Properly handle proxy_memory_limit_exceeded error for
 GetKeyServerLocationsRequest [release-7.1] (#10007)

* Properly handle proxy_memory_limit_exceeded error for GetKeyServerLocationsRequest

Also add buggify to inject the error in simulation.

* Add .git-blame-ignore-revs

* Fix FetchKeysError due to proxy_memory_limit_exceeded error

* Make commit_proxy_memory_limit_exceeded error a database level backoff

Since this error means the database is overloaded.

* Add backoff to getKeyLocation_internal as well

* Add trace events and adjust backoff

For each success, half the backoff until less than initial backoff value, then
set the backoff to 0.

* Simplify backoff calls per comment

* Fix format
---
 .git-blame-ignore-revs                |  4 ++
 fdbclient/DatabaseContext.h           | 11 +++-
 fdbclient/NativeAPI.actor.cpp         | 84 ++++++++++++++++++++-------
 fdbserver/CommitProxyServer.actor.cpp |  5 +-
 fdbserver/storageserver.actor.cpp     | 19 +++++-
 5 files changed, 96 insertions(+), 27 deletions(-)
 create mode 100644 .git-blame-ignore-revs

diff --git a/.git-blame-ignore-revs b/.git-blame-ignore-revs
new file mode 100644
index 00000000000..916311bab94
--- /dev/null
+++ b/.git-blame-ignore-revs
@@ -0,0 +1,4 @@
+# clang-format the entire codebase
+df90cc89de67ea4748c8cadd18e6fc4ce7fda12e
+2c788c233db56ccec4ed90d7da31887487b9f3b7
+69508b980f3cc5aabea6322f292e53b07bb27544
diff --git a/fdbclient/DatabaseContext.h b/fdbclient/DatabaseContext.h
index 455f7606bda..b051fe9a312 100644
--- a/fdbclient/DatabaseContext.h
+++ b/fdbclient/DatabaseContext.h
@@ -294,7 +294,7 @@ class DatabaseContext : public ReferenceCounted<DatabaseContext>, public FastAll
 	Future<Reference<CommitProxyInfo>> getCommitProxiesFuture(UseProvisionalProxies useProvisionalProxies);
 	Reference<GrvProxyInfo> getGrvProxies(UseProvisionalProxies useProvisionalProxies);
 	bool isCurrentGrvProxy(UID proxyId) const;
-	Future<Void> onProxiesChanged() const;
+	Future<Void> onProxiesChanged();
 	Future<HealthMetrics> getHealthMetrics(bool detailed);
 	// Pass a negative value for `shardLimit` to indicate no limit on the shard number.
 	Future<StorageMetrics> getStorageMetrics(KeyRange const& keys, int shardLimit);
@@ -656,9 +656,18 @@ class DatabaseContext : public ReferenceCounted<DatabaseContext>, public FastAll
 	std::unique_ptr<GlobalConfig> globalConfig;
 	EventCacheHolder connectToDatabaseEventCacheHolder;
 
+	// Gets a database level backoff delay future, time in seconds.
+	Future<Void> getBackoff() const { return backoffDelay > 0.0 ? delay(backoffDelay) : Future<Void>(Void()); }
+
+	// Updates internal Backoff state when a request fails or succeeds.
+	// E.g., commit_proxy_memory_limit_exceeded error means the database is overloaded
+	// and the client should back off more significantly than transaction-level errors.
+	void updateBackoff(const Error& err);
+
 private:
 	std::unordered_map<std::pair<int64_t, Key>, Reference<WatchMetadata>, boost::hash<std::pair<int64_t, Key>>>
 	    watchMap;
+	double backoffDelay = 0.0;
 };
 
 #endif
diff --git a/fdbclient/NativeAPI.actor.cpp b/fdbclient/NativeAPI.actor.cpp
index 9bbd0e70d0d..bdd46fc03fa 100644
--- a/fdbclient/NativeAPI.actor.cpp
+++ b/fdbclient/NativeAPI.actor.cpp
@@ -47,6 +47,7 @@
 #include "fdbclient/DatabaseContext.h"
 #include "fdbclient/GlobalConfig.actor.h"
 #include "fdbclient/IKnobCollection.h"
+#include "fdbclient/Knobs.h"
 #include "fdbclient/JsonBuilder.h"
 #include "fdbclient/KeyBackedTypes.h"
 #include "fdbclient/KeyRangeMap.h"
@@ -1998,7 +1999,8 @@ void DatabaseContext::clearFailedEndpointOnHealthyServer(const Endpoint& endpoin
 	failedEndpointsOnHealthyServersInfo.erase(endpoint);
 }
 
-Future<Void> DatabaseContext::onProxiesChanged() const {
+Future<Void> DatabaseContext::onProxiesChanged() {
+	backoffDelay = 0.0;
 	return this->proxiesChangeTrigger.onTrigger();
 }
 
@@ -2876,8 +2878,9 @@ ACTOR Future<KeyRangeLocationInfo> getKeyLocation_internal(Database cx,
 	if (debugID.present())
 		g_traceBatch.addEvent("TransactionDebug", debugID.get().first(), "NativeAPI.getKeyLocation.Before");
 
-	try {
-		loop {
+	loop {
+		try {
+			wait(cx->getBackoff());
 			++cx->transactionKeyServerLocationRequests;
 			choose {
 				when(wait(cx->onProxiesChanged())) {}
@@ -2904,20 +2907,27 @@ ACTOR Future<KeyRangeLocationInfo> getKeyLocation_internal(Database cx,
 					updateTssMappings(cx, rep);
 					updateTagMappings(cx, rep);
 
+					cx->updateBackoff(success());
 					return KeyRangeLocationInfo(
 					    rep.tenantEntry,
 					    KeyRange(toRelativeRange(rep.results[0].first, rep.tenantEntry.prefix), rep.arena),
 					    locationInfo);
 				}
 			}
-		}
-	} catch (Error& e) {
-		if (e.code() == error_code_tenant_not_found) {
-			ASSERT(tenant.present());
-			cx->invalidateCachedTenant(tenant.get());
-		}
+		} catch (Error& e) {
+			if (e.code() == error_code_proxy_memory_limit_exceeded) {
+				// Eats proxy_memory_limit_exceeded error from commit proxies
+				TraceEvent(SevWarnAlways, "CommitProxyOverloadedForKeyLocation").suppressFor(5);
+				cx->updateBackoff(e);
+				continue;
+			}
+			if (e.code() == error_code_tenant_not_found) {
+				ASSERT(tenant.present());
+				cx->invalidateCachedTenant(tenant.get());
+			}
 
-		throw;
+			throw;
+		}
 	}
 }
 
@@ -3010,6 +3020,30 @@ Future<KeyRangeLocationInfo> getKeyLocation(Reference<TransactionState> trState,
 	}
 }
 
+void DatabaseContext::updateBackoff(const Error& err) {
+	switch (err.code()) {
+	case error_code_success:
+		backoffDelay = backoffDelay / CLIENT_KNOBS->BACKOFF_GROWTH_RATE;
+		if (backoffDelay < CLIENT_KNOBS->DEFAULT_BACKOFF) {
+			backoffDelay = 0.0;
+		}
+		break;
+
+	case error_code_proxy_memory_limit_exceeded:
+		++transactionsResourceConstrained;
+		if (backoffDelay == 0.0) {
+			backoffDelay = CLIENT_KNOBS->DEFAULT_BACKOFF;
+		} else {
+			backoffDelay = std::min(backoffDelay * CLIENT_KNOBS->BACKOFF_GROWTH_RATE,
+			                        CLIENT_KNOBS->RESOURCE_CONSTRAINED_MAX_BACKOFF);
+		}
+		break;
+
+	default:
+		ASSERT_WE_THINK(false);
+	}
+}
+
 ACTOR Future<std::vector<KeyRangeLocationInfo>> getKeyRangeLocations_internal(
     Database cx,
     Optional<TenantName> tenant,
@@ -3024,8 +3058,9 @@ ACTOR Future<std::vector<KeyRangeLocationInfo>> getKeyRangeLocations_internal(
 	if (debugID.present())
 		g_traceBatch.addEvent("TransactionDebug", debugID.get().first(), "NativeAPI.getKeyLocations.Before");
 
-	try {
-		loop {
+	loop {
+		try {
+			wait(cx->getBackoff());
 			++cx->transactionKeyServerLocationRequests;
 			choose {
 				when(wait(cx->onProxiesChanged())) {}
@@ -3051,8 +3086,8 @@ ACTOR Future<std::vector<KeyRangeLocationInfo>> getKeyRangeLocations_internal(
 					state std::vector<KeyRangeLocationInfo> results;
 					state int shard = 0;
 					for (; shard < rep.results.size(); shard++) {
-						// FIXME: these shards are being inserted into the map sequentially, it would be much more CPU
-						// efficient to save the map pairs and insert them all at once.
+						// FIXME: these shards are being inserted into the map sequentially, it would be much more
+						// CPU efficient to save the map pairs and insert them all at once.
 						results.emplace_back(
 						    rep.tenantEntry,
 						    (toRelativeRange(rep.results[shard].first, rep.tenantEntry.prefix) & keys),
@@ -3063,17 +3098,24 @@ ACTOR Future<std::vector<KeyRangeLocationInfo>> getKeyRangeLocations_internal(
 					updateTssMappings(cx, rep);
 					updateTagMappings(cx, rep);
 
+					cx->updateBackoff(success());
 					return results;
 				}
 			}
-		}
-	} catch (Error& e) {
-		if (e.code() == error_code_tenant_not_found) {
-			ASSERT(tenant.present());
-			cx->invalidateCachedTenant(tenant.get());
-		}
+		} catch (Error& e) {
+			if (e.code() == error_code_proxy_memory_limit_exceeded) {
+				// Eats proxy_memory_limit_exceeded error from commit proxies
+				TraceEvent(SevWarnAlways, "CommitProxyOverloadedForRangeLocation").suppressFor(5);
+				cx->updateBackoff(e);
+				continue;
+			}
+			if (e.code() == error_code_tenant_not_found) {
+				ASSERT(tenant.present());
+				cx->invalidateCachedTenant(tenant.get());
+			}
 
-		throw;
+			throw;
+		}
 	}
 }
 
diff --git a/fdbserver/CommitProxyServer.actor.cpp b/fdbserver/CommitProxyServer.actor.cpp
index 1918a575a06..2af27aed41e 100644
--- a/fdbserver/CommitProxyServer.actor.cpp
+++ b/fdbserver/CommitProxyServer.actor.cpp
@@ -1805,8 +1805,9 @@ ACTOR static Future<Void> readRequestServer(CommitProxyInterface proxy,
 		GetKeyServerLocationsRequest req = waitNext(proxy.getKeyServersLocations.getFuture());
 		// WARNING: this code is run at a high priority, so it needs to do as little work as possible
 		if (req.limit != CLIENT_KNOBS->STORAGE_METRICS_SHARD_LIMIT && // Always do data distribution requests
-		    commitData->stats.keyServerLocationIn.getValue() - commitData->stats.keyServerLocationOut.getValue() >
-		        SERVER_KNOBS->KEY_LOCATION_MAX_QUEUE_SIZE) {
+		    (commitData->stats.keyServerLocationIn.getValue() - commitData->stats.keyServerLocationOut.getValue() >
+		         SERVER_KNOBS->KEY_LOCATION_MAX_QUEUE_SIZE ||
+		     (g_network->isSimulated() && BUGGIFY_WITH_PROB(0.001)))) {
 			++commitData->stats.keyServerLocationErrors;
 			req.reply.sendError(proxy_memory_limit_exceeded());
 			TraceEvent(SevWarnAlways, "ProxyLocationRequestThresholdExceeded").suppressFor(60);
diff --git a/fdbserver/storageserver.actor.cpp b/fdbserver/storageserver.actor.cpp
index fed740f79bd..0a2508d3a43 100644
--- a/fdbserver/storageserver.actor.cpp
+++ b/fdbserver/storageserver.actor.cpp
@@ -5787,6 +5787,21 @@ ACTOR Future<std::unordered_map<Key, Version>> dispatchChangeFeeds(StorageServer
 	}
 }
 
+bool fetchKeyCanRetry(const Error& e) {
+	switch (e.code()) {
+	case error_code_end_of_stream:
+	case error_code_connection_failed:
+	case error_code_transaction_too_old:
+	case error_code_future_version:
+	case error_code_process_behind:
+	case error_code_server_overloaded:
+	case error_code_proxy_memory_limit_exceeded:
+		return true;
+	default:
+		return false;
+	}
+}
+
 ACTOR Future<Void> fetchKeys(StorageServer* data, AddingShard* shard) {
 	state const UID fetchKeysID = deterministicRandom()->randomUniqueID();
 	state TraceInterval interval("FetchKeys");
@@ -5995,9 +6010,7 @@ ACTOR Future<Void> fetchKeys(StorageServer* data, AddingShard* shard) {
 					}
 				}
 			} catch (Error& e) {
-				if (e.code() != error_code_end_of_stream && e.code() != error_code_connection_failed &&
-				    e.code() != error_code_transaction_too_old && e.code() != error_code_future_version &&
-				    e.code() != error_code_process_behind && e.code() != error_code_server_overloaded) {
+				if (!fetchKeyCanRetry(e)) {
 					throw;
 				}
 				lastError = e;

From 863e338b47cfb3106e624af11c303ad9064deed7 Mon Sep 17 00:00:00 2001
From: neethuhaneesha <nbingi@apple.com>
Date: Wed, 26 Apr 2023 16:44:24 -0700
Subject: [PATCH 141/468] Enabling auto_prefix_mode to true in rocksdb.

---
 fdbclient/ServerKnobs.cpp                |  3 +--
 fdbclient/ServerKnobs.h                  |  1 -
 fdbserver/KeyValueStoreRocksDB.actor.cpp | 20 +++++++++-----------
 3 files changed, 10 insertions(+), 14 deletions(-)

diff --git a/fdbclient/ServerKnobs.cpp b/fdbclient/ServerKnobs.cpp
index a05116692dc..55e1f3119c9 100644
--- a/fdbclient/ServerKnobs.cpp
+++ b/fdbclient/ServerKnobs.cpp
@@ -376,7 +376,7 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi
 	init( ROCKSDB_LEVEL_STYLE_COMPACTION,                       true );
 	init( ROCKSDB_UNSAFE_AUTO_FSYNC,                           false );
 	init( ROCKSDB_PERIODIC_COMPACTION_SECONDS,                     0 );
-	init( ROCKSDB_PREFIX_LEN,                                      0 );
+	init( ROCKSDB_PREFIX_LEN,                                      0 ); if( randomize && BUGGIFY ) ROCKSDB_PREFIX_LEN = deterministicRandom()->randomInt(1, 20);
 	init( ROCKSDB_MEMTABLE_PREFIX_BLOOM_SIZE_RATIO,              0.1 );
 	init( ROCKSDB_BLOOM_BITS_PER_KEY,                             10 );
 	init( ROCKSDB_BLOOM_WHOLE_KEY_FILTERING,                   false );
@@ -420,7 +420,6 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi
 	// If ROCKSDB_SINGLEKEY_DELETES_ON_CLEARRANGE is enabled, disable ROCKSDB_ENABLE_CLEAR_RANGE_EAGER_READS knob.
 	// These knobs have contrary functionality.
 	init( ROCKSDB_SINGLEKEY_DELETES_ON_CLEARRANGE,              true );
-	init( ROCKSDB_SINGLEKEY_DELETES_BYTES_LIMIT,               10000 ); // 10KB
 	init( ROCKSDB_SINGLEKEY_DELETES_MAX,                         200 ); // Max rocksdb::delete calls in a transaction
 	init( ROCKSDB_ENABLE_CLEAR_RANGE_EAGER_READS,              false );
 	init( ROCKSDB_FORCE_DELETERANGE_FOR_CLEARRANGE,            false );
diff --git a/fdbclient/ServerKnobs.h b/fdbclient/ServerKnobs.h
index 81f8b4d2040..1d9635e1563 100644
--- a/fdbclient/ServerKnobs.h
+++ b/fdbclient/ServerKnobs.h
@@ -341,7 +341,6 @@ class ServerKnobs : public KnobsImpl<ServerKnobs> {
 	int ROCKSDB_CAN_COMMIT_DELAY_TIMES_ON_OVERLOAD;
 	bool ROCKSDB_DISABLE_WAL_EXPERIMENTAL;
 	bool ROCKSDB_SINGLEKEY_DELETES_ON_CLEARRANGE;
-	int64_t ROCKSDB_SINGLEKEY_DELETES_BYTES_LIMIT;
 	int ROCKSDB_SINGLEKEY_DELETES_MAX;
 	bool ROCKSDB_ENABLE_CLEAR_RANGE_EAGER_READS;
 	bool ROCKSDB_FORCE_DELETERANGE_FOR_CLEARRANGE;
diff --git a/fdbserver/KeyValueStoreRocksDB.actor.cpp b/fdbserver/KeyValueStoreRocksDB.actor.cpp
index 1a320ef68ab..d7f352d1b39 100644
--- a/fdbserver/KeyValueStoreRocksDB.actor.cpp
+++ b/fdbserver/KeyValueStoreRocksDB.actor.cpp
@@ -411,6 +411,7 @@ rocksdb::Options getOptions() {
 // Set some useful defaults desired for all reads.
 rocksdb::ReadOptions getReadOptions() {
 	rocksdb::ReadOptions options;
+	options.auto_prefix_mode = (SERVER_KNOBS->ROCKSDB_PREFIX_LEN > 0);
 	options.background_purge_on_iterator_cleanup = true;
 	return options;
 }
@@ -441,10 +442,11 @@ struct ReadIterator {
 	double creationTime;
 	KeyRange keyRange;
 	std::shared_ptr<rocksdb::Slice> beginSlice, endSlice;
-	ReadIterator(CF& cf, uint64_t index, DB& db, rocksdb::ReadOptions& options)
-	  : index(index), inUse(true), creationTime(now()), iter(db->NewIterator(options, cf)) {}
-	ReadIterator(CF& cf, uint64_t index, DB& db, rocksdb::ReadOptions options, KeyRange keyRange)
+	ReadIterator(CF& cf, uint64_t index, DB& db)
+	  : index(index), inUse(true), creationTime(now()), iter(db->NewIterator(getReadOptions(), cf)) {}
+	ReadIterator(CF& cf, uint64_t index, DB& db, KeyRange keyRange)
 	  : index(index), inUse(true), creationTime(now()), keyRange(keyRange) {
+		rocksdb::ReadOptions options = getReadOptions();
 		beginSlice = std::shared_ptr<rocksdb::Slice>(new rocksdb::Slice(toSlice(keyRange.begin)));
 		options.iterate_lower_bound = beginSlice.get();
 		endSlice = std::shared_ptr<rocksdb::Slice>(new rocksdb::Slice(toSlice(keyRange.end)));
@@ -468,10 +470,7 @@ gets deleted as the ref count becomes 0.
 */
 class ReadIteratorPool {
 public:
-	ReadIteratorPool(UID id, DB& db, CF& cf)
-	  : db(db), cf(cf), index(0), deletedUptoIndex(0), iteratorsReuseCount(0), readRangeOptions(getReadOptions()) {
-		readRangeOptions.background_purge_on_iterator_cleanup = true;
-		readRangeOptions.auto_prefix_mode = (SERVER_KNOBS->ROCKSDB_PREFIX_LEN > 0);
+	ReadIteratorPool(UID id, DB& db, CF& cf) : db(db), cf(cf), index(0), deletedUptoIndex(0), iteratorsReuseCount(0) {
 		TraceEvent("ReadIteratorPool", id)
 		    .detail("KnobRocksDBReadRangeReuseIterators", SERVER_KNOBS->ROCKSDB_READ_RANGE_REUSE_ITERATORS)
 		    .detail("KnobRocksDBReadRangeReuseBoundedIterators",
@@ -515,7 +514,7 @@ class ReadIteratorPool {
 			uint64_t readIteratorIndex = index;
 			mutex.unlock();
 
-			ReadIterator iter(cf, readIteratorIndex, db, readRangeOptions);
+			ReadIterator iter(cf, readIteratorIndex, db);
 			mutex.lock();
 			iteratorsMap.insert({ readIteratorIndex, iter });
 			mutex.unlock();
@@ -537,7 +536,7 @@ class ReadIteratorPool {
 			uint64_t readIteratorIndex = index;
 			mutex.unlock();
 
-			ReadIterator iter(cf, readIteratorIndex, db, readRangeOptions, keyRange);
+			ReadIterator iter(cf, readIteratorIndex, db, keyRange);
 			if (iteratorsMap.size() < SERVER_KNOBS->ROCKSDB_READ_RANGE_BOUNDED_ITERATORS_MAX_LIMIT) {
 				// Not storing more than ROCKSDB_READ_RANGE_BOUNDED_ITERATORS_MAX_LIMIT of iterators
 				// to avoid 'out of memory' issues.
@@ -548,7 +547,7 @@ class ReadIteratorPool {
 			return iter;
 		} else {
 			index++;
-			ReadIterator iter(cf, index, db, readRangeOptions, keyRange);
+			ReadIterator iter(cf, index, db, keyRange);
 			return iter;
 		}
 	}
@@ -594,7 +593,6 @@ class ReadIteratorPool {
 	std::unordered_map<int, ReadIterator>::iterator it;
 	DB& db;
 	CF& cf;
-	rocksdb::ReadOptions readRangeOptions;
 	std::mutex mutex;
 	// incrementing counter for every new iterator creation, to uniquely identify the iterator in returnIterator().
 	uint64_t index;

From 8bf1f88559c7205c90aa308fb1e813633393b3d5 Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Thu, 27 Apr 2023 13:42:01 -0700
Subject: [PATCH 142/468] Fix ConsistencyCheck_InvalidShardSize due to unfair
 split

ConsistencyCheck thinks the shard is too large and should be split. However, it
is possible that the split point returned by the Storage Server is discarded
due to "unfair" sizes after the split. Fix this problem by checking if this is
the case in ConsistencyCheck.
---
 .../workloads/ConsistencyCheck.actor.cpp      | 38 ++++++++++++++++---
 1 file changed, 32 insertions(+), 6 deletions(-)

diff --git a/fdbserver/workloads/ConsistencyCheck.actor.cpp b/fdbserver/workloads/ConsistencyCheck.actor.cpp
index 537bac2b868..ca4181e7d82 100644
--- a/fdbserver/workloads/ConsistencyCheck.actor.cpp
+++ b/fdbserver/workloads/ConsistencyCheck.actor.cpp
@@ -1058,9 +1058,11 @@ struct ConsistencyCheckWorkload : TestWorkload {
 	// Retrieves a vector of the storage servers' estimates for the size of a particular shard
 	// If a storage server can't be reached, its estimate will be -1
 	// If there is an error, then the returned vector will have 0 size
-	ACTOR Future<std::vector<int64_t>> getStorageSizeEstimate(std::vector<StorageServerInterface> storageServers,
-	                                                          KeyRangeRef shard) {
+	ACTOR Future<std::pair<std::vector<int64_t>, StorageMetrics>> getStorageSizeEstimate(
+	    std::vector<StorageServerInterface> storageServers,
+	    KeyRangeRef shard) {
 		state std::vector<int64_t> estimatedBytes;
+		state StorageMetrics metrics;
 
 		state WaitMetricsRequest req;
 		req.keys = shard;
@@ -1100,9 +1102,10 @@ struct ConsistencyCheckWorkload : TestWorkload {
 				else if (reply.present()) {
 					int64_t numBytes = reply.get().bytes;
 					estimatedBytes.push_back(numBytes);
-					if (firstValidStorageServer < 0)
+					if (firstValidStorageServer < 0) {
 						firstValidStorageServer = i;
-					else if (estimatedBytes[firstValidStorageServer] != numBytes) {
+						metrics = reply.get();
+					} else if (estimatedBytes[firstValidStorageServer] != numBytes) {
 						TraceEvent("ConsistencyCheck_InconsistentStorageMetrics")
 						    .detail("ByteEstimate1", estimatedBytes[firstValidStorageServer])
 						    .detail("ByteEstimate2", numBytes)
@@ -1125,7 +1128,7 @@ struct ConsistencyCheckWorkload : TestWorkload {
 			estimatedBytes.clear();
 		}
 
-		return estimatedBytes;
+		return std::make_pair(estimatedBytes, metrics);
 	}
 
 	// Comparison function used to compare map elements by value
@@ -1290,8 +1293,10 @@ struct ConsistencyCheckWorkload : TestWorkload {
 				}
 			}
 
-			state std::vector<int64_t> estimatedBytes =
+			std::pair<std::vector<int64_t>, StorageMetrics> estimatedBytesAndMetrics =
 			    wait(self->getStorageSizeEstimate(storageServerInterfaces, range));
+			state std::vector<int64_t> estimatedBytes = estimatedBytesAndMetrics.first;
+			state StorageMetrics estimated = estimatedBytesAndMetrics.second;
 
 			// Gets permitted size range of shard
 			int64_t maxShardSize = getMaxShardSize(dbSize);
@@ -1680,6 +1685,27 @@ struct ConsistencyCheckWorkload : TestWorkload {
 					self->testFailure(format("Shard size is more than %f std dev from estimate", failErrorNumStdDev));
 				}
 
+				// Check if the storage server returns split point for the shard. There are cases where
+				// the split point returned by storage server is discarded because it's an unfair split.
+				// See splitStorageMetrics() in NativeAPI.actor.cpp.
+				if (canSplit && sampledKeys > 5 && self->performQuiescentChecks) {
+					StorageMetrics splitMetrics;
+					splitMetrics.bytes = shardBounds.max.bytes / 2;
+					splitMetrics.bytesPerKSecond = range.begin >= keyServersKeys.begin
+					                                   ? splitMetrics.infinity
+					                                   : SERVER_KNOBS->SHARD_SPLIT_BYTES_PER_KSEC;
+					splitMetrics.iosPerKSecond = splitMetrics.infinity;
+					splitMetrics.bytesReadPerKSecond = splitMetrics.infinity; // Don't split by readBandwidth
+
+					Standalone<VectorRef<KeyRef>> splits =
+					    wait(cx->splitStorageMetrics(range, splitMetrics, estimated));
+					if (splits.size() <= 2) {
+						// Because the range's begin and end is included in splits, so this is the case
+						// the returned split is unfair and is discarded.
+						TraceEvent("ConsistencyCheck_DiscardSplits").detail("Range", range);
+						canSplit = false;
+					}
+				}
 				// In a quiescent database, check that the (estimated) size of the shard is within permitted bounds
 				// Min and max shard sizes have a 3 * shardBounds.permittedError.bytes cushion for error since shard
 				// sizes are not precise Shard splits ignore the first key in a shard, so its size shouldn't be

From 47ffc39d0092cbf07d992fa30d21482ca7e8b6b3 Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Mon, 24 Apr 2023 13:53:41 -0700
Subject: [PATCH 143/468] Fix worker caches

---
 fdbserver/worker.actor.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/fdbserver/worker.actor.cpp b/fdbserver/worker.actor.cpp
index e95eee465ca..15423e94b3d 100644
--- a/fdbserver/worker.actor.cpp
+++ b/fdbserver/worker.actor.cpp
@@ -2217,7 +2217,7 @@ ACTOR Future<Void> workerServer(Reference<IClusterConnectionRecord> connRecord,
 					ReplyPromise<InitializeBackupReply> backupReady = req.reply;
 					backupWorkerCache.set(req.reqId, backupReady.getFuture());
 					Future<Void> backupProcess = backupWorker(recruited, req, dbInfo);
-					backupProcess = storageCache.removeOnReady(req.reqId, backupProcess);
+					backupProcess = backupWorkerCache.removeOnReady(req.reqId, backupProcess);
 					errorForwarders.add(forwardError(errors, Role::BACKUP, recruited.id(), backupProcess));
 					TraceEvent("BackupInitRequest", req.reqId).detail("BackupId", recruited.id());
 					InitializeBackupReply reply(recruited, req.backupEpoch);

From fc903c4fe6c474321b9e483f76afd7bdf637e5e6 Mon Sep 17 00:00:00 2001
From: Heyang Zhou <zhy20000919@hotmail.com>
Date: Thu, 4 May 2023 03:32:22 +0800
Subject: [PATCH 144/468] Fix GlobalConfig_RetryableMigrationError when the
 database is locked (#10114)

* experiment: Enable LOCK_AWARE on GlobalConfig operations

* cleanup
---
 fdbclient/GlobalConfig.actor.cpp | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/fdbclient/GlobalConfig.actor.cpp b/fdbclient/GlobalConfig.actor.cpp
index c35919c3864..6b99ca8a321 100644
--- a/fdbclient/GlobalConfig.actor.cpp
+++ b/fdbclient/GlobalConfig.actor.cpp
@@ -176,6 +176,7 @@ ACTOR Future<Void> GlobalConfig::migrate(GlobalConfig* self) {
 		loop {
 			tr = makeReference<ReadYourWritesTransaction>(Database(Reference<DatabaseContext>::addRef(self->cx)));
 			tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
+			tr->setOption(FDBTransactionOptions::LOCK_AWARE);
 
 			try {
 				state Optional<Value> migrated = wait(tr->get(migratedKey));
@@ -239,6 +240,7 @@ ACTOR Future<Void> GlobalConfig::refresh(GlobalConfig* self) {
 		try {
 			tr = makeReference<ReadYourWritesTransaction>(Database(Reference<DatabaseContext>::addRef(self->cx)));
 			tr->setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
+			tr->setOption(FDBTransactionOptions::LOCK_AWARE);
 			RangeResult result = wait(tr->getRange(globalConfigDataKeys, CLIENT_KNOBS->TOO_MANY));
 			for (const auto& kv : result) {
 				KeyRef systemKey = kv.key.removePrefix(globalConfigKeysPrefix);

From 95d347cd69cd85b1530164fd7c529697b71a01fd Mon Sep 17 00:00:00 2001
From: Zhe Wu <halfprice@users.noreply.github.com>
Date: Wed, 3 May 2023 14:30:05 -0700
Subject: [PATCH 145/468] Increase MAX_STORAGE_COMMIT_TIME to be inline with
 LOW_PRIORITY_DURABILITY_LAG (#10116)

---
 fdbclient/ServerKnobs.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/fdbclient/ServerKnobs.cpp b/fdbclient/ServerKnobs.cpp
index 55e1f3119c9..0d4e98bba04 100644
--- a/fdbclient/ServerKnobs.cpp
+++ b/fdbclient/ServerKnobs.cpp
@@ -753,7 +753,7 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi
 	init( REPORT_DD_METRICS,                                    true );
 	init( DD_METRICS_REPORT_INTERVAL,                           30.0 );
 	init( FETCH_KEYS_TOO_LONG_TIME_CRITERIA,                   300.0 );
-	init( MAX_STORAGE_COMMIT_TIME,                             120.0 ); //The max fsync stall time on the storage server and tlog before marking a disk as failed
+	init( MAX_STORAGE_COMMIT_TIME,                             200.0 ); //The max fsync stall time on the storage server and tlog before marking a disk as failed
 	init( RANGESTREAM_LIMIT_BYTES,                               2e6 ); if( randomize && BUGGIFY ) RANGESTREAM_LIMIT_BYTES = 1;
 	init( CHANGEFEEDSTREAM_LIMIT_BYTES,                          1e6 ); if( randomize && BUGGIFY ) CHANGEFEEDSTREAM_LIMIT_BYTES = 1;
 	init( BLOBWORKERSTATUSSTREAM_LIMIT_BYTES,                    1e4 ); if( randomize && BUGGIFY ) BLOBWORKERSTATUSSTREAM_LIMIT_BYTES = 1;

From 5fe1bd8c2b2478238992c56fb49d30d1abb3678f Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyu_zhou@apple.com>
Date: Wed, 3 May 2023 20:54:14 -0700
Subject: [PATCH 146/468] Add 7.1.32, 7.1.33 release notes (#10125)

---
 .../source/release-notes/release-notes-710.rst      | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/documentation/sphinx/source/release-notes/release-notes-710.rst b/documentation/sphinx/source/release-notes/release-notes-710.rst
index eb711318fb5..71954a0e939 100644
--- a/documentation/sphinx/source/release-notes/release-notes-710.rst
+++ b/documentation/sphinx/source/release-notes/release-notes-710.rst
@@ -4,6 +4,19 @@
 Release Notes
 #############
 
+7.1.33
+======
+* Same as 7.1.32 release with AVX enabled.
+
+7.1.32
+======
+* Released with AVX disabled.
+* Increased MAX_STORAGE_COMMIT_TIME to reduce storage server IO timeout errors. `(PR #10116) <https://github.com/apple/foundationdb/pull/10116>`_
+* Fixed a CPU spinning issue on DR destination clusters. `(PR #10114) <https://github.com/apple/foundationdb/pull/10114>`_
+* Fixed a inconsistent read issue when using RocksDB engine. `(PR #10060) <https://github.com/apple/foundationdb/pull/10060>`_
+* Fixed a storage server failure issue and added client backoff when commit proxy is overloaded for key location requests. `(PR #10007) <https://github.com/apple/foundationdb/pull/10007>`_
+* Added gray failure detection of disconnected remote log routers. `(PR #9933) <https://github.com/apple/foundationdb/pull/9933>`_
+
 7.1.31
 ======
 * Same as 7.1.30 release with AVX enabled.

From 9608b9049e8d8dd99b18809b55484935ac5b4662 Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyu_zhou@apple.com>
Date: Thu, 4 May 2023 20:53:35 -0700
Subject: [PATCH 147/468] Increase buggified STORAGE_METRICS_SHARD_LIMIT value
 (#10149)

The previous buggified value 3 can be the same as key location size, thus
causing splitStorageMetrics() to stuck.
---
 fdbclient/ClientKnobs.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/fdbclient/ClientKnobs.cpp b/fdbclient/ClientKnobs.cpp
index 16c8191cee4..bbc315774a0 100644
--- a/fdbclient/ClientKnobs.cpp
+++ b/fdbclient/ClientKnobs.cpp
@@ -100,7 +100,7 @@ void ClientKnobs::initialize(Randomize randomize) {
 
 	init( GET_RANGE_SHARD_LIMIT,                     2 );
 	init( WARM_RANGE_SHARD_LIMIT,                  100 );
-	init( STORAGE_METRICS_SHARD_LIMIT,             100 ); if( randomize && BUGGIFY ) STORAGE_METRICS_SHARD_LIMIT = 3;
+	init( STORAGE_METRICS_SHARD_LIMIT,             100 ); if( randomize && BUGGIFY ) STORAGE_METRICS_SHARD_LIMIT = 10;
 	init( SHARD_COUNT_LIMIT,                        80 ); if( randomize && BUGGIFY ) SHARD_COUNT_LIMIT = 3;
 	init( STORAGE_METRICS_UNFAIR_SPLIT_LIMIT,  2.0/3.0 );
 	init( STORAGE_METRICS_TOO_MANY_SHARDS_DELAY,  15.0 );

From 71af9db3d50f4be9caa0b314e794888a558dc8f9 Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyu_zhou@apple.com>
Date: Fri, 5 May 2023 09:57:34 -0700
Subject: [PATCH 148/468] Add urllib3<2 to requirements.txt for Sphinx (#10160)

---
 documentation/sphinx/requirements.txt | 1 +
 1 file changed, 1 insertion(+)

diff --git a/documentation/sphinx/requirements.txt b/documentation/sphinx/requirements.txt
index 6742ef130ff..2c042569721 100644
--- a/documentation/sphinx/requirements.txt
+++ b/documentation/sphinx/requirements.txt
@@ -4,3 +4,4 @@ sphinx==5.1.1
 sphinx-bootstrap-theme==0.8.1
 docutils==0.19
 Jinja2==3.1.2
+urllib3<2

From 085c8bb8c2f5921bcc9e731a237867504a8edc79 Mon Sep 17 00:00:00 2001
From: Zhe Wu <halfprice@users.noreply.github.com>
Date: Fri, 5 May 2023 20:11:51 -0700
Subject: [PATCH 149/468] Update 7.1 minInvalidProtocolVersion to 7.4 to make
 downgrading from 7.3 to 7.1 work (#10146)

---
 flow/ProtocolVersion.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/flow/ProtocolVersion.h b/flow/ProtocolVersion.h
index a40c09c8c82..689dfae6a76 100644
--- a/flow/ProtocolVersion.h
+++ b/flow/ProtocolVersion.h
@@ -35,7 +35,7 @@ constexpr uint64_t currentProtocolVersionValue = 0x0FDB00B071010000LL;
 
 // The first protocol version that cannot be downgraded from. Ordinarily, this will be two release versions larger
 // than the current version, meaning that we only support downgrades between consecutive release versions.
-constexpr uint64_t minInvalidProtocolVersionValue = 0x0FDB00B073000000LL;
+constexpr uint64_t minInvalidProtocolVersionValue = 0x0FDB00B074000000LL;
 
 #define PROTOCOL_VERSION_FEATURE(v, x)                                                                                 \
 	static_assert((v & 0xF0FFFFLL) == 0 || v < 0x0FDB00B071000000LL, "Unexpected feature protocol version");           \

From 180ce51db2382139e8b9f5da3ee071c8fa86dcfb Mon Sep 17 00:00:00 2001
From: neethuhaneesha <nbingi@apple.com>
Date: Fri, 5 May 2023 20:42:25 -0700
Subject: [PATCH 150/468] Rocksdb paranoid file checks knob. (#10148)

---
 fdbclient/ServerKnobs.cpp                | 2 ++
 fdbclient/ServerKnobs.h                  | 1 +
 fdbserver/KeyValueStoreRocksDB.actor.cpp | 2 ++
 3 files changed, 5 insertions(+)

diff --git a/fdbclient/ServerKnobs.cpp b/fdbclient/ServerKnobs.cpp
index 0d4e98bba04..60cd24ef546 100644
--- a/fdbclient/ServerKnobs.cpp
+++ b/fdbclient/ServerKnobs.cpp
@@ -414,6 +414,8 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi
 	init( ROCKSDB_SOFT_PENDING_COMPACT_BYTES_LIMIT,      64000000000 ); // 64GB, Rocksdb option, Writes will slow down.
 	init( ROCKSDB_HARD_PENDING_COMPACT_BYTES_LIMIT,     100000000000 ); // 100GB, Rocksdb option, Writes will stall.
 	init( ROCKSDB_CAN_COMMIT_COMPACT_BYTES_LIMIT,        50000000000 ); // 50GB, Commit waits.
+	// Enabling ROCKSDB_PARANOID_FILE_CHECKS knob will have overhead. Be cautious to enable in prod.
+	init( ROCKSDB_PARANOID_FILE_CHECKS,                        false ); if( randomize && BUGGIFY ) ROCKSDB_PARANOID_FILE_CHECKS = deterministicRandom()->coinflip();
 	// Enable this knob only for experminatal purpose, never enable this in production.
 	// If enabled, all the committed in-memory memtable writes are lost on a crash.
 	init( ROCKSDB_DISABLE_WAL_EXPERIMENTAL,                    false );
diff --git a/fdbclient/ServerKnobs.h b/fdbclient/ServerKnobs.h
index 1d9635e1563..0ea56d81a27 100644
--- a/fdbclient/ServerKnobs.h
+++ b/fdbclient/ServerKnobs.h
@@ -337,6 +337,7 @@ class ServerKnobs : public KnobsImpl<ServerKnobs> {
 	int64_t ROCKSDB_SOFT_PENDING_COMPACT_BYTES_LIMIT;
 	int64_t ROCKSDB_HARD_PENDING_COMPACT_BYTES_LIMIT;
 	int64_t ROCKSDB_CAN_COMMIT_COMPACT_BYTES_LIMIT;
+	bool ROCKSDB_PARANOID_FILE_CHECKS;
 	int ROCKSDB_CAN_COMMIT_DELAY_ON_OVERLOAD;
 	int ROCKSDB_CAN_COMMIT_DELAY_TIMES_ON_OVERLOAD;
 	bool ROCKSDB_DISABLE_WAL_EXPERIMENTAL;
diff --git a/fdbserver/KeyValueStoreRocksDB.actor.cpp b/fdbserver/KeyValueStoreRocksDB.actor.cpp
index d7f352d1b39..6a3828e0f3c 100644
--- a/fdbserver/KeyValueStoreRocksDB.actor.cpp
+++ b/fdbserver/KeyValueStoreRocksDB.actor.cpp
@@ -321,6 +321,8 @@ rocksdb::ColumnFamilyOptions getCFOptions() {
 	if (SERVER_KNOBS->ROCKSDB_HARD_PENDING_COMPACT_BYTES_LIMIT > 0) {
 		options.hard_pending_compaction_bytes_limit = SERVER_KNOBS->ROCKSDB_HARD_PENDING_COMPACT_BYTES_LIMIT;
 	}
+	options.paranoid_file_checks = SERVER_KNOBS->ROCKSDB_PARANOID_FILE_CHECKS;
+
 	// Compact sstables when there's too much deleted stuff.
 	if (SERVER_KNOBS->ROCKSDB_ENABLE_COMPACT_ON_DELETION) {
 		// Creates a factory of a table property collector that marks a SST

From e0b8e93e5287e2410941e5f595bc6fe3ad1bee2b Mon Sep 17 00:00:00 2001
From: FoundationDB CI <foundationdb_ci@apple.com>
Date: Mon, 8 May 2023 18:39:14 +0000
Subject: [PATCH 151/468] disable AVX for 7.1.32 release

---
 cmake/ConfigureCompiler.cmake | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cmake/ConfigureCompiler.cmake b/cmake/ConfigureCompiler.cmake
index e52d8665ee8..7e50469ba6a 100644
--- a/cmake/ConfigureCompiler.cmake
+++ b/cmake/ConfigureCompiler.cmake
@@ -253,7 +253,7 @@ else()
       set(USE_AVX512F OFF)
     endif()
   endif()
-  set(USE_AVX ON CACHE BOOL "Enable AVX instructions")
+  set(USE_AVX OFF CACHE BOOL "Enable AVX instructions")
   if (USE_AVX)
     if (CMAKE_HOST_SYSTEM_PROCESSOR MATCHES "^x86")
       add_compile_options(-mavx)

From 1098aa0251f4c1cec6d6d5c5f34236eabf7c5e24 Mon Sep 17 00:00:00 2001
From: FoundationDB CI <foundationdb_ci@apple.com>
Date: Mon, 8 May 2023 18:39:41 +0000
Subject: [PATCH 152/468] enable AVX and update version for 7.1.33 release

---
 CMakeLists.txt                | 2 +-
 cmake/ConfigureCompiler.cmake | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 02e3fed12ca..935a12aab4a 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -23,7 +23,7 @@ else()
 endif()
 
 project(foundationdb
-  VERSION 7.1.32
+  VERSION 7.1.33
   DESCRIPTION "FoundationDB is a scalable, fault-tolerant, ordered key-value store with full ACID transactions."
   HOMEPAGE_URL "http://www.foundationdb.org/"
   LANGUAGES C CXX ASM)
diff --git a/cmake/ConfigureCompiler.cmake b/cmake/ConfigureCompiler.cmake
index 7e50469ba6a..e52d8665ee8 100644
--- a/cmake/ConfigureCompiler.cmake
+++ b/cmake/ConfigureCompiler.cmake
@@ -253,7 +253,7 @@ else()
       set(USE_AVX512F OFF)
     endif()
   endif()
-  set(USE_AVX OFF CACHE BOOL "Enable AVX instructions")
+  set(USE_AVX ON CACHE BOOL "Enable AVX instructions")
   if (USE_AVX)
     if (CMAKE_HOST_SYSTEM_PROCESSOR MATCHES "^x86")
       add_compile_options(-mavx)

From d7d0f6de1e354c01b225f33a8bf4cc227e756e4c Mon Sep 17 00:00:00 2001
From: FoundationDB CI <foundationdb_ci@apple.com>
Date: Tue, 9 May 2023 20:32:45 +0000
Subject: [PATCH 153/468] update version after 7.1.33 release

---
 CMakeLists.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 935a12aab4a..0d274e2c248 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -23,7 +23,7 @@ else()
 endif()
 
 project(foundationdb
-  VERSION 7.1.33
+  VERSION 7.1.34
   DESCRIPTION "FoundationDB is a scalable, fault-tolerant, ordered key-value store with full ACID transactions."
   HOMEPAGE_URL "http://www.foundationdb.org/"
   LANGUAGES C CXX ASM)

From 12ea1020bc401eebf0302d2e564afaa6f2203684 Mon Sep 17 00:00:00 2001
From: Zhanwei Wang <wang@hashdata.cn>
Date: Tue, 14 Jun 2022 04:33:05 +0800
Subject: [PATCH 154/468] Make backup work with s3 compatible service
 (#6355)(#6382) (#7324)

1. Support virtual hosting endpoint.

2. On-premise s3 compatible storage service may use IP instead of s3 form domain name,
especially for development/test environment.

Instead of parsing service and region from domain name,

1). Hard code "s3" as service name in v4 signature
2). Add new parameter to allow pass region name from url

3. Fix creating bucket issue on aws, adding request body.
---
 documentation/sphinx/source/backups.rst |   3 +-
 fdbclient/S3BlobStore.actor.cpp         | 121 ++++++++++++++++++------
 fdbclient/S3BlobStore.h                 |   8 +-
 3 files changed, 103 insertions(+), 29 deletions(-)

diff --git a/documentation/sphinx/source/backups.rst b/documentation/sphinx/source/backups.rst
index 9dcf5ffd8b5..793103340b6 100644
--- a/documentation/sphinx/source/backups.rst
+++ b/documentation/sphinx/source/backups.rst
@@ -83,7 +83,7 @@ For blob store backup locations, the Backup URL format is
 
 ::
 
-    blobstore://[<api_key>][:<secret>[:<security_token>]]@<hostname>[:<port>]/<name>?bucket=<bucket_name>[&<param>=<value>]...]
+    blobstore://[<api_key>][:<secret>[:<security_token>]]@<hostname>[:<port>]/<name>?bucket=<bucket_name>[&region=<region_name>][&<param>=<value>]...]
 
       <api_key> - API key to use for authentication. Optional.
       <secret> - API key's secret.  Optional.
@@ -92,6 +92,7 @@ For blob store backup locations, the Backup URL format is
       <port> - Remote port to connect to.  Optional.  Default is 80.
       <name> - Name of the backup within the backup bucket.  It can contain '/' characters in order to organize backups into a folder-like structure.
       <bucket_name> - Name of the bucket to use for backup data.
+      <region_name> - If <hostname> is not in s3 compatible form (s3.region-name.example.com) and aws v4 signature is enabled, region name is required.
       
       <param>=<value> - Optional URL parameters.  See below for details.
 
diff --git a/fdbclient/S3BlobStore.actor.cpp b/fdbclient/S3BlobStore.actor.cpp
index 3dc1f551020..f9ddf08b9e9 100644
--- a/fdbclient/S3BlobStore.actor.cpp
+++ b/fdbclient/S3BlobStore.actor.cpp
@@ -163,6 +163,32 @@ std::string S3BlobStoreEndpoint::BlobKnobs::getURLParameters() const {
 	return r;
 }
 
+std::string guessRegionFromDomain(std::string domain) {
+	static const std::vector<const char*> knownServices = { "s3.", "cos.", "oss-", "obs." };
+	boost::algorithm::to_lower(domain);
+
+	for (int i = 0; i < knownServices.size(); ++i) {
+		const char* service = knownServices[i];
+
+		std::size_t p = domain.find(service);
+
+		if (p == std::string::npos || (p >= 1 && domain[p - 1] != '.')) {
+			// eg. 127.0.0.1, example.com, s3-service.example.com, mys3.example.com
+			continue;
+		}
+
+		StringRef h(domain.c_str() + p);
+
+		if (!h.startsWith(LiteralStringRef("oss-"))) {
+			h.eat(service); // ignore s3 service
+		}
+
+		return h.eat(".").toString();
+	}
+
+	return "";
+}
+
 Reference<S3BlobStoreEndpoint> S3BlobStoreEndpoint::fromString(const std::string& url,
                                                                const Optional<std::string>& proxy,
                                                                std::string* resourceFromURL,
@@ -216,6 +242,8 @@ Reference<S3BlobStoreEndpoint> S3BlobStoreEndpoint::fromString(const std::string
 
 		StringRef service = h.eat();
 
+		std::string region = guessRegionFromDomain(host.toString());
+
 		BlobKnobs knobs;
 		HTTP::Headers extraHeaders;
 		while (1) {
@@ -245,6 +273,12 @@ Reference<S3BlobStoreEndpoint> S3BlobStoreEndpoint::fromString(const std::string
 				continue;
 			}
 
+			// overwrite s3 region from parameter
+			if (name == LiteralStringRef("region")) {
+				region = value.toString();
+				continue;
+			}
+
 			// See if the parameter is a knob
 			// First try setting a dummy value (all knobs are currently numeric) just to see if this parameter is known
 			// to S3BlobStoreEndpoint. If it is, then we will set it to a good value or throw below, so the dummy set
@@ -283,8 +317,13 @@ Reference<S3BlobStoreEndpoint> S3BlobStoreEndpoint::fromString(const std::string
 			creds = S3BlobStoreEndpoint::Credentials{ key.toString(), secret.toString(), securityToken.toString() };
 		}
 
+		if (region.empty() && CLIENT_KNOBS->HTTP_REQUEST_AWS_V4_HEADER) {
+			throw std::string(
+			    "Failed to get region from host or parameter in url, region is required for aws v4 signature");
+		}
+
 		return makeReference<S3BlobStoreEndpoint>(
-		    host.toString(), service.toString(), proxyHost, proxyPort, creds, knobs, extraHeaders);
+		    host.toString(), service.toString(), region, proxyHost, proxyPort, creds, knobs, extraHeaders);
 
 	} catch (std::string& err) {
 		if (error != nullptr)
@@ -350,10 +389,25 @@ std::string S3BlobStoreEndpoint::getResourceURL(std::string resource, std::strin
 	return r;
 }
 
+std::string constructResourcePath(Reference<S3BlobStoreEndpoint> b, std::string bucket, std::string object) {
+	std::string resource;
+
+	if (b->getHost().find(bucket + ".") != 0) {
+		resource += std::string("/") + bucket; // not virtual hosting mode
+	}
+
+	if (!object.empty()) {
+		resource += "/";
+		resource += object;
+	}
+
+	return std::move(resource);
+}
+
 ACTOR Future<bool> bucketExists_impl(Reference<S3BlobStoreEndpoint> b, std::string bucket) {
 	wait(b->requestRateRead->getAllowance(1));
 
-	std::string resource = std::string("/") + bucket;
+	std::string resource = constructResourcePath(b, bucket, "");
 	HTTP::Headers headers;
 
 	Reference<HTTP::Response> r = wait(b->doRequest("HEAD", resource, headers, nullptr, 0, { 200, 404 }));
@@ -367,7 +421,7 @@ Future<bool> S3BlobStoreEndpoint::bucketExists(std::string const& bucket) {
 ACTOR Future<bool> objectExists_impl(Reference<S3BlobStoreEndpoint> b, std::string bucket, std::string object) {
 	wait(b->requestRateRead->getAllowance(1));
 
-	std::string resource = std::string("/") + bucket + "/" + object;
+	std::string resource = constructResourcePath(b, bucket, object);
 	HTTP::Headers headers;
 
 	Reference<HTTP::Response> r = wait(b->doRequest("HEAD", resource, headers, nullptr, 0, { 200, 404 }));
@@ -381,7 +435,7 @@ Future<bool> S3BlobStoreEndpoint::objectExists(std::string const& bucket, std::s
 ACTOR Future<Void> deleteObject_impl(Reference<S3BlobStoreEndpoint> b, std::string bucket, std::string object) {
 	wait(b->requestRateDelete->getAllowance(1));
 
-	std::string resource = std::string("/") + bucket + "/" + object;
+	std::string resource = constructResourcePath(b, bucket, object);
 	HTTP::Headers headers;
 	// 200 or 204 means object successfully deleted, 404 means it already doesn't exist, so any of those are considered
 	// successful
@@ -473,9 +527,24 @@ ACTOR Future<Void> createBucket_impl(Reference<S3BlobStoreEndpoint> b, std::stri
 
 	bool exists = wait(b->bucketExists(bucket));
 	if (!exists) {
-		std::string resource = std::string("/") + bucket;
+		std::string resource = constructResourcePath(b, bucket, "");
 		HTTP::Headers headers;
-		Reference<HTTP::Response> r = wait(b->doRequest("PUT", resource, headers, nullptr, 0, { 200, 409 }));
+
+		std::string region = b->getRegion();
+		if (region.empty()) {
+			Reference<HTTP::Response> r = wait(b->doRequest("PUT", resource, headers, nullptr, 0, { 200, 409 }));
+		} else {
+			UnsentPacketQueue packets;
+			StringRef body(format("<CreateBucketConfiguration xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\">"
+			                      "  <LocationConstraint>%s</LocationConstraint>"
+			                      "</CreateBucketConfiguration>",
+			                      region.c_str()));
+			PacketWriter pw(packets.getWriteBuffer(), nullptr, Unversioned());
+			pw.serializeBytes(body);
+
+			Reference<HTTP::Response> r =
+			    wait(b->doRequest("PUT", resource, headers, &packets, body.size(), { 200, 409 }));
+		}
 	}
 	return Void();
 }
@@ -487,7 +556,7 @@ Future<Void> S3BlobStoreEndpoint::createBucket(std::string const& bucket) {
 ACTOR Future<int64_t> objectSize_impl(Reference<S3BlobStoreEndpoint> b, std::string bucket, std::string object) {
 	wait(b->requestRateRead->getAllowance(1));
 
-	std::string resource = std::string("/") + bucket + "/" + object;
+	std::string resource = constructResourcePath(b, bucket, object);
 	HTTP::Headers headers;
 
 	Reference<HTTP::Response> r = wait(b->doRequest("HEAD", resource, headers, nullptr, 0, { 200, 404 }));
@@ -922,8 +991,8 @@ ACTOR Future<Void> listObjectsStream_impl(Reference<S3BlobStoreEndpoint> bstore,
                                           int maxDepth,
                                           std::function<bool(std::string const&)> recurseFilter) {
 	// Request 1000 keys at a time, the maximum allowed
-	state std::string resource = "/";
-	resource.append(bucket);
+	state std::string resource = constructResourcePath(bstore, bucket, "");
+
 	resource.append("/?max-keys=1000");
 	if (prefix.present())
 		resource.append("&prefix=").append(HTTP::urlEncode(prefix.get()));
@@ -1282,10 +1351,6 @@ void S3BlobStoreEndpoint::setV4AuthHeaders(std::string const& verb,
 		amzDate = date;
 		dateStamp = datestamp;
 	}
-	// Extract service and region
-	StringRef hostRef(host);
-	std::string service = hostRef.eat(".").toString();
-	std::string region = hostRef.eat(".").toString();
 
 	// ************* TASK 1: CREATE A CANONICAL REQUEST *************
 	// Create Create canonical URI--the part of the URI from domain to query string (use '/' if no path)
@@ -1342,14 +1407,14 @@ void S3BlobStoreEndpoint::setV4AuthHeaders(std::string const& verb,
 
 	// ************* TASK 2: CREATE THE STRING TO SIGN*************
 	std::string algorithm = "AWS4-HMAC-SHA256";
-	std::string credentialScope = dateStamp + "/" + region + "/" + service + "/" + "aws4_request";
+	std::string credentialScope = dateStamp + "/" + region + "/s3/" + "aws4_request";
 	std::string stringToSign =
 	    algorithm + "\n" + amzDate + "\n" + credentialScope + "\n" + sha256_hex(canonicalRequest);
 
 	// ************* TASK 3: CALCULATE THE SIGNATURE *************
 	// Create the signing key using the function defined above.
-	std::string signingKey = hmac_sha256(
-	    hmac_sha256(hmac_sha256(hmac_sha256("AWS4" + secretKey, dateStamp), region), service), "aws4_request");
+	std::string signingKey =
+	    hmac_sha256(hmac_sha256(hmac_sha256(hmac_sha256("AWS4" + secretKey, dateStamp), region), "s3"), "aws4_request");
 	// Sign the string_to_sign using the signing_key
 	std::string signature = hmac_sha256_hex(signingKey, stringToSign);
 	// ************* TASK 4: ADD SIGNING INFORMATION TO THE Header *************
@@ -1418,7 +1483,7 @@ ACTOR Future<std::string> readEntireFile_impl(Reference<S3BlobStoreEndpoint> bst
                                               std::string object) {
 	wait(bstore->requestRateRead->getAllowance(1));
 
-	std::string resource = std::string("/") + bucket + "/" + object;
+	std::string resource = constructResourcePath(bstore, bucket, object);
 	HTTP::Headers headers;
 	Reference<HTTP::Response> r = wait(bstore->doRequest("GET", resource, headers, nullptr, 0, { 200, 404 }));
 	if (r->code == 404)
@@ -1443,7 +1508,7 @@ ACTOR Future<Void> writeEntireFileFromBuffer_impl(Reference<S3BlobStoreEndpoint>
 	wait(bstore->concurrentUploads.take());
 	state FlowLock::Releaser uploadReleaser(bstore->concurrentUploads, 1);
 
-	std::string resource = std::string("/") + bucket + "/" + object;
+	std::string resource = constructResourcePath(bstore, bucket, object);
 	HTTP::Headers headers;
 	// Send MD5 sum for content so blobstore can verify it
 	headers["Content-MD5"] = contentMD5;
@@ -1513,7 +1578,7 @@ ACTOR Future<int> readObject_impl(Reference<S3BlobStoreEndpoint> bstore,
 		return 0;
 	wait(bstore->requestRateRead->getAllowance(1));
 
-	std::string resource = std::string("/") + bucket + "/" + object;
+	std::string resource = constructResourcePath(bstore, bucket, object);
 	HTTP::Headers headers;
 	headers["Range"] = format("bytes=%lld-%lld", offset, offset + length - 1);
 	Reference<HTTP::Response> r = wait(bstore->doRequest("GET", resource, headers, nullptr, 0, { 200, 206, 404 }));
@@ -1540,7 +1605,8 @@ ACTOR static Future<std::string> beginMultiPartUpload_impl(Reference<S3BlobStore
                                                            std::string object) {
 	wait(bstore->requestRateWrite->getAllowance(1));
 
-	std::string resource = std::string("/") + bucket + "/" + object + "?uploads";
+	std::string resource = constructResourcePath(bstore, bucket, object);
+	resource += "?uploads";
 	HTTP::Headers headers;
 	if (!CLIENT_KNOBS->BLOBSTORE_ENCRYPTION_TYPE.empty())
 		headers["x-amz-server-side-encryption"] = CLIENT_KNOBS->BLOBSTORE_ENCRYPTION_TYPE;
@@ -1582,8 +1648,8 @@ ACTOR Future<std::string> uploadPart_impl(Reference<S3BlobStoreEndpoint> bstore,
 	wait(bstore->concurrentUploads.take());
 	state FlowLock::Releaser uploadReleaser(bstore->concurrentUploads, 1);
 
-	std::string resource =
-	    format("/%s/%s?partNumber=%d&uploadId=%s", bucket.c_str(), object.c_str(), partNumber, uploadID.c_str());
+	std::string resource = constructResourcePath(bstore, bucket, object);
+	resource += format("?partNumber=%d&uploadId=%s", partNumber, uploadID.c_str());
 	HTTP::Headers headers;
 	// Send MD5 sum for content so blobstore can verify it
 	headers["Content-MD5"] = contentMD5;
@@ -1635,7 +1701,8 @@ ACTOR Future<Void> finishMultiPartUpload_impl(Reference<S3BlobStoreEndpoint> bst
 		manifest += format("<Part><PartNumber>%d</PartNumber><ETag>%s</ETag></Part>\n", p.first, p.second.c_str());
 	manifest += "</CompleteMultipartUpload>";
 
-	std::string resource = format("/%s/%s?uploadId=%s", bucket.c_str(), object.c_str(), uploadID.c_str());
+	std::string resource = constructResourcePath(bstore, bucket, object);
+	resource += format("?uploadId=%s", uploadID.c_str());
 	HTTP::Headers headers;
 	PacketWriter pw(part_list.getWriteBuffer(manifest.size()), nullptr, Unversioned());
 	pw.serializeBytes(manifest);
@@ -1659,7 +1726,7 @@ TEST_CASE("/backup/s3/v4headers") {
 	S3BlobStoreEndpoint::Credentials creds{ "AKIAIOSFODNN7EXAMPLE", "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", "" }
 	// GET without query parameters
 	{
-		S3BlobStoreEndpoint s3("s3.amazonaws.com", "s3", "proxy", "port", creds);
+		S3BlobStoreEndpoint s3("s3.amazonaws.com", "443", "amazonaws", "proxy", "port", creds);
 		std::string verb("GET");
 		std::string resource("/test.txt");
 		HTTP::Headers headers;
@@ -1674,7 +1741,7 @@ TEST_CASE("/backup/s3/v4headers") {
 
 	// GET with query parameters
 	{
-		S3BlobStoreEndpoint s3("s3.amazonaws.com", "s3", "proxy", "port", creds);
+		S3BlobStoreEndpoint s3("s3.amazonaws.com", "443", "amazonaws", "proxy", "port", creds);
 		std::string verb("GET");
 		std::string resource("/test/examplebucket?Action=DescribeRegions&Version=2013-10-15");
 		HTTP::Headers headers;
@@ -1689,7 +1756,7 @@ TEST_CASE("/backup/s3/v4headers") {
 
 	// POST
 	{
-		S3BlobStoreEndpoint s3("s3.us-west-2.amazonaws.com", "s3", "proxy", "port", creds);
+		S3BlobStoreEndpoint s3("s3.us-west-2.amazonaws.com", "443", "us-west-2", "proxy", "port", creds);
 		std::string verb("POST");
 		std::string resource("/simple.json");
 		HTTP::Headers headers;
@@ -1706,4 +1773,4 @@ TEST_CASE("/backup/s3/v4headers") {
 	}
 
 	return Void();
-}
\ No newline at end of file
+}
diff --git a/fdbclient/S3BlobStore.h b/fdbclient/S3BlobStore.h
index a61a5958d85..86e0e54e32d 100644
--- a/fdbclient/S3BlobStore.h
+++ b/fdbclient/S3BlobStore.h
@@ -100,12 +100,13 @@ class S3BlobStoreEndpoint : public ReferenceCounted<S3BlobStoreEndpoint> {
 
 	S3BlobStoreEndpoint(std::string const& host,
 	                    std::string const& service,
+	                    std::string region,
 	                    Optional<std::string> const& proxyHost,
 	                    Optional<std::string> const& proxyPort,
 	                    Optional<Credentials> const& creds,
 	                    BlobKnobs const& knobs = BlobKnobs(),
 	                    HTTP::Headers extraHeaders = HTTP::Headers())
-	  : host(host), service(service), proxyHost(proxyHost), proxyPort(proxyPort),
+	  : host(host), service(service), region(region), proxyHost(proxyHost), proxyPort(proxyPort),
 	    useProxy(proxyHost.present() && proxyPort.present()), credentials(creds),
 	    lookupKey(creds.present() && creds.get().key.empty()),
 	    lookupSecret(creds.present() && creds.get().secret.empty()), knobs(knobs), extraHeaders(extraHeaders),
@@ -156,6 +157,7 @@ class S3BlobStoreEndpoint : public ReferenceCounted<S3BlobStoreEndpoint> {
 
 	std::string host;
 	std::string service;
+	std::string region;
 	Optional<std::string> proxyHost;
 	Optional<std::string> proxyPort;
 	bool useProxy;
@@ -193,6 +195,10 @@ class S3BlobStoreEndpoint : public ReferenceCounted<S3BlobStoreEndpoint> {
 	                      std::string date = "",
 	                      std::string datestamp = "");
 
+	std::string getHost() const { return host; }
+
+	std::string getRegion() const { return region; }
+
 	// Prepend the HTTP request header to the given PacketBuffer, returning the new head of the buffer chain
 	static PacketBuffer* writeRequestHeader(std::string const& request,
 	                                        HTTP::Headers const& headers,

From 149c58c12fc1766748f9fa9e78100da7db4d21e7 Mon Sep 17 00:00:00 2001
From: Zhe Wu <halfprice@users.noreply.github.com>
Date: Wed, 10 May 2023 20:55:38 -0700
Subject: [PATCH 155/468] [release 7.1] Status json should show tlog roles for
 old generation tlogs (#10207)

* Status json should show tlog roles for old generation tlogs

* Yeidl in processStatusFetcher when getting log and log router processes in earlier generations
---
 fdbserver/Status.actor.cpp | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/fdbserver/Status.actor.cpp b/fdbserver/Status.actor.cpp
index 06e1a6c5aaa..cf63e88d6c8 100644
--- a/fdbserver/Status.actor.cpp
+++ b/fdbserver/Status.actor.cpp
@@ -826,14 +826,23 @@ ACTOR static Future<JsonBuilderObject> processStatusFetcher(
 		}
 	}
 
-	for (auto& old : db->get().logSystemConfig.oldTLogs) {
-		for (auto& tLogSet : old.tLogs) {
+	state std::vector<OldTLogConf>::const_iterator oldTLogIter;
+	for (oldTLogIter = db->get().logSystemConfig.oldTLogs.begin();
+	     oldTLogIter != db->get().logSystemConfig.oldTLogs.end();
+	     ++oldTLogIter) {
+		for (auto& tLogSet : oldTLogIter->tLogs) {
+			for (auto& it : tLogSet.tLogs) {
+				if (it.present()) {
+					roles.addRole("log", it.interf());
+				}
+			}
 			for (auto& it : tLogSet.logRouters) {
 				if (it.present()) {
 					roles.addRole("router", it.interf());
 				}
 			}
 		}
+		wait(yield());
 	}
 
 	for (const auto& coordinator : coordinatorAddresses) {

From 188000bdd155ceb0476f508efc2336a423f71361 Mon Sep 17 00:00:00 2001
From: neethuhaneesha <nbingi@apple.com>
Date: Wed, 10 May 2023 15:27:41 -0700
Subject: [PATCH 156/468] RocksDB WAL archive options.

---
 fdbclient/ServerKnobs.cpp                |  2 ++
 fdbclient/ServerKnobs.h                  |  2 ++
 fdbserver/KeyValueStoreRocksDB.actor.cpp | 14 ++++++++++++++
 3 files changed, 18 insertions(+)

diff --git a/fdbclient/ServerKnobs.cpp b/fdbclient/ServerKnobs.cpp
index 60cd24ef546..c619ac9aeef 100644
--- a/fdbclient/ServerKnobs.cpp
+++ b/fdbclient/ServerKnobs.cpp
@@ -419,6 +419,8 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi
 	// Enable this knob only for experminatal purpose, never enable this in production.
 	// If enabled, all the committed in-memory memtable writes are lost on a crash.
 	init( ROCKSDB_DISABLE_WAL_EXPERIMENTAL,                    false );
+	init( ROCKSDB_WAL_TTL_SECONDS,                                 0 );
+	init( ROCKSDB_WAL_SIZE_LIMIT_MB,                               0 );
 	// If ROCKSDB_SINGLEKEY_DELETES_ON_CLEARRANGE is enabled, disable ROCKSDB_ENABLE_CLEAR_RANGE_EAGER_READS knob.
 	// These knobs have contrary functionality.
 	init( ROCKSDB_SINGLEKEY_DELETES_ON_CLEARRANGE,              true );
diff --git a/fdbclient/ServerKnobs.h b/fdbclient/ServerKnobs.h
index 0ea56d81a27..e3e7ff8e3a0 100644
--- a/fdbclient/ServerKnobs.h
+++ b/fdbclient/ServerKnobs.h
@@ -341,6 +341,8 @@ class ServerKnobs : public KnobsImpl<ServerKnobs> {
 	int ROCKSDB_CAN_COMMIT_DELAY_ON_OVERLOAD;
 	int ROCKSDB_CAN_COMMIT_DELAY_TIMES_ON_OVERLOAD;
 	bool ROCKSDB_DISABLE_WAL_EXPERIMENTAL;
+	int64_t ROCKSDB_WAL_TTL_SECONDS;
+	int64_t ROCKSDB_WAL_SIZE_LIMIT_MB;
 	bool ROCKSDB_SINGLEKEY_DELETES_ON_CLEARRANGE;
 	int ROCKSDB_SINGLEKEY_DELETES_MAX;
 	bool ROCKSDB_ENABLE_CLEAR_RANGE_EAGER_READS;
diff --git a/fdbserver/KeyValueStoreRocksDB.actor.cpp b/fdbserver/KeyValueStoreRocksDB.actor.cpp
index 6a3828e0f3c..b84d6136668 100644
--- a/fdbserver/KeyValueStoreRocksDB.actor.cpp
+++ b/fdbserver/KeyValueStoreRocksDB.actor.cpp
@@ -402,6 +402,20 @@ rocksdb::Options getOptions() {
 	if (SERVER_KNOBS->ROCKSDB_COMPACTION_READAHEAD_SIZE > 0) {
 		options.compaction_readahead_size = SERVER_KNOBS->ROCKSDB_COMPACTION_READAHEAD_SIZE;
 	}
+	// The following two fields affect how archived logs will be deleted.
+	// 1. If both set to 0, logs will be deleted asap and will not get into
+	//    the archive.
+	// 2. If WAL_ttl_seconds is 0 and WAL_size_limit_MB is not 0,
+	//    WAL files will be checked every 10 min and if total size is greater
+	//    then WAL_size_limit_MB, they will be deleted starting with the
+	//    earliest until size_limit is met. All empty files will be deleted.
+	// 3. If WAL_ttl_seconds is not 0 and WAL_size_limit_MB is 0, then
+	//    WAL files will be checked every WAL_ttl_seconds / 2 and those that
+	//    are older than WAL_ttl_seconds will be deleted.
+	// 4. If both are not 0, WAL files will be checked every 10 min and both
+	//    checks will be performed with ttl being first.
+	options.WAL_ttl_seconds = SERVER_KNOBS->ROCKSDB_WAL_TTL_SECONDS;
+	options.WAL_size_limit_MB = SERVER_KNOBS->ROCKSDB_WAL_SIZE_LIMIT_MB;
 
 	options.statistics = rocksdb::CreateDBStatistics();
 	options.statistics->set_stats_level(rocksdb::StatsLevel(SERVER_KNOBS->ROCKSDB_STATS_LEVEL));

From d1b458f8de80f527ddad840b393569da1cc7f81f Mon Sep 17 00:00:00 2001
From: He Liu <86634338+liquid-helium@users.noreply.github.com>
Date: Thu, 27 Apr 2023 12:32:04 -0700
Subject: [PATCH 157/468] Upgrade to RocksDB 8.1.1 (#10039)

* Upgrade to RocksDB 8.1.1

* Revert an ASSERT on the previous rocksdb bug. The test was a guard to show the failure.
---
 cmake/CompileRocksDB.cmake                    |    6 +-
 fdbserver/FDBRocksDBVersion.h                 |   28 +
 fdbserver/KeyValueStoreRocksDB.actor.cpp      |   25 +-
 .../KeyValueStoreShardedRocksDB.actor.cpp     | 4120 +++++++++++++++++
 fdbserver/RocksDBCheckpointUtils.actor.h      |   14 +-
 5 files changed, 4178 insertions(+), 15 deletions(-)
 create mode 100644 fdbserver/FDBRocksDBVersion.h
 create mode 100644 fdbserver/KeyValueStoreShardedRocksDB.actor.cpp

diff --git a/cmake/CompileRocksDB.cmake b/cmake/CompileRocksDB.cmake
index b1f5d8e8e35..709c61dfb9d 100644
--- a/cmake/CompileRocksDB.cmake
+++ b/cmake/CompileRocksDB.cmake
@@ -1,6 +1,6 @@
 # FindRocksDB
 
-find_package(RocksDB 7.10.2)
+find_package(RocksDB 8.1.1)
 
 include(ExternalProject)
 
@@ -54,8 +54,8 @@ if(ROCKSDB_FOUND)
     ${BINARY_DIR}/librocksdb.a)
 else()
   ExternalProject_Add(rocksdb
-    URL https://github.com/facebook/rocksdb/archive/refs/tags/v7.10.2.tar.gz
-    URL_HASH SHA256=4619ae7308cd3d11cdd36f0bfad3fb03a1ad399ca333f192b77b6b95b08e2f78
+    URL https://github.com/facebook/rocksdb/archive/refs/tags/v8.1.1.tar.gz
+    URL_HASH SHA256=9102704e169cfb53e7724a30750eeeb3e71307663852f01fa08d5a320e6155a8
     CMAKE_ARGS ${RocksDB_CMAKE_ARGS}
     BUILD_BYPRODUCTS <BINARY_DIR>/librocksdb.a
     INSTALL_COMMAND ""
diff --git a/fdbserver/FDBRocksDBVersion.h b/fdbserver/FDBRocksDBVersion.h
new file mode 100644
index 00000000000..d8c1272a168
--- /dev/null
+++ b/fdbserver/FDBRocksDBVersion.h
@@ -0,0 +1,28 @@
+/*
+ * FDBRocksDBVersion.h
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2023 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef FDBSERVER_FDBROCKSDBVERSION_H
+#define FDBSERVER_FDBROCKSDBVERSION_H
+
+#define FDB_ROCKSDB_MAJOR 8
+#define FDB_ROCKSDB_MINOR 1
+#define FDB_ROCKSDB_PATCH 1
+
+#endif
\ No newline at end of file
diff --git a/fdbserver/KeyValueStoreRocksDB.actor.cpp b/fdbserver/KeyValueStoreRocksDB.actor.cpp
index b84d6136668..2aaa2b020af 100644
--- a/fdbserver/KeyValueStoreRocksDB.actor.cpp
+++ b/fdbserver/KeyValueStoreRocksDB.actor.cpp
@@ -50,6 +50,7 @@
 #endif
 #include "fdbclient/SystemData.h"
 #include "fdbserver/CoroFlow.h"
+#include "fdbserver/FDBRocksDBVersion.h"
 #include "flow/flow.h"
 #include "flow/IThreadPool.h"
 #include "flow/ThreadHelper.actor.h"
@@ -68,9 +69,10 @@
 
 #ifdef SSD_ROCKSDB_EXPERIMENTAL
 
-// Enforcing rocksdb version to be 7.10.2
-static_assert((ROCKSDB_MAJOR == 7 && ROCKSDB_MINOR == 10 && ROCKSDB_PATCH == 2),
-              "Unsupported rocksdb version. Update the rocksdb to 7.10.2 version");
+// Enforcing rocksdb version.
+static_assert((ROCKSDB_MAJOR == FDB_ROCKSDB_MAJOR && ROCKSDB_MINOR == FDB_ROCKSDB_MINOR &&
+               ROCKSDB_PATCH == FDB_ROCKSDB_PATCH),
+              "Unsupported rocksdb version.");
 
 namespace {
 using rocksdb::BackgroundErrorReason;
@@ -932,7 +934,8 @@ ACTOR Future<Void> rocksDBMetricLogger(UID id,
 		{ "BloomFilterUseful", rocksdb::BLOOM_FILTER_USEFUL, 0 },
 		{ "BloomFilterFullPositive", rocksdb::BLOOM_FILTER_FULL_POSITIVE, 0 },
 		{ "BloomFilterTruePositive", rocksdb::BLOOM_FILTER_FULL_TRUE_POSITIVE, 0 },
-		{ "BloomFilterMicros", rocksdb::BLOOM_FILTER_MICROS, 0 },
+		// Deprecated in RocksDB 8.0
+		// { "BloomFilterMicros", rocksdb::BLOOM_FILTER_MICROS, 0 },
 		{ "MemtableHit", rocksdb::MEMTABLE_HIT, 0 },
 		{ "MemtableMiss", rocksdb::MEMTABLE_MISS, 0 },
 		{ "GetHitL0", rocksdb::GET_HIT_L0, 0 },
@@ -945,8 +948,9 @@ ACTOR Future<Void> rocksDBMetricLogger(UID id,
 		{ "CountDBPrev", rocksdb::NUMBER_DB_PREV, 0 },
 		{ "BloomFilterPrefixChecked", rocksdb::BLOOM_FILTER_PREFIX_CHECKED, 0 },
 		{ "BloomFilterPrefixUseful", rocksdb::BLOOM_FILTER_PREFIX_USEFUL, 0 },
-		{ "BlockCacheCompressedMiss", rocksdb::BLOCK_CACHE_COMPRESSED_MISS, 0 },
-		{ "BlockCacheCompressedHit", rocksdb::BLOCK_CACHE_COMPRESSED_HIT, 0 },
+		// Deprecated in RocksDB 8.0
+		// { "BlockCacheCompressedMiss", rocksdb::BLOCK_CACHE_COMPRESSED_MISS, 0 },
+		// { "BlockCacheCompressedHit", rocksdb::BLOCK_CACHE_COMPRESSED_HIT, 0 },
 		{ "CountWalFileSyncs", rocksdb::WAL_FILE_SYNCED, 0 },
 		{ "CountWalFileBytes", rocksdb::WAL_FILE_BYTES, 0 },
 		{ "CompactReadBytes", rocksdb::COMPACT_READ_BYTES, 0 },
@@ -976,10 +980,11 @@ ACTOR Future<Void> rocksDBMetricLogger(UID id,
 		  rocksdb::COMPRESSION_TIMES_NANOS }, // enabled if rocksdb::StatsLevel > kExceptDetailedTimers(3)
 		{ "DecompressionTimeNanos",
 		  rocksdb::DECOMPRESSION_TIMES_NANOS }, // enabled if rocksdb::StatsLevel > kExceptDetailedTimers(3)
-		{ "HardRateLimitDelayCount",
-		  rocksdb::HARD_RATE_LIMIT_DELAY_COUNT }, // enabled if rocksdb::StatsLevel > kExceptHistogramOrTimers(1)
-		{ "SoftRateLimitDelayCount",
-		  rocksdb::SOFT_RATE_LIMIT_DELAY_COUNT }, // enabled if rocksdb::StatsLevel > kExceptHistogramOrTimers(1)
+		// Deprecated in RocksDB 8.0
+		// { "HardRateLimitDelayCount",
+		//   rocksdb::HARD_RATE_LIMIT_DELAY_COUNT }, // enabled if rocksdb::StatsLevel > kExceptHistogramOrTimers(1)
+		// { "SoftRateLimitDelayCount",
+		//   rocksdb::SOFT_RATE_LIMIT_DELAY_COUNT }, // enabled if rocksdb::StatsLevel > kExceptHistogramOrTimers(1)
 		{ "WriteStall", rocksdb::WRITE_STALL }, // enabled if rocksdb::StatsLevel > kExceptHistogramOrTimers(1)
 	};
 	state std::vector<std::pair<const char*, std::string>> intPropertyStats = {
diff --git a/fdbserver/KeyValueStoreShardedRocksDB.actor.cpp b/fdbserver/KeyValueStoreShardedRocksDB.actor.cpp
new file mode 100644
index 00000000000..694862d64c4
--- /dev/null
+++ b/fdbserver/KeyValueStoreShardedRocksDB.actor.cpp
@@ -0,0 +1,4120 @@
+#include "fdbclient/FDBTypes.h"
+#ifdef SSD_ROCKSDB_EXPERIMENTAL
+
+#include "fdbclient/KeyRangeMap.h"
+#include "fdbclient/SystemData.h"
+#include "flow/flow.h"
+#include "flow/serialize.h"
+#include <rocksdb/c.h>
+#include <rocksdb/cache.h>
+#include <rocksdb/db.h>
+#include <rocksdb/filter_policy.h>
+#include <rocksdb/listener.h>
+#include <rocksdb/metadata.h>
+#include <rocksdb/options.h>
+#include <rocksdb/perf_context.h>
+#include <rocksdb/rate_limiter.h>
+#include <rocksdb/slice_transform.h>
+#include <rocksdb/statistics.h>
+#include <rocksdb/table.h>
+#include <rocksdb/utilities/checkpoint.h>
+#include <rocksdb/utilities/table_properties_collectors.h>
+#include <rocksdb/version.h>
+#if defined __has_include
+#if __has_include(<liburing.h>)
+#include <liburing.h>
+#endif
+#endif
+#include "fdbclient/SystemData.h"
+#include "fdbserver/CoroFlow.h"
+#include "fdbserver/FDBRocksDBVersion.h"
+#include "flow/flow.h"
+#include "flow/IThreadPool.h"
+#include "flow/ThreadHelper.actor.h"
+#include "flow/Histogram.h"
+
+#include <memory>
+#include <tuple>
+#include <vector>
+
+#endif // SSD_ROCKSDB_EXPERIMENTAL
+
+#include "fdbserver/IKeyValueStore.h"
+#include "fdbserver/RocksDBCheckpointUtils.actor.h"
+#include "flow/actorcompiler.h" // has to be last include
+
+#ifdef SSD_ROCKSDB_EXPERIMENTAL
+
+// Enforcing rocksdb version.
+static_assert((ROCKSDB_MAJOR == FDB_ROCKSDB_MAJOR && ROCKSDB_MINOR == FDB_ROCKSDB_MINOR &&
+               ROCKSDB_PATCH == FDB_ROCKSDB_PATCH),
+              "Unsupported rocksdb version.");
+
+const std::string rocksDataFolderSuffix = "-data";
+const std::string METADATA_SHARD_ID = "kvs-metadata";
+const std::string DEFAULT_CF_NAME = "default"; // `specialKeys` is stored in this culoumn family.
+const KeyRef shardMappingPrefix("\xff\xff/ShardMapping/"_sr);
+// TODO: move constants to a header file.
+const KeyRef persistVersion = "\xff\xffVersion"_sr;
+const StringRef ROCKSDBSTORAGE_HISTOGRAM_GROUP = "RocksDBStorage"_sr;
+const StringRef ROCKSDB_COMMIT_LATENCY_HISTOGRAM = "RocksDBCommitLatency"_sr;
+const StringRef ROCKSDB_COMMIT_ACTION_HISTOGRAM = "RocksDBCommitAction"_sr;
+const StringRef ROCKSDB_COMMIT_QUEUEWAIT_HISTOGRAM = "RocksDBCommitQueueWait"_sr;
+const StringRef ROCKSDB_WRITE_HISTOGRAM = "RocksDBWrite"_sr;
+const StringRef ROCKSDB_DELETE_COMPACTRANGE_HISTOGRAM = "RocksDBDeleteCompactRange"_sr;
+const StringRef ROCKSDB_READRANGE_LATENCY_HISTOGRAM = "RocksDBReadRangeLatency"_sr;
+const StringRef ROCKSDB_READVALUE_LATENCY_HISTOGRAM = "RocksDBReadValueLatency"_sr;
+const StringRef ROCKSDB_READPREFIX_LATENCY_HISTOGRAM = "RocksDBReadPrefixLatency"_sr;
+const StringRef ROCKSDB_READRANGE_ACTION_HISTOGRAM = "RocksDBReadRangeAction"_sr;
+const StringRef ROCKSDB_READVALUE_ACTION_HISTOGRAM = "RocksDBReadValueAction"_sr;
+const StringRef ROCKSDB_READPREFIX_ACTION_HISTOGRAM = "RocksDBReadPrefixAction"_sr;
+const StringRef ROCKSDB_READRANGE_QUEUEWAIT_HISTOGRAM = "RocksDBReadRangeQueueWait"_sr;
+const StringRef ROCKSDB_READVALUE_QUEUEWAIT_HISTOGRAM = "RocksDBReadValueQueueWait"_sr;
+const StringRef ROCKSDB_READPREFIX_QUEUEWAIT_HISTOGRAM = "RocksDBReadPrefixQueueWait"_sr;
+const StringRef ROCKSDB_READRANGE_NEWITERATOR_HISTOGRAM = "RocksDBReadRangeNewIterator"_sr;
+const StringRef ROCKSDB_READVALUE_GET_HISTOGRAM = "RocksDBReadValueGet"_sr;
+const StringRef ROCKSDB_READPREFIX_GET_HISTOGRAM = "RocksDBReadPrefixGet"_sr;
+
+namespace {
+struct PhysicalShard;
+struct DataShard;
+struct ReadIterator;
+struct ShardedRocksDBKeyValueStore;
+
+using rocksdb::BackgroundErrorReason;
+
+// Returns string representation of RocksDB background error reason.
+// Error reason code:
+// https://github.com/facebook/rocksdb/blob/12d798ac06bcce36be703b057d5f5f4dab3b270c/include/rocksdb/listener.h#L125
+// This function needs to be updated when error code changes.
+std::string getErrorReason(BackgroundErrorReason reason) {
+	switch (reason) {
+	case BackgroundErrorReason::kFlush:
+		return format("%d Flush", reason);
+	case BackgroundErrorReason::kCompaction:
+		return format("%d Compaction", reason);
+	case BackgroundErrorReason::kWriteCallback:
+		return format("%d WriteCallback", reason);
+	case BackgroundErrorReason::kMemTable:
+		return format("%d MemTable", reason);
+	case BackgroundErrorReason::kManifestWrite:
+		return format("%d ManifestWrite", reason);
+	case BackgroundErrorReason::kFlushNoWAL:
+		return format("%d FlushNoWAL", reason);
+	case BackgroundErrorReason::kManifestWriteNoWAL:
+		return format("%d ManifestWriteNoWAL", reason);
+	default:
+		return format("%d Unknown", reason);
+	}
+}
+
+// Background error handling is tested with Chaos test.
+// TODO: Test background error in simulation. RocksDB doesn't use flow IO in simulation, which limits our ability to
+// inject IO errors. We could implement rocksdb::FileSystem using flow IO to unblock simulation. Also, trace event is
+// not available on background threads because trace event requires setting up special thread locals. Using trace event
+// could potentially cause segmentation fault.
+class RocksDBErrorListener : public rocksdb::EventListener {
+public:
+	RocksDBErrorListener(){};
+	void OnBackgroundError(rocksdb::BackgroundErrorReason reason, rocksdb::Status* bg_error) override {
+		TraceEvent(SevError, "ShardedRocksDBBGError")
+		    .detail("Reason", getErrorReason(reason))
+		    .detail("ShardedRocksDBSeverity", bg_error->severity())
+		    .detail("Status", bg_error->ToString());
+		std::unique_lock<std::mutex> lock(mutex);
+		if (!errorPromise.isValid())
+			return;
+		// RocksDB generates two types of background errors, IO Error and Corruption
+		// Error type and severity map could be found at
+		// https://github.com/facebook/rocksdb/blob/2e09a54c4fb82e88bcaa3e7cfa8ccbbbbf3635d5/db/error_handler.cc#L138.
+		// All background errors will be treated as storage engine failure. Send the error to storage server.
+		if (bg_error->IsIOError()) {
+			errorPromise.sendError(io_error());
+		} else if (bg_error->IsCorruption()) {
+			errorPromise.sendError(file_corrupt());
+		} else {
+			errorPromise.sendError(unknown_error());
+		}
+	}
+	Future<Void> getFuture() {
+		std::unique_lock<std::mutex> lock(mutex);
+		return errorPromise.getFuture();
+	}
+	~RocksDBErrorListener() {
+		std::unique_lock<std::mutex> lock(mutex);
+		if (!errorPromise.isValid())
+			return;
+		errorPromise.send(Never());
+	}
+
+private:
+	ThreadReturnPromise<Void> errorPromise;
+	std::mutex mutex;
+};
+
+// Encapsulation of shared states.
+struct ShardedRocksDBState {
+	bool closing = false;
+};
+
+std::shared_ptr<rocksdb::Cache> rocksdb_block_cache = nullptr;
+
+rocksdb::ExportImportFilesMetaData getMetaData(const CheckpointMetaData& checkpoint) {
+	rocksdb::ExportImportFilesMetaData metaData;
+	if (checkpoint.getFormat() != DataMoveRocksCF) {
+		return metaData;
+	}
+
+	RocksDBColumnFamilyCheckpoint rocksCF = getRocksCF(checkpoint);
+	metaData.db_comparator_name = rocksCF.dbComparatorName;
+
+	for (const LiveFileMetaData& fileMetaData : rocksCF.sstFiles) {
+		rocksdb::LiveFileMetaData liveFileMetaData;
+		liveFileMetaData.relative_filename = fileMetaData.relative_filename;
+		liveFileMetaData.directory = fileMetaData.directory;
+		liveFileMetaData.file_number = fileMetaData.file_number;
+		liveFileMetaData.file_type = static_cast<rocksdb::FileType>(fileMetaData.file_type);
+		liveFileMetaData.size = fileMetaData.size;
+		liveFileMetaData.temperature = static_cast<rocksdb::Temperature>(fileMetaData.temperature);
+		liveFileMetaData.file_checksum = fileMetaData.file_checksum;
+		liveFileMetaData.file_checksum_func_name = fileMetaData.file_checksum_func_name;
+		liveFileMetaData.smallest_seqno = fileMetaData.smallest_seqno;
+		liveFileMetaData.largest_seqno = fileMetaData.largest_seqno;
+		liveFileMetaData.smallestkey = fileMetaData.smallestkey;
+		liveFileMetaData.largestkey = fileMetaData.largestkey;
+		liveFileMetaData.num_reads_sampled = fileMetaData.num_reads_sampled;
+		liveFileMetaData.being_compacted = fileMetaData.being_compacted;
+		liveFileMetaData.num_entries = fileMetaData.num_entries;
+		liveFileMetaData.num_deletions = fileMetaData.num_deletions;
+		liveFileMetaData.oldest_blob_file_number = fileMetaData.oldest_blob_file_number;
+		liveFileMetaData.oldest_ancester_time = fileMetaData.oldest_ancester_time;
+		liveFileMetaData.file_creation_time = fileMetaData.file_creation_time;
+		liveFileMetaData.smallest = fileMetaData.smallest;
+		liveFileMetaData.largest = fileMetaData.largest;
+		liveFileMetaData.file_type = rocksdb::kTableFile;
+		liveFileMetaData.epoch_number = fileMetaData.epoch_number;
+		liveFileMetaData.name = fileMetaData.name;
+		liveFileMetaData.db_path = fileMetaData.db_path;
+		liveFileMetaData.column_family_name = fileMetaData.column_family_name;
+		liveFileMetaData.level = fileMetaData.level;
+		metaData.files.push_back(liveFileMetaData);
+	}
+
+	return metaData;
+}
+
+void populateMetaData(CheckpointMetaData* checkpoint, const rocksdb::ExportImportFilesMetaData* metaData) {
+	RocksDBColumnFamilyCheckpoint rocksCF;
+	if (metaData != nullptr) {
+		rocksCF.dbComparatorName = metaData->db_comparator_name;
+		for (const rocksdb::LiveFileMetaData& fileMetaData : metaData->files) {
+			LiveFileMetaData liveFileMetaData;
+			liveFileMetaData.relative_filename = fileMetaData.relative_filename;
+			liveFileMetaData.directory = fileMetaData.directory;
+			liveFileMetaData.file_number = fileMetaData.file_number;
+			liveFileMetaData.file_type = static_cast<int>(fileMetaData.file_type);
+			liveFileMetaData.size = fileMetaData.size;
+			liveFileMetaData.temperature = static_cast<uint8_t>(fileMetaData.temperature);
+			liveFileMetaData.file_checksum = fileMetaData.file_checksum;
+			liveFileMetaData.file_checksum_func_name = fileMetaData.file_checksum_func_name;
+			liveFileMetaData.smallest_seqno = fileMetaData.smallest_seqno;
+			liveFileMetaData.largest_seqno = fileMetaData.largest_seqno;
+			liveFileMetaData.smallestkey = fileMetaData.smallestkey;
+			liveFileMetaData.largestkey = fileMetaData.largestkey;
+			liveFileMetaData.num_reads_sampled = fileMetaData.num_reads_sampled;
+			liveFileMetaData.being_compacted = fileMetaData.being_compacted;
+			liveFileMetaData.num_entries = fileMetaData.num_entries;
+			liveFileMetaData.num_deletions = fileMetaData.num_deletions;
+			liveFileMetaData.oldest_blob_file_number = fileMetaData.oldest_blob_file_number;
+			liveFileMetaData.oldest_ancester_time = fileMetaData.oldest_ancester_time;
+			liveFileMetaData.file_creation_time = fileMetaData.file_creation_time;
+			liveFileMetaData.smallest = fileMetaData.smallest;
+			liveFileMetaData.largest = fileMetaData.largest;
+			liveFileMetaData.epoch_number = fileMetaData.epoch_number;
+			liveFileMetaData.name = fileMetaData.name;
+			liveFileMetaData.db_path = fileMetaData.db_path;
+			liveFileMetaData.column_family_name = fileMetaData.column_family_name;
+			liveFileMetaData.level = fileMetaData.level;
+			rocksCF.sstFiles.push_back(liveFileMetaData);
+		}
+	}
+	checkpoint->setFormat(DataMoveRocksCF);
+	checkpoint->serializedCheckpoint = ObjectWriter::toValue(rocksCF, IncludeVersion());
+}
+
+const rocksdb::Slice toSlice(StringRef s) {
+	return rocksdb::Slice(reinterpret_cast<const char*>(s.begin()), s.size());
+}
+
+StringRef toStringRef(rocksdb::Slice s) {
+	return StringRef(reinterpret_cast<const uint8_t*>(s.data()), s.size());
+}
+
+std::string getShardMappingKey(KeyRef key, StringRef prefix) {
+	return prefix.toString() + key.toString();
+}
+
+void logRocksDBError(const rocksdb::Status& status, const std::string& method) {
+	auto level = status.IsTimedOut() ? SevWarn : SevError;
+	TraceEvent e(level, "ShardedRocksDBError");
+	e.setMaxFieldLength(10000)
+	    .detail("Error", status.ToString())
+	    .detail("Method", method)
+	    .detail("ShardedRocksDBSeverity", status.severity());
+	if (status.IsIOError()) {
+		e.detail("SubCode", status.subcode());
+	}
+}
+
+// TODO: define shard ops.
+enum class ShardOp {
+	CREATE,
+	OPEN,
+	DESTROY,
+	CLOSE,
+	MODIFY_RANGE,
+};
+
+const char* ShardOpToString(ShardOp op) {
+	switch (op) {
+	case ShardOp::CREATE:
+		return "CREATE";
+	case ShardOp::OPEN:
+		return "OPEN";
+	case ShardOp::DESTROY:
+		return "DESTROY";
+	case ShardOp::CLOSE:
+		return "CLOSE";
+	case ShardOp::MODIFY_RANGE:
+		return "MODIFY_RANGE";
+	default:
+		return "Unknown";
+	}
+}
+void logShardEvent(StringRef name, ShardOp op, Severity severity = SevInfo, const std::string& message = "") {
+	TraceEvent e(severity, "ShardedRocksDBKVSShardEvent");
+	e.detail("Name", name).detail("Action", ShardOpToString(op));
+	if (!message.empty()) {
+		e.detail("Message", message);
+	}
+}
+void logShardEvent(StringRef name,
+                   KeyRangeRef range,
+                   ShardOp op,
+                   Severity severity = SevInfo,
+                   const std::string& message = "") {
+	TraceEvent e(severity, "ShardedRocksDBKVSShardEvent");
+	e.detail("Name", name).detail("Action", ShardOpToString(op)).detail("Begin", range.begin).detail("End", range.end);
+	if (message != "") {
+		e.detail("Message", message);
+	}
+}
+
+Error statusToError(const rocksdb::Status& s) {
+	if (s.IsIOError()) {
+		return io_error();
+	} else if (s.IsTimedOut()) {
+		return key_value_store_deadline_exceeded();
+	} else {
+		return unknown_error();
+	}
+}
+
+rocksdb::ColumnFamilyOptions getCFOptions() {
+	rocksdb::ColumnFamilyOptions options;
+	options.disable_auto_compactions = SERVER_KNOBS->ROCKSDB_DISABLE_AUTO_COMPACTIONS;
+	options.level_compaction_dynamic_level_bytes = SERVER_KNOBS->ROCKSDB_LEVEL_COMPACTION_DYNAMIC_LEVEL_BYTES;
+	options.OptimizeLevelStyleCompaction(SERVER_KNOBS->ROCKSDB_MEMTABLE_BYTES);
+	if (SERVER_KNOBS->ROCKSDB_PERIODIC_COMPACTION_SECONDS > 0) {
+		options.periodic_compaction_seconds = SERVER_KNOBS->ROCKSDB_PERIODIC_COMPACTION_SECONDS;
+	}
+	// Compact sstables when there's too much deleted stuff.
+	options.table_properties_collector_factories = { rocksdb::NewCompactOnDeletionCollectorFactory(128, 1) };
+
+	rocksdb::BlockBasedTableOptions bbOpts;
+	// TODO: Add a knob for the block cache size. (Default is 8 MB)
+	if (SERVER_KNOBS->ROCKSDB_PREFIX_LEN > 0) {
+		// Prefix blooms are used during Seek.
+		options.prefix_extractor.reset(rocksdb::NewFixedPrefixTransform(SERVER_KNOBS->ROCKSDB_PREFIX_LEN));
+
+		// Also turn on bloom filters in the memtable.
+		// TODO: Make a knob for this as well.
+		options.memtable_prefix_bloom_size_ratio = 0.1;
+
+		// 5 -- Can be read by RocksDB's versions since 6.6.0. Full and partitioned
+		// filters use a generally faster and more accurate Bloom filter
+		// implementation, with a different schema.
+		// https://github.com/facebook/rocksdb/blob/b77569f18bfc77fb1d8a0b3218f6ecf571bc4988/include/rocksdb/table.h#L391
+		bbOpts.format_version = 5;
+
+		// Create and apply a bloom filter using the 10 bits
+		// which should yield a ~1% false positive rate:
+		// https://github.com/facebook/rocksdb/wiki/RocksDB-Bloom-Filter#full-filters-new-format
+		bbOpts.filter_policy.reset(rocksdb::NewBloomFilterPolicy(10));
+
+		// The whole key blooms are only used for point lookups.
+		// https://github.com/facebook/rocksdb/wiki/RocksDB-Bloom-Filter#prefix-vs-whole-key
+		bbOpts.whole_key_filtering = false;
+	}
+
+	if (rocksdb_block_cache == nullptr && SERVER_KNOBS->ROCKSDB_BLOCK_CACHE_SIZE > 0) {
+		rocksdb_block_cache = rocksdb::NewLRUCache(SERVER_KNOBS->ROCKSDB_BLOCK_CACHE_SIZE);
+	}
+	bbOpts.block_cache = rocksdb_block_cache;
+
+	options.table_factory.reset(rocksdb::NewBlockBasedTableFactory(bbOpts));
+
+	return options;
+}
+
+rocksdb::Options getOptions() {
+	rocksdb::Options options({}, getCFOptions());
+	options.avoid_unnecessary_blocking_io = true;
+	options.create_if_missing = true;
+	options.atomic_flush = SERVER_KNOBS->ROCKSDB_ATOMIC_FLUSH;
+	if (SERVER_KNOBS->ROCKSDB_BACKGROUND_PARALLELISM > 0) {
+		options.IncreaseParallelism(SERVER_KNOBS->ROCKSDB_BACKGROUND_PARALLELISM);
+	}
+
+	options.delete_obsolete_files_period_micros = SERVER_KNOBS->ROCKSDB_DELETE_OBSOLETE_FILE_PERIOD * 1000000;
+	options.max_total_wal_size = SERVER_KNOBS->ROCKSDB_MAX_TOTAL_WAL_SIZE;
+	options.max_subcompactions = SERVER_KNOBS->ROCKSDB_MAX_SUBCOMPACTIONS;
+	options.max_background_jobs = SERVER_KNOBS->ROCKSDB_MAX_BACKGROUND_JOBS;
+
+	options.db_write_buffer_size = SERVER_KNOBS->ROCKSDB_WRITE_BUFFER_SIZE;
+	options.write_buffer_size = SERVER_KNOBS->ROCKSDB_CF_WRITE_BUFFER_SIZE;
+	options.statistics = rocksdb::CreateDBStatistics();
+	options.statistics->set_stats_level(rocksdb::kExceptHistogramOrTimers);
+	options.db_log_dir = g_network->isSimulated() ? "" : SERVER_KNOBS->LOG_DIRECTORY;
+	return options;
+}
+
+// Set some useful defaults desired for all reads.
+rocksdb::ReadOptions getReadOptions() {
+	rocksdb::ReadOptions options;
+	options.background_purge_on_iterator_cleanup = true;
+	options.auto_prefix_mode = (SERVER_KNOBS->ROCKSDB_PREFIX_LEN > 0);
+	return options;
+}
+
+struct ReadIterator {
+	uint64_t index; // incrementing counter to uniquely identify read iterator.
+	bool inUse;
+	std::shared_ptr<rocksdb::Iterator> iter;
+	double creationTime;
+	KeyRange keyRange;
+	std::shared_ptr<rocksdb::Slice> beginSlice, endSlice;
+
+	ReadIterator(rocksdb::ColumnFamilyHandle* cf, uint64_t index, rocksdb::DB* db)
+	  : index(index), inUse(true), creationTime(now()), iter(db->NewIterator(getReadOptions(), cf)) {}
+	ReadIterator(rocksdb::ColumnFamilyHandle* cf, uint64_t index, rocksdb::DB* db, const KeyRange& range)
+	  : index(index), inUse(true), creationTime(now()), keyRange(range) {
+		auto options = getReadOptions();
+		beginSlice = std::shared_ptr<rocksdb::Slice>(new rocksdb::Slice(toSlice(keyRange.begin)));
+		options.iterate_lower_bound = beginSlice.get();
+		endSlice = std::shared_ptr<rocksdb::Slice>(new rocksdb::Slice(toSlice(keyRange.end)));
+		options.iterate_upper_bound = endSlice.get();
+		iter = std::shared_ptr<rocksdb::Iterator>(db->NewIterator(options, cf));
+	}
+};
+
+/*
+ReadIteratorPool: Collection of iterators. Reuses iterators on non-concurrent multiple read operations,
+instead of creating and deleting for every read.
+
+Read: IteratorPool provides an unused iterator if exists or creates and gives a new iterator.
+Returns back the iterator after the read is done.
+
+Write: Iterators in the pool are deleted, forcing new iterator creation on next reads. The iterators
+which are currently used by the reads can continue using the iterator as it is a shared_ptr. Once
+the read is processed, shared_ptr goes out of scope and gets deleted. Eventually the iterator object
+gets deleted as the ref count becomes 0.
+*/
+class ReadIteratorPool {
+public:
+	ReadIteratorPool(rocksdb::DB* db, rocksdb::ColumnFamilyHandle* cf, const std::string& path)
+	  : db(db), cf(cf), index(0), iteratorsReuseCount(0) {
+		ASSERT(db);
+		ASSERT(cf);
+		TraceEvent(SevVerbose, "ShardedRocksReadIteratorPool")
+		    .detail("Path", path)
+		    .detail("KnobRocksDBReadRangeReuseIterators", SERVER_KNOBS->ROCKSDB_READ_RANGE_REUSE_ITERATORS)
+		    .detail("KnobRocksDBPrefixLen", SERVER_KNOBS->ROCKSDB_PREFIX_LEN);
+	}
+
+	// Called on every db commit.
+	void update() {
+		if (SERVER_KNOBS->ROCKSDB_READ_RANGE_REUSE_ITERATORS) {
+			std::lock_guard<std::mutex> lock(mutex);
+			iteratorsMap.clear();
+		}
+	}
+
+	// Called on every read operation.
+	ReadIterator getIterator(const KeyRange& range) {
+		// Shared iterators are not bounded.
+		if (SERVER_KNOBS->ROCKSDB_READ_RANGE_REUSE_ITERATORS) {
+			std::lock_guard<std::mutex> lock(mutex);
+			for (it = iteratorsMap.begin(); it != iteratorsMap.end(); it++) {
+				if (!it->second.inUse) {
+					it->second.inUse = true;
+					iteratorsReuseCount++;
+					return it->second;
+				}
+			}
+			index++;
+			ReadIterator iter(cf, index, db);
+			iteratorsMap.insert({ index, iter });
+			return iter;
+		} else {
+			index++;
+			ReadIterator iter(cf, index, db, range);
+			return iter;
+		}
+	}
+
+	// Called on every read operation, after the keys are collected.
+	void returnIterator(ReadIterator& iter) {
+		if (SERVER_KNOBS->ROCKSDB_READ_RANGE_REUSE_ITERATORS) {
+			std::lock_guard<std::mutex> lock(mutex);
+			it = iteratorsMap.find(iter.index);
+			// iterator found: put the iterator back to the pool(inUse=false).
+			// iterator not found: update would have removed the iterator from pool, so nothing to do.
+			if (it != iteratorsMap.end()) {
+				ASSERT(it->second.inUse);
+				it->second.inUse = false;
+			}
+		}
+	}
+
+	// Called for every ROCKSDB_READ_RANGE_ITERATOR_REFRESH_TIME seconds in a loop.
+	void refreshIterators() {
+		std::lock_guard<std::mutex> lock(mutex);
+		it = iteratorsMap.begin();
+		while (it != iteratorsMap.end()) {
+			if (now() - it->second.creationTime > SERVER_KNOBS->ROCKSDB_READ_RANGE_ITERATOR_REFRESH_TIME) {
+				it = iteratorsMap.erase(it);
+			} else {
+				it++;
+			}
+		}
+	}
+
+	uint64_t numReadIteratorsCreated() { return index; }
+
+	uint64_t numTimesReadIteratorsReused() { return iteratorsReuseCount; }
+
+private:
+	std::unordered_map<int, ReadIterator> iteratorsMap;
+	std::unordered_map<int, ReadIterator>::iterator it;
+	rocksdb::DB* db;
+	rocksdb::ColumnFamilyHandle* cf;
+	std::mutex mutex;
+	// incrementing counter for every new iterator creation, to uniquely identify the iterator in returnIterator().
+	uint64_t index;
+	uint64_t iteratorsReuseCount;
+};
+
+ACTOR Future<Void> flowLockLogger(const FlowLock* readLock, const FlowLock* fetchLock) {
+	loop {
+		wait(delay(SERVER_KNOBS->ROCKSDB_METRICS_DELAY));
+		TraceEvent e("ShardedRocksDBFlowLock");
+		e.detail("ReadAvailable", readLock->available());
+		e.detail("ReadActivePermits", readLock->activePermits());
+		e.detail("ReadWaiters", readLock->waiters());
+		e.detail("FetchAvailable", fetchLock->available());
+		e.detail("FetchActivePermits", fetchLock->activePermits());
+		e.detail("FetchWaiters", fetchLock->waiters());
+	}
+}
+
+// DataShard represents a key range (logical shard) in FDB. A DataShard is assigned to a specific physical shard.
+struct DataShard {
+	DataShard(KeyRange range, PhysicalShard* physicalShard) : range(range), physicalShard(physicalShard) {}
+
+	bool initialized() const;
+
+	KeyRange range;
+	PhysicalShard* physicalShard;
+};
+
+// PhysicalShard represent a collection of logical shards. A PhysicalShard could have one or more DataShards. A
+// PhysicalShard is stored as a column family in rocksdb. Each PhysicalShard has its own iterator pool.
+struct PhysicalShard {
+	PhysicalShard(rocksdb::DB* db, std::string id, const rocksdb::ColumnFamilyOptions& options)
+	  : db(db), id(id), cfOptions(options), isInitialized(false) {}
+	PhysicalShard(rocksdb::DB* db, std::string id, rocksdb::ColumnFamilyHandle* handle)
+	  : db(db), id(id), cf(handle), isInitialized(true) {
+		ASSERT(cf);
+		readIterPool = std::make_shared<ReadIteratorPool>(db, cf, id);
+	}
+
+	rocksdb::Status init() {
+		if (cf) {
+			return rocksdb::Status::OK();
+		}
+		auto status = db->CreateColumnFamily(cfOptions, id, &cf);
+		if (!status.ok()) {
+			logRocksDBError(status, "AddCF");
+			return status;
+		}
+		readIterPool = std::make_shared<ReadIteratorPool>(db, cf, id);
+		this->isInitialized.store(true);
+		return status;
+	}
+
+	// Restore from the checkpoint.
+	rocksdb::Status restore(const CheckpointMetaData& checkpoint) {
+		const CheckpointFormat format = checkpoint.getFormat();
+		rocksdb::Status status;
+		if (format == DataMoveRocksCF) {
+			rocksdb::ExportImportFilesMetaData metaData = getMetaData(checkpoint);
+			if (metaData.files.empty()) {
+				TraceEvent(SevInfo, "RocksDBRestoreEmptyShard")
+				    .detail("Shard", id)
+				    .detail("CheckpointID", checkpoint.checkpointID);
+				status = db->CreateColumnFamily(getCFOptions(), id, &cf);
+			} else {
+				TraceEvent(SevInfo, "RocksDBRestoreCF");
+				rocksdb::ImportColumnFamilyOptions importOptions;
+				importOptions.move_files = SERVER_KNOBS->ROCKSDB_IMPORT_MOVE_FILES;
+				status = db->CreateColumnFamilyWithImport(getCFOptions(), id, importOptions, metaData, &cf);
+				TraceEvent(SevInfo, "RocksDBRestoreCFEnd").detail("Status", status.ToString());
+			}
+		} else if (format == RocksDBKeyValues) {
+			ASSERT(cf != nullptr);
+			std::vector<std::string> sstFiles;
+			const RocksDBCheckpointKeyValues rcp = getRocksKeyValuesCheckpoint(checkpoint);
+			for (const auto& file : rcp.fetchedFiles) {
+				if (file.path != emptySstFilePath) {
+					sstFiles.push_back(file.path);
+				}
+			}
+
+			TraceEvent(SevDebug, "RocksDBRestoreFiles")
+			    .detail("Shard", id)
+			    .detail("CheckpointID", checkpoint.checkpointID)
+			    .detail("Files", describe(sstFiles));
+
+			if (!sstFiles.empty()) {
+				rocksdb::IngestExternalFileOptions ingestOptions;
+				ingestOptions.move_files = SERVER_KNOBS->ROCKSDB_IMPORT_MOVE_FILES;
+				ingestOptions.verify_checksums_before_ingest = SERVER_KNOBS->ROCKSDB_VERIFY_CHECKSUM_BEFORE_RESTORE;
+				status = db->IngestExternalFile(cf, sstFiles, ingestOptions);
+			} else {
+				TraceEvent(SevWarn, "RocksDBServeRestoreEmptyRange")
+				    .detail("Shard", id)
+				    .detail("RocksKeyValuesCheckpoint", rcp.toString())
+				    .detail("Checkpoint", checkpoint.toString());
+			}
+			TraceEvent(SevInfo, "RocksDBServeRestoreFiles")
+			    .detail("Shard", id)
+			    .detail("CFName", cf->GetName())
+			    .detail("Checkpoint", checkpoint.toString())
+			    .detail("RestoredFiles", describe(sstFiles));
+		} else {
+			throw not_implemented();
+		}
+
+		TraceEvent(status.ok() ? SevInfo : SevWarnAlways, "PhysicalShardRestoreEnd")
+		    .detail("Status", status.ToString())
+		    .detail("Shard", id)
+		    .detail("CFName", cf == nullptr ? "" : cf->GetName())
+		    .detail("Checkpoint", checkpoint.toString());
+		if (status.ok()) {
+			if (!this->isInitialized) {
+				readIterPool = std::make_shared<ReadIteratorPool>(db, cf, id);
+				this->isInitialized.store(true);
+			} else {
+				refreshReadIteratorPool();
+			}
+		}
+
+		return status;
+	}
+
+	bool initialized() { return this->isInitialized.load(); }
+
+	void refreshReadIteratorPool() {
+		ASSERT(this->readIterPool != nullptr);
+		this->readIterPool->refreshIterators();
+	}
+
+	std::vector<KeyRange> getAllRanges() const {
+		std::vector<KeyRange> res;
+		for (const auto& [key, shard] : dataShards) {
+			if (shard != nullptr) {
+				res.push_back(shard->range);
+			}
+		}
+		return res;
+	}
+
+	std::string toString() {
+		std::string ret = "[ID]: " + this->id + ", [CF]: ";
+		if (initialized()) {
+			ret += std::to_string(this->cf->GetID());
+		} else {
+			ret += "Not initialized";
+		}
+		return ret;
+	}
+
+	~PhysicalShard() {
+		logShardEvent(id, ShardOp::CLOSE);
+		isInitialized.store(false);
+		readIterPool.reset();
+
+		// Deleting default column family is not allowed.
+		if (id == DEFAULT_CF_NAME) {
+			return;
+		}
+
+		if (deletePending) {
+			auto s = db->DropColumnFamily(cf);
+			if (!s.ok()) {
+				logRocksDBError(s, "DestroyShard");
+				logShardEvent(id, ShardOp::DESTROY, SevError, s.ToString());
+				return;
+			}
+		}
+		auto s = db->DestroyColumnFamilyHandle(cf);
+		if (!s.ok()) {
+			logRocksDBError(s, "DestroyCFHandle");
+			logShardEvent(id, ShardOp::DESTROY, SevError, s.ToString());
+			return;
+		}
+		logShardEvent(id, ShardOp::DESTROY);
+	}
+
+	rocksdb::DB* db;
+	std::string id;
+	rocksdb::ColumnFamilyOptions cfOptions;
+	rocksdb::ColumnFamilyHandle* cf = nullptr;
+	std::unordered_map<std::string, std::unique_ptr<DataShard>> dataShards;
+	std::shared_ptr<ReadIteratorPool> readIterPool;
+	bool deletePending = false;
+	std::atomic<bool> isInitialized;
+	double deleteTimeSec;
+};
+
+int readRangeInDb(PhysicalShard* shard, const KeyRangeRef range, int rowLimit, int byteLimit, RangeResult* result) {
+	if (rowLimit == 0 || byteLimit == 0) {
+		return 0;
+	}
+
+	int accumulatedBytes = 0;
+	rocksdb::Status s;
+
+	// When using a prefix extractor, ensure that keys are returned in order even if they cross
+	// a prefix boundary.
+	if (rowLimit >= 0) {
+		ReadIterator readIter = shard->readIterPool->getIterator(range);
+		auto cursor = readIter.iter;
+		cursor->Seek(toSlice(range.begin));
+		while (cursor->Valid() && toStringRef(cursor->key()) < range.end) {
+			KeyValueRef kv(toStringRef(cursor->key()), toStringRef(cursor->value()));
+			accumulatedBytes += sizeof(KeyValueRef) + kv.expectedSize();
+			result->push_back_deep(result->arena(), kv);
+			// Calling `cursor->Next()` is potentially expensive, so short-circut here just in case.
+			if (result->size() >= rowLimit || accumulatedBytes >= byteLimit) {
+				break;
+			}
+			cursor->Next();
+		}
+		s = cursor->status();
+		shard->readIterPool->returnIterator(readIter);
+	} else {
+		ReadIterator readIter = shard->readIterPool->getIterator(range);
+		auto cursor = readIter.iter;
+		cursor->SeekForPrev(toSlice(range.end));
+		if (cursor->Valid() && toStringRef(cursor->key()) == range.end) {
+			cursor->Prev();
+		}
+		while (cursor->Valid() && toStringRef(cursor->key()) >= range.begin) {
+			KeyValueRef kv(toStringRef(cursor->key()), toStringRef(cursor->value()));
+			accumulatedBytes += sizeof(KeyValueRef) + kv.expectedSize();
+			result->push_back_deep(result->arena(), kv);
+			// Calling `cursor->Prev()` is potentially expensive, so short-circut here just in case.
+			if (result->size() >= -rowLimit || accumulatedBytes >= byteLimit) {
+				break;
+			}
+			cursor->Prev();
+		}
+		s = cursor->status();
+		shard->readIterPool->returnIterator(readIter);
+	}
+
+	if (!s.ok()) {
+		logRocksDBError(s, "ReadRange");
+		// The data writen to the arena is not erased, which will leave RangeResult in a dirty state. The RangeResult
+		// should never be returned to user.
+		return -1;
+	}
+	return accumulatedBytes;
+}
+
+// Manages physical shards and maintains logical shard mapping.
+class ShardManager {
+public:
+	ShardManager(std::string path, UID logId, const rocksdb::Options& options)
+	  : path(path), logId(logId), dbOptions(options), dataShardMap(nullptr, specialKeys.end) {}
+
+	ACTOR static Future<Void> shardMetricsLogger(std::shared_ptr<ShardedRocksDBState> rState,
+	                                             Future<Void> openFuture,
+	                                             ShardManager* shardManager) {
+		state std::unordered_map<std::string, std::shared_ptr<PhysicalShard>>* physicalShards =
+		    shardManager->getAllShards();
+
+		try {
+			wait(openFuture);
+			loop {
+				wait(delay(SERVER_KNOBS->ROCKSDB_METRICS_DELAY));
+				if (rState->closing) {
+					break;
+				}
+				TraceEvent(SevInfo, "ShardedRocksKVSPhysialShardMetrics")
+				    .detail("NumActiveShards", shardManager->numActiveShards())
+				    .detail("TotalPhysicalShards", shardManager->numPhysicalShards());
+
+				for (auto& [id, shard] : *physicalShards) {
+					if (!shard->initialized()) {
+						continue;
+					}
+					std::string propValue = "";
+					ASSERT(shard->db->GetProperty(shard->cf, rocksdb::DB::Properties::kCFStats, &propValue));
+					TraceEvent(SevInfo, "PhysicalShardCFStats")
+					    .detail("PhysicalShardID", id)
+					    .detail("Detail", propValue);
+
+					// Get compression ratio for each level.
+					rocksdb::ColumnFamilyMetaData cfMetadata;
+					shard->db->GetColumnFamilyMetaData(shard->cf, &cfMetadata);
+					TraceEvent e(SevInfo, "PhysicalShardLevelStats");
+					e.detail("PhysicalShardID", id);
+					std::string levelProp;
+					for (auto it = cfMetadata.levels.begin(); it != cfMetadata.levels.end(); ++it) {
+						std::string propValue = "";
+						ASSERT(shard->db->GetProperty(shard->cf,
+						                              rocksdb::DB::Properties::kCompressionRatioAtLevelPrefix +
+						                                  std::to_string(it->level),
+						                              &propValue));
+						e.detail("Level" + std::to_string(it->level), std::to_string(it->size) + " " + propValue);
+					}
+				}
+			}
+		} catch (Error& e) {
+			if (e.code() != error_code_actor_cancelled) {
+				TraceEvent(SevError, "ShardedRocksShardMetricsLoggerError").errorUnsuppressed(e);
+			}
+		}
+		return Void();
+	}
+
+	rocksdb::Status init() {
+		// Open instance.
+		TraceEvent(SevInfo, "ShardedRocksShardManagerInitBegin", this->logId).detail("DataPath", path);
+		std::vector<std::string> columnFamilies;
+		rocksdb::Status status = rocksdb::DB::ListColumnFamilies(dbOptions, path, &columnFamilies);
+
+		std::vector<rocksdb::ColumnFamilyDescriptor> descriptors;
+		bool foundMetadata = false;
+		for (const auto& name : columnFamilies) {
+			if (name == METADATA_SHARD_ID) {
+				foundMetadata = true;
+			}
+			descriptors.push_back(rocksdb::ColumnFamilyDescriptor{ name, rocksdb::ColumnFamilyOptions(dbOptions) });
+		}
+
+		ASSERT(foundMetadata || descriptors.size() == 0);
+
+		// Add default column family if it's a newly opened database.
+		if (descriptors.size() == 0) {
+			descriptors.push_back(
+			    rocksdb::ColumnFamilyDescriptor{ DEFAULT_CF_NAME, rocksdb::ColumnFamilyOptions(dbOptions) });
+		}
+
+		std::vector<rocksdb::ColumnFamilyHandle*> handles;
+		status = rocksdb::DB::Open(dbOptions, path, descriptors, &handles, &db);
+		if (!status.ok()) {
+			logRocksDBError(status, "Open");
+			return status;
+		}
+
+		if (foundMetadata) {
+			TraceEvent(SevInfo, "ShardedRocksInitLoadPhysicalShards", this->logId)
+			    .detail("PhysicalShardCount", handles.size());
+
+			std::shared_ptr<PhysicalShard> metadataShard = nullptr;
+			for (auto handle : handles) {
+				auto shard = std::make_shared<PhysicalShard>(db, handle->GetName(), handle);
+				if (shard->id == METADATA_SHARD_ID) {
+					metadataShard = shard;
+				}
+				physicalShards[shard->id] = shard;
+				columnFamilyMap[handle->GetID()] = handle;
+				TraceEvent(SevVerbose, "ShardedRocksInitPhysicalShard", this->logId)
+				    .detail("PhysicalShardID", shard->id);
+			}
+
+			std::set<std::string> unusedShards(columnFamilies.begin(), columnFamilies.end());
+			unusedShards.erase(METADATA_SHARD_ID);
+			unusedShards.erase(DEFAULT_CF_NAME);
+
+			KeyRange keyRange = prefixRange(shardMappingPrefix);
+			while (true) {
+				RangeResult metadata;
+				const int bytes = readRangeInDb(metadataShard.get(),
+				                                keyRange,
+				                                std::max(2, SERVER_KNOBS->ROCKSDB_READ_RANGE_ROW_LIMIT),
+				                                UINT16_MAX,
+				                                &metadata);
+				if (bytes <= 0) {
+					break;
+				}
+
+				ASSERT_GT(metadata.size(), 0);
+				for (int i = 0; i < metadata.size() - 1; ++i) {
+					const std::string name = metadata[i].value.toString();
+					KeyRangeRef range(metadata[i].key.removePrefix(shardMappingPrefix),
+					                  metadata[i + 1].key.removePrefix(shardMappingPrefix));
+					TraceEvent(SevVerbose, "DecodeShardMapping", this->logId)
+					    .detail("Range", range)
+					    .detail("Name", name);
+
+					// Empty name indicates the shard doesn't belong to the SS/KVS.
+					if (name.empty()) {
+						continue;
+					}
+
+					auto it = physicalShards.find(name);
+					// Raise error if physical shard is missing.
+					if (it == physicalShards.end()) {
+						TraceEvent(SevError, "ShardedRocksDB", this->logId).detail("MissingShard", name);
+						return rocksdb::Status::NotFound();
+					}
+
+					std::unique_ptr<DataShard> dataShard = std::make_unique<DataShard>(range, it->second.get());
+					dataShardMap.insert(range, dataShard.get());
+					it->second->dataShards[range.begin.toString()] = std::move(dataShard);
+					activePhysicalShardIds.emplace(name);
+					unusedShards.erase(name);
+				}
+
+				if (metadata.back().key.removePrefix(shardMappingPrefix) == specialKeys.end) {
+					TraceEvent(SevVerbose, "ShardedRocksLoadShardMappingEnd", this->logId);
+					break;
+				}
+
+				// Read from the current last key since the shard begining with it hasn't been processed.
+				if (metadata.size() == 1 && metadata.back().value.toString().empty()) {
+					// Should not happen, just being paranoid.
+					keyRange = KeyRangeRef(keyAfter(metadata.back().key), keyRange.end);
+				} else {
+					keyRange = KeyRangeRef(metadata.back().key, keyRange.end);
+				}
+			}
+
+			for (const auto& name : unusedShards) {
+				TraceEvent(SevDebug, "UnusedShardName", logId).detail("Name", name);
+				auto it = physicalShards.find(name);
+				ASSERT(it != physicalShards.end());
+				auto shard = it->second;
+				if (shard->dataShards.size() == 0) {
+					shard->deleteTimeSec = now();
+					pendingDeletionShards.push_back(name);
+				}
+			}
+			if (unusedShards.size() > 0) {
+				TraceEvent("ShardedRocksDB", logId).detail("CleanUpUnusedShards", unusedShards.size());
+			}
+		} else {
+			// DB is opened with default shard.
+			ASSERT(handles.size() == 1);
+
+			// Add SpecialKeys range. This range should not be modified.
+			std::shared_ptr<PhysicalShard> defaultShard =
+			    std::make_shared<PhysicalShard>(db, DEFAULT_CF_NAME, handles[0]);
+			columnFamilyMap[defaultShard->cf->GetID()] = defaultShard->cf;
+			std::unique_ptr<DataShard> dataShard = std::make_unique<DataShard>(specialKeys, defaultShard.get());
+			dataShardMap.insert(specialKeys, dataShard.get());
+			defaultShard->dataShards[specialKeys.begin.toString()] = std::move(dataShard);
+			physicalShards[defaultShard->id] = defaultShard;
+
+			// Create metadata shard.
+			auto metadataShard =
+			    std::make_shared<PhysicalShard>(db, METADATA_SHARD_ID, rocksdb::ColumnFamilyOptions(dbOptions));
+			metadataShard->init();
+			columnFamilyMap[metadataShard->cf->GetID()] = metadataShard->cf;
+			physicalShards[METADATA_SHARD_ID] = metadataShard;
+
+			// Write special key range metadata.
+			writeBatch = std::make_unique<rocksdb::WriteBatch>();
+			dirtyShards = std::make_unique<std::set<PhysicalShard*>>();
+			persistRangeMapping(specialKeys, true);
+			rocksdb::WriteOptions options;
+			status = db->Write(options, writeBatch.get());
+			if (!status.ok()) {
+				return status;
+			}
+			metadataShard->readIterPool->update();
+			TraceEvent(SevInfo, "ShardedRocksInitializeMetaDataShard", this->logId)
+			    .detail("MetadataShardCF", metadataShard->cf->GetID());
+		}
+
+		writeBatch = std::make_unique<rocksdb::WriteBatch>();
+		dirtyShards = std::make_unique<std::set<PhysicalShard*>>();
+
+		TraceEvent(SevInfo, "ShardedRocksShardManagerInitEnd", this->logId).detail("DataPath", path);
+		return status;
+	}
+
+	DataShard* getDataShard(KeyRef key) {
+		DataShard* shard = dataShardMap[key];
+		ASSERT(shard == nullptr || shard->range.contains(key));
+		return shard;
+	}
+
+	PhysicalShard* getSpecialKeysShard() {
+		auto it = physicalShards.find(DEFAULT_CF_NAME);
+		ASSERT(it != physicalShards.end());
+		return it->second.get();
+	}
+
+	PhysicalShard* getMetaDataShard() {
+		auto it = physicalShards.find(METADATA_SHARD_ID);
+		ASSERT(it != physicalShards.end());
+		return it->second.get();
+	}
+
+	// Returns the physical shard that hosting all `ranges`; Returns nullptr if such a physical shard does not exists.
+	PhysicalShard* getPhysicalShardForAllRanges(std::vector<KeyRange> ranges) {
+		PhysicalShard* result = nullptr;
+		for (const auto& range : ranges) {
+			auto rangeIterator = this->dataShardMap.intersectingRanges(range);
+			for (auto it = rangeIterator.begin(); it != rangeIterator.end(); ++it) {
+				if (it.value() == nullptr) {
+					TraceEvent(SevWarn, "ShardedRocksDBRangeNotInKVS", logId).detail("Range", range);
+					return nullptr;
+				} else {
+					ASSERT(it.value()->physicalShard != nullptr);
+					PhysicalShard* ps = it.value()->physicalShard;
+					if (result == nullptr) {
+						result = ps;
+					} else if (ps != result) {
+						TraceEvent(SevWarn, "ShardedRocksDBRangeOnMultipleShards", logId)
+						    .detail("Ranges", describe(ranges))
+						    .detail("PhysicalShard", result->id)
+						    .detail("SecondPhysicalShard", ps->id);
+						return nullptr;
+					}
+				}
+			}
+		}
+
+		return result;
+	}
+
+	std::vector<DataShard*> getDataShardsByRange(KeyRangeRef range) {
+		std::vector<DataShard*> result;
+		auto rangeIterator = dataShardMap.intersectingRanges(range);
+
+		for (auto it = rangeIterator.begin(); it != rangeIterator.end(); ++it) {
+			if (it.value() == nullptr) {
+				TraceEvent(SevVerbose, "ShardedRocksDB")
+				    .detail("Info", "ShardNotFound")
+				    .detail("BeginKey", range.begin)
+				    .detail("EndKey", range.end);
+				continue;
+			}
+			result.push_back(it.value());
+		}
+		return result;
+	}
+
+	PhysicalShard* addRange(KeyRange range, std::string id) {
+		TraceEvent(SevVerbose, "ShardedRocksAddRangeBegin", this->logId)
+		    .detail("Range", range)
+		    .detail("PhysicalShardID", id);
+
+		// Newly added range should not overlap with any existing range.
+		auto ranges = dataShardMap.intersectingRanges(range);
+
+		for (auto it = ranges.begin(); it != ranges.end(); ++it) {
+			if (it.value()) {
+				if (it.value()->physicalShard->id == id) {
+					TraceEvent(SevError, "ShardedRocksDBAddRange")
+					    .detail("ErrorType", "RangeAlreadyExist")
+					    .detail("IntersectingRange", it->range())
+					    .detail("DataShardRange", it->value()->range)
+					    .detail("ExpectedShardId", id)
+					    .detail("PhysicalShardID", it->value()->physicalShard->toString());
+				} else {
+					TraceEvent(SevError, "ShardedRocksDBAddRange")
+					    .detail("ErrorType", "ConflictingRange")
+					    .detail("IntersectingRange", it->range())
+					    .detail("DataShardRange", it->value()->range)
+					    .detail("ExpectedShardId", id)
+					    .detail("PhysicalShardID", it->value()->physicalShard->toString());
+				}
+				return nullptr;
+			}
+		}
+
+		auto [it, inserted] = physicalShards.emplace(
+		    id, std::make_shared<PhysicalShard>(db, id, rocksdb::ColumnFamilyOptions(dbOptions)));
+		std::shared_ptr<PhysicalShard>& shard = it->second;
+
+		activePhysicalShardIds.emplace(id);
+
+		auto dataShard = std::make_unique<DataShard>(range, shard.get());
+		dataShardMap.insert(range, dataShard.get());
+		shard->dataShards[range.begin.toString()] = std::move(dataShard);
+
+		validate();
+
+		TraceEvent(SevInfo, "ShardedRocksDBRangeAdded", this->logId)
+		    .detail("Range", range)
+		    .detail("PhysicalShardID", id);
+
+		return shard.get();
+	}
+
+	std::vector<std::string> removeRange(KeyRange range) {
+		TraceEvent(SevInfo, "ShardedRocksRemoveRangeBegin", this->logId).detail("Range", range);
+		std::vector<std::string> shardIds;
+
+		std::vector<DataShard*> newShards;
+		auto ranges = dataShardMap.intersectingRanges(range);
+
+		for (auto it = ranges.begin(); it != ranges.end(); ++it) {
+			if (!it.value()) {
+				TraceEvent(SevDebug, "ShardedRocksDB")
+				    .detail("Info", "RemoveNonExistentRange")
+				    .detail("BeginKey", range.begin)
+				    .detail("EndKey", range.end);
+				continue;
+			}
+			auto existingShard = it.value()->physicalShard;
+			auto shardRange = it.range();
+
+			if (SERVER_KNOBS->ROCKSDB_EMPTY_RANGE_CHECK && existingShard->initialized()) {
+				// Enable consistency validation.
+				RangeResult rangeResult;
+				auto bytesRead = readRangeInDb(existingShard, range, 1, UINT16_MAX, &rangeResult);
+				if (bytesRead > 0) {
+					TraceEvent(SevError, "ShardedRocksDBRangeNotEmpty")
+					    .detail("PhysicalShard", existingShard->toString())
+					    .detail("Range", range)
+					    .detail("DataShardRange", shardRange);
+				}
+
+				// Force clear range.
+				writeBatch->DeleteRange(it.value()->physicalShard->cf, toSlice(range.begin), toSlice(range.end));
+				dirtyShards->insert(it.value()->physicalShard);
+			}
+
+			TraceEvent(SevDebug, "ShardedRocksRemoveRange")
+			    .detail("Range", range)
+			    .detail("IntersectingRange", shardRange)
+			    .detail("DataShardRange", it.value()->range)
+			    .detail("PhysicalShard", existingShard->toString());
+
+			ASSERT(it.value()->range == shardRange); // Ranges should be consistent.
+
+			if (range.contains(shardRange)) {
+				existingShard->dataShards.erase(shardRange.begin.toString());
+				TraceEvent(SevInfo, "ShardedRocksRemovedRange")
+				    .detail("Range", range)
+				    .detail("RemovedRange", shardRange)
+				    .detail("PhysicalShard", existingShard->toString());
+				if (existingShard->dataShards.size() == 0) {
+					TraceEvent(SevDebug, "ShardedRocksDB").detail("EmptyShardId", existingShard->id);
+					shardIds.push_back(existingShard->id);
+					existingShard->deleteTimeSec = now();
+					pendingDeletionShards.push_back(existingShard->id);
+					activePhysicalShardIds.erase(existingShard->id);
+				}
+				continue;
+			}
+
+			// Range modification could result in more than one segments. Remove the original segment key here.
+			existingShard->dataShards.erase(shardRange.begin.toString());
+			if (shardRange.begin < range.begin) {
+				auto dataShard =
+				    std::make_unique<DataShard>(KeyRange(KeyRangeRef(shardRange.begin, range.begin)), existingShard);
+
+				newShards.push_back(dataShard.get());
+				const std::string msg = "Shrink shard from " + Traceable<KeyRangeRef>::toString(shardRange) + " to " +
+				                        Traceable<KeyRangeRef>::toString(dataShard->range);
+
+				existingShard->dataShards[shardRange.begin.toString()] = std::move(dataShard);
+				logShardEvent(existingShard->id, shardRange, ShardOp::MODIFY_RANGE, SevInfo, msg);
+			}
+
+			if (shardRange.end > range.end) {
+				auto dataShard =
+				    std::make_unique<DataShard>(KeyRange(KeyRangeRef(range.end, shardRange.end)), existingShard);
+
+				newShards.push_back(dataShard.get());
+				const std::string msg = "Shrink shard from " + Traceable<KeyRangeRef>::toString(shardRange) + " to " +
+				                        Traceable<KeyRangeRef>::toString(dataShard->range);
+
+				existingShard->dataShards[range.end.toString()] = std::move(dataShard);
+				logShardEvent(existingShard->id, shardRange, ShardOp::MODIFY_RANGE, SevInfo, msg);
+			}
+		}
+
+		dataShardMap.insert(range, nullptr);
+		for (DataShard* shard : newShards) {
+			dataShardMap.insert(shard->range, shard);
+		}
+
+		validate();
+
+		TraceEvent(SevInfo, "ShardedRocksRemoveRangeEnd", this->logId).detail("Range", range);
+
+		return shardIds;
+	}
+
+	std::vector<std::shared_ptr<PhysicalShard>> getPendingDeletionShards(double cleanUpDelay) {
+		std::vector<std::shared_ptr<PhysicalShard>> emptyShards;
+		double currentTime = now();
+
+		TraceEvent(SevInfo, "ShardedRocksDB", logId)
+		    .detail("PendingDeletionShardQueueSize", pendingDeletionShards.size());
+		while (!pendingDeletionShards.empty()) {
+			const auto id = pendingDeletionShards.front();
+			auto it = physicalShards.find(id);
+			if (it == physicalShards.end() || it->second->dataShards.size() != 0) {
+				pendingDeletionShards.pop_front();
+				continue;
+			}
+			if (currentTime - it->second->deleteTimeSec > cleanUpDelay) {
+				pendingDeletionShards.pop_front();
+				emptyShards.push_back(it->second);
+				physicalShards.erase(id);
+			} else {
+				break;
+			}
+		}
+		return emptyShards;
+	}
+
+	void put(KeyRef key, ValueRef value) {
+		auto it = dataShardMap.rangeContaining(key);
+		if (!it.value()) {
+			TraceEvent(SevError, "ShardedRocksDB").detail("Error", "write to non-exist shard").detail("WriteKey", key);
+			return;
+		}
+		TraceEvent(SevVerbose, "ShardedRocksShardManagerPut", this->logId)
+		    .detail("WriteKey", key)
+		    .detail("Value", value)
+		    .detail("MapRange", it.range())
+		    .detail("ShardRange", it.value()->range);
+		ASSERT(it.value()->range == (KeyRangeRef)it.range());
+		ASSERT(writeBatch != nullptr);
+		ASSERT(dirtyShards != nullptr);
+		writeBatch->Put(it.value()->physicalShard->cf, toSlice(key), toSlice(value));
+		dirtyShards->insert(it.value()->physicalShard);
+		TraceEvent(SevVerbose, "ShardedRocksShardManagerPutEnd", this->logId)
+		    .detail("WriteKey", key)
+		    .detail("Value", value);
+	}
+
+	void clear(KeyRef key) {
+		auto it = dataShardMap.rangeContaining(key);
+		if (!it.value()) {
+			return;
+		}
+		writeBatch->Delete(it.value()->physicalShard->cf, toSlice(key));
+		dirtyShards->insert(it.value()->physicalShard);
+	}
+
+	void clearRange(KeyRangeRef range) {
+		auto rangeIterator = dataShardMap.intersectingRanges(range);
+
+		for (auto it = rangeIterator.begin(); it != rangeIterator.end(); ++it) {
+			if (it.value() == nullptr) {
+				TraceEvent(SevDebug, "ShardedRocksDB").detail("ClearNonExistentRange", it.range());
+				continue;
+			}
+			writeBatch->DeleteRange(it.value()->physicalShard->cf, toSlice(range.begin), toSlice(range.end));
+			dirtyShards->insert(it.value()->physicalShard);
+		}
+	}
+
+	void populateRangeMappingMutations(rocksdb::WriteBatch* writeBatch, KeyRangeRef range, bool isAdd) {
+		TraceEvent(SevDebug, "ShardedRocksDB", this->logId)
+		    .detail("Info", "RangeToPersist")
+		    .detail("BeginKey", range.begin)
+		    .detail("EndKey", range.end);
+		PhysicalShard* metadataShard = getMetaDataShard();
+
+		writeBatch->DeleteRange(metadataShard->cf,
+		                        getShardMappingKey(range.begin, shardMappingPrefix),
+		                        getShardMappingKey(range.end, shardMappingPrefix));
+
+		KeyRef lastKey = range.end;
+		if (isAdd) {
+			auto ranges = dataShardMap.intersectingRanges(range);
+			for (auto it = ranges.begin(); it != ranges.end(); ++it) {
+				if (it.value()) {
+					ASSERT(it.range() == it.value()->range);
+					// Non-empty range.
+					writeBatch->Put(metadataShard->cf,
+					                getShardMappingKey(it.range().begin, shardMappingPrefix),
+					                it.value()->physicalShard->id);
+					TraceEvent(SevDebug, "ShardedRocksDB", this->logId)
+					    .detail("Action", "PersistRangeMapping")
+					    .detail("BeginKey", it.range().begin)
+					    .detail("EndKey", it.range().end)
+					    .detail("PhysicalShardID", it.value()->physicalShard->id);
+
+				} else {
+					// Empty range.
+					writeBatch->Put(metadataShard->cf, getShardMappingKey(it.range().begin, shardMappingPrefix), "");
+					TraceEvent(SevDebug, "ShardedRocksDB", this->logId)
+					    .detail("Action", "PersistRangeMapping")
+					    .detail("BeginKey", it.range().begin)
+					    .detail("EndKey", it.range().end)
+					    .detail("PhysicalShardID", "None");
+				}
+				lastKey = it.range().end;
+			}
+		} else {
+			writeBatch->Put(metadataShard->cf, getShardMappingKey(range.begin, shardMappingPrefix), "");
+			TraceEvent(SevDebug, "ShardedRocksDB", this->logId)
+			    .detail("Action", "PersistRangeMapping")
+			    .detail("RemoveRange", "True")
+			    .detail("BeginKey", range.begin)
+			    .detail("EndKey", range.end);
+		}
+
+		DataShard* nextShard = nullptr;
+		if (lastKey <= allKeys.end) {
+			nextShard = dataShardMap.rangeContaining(lastKey).value();
+		}
+		writeBatch->Put(metadataShard->cf,
+		                getShardMappingKey(lastKey, shardMappingPrefix),
+		                nextShard == nullptr ? "" : nextShard->physicalShard->id);
+		TraceEvent(SevDebug, "ShardedRocksDB", this->logId)
+		    .detail("Action", "PersistRangeMappingEnd")
+		    .detail("NextShardKey", lastKey)
+		    .detail("Value", nextShard == nullptr ? "" : nextShard->physicalShard->id);
+	}
+
+	void persistRangeMapping(KeyRangeRef range, bool isAdd) {
+		populateRangeMappingMutations(writeBatch.get(), range, isAdd);
+		dirtyShards->insert(getMetaDataShard());
+	}
+
+	std::unique_ptr<rocksdb::WriteBatch> getWriteBatch() {
+		std::unique_ptr<rocksdb::WriteBatch> existingWriteBatch = std::move(writeBatch);
+		writeBatch = std::make_unique<rocksdb::WriteBatch>();
+		return existingWriteBatch;
+	}
+
+	std::unique_ptr<std::set<PhysicalShard*>> getDirtyShards() {
+		std::unique_ptr<std::set<PhysicalShard*>> existingShards = std::move(dirtyShards);
+		dirtyShards = std::make_unique<std::set<PhysicalShard*>>();
+		return existingShards;
+	}
+
+	void closeAllShards() {
+		columnFamilyMap.clear();
+		physicalShards.clear();
+		// Close DB.
+		auto s = db->Close();
+		if (!s.ok()) {
+			logRocksDBError(s, "Close");
+			return;
+		}
+		TraceEvent("ShardedRocksDB", this->logId).detail("Info", "DBClosed");
+	}
+
+	void destroyAllShards() {
+		columnFamilyMap.clear();
+		for (auto& [_, shard] : physicalShards) {
+			shard->deletePending = true;
+		}
+		physicalShards.clear();
+		// Close DB.
+		auto s = db->Close();
+		if (!s.ok()) {
+			logRocksDBError(s, "Close");
+			return;
+		}
+		s = rocksdb::DestroyDB(path, getOptions());
+		if (!s.ok()) {
+			logRocksDBError(s, "DestroyDB");
+		}
+		TraceEvent("ShardedRocksDB", this->logId).detail("Info", "DBDestroyed");
+	}
+
+	rocksdb::DB* getDb() const { return db; }
+
+	std::unordered_map<std::string, std::shared_ptr<PhysicalShard>>* getAllShards() { return &physicalShards; }
+
+	std::unordered_map<uint32_t, rocksdb::ColumnFamilyHandle*>* getColumnFamilyMap() { return &columnFamilyMap; }
+
+	std::vector<rocksdb::ColumnFamilyHandle*> getColumnFamilies() {
+		std::vector<rocksdb::ColumnFamilyHandle*> res;
+		for (auto& [id, cf] : columnFamilyMap) {
+			res.push_back(cf);
+		}
+		return res;
+	}
+
+	size_t numPhysicalShards() const { return physicalShards.size(); };
+
+	size_t numActiveShards() const { return activePhysicalShardIds.size(); };
+
+	std::vector<std::pair<KeyRange, std::string>> getDataMapping() {
+		std::vector<std::pair<KeyRange, std::string>> dataMap;
+		for (auto it : dataShardMap.ranges()) {
+			if (!it.value()) {
+				continue;
+			}
+			dataMap.push_back(std::make_pair(it.range(), it.value()->physicalShard->id));
+		}
+		return dataMap;
+	}
+
+	CoalescedKeyRangeMap<std::string> getExistingRanges() {
+		CoalescedKeyRangeMap<std::string> existingRanges;
+		existingRanges.insert(allKeys, "");
+		for (auto it : dataShardMap.intersectingRanges(allKeys)) {
+			if (!it.value()) {
+				continue;
+			}
+
+			if (it.value()->physicalShard->id == "kvs-metadata") {
+				continue;
+			}
+
+			existingRanges.insert(it.range(), it.value()->physicalShard->id);
+		}
+		return existingRanges;
+	}
+
+	void validate() {
+		TraceEvent(SevVerbose, "ShardedRocksValidateShardManager", this->logId);
+		for (auto s = dataShardMap.ranges().begin(); s != dataShardMap.ranges().end(); ++s) {
+			TraceEvent e(SevVerbose, "ShardedRocksValidateDataShardMap", this->logId);
+			e.detail("Range", s->range());
+			const DataShard* shard = s->value();
+			e.detail("ShardAddress", reinterpret_cast<std::uintptr_t>(shard));
+			if (shard != nullptr) {
+				e.detail("PhysicalShard", shard->physicalShard->id);
+			} else {
+				e.detail("Shard", "Empty");
+			}
+			if (shard != nullptr) {
+				if (shard->range != static_cast<KeyRangeRef>(s->range())) {
+					TraceEvent(SevWarn, "ShardRangeMismatch").detail("Range", s->range());
+				}
+
+				ASSERT(shard->range == static_cast<KeyRangeRef>(s->range()));
+				ASSERT(shard->physicalShard != nullptr);
+				auto it = shard->physicalShard->dataShards.find(shard->range.begin.toString());
+				ASSERT(it != shard->physicalShard->dataShards.end());
+				ASSERT(it->second.get() == shard);
+			}
+		}
+	}
+
+private:
+	const std::string path;
+	const UID logId;
+	rocksdb::Options dbOptions;
+	rocksdb::DB* db = nullptr;
+	std::unordered_map<std::string, std::shared_ptr<PhysicalShard>> physicalShards;
+	std::unordered_set<std::string> activePhysicalShardIds;
+	// Stores mapping between cf id and cf handle, used during compaction.
+	std::unordered_map<uint32_t, rocksdb::ColumnFamilyHandle*> columnFamilyMap;
+	std::unique_ptr<rocksdb::WriteBatch> writeBatch;
+	std::unique_ptr<std::set<PhysicalShard*>> dirtyShards;
+	KeyRangeMap<DataShard*> dataShardMap;
+	std::deque<std::string> pendingDeletionShards;
+};
+
+class RocksDBMetrics {
+public:
+	RocksDBMetrics(UID debugID, std::shared_ptr<rocksdb::Statistics> stats);
+	void logStats(rocksdb::DB* db);
+	// PerfContext
+	void resetPerfContext();
+	void collectPerfContext(int index);
+	void logPerfContext(bool ignoreZeroMetric);
+	// For Readers
+	Reference<Histogram> getReadRangeLatencyHistogram(int index);
+	Reference<Histogram> getReadValueLatencyHistogram(int index);
+	Reference<Histogram> getReadPrefixLatencyHistogram(int index);
+	Reference<Histogram> getReadRangeActionHistogram(int index);
+	Reference<Histogram> getReadValueActionHistogram(int index);
+	Reference<Histogram> getReadPrefixActionHistogram(int index);
+	Reference<Histogram> getReadRangeQueueWaitHistogram(int index);
+	Reference<Histogram> getReadValueQueueWaitHistogram(int index);
+	Reference<Histogram> getReadPrefixQueueWaitHistogram(int index);
+	Reference<Histogram> getReadRangeNewIteratorHistogram(int index);
+	Reference<Histogram> getReadValueGetHistogram(int index);
+	Reference<Histogram> getReadPrefixGetHistogram(int index);
+	// For Writer
+	Reference<Histogram> getCommitLatencyHistogram();
+	Reference<Histogram> getCommitActionHistogram();
+	Reference<Histogram> getCommitQueueWaitHistogram();
+	Reference<Histogram> getWriteHistogram();
+	Reference<Histogram> getDeleteCompactRangeHistogram();
+	// Stat for Memory Usage
+	void logMemUsage(rocksdb::DB* db);
+
+private:
+	const UID debugID;
+	// Global Statistic Input to RocksDB DB instance
+	std::shared_ptr<rocksdb::Statistics> stats;
+	// Statistic Output from RocksDB
+	std::vector<std::tuple<const char*, uint32_t, uint64_t>> tickerStats;
+	std::vector<std::pair<const char*, std::string>> intPropertyStats;
+	// Iterator Pool Stats
+	std::unordered_map<std::string, uint64_t> readIteratorPoolStats;
+	// PerfContext
+	std::vector<std::tuple<const char*, int, std::vector<uint64_t>>> perfContextMetrics;
+	// Readers Histogram
+	std::vector<Reference<Histogram>> readRangeLatencyHistograms;
+	std::vector<Reference<Histogram>> readValueLatencyHistograms;
+	std::vector<Reference<Histogram>> readPrefixLatencyHistograms;
+	std::vector<Reference<Histogram>> readRangeActionHistograms;
+	std::vector<Reference<Histogram>> readValueActionHistograms;
+	std::vector<Reference<Histogram>> readPrefixActionHistograms;
+	std::vector<Reference<Histogram>> readRangeQueueWaitHistograms;
+	std::vector<Reference<Histogram>> readValueQueueWaitHistograms;
+	std::vector<Reference<Histogram>> readPrefixQueueWaitHistograms;
+	std::vector<Reference<Histogram>> readRangeNewIteratorHistograms; // Zhe: haven't used?
+	std::vector<Reference<Histogram>> readValueGetHistograms;
+	std::vector<Reference<Histogram>> readPrefixGetHistograms;
+	// Writer Histogram
+	Reference<Histogram> commitLatencyHistogram;
+	Reference<Histogram> commitActionHistogram;
+	Reference<Histogram> commitQueueWaitHistogram;
+	Reference<Histogram> writeHistogram;
+	Reference<Histogram> deleteCompactRangeHistogram;
+
+	uint64_t getRocksdbPerfcontextMetric(int metric);
+};
+
+// We have 4 readers and 1 writer. Following input index denotes the
+// id assigned to the reader thread when creating it
+Reference<Histogram> RocksDBMetrics::getReadRangeLatencyHistogram(int index) {
+	return readRangeLatencyHistograms[index];
+}
+Reference<Histogram> RocksDBMetrics::getReadValueLatencyHistogram(int index) {
+	return readValueLatencyHistograms[index];
+}
+Reference<Histogram> RocksDBMetrics::getReadPrefixLatencyHistogram(int index) {
+	return readPrefixLatencyHistograms[index];
+}
+Reference<Histogram> RocksDBMetrics::getReadRangeActionHistogram(int index) {
+	return readRangeActionHistograms[index];
+}
+Reference<Histogram> RocksDBMetrics::getReadValueActionHistogram(int index) {
+	return readValueActionHistograms[index];
+}
+Reference<Histogram> RocksDBMetrics::getReadPrefixActionHistogram(int index) {
+	return readPrefixActionHistograms[index];
+}
+Reference<Histogram> RocksDBMetrics::getReadRangeQueueWaitHistogram(int index) {
+	return readRangeQueueWaitHistograms[index];
+}
+Reference<Histogram> RocksDBMetrics::getReadValueQueueWaitHistogram(int index) {
+	return readValueQueueWaitHistograms[index];
+}
+Reference<Histogram> RocksDBMetrics::getReadPrefixQueueWaitHistogram(int index) {
+	return readPrefixQueueWaitHistograms[index];
+}
+Reference<Histogram> RocksDBMetrics::getReadRangeNewIteratorHistogram(int index) {
+	return readRangeNewIteratorHistograms[index];
+}
+Reference<Histogram> RocksDBMetrics::getReadValueGetHistogram(int index) {
+	return readValueGetHistograms[index];
+}
+Reference<Histogram> RocksDBMetrics::getReadPrefixGetHistogram(int index) {
+	return readPrefixGetHistograms[index];
+}
+Reference<Histogram> RocksDBMetrics::getCommitLatencyHistogram() {
+	return commitLatencyHistogram;
+}
+Reference<Histogram> RocksDBMetrics::getCommitActionHistogram() {
+	return commitActionHistogram;
+}
+Reference<Histogram> RocksDBMetrics::getCommitQueueWaitHistogram() {
+	return commitQueueWaitHistogram;
+}
+Reference<Histogram> RocksDBMetrics::getWriteHistogram() {
+	return writeHistogram;
+}
+Reference<Histogram> RocksDBMetrics::getDeleteCompactRangeHistogram() {
+	return deleteCompactRangeHistogram;
+}
+
+RocksDBMetrics::RocksDBMetrics(UID debugID, std::shared_ptr<rocksdb::Statistics> stats)
+  : debugID(debugID), stats(stats) {
+	tickerStats = {
+		{ "StallMicros", rocksdb::STALL_MICROS, 0 },
+		{ "BytesRead", rocksdb::BYTES_READ, 0 },
+		{ "IterBytesRead", rocksdb::ITER_BYTES_READ, 0 },
+		{ "BytesWritten", rocksdb::BYTES_WRITTEN, 0 },
+		{ "BlockCacheMisses", rocksdb::BLOCK_CACHE_MISS, 0 },
+		{ "BlockCacheHits", rocksdb::BLOCK_CACHE_HIT, 0 },
+		{ "BloomFilterUseful", rocksdb::BLOOM_FILTER_USEFUL, 0 },
+		{ "BloomFilterFullPositive", rocksdb::BLOOM_FILTER_FULL_POSITIVE, 0 },
+		{ "BloomFilterTruePositive", rocksdb::BLOOM_FILTER_FULL_TRUE_POSITIVE, 0 },
+		// Deprecated in RocksDB 8.0
+		// { "BloomFilterMicros", rocksdb::BLOOM_FILTER_MICROS, 0 },
+		{ "MemtableHit", rocksdb::MEMTABLE_HIT, 0 },
+		{ "MemtableMiss", rocksdb::MEMTABLE_MISS, 0 },
+		{ "GetHitL0", rocksdb::GET_HIT_L0, 0 },
+		{ "GetHitL1", rocksdb::GET_HIT_L1, 0 },
+		{ "GetHitL2AndUp", rocksdb::GET_HIT_L2_AND_UP, 0 },
+		{ "CountKeysWritten", rocksdb::NUMBER_KEYS_WRITTEN, 0 },
+		{ "CountKeysRead", rocksdb::NUMBER_KEYS_READ, 0 },
+		{ "CountDBSeek", rocksdb::NUMBER_DB_SEEK, 0 },
+		{ "CountDBNext", rocksdb::NUMBER_DB_NEXT, 0 },
+		{ "CountDBPrev", rocksdb::NUMBER_DB_PREV, 0 },
+		{ "BloomFilterPrefixChecked", rocksdb::BLOOM_FILTER_PREFIX_CHECKED, 0 },
+		{ "BloomFilterPrefixUseful", rocksdb::BLOOM_FILTER_PREFIX_USEFUL, 0 },
+		// Deprecated in RocksDB 8.0
+		// { "BlockCacheCompressedMiss", rocksdb::BLOCK_CACHE_COMPRESSED_MISS, 0 },
+		// { "BlockCacheCompressedHit", rocksdb::BLOCK_CACHE_COMPRESSED_HIT, 0 },
+		{ "CountWalFileSyncs", rocksdb::WAL_FILE_SYNCED, 0 },
+		{ "CountWalFileBytes", rocksdb::WAL_FILE_BYTES, 0 },
+		{ "CompactReadBytes", rocksdb::COMPACT_READ_BYTES, 0 },
+		{ "CompactWriteBytes", rocksdb::COMPACT_WRITE_BYTES, 0 },
+		{ "FlushWriteBytes", rocksdb::FLUSH_WRITE_BYTES, 0 },
+		{ "CountBlocksCompressed", rocksdb::NUMBER_BLOCK_COMPRESSED, 0 },
+		{ "CountBlocksDecompressed", rocksdb::NUMBER_BLOCK_DECOMPRESSED, 0 },
+		{ "RowCacheHit", rocksdb::ROW_CACHE_HIT, 0 },
+		{ "RowCacheMiss", rocksdb::ROW_CACHE_MISS, 0 },
+		{ "CountIterSkippedKeys", rocksdb::NUMBER_ITER_SKIP, 0 },
+
+	};
+
+	intPropertyStats = {
+		{ "NumImmutableMemtables", rocksdb::DB::Properties::kNumImmutableMemTable },
+		{ "NumImmutableMemtablesFlushed", rocksdb::DB::Properties::kNumImmutableMemTableFlushed },
+		{ "IsMemtableFlushPending", rocksdb::DB::Properties::kMemTableFlushPending },
+		{ "NumRunningFlushes", rocksdb::DB::Properties::kNumRunningFlushes },
+		{ "IsCompactionPending", rocksdb::DB::Properties::kCompactionPending },
+		{ "NumRunningCompactions", rocksdb::DB::Properties::kNumRunningCompactions },
+		{ "CumulativeBackgroundErrors", rocksdb::DB::Properties::kBackgroundErrors },
+		{ "CurrentSizeActiveMemtable", rocksdb::DB::Properties::kCurSizeActiveMemTable },
+		{ "AllMemtablesBytes", rocksdb::DB::Properties::kCurSizeAllMemTables },
+		{ "ActiveMemtableBytes", rocksdb::DB::Properties::kSizeAllMemTables },
+		{ "CountEntriesActiveMemtable", rocksdb::DB::Properties::kNumEntriesActiveMemTable },
+		{ "CountEntriesImmutMemtables", rocksdb::DB::Properties::kNumEntriesImmMemTables },
+		{ "CountDeletesActiveMemtable", rocksdb::DB::Properties::kNumDeletesActiveMemTable },
+		{ "CountDeletesImmutMemtables", rocksdb::DB::Properties::kNumDeletesImmMemTables },
+		{ "EstimatedCountKeys", rocksdb::DB::Properties::kEstimateNumKeys },
+		{ "EstimateSstReaderBytes", rocksdb::DB::Properties::kEstimateTableReadersMem },
+		{ "CountActiveSnapshots", rocksdb::DB::Properties::kNumSnapshots },
+		{ "OldestSnapshotTime", rocksdb::DB::Properties::kOldestSnapshotTime },
+		{ "CountLiveVersions", rocksdb::DB::Properties::kNumLiveVersions },
+		{ "EstimateLiveDataSize", rocksdb::DB::Properties::kEstimateLiveDataSize },
+		{ "BaseLevel", rocksdb::DB::Properties::kBaseLevel },
+		{ "EstPendCompactBytes", rocksdb::DB::Properties::kEstimatePendingCompactionBytes },
+		{ "BlockCacheUsage", rocksdb::DB::Properties::kBlockCacheUsage },
+		{ "BlockCachePinnedUsage", rocksdb::DB::Properties::kBlockCachePinnedUsage },
+		{ "LiveSstFilesSize", rocksdb::DB::Properties::kLiveSstFilesSize },
+	};
+
+	perfContextMetrics = {
+		{ "UserKeyComparisonCount", rocksdb_user_key_comparison_count, {} },
+		{ "BlockCacheHitCount", rocksdb_block_cache_hit_count, {} },
+		{ "BlockReadCount", rocksdb_block_read_count, {} },
+		{ "BlockReadByte", rocksdb_block_read_byte, {} },
+		{ "BlockReadTime", rocksdb_block_read_time, {} },
+		{ "BlockChecksumTime", rocksdb_block_checksum_time, {} },
+		{ "BlockDecompressTime", rocksdb_block_decompress_time, {} },
+		{ "GetReadBytes", rocksdb_get_read_bytes, {} },
+		{ "MultigetReadBytes", rocksdb_multiget_read_bytes, {} },
+		{ "IterReadBytes", rocksdb_iter_read_bytes, {} },
+		{ "InternalKeySkippedCount", rocksdb_internal_key_skipped_count, {} },
+		{ "InternalDeleteSkippedCount", rocksdb_internal_delete_skipped_count, {} },
+		{ "InternalRecentSkippedCount", rocksdb_internal_recent_skipped_count, {} },
+		{ "InternalMergeCount", rocksdb_internal_merge_count, {} },
+		{ "GetSnapshotTime", rocksdb_get_snapshot_time, {} },
+		{ "GetFromMemtableTime", rocksdb_get_from_memtable_time, {} },
+		{ "GetFromMemtableCount", rocksdb_get_from_memtable_count, {} },
+		{ "GetPostProcessTime", rocksdb_get_post_process_time, {} },
+		{ "GetFromOutputFilesTime", rocksdb_get_from_output_files_time, {} },
+		{ "SeekOnMemtableTime", rocksdb_seek_on_memtable_time, {} },
+		{ "SeekOnMemtableCount", rocksdb_seek_on_memtable_count, {} },
+		{ "NextOnMemtableCount", rocksdb_next_on_memtable_count, {} },
+		{ "PrevOnMemtableCount", rocksdb_prev_on_memtable_count, {} },
+		{ "SeekChildSeekTime", rocksdb_seek_child_seek_time, {} },
+		{ "SeekChildSeekCount", rocksdb_seek_child_seek_count, {} },
+		{ "SeekMinHeapTime", rocksdb_seek_min_heap_time, {} },
+		{ "SeekMaxHeapTime", rocksdb_seek_max_heap_time, {} },
+		{ "SeekInternalSeekTime", rocksdb_seek_internal_seek_time, {} },
+		{ "FindNextUserEntryTime", rocksdb_find_next_user_entry_time, {} },
+		{ "WriteWalTime", rocksdb_write_wal_time, {} },
+		{ "WriteMemtableTime", rocksdb_write_memtable_time, {} },
+		{ "WriteDelayTime", rocksdb_write_delay_time, {} },
+		{ "WritePreAndPostProcessTime", rocksdb_write_pre_and_post_process_time, {} },
+		{ "DbMutexLockNanos", rocksdb_db_mutex_lock_nanos, {} },
+		{ "DbConditionWaitNanos", rocksdb_db_condition_wait_nanos, {} },
+		{ "MergeOperatorTimeNanos", rocksdb_merge_operator_time_nanos, {} },
+		{ "ReadIndexBlockNanos", rocksdb_read_index_block_nanos, {} },
+		{ "ReadFilterBlockNanos", rocksdb_read_filter_block_nanos, {} },
+		{ "NewTableBlockIterNanos", rocksdb_new_table_block_iter_nanos, {} },
+		{ "NewTableIteratorNanos", rocksdb_new_table_iterator_nanos, {} },
+		{ "BlockSeekNanos", rocksdb_block_seek_nanos, {} },
+		{ "FindTableNanos", rocksdb_find_table_nanos, {} },
+		{ "BloomMemtableHitCount", rocksdb_bloom_memtable_hit_count, {} },
+		{ "BloomMemtableMissCount", rocksdb_bloom_memtable_miss_count, {} },
+		{ "BloomSstHitCount", rocksdb_bloom_sst_hit_count, {} },
+		{ "BloomSstMissCount", rocksdb_bloom_sst_miss_count, {} },
+		{ "KeyLockWaitTime", rocksdb_key_lock_wait_time, {} },
+		{ "KeyLockWaitCount", rocksdb_key_lock_wait_count, {} },
+		{ "EnvNewSequentialFileNanos", rocksdb_env_new_sequential_file_nanos, {} },
+		{ "EnvNewRandomAccessFileNanos", rocksdb_env_new_random_access_file_nanos, {} },
+		{ "EnvNewWritableFileNanos", rocksdb_env_new_writable_file_nanos, {} },
+		{ "EnvReuseWritableFileNanos", rocksdb_env_reuse_writable_file_nanos, {} },
+		{ "EnvNewRandomRwFileNanos", rocksdb_env_new_random_rw_file_nanos, {} },
+		{ "EnvNewDirectoryNanos", rocksdb_env_new_directory_nanos, {} },
+		{ "EnvFileExistsNanos", rocksdb_env_file_exists_nanos, {} },
+		{ "EnvGetChildrenNanos", rocksdb_env_get_children_nanos, {} },
+		{ "EnvGetChildrenFileAttributesNanos", rocksdb_env_get_children_file_attributes_nanos, {} },
+		{ "EnvDeleteFileNanos", rocksdb_env_delete_file_nanos, {} },
+		{ "EnvCreateDirNanos", rocksdb_env_create_dir_nanos, {} },
+		{ "EnvCreateDirIfMissingNanos", rocksdb_env_create_dir_if_missing_nanos, {} },
+		{ "EnvDeleteDirNanos", rocksdb_env_delete_dir_nanos, {} },
+		{ "EnvGetFileSizeNanos", rocksdb_env_get_file_size_nanos, {} },
+		{ "EnvGetFileModificationTimeNanos", rocksdb_env_get_file_modification_time_nanos, {} },
+		{ "EnvRenameFileNanos", rocksdb_env_rename_file_nanos, {} },
+		{ "EnvLinkFileNanos", rocksdb_env_link_file_nanos, {} },
+		{ "EnvLockFileNanos", rocksdb_env_lock_file_nanos, {} },
+		{ "EnvUnlockFileNanos", rocksdb_env_unlock_file_nanos, {} },
+		{ "EnvNewLoggerNanos", rocksdb_env_new_logger_nanos, {} },
+	};
+	for (auto& [name, metric, vals] : perfContextMetrics) { // readers, then writer
+		for (int i = 0; i < SERVER_KNOBS->ROCKSDB_READ_PARALLELISM; i++) {
+			vals.push_back(0); // add reader
+		}
+		vals.push_back(0); // add writer
+	}
+	for (int i = 0; i < SERVER_KNOBS->ROCKSDB_READ_PARALLELISM; i++) {
+		readRangeLatencyHistograms.push_back(Histogram::getHistogram(
+		    ROCKSDBSTORAGE_HISTOGRAM_GROUP, ROCKSDB_READRANGE_LATENCY_HISTOGRAM, Histogram::Unit::milliseconds));
+		readValueLatencyHistograms.push_back(Histogram::getHistogram(
+		    ROCKSDBSTORAGE_HISTOGRAM_GROUP, ROCKSDB_READVALUE_LATENCY_HISTOGRAM, Histogram::Unit::milliseconds));
+		readPrefixLatencyHistograms.push_back(Histogram::getHistogram(
+		    ROCKSDBSTORAGE_HISTOGRAM_GROUP, ROCKSDB_READPREFIX_LATENCY_HISTOGRAM, Histogram::Unit::milliseconds));
+		readRangeActionHistograms.push_back(Histogram::getHistogram(
+		    ROCKSDBSTORAGE_HISTOGRAM_GROUP, ROCKSDB_READRANGE_ACTION_HISTOGRAM, Histogram::Unit::milliseconds));
+		readValueActionHistograms.push_back(Histogram::getHistogram(
+		    ROCKSDBSTORAGE_HISTOGRAM_GROUP, ROCKSDB_READVALUE_ACTION_HISTOGRAM, Histogram::Unit::milliseconds));
+		readPrefixActionHistograms.push_back(Histogram::getHistogram(
+		    ROCKSDBSTORAGE_HISTOGRAM_GROUP, ROCKSDB_READPREFIX_ACTION_HISTOGRAM, Histogram::Unit::milliseconds));
+		readRangeQueueWaitHistograms.push_back(Histogram::getHistogram(
+		    ROCKSDBSTORAGE_HISTOGRAM_GROUP, ROCKSDB_READRANGE_QUEUEWAIT_HISTOGRAM, Histogram::Unit::milliseconds));
+		readValueQueueWaitHistograms.push_back(Histogram::getHistogram(
+		    ROCKSDBSTORAGE_HISTOGRAM_GROUP, ROCKSDB_READVALUE_QUEUEWAIT_HISTOGRAM, Histogram::Unit::milliseconds));
+		readPrefixQueueWaitHistograms.push_back(Histogram::getHistogram(
+		    ROCKSDBSTORAGE_HISTOGRAM_GROUP, ROCKSDB_READPREFIX_QUEUEWAIT_HISTOGRAM, Histogram::Unit::milliseconds));
+		readRangeNewIteratorHistograms.push_back(Histogram::getHistogram(
+		    ROCKSDBSTORAGE_HISTOGRAM_GROUP, ROCKSDB_READRANGE_NEWITERATOR_HISTOGRAM, Histogram::Unit::milliseconds));
+		readValueGetHistograms.push_back(Histogram::getHistogram(
+		    ROCKSDBSTORAGE_HISTOGRAM_GROUP, ROCKSDB_READVALUE_GET_HISTOGRAM, Histogram::Unit::milliseconds));
+		readPrefixGetHistograms.push_back(Histogram::getHistogram(
+		    ROCKSDBSTORAGE_HISTOGRAM_GROUP, ROCKSDB_READPREFIX_GET_HISTOGRAM, Histogram::Unit::milliseconds));
+	}
+	commitLatencyHistogram = Histogram::getHistogram(
+	    ROCKSDBSTORAGE_HISTOGRAM_GROUP, ROCKSDB_COMMIT_LATENCY_HISTOGRAM, Histogram::Unit::milliseconds);
+	commitActionHistogram = Histogram::getHistogram(
+	    ROCKSDBSTORAGE_HISTOGRAM_GROUP, ROCKSDB_COMMIT_ACTION_HISTOGRAM, Histogram::Unit::milliseconds);
+	commitQueueWaitHistogram = Histogram::getHistogram(
+	    ROCKSDBSTORAGE_HISTOGRAM_GROUP, ROCKSDB_COMMIT_QUEUEWAIT_HISTOGRAM, Histogram::Unit::milliseconds);
+	writeHistogram =
+	    Histogram::getHistogram(ROCKSDBSTORAGE_HISTOGRAM_GROUP, ROCKSDB_WRITE_HISTOGRAM, Histogram::Unit::milliseconds);
+	deleteCompactRangeHistogram = Histogram::getHistogram(
+	    ROCKSDBSTORAGE_HISTOGRAM_GROUP, ROCKSDB_DELETE_COMPACTRANGE_HISTOGRAM, Histogram::Unit::milliseconds);
+}
+
+void RocksDBMetrics::logStats(rocksdb::DB* db) {
+	TraceEvent e(SevInfo, "ShardedRocksDBMetrics", debugID);
+	uint64_t stat;
+	for (auto& [name, ticker, cumulation] : tickerStats) {
+		stat = stats->getTickerCount(ticker);
+		e.detail(name, stat - cumulation);
+		cumulation = stat;
+	}
+	for (auto& [name, property] : intPropertyStats) {
+		stat = 0;
+		ASSERT(db->GetAggregatedIntProperty(property, &stat));
+		e.detail(name, stat);
+	}
+}
+
+void RocksDBMetrics::logMemUsage(rocksdb::DB* db) {
+	TraceEvent e(SevInfo, "ShardedRocksDBMemMetrics", debugID);
+	uint64_t stat;
+	ASSERT(db != nullptr);
+	ASSERT(db->GetAggregatedIntProperty(rocksdb::DB::Properties::kBlockCacheUsage, &stat));
+	e.detail("BlockCacheUsage", stat);
+	ASSERT(db->GetAggregatedIntProperty(rocksdb::DB::Properties::kEstimateTableReadersMem, &stat));
+	e.detail("EstimateSstReaderBytes", stat);
+	ASSERT(db->GetAggregatedIntProperty(rocksdb::DB::Properties::kCurSizeAllMemTables, &stat));
+	e.detail("AllMemtablesBytes", stat);
+	ASSERT(db->GetAggregatedIntProperty(rocksdb::DB::Properties::kBlockCachePinnedUsage, &stat));
+	e.detail("BlockCachePinnedUsage", stat);
+}
+
+void RocksDBMetrics::resetPerfContext() {
+	rocksdb::SetPerfLevel(rocksdb::PerfLevel::kEnableCount);
+	rocksdb::get_perf_context()->Reset();
+}
+
+void RocksDBMetrics::collectPerfContext(int index) {
+	for (auto& [name, metric, vals] : perfContextMetrics) {
+		vals[index] = getRocksdbPerfcontextMetric(metric);
+	}
+}
+
+void RocksDBMetrics::logPerfContext(bool ignoreZeroMetric) {
+	TraceEvent e(SevInfo, "ShardedRocksDBPerfContextMetrics", debugID);
+	e.setMaxEventLength(20000);
+	for (auto& [name, metric, vals] : perfContextMetrics) {
+		uint64_t s = 0;
+		for (auto& v : vals) {
+			s = s + v;
+		}
+		if (ignoreZeroMetric && s == 0)
+			continue;
+		e.detail(name, s);
+	}
+}
+
+uint64_t RocksDBMetrics::getRocksdbPerfcontextMetric(int metric) {
+	switch (metric) {
+	case rocksdb_user_key_comparison_count:
+		return rocksdb::get_perf_context()->user_key_comparison_count;
+	case rocksdb_block_cache_hit_count:
+		return rocksdb::get_perf_context()->block_cache_hit_count;
+	case rocksdb_block_read_count:
+		return rocksdb::get_perf_context()->block_read_count;
+	case rocksdb_block_read_byte:
+		return rocksdb::get_perf_context()->block_read_byte;
+	case rocksdb_block_read_time:
+		return rocksdb::get_perf_context()->block_read_time;
+	case rocksdb_block_checksum_time:
+		return rocksdb::get_perf_context()->block_checksum_time;
+	case rocksdb_block_decompress_time:
+		return rocksdb::get_perf_context()->block_decompress_time;
+	case rocksdb_get_read_bytes:
+		return rocksdb::get_perf_context()->get_read_bytes;
+	case rocksdb_multiget_read_bytes:
+		return rocksdb::get_perf_context()->multiget_read_bytes;
+	case rocksdb_iter_read_bytes:
+		return rocksdb::get_perf_context()->iter_read_bytes;
+	case rocksdb_internal_key_skipped_count:
+		return rocksdb::get_perf_context()->internal_key_skipped_count;
+	case rocksdb_internal_delete_skipped_count:
+		return rocksdb::get_perf_context()->internal_delete_skipped_count;
+	case rocksdb_internal_recent_skipped_count:
+		return rocksdb::get_perf_context()->internal_recent_skipped_count;
+	case rocksdb_internal_merge_count:
+		return rocksdb::get_perf_context()->internal_merge_count;
+	case rocksdb_get_snapshot_time:
+		return rocksdb::get_perf_context()->get_snapshot_time;
+	case rocksdb_get_from_memtable_time:
+		return rocksdb::get_perf_context()->get_from_memtable_time;
+	case rocksdb_get_from_memtable_count:
+		return rocksdb::get_perf_context()->get_from_memtable_count;
+	case rocksdb_get_post_process_time:
+		return rocksdb::get_perf_context()->get_post_process_time;
+	case rocksdb_get_from_output_files_time:
+		return rocksdb::get_perf_context()->get_from_output_files_time;
+	case rocksdb_seek_on_memtable_time:
+		return rocksdb::get_perf_context()->seek_on_memtable_time;
+	case rocksdb_seek_on_memtable_count:
+		return rocksdb::get_perf_context()->seek_on_memtable_count;
+	case rocksdb_next_on_memtable_count:
+		return rocksdb::get_perf_context()->next_on_memtable_count;
+	case rocksdb_prev_on_memtable_count:
+		return rocksdb::get_perf_context()->prev_on_memtable_count;
+	case rocksdb_seek_child_seek_time:
+		return rocksdb::get_perf_context()->seek_child_seek_time;
+	case rocksdb_seek_child_seek_count:
+		return rocksdb::get_perf_context()->seek_child_seek_count;
+	case rocksdb_seek_min_heap_time:
+		return rocksdb::get_perf_context()->seek_min_heap_time;
+	case rocksdb_seek_max_heap_time:
+		return rocksdb::get_perf_context()->seek_max_heap_time;
+	case rocksdb_seek_internal_seek_time:
+		return rocksdb::get_perf_context()->seek_internal_seek_time;
+	case rocksdb_find_next_user_entry_time:
+		return rocksdb::get_perf_context()->find_next_user_entry_time;
+	case rocksdb_write_wal_time:
+		return rocksdb::get_perf_context()->write_wal_time;
+	case rocksdb_write_memtable_time:
+		return rocksdb::get_perf_context()->write_memtable_time;
+	case rocksdb_write_delay_time:
+		return rocksdb::get_perf_context()->write_delay_time;
+	case rocksdb_write_pre_and_post_process_time:
+		return rocksdb::get_perf_context()->write_pre_and_post_process_time;
+	case rocksdb_db_mutex_lock_nanos:
+		return rocksdb::get_perf_context()->db_mutex_lock_nanos;
+	case rocksdb_db_condition_wait_nanos:
+		return rocksdb::get_perf_context()->db_condition_wait_nanos;
+	case rocksdb_merge_operator_time_nanos:
+		return rocksdb::get_perf_context()->merge_operator_time_nanos;
+	case rocksdb_read_index_block_nanos:
+		return rocksdb::get_perf_context()->read_index_block_nanos;
+	case rocksdb_read_filter_block_nanos:
+		return rocksdb::get_perf_context()->read_filter_block_nanos;
+	case rocksdb_new_table_block_iter_nanos:
+		return rocksdb::get_perf_context()->new_table_block_iter_nanos;
+	case rocksdb_new_table_iterator_nanos:
+		return rocksdb::get_perf_context()->new_table_iterator_nanos;
+	case rocksdb_block_seek_nanos:
+		return rocksdb::get_perf_context()->block_seek_nanos;
+	case rocksdb_find_table_nanos:
+		return rocksdb::get_perf_context()->find_table_nanos;
+	case rocksdb_bloom_memtable_hit_count:
+		return rocksdb::get_perf_context()->bloom_memtable_hit_count;
+	case rocksdb_bloom_memtable_miss_count:
+		return rocksdb::get_perf_context()->bloom_memtable_miss_count;
+	case rocksdb_bloom_sst_hit_count:
+		return rocksdb::get_perf_context()->bloom_sst_hit_count;
+	case rocksdb_bloom_sst_miss_count:
+		return rocksdb::get_perf_context()->bloom_sst_miss_count;
+	case rocksdb_key_lock_wait_time:
+		return rocksdb::get_perf_context()->key_lock_wait_time;
+	case rocksdb_key_lock_wait_count:
+		return rocksdb::get_perf_context()->key_lock_wait_count;
+	case rocksdb_env_new_sequential_file_nanos:
+		return rocksdb::get_perf_context()->env_new_sequential_file_nanos;
+	case rocksdb_env_new_random_access_file_nanos:
+		return rocksdb::get_perf_context()->env_new_random_access_file_nanos;
+	case rocksdb_env_new_writable_file_nanos:
+		return rocksdb::get_perf_context()->env_new_writable_file_nanos;
+	case rocksdb_env_reuse_writable_file_nanos:
+		return rocksdb::get_perf_context()->env_reuse_writable_file_nanos;
+	case rocksdb_env_new_random_rw_file_nanos:
+		return rocksdb::get_perf_context()->env_new_random_rw_file_nanos;
+	case rocksdb_env_new_directory_nanos:
+		return rocksdb::get_perf_context()->env_new_directory_nanos;
+	case rocksdb_env_file_exists_nanos:
+		return rocksdb::get_perf_context()->env_file_exists_nanos;
+	case rocksdb_env_get_children_nanos:
+		return rocksdb::get_perf_context()->env_get_children_nanos;
+	case rocksdb_env_get_children_file_attributes_nanos:
+		return rocksdb::get_perf_context()->env_get_children_file_attributes_nanos;
+	case rocksdb_env_delete_file_nanos:
+		return rocksdb::get_perf_context()->env_delete_file_nanos;
+	case rocksdb_env_create_dir_nanos:
+		return rocksdb::get_perf_context()->env_create_dir_nanos;
+	case rocksdb_env_create_dir_if_missing_nanos:
+		return rocksdb::get_perf_context()->env_create_dir_if_missing_nanos;
+	case rocksdb_env_delete_dir_nanos:
+		return rocksdb::get_perf_context()->env_delete_dir_nanos;
+	case rocksdb_env_get_file_size_nanos:
+		return rocksdb::get_perf_context()->env_get_file_size_nanos;
+	case rocksdb_env_get_file_modification_time_nanos:
+		return rocksdb::get_perf_context()->env_get_file_modification_time_nanos;
+	case rocksdb_env_rename_file_nanos:
+		return rocksdb::get_perf_context()->env_rename_file_nanos;
+	case rocksdb_env_link_file_nanos:
+		return rocksdb::get_perf_context()->env_link_file_nanos;
+	case rocksdb_env_lock_file_nanos:
+		return rocksdb::get_perf_context()->env_lock_file_nanos;
+	case rocksdb_env_unlock_file_nanos:
+		return rocksdb::get_perf_context()->env_unlock_file_nanos;
+	case rocksdb_env_new_logger_nanos:
+		return rocksdb::get_perf_context()->env_new_logger_nanos;
+	default:
+		break;
+	}
+	return 0;
+}
+
+ACTOR Future<Void> rocksDBAggregatedMetricsLogger(std::shared_ptr<ShardedRocksDBState> rState,
+                                                  Future<Void> openFuture,
+                                                  std::shared_ptr<RocksDBMetrics> rocksDBMetrics,
+                                                  ShardManager* shardManager) {
+	try {
+		wait(openFuture);
+		state rocksdb::DB* db = shardManager->getDb();
+		loop {
+			wait(delay(SERVER_KNOBS->ROCKSDB_METRICS_DELAY));
+			if (rState->closing) {
+				break;
+			}
+			rocksDBMetrics->logStats(db);
+			rocksDBMetrics->logMemUsage(db);
+			if (SERVER_KNOBS->ROCKSDB_PERFCONTEXT_SAMPLE_RATE != 0) {
+				rocksDBMetrics->logPerfContext(true);
+			}
+		}
+	} catch (Error& e) {
+		if (e.code() != error_code_actor_cancelled) {
+			TraceEvent(SevError, "ShardedRocksDBMetricsError").errorUnsuppressed(e);
+		}
+	}
+	return Void();
+}
+
+struct ShardedRocksDBKeyValueStore : IKeyValueStore {
+	using CF = rocksdb::ColumnFamilyHandle*;
+
+	ACTOR static Future<Void> refreshReadIteratorPools(
+	    std::shared_ptr<ShardedRocksDBState> rState,
+	    Future<Void> readyToStart,
+	    std::unordered_map<std::string, std::shared_ptr<PhysicalShard>>* physicalShards) {
+		state Reference<Histogram> histogram = Histogram::getHistogram(
+		    ROCKSDBSTORAGE_HISTOGRAM_GROUP, "TimeSpentRefreshIterators"_sr, Histogram::Unit::milliseconds);
+
+		if (SERVER_KNOBS->ROCKSDB_READ_RANGE_REUSE_ITERATORS) {
+			try {
+				wait(readyToStart);
+				loop {
+					wait(delay(SERVER_KNOBS->ROCKSDB_READ_RANGE_ITERATOR_REFRESH_TIME));
+					if (rState->closing) {
+						break;
+					}
+					double startTime = timer_monotonic();
+					for (auto& [_, shard] : *physicalShards) {
+						if (shard->initialized()) {
+							shard->refreshReadIteratorPool();
+						}
+					}
+					histogram->sample(timer_monotonic() - startTime);
+				}
+			} catch (Error& e) {
+				if (e.code() != error_code_actor_cancelled) {
+					TraceEvent(SevError, "RefreshReadIteratorPoolError").errorUnsuppressed(e);
+				}
+			}
+		}
+
+		return Void();
+	}
+
+	ACTOR static Future<Void> doRestore(ShardedRocksDBKeyValueStore* self,
+	                                    std::string shardId,
+	                                    std::vector<KeyRange> ranges,
+	                                    std::vector<CheckpointMetaData> checkpoints) {
+		for (const KeyRange& range : ranges) {
+			std::vector<DataShard*> shards = self->shardManager.getDataShardsByRange(range);
+			if (!shards.empty()) {
+				TraceEvent te(SevWarnAlways, "RestoreRangesNotEmpty", self->id);
+				te.detail("Range", range);
+				te.detail("RestoreShardID", shardId);
+				for (int i = 0; i < shards.size(); ++i) {
+					te.detail("DataShard-" + std::to_string(i), shards[i]->range);
+				}
+				te.log();
+				throw failed_to_restore_checkpoint();
+			}
+		}
+		for (const KeyRange& range : ranges) {
+			self->shardManager.addRange(range, shardId);
+		}
+		TraceEvent(SevDebug, "ShardedRocksRestoreAddRange", self->id)
+		    .detail("Ranges", describe(ranges))
+		    .detail("ShardID", shardId)
+		    .detail("Checkpoints", describe(checkpoints));
+		auto a = new Writer::RestoreAction(&self->shardManager, self->path, shardId, ranges, checkpoints);
+		auto res = a->done.getFuture();
+		self->writeThread->post(a);
+
+		try {
+			wait(res);
+		} catch (Error& e) {
+			for (const KeyRange& range : ranges) {
+				self->shardManager.removeRange(range);
+			}
+			throw;
+		}
+
+		return Void();
+	}
+
+	struct Writer : IThreadPoolReceiver {
+		const UID logId;
+		int threadIndex;
+		std::unordered_map<uint32_t, rocksdb::ColumnFamilyHandle*>* columnFamilyMap;
+		std::shared_ptr<RocksDBMetrics> rocksDBMetrics;
+		std::shared_ptr<rocksdb::RateLimiter> rateLimiter;
+		double sampleStartTime;
+
+		explicit Writer(UID logId,
+		                int threadIndex,
+		                std::unordered_map<uint32_t, rocksdb::ColumnFamilyHandle*>* columnFamilyMap,
+		                std::shared_ptr<RocksDBMetrics> rocksDBMetrics)
+		  : logId(logId), threadIndex(threadIndex), columnFamilyMap(columnFamilyMap), rocksDBMetrics(rocksDBMetrics),
+		    rateLimiter(SERVER_KNOBS->ROCKSDB_WRITE_RATE_LIMITER_BYTES_PER_SEC > 0
+		                    ? rocksdb::NewGenericRateLimiter(
+		                          SERVER_KNOBS->ROCKSDB_WRITE_RATE_LIMITER_BYTES_PER_SEC, // rate_bytes_per_sec
+		                          100 * 1000, // refill_period_us
+		                          10, // fairness
+		                          rocksdb::RateLimiter::Mode::kWritesOnly,
+		                          SERVER_KNOBS->ROCKSDB_WRITE_RATE_LIMITER_AUTO_TUNE)
+		                    : nullptr),
+		    sampleStartTime(now()) {}
+
+		~Writer() override {}
+
+		void init() override {}
+
+		void sample() {
+			if (SERVER_KNOBS->ROCKSDB_METRICS_SAMPLE_INTERVAL > 0 &&
+			    now() - sampleStartTime >= SERVER_KNOBS->ROCKSDB_METRICS_SAMPLE_INTERVAL) {
+				rocksDBMetrics->collectPerfContext(threadIndex);
+				rocksDBMetrics->resetPerfContext();
+				sampleStartTime = now();
+			}
+		}
+
+		struct OpenAction : TypedAction<Writer, OpenAction> {
+			ShardManager* shardManager;
+			ThreadReturnPromise<Void> done;
+			Optional<Future<Void>>& metrics;
+			const FlowLock* readLock;
+			const FlowLock* fetchLock;
+			std::shared_ptr<RocksDBErrorListener> errorListener;
+
+			OpenAction(ShardManager* shardManager,
+			           Optional<Future<Void>>& metrics,
+			           const FlowLock* readLock,
+			           const FlowLock* fetchLock,
+			           std::shared_ptr<RocksDBErrorListener> errorListener)
+			  : shardManager(shardManager), metrics(metrics), readLock(readLock), fetchLock(fetchLock),
+			    errorListener(errorListener) {}
+
+			double getTimeEstimate() const override { return SERVER_KNOBS->COMMIT_TIME_ESTIMATE; }
+		};
+
+		void action(OpenAction& a) {
+			auto status = a.shardManager->init();
+
+			if (!status.ok()) {
+				logRocksDBError(status, "Open");
+				a.done.sendError(statusToError(status));
+				return;
+			}
+
+			TraceEvent(SevInfo, "ShardedRocksDB").detail("Method", "Open");
+			a.done.send(Void());
+		}
+
+		struct AddShardAction : TypedAction<Writer, AddShardAction> {
+			PhysicalShard* shard;
+			ThreadReturnPromise<Void> done;
+
+			AddShardAction(PhysicalShard* shard) : shard(shard) { ASSERT(shard); }
+			double getTimeEstimate() const override { return SERVER_KNOBS->COMMIT_TIME_ESTIMATE; }
+		};
+
+		void action(AddShardAction& a) {
+			auto s = a.shard->init();
+			if (!s.ok()) {
+				a.done.sendError(statusToError(s));
+				return;
+			}
+			ASSERT(a.shard->cf);
+			(*columnFamilyMap)[a.shard->cf->GetID()] = a.shard->cf;
+			a.done.send(Void());
+		}
+
+		struct RemoveShardAction : TypedAction<Writer, RemoveShardAction> {
+			std::vector<std::shared_ptr<PhysicalShard>> shards;
+			ThreadReturnPromise<Void> done;
+
+			RemoveShardAction(std::vector<std::shared_ptr<PhysicalShard>>& shards) : shards(shards) {}
+			double getTimeEstimate() const override { return SERVER_KNOBS->COMMIT_TIME_ESTIMATE; }
+		};
+
+		void action(RemoveShardAction& a) {
+			for (auto& shard : a.shards) {
+				shard->deletePending = true;
+				columnFamilyMap->erase(shard->cf->GetID());
+			}
+			a.shards.clear();
+			a.done.send(Void());
+		}
+
+		struct CommitAction : TypedAction<Writer, CommitAction> {
+			rocksdb::DB* db;
+			std::unique_ptr<rocksdb::WriteBatch> writeBatch;
+			std::unique_ptr<std::set<PhysicalShard*>> dirtyShards;
+			const std::unordered_map<uint32_t, rocksdb::ColumnFamilyHandle*>* columnFamilyMap;
+			ThreadReturnPromise<Void> done;
+			double startTime;
+			bool getHistograms;
+			bool logShardMemUsage;
+			double getTimeEstimate() const override { return SERVER_KNOBS->COMMIT_TIME_ESTIMATE; }
+			CommitAction(rocksdb::DB* db,
+			             std::unique_ptr<rocksdb::WriteBatch> writeBatch,
+			             std::unique_ptr<std::set<PhysicalShard*>> dirtyShards,
+			             std::unordered_map<uint32_t, rocksdb::ColumnFamilyHandle*>* columnFamilyMap)
+			  : db(db), writeBatch(std::move(writeBatch)), dirtyShards(std::move(dirtyShards)),
+			    columnFamilyMap(columnFamilyMap) {
+				if (deterministicRandom()->random01() < SERVER_KNOBS->ROCKSDB_HISTOGRAMS_SAMPLE_RATE) {
+					getHistograms = true;
+					startTime = timer_monotonic();
+				} else {
+					getHistograms = false;
+				}
+			}
+		};
+
+		struct DeleteVisitor : public rocksdb::WriteBatch::Handler {
+			std::vector<std::pair<uint32_t, KeyRange>>* deletes;
+
+			DeleteVisitor(std::vector<std::pair<uint32_t, KeyRange>>* deletes) : deletes(deletes) { ASSERT(deletes); }
+
+			rocksdb::Status DeleteRangeCF(uint32_t column_family_id,
+			                              const rocksdb::Slice& begin,
+			                              const rocksdb::Slice& end) override {
+				deletes->push_back(
+				    std::make_pair(column_family_id, KeyRange(KeyRangeRef(toStringRef(begin), toStringRef(end)))));
+				return rocksdb::Status::OK();
+			}
+
+			rocksdb::Status PutCF(uint32_t column_family_id,
+			                      const rocksdb::Slice& key,
+			                      const rocksdb::Slice& value) override {
+				return rocksdb::Status::OK();
+			}
+
+			rocksdb::Status DeleteCF(uint32_t column_family_id, const rocksdb::Slice& key) override {
+				return rocksdb::Status::OK();
+			}
+
+			rocksdb::Status SingleDeleteCF(uint32_t column_family_id, const rocksdb::Slice& key) override {
+				return rocksdb::Status::OK();
+			}
+
+			rocksdb::Status MergeCF(uint32_t column_family_id,
+			                        const rocksdb::Slice& key,
+			                        const rocksdb::Slice& value) override {
+				return rocksdb::Status::OK();
+			}
+		};
+
+		rocksdb::Status doCommit(rocksdb::WriteBatch* batch,
+		                         rocksdb::DB* db,
+		                         std::vector<std::pair<uint32_t, KeyRange>>* deletes,
+		                         bool sample) {
+			if (SERVER_KNOBS->ROCKSDB_SUGGEST_COMPACT_CLEAR_RANGE) {
+				DeleteVisitor dv(deletes);
+				rocksdb::Status s = batch->Iterate(&dv);
+				if (!s.ok()) {
+					logRocksDBError(s, "CommitDeleteVisitor");
+					return s;
+				}
+
+				// If there are any range deletes, we should have added them to be deleted.
+				ASSERT(!deletes->empty() || !batch->HasDeleteRange());
+			}
+
+			rocksdb::WriteOptions options;
+			options.sync = !SERVER_KNOBS->ROCKSDB_UNSAFE_AUTO_FSYNC;
+
+			double writeBeginTime = sample ? timer_monotonic() : 0;
+			rocksdb::Status s = db->Write(options, batch);
+			if (sample) {
+				rocksDBMetrics->getWriteHistogram()->sampleSeconds(timer_monotonic() - writeBeginTime);
+			}
+			if (!s.ok()) {
+				logRocksDBError(s, "Commit");
+				return s;
+			}
+
+			return s;
+		}
+
+		void action(CommitAction& a) {
+			double commitBeginTime;
+			if (a.getHistograms) {
+				commitBeginTime = timer_monotonic();
+				rocksDBMetrics->getCommitQueueWaitHistogram()->sampleSeconds(commitBeginTime - a.startTime);
+			}
+			std::vector<std::pair<uint32_t, KeyRange>> deletes;
+			auto s = doCommit(a.writeBatch.get(), a.db, &deletes, a.getHistograms);
+			if (!s.ok()) {
+				a.done.sendError(statusToError(s));
+				return;
+			}
+
+			for (auto shard : *(a.dirtyShards)) {
+				shard->readIterPool->update();
+			}
+
+			a.done.send(Void());
+			if (SERVER_KNOBS->ROCKSDB_SUGGEST_COMPACT_CLEAR_RANGE) {
+				for (const auto& [id, range] : deletes) {
+					auto cf = columnFamilyMap->find(id);
+					ASSERT(cf != columnFamilyMap->end());
+					auto begin = toSlice(range.begin);
+					auto end = toSlice(range.end);
+
+					ASSERT(a.db->SuggestCompactRange(cf->second, &begin, &end).ok());
+				}
+			}
+
+			if (a.getHistograms) {
+				double currTime = timer_monotonic();
+				rocksDBMetrics->getCommitActionHistogram()->sampleSeconds(currTime - commitBeginTime);
+				rocksDBMetrics->getCommitLatencyHistogram()->sampleSeconds(currTime - a.startTime);
+			}
+
+			sample();
+		}
+
+		struct CloseAction : TypedAction<Writer, CloseAction> {
+			ShardManager* shardManager;
+			ThreadReturnPromise<Void> done;
+			bool deleteOnClose;
+			CloseAction(ShardManager* shardManager, bool deleteOnClose)
+			  : shardManager(shardManager), deleteOnClose(deleteOnClose) {}
+			double getTimeEstimate() const override { return SERVER_KNOBS->COMMIT_TIME_ESTIMATE; }
+		};
+
+		void action(CloseAction& a) {
+			if (a.deleteOnClose) {
+				a.shardManager->destroyAllShards();
+			} else {
+				a.shardManager->closeAllShards();
+			}
+			TraceEvent(SevInfo, "ShardedRocksDB").detail("Method", "Close");
+			a.done.send(Void());
+		}
+
+		struct CheckpointAction : TypedAction<Writer, CheckpointAction> {
+			CheckpointAction(ShardManager* shardManager, const CheckpointRequest& request)
+			  : shardManager(shardManager), request(request) {}
+
+			double getTimeEstimate() const override { return SERVER_KNOBS->COMMIT_TIME_ESTIMATE; }
+
+			ShardManager* shardManager;
+			const CheckpointRequest request;
+			ThreadReturnPromise<CheckpointMetaData> reply;
+		};
+
+		void action(CheckpointAction& a) {
+			TraceEvent(SevInfo, "ShardedRocksCheckpointBegin", logId)
+			    .detail("CheckpointID", a.request.checkpointID)
+			    .detail("Version", a.request.version)
+			    .detail("Ranges", describe(a.request.ranges))
+			    .detail("Format", static_cast<int>(a.request.format))
+			    .detail("CheckpointDir", a.request.checkpointDir);
+
+			rocksdb::Status s;
+			rocksdb::Checkpoint* checkpoint = nullptr;
+
+			PhysicalShard* ps = a.shardManager->getPhysicalShardForAllRanges(a.request.ranges);
+			if (ps == nullptr) {
+				TraceEvent(SevInfo, "ShardedRocksCheckpointInvalidPhysicalShard", logId)
+				    .detail("CheckpointID", a.request.checkpointID)
+				    .detail("Version", a.request.version)
+				    .detail("Ranges", describe(a.request.ranges))
+				    .detail("Format", static_cast<int>(a.request.format))
+				    .detail("CheckpointDir", a.request.checkpointDir);
+				a.reply.sendError(failed_to_create_checkpoint());
+				return;
+			}
+
+			TraceEvent(SevDebug, "ShardedRocksCheckpointCF", logId)
+			    .detail("CFName", ps->cf->GetName())
+			    .detail("CheckpointID", a.request.checkpointID)
+			    .detail("Version", a.request.version)
+			    .detail("Ranges", describe(a.request.ranges))
+			    .detail("PhysicalShardRanges", describe(ps->getAllRanges()))
+			    .detail("Format", static_cast<int>(a.request.format))
+			    .detail("CheckpointDir", a.request.checkpointDir);
+
+			rocksdb::PinnableSlice value;
+			rocksdb::ReadOptions readOptions = getReadOptions();
+			s = a.shardManager->getDb()->Get(
+			    readOptions, a.shardManager->getSpecialKeysShard()->cf, toSlice(persistVersion), &value);
+
+			if (!s.ok() && !s.IsNotFound()) {
+				logRocksDBError(s, "ShardedRocksCheckpointReadPersistVersion");
+				a.reply.sendError(failed_to_create_checkpoint());
+				return;
+			}
+
+			const Version version = s.IsNotFound()
+			                            ? latestVersion
+			                            : BinaryReader::fromStringRef<Version>(toStringRef(value), Unversioned());
+
+			TraceEvent(SevDebug, "ShardedRocksCheckpointVersion", logId)
+			    .detail("CheckpointVersion", a.request.version)
+			    .detail("PersistVersion", version);
+			ASSERT(a.request.version == version || a.request.version == latestVersion);
+
+			CheckpointMetaData res(ps->getAllRanges(), version, a.request.format, a.request.checkpointID);
+			s = rocksdb::Checkpoint::Create(a.shardManager->getDb(), &checkpoint);
+			if (!s.ok()) {
+				logRocksDBError(s, "CreateRocksDBCheckpoint");
+				a.reply.sendError(failed_to_create_checkpoint());
+				return;
+			}
+
+			const std::string& checkpointDir = abspath(a.request.checkpointDir);
+			platform::eraseDirectoryRecursive(checkpointDir);
+			if (a.request.format == DataMoveRocksCF) {
+				rocksdb::ExportImportFilesMetaData* pMetadata = nullptr;
+				s = checkpoint->ExportColumnFamily(ps->cf, checkpointDir, &pMetadata);
+				if (!s.ok()) {
+					logRocksDBError(s, "CheckpointExportColumnFamily");
+					a.reply.sendError(failed_to_create_checkpoint());
+					return;
+				}
+
+				populateMetaData(&res, pMetadata);
+				rocksdb::ExportImportFilesMetaData metadata = *pMetadata;
+				delete pMetadata;
+				if (SERVER_KNOBS->ROCKSDB_ENABLE_CHECKPOINT_VALIDATION) {
+					rocksdb::ImportColumnFamilyOptions importOptions;
+					importOptions.move_files = false;
+					rocksdb::ColumnFamilyHandle* handle;
+					const std::string cfName = deterministicRandom()->randomAlphaNumeric(8);
+					s = a.shardManager->getDb()->CreateColumnFamilyWithImport(
+					    getCFOptions(), cfName, importOptions, metadata, &handle);
+					if (!s.ok()) {
+						TraceEvent(SevError, "ShardedRocksCheckpointValidateImportError", logId)
+						    .detail("Status", s.ToString())
+						    .detail("CheckpointID", a.request.checkpointID)
+						    .detail("Ranges", describe(a.request.ranges))
+						    .detail("CheckDir", a.request.checkpointDir)
+						    .detail("CheckpointVersion", a.request.version)
+						    .detail("PersistVersion", version);
+						a.reply.sendError(failed_to_create_checkpoint());
+						return;
+					}
+
+					auto options = getReadOptions();
+					rocksdb::Iterator* oriIter = a.shardManager->getDb()->NewIterator(options, ps->cf);
+					rocksdb::Iterator* resIter = a.shardManager->getDb()->NewIterator(options, handle);
+					oriIter->SeekToFirst();
+					resIter->SeekToFirst();
+					while (oriIter->Valid() && resIter->Valid()) {
+						if (oriIter->key().compare(resIter->key()) != 0 ||
+						    oriIter->value().compare(resIter->value()) != 0) {
+							TraceEvent(SevError, "ShardedRocksCheckpointValidateError", logId)
+							    .detail("KeyInDB", toStringRef(oriIter->key()))
+							    .detail("ValueInDB", toStringRef(oriIter->value()))
+							    .detail("KeyInCheckpoint", toStringRef(resIter->key()))
+							    .detail("ValueInCheckpoint", toStringRef(resIter->value()));
+
+							a.reply.sendError(failed_to_create_checkpoint());
+							return;
+						}
+						oriIter->Next();
+						resIter->Next();
+					}
+					if (oriIter->Valid() || resIter->Valid()) {
+						a.reply.sendError(failed_to_create_checkpoint());
+						return;
+					}
+					delete oriIter;
+					delete resIter;
+
+					s = a.shardManager->getDb()->DropColumnFamily(handle);
+					if (!s.ok()) {
+						logRocksDBError(s, "CheckpointDropColumnFamily");
+						a.reply.sendError(failed_to_create_checkpoint());
+						return;
+					}
+					s = a.shardManager->getDb()->DestroyColumnFamilyHandle(handle);
+					if (!s.ok()) {
+						logRocksDBError(s, "CheckpointDestroyColumnFamily");
+						a.reply.sendError(failed_to_create_checkpoint());
+						return;
+					}
+					TraceEvent(SevDebug, "ShardedRocksCheckpointValidateSuccess", logId)
+					    .detail("CheckpointVersion", a.request.version)
+					    .detail("PersistVersion", version);
+				}
+			} else {
+				if (checkpoint != nullptr) {
+					delete checkpoint;
+				}
+				a.reply.sendError(not_implemented());
+				return;
+			}
+
+			res.setState(CheckpointMetaData::Complete);
+			res.dir = a.request.checkpointDir;
+			a.reply.send(res);
+
+			if (checkpoint != nullptr) {
+				delete checkpoint;
+			}
+			TraceEvent(SevInfo, "ShardedRocksCheckpointEnd", logId).detail("Checkpoint", res.toString());
+		}
+
+		struct RestoreAction : TypedAction<Writer, RestoreAction> {
+			RestoreAction(ShardManager* shardManager,
+			              const std::string& path,
+			              const std::string& shardId,
+			              const std::vector<KeyRange>& ranges,
+			              const std::vector<CheckpointMetaData>& checkpoints)
+			  : shardManager(shardManager), path(path), shardId(shardId), ranges(ranges), checkpoints(checkpoints) {}
+
+			double getTimeEstimate() const override { return SERVER_KNOBS->COMMIT_TIME_ESTIMATE; }
+
+			ShardManager* shardManager;
+			const std::string path;
+			const std::string shardId;
+			std::vector<KeyRange> ranges;
+			std::vector<CheckpointMetaData> checkpoints;
+			ThreadReturnPromise<Void> done;
+		};
+
+		void action(RestoreAction& a) {
+			TraceEvent(SevInfo, "ShardedRocksDBRestoreBegin", logId)
+			    .detail("Path", a.path)
+			    .detail("Checkpoints", describe(a.checkpoints));
+
+			ASSERT(!a.checkpoints.empty());
+
+			PhysicalShard* ps = a.shardManager->getPhysicalShardForAllRanges(a.ranges);
+			ASSERT(ps != nullptr);
+
+			const CheckpointFormat format = a.checkpoints[0].getFormat();
+			for (int i = 1; i < a.checkpoints.size(); ++i) {
+				if (a.checkpoints[i].getFormat() != format) {
+					throw invalid_checkpoint_format();
+				}
+			}
+
+			rocksdb::Status status;
+			rocksdb::WriteBatch writeBatch;
+			rocksdb::WriteOptions options;
+			options.sync = !SERVER_KNOBS->ROCKSDB_UNSAFE_AUTO_FSYNC;
+
+			if (format == DataMoveRocksCF) {
+				if (a.checkpoints.size() > 1) {
+					TraceEvent(SevError, "ShardedRocksDBRestoreMultipleCFs", logId)
+					    .detail("Path", a.path)
+					    .detail("Checkpoints", describe(a.checkpoints));
+					a.done.sendError(failed_to_restore_checkpoint());
+					return;
+				}
+				CheckpointMetaData& checkpoint = a.checkpoints.front();
+				std::sort(a.ranges.begin(), a.ranges.end(), KeyRangeRef::ArbitraryOrder());
+				std::sort(checkpoint.ranges.begin(), checkpoint.ranges.end(), KeyRangeRef::ArbitraryOrder());
+				if (a.ranges.empty() || checkpoint.ranges.empty() || a.ranges.size() > checkpoint.ranges.size() ||
+				    a.ranges.front().begin != checkpoint.ranges.front().begin) {
+					TraceEvent(SevError, "ShardedRocksDBRestoreFailed", logId)
+					    .detail("Path", a.path)
+					    .detail("Ranges", describe(a.ranges))
+					    .detail("Checkpoints", describe(a.checkpoints));
+					a.done.sendError(failed_to_restore_checkpoint());
+					return;
+				}
+
+				for (int i = 0; i < a.ranges.size(); ++i) {
+					if (a.ranges[i] != checkpoint.ranges[i] && i != a.ranges.size() - 1) {
+						TraceEvent(SevError, "ShardedRocksDBRestoreFailed", logId)
+						    .detail("Path", a.path)
+						    .detail("Ranges", describe(a.ranges))
+						    .detail("Checkpoints", describe(a.checkpoints));
+						a.done.sendError(failed_to_restore_checkpoint());
+						return;
+					}
+				}
+
+				ASSERT(!ps->initialized());
+				TraceEvent(SevDebug, "ShardedRocksRestoringCF", logId)
+				    .detail("Path", a.path)
+				    .detail("Checkpoints", describe(a.checkpoints))
+				    .detail("RocksDBCF", getRocksCF(a.checkpoints[0]).toString());
+
+				status = ps->restore(a.checkpoints[0]);
+
+				if (!status.ok()) {
+					a.done.sendError(failed_to_restore_checkpoint());
+					return;
+				} else {
+					ASSERT(ps->initialized());
+					(*columnFamilyMap)[ps->cf->GetID()] = ps->cf;
+					TraceEvent(SevInfo, "RocksDBRestoreCFSuccess", logId)
+					    .detail("Path", a.path)
+					    .detail("ColumnFaminly", ps->cf->GetName())
+					    .detail("Checkpoints", describe(a.checkpoints));
+
+					// Remove the extra data.
+					KeyRangeRef cRange(a.ranges.back().end, checkpoint.ranges.back().end);
+					if (!cRange.empty()) {
+						TraceEvent(SevInfo, "RocksDBRestoreCFRemoveExtraRange", logId)
+						    .detail("Path", a.path)
+						    .detail("Checkpoints", describe(a.checkpoints))
+						    .detail("RestoreRanges", describe(a.ranges))
+						    .detail("ClearExtraRange", cRange);
+						writeBatch.DeleteRange(ps->cf, toSlice(cRange.begin), toSlice(cRange.end));
+						status = a.shardManager->getDb()->Write(options, &writeBatch);
+						if (!status.ok()) {
+							logRocksDBError(status, "RestoreClearnExtraRanges");
+							a.done.sendError(statusToError(status));
+							return;
+						}
+					}
+				}
+			} else if (format == RocksDBKeyValues) {
+				// Make sure the files are complete for the desired ranges.
+				std::vector<KeyRange> fetchedRanges;
+				std::vector<KeyRange> intendedRanges(a.ranges.begin(), a.ranges.end());
+				std::vector<RocksDBCheckpointKeyValues> rkvs;
+				for (const auto& checkpoint : a.checkpoints) {
+					rkvs.push_back(getRocksKeyValuesCheckpoint(checkpoint));
+					ASSERT(!rkvs.back().fetchedFiles.empty());
+					for (const auto& file : rkvs.back().fetchedFiles) {
+						fetchedRanges.push_back(file.range);
+					}
+				}
+				// Verify that the collective fetchedRanges is the same as the collective intendedRanges.
+				std::sort(fetchedRanges.begin(), fetchedRanges.end(), KeyRangeRef::ArbitraryOrder());
+				std::sort(intendedRanges.begin(), intendedRanges.end(), KeyRangeRef::ArbitraryOrder());
+				int i = 0, j = 0;
+				while (i < fetchedRanges.size() && j < intendedRanges.size()) {
+					if (fetchedRanges[i].begin != intendedRanges[j].begin) {
+						break;
+					} else if (fetchedRanges[i] == intendedRanges[j]) {
+						++i;
+						++j;
+					} else if (fetchedRanges[i].contains(intendedRanges[j])) {
+						fetchedRanges[i] = KeyRangeRef(intendedRanges[j].end, fetchedRanges[i].end);
+						++j;
+					} else if (intendedRanges[j].contains(fetchedRanges[i])) {
+						intendedRanges[j] = KeyRangeRef(fetchedRanges[i].end, intendedRanges[j].end);
+						++i;
+					} else {
+						break;
+					}
+				}
+				if (i != fetchedRanges.size() || j != intendedRanges.size()) {
+					TraceEvent(SevError, "ShardedRocksDBRestoreFailed", logId)
+					    .detail("Reason", "RestoreFilesRangesMismatch")
+					    .detail("Ranges", describe(a.ranges))
+					    .setMaxFieldLength(1000)
+					    .detail("FetchedFiles", describe(rkvs));
+					a.done.sendError(failed_to_restore_checkpoint());
+					return;
+				}
+
+				if (!ps->initialized()) {
+					TraceEvent(SevDebug, "ShardedRocksRestoreInitPS", logId)
+					    .detail("Path", a.path)
+					    .detail("Checkpoints", describe(a.checkpoints))
+					    .detail("PhysicalShard", ps->toString());
+					status = ps->init();
+					if (!status.ok()) {
+						logRocksDBError(status, "RestoreInitPhysicalShard");
+						a.done.sendError(statusToError(status));
+						return;
+					}
+					(*columnFamilyMap)[ps->cf->GetID()] = ps->cf;
+				}
+				if (!status.ok()) {
+					logRocksDBError(status, "RestoreInitPhysicalShard");
+					a.done.sendError(statusToError(status));
+					return;
+				}
+
+				for (const auto& checkpoint : a.checkpoints) {
+					status = ps->restore(checkpoint);
+					if (!status.ok()) {
+						TraceEvent(SevWarnAlways, "ShardedRocksIngestFileError", logId)
+						    .detail("Error", status.ToString())
+						    .detail("Path", a.path)
+						    .detail("Checkpoint", checkpoint.toString())
+						    .detail("PhysicalShard", ps->toString());
+						break;
+					}
+				}
+
+				if (!status.ok()) {
+					TraceEvent(SevWarnAlways, "ShardedRocksRestoreError", logId).detail("Status", status.ToString());
+					for (const auto& range : a.ranges) {
+						writeBatch.DeleteRange(ps->cf, toSlice(range.begin), toSlice(range.end));
+					}
+					TraceEvent(SevInfo, "ShardedRocksRevertRestore", logId)
+					    .detail("Path", a.path)
+					    .detail("Checkpoints", describe(a.checkpoints))
+					    .detail("PhysicalShard", ps->toString())
+					    .detail("RestoreRanges", describe(a.ranges));
+
+					rocksdb::Status s = a.shardManager->getDb()->Write(options, &writeBatch);
+					if (!s.ok()) {
+						TraceEvent(SevError, "ShardedRocksRevertRestoreError", logId)
+						    .detail("Error", s.ToString())
+						    .detail("Path", a.path)
+						    .detail("PhysicalShard", ps->toString())
+						    .detail("RestoreRanges", describe(a.ranges));
+					}
+					a.done.sendError(failed_to_restore_checkpoint());
+					return;
+				}
+			} else if (format == RocksDB) {
+				a.done.sendError(not_implemented());
+				return;
+			}
+
+			TraceEvent(SevInfo, "ShardedRocksDBRestoreEnd", logId)
+			    .detail("Path", a.path)
+			    .detail("Checkpoints", describe(a.checkpoints));
+			a.done.send(Void());
+		}
+	};
+
+	struct Reader : IThreadPoolReceiver {
+		const UID logId;
+		double readValueTimeout;
+		double readValuePrefixTimeout;
+		double readRangeTimeout;
+		int threadIndex;
+		std::shared_ptr<RocksDBMetrics> rocksDBMetrics;
+		double sampleStartTime;
+
+		explicit Reader(UID logId, int threadIndex, std::shared_ptr<RocksDBMetrics> rocksDBMetrics)
+		  : logId(logId), threadIndex(threadIndex), rocksDBMetrics(rocksDBMetrics), sampleStartTime(now()) {
+			readValueTimeout = SERVER_KNOBS->ROCKSDB_READ_VALUE_TIMEOUT;
+			readValuePrefixTimeout = SERVER_KNOBS->ROCKSDB_READ_VALUE_PREFIX_TIMEOUT;
+			readRangeTimeout = SERVER_KNOBS->ROCKSDB_READ_RANGE_TIMEOUT;
+		}
+
+		void init() override {}
+
+		void sample() {
+			if (SERVER_KNOBS->ROCKSDB_METRICS_SAMPLE_INTERVAL > 0 &&
+			    now() - sampleStartTime >= SERVER_KNOBS->ROCKSDB_METRICS_SAMPLE_INTERVAL) {
+				rocksDBMetrics->collectPerfContext(threadIndex);
+				rocksDBMetrics->resetPerfContext();
+				sampleStartTime = now();
+			}
+		}
+		struct ReadValueAction : TypedAction<Reader, ReadValueAction> {
+			Key key;
+			PhysicalShard* shard;
+			Optional<UID> debugID;
+			double startTime;
+			bool getHistograms;
+			bool logShardMemUsage;
+			ThreadReturnPromise<Optional<Value>> result;
+
+			ReadValueAction(KeyRef key, PhysicalShard* shard, Optional<UID> debugID)
+			  : key(key), shard(shard), debugID(debugID), startTime(timer_monotonic()),
+			    getHistograms(
+			        (deterministicRandom()->random01() < SERVER_KNOBS->ROCKSDB_HISTOGRAMS_SAMPLE_RATE) ? true : false) {
+			}
+
+			double getTimeEstimate() const override { return SERVER_KNOBS->READ_VALUE_TIME_ESTIMATE; }
+		};
+
+		void action(ReadValueAction& a) {
+			double readBeginTime = timer_monotonic();
+			if (a.getHistograms) {
+				rocksDBMetrics->getReadValueQueueWaitHistogram(threadIndex)->sampleSeconds(readBeginTime - a.startTime);
+			}
+			Optional<TraceBatch> traceBatch;
+			if (a.debugID.present()) {
+				traceBatch = { TraceBatch{} };
+				traceBatch.get().addEvent("GetValueDebug", a.debugID.get().first(), "Reader.Before");
+			}
+			if (SERVER_KNOBS->ROCKSDB_SET_READ_TIMEOUT && readBeginTime - a.startTime > readValueTimeout) {
+				TraceEvent(SevWarn, "ShardedRocksDBError")
+				    .detail("Error", "Read value request timedout")
+				    .detail("Method", "ReadValueAction")
+				    .detail("Timeout value", readValueTimeout);
+				a.result.sendError(key_value_store_deadline_exceeded());
+				return;
+			}
+
+			rocksdb::PinnableSlice value;
+			auto options = getReadOptions();
+
+			auto db = a.shard->db;
+			if (SERVER_KNOBS->ROCKSDB_SET_READ_TIMEOUT) {
+				uint64_t deadlineMircos =
+				    db->GetEnv()->NowMicros() + (readValueTimeout - (timer_monotonic() - a.startTime)) * 1000000;
+				std::chrono::seconds deadlineSeconds(deadlineMircos / 1000000);
+				options.deadline = std::chrono::duration_cast<std::chrono::microseconds>(deadlineSeconds);
+			}
+			double dbGetBeginTime = a.getHistograms ? timer_monotonic() : 0;
+			auto s = db->Get(options, a.shard->cf, toSlice(a.key), &value);
+
+			if (a.getHistograms) {
+				rocksDBMetrics->getReadValueGetHistogram(threadIndex)
+				    ->sampleSeconds(timer_monotonic() - dbGetBeginTime);
+			}
+
+			if (a.debugID.present()) {
+				traceBatch.get().addEvent("GetValueDebug", a.debugID.get().first(), "Reader.After");
+				traceBatch.get().dump();
+			}
+			if (s.ok()) {
+				a.result.send(Value(toStringRef(value)));
+			} else if (s.IsNotFound()) {
+				a.result.send(Optional<Value>());
+			} else {
+				logRocksDBError(s, "ReadValue");
+				a.result.sendError(statusToError(s));
+			}
+
+			if (a.getHistograms) {
+				double currTime = timer_monotonic();
+				rocksDBMetrics->getReadValueActionHistogram(threadIndex)->sampleSeconds(currTime - readBeginTime);
+				rocksDBMetrics->getReadValueLatencyHistogram(threadIndex)->sampleSeconds(currTime - a.startTime);
+			}
+
+			sample();
+		}
+
+		struct ReadValuePrefixAction : TypedAction<Reader, ReadValuePrefixAction> {
+			Key key;
+			int maxLength;
+			PhysicalShard* shard;
+			Optional<UID> debugID;
+			double startTime;
+			bool getHistograms;
+			bool logShardMemUsage;
+			ThreadReturnPromise<Optional<Value>> result;
+
+			ReadValuePrefixAction(Key key, int maxLength, PhysicalShard* shard, Optional<UID> debugID)
+			  : key(key), maxLength(maxLength), shard(shard), debugID(debugID), startTime(timer_monotonic()),
+			    getHistograms((deterministicRandom()->random01() < SERVER_KNOBS->ROCKSDB_HISTOGRAMS_SAMPLE_RATE)
+			                      ? true
+			                      : false){};
+			double getTimeEstimate() const override { return SERVER_KNOBS->READ_VALUE_TIME_ESTIMATE; }
+		};
+
+		void action(ReadValuePrefixAction& a) {
+			double readBeginTime = timer_monotonic();
+			if (a.getHistograms) {
+				rocksDBMetrics->getReadPrefixQueueWaitHistogram(threadIndex)
+				    ->sampleSeconds(readBeginTime - a.startTime);
+			}
+			Optional<TraceBatch> traceBatch;
+			if (a.debugID.present()) {
+				traceBatch = { TraceBatch{} };
+				traceBatch.get().addEvent("GetValuePrefixDebug",
+				                          a.debugID.get().first(),
+				                          "Reader.Before"); //.detail("TaskID", g_network->getCurrentTask());
+			}
+			if (SERVER_KNOBS->ROCKSDB_SET_READ_TIMEOUT && readBeginTime - a.startTime > readValuePrefixTimeout) {
+				TraceEvent(SevWarn, "ShardedRocksDBError")
+				    .detail("Error", "Read value prefix request timedout")
+				    .detail("Method", "ReadValuePrefixAction")
+				    .detail("Timeout value", readValuePrefixTimeout);
+				a.result.sendError(key_value_store_deadline_exceeded());
+				return;
+			}
+
+			rocksdb::PinnableSlice value;
+			auto options = getReadOptions();
+			auto db = a.shard->db;
+			if (SERVER_KNOBS->ROCKSDB_SET_READ_TIMEOUT) {
+				uint64_t deadlineMircos =
+				    db->GetEnv()->NowMicros() + (readValuePrefixTimeout - (timer_monotonic() - a.startTime)) * 1000000;
+				std::chrono::seconds deadlineSeconds(deadlineMircos / 1000000);
+				options.deadline = std::chrono::duration_cast<std::chrono::microseconds>(deadlineSeconds);
+			}
+
+			double dbGetBeginTime = a.getHistograms ? timer_monotonic() : 0;
+			auto s = db->Get(options, a.shard->cf, toSlice(a.key), &value);
+
+			if (a.getHistograms) {
+				rocksDBMetrics->getReadPrefixGetHistogram(threadIndex)
+				    ->sampleSeconds(timer_monotonic() - dbGetBeginTime);
+			}
+
+			if (a.debugID.present()) {
+				traceBatch.get().addEvent("GetValuePrefixDebug",
+				                          a.debugID.get().first(),
+				                          "Reader.After"); //.detail("TaskID", g_network->getCurrentTask());
+				traceBatch.get().dump();
+			}
+			if (s.ok()) {
+				a.result.send(Value(StringRef(reinterpret_cast<const uint8_t*>(value.data()),
+				                              std::min(value.size(), size_t(a.maxLength)))));
+			} else if (s.IsNotFound()) {
+				a.result.send(Optional<Value>());
+			} else {
+				logRocksDBError(s, "ReadValuePrefix");
+				a.result.sendError(statusToError(s));
+			}
+			if (a.getHistograms) {
+				double currTime = timer_monotonic();
+				rocksDBMetrics->getReadPrefixActionHistogram(threadIndex)->sampleSeconds(currTime - readBeginTime);
+				rocksDBMetrics->getReadPrefixLatencyHistogram(threadIndex)->sampleSeconds(currTime - a.startTime);
+			}
+
+			sample();
+		}
+
+		struct ReadRangeAction : TypedAction<Reader, ReadRangeAction>, FastAllocated<ReadRangeAction> {
+			KeyRange keys;
+			std::vector<std::pair<PhysicalShard*, KeyRange>> shardRanges;
+			int rowLimit, byteLimit;
+			double startTime;
+			bool getHistograms;
+			bool logShardMemUsage;
+			ThreadReturnPromise<RangeResult> result;
+			ReadRangeAction(KeyRange keys, std::vector<DataShard*> shards, int rowLimit, int byteLimit)
+			  : keys(keys), rowLimit(rowLimit), byteLimit(byteLimit), startTime(timer_monotonic()),
+			    getHistograms(
+			        (deterministicRandom()->random01() < SERVER_KNOBS->ROCKSDB_HISTOGRAMS_SAMPLE_RATE) ? true : false) {
+				std::set<PhysicalShard*> usedShards;
+				for (const DataShard* shard : shards) {
+					ASSERT(shard);
+					shardRanges.emplace_back(shard->physicalShard, keys & shard->range);
+					usedShards.insert(shard->physicalShard);
+				}
+				if (usedShards.size() != shards.size()) {
+					TraceEvent("ReadRangeMetrics")
+					    .detail("NumPhysicalShards", usedShards.size())
+					    .detail("NumDataShards", shards.size());
+				}
+			}
+			double getTimeEstimate() const override { return SERVER_KNOBS->READ_RANGE_TIME_ESTIMATE; }
+		};
+
+		void action(ReadRangeAction& a) {
+			double readBeginTime = timer_monotonic();
+			if (a.getHistograms) {
+				rocksDBMetrics->getReadRangeQueueWaitHistogram(threadIndex)->sampleSeconds(readBeginTime - a.startTime);
+			}
+			if (SERVER_KNOBS->ROCKSDB_SET_READ_TIMEOUT && readBeginTime - a.startTime > readRangeTimeout) {
+				TraceEvent(SevWarn, "ShardedRocksKVSReadTimeout")
+				    .detail("Error", "Read range request timedout")
+				    .detail("Method", "ReadRangeAction")
+				    .detail("Timeout value", readRangeTimeout);
+				a.result.sendError(key_value_store_deadline_exceeded());
+				return;
+			}
+
+			int rowLimit = a.rowLimit;
+			int byteLimit = a.byteLimit;
+			RangeResult result;
+
+			if (rowLimit == 0 || byteLimit == 0) {
+				a.result.send(result);
+			}
+			if (rowLimit < 0) {
+				// Reverses the shard order so we could read range in reverse direction.
+				std::reverse(a.shardRanges.begin(), a.shardRanges.end());
+			}
+
+			// TODO: consider multi-thread reads. It's possible to read multiple shards in parallel. However, the
+			// number of rows to read needs to be calculated based on the previous read result. We may read more
+			// than we expected when parallel read is used when the previous result is not available. It's unlikely
+			// to get to performance improvement when the actual number of rows to read is very small.
+			int accumulatedBytes = 0;
+			int numShards = 0;
+			for (auto& [shard, range] : a.shardRanges) {
+				if (shard == nullptr || !shard->initialized()) {
+					TraceEvent(SevWarn, "ShardedRocksReadRangeShardNotReady", logId)
+					    .detail("Range", range)
+					    .detail("Reason", shard == nullptr ? "Not Exist" : "Not Initialized");
+					continue;
+				}
+				auto bytesRead = readRangeInDb(shard, range, rowLimit, byteLimit, &result);
+				if (bytesRead < 0) {
+					// Error reading an instance.
+					a.result.sendError(internal_error());
+					return;
+				}
+				byteLimit -= bytesRead;
+				accumulatedBytes += bytesRead;
+				++numShards;
+				if (result.size() >= abs(a.rowLimit) || accumulatedBytes >= a.byteLimit) {
+					break;
+				}
+
+				if (SERVER_KNOBS->ROCKSDB_SET_READ_TIMEOUT && timer_monotonic() - a.startTime > readRangeTimeout) {
+					TraceEvent(SevInfo, "ShardedRocksDBTimeout")
+					    .detail("Action", "ReadRange")
+					    .detail("ShardsRead", numShards)
+					    .detail("BytesRead", accumulatedBytes);
+					a.result.sendError(key_value_store_deadline_exceeded());
+					return;
+				}
+			}
+
+			result.more =
+			    (result.size() == a.rowLimit) || (result.size() == -a.rowLimit) || (accumulatedBytes >= a.byteLimit);
+			a.result.send(result);
+			if (a.getHistograms) {
+				double currTime = timer_monotonic();
+				rocksDBMetrics->getReadRangeActionHistogram(threadIndex)->sampleSeconds(currTime - readBeginTime);
+				rocksDBMetrics->getReadRangeLatencyHistogram(threadIndex)->sampleSeconds(currTime - a.startTime);
+				if (a.shardRanges.size() > 1) {
+					TraceEvent(SevInfo, "ShardedRocksDB").detail("ReadRangeShards", a.shardRanges.size());
+				}
+			}
+
+			sample();
+		}
+	};
+
+	struct Counters {
+		CounterCollection cc;
+		Counter immediateThrottle;
+		Counter failedToAcquire;
+
+		Counters()
+		  : cc("RocksDBThrottle"), immediateThrottle("ImmediateThrottle", cc), failedToAcquire("failedToAcquire", cc) {}
+	};
+
+	// Persist shard mappinng key range should not be in shardMap.
+	explicit ShardedRocksDBKeyValueStore(const std::string& path, UID id)
+	  : rState(std::make_shared<ShardedRocksDBState>()), path(path), id(id),
+	    readSemaphore(SERVER_KNOBS->ROCKSDB_READ_QUEUE_SOFT_MAX),
+	    fetchSemaphore(SERVER_KNOBS->ROCKSDB_FETCH_QUEUE_SOFT_MAX),
+	    numReadWaiters(SERVER_KNOBS->ROCKSDB_READ_QUEUE_HARD_MAX - SERVER_KNOBS->ROCKSDB_READ_QUEUE_SOFT_MAX),
+	    numFetchWaiters(SERVER_KNOBS->ROCKSDB_FETCH_QUEUE_HARD_MAX - SERVER_KNOBS->ROCKSDB_FETCH_QUEUE_SOFT_MAX),
+	    errorListener(std::make_shared<RocksDBErrorListener>()), errorFuture(errorListener->getFuture()),
+	    dbOptions(getOptions()), shardManager(path, id, dbOptions),
+	    rocksDBMetrics(std::make_shared<RocksDBMetrics>(id, dbOptions.statistics)) {
+		// In simluation, run the reader/writer threads as Coro threads (i.e. in the network thread. The storage
+		// engine is still multi-threaded as background compaction threads are still present. Reads/writes to disk
+		// will also block the network thread in a way that would be unacceptable in production but is a necessary
+		// evil here. When performing the reads in background threads in simulation, the event loop thinks there is
+		// no work to do and advances time faster than 1 sec/sec. By the time the blocking read actually finishes,
+		// simulation has advanced time by more than 5 seconds, so every read fails with a transaction_too_old
+		// error. Doing blocking IO on the main thread solves this issue. There are almost certainly better fixes,
+		// but my goal was to get a less invasive change merged first and work on a more realistic version if/when
+		// we think that would provide substantially more confidence in the correctness.
+		// TODO: Adapt the simulation framework to not advance time quickly when background reads/writes are
+		// occurring.
+		if (g_network->isSimulated()) {
+			TraceEvent(SevDebug, "ShardedRocksDB").detail("Info", "Use Coro threads in simulation.");
+			writeThread = CoroThreadPool::createThreadPool();
+			readThreads = CoroThreadPool::createThreadPool();
+		} else {
+			writeThread = createGenericThreadPool(/*stackSize=*/0, SERVER_KNOBS->ROCKSDB_WRITER_THREAD_PRIORITY);
+			readThreads = createGenericThreadPool(/*stackSize=*/0, SERVER_KNOBS->ROCKSDB_READER_THREAD_PRIORITY);
+		}
+		writeThread->addThread(new Writer(id, 0, shardManager.getColumnFamilyMap(), rocksDBMetrics), "fdb-rocksdb-wr");
+		TraceEvent("ShardedRocksDBReadThreads", id)
+		    .detail("KnobRocksDBReadParallelism", SERVER_KNOBS->ROCKSDB_READ_PARALLELISM);
+		for (unsigned i = 0; i < SERVER_KNOBS->ROCKSDB_READ_PARALLELISM; ++i) {
+			readThreads->addThread(new Reader(id, i, rocksDBMetrics), "fdb-rocksdb-re");
+		}
+	}
+
+	Future<Void> getError() const override { return errorFuture; }
+
+	ACTOR static void doClose(ShardedRocksDBKeyValueStore* self, bool deleteOnClose) {
+		self->rState->closing = true;
+		// The metrics future retains a reference to the DB, so stop it before we delete it.
+		self->metrics.reset();
+		self->refreshHolder.cancel();
+		self->cleanUpJob.cancel();
+
+		wait(self->readThreads->stop());
+		auto a = new Writer::CloseAction(&self->shardManager, deleteOnClose);
+		auto f = a->done.getFuture();
+		self->writeThread->post(a);
+		wait(f);
+		wait(self->writeThread->stop());
+		if (self->closePromise.canBeSet())
+			self->closePromise.send(Void());
+		delete self;
+	}
+
+	Future<Void> onClosed() const override { return closePromise.getFuture(); }
+
+	void dispose() override { doClose(this, true); }
+
+	void close() override { doClose(this, false); }
+
+	KeyValueStoreType getType() const override { return KeyValueStoreType(KeyValueStoreType::SSD_SHARDED_ROCKSDB); }
+
+	bool shardAware() const override { return true; }
+
+	Future<Void> init() override {
+		if (openFuture.isValid()) {
+			return openFuture;
+			// Restore durable state if KVS is open. KVS will be re-initialized during rollback. To avoid the cost
+			// of opening and closing multiple rocksdb instances, we reconcile the shard map using persist shard
+			// mapping data.
+		} else {
+			auto a = std::make_unique<Writer::OpenAction>(
+			    &shardManager, metrics, &readSemaphore, &fetchSemaphore, errorListener);
+			openFuture = a->done.getFuture();
+			this->metrics = ShardManager::shardMetricsLogger(this->rState, openFuture, &shardManager) &&
+			                rocksDBAggregatedMetricsLogger(this->rState, openFuture, rocksDBMetrics, &shardManager);
+			this->refreshHolder = refreshReadIteratorPools(this->rState, openFuture, shardManager.getAllShards());
+			this->cleanUpJob = emptyShardCleaner(this->rState, openFuture, &shardManager, writeThread);
+			writeThread->post(a.release());
+			return openFuture;
+		}
+	}
+
+	Future<Void> addRange(KeyRangeRef range, std::string id) override {
+		auto shard = shardManager.addRange(range, id);
+		if (shard->initialized()) {
+			return Void();
+		}
+		auto a = new Writer::AddShardAction(shard);
+		Future<Void> res = a->done.getFuture();
+		writeThread->post(a);
+		return res;
+	}
+
+	void set(KeyValueRef kv, const Arena*) override { shardManager.put(kv.key, kv.value); }
+
+	void clear(KeyRangeRef range, const Arena*) override {
+		if (range.singleKeyRange()) {
+			shardManager.clear(range.begin);
+		} else {
+			shardManager.clearRange(range);
+		}
+	}
+
+	// Checks and waits for few seconds if rocskdb is overloaded.
+	ACTOR Future<Void> checkRocksdbState(rocksdb::DB* db) {
+		state uint64_t estPendCompactBytes;
+		state int count = SERVER_KNOBS->ROCKSDB_CAN_COMMIT_DELAY_TIMES_ON_OVERLOAD;
+		db->GetAggregatedIntProperty(rocksdb::DB::Properties::kEstimatePendingCompactionBytes, &estPendCompactBytes);
+		while (count && estPendCompactBytes > SERVER_KNOBS->ROCKSDB_CAN_COMMIT_COMPACT_BYTES_LIMIT) {
+			wait(delay(SERVER_KNOBS->ROCKSDB_CAN_COMMIT_DELAY_ON_OVERLOAD));
+			count--;
+			db->GetAggregatedIntProperty(rocksdb::DB::Properties::kEstimatePendingCompactionBytes,
+			                             &estPendCompactBytes);
+		}
+
+		return Void();
+	}
+
+	Future<Void> canCommit() override { return checkRocksdbState(shardManager.getDb()); }
+
+	Future<Void> commit(bool) override {
+		auto a = new Writer::CommitAction(shardManager.getDb(),
+		                                  shardManager.getWriteBatch(),
+		                                  shardManager.getDirtyShards(),
+		                                  shardManager.getColumnFamilyMap());
+		auto res = a->done.getFuture();
+		writeThread->post(a);
+		return res;
+	}
+
+	void checkWaiters(const FlowLock& semaphore, int maxWaiters) {
+		if (semaphore.waiters() > maxWaiters) {
+			++counters.immediateThrottle;
+			throw server_overloaded();
+		}
+	}
+
+	// We don't throttle eager reads and reads to the FF keyspace because FDB struggles when those reads fail.
+	// Thus far, they have been low enough volume to not cause an issue.
+	static bool shouldThrottle(ReadType type, KeyRef key) {
+		return type != ReadType::EAGER && !(key.startsWith(systemKeys.begin));
+	}
+
+	ACTOR template <class Action>
+	static Future<Optional<Value>> read(Action* action, FlowLock* semaphore, IThreadPool* pool, Counter* counter) {
+		state std::unique_ptr<Action> a(action);
+		state Optional<Void> slot = wait(timeout(semaphore->take(), SERVER_KNOBS->ROCKSDB_READ_QUEUE_WAIT));
+		if (!slot.present()) {
+			++(*counter);
+			throw server_overloaded();
+		}
+
+		state FlowLock::Releaser release(*semaphore);
+
+		auto fut = a->result.getFuture();
+		pool->post(a.release());
+		Optional<Value> result = wait(fut);
+
+		return result;
+	}
+
+	Future<Optional<Value>> readValue(KeyRef key, Optional<ReadOptions> options) override {
+		auto* shard = shardManager.getDataShard(key);
+		if (shard == nullptr || !shard->physicalShard->initialized()) {
+			// TODO: read non-exist system key range should not cause an error.
+			TraceEvent(SevWarn, "ShardedRocksDB", this->id)
+			    .detail("Detail", "Read non-exist key range")
+			    .detail("ReadKey", key);
+			return Optional<Value>();
+		}
+
+		ReadType type = ReadType::NORMAL;
+		Optional<UID> debugID;
+
+		if (options.present()) {
+			type = options.get().type;
+			debugID = options.get().debugID;
+		}
+
+		if (!shouldThrottle(type, key)) {
+			auto a = new Reader::ReadValueAction(key, shard->physicalShard, debugID);
+			auto res = a->result.getFuture();
+			readThreads->post(a);
+			return res;
+		}
+
+		auto& semaphore = (type == ReadType::FETCH) ? fetchSemaphore : readSemaphore;
+		int maxWaiters = (type == ReadType::FETCH) ? numFetchWaiters : numReadWaiters;
+
+		checkWaiters(semaphore, maxWaiters);
+		auto a = std::make_unique<Reader::ReadValueAction>(key, shard->physicalShard, debugID);
+		return read(a.release(), &semaphore, readThreads.getPtr(), &counters.failedToAcquire);
+	}
+
+	Future<Optional<Value>> readValuePrefix(KeyRef key, int maxLength, Optional<ReadOptions> options) override {
+		auto* shard = shardManager.getDataShard(key);
+		if (shard == nullptr || !shard->physicalShard->initialized()) {
+			// TODO: read non-exist system key range should not cause an error.
+			TraceEvent(SevWarn, "ShardedRocksDB", this->id)
+			    .detail("Detail", "Read non-exist key range")
+			    .detail("ReadKey", key);
+			return Optional<Value>();
+		}
+
+		ReadType type = ReadType::NORMAL;
+		Optional<UID> debugID;
+
+		if (options.present()) {
+			type = options.get().type;
+			debugID = options.get().debugID;
+		}
+
+		if (!shouldThrottle(type, key)) {
+			auto a = new Reader::ReadValuePrefixAction(key, maxLength, shard->physicalShard, debugID);
+			auto res = a->result.getFuture();
+			readThreads->post(a);
+			return res;
+		}
+
+		auto& semaphore = (type == ReadType::FETCH) ? fetchSemaphore : readSemaphore;
+		int maxWaiters = (type == ReadType::FETCH) ? numFetchWaiters : numReadWaiters;
+
+		checkWaiters(semaphore, maxWaiters);
+		auto a = std::make_unique<Reader::ReadValuePrefixAction>(key, maxLength, shard->physicalShard, debugID);
+		return read(a.release(), &semaphore, readThreads.getPtr(), &counters.failedToAcquire);
+	}
+
+	ACTOR static Future<Standalone<RangeResultRef>> read(Reader::ReadRangeAction* action,
+	                                                     FlowLock* semaphore,
+	                                                     IThreadPool* pool,
+	                                                     Counter* counter) {
+		state std::unique_ptr<Reader::ReadRangeAction> a(action);
+		state Optional<Void> slot = wait(timeout(semaphore->take(), SERVER_KNOBS->ROCKSDB_READ_QUEUE_WAIT));
+		if (!slot.present()) {
+			++(*counter);
+			throw server_overloaded();
+		}
+
+		state FlowLock::Releaser release(*semaphore);
+
+		auto fut = a->result.getFuture();
+		pool->post(a.release());
+		Standalone<RangeResultRef> result = wait(fut);
+
+		return result;
+	}
+
+	Future<RangeResult> readRange(KeyRangeRef keys,
+	                              int rowLimit,
+	                              int byteLimit,
+	                              Optional<ReadOptions> options = Optional<ReadOptions>()) override {
+		TraceEvent(SevVerbose, "ShardedRocksReadRangeBegin", this->id).detail("Range", keys);
+		auto shards = shardManager.getDataShardsByRange(keys);
+
+		ReadType type = ReadType::NORMAL;
+		if (options.present()) {
+			type = options.get().type;
+		}
+
+		if (!shouldThrottle(type, keys.begin)) {
+			auto a = new Reader::ReadRangeAction(keys, shards, rowLimit, byteLimit);
+			auto res = a->result.getFuture();
+			readThreads->post(a);
+			return res;
+		}
+
+		auto& semaphore = (type == ReadType::FETCH) ? fetchSemaphore : readSemaphore;
+		int maxWaiters = (type == ReadType::FETCH) ? numFetchWaiters : numReadWaiters;
+		checkWaiters(semaphore, maxWaiters);
+
+		auto a = std::make_unique<Reader::ReadRangeAction>(keys, shards, rowLimit, byteLimit);
+		return read(a.release(), &semaphore, readThreads.getPtr(), &counters.failedToAcquire);
+	}
+
+	ACTOR static Future<Void> emptyShardCleaner(std::shared_ptr<ShardedRocksDBState> rState,
+	                                            Future<Void> openFuture,
+	                                            ShardManager* shardManager,
+	                                            Reference<IThreadPool> writeThread) {
+		state double cleanUpDelay = SERVER_KNOBS->ROCKSDB_PHYSICAL_SHARD_CLEAN_UP_DELAY;
+		state double cleanUpPeriod = cleanUpDelay * 2;
+		try {
+			wait(openFuture);
+			loop {
+				wait(delay(cleanUpPeriod));
+				if (rState->closing) {
+					break;
+				}
+				auto shards = shardManager->getPendingDeletionShards(cleanUpDelay);
+				if (shards.size() > 0) {
+					auto a = new Writer::RemoveShardAction(shards);
+					Future<Void> f = a->done.getFuture();
+					writeThread->post(a);
+					TraceEvent(SevInfo, "ShardedRocksDB").detail("DeleteEmptyShards", shards.size());
+					wait(f);
+				}
+			}
+		} catch (Error& e) {
+			if (e.code() != error_code_actor_cancelled) {
+				TraceEvent(SevError, "DeleteEmptyShardsError").errorUnsuppressed(e);
+			}
+		}
+		return Void();
+	}
+
+	StorageBytes getStorageBytes() const override {
+		uint64_t live = 0;
+		ASSERT(shardManager.getDb()->GetAggregatedIntProperty(rocksdb::DB::Properties::kLiveSstFilesSize, &live));
+
+		int64_t free;
+		int64_t total;
+		g_network->getDiskBytes(path, free, total);
+		return StorageBytes(free, total, live, free);
+	}
+
+	Future<CheckpointMetaData> checkpoint(const CheckpointRequest& request) override {
+		auto a = new Writer::CheckpointAction(&shardManager, request);
+
+		auto res = a->reply.getFuture();
+		writeThread->post(a);
+		return res;
+	}
+
+	Future<Void> restore(const std::string& shardId,
+	                     const std::vector<KeyRange>& ranges,
+	                     const std::vector<CheckpointMetaData>& checkpoints) override {
+		return doRestore(this, shardId, ranges, checkpoints);
+	}
+
+	std::vector<std::string> removeRange(KeyRangeRef range) override { return shardManager.removeRange(range); }
+
+	void persistRangeMapping(KeyRangeRef range, bool isAdd) override {
+		return shardManager.persistRangeMapping(range, isAdd);
+	}
+
+	// Used for debugging shard mapping issue.
+	std::vector<std::pair<KeyRange, std::string>> getDataMapping() { return shardManager.getDataMapping(); }
+
+	Future<EncryptionAtRestMode> encryptionMode() override {
+		return EncryptionAtRestMode(EncryptionAtRestMode::DISABLED);
+	}
+
+	CoalescedKeyRangeMap<std::string> getExistingRanges() override { return shardManager.getExistingRanges(); }
+
+	std::shared_ptr<ShardedRocksDBState> rState;
+	rocksdb::Options dbOptions;
+	ShardManager shardManager;
+	std::shared_ptr<RocksDBMetrics> rocksDBMetrics;
+	std::string path;
+	UID id;
+	Reference<IThreadPool> writeThread;
+	Reference<IThreadPool> readThreads;
+	std::shared_ptr<RocksDBErrorListener> errorListener;
+	Future<Void> errorFuture;
+	Promise<Void> closePromise;
+	Future<Void> openFuture;
+	Optional<Future<Void>> metrics;
+	FlowLock readSemaphore;
+	int numReadWaiters;
+	FlowLock fetchSemaphore;
+	int numFetchWaiters;
+	Counters counters;
+	Future<Void> refreshHolder;
+	Future<Void> cleanUpJob;
+};
+
+ACTOR Future<Void> testCheckpointRestore(IKeyValueStore* kvStore, std::vector<KeyRange> ranges) {
+	state std::string checkpointDir = "checkpoint" + deterministicRandom()->randomAlphaNumeric(5);
+	platform::eraseDirectoryRecursive(checkpointDir);
+	CheckpointRequest request(
+	    latestVersion, ranges, DataMoveRocksCF, deterministicRandom()->randomUniqueID(), checkpointDir);
+	state CheckpointMetaData checkpoint = wait(kvStore->checkpoint(request));
+	RocksDBColumnFamilyCheckpoint rocksCF = getRocksCF(checkpoint);
+
+	TraceEvent(SevDebug, "ShardedRocksCheckpointTest")
+	    .detail("Checkpoint", checkpoint.toString())
+	    .detail("ColumnFamily", rocksCF.toString());
+
+	state std::string rocksDBRestoreDir = "sharded-rocks-restore" + deterministicRandom()->randomAlphaNumeric(5);
+	platform::eraseDirectoryRecursive(rocksDBRestoreDir);
+	state IKeyValueStore* restoreKv = keyValueStoreShardedRocksDB(
+	    rocksDBRestoreDir, deterministicRandom()->randomUniqueID(), KeyValueStoreType::SSD_SHARDED_ROCKSDB);
+	wait(restoreKv->init());
+	try {
+		const std::string shardId = "restoredShard";
+		wait(restoreKv->restore(shardId, ranges, { checkpoint }));
+	} catch (Error& e) {
+		TraceEvent(SevWarnAlways, "TestRestoreCheckpointError")
+		    .errorUnsuppressed(e)
+		    .detail("Checkpoint", checkpoint.toString());
+		throw;
+	}
+
+	state int i = 0;
+	for (; i < ranges.size(); ++i) {
+		state RangeResult restoreResult = wait(restoreKv->readRange(ranges[i]));
+		RangeResult result = wait(kvStore->readRange(ranges[i]));
+		ASSERT(!restoreResult.more && !result.more);
+		ASSERT(restoreResult.size() == result.size());
+		for (int i = 0; i < result.size(); ++i) {
+			TraceEvent(SevDebug, "ReadKeyValueFromRestoredRocks")
+			    .detail("Key", result[i].key)
+			    .detail("Value", result[i].value)
+			    .detail("RestoreKey", restoreResult[i].key)
+			    .detail("RestoreValue", restoreResult[i].value);
+			ASSERT(result[i].key == restoreResult[i].key);
+			ASSERT(result[i].value == restoreResult[i].value);
+		}
+	}
+
+	Future<Void> restoreKvClose = restoreKv->onClosed();
+	restoreKv->close();
+	wait(restoreKvClose);
+	return Void();
+}
+} // namespace
+
+#endif // SSD_ROCKSDB_EXPERIMENTAL
+
+IKeyValueStore* keyValueStoreShardedRocksDB(std::string const& path,
+                                            UID logID,
+                                            KeyValueStoreType storeType,
+                                            bool checkChecksums,
+                                            bool checkIntegrity) {
+#ifdef SSD_ROCKSDB_EXPERIMENTAL
+	return new ShardedRocksDBKeyValueStore(path, logID);
+#else
+	TraceEvent(SevError, "ShardedRocksDBEngineInitFailure").detail("Reason", "Built without RocksDB");
+	ASSERT(false);
+	return nullptr;
+#endif // SSD_ROCKSDB_EXPERIMENTAL
+}
+
+#ifdef SSD_ROCKSDB_EXPERIMENTAL
+#include "flow/UnitTest.h"
+
+namespace {
+TEST_CASE("noSim/ShardedRocksDB/Initialization") {
+	state const std::string rocksDBTestDir = "sharded-rocksdb-test-db";
+	platform::eraseDirectoryRecursive(rocksDBTestDir);
+
+	state IKeyValueStore* kvStore =
+	    new ShardedRocksDBKeyValueStore(rocksDBTestDir, deterministicRandom()->randomUniqueID());
+	wait(kvStore->init());
+
+	Future<Void> closed = kvStore->onClosed();
+	kvStore->dispose();
+	wait(closed);
+	return Void();
+}
+
+TEST_CASE("noSim/ShardedRocksDB/SingleShardRead") {
+	state const std::string rocksDBTestDir = "sharded-rocksdb-test-db";
+	platform::eraseDirectoryRecursive(rocksDBTestDir);
+
+	state IKeyValueStore* kvStore =
+	    new ShardedRocksDBKeyValueStore(rocksDBTestDir, deterministicRandom()->randomUniqueID());
+	wait(kvStore->init());
+
+	KeyRangeRef range("a"_sr, "b"_sr);
+	wait(kvStore->addRange(range, "shard-1"));
+
+	kvStore->set({ "a"_sr, "foo"_sr });
+	kvStore->set({ "ac"_sr, "bar"_sr });
+	wait(kvStore->commit(false));
+
+	Optional<Value> val = wait(kvStore->readValue("a"_sr));
+	ASSERT(Optional<Value>("foo"_sr) == val);
+	{
+		Optional<Value> val = wait(kvStore->readValue("ac"_sr));
+		ASSERT(Optional<Value>("bar"_sr) == val);
+	}
+
+	Future<Void> closed = kvStore->onClosed();
+	kvStore->dispose();
+	wait(closed);
+	return Void();
+}
+
+TEST_CASE("noSim/ShardedRocksDB/RangeOps") {
+	state std::string rocksDBTestDir = "sharded-rocksdb-kvs-test-db";
+	platform::eraseDirectoryRecursive(rocksDBTestDir);
+
+	state IKeyValueStore* kvStore =
+	    new ShardedRocksDBKeyValueStore(rocksDBTestDir, deterministicRandom()->randomUniqueID());
+	wait(kvStore->init());
+
+	std::vector<Future<Void>> addRangeFutures;
+	addRangeFutures.push_back(kvStore->addRange(KeyRangeRef("0"_sr, "3"_sr), "shard-1"));
+	addRangeFutures.push_back(kvStore->addRange(KeyRangeRef("4"_sr, "7"_sr), "shard-2"));
+
+	wait(waitForAll(addRangeFutures));
+
+	kvStore->persistRangeMapping(KeyRangeRef("0"_sr, "7"_sr), true);
+
+	// write to shard 1
+	state RangeResult expectedRows;
+	for (int i = 0; i < 30; ++i) {
+		std::string key = format("%02d", i);
+		std::string value = std::to_string(i);
+		kvStore->set({ key, value });
+		expectedRows.push_back_deep(expectedRows.arena(), { key, value });
+	}
+
+	// write to shard 2
+	for (int i = 40; i < 70; ++i) {
+		std::string key = format("%02d", i);
+		std::string value = std::to_string(i);
+		kvStore->set({ key, value });
+		expectedRows.push_back_deep(expectedRows.arena(), { key, value });
+	}
+
+	wait(kvStore->commit(false));
+	Future<Void> closed = kvStore->onClosed();
+	kvStore->close();
+	wait(closed);
+	kvStore = new ShardedRocksDBKeyValueStore(rocksDBTestDir, deterministicRandom()->randomUniqueID());
+	wait(kvStore->init());
+
+	// Point read
+	state int i = 0;
+	for (i = 0; i < expectedRows.size(); ++i) {
+		Optional<Value> val = wait(kvStore->readValue(expectedRows[i].key));
+		ASSERT(val == Optional<Value>(expectedRows[i].value));
+	}
+
+	// Range read
+	// Read forward full range.
+	RangeResult result = wait(kvStore->readRange(KeyRangeRef("0"_sr, ":"_sr), 1000, 10000));
+	ASSERT_EQ(result.size(), expectedRows.size());
+	for (int i = 0; i < expectedRows.size(); ++i) {
+		ASSERT(result[i] == expectedRows[i]);
+	}
+
+	// Read backward full range.
+	{
+		RangeResult result = wait(kvStore->readRange(KeyRangeRef("0"_sr, ":"_sr), -1000, 10000));
+		ASSERT_EQ(result.size(), expectedRows.size());
+		for (int i = 0; i < expectedRows.size(); ++i) {
+			ASSERT(result[i] == expectedRows[59 - i]);
+		}
+	}
+
+	// Forward with row limit.
+	{
+		RangeResult result = wait(kvStore->readRange(KeyRangeRef("2"_sr, "6"_sr), 10, 10000));
+		ASSERT_EQ(result.size(), 10);
+		for (int i = 0; i < 10; ++i) {
+			ASSERT(result[i] == expectedRows[20 + i]);
+		}
+	}
+
+	// Add another range on shard-1.
+	wait(kvStore->addRange(KeyRangeRef("7"_sr, "9"_sr), "shard-1"));
+	kvStore->persistRangeMapping(KeyRangeRef("7"_sr, "9"_sr), true);
+
+	for (i = 70; i < 90; ++i) {
+		std::string key = format("%02d", i);
+		std::string value = std::to_string(i);
+		kvStore->set({ key, value });
+		expectedRows.push_back_deep(expectedRows.arena(), { key, value });
+	}
+
+	wait(kvStore->commit(false));
+
+	{
+		Future<Void> closed = kvStore->onClosed();
+		kvStore->close();
+		wait(closed);
+	}
+	kvStore = new ShardedRocksDBKeyValueStore(rocksDBTestDir, deterministicRandom()->randomUniqueID());
+	wait(kvStore->init());
+
+	// Read all values.
+	{
+		RangeResult result = wait(kvStore->readRange(KeyRangeRef("0"_sr, ":"_sr), 1000, 10000));
+		ASSERT_EQ(result.size(), expectedRows.size());
+		for (int i = 0; i < expectedRows.size(); ++i) {
+			ASSERT(result[i] == expectedRows[i]);
+		}
+	}
+
+	// Read partial range with row limit
+	{
+		RangeResult result = wait(kvStore->readRange(KeyRangeRef("5"_sr, ":"_sr), 35, 10000));
+		ASSERT_EQ(result.size(), 35);
+		for (int i = 0; i < result.size(); ++i) {
+			ASSERT(result[i] == expectedRows[40 + i]);
+		}
+	}
+
+	// Clear a range on a single shard.
+	kvStore->clear(KeyRangeRef("40"_sr, "45"_sr));
+	wait(kvStore->commit(false));
+
+	{
+		RangeResult result = wait(kvStore->readRange(KeyRangeRef("4"_sr, "5"_sr), 20, 10000));
+		ASSERT_EQ(result.size(), 5);
+	}
+
+	// Clear a single value.
+	kvStore->clear(KeyRangeRef("01"_sr, keyAfter("01"_sr)));
+	wait(kvStore->commit(false));
+
+	Optional<Value> val = wait(kvStore->readValue("01"_sr));
+	ASSERT(!val.present());
+
+	// Clear a range spanning on multiple shards.
+	kvStore->clear(KeyRangeRef("1"_sr, "8"_sr));
+	wait(kvStore->commit(false));
+
+	{
+		Future<Void> closed = kvStore->onClosed();
+		kvStore->close();
+		wait(closed);
+	}
+	kvStore = new ShardedRocksDBKeyValueStore(rocksDBTestDir, deterministicRandom()->randomUniqueID());
+	wait(kvStore->init());
+
+	{
+		RangeResult result = wait(kvStore->readRange(KeyRangeRef("1"_sr, "8"_sr), 1000, 10000));
+		ASSERT_EQ(result.size(), 0);
+	}
+
+	{
+		RangeResult result = wait(kvStore->readRange(KeyRangeRef("0"_sr, ":"_sr), 1000, 10000));
+		ASSERT_EQ(result.size(), 19);
+	}
+
+	{
+		Future<Void> closed = kvStore->onClosed();
+		kvStore->dispose();
+		wait(closed);
+	}
+
+	return Void();
+}
+
+TEST_CASE("noSim/ShardedRocksDB/ShardOps") {
+	state std::string rocksDBTestDir = "sharded-rocksdb-kvs-test-db";
+	platform::eraseDirectoryRecursive(rocksDBTestDir);
+
+	state ShardedRocksDBKeyValueStore* rocksdbStore =
+	    new ShardedRocksDBKeyValueStore(rocksDBTestDir, deterministicRandom()->randomUniqueID());
+	state IKeyValueStore* kvStore = rocksdbStore;
+	wait(kvStore->init());
+
+	// Add some ranges.
+	{
+		std::vector<Future<Void>> addRangeFutures;
+		addRangeFutures.push_back(kvStore->addRange(KeyRangeRef("a"_sr, "c"_sr), "shard-1"));
+		addRangeFutures.push_back(kvStore->addRange(KeyRangeRef("c"_sr, "f"_sr), "shard-2"));
+
+		wait(waitForAll(addRangeFutures));
+	}
+
+	{
+		std::vector<Future<Void>> addRangeFutures;
+		addRangeFutures.push_back(kvStore->addRange(KeyRangeRef("x"_sr, "z"_sr), "shard-1"));
+		addRangeFutures.push_back(kvStore->addRange(KeyRangeRef("l"_sr, "n"_sr), "shard-3"));
+
+		wait(waitForAll(addRangeFutures));
+	}
+
+	// Remove single range.
+	std::vector<std::string> shardsToCleanUp;
+	auto shardIds = kvStore->removeRange(KeyRangeRef("b"_sr, "c"_sr));
+	// Remove range didn't create empty shard.
+	ASSERT_EQ(shardIds.size(), 0);
+
+	// Remove range spanning on multiple shards.
+	shardIds = kvStore->removeRange(KeyRangeRef("c"_sr, "m"_sr));
+	sort(shardIds.begin(), shardIds.end());
+	int count = std::unique(shardIds.begin(), shardIds.end()) - shardIds.begin();
+	ASSERT_EQ(count, 1);
+	ASSERT(shardIds[0] == "shard-2");
+
+	// Add more ranges.
+	std::vector<Future<Void>> addRangeFutures;
+	addRangeFutures.push_back(kvStore->addRange(KeyRangeRef("b"_sr, "g"_sr), "shard-1"));
+	addRangeFutures.push_back(kvStore->addRange(KeyRangeRef("l"_sr, "m"_sr), "shard-2"));
+	addRangeFutures.push_back(kvStore->addRange(KeyRangeRef("u"_sr, "v"_sr), "shard-3"));
+
+	wait(waitForAll(addRangeFutures));
+
+	auto dataMap = rocksdbStore->getDataMapping();
+	state std::vector<std::pair<KeyRange, std::string>> mapping;
+	mapping.push_back(std::make_pair(KeyRange(KeyRangeRef("a"_sr, "b"_sr)), "shard-1"));
+	mapping.push_back(std::make_pair(KeyRange(KeyRangeRef("b"_sr, "g"_sr)), "shard-1"));
+	mapping.push_back(std::make_pair(KeyRange(KeyRangeRef("l"_sr, "m"_sr)), "shard-2"));
+	mapping.push_back(std::make_pair(KeyRange(KeyRangeRef("m"_sr, "n"_sr)), "shard-3"));
+	mapping.push_back(std::make_pair(KeyRange(KeyRangeRef("u"_sr, "v"_sr)), "shard-3"));
+	mapping.push_back(std::make_pair(KeyRange(KeyRangeRef("x"_sr, "z"_sr)), "shard-1"));
+	mapping.push_back(std::make_pair(specialKeys, DEFAULT_CF_NAME));
+
+	for (auto it = dataMap.begin(); it != dataMap.end(); ++it) {
+		std::cout << "Begin " << it->first.begin.toString() << ", End " << it->first.end.toString() << ", id "
+		          << it->second << "\n";
+	}
+	ASSERT(dataMap == mapping);
+
+	kvStore->persistRangeMapping(KeyRangeRef("a"_sr, "z"_sr), true);
+	wait(kvStore->commit(false));
+
+	// Restart.
+	Future<Void> closed = kvStore->onClosed();
+	kvStore->close();
+	wait(closed);
+
+	rocksdbStore = new ShardedRocksDBKeyValueStore(rocksDBTestDir, deterministicRandom()->randomUniqueID());
+	kvStore = rocksdbStore;
+	wait(kvStore->init());
+
+	{
+		auto dataMap = rocksdbStore->getDataMapping();
+		for (auto it = dataMap.begin(); it != dataMap.end(); ++it) {
+			std::cout << "Begin " << it->first.begin.toString() << ", End " << it->first.end.toString() << ", id "
+			          << it->second << "\n";
+		}
+		ASSERT(dataMap == mapping);
+	}
+
+	// Remove all the ranges.
+	{
+		state std::vector<std::string> shardsToCleanUp = kvStore->removeRange(KeyRangeRef("a"_sr, "z"_sr));
+		ASSERT_EQ(shardsToCleanUp.size(), 3);
+
+		// Add another range to shard-2.
+		wait(kvStore->addRange(KeyRangeRef("h"_sr, "i"_sr), "shard-2"));
+	}
+	{
+		auto dataMap = rocksdbStore->getDataMapping();
+		ASSERT_EQ(dataMap.size(), 2);
+		ASSERT(dataMap[0].second == "shard-2");
+	}
+
+	{
+		Future<Void> closed = kvStore->onClosed();
+		kvStore->dispose();
+		wait(closed);
+	}
+	return Void();
+}
+
+TEST_CASE("noSim/ShardedRocksDB/Metadata") {
+	state std::string rocksDBTestDir = "sharded-rocksdb-kvs-test-db";
+	state Key testSpecialKey = "\xff\xff/TestKey"_sr;
+	state Value testSpecialValue = "\xff\xff/TestValue"_sr;
+	platform::eraseDirectoryRecursive(rocksDBTestDir);
+
+	state ShardedRocksDBKeyValueStore* rocksdbStore =
+	    new ShardedRocksDBKeyValueStore(rocksDBTestDir, deterministicRandom()->randomUniqueID());
+	state IKeyValueStore* kvStore = rocksdbStore;
+	wait(kvStore->init());
+
+	Optional<Value> val = wait(kvStore->readValue(testSpecialKey));
+	ASSERT(!val.present());
+
+	kvStore->set(KeyValueRef(testSpecialKey, testSpecialValue));
+	wait(kvStore->commit(false));
+
+	{
+		Optional<Value> val = wait(kvStore->readValue(testSpecialKey));
+		ASSERT(val.get() == testSpecialValue);
+	}
+
+	// Add some ranges.
+	std::vector<Future<Void>> addRangeFutures;
+	addRangeFutures.push_back(kvStore->addRange(KeyRangeRef("a"_sr, "c"_sr), "shard-1"));
+	addRangeFutures.push_back(kvStore->addRange(KeyRangeRef("c"_sr, "f"_sr), "shard-2"));
+	kvStore->persistRangeMapping(KeyRangeRef("a"_sr, "f"_sr), true);
+
+	wait(waitForAll(addRangeFutures));
+	kvStore->set(KeyValueRef("a1"_sr, "foo"_sr));
+	kvStore->set(KeyValueRef("d1"_sr, "bar"_sr));
+	wait(kvStore->commit(false));
+
+	// Restart.
+	Future<Void> closed = kvStore->onClosed();
+	kvStore->close();
+	wait(closed);
+	rocksdbStore = new ShardedRocksDBKeyValueStore(rocksDBTestDir, deterministicRandom()->randomUniqueID());
+	kvStore = rocksdbStore;
+	wait(kvStore->init());
+
+	{
+		Optional<Value> val = wait(kvStore->readValue(testSpecialKey));
+		ASSERT(val.get() == testSpecialValue);
+	}
+
+	// Read value back.
+	{
+		Optional<Value> val = wait(kvStore->readValue("a1"_sr));
+		ASSERT(val == Optional<Value>("foo"_sr));
+	}
+	{
+		Optional<Value> val = wait(kvStore->readValue("d1"_sr));
+		ASSERT(val == Optional<Value>("bar"_sr));
+	}
+
+	// Remove range containing a1.
+	kvStore->persistRangeMapping(KeyRangeRef("a"_sr, "b"_sr), false);
+	auto shardIds = kvStore->removeRange(KeyRangeRef("a"_sr, "b"_sr));
+	wait(kvStore->commit(false));
+
+	// Read a1.
+	{
+		Optional<Value> val = wait(kvStore->readValue("a1"_sr));
+		ASSERT(!val.present());
+	}
+
+	// Restart.
+	{
+		Future<Void> closed = kvStore->onClosed();
+		kvStore->close();
+		wait(closed);
+	}
+	rocksdbStore = new ShardedRocksDBKeyValueStore(rocksDBTestDir, deterministicRandom()->randomUniqueID());
+	kvStore = rocksdbStore;
+	wait(kvStore->init());
+
+	// Read again.
+	{
+		Optional<Value> val = wait(kvStore->readValue("a1"_sr));
+		ASSERT(!val.present());
+	}
+	{
+		Optional<Value> val = wait(kvStore->readValue("d1"_sr));
+		ASSERT(val == Optional<Value>("bar"_sr));
+	}
+
+	auto mapping = rocksdbStore->getDataMapping();
+	ASSERT(mapping.size() == 3);
+
+	// Remove all the ranges.
+	kvStore->removeRange(KeyRangeRef("a"_sr, "f"_sr));
+	mapping = rocksdbStore->getDataMapping();
+	ASSERT(mapping.size() == 1);
+
+	// Restart.
+	{
+		Future<Void> closed = kvStore->onClosed();
+		kvStore->close();
+		wait(closed);
+	}
+	rocksdbStore = new ShardedRocksDBKeyValueStore(rocksDBTestDir, deterministicRandom()->randomUniqueID());
+	kvStore = rocksdbStore;
+	wait(kvStore->init());
+
+	// Because range metadata was not committed, ranges should be restored.
+	{
+		auto mapping = rocksdbStore->getDataMapping();
+		ASSERT(mapping.size() == 3);
+
+		// Remove ranges again.
+		kvStore->persistRangeMapping(KeyRangeRef("a"_sr, "f"_sr), false);
+		kvStore->removeRange(KeyRangeRef("a"_sr, "f"_sr));
+
+		mapping = rocksdbStore->getDataMapping();
+		ASSERT(mapping.size() == 1);
+
+		wait(kvStore->commit(false));
+	}
+
+	// Restart.
+	{
+		Future<Void> closed = kvStore->onClosed();
+		kvStore->close();
+		wait(closed);
+	}
+
+	rocksdbStore = new ShardedRocksDBKeyValueStore(rocksDBTestDir, deterministicRandom()->randomUniqueID());
+	kvStore = rocksdbStore;
+	wait(kvStore->init());
+
+	// No range available.
+	{
+		auto mapping = rocksdbStore->getDataMapping();
+		for (auto it = mapping.begin(); it != mapping.end(); ++it) {
+			std::cout << "Begin " << it->first.begin.toString() << ", End " << it->first.end.toString() << ", id "
+			          << it->second << "\n";
+		}
+		ASSERT(mapping.size() == 1);
+	}
+
+	{
+		Future<Void> closed = kvStore->onClosed();
+		kvStore->dispose();
+		wait(closed);
+	}
+
+	return Void();
+}
+
+TEST_CASE("noSim/ShardedRocksDB/CheckpointBasic") {
+	state std::string rocksDBTestDir = "sharded-rocks-checkpoint-restore";
+	state std::map<Key, Value> kvs({ { "a"_sr, "TestValueA"_sr },
+	                                 { "ab"_sr, "TestValueAB"_sr },
+	                                 { "ad"_sr, "TestValueAD"_sr },
+	                                 { "b"_sr, "TestValueB"_sr },
+	                                 { "ba"_sr, "TestValueBA"_sr },
+	                                 { "c"_sr, "TestValueC"_sr },
+	                                 { "d"_sr, "TestValueD"_sr },
+	                                 { "e"_sr, "TestValueE"_sr },
+	                                 { "h"_sr, "TestValueH"_sr },
+	                                 { "ha"_sr, "TestValueHA"_sr } });
+	platform::eraseDirectoryRecursive(rocksDBTestDir);
+	state IKeyValueStore* kvStore =
+	    new ShardedRocksDBKeyValueStore(rocksDBTestDir, deterministicRandom()->randomUniqueID());
+	wait(kvStore->init());
+
+	// Add some ranges.
+	std::vector<Future<Void>> addRangeFutures;
+	addRangeFutures.push_back(kvStore->addRange(KeyRangeRef("a"_sr, "c"_sr), "shard-1"));
+	addRangeFutures.push_back(kvStore->addRange(KeyRangeRef("c"_sr, "f"_sr), "shard-2"));
+	addRangeFutures.push_back(kvStore->addRange(KeyRangeRef("h"_sr, "k"_sr), "shard-1"));
+	kvStore->persistRangeMapping(KeyRangeRef("a"_sr, "f"_sr), true);
+	wait(waitForAll(addRangeFutures) && kvStore->commit(false));
+
+	for (const auto& [k, v] : kvs) {
+		kvStore->set(KeyValueRef(k, v));
+	}
+	wait(kvStore->commit(false));
+
+	state std::string checkpointDir = "checkpoint";
+	platform::eraseDirectoryRecursive(checkpointDir);
+
+	// Checkpoint iterator returns only the desired keyrange, i.e., ["ab", "b"].
+	CheckpointRequest request(latestVersion,
+	                          { KeyRangeRef("a"_sr, "c"_sr), KeyRangeRef("h"_sr, "k"_sr) },
+	                          DataMoveRocksCF,
+	                          deterministicRandom()->randomUniqueID(),
+	                          checkpointDir);
+	CheckpointMetaData metaData = wait(kvStore->checkpoint(request));
+
+	state Standalone<StringRef> token = BinaryWriter::toValue(KeyRangeRef("a"_sr, "k"_sr), IncludeVersion());
+	state ICheckpointReader* cpReader =
+	    newCheckpointReader(metaData, CheckpointAsKeyValues::True, deterministicRandom()->randomUniqueID());
+	ASSERT(cpReader != nullptr);
+	wait(cpReader->init(token));
+	state KeyRange testRange(KeyRangeRef("ab"_sr, "b"_sr));
+	state std::unique_ptr<ICheckpointIterator> iter0 = cpReader->getIterator(testRange);
+	state int numKeys = 0;
+	try {
+		loop {
+			RangeResult res = wait(iter0->nextBatch(CLIENT_KNOBS->REPLY_BYTE_LIMIT, CLIENT_KNOBS->REPLY_BYTE_LIMIT));
+			for (const auto& kv : res) {
+				ASSERT(testRange.contains(kv.key));
+				ASSERT(kvs[kv.key] == kv.value);
+				++numKeys;
+			}
+		}
+	} catch (Error& e) {
+		ASSERT(e.code() == error_code_end_of_stream);
+		ASSERT(numKeys == 2);
+	}
+
+	testRange = KeyRangeRef("a"_sr, "k"_sr);
+	state std::unique_ptr<ICheckpointIterator> iter1 = cpReader->getIterator(testRange);
+	try {
+		numKeys = 0;
+		loop {
+			RangeResult res = wait(iter1->nextBatch(CLIENT_KNOBS->REPLY_BYTE_LIMIT, CLIENT_KNOBS->REPLY_BYTE_LIMIT));
+			for (const auto& kv : res) {
+				ASSERT(testRange.contains(kv.key));
+				ASSERT(kvs[kv.key] == kv.value);
+				++numKeys;
+			}
+		}
+	} catch (Error& e) {
+		ASSERT(e.code() == error_code_end_of_stream);
+		ASSERT(numKeys == 7);
+	}
+
+	iter0.reset();
+	iter1.reset();
+	ASSERT(!cpReader->inUse());
+	TraceEvent(SevDebug, "ShardedRocksCheckpointReaaderTested");
+	std::vector<Future<Void>> closes;
+	closes.push_back(cpReader->close());
+	closes.push_back(kvStore->onClosed());
+	kvStore->dispose();
+	wait(waitForAll(closes));
+
+	platform::eraseDirectoryRecursive(rocksDBTestDir);
+	platform::eraseDirectoryRecursive(checkpointDir);
+
+	return Void();
+}
+
+TEST_CASE("noSim/ShardedRocksDB/CheckpointRestore") {
+	state std::string rocksDBTestDir = "sharded-rocks-checkpoint" + deterministicRandom()->randomAlphaNumeric(5);
+	state std::map<Key, Value> kvs({ { "ab"_sr, "TestValueAB"_sr },
+	                                 { "ad"_sr, "TestValueAD"_sr },
+	                                 { "b"_sr, "TestValueB"_sr },
+	                                 { "ba"_sr, "TestValueBA"_sr },
+	                                 { "c"_sr, "TestValueC"_sr },
+	                                 { "d"_sr, "TestValueD"_sr },
+	                                 { "e"_sr, "TestValueE"_sr },
+	                                 { "h"_sr, "TestValueH"_sr },
+	                                 { "ha"_sr, "TestValueHA"_sr } });
+	platform::eraseDirectoryRecursive(rocksDBTestDir);
+	state IKeyValueStore* kvStore =
+	    new ShardedRocksDBKeyValueStore(rocksDBTestDir, deterministicRandom()->randomUniqueID());
+	wait(kvStore->init());
+
+	state std::string shardId = "shard_1";
+	state std::string emptyShardId = "shard_2";
+	state KeyRangeRef rangeK(""_sr, "k"_sr);
+	state KeyRangeRef rangeKz("z1"_sr, "z3"_sr);
+	// // Add some ranges.
+	std::vector<Future<Void>> addRangeFutures;
+	addRangeFutures.push_back(kvStore->addRange(rangeK, shardId));
+	addRangeFutures.push_back(kvStore->addRange(rangeKz, emptyShardId));
+	kvStore->persistRangeMapping(rangeK, true);
+	kvStore->persistRangeMapping(rangeKz, true);
+	wait(waitForAll(addRangeFutures));
+	wait(kvStore->commit(false));
+
+	for (const auto& [k, v] : kvs) {
+		kvStore->set(KeyValueRef(k, v));
+	}
+	wait(kvStore->commit(false));
+	kvStore->clear(KeyRangeRef(""_sr, "z"_sr));
+	wait(kvStore->commit(false));
+
+	state Error err;
+	try {
+		wait(testCheckpointRestore(kvStore, { rangeK }));
+	} catch (Error& e) {
+		TraceEvent(SevError, "TestCheckpointRestoreError").errorUnsuppressed(e);
+		err = e;
+	}
+	// This will fail once RocksDB is upgraded to 8.1.
+	// ASSERT(err.code() == error_code_failed_to_restore_checkpoint);
+
+	try {
+		wait(testCheckpointRestore(kvStore, { rangeKz }));
+	} catch (Error& e) {
+		TraceEvent("TestCheckpointRestoreError").errorUnsuppressed(e);
+		err = e;
+	}
+
+	std::vector<Future<Void>> closes;
+	closes.push_back(kvStore->onClosed());
+	kvStore->close();
+	wait(waitForAll(closes));
+
+	platform::eraseDirectoryRecursive(rocksDBTestDir);
+
+	return Void();
+}
+
+TEST_CASE("noSim/ShardedRocksDB/RocksDBSstFileWriter") {
+	state std::string localFile = "rocksdb-sst-file-dump.sst";
+	state std::unique_ptr<IRocksDBSstFileWriter> sstWriter = newRocksDBSstFileWriter();
+	// Write nothing to sst file
+	sstWriter->open(localFile);
+	bool anyFileCreated = sstWriter->finish();
+	ASSERT(!anyFileCreated);
+	// Write kvs1 to sst file
+	state std::map<Key, Value> kvs1({ { "a"_sr, "1"_sr },
+	                                  { "ab"_sr, "12"_sr },
+	                                  { "ad"_sr, "14"_sr },
+	                                  { "b"_sr, "2"_sr },
+	                                  { "ba"_sr, "21"_sr },
+	                                  { "c"_sr, "3"_sr },
+	                                  { "d"_sr, "4"_sr },
+	                                  { "e"_sr, "5"_sr },
+	                                  { "h"_sr, "8"_sr },
+	                                  { "ha"_sr, "81"_sr } });
+	sstWriter = newRocksDBSstFileWriter();
+	sstWriter->open(localFile);
+	for (const auto& [key, value] : kvs1) {
+		sstWriter->write(key, value);
+	}
+	anyFileCreated = sstWriter->finish();
+	ASSERT(anyFileCreated);
+	// Write kvs2 to the same sst file where kvs2 keys are different from kvs1
+	state std::map<Key, Value> kvs2({ { "fa"_sr, "61"_sr },
+	                                  { "fab"_sr, "612"_sr },
+	                                  { "fad"_sr, "614"_sr },
+	                                  { "fb"_sr, "62"_sr },
+	                                  { "fba"_sr, "621"_sr },
+	                                  { "fc"_sr, "63"_sr },
+	                                  { "fd"_sr, "64"_sr },
+	                                  { "fe"_sr, "65"_sr },
+	                                  { "fh"_sr, "68"_sr },
+	                                  { "fha"_sr, "681"_sr } });
+	sstWriter->open(localFile);
+	for (const auto& [key, value] : kvs2) {
+		sstWriter->write(key, value);
+	}
+	anyFileCreated = sstWriter->finish();
+	ASSERT(anyFileCreated);
+	// Write kvs3 to the same sst file where kvs3 modifies values of kvs2
+	state std::map<Key, Value> kvs3({ { "fa"_sr, "1"_sr },
+	                                  { "fab"_sr, "12"_sr },
+	                                  { "fad"_sr, "14"_sr },
+	                                  { "fb"_sr, "2"_sr },
+	                                  { "fba"_sr, "21"_sr },
+	                                  { "fc"_sr, "3"_sr },
+	                                  { "fd"_sr, "4"_sr },
+	                                  { "fe"_sr, "5"_sr },
+	                                  { "fh"_sr, "8"_sr },
+	                                  { "fha"_sr, "81"_sr } });
+	sstWriter->open(localFile);
+	for (const auto& [key, value] : kvs3) {
+		sstWriter->write(key, value);
+	}
+	anyFileCreated = sstWriter->finish();
+	ASSERT(anyFileCreated);
+	// Check: sst only contains kv of kvs3
+	rocksdb::Status status;
+	rocksdb::IngestExternalFileOptions ingestOptions;
+	rocksdb::DB* db;
+	rocksdb::Options options;
+	options.create_if_missing = true;
+	status = rocksdb::DB::Open(options, "testdb", &db);
+	ASSERT(status.ok());
+	status = db->IngestExternalFile({ localFile }, ingestOptions);
+	ASSERT(status.ok());
+	std::string value;
+	for (const auto& [key, targetValue] : kvs1) {
+		status = db->Get(rocksdb::ReadOptions(), key.toString(), &value);
+		ASSERT(status.IsNotFound());
+	}
+	for (const auto& [key, targetValue] : kvs2) {
+		status = db->Get(rocksdb::ReadOptions(), key.toString(), &value);
+		ASSERT(value != targetValue.toString());
+	}
+	for (const auto& [key, targetValue] : kvs3) {
+		status = db->Get(rocksdb::ReadOptions(), key.toString(), &value);
+		ASSERT(status.ok());
+		ASSERT(value == targetValue.toString());
+	}
+	delete db;
+	return Void();
+}
+
+} // namespace
+
+#endif // SSD_ROCKSDB_EXPERIMENTAL
diff --git a/fdbserver/RocksDBCheckpointUtils.actor.h b/fdbserver/RocksDBCheckpointUtils.actor.h
index 23979f22e08..211e6430c75 100644
--- a/fdbserver/RocksDBCheckpointUtils.actor.h
+++ b/fdbserver/RocksDBCheckpointUtils.actor.h
@@ -129,6 +129,12 @@ struct SstFileMetaData {
 	// DEPRECATED: replaced by `directory` in base struct
 	std::string db_path;
 
+	// These bounds define the effective key range for range tombstones
+	// in this file.
+	// Currently only used by CreateColumnFamilyWithImport().
+	std::string smallest{}; // Smallest internal key served by table
+	std::string largest{}; // Largest internal key served by table
+
 	template <class Ar>
 	void serialize(Ar& ar) {
 		serializer(ar,
@@ -153,7 +159,9 @@ struct SstFileMetaData {
 		           file_creation_time,
 		           epoch_number,
 		           name,
-		           db_path);
+		           db_path,
+		           smallest,
+		           largest);
 	}
 };
 
@@ -192,7 +200,9 @@ struct LiveFileMetaData : public SstFileMetaData {
 		           SstFileMetaData::db_path,
 		           column_family_name,
 		           level,
-		           fetched);
+		           fetched,
+		           SstFileMetaData::smallest,
+		           SstFileMetaData::largest);
 	}
 };
 

From 4aa13662e97931fb81957fdf70e802f18339e81c Mon Sep 17 00:00:00 2001
From: He Liu <heliu05023@gmail.com>
Date: Wed, 17 May 2023 17:01:08 -0700
Subject: [PATCH 158/468] Added FDBRocksDBVersion.h.

---
 fdbserver/CMakeLists.txt | 1 +
 1 file changed, 1 insertion(+)

diff --git a/fdbserver/CMakeLists.txt b/fdbserver/CMakeLists.txt
index fdbe48ec414..d08d120dabb 100644
--- a/fdbserver/CMakeLists.txt
+++ b/fdbserver/CMakeLists.txt
@@ -43,6 +43,7 @@ set(FDBSERVER_SRCS
   EncryptKeyProxyInterface.h
   FDBExecHelper.actor.cpp
   FDBExecHelper.actor.h
+  FDBRocksDBVersion.h
   fdbserver.actor.cpp
   GrvProxyServer.actor.cpp
   IConfigConsumer.cpp

From 3cad363cf58479818b5841fd332e36c30aa98a8a Mon Sep 17 00:00:00 2001
From: He Liu <heliu05023@gmail.com>
Date: Wed, 17 May 2023 17:05:18 -0700
Subject: [PATCH 159/468] Removed KeyValueStoreShardedRocksDB.actor.cpp

---
 .../KeyValueStoreShardedRocksDB.actor.cpp     | 4120 -----------------
 1 file changed, 4120 deletions(-)
 delete mode 100644 fdbserver/KeyValueStoreShardedRocksDB.actor.cpp

diff --git a/fdbserver/KeyValueStoreShardedRocksDB.actor.cpp b/fdbserver/KeyValueStoreShardedRocksDB.actor.cpp
deleted file mode 100644
index 694862d64c4..00000000000
--- a/fdbserver/KeyValueStoreShardedRocksDB.actor.cpp
+++ /dev/null
@@ -1,4120 +0,0 @@
-#include "fdbclient/FDBTypes.h"
-#ifdef SSD_ROCKSDB_EXPERIMENTAL
-
-#include "fdbclient/KeyRangeMap.h"
-#include "fdbclient/SystemData.h"
-#include "flow/flow.h"
-#include "flow/serialize.h"
-#include <rocksdb/c.h>
-#include <rocksdb/cache.h>
-#include <rocksdb/db.h>
-#include <rocksdb/filter_policy.h>
-#include <rocksdb/listener.h>
-#include <rocksdb/metadata.h>
-#include <rocksdb/options.h>
-#include <rocksdb/perf_context.h>
-#include <rocksdb/rate_limiter.h>
-#include <rocksdb/slice_transform.h>
-#include <rocksdb/statistics.h>
-#include <rocksdb/table.h>
-#include <rocksdb/utilities/checkpoint.h>
-#include <rocksdb/utilities/table_properties_collectors.h>
-#include <rocksdb/version.h>
-#if defined __has_include
-#if __has_include(<liburing.h>)
-#include <liburing.h>
-#endif
-#endif
-#include "fdbclient/SystemData.h"
-#include "fdbserver/CoroFlow.h"
-#include "fdbserver/FDBRocksDBVersion.h"
-#include "flow/flow.h"
-#include "flow/IThreadPool.h"
-#include "flow/ThreadHelper.actor.h"
-#include "flow/Histogram.h"
-
-#include <memory>
-#include <tuple>
-#include <vector>
-
-#endif // SSD_ROCKSDB_EXPERIMENTAL
-
-#include "fdbserver/IKeyValueStore.h"
-#include "fdbserver/RocksDBCheckpointUtils.actor.h"
-#include "flow/actorcompiler.h" // has to be last include
-
-#ifdef SSD_ROCKSDB_EXPERIMENTAL
-
-// Enforcing rocksdb version.
-static_assert((ROCKSDB_MAJOR == FDB_ROCKSDB_MAJOR && ROCKSDB_MINOR == FDB_ROCKSDB_MINOR &&
-               ROCKSDB_PATCH == FDB_ROCKSDB_PATCH),
-              "Unsupported rocksdb version.");
-
-const std::string rocksDataFolderSuffix = "-data";
-const std::string METADATA_SHARD_ID = "kvs-metadata";
-const std::string DEFAULT_CF_NAME = "default"; // `specialKeys` is stored in this culoumn family.
-const KeyRef shardMappingPrefix("\xff\xff/ShardMapping/"_sr);
-// TODO: move constants to a header file.
-const KeyRef persistVersion = "\xff\xffVersion"_sr;
-const StringRef ROCKSDBSTORAGE_HISTOGRAM_GROUP = "RocksDBStorage"_sr;
-const StringRef ROCKSDB_COMMIT_LATENCY_HISTOGRAM = "RocksDBCommitLatency"_sr;
-const StringRef ROCKSDB_COMMIT_ACTION_HISTOGRAM = "RocksDBCommitAction"_sr;
-const StringRef ROCKSDB_COMMIT_QUEUEWAIT_HISTOGRAM = "RocksDBCommitQueueWait"_sr;
-const StringRef ROCKSDB_WRITE_HISTOGRAM = "RocksDBWrite"_sr;
-const StringRef ROCKSDB_DELETE_COMPACTRANGE_HISTOGRAM = "RocksDBDeleteCompactRange"_sr;
-const StringRef ROCKSDB_READRANGE_LATENCY_HISTOGRAM = "RocksDBReadRangeLatency"_sr;
-const StringRef ROCKSDB_READVALUE_LATENCY_HISTOGRAM = "RocksDBReadValueLatency"_sr;
-const StringRef ROCKSDB_READPREFIX_LATENCY_HISTOGRAM = "RocksDBReadPrefixLatency"_sr;
-const StringRef ROCKSDB_READRANGE_ACTION_HISTOGRAM = "RocksDBReadRangeAction"_sr;
-const StringRef ROCKSDB_READVALUE_ACTION_HISTOGRAM = "RocksDBReadValueAction"_sr;
-const StringRef ROCKSDB_READPREFIX_ACTION_HISTOGRAM = "RocksDBReadPrefixAction"_sr;
-const StringRef ROCKSDB_READRANGE_QUEUEWAIT_HISTOGRAM = "RocksDBReadRangeQueueWait"_sr;
-const StringRef ROCKSDB_READVALUE_QUEUEWAIT_HISTOGRAM = "RocksDBReadValueQueueWait"_sr;
-const StringRef ROCKSDB_READPREFIX_QUEUEWAIT_HISTOGRAM = "RocksDBReadPrefixQueueWait"_sr;
-const StringRef ROCKSDB_READRANGE_NEWITERATOR_HISTOGRAM = "RocksDBReadRangeNewIterator"_sr;
-const StringRef ROCKSDB_READVALUE_GET_HISTOGRAM = "RocksDBReadValueGet"_sr;
-const StringRef ROCKSDB_READPREFIX_GET_HISTOGRAM = "RocksDBReadPrefixGet"_sr;
-
-namespace {
-struct PhysicalShard;
-struct DataShard;
-struct ReadIterator;
-struct ShardedRocksDBKeyValueStore;
-
-using rocksdb::BackgroundErrorReason;
-
-// Returns string representation of RocksDB background error reason.
-// Error reason code:
-// https://github.com/facebook/rocksdb/blob/12d798ac06bcce36be703b057d5f5f4dab3b270c/include/rocksdb/listener.h#L125
-// This function needs to be updated when error code changes.
-std::string getErrorReason(BackgroundErrorReason reason) {
-	switch (reason) {
-	case BackgroundErrorReason::kFlush:
-		return format("%d Flush", reason);
-	case BackgroundErrorReason::kCompaction:
-		return format("%d Compaction", reason);
-	case BackgroundErrorReason::kWriteCallback:
-		return format("%d WriteCallback", reason);
-	case BackgroundErrorReason::kMemTable:
-		return format("%d MemTable", reason);
-	case BackgroundErrorReason::kManifestWrite:
-		return format("%d ManifestWrite", reason);
-	case BackgroundErrorReason::kFlushNoWAL:
-		return format("%d FlushNoWAL", reason);
-	case BackgroundErrorReason::kManifestWriteNoWAL:
-		return format("%d ManifestWriteNoWAL", reason);
-	default:
-		return format("%d Unknown", reason);
-	}
-}
-
-// Background error handling is tested with Chaos test.
-// TODO: Test background error in simulation. RocksDB doesn't use flow IO in simulation, which limits our ability to
-// inject IO errors. We could implement rocksdb::FileSystem using flow IO to unblock simulation. Also, trace event is
-// not available on background threads because trace event requires setting up special thread locals. Using trace event
-// could potentially cause segmentation fault.
-class RocksDBErrorListener : public rocksdb::EventListener {
-public:
-	RocksDBErrorListener(){};
-	void OnBackgroundError(rocksdb::BackgroundErrorReason reason, rocksdb::Status* bg_error) override {
-		TraceEvent(SevError, "ShardedRocksDBBGError")
-		    .detail("Reason", getErrorReason(reason))
-		    .detail("ShardedRocksDBSeverity", bg_error->severity())
-		    .detail("Status", bg_error->ToString());
-		std::unique_lock<std::mutex> lock(mutex);
-		if (!errorPromise.isValid())
-			return;
-		// RocksDB generates two types of background errors, IO Error and Corruption
-		// Error type and severity map could be found at
-		// https://github.com/facebook/rocksdb/blob/2e09a54c4fb82e88bcaa3e7cfa8ccbbbbf3635d5/db/error_handler.cc#L138.
-		// All background errors will be treated as storage engine failure. Send the error to storage server.
-		if (bg_error->IsIOError()) {
-			errorPromise.sendError(io_error());
-		} else if (bg_error->IsCorruption()) {
-			errorPromise.sendError(file_corrupt());
-		} else {
-			errorPromise.sendError(unknown_error());
-		}
-	}
-	Future<Void> getFuture() {
-		std::unique_lock<std::mutex> lock(mutex);
-		return errorPromise.getFuture();
-	}
-	~RocksDBErrorListener() {
-		std::unique_lock<std::mutex> lock(mutex);
-		if (!errorPromise.isValid())
-			return;
-		errorPromise.send(Never());
-	}
-
-private:
-	ThreadReturnPromise<Void> errorPromise;
-	std::mutex mutex;
-};
-
-// Encapsulation of shared states.
-struct ShardedRocksDBState {
-	bool closing = false;
-};
-
-std::shared_ptr<rocksdb::Cache> rocksdb_block_cache = nullptr;
-
-rocksdb::ExportImportFilesMetaData getMetaData(const CheckpointMetaData& checkpoint) {
-	rocksdb::ExportImportFilesMetaData metaData;
-	if (checkpoint.getFormat() != DataMoveRocksCF) {
-		return metaData;
-	}
-
-	RocksDBColumnFamilyCheckpoint rocksCF = getRocksCF(checkpoint);
-	metaData.db_comparator_name = rocksCF.dbComparatorName;
-
-	for (const LiveFileMetaData& fileMetaData : rocksCF.sstFiles) {
-		rocksdb::LiveFileMetaData liveFileMetaData;
-		liveFileMetaData.relative_filename = fileMetaData.relative_filename;
-		liveFileMetaData.directory = fileMetaData.directory;
-		liveFileMetaData.file_number = fileMetaData.file_number;
-		liveFileMetaData.file_type = static_cast<rocksdb::FileType>(fileMetaData.file_type);
-		liveFileMetaData.size = fileMetaData.size;
-		liveFileMetaData.temperature = static_cast<rocksdb::Temperature>(fileMetaData.temperature);
-		liveFileMetaData.file_checksum = fileMetaData.file_checksum;
-		liveFileMetaData.file_checksum_func_name = fileMetaData.file_checksum_func_name;
-		liveFileMetaData.smallest_seqno = fileMetaData.smallest_seqno;
-		liveFileMetaData.largest_seqno = fileMetaData.largest_seqno;
-		liveFileMetaData.smallestkey = fileMetaData.smallestkey;
-		liveFileMetaData.largestkey = fileMetaData.largestkey;
-		liveFileMetaData.num_reads_sampled = fileMetaData.num_reads_sampled;
-		liveFileMetaData.being_compacted = fileMetaData.being_compacted;
-		liveFileMetaData.num_entries = fileMetaData.num_entries;
-		liveFileMetaData.num_deletions = fileMetaData.num_deletions;
-		liveFileMetaData.oldest_blob_file_number = fileMetaData.oldest_blob_file_number;
-		liveFileMetaData.oldest_ancester_time = fileMetaData.oldest_ancester_time;
-		liveFileMetaData.file_creation_time = fileMetaData.file_creation_time;
-		liveFileMetaData.smallest = fileMetaData.smallest;
-		liveFileMetaData.largest = fileMetaData.largest;
-		liveFileMetaData.file_type = rocksdb::kTableFile;
-		liveFileMetaData.epoch_number = fileMetaData.epoch_number;
-		liveFileMetaData.name = fileMetaData.name;
-		liveFileMetaData.db_path = fileMetaData.db_path;
-		liveFileMetaData.column_family_name = fileMetaData.column_family_name;
-		liveFileMetaData.level = fileMetaData.level;
-		metaData.files.push_back(liveFileMetaData);
-	}
-
-	return metaData;
-}
-
-void populateMetaData(CheckpointMetaData* checkpoint, const rocksdb::ExportImportFilesMetaData* metaData) {
-	RocksDBColumnFamilyCheckpoint rocksCF;
-	if (metaData != nullptr) {
-		rocksCF.dbComparatorName = metaData->db_comparator_name;
-		for (const rocksdb::LiveFileMetaData& fileMetaData : metaData->files) {
-			LiveFileMetaData liveFileMetaData;
-			liveFileMetaData.relative_filename = fileMetaData.relative_filename;
-			liveFileMetaData.directory = fileMetaData.directory;
-			liveFileMetaData.file_number = fileMetaData.file_number;
-			liveFileMetaData.file_type = static_cast<int>(fileMetaData.file_type);
-			liveFileMetaData.size = fileMetaData.size;
-			liveFileMetaData.temperature = static_cast<uint8_t>(fileMetaData.temperature);
-			liveFileMetaData.file_checksum = fileMetaData.file_checksum;
-			liveFileMetaData.file_checksum_func_name = fileMetaData.file_checksum_func_name;
-			liveFileMetaData.smallest_seqno = fileMetaData.smallest_seqno;
-			liveFileMetaData.largest_seqno = fileMetaData.largest_seqno;
-			liveFileMetaData.smallestkey = fileMetaData.smallestkey;
-			liveFileMetaData.largestkey = fileMetaData.largestkey;
-			liveFileMetaData.num_reads_sampled = fileMetaData.num_reads_sampled;
-			liveFileMetaData.being_compacted = fileMetaData.being_compacted;
-			liveFileMetaData.num_entries = fileMetaData.num_entries;
-			liveFileMetaData.num_deletions = fileMetaData.num_deletions;
-			liveFileMetaData.oldest_blob_file_number = fileMetaData.oldest_blob_file_number;
-			liveFileMetaData.oldest_ancester_time = fileMetaData.oldest_ancester_time;
-			liveFileMetaData.file_creation_time = fileMetaData.file_creation_time;
-			liveFileMetaData.smallest = fileMetaData.smallest;
-			liveFileMetaData.largest = fileMetaData.largest;
-			liveFileMetaData.epoch_number = fileMetaData.epoch_number;
-			liveFileMetaData.name = fileMetaData.name;
-			liveFileMetaData.db_path = fileMetaData.db_path;
-			liveFileMetaData.column_family_name = fileMetaData.column_family_name;
-			liveFileMetaData.level = fileMetaData.level;
-			rocksCF.sstFiles.push_back(liveFileMetaData);
-		}
-	}
-	checkpoint->setFormat(DataMoveRocksCF);
-	checkpoint->serializedCheckpoint = ObjectWriter::toValue(rocksCF, IncludeVersion());
-}
-
-const rocksdb::Slice toSlice(StringRef s) {
-	return rocksdb::Slice(reinterpret_cast<const char*>(s.begin()), s.size());
-}
-
-StringRef toStringRef(rocksdb::Slice s) {
-	return StringRef(reinterpret_cast<const uint8_t*>(s.data()), s.size());
-}
-
-std::string getShardMappingKey(KeyRef key, StringRef prefix) {
-	return prefix.toString() + key.toString();
-}
-
-void logRocksDBError(const rocksdb::Status& status, const std::string& method) {
-	auto level = status.IsTimedOut() ? SevWarn : SevError;
-	TraceEvent e(level, "ShardedRocksDBError");
-	e.setMaxFieldLength(10000)
-	    .detail("Error", status.ToString())
-	    .detail("Method", method)
-	    .detail("ShardedRocksDBSeverity", status.severity());
-	if (status.IsIOError()) {
-		e.detail("SubCode", status.subcode());
-	}
-}
-
-// TODO: define shard ops.
-enum class ShardOp {
-	CREATE,
-	OPEN,
-	DESTROY,
-	CLOSE,
-	MODIFY_RANGE,
-};
-
-const char* ShardOpToString(ShardOp op) {
-	switch (op) {
-	case ShardOp::CREATE:
-		return "CREATE";
-	case ShardOp::OPEN:
-		return "OPEN";
-	case ShardOp::DESTROY:
-		return "DESTROY";
-	case ShardOp::CLOSE:
-		return "CLOSE";
-	case ShardOp::MODIFY_RANGE:
-		return "MODIFY_RANGE";
-	default:
-		return "Unknown";
-	}
-}
-void logShardEvent(StringRef name, ShardOp op, Severity severity = SevInfo, const std::string& message = "") {
-	TraceEvent e(severity, "ShardedRocksDBKVSShardEvent");
-	e.detail("Name", name).detail("Action", ShardOpToString(op));
-	if (!message.empty()) {
-		e.detail("Message", message);
-	}
-}
-void logShardEvent(StringRef name,
-                   KeyRangeRef range,
-                   ShardOp op,
-                   Severity severity = SevInfo,
-                   const std::string& message = "") {
-	TraceEvent e(severity, "ShardedRocksDBKVSShardEvent");
-	e.detail("Name", name).detail("Action", ShardOpToString(op)).detail("Begin", range.begin).detail("End", range.end);
-	if (message != "") {
-		e.detail("Message", message);
-	}
-}
-
-Error statusToError(const rocksdb::Status& s) {
-	if (s.IsIOError()) {
-		return io_error();
-	} else if (s.IsTimedOut()) {
-		return key_value_store_deadline_exceeded();
-	} else {
-		return unknown_error();
-	}
-}
-
-rocksdb::ColumnFamilyOptions getCFOptions() {
-	rocksdb::ColumnFamilyOptions options;
-	options.disable_auto_compactions = SERVER_KNOBS->ROCKSDB_DISABLE_AUTO_COMPACTIONS;
-	options.level_compaction_dynamic_level_bytes = SERVER_KNOBS->ROCKSDB_LEVEL_COMPACTION_DYNAMIC_LEVEL_BYTES;
-	options.OptimizeLevelStyleCompaction(SERVER_KNOBS->ROCKSDB_MEMTABLE_BYTES);
-	if (SERVER_KNOBS->ROCKSDB_PERIODIC_COMPACTION_SECONDS > 0) {
-		options.periodic_compaction_seconds = SERVER_KNOBS->ROCKSDB_PERIODIC_COMPACTION_SECONDS;
-	}
-	// Compact sstables when there's too much deleted stuff.
-	options.table_properties_collector_factories = { rocksdb::NewCompactOnDeletionCollectorFactory(128, 1) };
-
-	rocksdb::BlockBasedTableOptions bbOpts;
-	// TODO: Add a knob for the block cache size. (Default is 8 MB)
-	if (SERVER_KNOBS->ROCKSDB_PREFIX_LEN > 0) {
-		// Prefix blooms are used during Seek.
-		options.prefix_extractor.reset(rocksdb::NewFixedPrefixTransform(SERVER_KNOBS->ROCKSDB_PREFIX_LEN));
-
-		// Also turn on bloom filters in the memtable.
-		// TODO: Make a knob for this as well.
-		options.memtable_prefix_bloom_size_ratio = 0.1;
-
-		// 5 -- Can be read by RocksDB's versions since 6.6.0. Full and partitioned
-		// filters use a generally faster and more accurate Bloom filter
-		// implementation, with a different schema.
-		// https://github.com/facebook/rocksdb/blob/b77569f18bfc77fb1d8a0b3218f6ecf571bc4988/include/rocksdb/table.h#L391
-		bbOpts.format_version = 5;
-
-		// Create and apply a bloom filter using the 10 bits
-		// which should yield a ~1% false positive rate:
-		// https://github.com/facebook/rocksdb/wiki/RocksDB-Bloom-Filter#full-filters-new-format
-		bbOpts.filter_policy.reset(rocksdb::NewBloomFilterPolicy(10));
-
-		// The whole key blooms are only used for point lookups.
-		// https://github.com/facebook/rocksdb/wiki/RocksDB-Bloom-Filter#prefix-vs-whole-key
-		bbOpts.whole_key_filtering = false;
-	}
-
-	if (rocksdb_block_cache == nullptr && SERVER_KNOBS->ROCKSDB_BLOCK_CACHE_SIZE > 0) {
-		rocksdb_block_cache = rocksdb::NewLRUCache(SERVER_KNOBS->ROCKSDB_BLOCK_CACHE_SIZE);
-	}
-	bbOpts.block_cache = rocksdb_block_cache;
-
-	options.table_factory.reset(rocksdb::NewBlockBasedTableFactory(bbOpts));
-
-	return options;
-}
-
-rocksdb::Options getOptions() {
-	rocksdb::Options options({}, getCFOptions());
-	options.avoid_unnecessary_blocking_io = true;
-	options.create_if_missing = true;
-	options.atomic_flush = SERVER_KNOBS->ROCKSDB_ATOMIC_FLUSH;
-	if (SERVER_KNOBS->ROCKSDB_BACKGROUND_PARALLELISM > 0) {
-		options.IncreaseParallelism(SERVER_KNOBS->ROCKSDB_BACKGROUND_PARALLELISM);
-	}
-
-	options.delete_obsolete_files_period_micros = SERVER_KNOBS->ROCKSDB_DELETE_OBSOLETE_FILE_PERIOD * 1000000;
-	options.max_total_wal_size = SERVER_KNOBS->ROCKSDB_MAX_TOTAL_WAL_SIZE;
-	options.max_subcompactions = SERVER_KNOBS->ROCKSDB_MAX_SUBCOMPACTIONS;
-	options.max_background_jobs = SERVER_KNOBS->ROCKSDB_MAX_BACKGROUND_JOBS;
-
-	options.db_write_buffer_size = SERVER_KNOBS->ROCKSDB_WRITE_BUFFER_SIZE;
-	options.write_buffer_size = SERVER_KNOBS->ROCKSDB_CF_WRITE_BUFFER_SIZE;
-	options.statistics = rocksdb::CreateDBStatistics();
-	options.statistics->set_stats_level(rocksdb::kExceptHistogramOrTimers);
-	options.db_log_dir = g_network->isSimulated() ? "" : SERVER_KNOBS->LOG_DIRECTORY;
-	return options;
-}
-
-// Set some useful defaults desired for all reads.
-rocksdb::ReadOptions getReadOptions() {
-	rocksdb::ReadOptions options;
-	options.background_purge_on_iterator_cleanup = true;
-	options.auto_prefix_mode = (SERVER_KNOBS->ROCKSDB_PREFIX_LEN > 0);
-	return options;
-}
-
-struct ReadIterator {
-	uint64_t index; // incrementing counter to uniquely identify read iterator.
-	bool inUse;
-	std::shared_ptr<rocksdb::Iterator> iter;
-	double creationTime;
-	KeyRange keyRange;
-	std::shared_ptr<rocksdb::Slice> beginSlice, endSlice;
-
-	ReadIterator(rocksdb::ColumnFamilyHandle* cf, uint64_t index, rocksdb::DB* db)
-	  : index(index), inUse(true), creationTime(now()), iter(db->NewIterator(getReadOptions(), cf)) {}
-	ReadIterator(rocksdb::ColumnFamilyHandle* cf, uint64_t index, rocksdb::DB* db, const KeyRange& range)
-	  : index(index), inUse(true), creationTime(now()), keyRange(range) {
-		auto options = getReadOptions();
-		beginSlice = std::shared_ptr<rocksdb::Slice>(new rocksdb::Slice(toSlice(keyRange.begin)));
-		options.iterate_lower_bound = beginSlice.get();
-		endSlice = std::shared_ptr<rocksdb::Slice>(new rocksdb::Slice(toSlice(keyRange.end)));
-		options.iterate_upper_bound = endSlice.get();
-		iter = std::shared_ptr<rocksdb::Iterator>(db->NewIterator(options, cf));
-	}
-};
-
-/*
-ReadIteratorPool: Collection of iterators. Reuses iterators on non-concurrent multiple read operations,
-instead of creating and deleting for every read.
-
-Read: IteratorPool provides an unused iterator if exists or creates and gives a new iterator.
-Returns back the iterator after the read is done.
-
-Write: Iterators in the pool are deleted, forcing new iterator creation on next reads. The iterators
-which are currently used by the reads can continue using the iterator as it is a shared_ptr. Once
-the read is processed, shared_ptr goes out of scope and gets deleted. Eventually the iterator object
-gets deleted as the ref count becomes 0.
-*/
-class ReadIteratorPool {
-public:
-	ReadIteratorPool(rocksdb::DB* db, rocksdb::ColumnFamilyHandle* cf, const std::string& path)
-	  : db(db), cf(cf), index(0), iteratorsReuseCount(0) {
-		ASSERT(db);
-		ASSERT(cf);
-		TraceEvent(SevVerbose, "ShardedRocksReadIteratorPool")
-		    .detail("Path", path)
-		    .detail("KnobRocksDBReadRangeReuseIterators", SERVER_KNOBS->ROCKSDB_READ_RANGE_REUSE_ITERATORS)
-		    .detail("KnobRocksDBPrefixLen", SERVER_KNOBS->ROCKSDB_PREFIX_LEN);
-	}
-
-	// Called on every db commit.
-	void update() {
-		if (SERVER_KNOBS->ROCKSDB_READ_RANGE_REUSE_ITERATORS) {
-			std::lock_guard<std::mutex> lock(mutex);
-			iteratorsMap.clear();
-		}
-	}
-
-	// Called on every read operation.
-	ReadIterator getIterator(const KeyRange& range) {
-		// Shared iterators are not bounded.
-		if (SERVER_KNOBS->ROCKSDB_READ_RANGE_REUSE_ITERATORS) {
-			std::lock_guard<std::mutex> lock(mutex);
-			for (it = iteratorsMap.begin(); it != iteratorsMap.end(); it++) {
-				if (!it->second.inUse) {
-					it->second.inUse = true;
-					iteratorsReuseCount++;
-					return it->second;
-				}
-			}
-			index++;
-			ReadIterator iter(cf, index, db);
-			iteratorsMap.insert({ index, iter });
-			return iter;
-		} else {
-			index++;
-			ReadIterator iter(cf, index, db, range);
-			return iter;
-		}
-	}
-
-	// Called on every read operation, after the keys are collected.
-	void returnIterator(ReadIterator& iter) {
-		if (SERVER_KNOBS->ROCKSDB_READ_RANGE_REUSE_ITERATORS) {
-			std::lock_guard<std::mutex> lock(mutex);
-			it = iteratorsMap.find(iter.index);
-			// iterator found: put the iterator back to the pool(inUse=false).
-			// iterator not found: update would have removed the iterator from pool, so nothing to do.
-			if (it != iteratorsMap.end()) {
-				ASSERT(it->second.inUse);
-				it->second.inUse = false;
-			}
-		}
-	}
-
-	// Called for every ROCKSDB_READ_RANGE_ITERATOR_REFRESH_TIME seconds in a loop.
-	void refreshIterators() {
-		std::lock_guard<std::mutex> lock(mutex);
-		it = iteratorsMap.begin();
-		while (it != iteratorsMap.end()) {
-			if (now() - it->second.creationTime > SERVER_KNOBS->ROCKSDB_READ_RANGE_ITERATOR_REFRESH_TIME) {
-				it = iteratorsMap.erase(it);
-			} else {
-				it++;
-			}
-		}
-	}
-
-	uint64_t numReadIteratorsCreated() { return index; }
-
-	uint64_t numTimesReadIteratorsReused() { return iteratorsReuseCount; }
-
-private:
-	std::unordered_map<int, ReadIterator> iteratorsMap;
-	std::unordered_map<int, ReadIterator>::iterator it;
-	rocksdb::DB* db;
-	rocksdb::ColumnFamilyHandle* cf;
-	std::mutex mutex;
-	// incrementing counter for every new iterator creation, to uniquely identify the iterator in returnIterator().
-	uint64_t index;
-	uint64_t iteratorsReuseCount;
-};
-
-ACTOR Future<Void> flowLockLogger(const FlowLock* readLock, const FlowLock* fetchLock) {
-	loop {
-		wait(delay(SERVER_KNOBS->ROCKSDB_METRICS_DELAY));
-		TraceEvent e("ShardedRocksDBFlowLock");
-		e.detail("ReadAvailable", readLock->available());
-		e.detail("ReadActivePermits", readLock->activePermits());
-		e.detail("ReadWaiters", readLock->waiters());
-		e.detail("FetchAvailable", fetchLock->available());
-		e.detail("FetchActivePermits", fetchLock->activePermits());
-		e.detail("FetchWaiters", fetchLock->waiters());
-	}
-}
-
-// DataShard represents a key range (logical shard) in FDB. A DataShard is assigned to a specific physical shard.
-struct DataShard {
-	DataShard(KeyRange range, PhysicalShard* physicalShard) : range(range), physicalShard(physicalShard) {}
-
-	bool initialized() const;
-
-	KeyRange range;
-	PhysicalShard* physicalShard;
-};
-
-// PhysicalShard represent a collection of logical shards. A PhysicalShard could have one or more DataShards. A
-// PhysicalShard is stored as a column family in rocksdb. Each PhysicalShard has its own iterator pool.
-struct PhysicalShard {
-	PhysicalShard(rocksdb::DB* db, std::string id, const rocksdb::ColumnFamilyOptions& options)
-	  : db(db), id(id), cfOptions(options), isInitialized(false) {}
-	PhysicalShard(rocksdb::DB* db, std::string id, rocksdb::ColumnFamilyHandle* handle)
-	  : db(db), id(id), cf(handle), isInitialized(true) {
-		ASSERT(cf);
-		readIterPool = std::make_shared<ReadIteratorPool>(db, cf, id);
-	}
-
-	rocksdb::Status init() {
-		if (cf) {
-			return rocksdb::Status::OK();
-		}
-		auto status = db->CreateColumnFamily(cfOptions, id, &cf);
-		if (!status.ok()) {
-			logRocksDBError(status, "AddCF");
-			return status;
-		}
-		readIterPool = std::make_shared<ReadIteratorPool>(db, cf, id);
-		this->isInitialized.store(true);
-		return status;
-	}
-
-	// Restore from the checkpoint.
-	rocksdb::Status restore(const CheckpointMetaData& checkpoint) {
-		const CheckpointFormat format = checkpoint.getFormat();
-		rocksdb::Status status;
-		if (format == DataMoveRocksCF) {
-			rocksdb::ExportImportFilesMetaData metaData = getMetaData(checkpoint);
-			if (metaData.files.empty()) {
-				TraceEvent(SevInfo, "RocksDBRestoreEmptyShard")
-				    .detail("Shard", id)
-				    .detail("CheckpointID", checkpoint.checkpointID);
-				status = db->CreateColumnFamily(getCFOptions(), id, &cf);
-			} else {
-				TraceEvent(SevInfo, "RocksDBRestoreCF");
-				rocksdb::ImportColumnFamilyOptions importOptions;
-				importOptions.move_files = SERVER_KNOBS->ROCKSDB_IMPORT_MOVE_FILES;
-				status = db->CreateColumnFamilyWithImport(getCFOptions(), id, importOptions, metaData, &cf);
-				TraceEvent(SevInfo, "RocksDBRestoreCFEnd").detail("Status", status.ToString());
-			}
-		} else if (format == RocksDBKeyValues) {
-			ASSERT(cf != nullptr);
-			std::vector<std::string> sstFiles;
-			const RocksDBCheckpointKeyValues rcp = getRocksKeyValuesCheckpoint(checkpoint);
-			for (const auto& file : rcp.fetchedFiles) {
-				if (file.path != emptySstFilePath) {
-					sstFiles.push_back(file.path);
-				}
-			}
-
-			TraceEvent(SevDebug, "RocksDBRestoreFiles")
-			    .detail("Shard", id)
-			    .detail("CheckpointID", checkpoint.checkpointID)
-			    .detail("Files", describe(sstFiles));
-
-			if (!sstFiles.empty()) {
-				rocksdb::IngestExternalFileOptions ingestOptions;
-				ingestOptions.move_files = SERVER_KNOBS->ROCKSDB_IMPORT_MOVE_FILES;
-				ingestOptions.verify_checksums_before_ingest = SERVER_KNOBS->ROCKSDB_VERIFY_CHECKSUM_BEFORE_RESTORE;
-				status = db->IngestExternalFile(cf, sstFiles, ingestOptions);
-			} else {
-				TraceEvent(SevWarn, "RocksDBServeRestoreEmptyRange")
-				    .detail("Shard", id)
-				    .detail("RocksKeyValuesCheckpoint", rcp.toString())
-				    .detail("Checkpoint", checkpoint.toString());
-			}
-			TraceEvent(SevInfo, "RocksDBServeRestoreFiles")
-			    .detail("Shard", id)
-			    .detail("CFName", cf->GetName())
-			    .detail("Checkpoint", checkpoint.toString())
-			    .detail("RestoredFiles", describe(sstFiles));
-		} else {
-			throw not_implemented();
-		}
-
-		TraceEvent(status.ok() ? SevInfo : SevWarnAlways, "PhysicalShardRestoreEnd")
-		    .detail("Status", status.ToString())
-		    .detail("Shard", id)
-		    .detail("CFName", cf == nullptr ? "" : cf->GetName())
-		    .detail("Checkpoint", checkpoint.toString());
-		if (status.ok()) {
-			if (!this->isInitialized) {
-				readIterPool = std::make_shared<ReadIteratorPool>(db, cf, id);
-				this->isInitialized.store(true);
-			} else {
-				refreshReadIteratorPool();
-			}
-		}
-
-		return status;
-	}
-
-	bool initialized() { return this->isInitialized.load(); }
-
-	void refreshReadIteratorPool() {
-		ASSERT(this->readIterPool != nullptr);
-		this->readIterPool->refreshIterators();
-	}
-
-	std::vector<KeyRange> getAllRanges() const {
-		std::vector<KeyRange> res;
-		for (const auto& [key, shard] : dataShards) {
-			if (shard != nullptr) {
-				res.push_back(shard->range);
-			}
-		}
-		return res;
-	}
-
-	std::string toString() {
-		std::string ret = "[ID]: " + this->id + ", [CF]: ";
-		if (initialized()) {
-			ret += std::to_string(this->cf->GetID());
-		} else {
-			ret += "Not initialized";
-		}
-		return ret;
-	}
-
-	~PhysicalShard() {
-		logShardEvent(id, ShardOp::CLOSE);
-		isInitialized.store(false);
-		readIterPool.reset();
-
-		// Deleting default column family is not allowed.
-		if (id == DEFAULT_CF_NAME) {
-			return;
-		}
-
-		if (deletePending) {
-			auto s = db->DropColumnFamily(cf);
-			if (!s.ok()) {
-				logRocksDBError(s, "DestroyShard");
-				logShardEvent(id, ShardOp::DESTROY, SevError, s.ToString());
-				return;
-			}
-		}
-		auto s = db->DestroyColumnFamilyHandle(cf);
-		if (!s.ok()) {
-			logRocksDBError(s, "DestroyCFHandle");
-			logShardEvent(id, ShardOp::DESTROY, SevError, s.ToString());
-			return;
-		}
-		logShardEvent(id, ShardOp::DESTROY);
-	}
-
-	rocksdb::DB* db;
-	std::string id;
-	rocksdb::ColumnFamilyOptions cfOptions;
-	rocksdb::ColumnFamilyHandle* cf = nullptr;
-	std::unordered_map<std::string, std::unique_ptr<DataShard>> dataShards;
-	std::shared_ptr<ReadIteratorPool> readIterPool;
-	bool deletePending = false;
-	std::atomic<bool> isInitialized;
-	double deleteTimeSec;
-};
-
-int readRangeInDb(PhysicalShard* shard, const KeyRangeRef range, int rowLimit, int byteLimit, RangeResult* result) {
-	if (rowLimit == 0 || byteLimit == 0) {
-		return 0;
-	}
-
-	int accumulatedBytes = 0;
-	rocksdb::Status s;
-
-	// When using a prefix extractor, ensure that keys are returned in order even if they cross
-	// a prefix boundary.
-	if (rowLimit >= 0) {
-		ReadIterator readIter = shard->readIterPool->getIterator(range);
-		auto cursor = readIter.iter;
-		cursor->Seek(toSlice(range.begin));
-		while (cursor->Valid() && toStringRef(cursor->key()) < range.end) {
-			KeyValueRef kv(toStringRef(cursor->key()), toStringRef(cursor->value()));
-			accumulatedBytes += sizeof(KeyValueRef) + kv.expectedSize();
-			result->push_back_deep(result->arena(), kv);
-			// Calling `cursor->Next()` is potentially expensive, so short-circut here just in case.
-			if (result->size() >= rowLimit || accumulatedBytes >= byteLimit) {
-				break;
-			}
-			cursor->Next();
-		}
-		s = cursor->status();
-		shard->readIterPool->returnIterator(readIter);
-	} else {
-		ReadIterator readIter = shard->readIterPool->getIterator(range);
-		auto cursor = readIter.iter;
-		cursor->SeekForPrev(toSlice(range.end));
-		if (cursor->Valid() && toStringRef(cursor->key()) == range.end) {
-			cursor->Prev();
-		}
-		while (cursor->Valid() && toStringRef(cursor->key()) >= range.begin) {
-			KeyValueRef kv(toStringRef(cursor->key()), toStringRef(cursor->value()));
-			accumulatedBytes += sizeof(KeyValueRef) + kv.expectedSize();
-			result->push_back_deep(result->arena(), kv);
-			// Calling `cursor->Prev()` is potentially expensive, so short-circut here just in case.
-			if (result->size() >= -rowLimit || accumulatedBytes >= byteLimit) {
-				break;
-			}
-			cursor->Prev();
-		}
-		s = cursor->status();
-		shard->readIterPool->returnIterator(readIter);
-	}
-
-	if (!s.ok()) {
-		logRocksDBError(s, "ReadRange");
-		// The data writen to the arena is not erased, which will leave RangeResult in a dirty state. The RangeResult
-		// should never be returned to user.
-		return -1;
-	}
-	return accumulatedBytes;
-}
-
-// Manages physical shards and maintains logical shard mapping.
-class ShardManager {
-public:
-	ShardManager(std::string path, UID logId, const rocksdb::Options& options)
-	  : path(path), logId(logId), dbOptions(options), dataShardMap(nullptr, specialKeys.end) {}
-
-	ACTOR static Future<Void> shardMetricsLogger(std::shared_ptr<ShardedRocksDBState> rState,
-	                                             Future<Void> openFuture,
-	                                             ShardManager* shardManager) {
-		state std::unordered_map<std::string, std::shared_ptr<PhysicalShard>>* physicalShards =
-		    shardManager->getAllShards();
-
-		try {
-			wait(openFuture);
-			loop {
-				wait(delay(SERVER_KNOBS->ROCKSDB_METRICS_DELAY));
-				if (rState->closing) {
-					break;
-				}
-				TraceEvent(SevInfo, "ShardedRocksKVSPhysialShardMetrics")
-				    .detail("NumActiveShards", shardManager->numActiveShards())
-				    .detail("TotalPhysicalShards", shardManager->numPhysicalShards());
-
-				for (auto& [id, shard] : *physicalShards) {
-					if (!shard->initialized()) {
-						continue;
-					}
-					std::string propValue = "";
-					ASSERT(shard->db->GetProperty(shard->cf, rocksdb::DB::Properties::kCFStats, &propValue));
-					TraceEvent(SevInfo, "PhysicalShardCFStats")
-					    .detail("PhysicalShardID", id)
-					    .detail("Detail", propValue);
-
-					// Get compression ratio for each level.
-					rocksdb::ColumnFamilyMetaData cfMetadata;
-					shard->db->GetColumnFamilyMetaData(shard->cf, &cfMetadata);
-					TraceEvent e(SevInfo, "PhysicalShardLevelStats");
-					e.detail("PhysicalShardID", id);
-					std::string levelProp;
-					for (auto it = cfMetadata.levels.begin(); it != cfMetadata.levels.end(); ++it) {
-						std::string propValue = "";
-						ASSERT(shard->db->GetProperty(shard->cf,
-						                              rocksdb::DB::Properties::kCompressionRatioAtLevelPrefix +
-						                                  std::to_string(it->level),
-						                              &propValue));
-						e.detail("Level" + std::to_string(it->level), std::to_string(it->size) + " " + propValue);
-					}
-				}
-			}
-		} catch (Error& e) {
-			if (e.code() != error_code_actor_cancelled) {
-				TraceEvent(SevError, "ShardedRocksShardMetricsLoggerError").errorUnsuppressed(e);
-			}
-		}
-		return Void();
-	}
-
-	rocksdb::Status init() {
-		// Open instance.
-		TraceEvent(SevInfo, "ShardedRocksShardManagerInitBegin", this->logId).detail("DataPath", path);
-		std::vector<std::string> columnFamilies;
-		rocksdb::Status status = rocksdb::DB::ListColumnFamilies(dbOptions, path, &columnFamilies);
-
-		std::vector<rocksdb::ColumnFamilyDescriptor> descriptors;
-		bool foundMetadata = false;
-		for (const auto& name : columnFamilies) {
-			if (name == METADATA_SHARD_ID) {
-				foundMetadata = true;
-			}
-			descriptors.push_back(rocksdb::ColumnFamilyDescriptor{ name, rocksdb::ColumnFamilyOptions(dbOptions) });
-		}
-
-		ASSERT(foundMetadata || descriptors.size() == 0);
-
-		// Add default column family if it's a newly opened database.
-		if (descriptors.size() == 0) {
-			descriptors.push_back(
-			    rocksdb::ColumnFamilyDescriptor{ DEFAULT_CF_NAME, rocksdb::ColumnFamilyOptions(dbOptions) });
-		}
-
-		std::vector<rocksdb::ColumnFamilyHandle*> handles;
-		status = rocksdb::DB::Open(dbOptions, path, descriptors, &handles, &db);
-		if (!status.ok()) {
-			logRocksDBError(status, "Open");
-			return status;
-		}
-
-		if (foundMetadata) {
-			TraceEvent(SevInfo, "ShardedRocksInitLoadPhysicalShards", this->logId)
-			    .detail("PhysicalShardCount", handles.size());
-
-			std::shared_ptr<PhysicalShard> metadataShard = nullptr;
-			for (auto handle : handles) {
-				auto shard = std::make_shared<PhysicalShard>(db, handle->GetName(), handle);
-				if (shard->id == METADATA_SHARD_ID) {
-					metadataShard = shard;
-				}
-				physicalShards[shard->id] = shard;
-				columnFamilyMap[handle->GetID()] = handle;
-				TraceEvent(SevVerbose, "ShardedRocksInitPhysicalShard", this->logId)
-				    .detail("PhysicalShardID", shard->id);
-			}
-
-			std::set<std::string> unusedShards(columnFamilies.begin(), columnFamilies.end());
-			unusedShards.erase(METADATA_SHARD_ID);
-			unusedShards.erase(DEFAULT_CF_NAME);
-
-			KeyRange keyRange = prefixRange(shardMappingPrefix);
-			while (true) {
-				RangeResult metadata;
-				const int bytes = readRangeInDb(metadataShard.get(),
-				                                keyRange,
-				                                std::max(2, SERVER_KNOBS->ROCKSDB_READ_RANGE_ROW_LIMIT),
-				                                UINT16_MAX,
-				                                &metadata);
-				if (bytes <= 0) {
-					break;
-				}
-
-				ASSERT_GT(metadata.size(), 0);
-				for (int i = 0; i < metadata.size() - 1; ++i) {
-					const std::string name = metadata[i].value.toString();
-					KeyRangeRef range(metadata[i].key.removePrefix(shardMappingPrefix),
-					                  metadata[i + 1].key.removePrefix(shardMappingPrefix));
-					TraceEvent(SevVerbose, "DecodeShardMapping", this->logId)
-					    .detail("Range", range)
-					    .detail("Name", name);
-
-					// Empty name indicates the shard doesn't belong to the SS/KVS.
-					if (name.empty()) {
-						continue;
-					}
-
-					auto it = physicalShards.find(name);
-					// Raise error if physical shard is missing.
-					if (it == physicalShards.end()) {
-						TraceEvent(SevError, "ShardedRocksDB", this->logId).detail("MissingShard", name);
-						return rocksdb::Status::NotFound();
-					}
-
-					std::unique_ptr<DataShard> dataShard = std::make_unique<DataShard>(range, it->second.get());
-					dataShardMap.insert(range, dataShard.get());
-					it->second->dataShards[range.begin.toString()] = std::move(dataShard);
-					activePhysicalShardIds.emplace(name);
-					unusedShards.erase(name);
-				}
-
-				if (metadata.back().key.removePrefix(shardMappingPrefix) == specialKeys.end) {
-					TraceEvent(SevVerbose, "ShardedRocksLoadShardMappingEnd", this->logId);
-					break;
-				}
-
-				// Read from the current last key since the shard begining with it hasn't been processed.
-				if (metadata.size() == 1 && metadata.back().value.toString().empty()) {
-					// Should not happen, just being paranoid.
-					keyRange = KeyRangeRef(keyAfter(metadata.back().key), keyRange.end);
-				} else {
-					keyRange = KeyRangeRef(metadata.back().key, keyRange.end);
-				}
-			}
-
-			for (const auto& name : unusedShards) {
-				TraceEvent(SevDebug, "UnusedShardName", logId).detail("Name", name);
-				auto it = physicalShards.find(name);
-				ASSERT(it != physicalShards.end());
-				auto shard = it->second;
-				if (shard->dataShards.size() == 0) {
-					shard->deleteTimeSec = now();
-					pendingDeletionShards.push_back(name);
-				}
-			}
-			if (unusedShards.size() > 0) {
-				TraceEvent("ShardedRocksDB", logId).detail("CleanUpUnusedShards", unusedShards.size());
-			}
-		} else {
-			// DB is opened with default shard.
-			ASSERT(handles.size() == 1);
-
-			// Add SpecialKeys range. This range should not be modified.
-			std::shared_ptr<PhysicalShard> defaultShard =
-			    std::make_shared<PhysicalShard>(db, DEFAULT_CF_NAME, handles[0]);
-			columnFamilyMap[defaultShard->cf->GetID()] = defaultShard->cf;
-			std::unique_ptr<DataShard> dataShard = std::make_unique<DataShard>(specialKeys, defaultShard.get());
-			dataShardMap.insert(specialKeys, dataShard.get());
-			defaultShard->dataShards[specialKeys.begin.toString()] = std::move(dataShard);
-			physicalShards[defaultShard->id] = defaultShard;
-
-			// Create metadata shard.
-			auto metadataShard =
-			    std::make_shared<PhysicalShard>(db, METADATA_SHARD_ID, rocksdb::ColumnFamilyOptions(dbOptions));
-			metadataShard->init();
-			columnFamilyMap[metadataShard->cf->GetID()] = metadataShard->cf;
-			physicalShards[METADATA_SHARD_ID] = metadataShard;
-
-			// Write special key range metadata.
-			writeBatch = std::make_unique<rocksdb::WriteBatch>();
-			dirtyShards = std::make_unique<std::set<PhysicalShard*>>();
-			persistRangeMapping(specialKeys, true);
-			rocksdb::WriteOptions options;
-			status = db->Write(options, writeBatch.get());
-			if (!status.ok()) {
-				return status;
-			}
-			metadataShard->readIterPool->update();
-			TraceEvent(SevInfo, "ShardedRocksInitializeMetaDataShard", this->logId)
-			    .detail("MetadataShardCF", metadataShard->cf->GetID());
-		}
-
-		writeBatch = std::make_unique<rocksdb::WriteBatch>();
-		dirtyShards = std::make_unique<std::set<PhysicalShard*>>();
-
-		TraceEvent(SevInfo, "ShardedRocksShardManagerInitEnd", this->logId).detail("DataPath", path);
-		return status;
-	}
-
-	DataShard* getDataShard(KeyRef key) {
-		DataShard* shard = dataShardMap[key];
-		ASSERT(shard == nullptr || shard->range.contains(key));
-		return shard;
-	}
-
-	PhysicalShard* getSpecialKeysShard() {
-		auto it = physicalShards.find(DEFAULT_CF_NAME);
-		ASSERT(it != physicalShards.end());
-		return it->second.get();
-	}
-
-	PhysicalShard* getMetaDataShard() {
-		auto it = physicalShards.find(METADATA_SHARD_ID);
-		ASSERT(it != physicalShards.end());
-		return it->second.get();
-	}
-
-	// Returns the physical shard that hosting all `ranges`; Returns nullptr if such a physical shard does not exists.
-	PhysicalShard* getPhysicalShardForAllRanges(std::vector<KeyRange> ranges) {
-		PhysicalShard* result = nullptr;
-		for (const auto& range : ranges) {
-			auto rangeIterator = this->dataShardMap.intersectingRanges(range);
-			for (auto it = rangeIterator.begin(); it != rangeIterator.end(); ++it) {
-				if (it.value() == nullptr) {
-					TraceEvent(SevWarn, "ShardedRocksDBRangeNotInKVS", logId).detail("Range", range);
-					return nullptr;
-				} else {
-					ASSERT(it.value()->physicalShard != nullptr);
-					PhysicalShard* ps = it.value()->physicalShard;
-					if (result == nullptr) {
-						result = ps;
-					} else if (ps != result) {
-						TraceEvent(SevWarn, "ShardedRocksDBRangeOnMultipleShards", logId)
-						    .detail("Ranges", describe(ranges))
-						    .detail("PhysicalShard", result->id)
-						    .detail("SecondPhysicalShard", ps->id);
-						return nullptr;
-					}
-				}
-			}
-		}
-
-		return result;
-	}
-
-	std::vector<DataShard*> getDataShardsByRange(KeyRangeRef range) {
-		std::vector<DataShard*> result;
-		auto rangeIterator = dataShardMap.intersectingRanges(range);
-
-		for (auto it = rangeIterator.begin(); it != rangeIterator.end(); ++it) {
-			if (it.value() == nullptr) {
-				TraceEvent(SevVerbose, "ShardedRocksDB")
-				    .detail("Info", "ShardNotFound")
-				    .detail("BeginKey", range.begin)
-				    .detail("EndKey", range.end);
-				continue;
-			}
-			result.push_back(it.value());
-		}
-		return result;
-	}
-
-	PhysicalShard* addRange(KeyRange range, std::string id) {
-		TraceEvent(SevVerbose, "ShardedRocksAddRangeBegin", this->logId)
-		    .detail("Range", range)
-		    .detail("PhysicalShardID", id);
-
-		// Newly added range should not overlap with any existing range.
-		auto ranges = dataShardMap.intersectingRanges(range);
-
-		for (auto it = ranges.begin(); it != ranges.end(); ++it) {
-			if (it.value()) {
-				if (it.value()->physicalShard->id == id) {
-					TraceEvent(SevError, "ShardedRocksDBAddRange")
-					    .detail("ErrorType", "RangeAlreadyExist")
-					    .detail("IntersectingRange", it->range())
-					    .detail("DataShardRange", it->value()->range)
-					    .detail("ExpectedShardId", id)
-					    .detail("PhysicalShardID", it->value()->physicalShard->toString());
-				} else {
-					TraceEvent(SevError, "ShardedRocksDBAddRange")
-					    .detail("ErrorType", "ConflictingRange")
-					    .detail("IntersectingRange", it->range())
-					    .detail("DataShardRange", it->value()->range)
-					    .detail("ExpectedShardId", id)
-					    .detail("PhysicalShardID", it->value()->physicalShard->toString());
-				}
-				return nullptr;
-			}
-		}
-
-		auto [it, inserted] = physicalShards.emplace(
-		    id, std::make_shared<PhysicalShard>(db, id, rocksdb::ColumnFamilyOptions(dbOptions)));
-		std::shared_ptr<PhysicalShard>& shard = it->second;
-
-		activePhysicalShardIds.emplace(id);
-
-		auto dataShard = std::make_unique<DataShard>(range, shard.get());
-		dataShardMap.insert(range, dataShard.get());
-		shard->dataShards[range.begin.toString()] = std::move(dataShard);
-
-		validate();
-
-		TraceEvent(SevInfo, "ShardedRocksDBRangeAdded", this->logId)
-		    .detail("Range", range)
-		    .detail("PhysicalShardID", id);
-
-		return shard.get();
-	}
-
-	std::vector<std::string> removeRange(KeyRange range) {
-		TraceEvent(SevInfo, "ShardedRocksRemoveRangeBegin", this->logId).detail("Range", range);
-		std::vector<std::string> shardIds;
-
-		std::vector<DataShard*> newShards;
-		auto ranges = dataShardMap.intersectingRanges(range);
-
-		for (auto it = ranges.begin(); it != ranges.end(); ++it) {
-			if (!it.value()) {
-				TraceEvent(SevDebug, "ShardedRocksDB")
-				    .detail("Info", "RemoveNonExistentRange")
-				    .detail("BeginKey", range.begin)
-				    .detail("EndKey", range.end);
-				continue;
-			}
-			auto existingShard = it.value()->physicalShard;
-			auto shardRange = it.range();
-
-			if (SERVER_KNOBS->ROCKSDB_EMPTY_RANGE_CHECK && existingShard->initialized()) {
-				// Enable consistency validation.
-				RangeResult rangeResult;
-				auto bytesRead = readRangeInDb(existingShard, range, 1, UINT16_MAX, &rangeResult);
-				if (bytesRead > 0) {
-					TraceEvent(SevError, "ShardedRocksDBRangeNotEmpty")
-					    .detail("PhysicalShard", existingShard->toString())
-					    .detail("Range", range)
-					    .detail("DataShardRange", shardRange);
-				}
-
-				// Force clear range.
-				writeBatch->DeleteRange(it.value()->physicalShard->cf, toSlice(range.begin), toSlice(range.end));
-				dirtyShards->insert(it.value()->physicalShard);
-			}
-
-			TraceEvent(SevDebug, "ShardedRocksRemoveRange")
-			    .detail("Range", range)
-			    .detail("IntersectingRange", shardRange)
-			    .detail("DataShardRange", it.value()->range)
-			    .detail("PhysicalShard", existingShard->toString());
-
-			ASSERT(it.value()->range == shardRange); // Ranges should be consistent.
-
-			if (range.contains(shardRange)) {
-				existingShard->dataShards.erase(shardRange.begin.toString());
-				TraceEvent(SevInfo, "ShardedRocksRemovedRange")
-				    .detail("Range", range)
-				    .detail("RemovedRange", shardRange)
-				    .detail("PhysicalShard", existingShard->toString());
-				if (existingShard->dataShards.size() == 0) {
-					TraceEvent(SevDebug, "ShardedRocksDB").detail("EmptyShardId", existingShard->id);
-					shardIds.push_back(existingShard->id);
-					existingShard->deleteTimeSec = now();
-					pendingDeletionShards.push_back(existingShard->id);
-					activePhysicalShardIds.erase(existingShard->id);
-				}
-				continue;
-			}
-
-			// Range modification could result in more than one segments. Remove the original segment key here.
-			existingShard->dataShards.erase(shardRange.begin.toString());
-			if (shardRange.begin < range.begin) {
-				auto dataShard =
-				    std::make_unique<DataShard>(KeyRange(KeyRangeRef(shardRange.begin, range.begin)), existingShard);
-
-				newShards.push_back(dataShard.get());
-				const std::string msg = "Shrink shard from " + Traceable<KeyRangeRef>::toString(shardRange) + " to " +
-				                        Traceable<KeyRangeRef>::toString(dataShard->range);
-
-				existingShard->dataShards[shardRange.begin.toString()] = std::move(dataShard);
-				logShardEvent(existingShard->id, shardRange, ShardOp::MODIFY_RANGE, SevInfo, msg);
-			}
-
-			if (shardRange.end > range.end) {
-				auto dataShard =
-				    std::make_unique<DataShard>(KeyRange(KeyRangeRef(range.end, shardRange.end)), existingShard);
-
-				newShards.push_back(dataShard.get());
-				const std::string msg = "Shrink shard from " + Traceable<KeyRangeRef>::toString(shardRange) + " to " +
-				                        Traceable<KeyRangeRef>::toString(dataShard->range);
-
-				existingShard->dataShards[range.end.toString()] = std::move(dataShard);
-				logShardEvent(existingShard->id, shardRange, ShardOp::MODIFY_RANGE, SevInfo, msg);
-			}
-		}
-
-		dataShardMap.insert(range, nullptr);
-		for (DataShard* shard : newShards) {
-			dataShardMap.insert(shard->range, shard);
-		}
-
-		validate();
-
-		TraceEvent(SevInfo, "ShardedRocksRemoveRangeEnd", this->logId).detail("Range", range);
-
-		return shardIds;
-	}
-
-	std::vector<std::shared_ptr<PhysicalShard>> getPendingDeletionShards(double cleanUpDelay) {
-		std::vector<std::shared_ptr<PhysicalShard>> emptyShards;
-		double currentTime = now();
-
-		TraceEvent(SevInfo, "ShardedRocksDB", logId)
-		    .detail("PendingDeletionShardQueueSize", pendingDeletionShards.size());
-		while (!pendingDeletionShards.empty()) {
-			const auto id = pendingDeletionShards.front();
-			auto it = physicalShards.find(id);
-			if (it == physicalShards.end() || it->second->dataShards.size() != 0) {
-				pendingDeletionShards.pop_front();
-				continue;
-			}
-			if (currentTime - it->second->deleteTimeSec > cleanUpDelay) {
-				pendingDeletionShards.pop_front();
-				emptyShards.push_back(it->second);
-				physicalShards.erase(id);
-			} else {
-				break;
-			}
-		}
-		return emptyShards;
-	}
-
-	void put(KeyRef key, ValueRef value) {
-		auto it = dataShardMap.rangeContaining(key);
-		if (!it.value()) {
-			TraceEvent(SevError, "ShardedRocksDB").detail("Error", "write to non-exist shard").detail("WriteKey", key);
-			return;
-		}
-		TraceEvent(SevVerbose, "ShardedRocksShardManagerPut", this->logId)
-		    .detail("WriteKey", key)
-		    .detail("Value", value)
-		    .detail("MapRange", it.range())
-		    .detail("ShardRange", it.value()->range);
-		ASSERT(it.value()->range == (KeyRangeRef)it.range());
-		ASSERT(writeBatch != nullptr);
-		ASSERT(dirtyShards != nullptr);
-		writeBatch->Put(it.value()->physicalShard->cf, toSlice(key), toSlice(value));
-		dirtyShards->insert(it.value()->physicalShard);
-		TraceEvent(SevVerbose, "ShardedRocksShardManagerPutEnd", this->logId)
-		    .detail("WriteKey", key)
-		    .detail("Value", value);
-	}
-
-	void clear(KeyRef key) {
-		auto it = dataShardMap.rangeContaining(key);
-		if (!it.value()) {
-			return;
-		}
-		writeBatch->Delete(it.value()->physicalShard->cf, toSlice(key));
-		dirtyShards->insert(it.value()->physicalShard);
-	}
-
-	void clearRange(KeyRangeRef range) {
-		auto rangeIterator = dataShardMap.intersectingRanges(range);
-
-		for (auto it = rangeIterator.begin(); it != rangeIterator.end(); ++it) {
-			if (it.value() == nullptr) {
-				TraceEvent(SevDebug, "ShardedRocksDB").detail("ClearNonExistentRange", it.range());
-				continue;
-			}
-			writeBatch->DeleteRange(it.value()->physicalShard->cf, toSlice(range.begin), toSlice(range.end));
-			dirtyShards->insert(it.value()->physicalShard);
-		}
-	}
-
-	void populateRangeMappingMutations(rocksdb::WriteBatch* writeBatch, KeyRangeRef range, bool isAdd) {
-		TraceEvent(SevDebug, "ShardedRocksDB", this->logId)
-		    .detail("Info", "RangeToPersist")
-		    .detail("BeginKey", range.begin)
-		    .detail("EndKey", range.end);
-		PhysicalShard* metadataShard = getMetaDataShard();
-
-		writeBatch->DeleteRange(metadataShard->cf,
-		                        getShardMappingKey(range.begin, shardMappingPrefix),
-		                        getShardMappingKey(range.end, shardMappingPrefix));
-
-		KeyRef lastKey = range.end;
-		if (isAdd) {
-			auto ranges = dataShardMap.intersectingRanges(range);
-			for (auto it = ranges.begin(); it != ranges.end(); ++it) {
-				if (it.value()) {
-					ASSERT(it.range() == it.value()->range);
-					// Non-empty range.
-					writeBatch->Put(metadataShard->cf,
-					                getShardMappingKey(it.range().begin, shardMappingPrefix),
-					                it.value()->physicalShard->id);
-					TraceEvent(SevDebug, "ShardedRocksDB", this->logId)
-					    .detail("Action", "PersistRangeMapping")
-					    .detail("BeginKey", it.range().begin)
-					    .detail("EndKey", it.range().end)
-					    .detail("PhysicalShardID", it.value()->physicalShard->id);
-
-				} else {
-					// Empty range.
-					writeBatch->Put(metadataShard->cf, getShardMappingKey(it.range().begin, shardMappingPrefix), "");
-					TraceEvent(SevDebug, "ShardedRocksDB", this->logId)
-					    .detail("Action", "PersistRangeMapping")
-					    .detail("BeginKey", it.range().begin)
-					    .detail("EndKey", it.range().end)
-					    .detail("PhysicalShardID", "None");
-				}
-				lastKey = it.range().end;
-			}
-		} else {
-			writeBatch->Put(metadataShard->cf, getShardMappingKey(range.begin, shardMappingPrefix), "");
-			TraceEvent(SevDebug, "ShardedRocksDB", this->logId)
-			    .detail("Action", "PersistRangeMapping")
-			    .detail("RemoveRange", "True")
-			    .detail("BeginKey", range.begin)
-			    .detail("EndKey", range.end);
-		}
-
-		DataShard* nextShard = nullptr;
-		if (lastKey <= allKeys.end) {
-			nextShard = dataShardMap.rangeContaining(lastKey).value();
-		}
-		writeBatch->Put(metadataShard->cf,
-		                getShardMappingKey(lastKey, shardMappingPrefix),
-		                nextShard == nullptr ? "" : nextShard->physicalShard->id);
-		TraceEvent(SevDebug, "ShardedRocksDB", this->logId)
-		    .detail("Action", "PersistRangeMappingEnd")
-		    .detail("NextShardKey", lastKey)
-		    .detail("Value", nextShard == nullptr ? "" : nextShard->physicalShard->id);
-	}
-
-	void persistRangeMapping(KeyRangeRef range, bool isAdd) {
-		populateRangeMappingMutations(writeBatch.get(), range, isAdd);
-		dirtyShards->insert(getMetaDataShard());
-	}
-
-	std::unique_ptr<rocksdb::WriteBatch> getWriteBatch() {
-		std::unique_ptr<rocksdb::WriteBatch> existingWriteBatch = std::move(writeBatch);
-		writeBatch = std::make_unique<rocksdb::WriteBatch>();
-		return existingWriteBatch;
-	}
-
-	std::unique_ptr<std::set<PhysicalShard*>> getDirtyShards() {
-		std::unique_ptr<std::set<PhysicalShard*>> existingShards = std::move(dirtyShards);
-		dirtyShards = std::make_unique<std::set<PhysicalShard*>>();
-		return existingShards;
-	}
-
-	void closeAllShards() {
-		columnFamilyMap.clear();
-		physicalShards.clear();
-		// Close DB.
-		auto s = db->Close();
-		if (!s.ok()) {
-			logRocksDBError(s, "Close");
-			return;
-		}
-		TraceEvent("ShardedRocksDB", this->logId).detail("Info", "DBClosed");
-	}
-
-	void destroyAllShards() {
-		columnFamilyMap.clear();
-		for (auto& [_, shard] : physicalShards) {
-			shard->deletePending = true;
-		}
-		physicalShards.clear();
-		// Close DB.
-		auto s = db->Close();
-		if (!s.ok()) {
-			logRocksDBError(s, "Close");
-			return;
-		}
-		s = rocksdb::DestroyDB(path, getOptions());
-		if (!s.ok()) {
-			logRocksDBError(s, "DestroyDB");
-		}
-		TraceEvent("ShardedRocksDB", this->logId).detail("Info", "DBDestroyed");
-	}
-
-	rocksdb::DB* getDb() const { return db; }
-
-	std::unordered_map<std::string, std::shared_ptr<PhysicalShard>>* getAllShards() { return &physicalShards; }
-
-	std::unordered_map<uint32_t, rocksdb::ColumnFamilyHandle*>* getColumnFamilyMap() { return &columnFamilyMap; }
-
-	std::vector<rocksdb::ColumnFamilyHandle*> getColumnFamilies() {
-		std::vector<rocksdb::ColumnFamilyHandle*> res;
-		for (auto& [id, cf] : columnFamilyMap) {
-			res.push_back(cf);
-		}
-		return res;
-	}
-
-	size_t numPhysicalShards() const { return physicalShards.size(); };
-
-	size_t numActiveShards() const { return activePhysicalShardIds.size(); };
-
-	std::vector<std::pair<KeyRange, std::string>> getDataMapping() {
-		std::vector<std::pair<KeyRange, std::string>> dataMap;
-		for (auto it : dataShardMap.ranges()) {
-			if (!it.value()) {
-				continue;
-			}
-			dataMap.push_back(std::make_pair(it.range(), it.value()->physicalShard->id));
-		}
-		return dataMap;
-	}
-
-	CoalescedKeyRangeMap<std::string> getExistingRanges() {
-		CoalescedKeyRangeMap<std::string> existingRanges;
-		existingRanges.insert(allKeys, "");
-		for (auto it : dataShardMap.intersectingRanges(allKeys)) {
-			if (!it.value()) {
-				continue;
-			}
-
-			if (it.value()->physicalShard->id == "kvs-metadata") {
-				continue;
-			}
-
-			existingRanges.insert(it.range(), it.value()->physicalShard->id);
-		}
-		return existingRanges;
-	}
-
-	void validate() {
-		TraceEvent(SevVerbose, "ShardedRocksValidateShardManager", this->logId);
-		for (auto s = dataShardMap.ranges().begin(); s != dataShardMap.ranges().end(); ++s) {
-			TraceEvent e(SevVerbose, "ShardedRocksValidateDataShardMap", this->logId);
-			e.detail("Range", s->range());
-			const DataShard* shard = s->value();
-			e.detail("ShardAddress", reinterpret_cast<std::uintptr_t>(shard));
-			if (shard != nullptr) {
-				e.detail("PhysicalShard", shard->physicalShard->id);
-			} else {
-				e.detail("Shard", "Empty");
-			}
-			if (shard != nullptr) {
-				if (shard->range != static_cast<KeyRangeRef>(s->range())) {
-					TraceEvent(SevWarn, "ShardRangeMismatch").detail("Range", s->range());
-				}
-
-				ASSERT(shard->range == static_cast<KeyRangeRef>(s->range()));
-				ASSERT(shard->physicalShard != nullptr);
-				auto it = shard->physicalShard->dataShards.find(shard->range.begin.toString());
-				ASSERT(it != shard->physicalShard->dataShards.end());
-				ASSERT(it->second.get() == shard);
-			}
-		}
-	}
-
-private:
-	const std::string path;
-	const UID logId;
-	rocksdb::Options dbOptions;
-	rocksdb::DB* db = nullptr;
-	std::unordered_map<std::string, std::shared_ptr<PhysicalShard>> physicalShards;
-	std::unordered_set<std::string> activePhysicalShardIds;
-	// Stores mapping between cf id and cf handle, used during compaction.
-	std::unordered_map<uint32_t, rocksdb::ColumnFamilyHandle*> columnFamilyMap;
-	std::unique_ptr<rocksdb::WriteBatch> writeBatch;
-	std::unique_ptr<std::set<PhysicalShard*>> dirtyShards;
-	KeyRangeMap<DataShard*> dataShardMap;
-	std::deque<std::string> pendingDeletionShards;
-};
-
-class RocksDBMetrics {
-public:
-	RocksDBMetrics(UID debugID, std::shared_ptr<rocksdb::Statistics> stats);
-	void logStats(rocksdb::DB* db);
-	// PerfContext
-	void resetPerfContext();
-	void collectPerfContext(int index);
-	void logPerfContext(bool ignoreZeroMetric);
-	// For Readers
-	Reference<Histogram> getReadRangeLatencyHistogram(int index);
-	Reference<Histogram> getReadValueLatencyHistogram(int index);
-	Reference<Histogram> getReadPrefixLatencyHistogram(int index);
-	Reference<Histogram> getReadRangeActionHistogram(int index);
-	Reference<Histogram> getReadValueActionHistogram(int index);
-	Reference<Histogram> getReadPrefixActionHistogram(int index);
-	Reference<Histogram> getReadRangeQueueWaitHistogram(int index);
-	Reference<Histogram> getReadValueQueueWaitHistogram(int index);
-	Reference<Histogram> getReadPrefixQueueWaitHistogram(int index);
-	Reference<Histogram> getReadRangeNewIteratorHistogram(int index);
-	Reference<Histogram> getReadValueGetHistogram(int index);
-	Reference<Histogram> getReadPrefixGetHistogram(int index);
-	// For Writer
-	Reference<Histogram> getCommitLatencyHistogram();
-	Reference<Histogram> getCommitActionHistogram();
-	Reference<Histogram> getCommitQueueWaitHistogram();
-	Reference<Histogram> getWriteHistogram();
-	Reference<Histogram> getDeleteCompactRangeHistogram();
-	// Stat for Memory Usage
-	void logMemUsage(rocksdb::DB* db);
-
-private:
-	const UID debugID;
-	// Global Statistic Input to RocksDB DB instance
-	std::shared_ptr<rocksdb::Statistics> stats;
-	// Statistic Output from RocksDB
-	std::vector<std::tuple<const char*, uint32_t, uint64_t>> tickerStats;
-	std::vector<std::pair<const char*, std::string>> intPropertyStats;
-	// Iterator Pool Stats
-	std::unordered_map<std::string, uint64_t> readIteratorPoolStats;
-	// PerfContext
-	std::vector<std::tuple<const char*, int, std::vector<uint64_t>>> perfContextMetrics;
-	// Readers Histogram
-	std::vector<Reference<Histogram>> readRangeLatencyHistograms;
-	std::vector<Reference<Histogram>> readValueLatencyHistograms;
-	std::vector<Reference<Histogram>> readPrefixLatencyHistograms;
-	std::vector<Reference<Histogram>> readRangeActionHistograms;
-	std::vector<Reference<Histogram>> readValueActionHistograms;
-	std::vector<Reference<Histogram>> readPrefixActionHistograms;
-	std::vector<Reference<Histogram>> readRangeQueueWaitHistograms;
-	std::vector<Reference<Histogram>> readValueQueueWaitHistograms;
-	std::vector<Reference<Histogram>> readPrefixQueueWaitHistograms;
-	std::vector<Reference<Histogram>> readRangeNewIteratorHistograms; // Zhe: haven't used?
-	std::vector<Reference<Histogram>> readValueGetHistograms;
-	std::vector<Reference<Histogram>> readPrefixGetHistograms;
-	// Writer Histogram
-	Reference<Histogram> commitLatencyHistogram;
-	Reference<Histogram> commitActionHistogram;
-	Reference<Histogram> commitQueueWaitHistogram;
-	Reference<Histogram> writeHistogram;
-	Reference<Histogram> deleteCompactRangeHistogram;
-
-	uint64_t getRocksdbPerfcontextMetric(int metric);
-};
-
-// We have 4 readers and 1 writer. Following input index denotes the
-// id assigned to the reader thread when creating it
-Reference<Histogram> RocksDBMetrics::getReadRangeLatencyHistogram(int index) {
-	return readRangeLatencyHistograms[index];
-}
-Reference<Histogram> RocksDBMetrics::getReadValueLatencyHistogram(int index) {
-	return readValueLatencyHistograms[index];
-}
-Reference<Histogram> RocksDBMetrics::getReadPrefixLatencyHistogram(int index) {
-	return readPrefixLatencyHistograms[index];
-}
-Reference<Histogram> RocksDBMetrics::getReadRangeActionHistogram(int index) {
-	return readRangeActionHistograms[index];
-}
-Reference<Histogram> RocksDBMetrics::getReadValueActionHistogram(int index) {
-	return readValueActionHistograms[index];
-}
-Reference<Histogram> RocksDBMetrics::getReadPrefixActionHistogram(int index) {
-	return readPrefixActionHistograms[index];
-}
-Reference<Histogram> RocksDBMetrics::getReadRangeQueueWaitHistogram(int index) {
-	return readRangeQueueWaitHistograms[index];
-}
-Reference<Histogram> RocksDBMetrics::getReadValueQueueWaitHistogram(int index) {
-	return readValueQueueWaitHistograms[index];
-}
-Reference<Histogram> RocksDBMetrics::getReadPrefixQueueWaitHistogram(int index) {
-	return readPrefixQueueWaitHistograms[index];
-}
-Reference<Histogram> RocksDBMetrics::getReadRangeNewIteratorHistogram(int index) {
-	return readRangeNewIteratorHistograms[index];
-}
-Reference<Histogram> RocksDBMetrics::getReadValueGetHistogram(int index) {
-	return readValueGetHistograms[index];
-}
-Reference<Histogram> RocksDBMetrics::getReadPrefixGetHistogram(int index) {
-	return readPrefixGetHistograms[index];
-}
-Reference<Histogram> RocksDBMetrics::getCommitLatencyHistogram() {
-	return commitLatencyHistogram;
-}
-Reference<Histogram> RocksDBMetrics::getCommitActionHistogram() {
-	return commitActionHistogram;
-}
-Reference<Histogram> RocksDBMetrics::getCommitQueueWaitHistogram() {
-	return commitQueueWaitHistogram;
-}
-Reference<Histogram> RocksDBMetrics::getWriteHistogram() {
-	return writeHistogram;
-}
-Reference<Histogram> RocksDBMetrics::getDeleteCompactRangeHistogram() {
-	return deleteCompactRangeHistogram;
-}
-
-RocksDBMetrics::RocksDBMetrics(UID debugID, std::shared_ptr<rocksdb::Statistics> stats)
-  : debugID(debugID), stats(stats) {
-	tickerStats = {
-		{ "StallMicros", rocksdb::STALL_MICROS, 0 },
-		{ "BytesRead", rocksdb::BYTES_READ, 0 },
-		{ "IterBytesRead", rocksdb::ITER_BYTES_READ, 0 },
-		{ "BytesWritten", rocksdb::BYTES_WRITTEN, 0 },
-		{ "BlockCacheMisses", rocksdb::BLOCK_CACHE_MISS, 0 },
-		{ "BlockCacheHits", rocksdb::BLOCK_CACHE_HIT, 0 },
-		{ "BloomFilterUseful", rocksdb::BLOOM_FILTER_USEFUL, 0 },
-		{ "BloomFilterFullPositive", rocksdb::BLOOM_FILTER_FULL_POSITIVE, 0 },
-		{ "BloomFilterTruePositive", rocksdb::BLOOM_FILTER_FULL_TRUE_POSITIVE, 0 },
-		// Deprecated in RocksDB 8.0
-		// { "BloomFilterMicros", rocksdb::BLOOM_FILTER_MICROS, 0 },
-		{ "MemtableHit", rocksdb::MEMTABLE_HIT, 0 },
-		{ "MemtableMiss", rocksdb::MEMTABLE_MISS, 0 },
-		{ "GetHitL0", rocksdb::GET_HIT_L0, 0 },
-		{ "GetHitL1", rocksdb::GET_HIT_L1, 0 },
-		{ "GetHitL2AndUp", rocksdb::GET_HIT_L2_AND_UP, 0 },
-		{ "CountKeysWritten", rocksdb::NUMBER_KEYS_WRITTEN, 0 },
-		{ "CountKeysRead", rocksdb::NUMBER_KEYS_READ, 0 },
-		{ "CountDBSeek", rocksdb::NUMBER_DB_SEEK, 0 },
-		{ "CountDBNext", rocksdb::NUMBER_DB_NEXT, 0 },
-		{ "CountDBPrev", rocksdb::NUMBER_DB_PREV, 0 },
-		{ "BloomFilterPrefixChecked", rocksdb::BLOOM_FILTER_PREFIX_CHECKED, 0 },
-		{ "BloomFilterPrefixUseful", rocksdb::BLOOM_FILTER_PREFIX_USEFUL, 0 },
-		// Deprecated in RocksDB 8.0
-		// { "BlockCacheCompressedMiss", rocksdb::BLOCK_CACHE_COMPRESSED_MISS, 0 },
-		// { "BlockCacheCompressedHit", rocksdb::BLOCK_CACHE_COMPRESSED_HIT, 0 },
-		{ "CountWalFileSyncs", rocksdb::WAL_FILE_SYNCED, 0 },
-		{ "CountWalFileBytes", rocksdb::WAL_FILE_BYTES, 0 },
-		{ "CompactReadBytes", rocksdb::COMPACT_READ_BYTES, 0 },
-		{ "CompactWriteBytes", rocksdb::COMPACT_WRITE_BYTES, 0 },
-		{ "FlushWriteBytes", rocksdb::FLUSH_WRITE_BYTES, 0 },
-		{ "CountBlocksCompressed", rocksdb::NUMBER_BLOCK_COMPRESSED, 0 },
-		{ "CountBlocksDecompressed", rocksdb::NUMBER_BLOCK_DECOMPRESSED, 0 },
-		{ "RowCacheHit", rocksdb::ROW_CACHE_HIT, 0 },
-		{ "RowCacheMiss", rocksdb::ROW_CACHE_MISS, 0 },
-		{ "CountIterSkippedKeys", rocksdb::NUMBER_ITER_SKIP, 0 },
-
-	};
-
-	intPropertyStats = {
-		{ "NumImmutableMemtables", rocksdb::DB::Properties::kNumImmutableMemTable },
-		{ "NumImmutableMemtablesFlushed", rocksdb::DB::Properties::kNumImmutableMemTableFlushed },
-		{ "IsMemtableFlushPending", rocksdb::DB::Properties::kMemTableFlushPending },
-		{ "NumRunningFlushes", rocksdb::DB::Properties::kNumRunningFlushes },
-		{ "IsCompactionPending", rocksdb::DB::Properties::kCompactionPending },
-		{ "NumRunningCompactions", rocksdb::DB::Properties::kNumRunningCompactions },
-		{ "CumulativeBackgroundErrors", rocksdb::DB::Properties::kBackgroundErrors },
-		{ "CurrentSizeActiveMemtable", rocksdb::DB::Properties::kCurSizeActiveMemTable },
-		{ "AllMemtablesBytes", rocksdb::DB::Properties::kCurSizeAllMemTables },
-		{ "ActiveMemtableBytes", rocksdb::DB::Properties::kSizeAllMemTables },
-		{ "CountEntriesActiveMemtable", rocksdb::DB::Properties::kNumEntriesActiveMemTable },
-		{ "CountEntriesImmutMemtables", rocksdb::DB::Properties::kNumEntriesImmMemTables },
-		{ "CountDeletesActiveMemtable", rocksdb::DB::Properties::kNumDeletesActiveMemTable },
-		{ "CountDeletesImmutMemtables", rocksdb::DB::Properties::kNumDeletesImmMemTables },
-		{ "EstimatedCountKeys", rocksdb::DB::Properties::kEstimateNumKeys },
-		{ "EstimateSstReaderBytes", rocksdb::DB::Properties::kEstimateTableReadersMem },
-		{ "CountActiveSnapshots", rocksdb::DB::Properties::kNumSnapshots },
-		{ "OldestSnapshotTime", rocksdb::DB::Properties::kOldestSnapshotTime },
-		{ "CountLiveVersions", rocksdb::DB::Properties::kNumLiveVersions },
-		{ "EstimateLiveDataSize", rocksdb::DB::Properties::kEstimateLiveDataSize },
-		{ "BaseLevel", rocksdb::DB::Properties::kBaseLevel },
-		{ "EstPendCompactBytes", rocksdb::DB::Properties::kEstimatePendingCompactionBytes },
-		{ "BlockCacheUsage", rocksdb::DB::Properties::kBlockCacheUsage },
-		{ "BlockCachePinnedUsage", rocksdb::DB::Properties::kBlockCachePinnedUsage },
-		{ "LiveSstFilesSize", rocksdb::DB::Properties::kLiveSstFilesSize },
-	};
-
-	perfContextMetrics = {
-		{ "UserKeyComparisonCount", rocksdb_user_key_comparison_count, {} },
-		{ "BlockCacheHitCount", rocksdb_block_cache_hit_count, {} },
-		{ "BlockReadCount", rocksdb_block_read_count, {} },
-		{ "BlockReadByte", rocksdb_block_read_byte, {} },
-		{ "BlockReadTime", rocksdb_block_read_time, {} },
-		{ "BlockChecksumTime", rocksdb_block_checksum_time, {} },
-		{ "BlockDecompressTime", rocksdb_block_decompress_time, {} },
-		{ "GetReadBytes", rocksdb_get_read_bytes, {} },
-		{ "MultigetReadBytes", rocksdb_multiget_read_bytes, {} },
-		{ "IterReadBytes", rocksdb_iter_read_bytes, {} },
-		{ "InternalKeySkippedCount", rocksdb_internal_key_skipped_count, {} },
-		{ "InternalDeleteSkippedCount", rocksdb_internal_delete_skipped_count, {} },
-		{ "InternalRecentSkippedCount", rocksdb_internal_recent_skipped_count, {} },
-		{ "InternalMergeCount", rocksdb_internal_merge_count, {} },
-		{ "GetSnapshotTime", rocksdb_get_snapshot_time, {} },
-		{ "GetFromMemtableTime", rocksdb_get_from_memtable_time, {} },
-		{ "GetFromMemtableCount", rocksdb_get_from_memtable_count, {} },
-		{ "GetPostProcessTime", rocksdb_get_post_process_time, {} },
-		{ "GetFromOutputFilesTime", rocksdb_get_from_output_files_time, {} },
-		{ "SeekOnMemtableTime", rocksdb_seek_on_memtable_time, {} },
-		{ "SeekOnMemtableCount", rocksdb_seek_on_memtable_count, {} },
-		{ "NextOnMemtableCount", rocksdb_next_on_memtable_count, {} },
-		{ "PrevOnMemtableCount", rocksdb_prev_on_memtable_count, {} },
-		{ "SeekChildSeekTime", rocksdb_seek_child_seek_time, {} },
-		{ "SeekChildSeekCount", rocksdb_seek_child_seek_count, {} },
-		{ "SeekMinHeapTime", rocksdb_seek_min_heap_time, {} },
-		{ "SeekMaxHeapTime", rocksdb_seek_max_heap_time, {} },
-		{ "SeekInternalSeekTime", rocksdb_seek_internal_seek_time, {} },
-		{ "FindNextUserEntryTime", rocksdb_find_next_user_entry_time, {} },
-		{ "WriteWalTime", rocksdb_write_wal_time, {} },
-		{ "WriteMemtableTime", rocksdb_write_memtable_time, {} },
-		{ "WriteDelayTime", rocksdb_write_delay_time, {} },
-		{ "WritePreAndPostProcessTime", rocksdb_write_pre_and_post_process_time, {} },
-		{ "DbMutexLockNanos", rocksdb_db_mutex_lock_nanos, {} },
-		{ "DbConditionWaitNanos", rocksdb_db_condition_wait_nanos, {} },
-		{ "MergeOperatorTimeNanos", rocksdb_merge_operator_time_nanos, {} },
-		{ "ReadIndexBlockNanos", rocksdb_read_index_block_nanos, {} },
-		{ "ReadFilterBlockNanos", rocksdb_read_filter_block_nanos, {} },
-		{ "NewTableBlockIterNanos", rocksdb_new_table_block_iter_nanos, {} },
-		{ "NewTableIteratorNanos", rocksdb_new_table_iterator_nanos, {} },
-		{ "BlockSeekNanos", rocksdb_block_seek_nanos, {} },
-		{ "FindTableNanos", rocksdb_find_table_nanos, {} },
-		{ "BloomMemtableHitCount", rocksdb_bloom_memtable_hit_count, {} },
-		{ "BloomMemtableMissCount", rocksdb_bloom_memtable_miss_count, {} },
-		{ "BloomSstHitCount", rocksdb_bloom_sst_hit_count, {} },
-		{ "BloomSstMissCount", rocksdb_bloom_sst_miss_count, {} },
-		{ "KeyLockWaitTime", rocksdb_key_lock_wait_time, {} },
-		{ "KeyLockWaitCount", rocksdb_key_lock_wait_count, {} },
-		{ "EnvNewSequentialFileNanos", rocksdb_env_new_sequential_file_nanos, {} },
-		{ "EnvNewRandomAccessFileNanos", rocksdb_env_new_random_access_file_nanos, {} },
-		{ "EnvNewWritableFileNanos", rocksdb_env_new_writable_file_nanos, {} },
-		{ "EnvReuseWritableFileNanos", rocksdb_env_reuse_writable_file_nanos, {} },
-		{ "EnvNewRandomRwFileNanos", rocksdb_env_new_random_rw_file_nanos, {} },
-		{ "EnvNewDirectoryNanos", rocksdb_env_new_directory_nanos, {} },
-		{ "EnvFileExistsNanos", rocksdb_env_file_exists_nanos, {} },
-		{ "EnvGetChildrenNanos", rocksdb_env_get_children_nanos, {} },
-		{ "EnvGetChildrenFileAttributesNanos", rocksdb_env_get_children_file_attributes_nanos, {} },
-		{ "EnvDeleteFileNanos", rocksdb_env_delete_file_nanos, {} },
-		{ "EnvCreateDirNanos", rocksdb_env_create_dir_nanos, {} },
-		{ "EnvCreateDirIfMissingNanos", rocksdb_env_create_dir_if_missing_nanos, {} },
-		{ "EnvDeleteDirNanos", rocksdb_env_delete_dir_nanos, {} },
-		{ "EnvGetFileSizeNanos", rocksdb_env_get_file_size_nanos, {} },
-		{ "EnvGetFileModificationTimeNanos", rocksdb_env_get_file_modification_time_nanos, {} },
-		{ "EnvRenameFileNanos", rocksdb_env_rename_file_nanos, {} },
-		{ "EnvLinkFileNanos", rocksdb_env_link_file_nanos, {} },
-		{ "EnvLockFileNanos", rocksdb_env_lock_file_nanos, {} },
-		{ "EnvUnlockFileNanos", rocksdb_env_unlock_file_nanos, {} },
-		{ "EnvNewLoggerNanos", rocksdb_env_new_logger_nanos, {} },
-	};
-	for (auto& [name, metric, vals] : perfContextMetrics) { // readers, then writer
-		for (int i = 0; i < SERVER_KNOBS->ROCKSDB_READ_PARALLELISM; i++) {
-			vals.push_back(0); // add reader
-		}
-		vals.push_back(0); // add writer
-	}
-	for (int i = 0; i < SERVER_KNOBS->ROCKSDB_READ_PARALLELISM; i++) {
-		readRangeLatencyHistograms.push_back(Histogram::getHistogram(
-		    ROCKSDBSTORAGE_HISTOGRAM_GROUP, ROCKSDB_READRANGE_LATENCY_HISTOGRAM, Histogram::Unit::milliseconds));
-		readValueLatencyHistograms.push_back(Histogram::getHistogram(
-		    ROCKSDBSTORAGE_HISTOGRAM_GROUP, ROCKSDB_READVALUE_LATENCY_HISTOGRAM, Histogram::Unit::milliseconds));
-		readPrefixLatencyHistograms.push_back(Histogram::getHistogram(
-		    ROCKSDBSTORAGE_HISTOGRAM_GROUP, ROCKSDB_READPREFIX_LATENCY_HISTOGRAM, Histogram::Unit::milliseconds));
-		readRangeActionHistograms.push_back(Histogram::getHistogram(
-		    ROCKSDBSTORAGE_HISTOGRAM_GROUP, ROCKSDB_READRANGE_ACTION_HISTOGRAM, Histogram::Unit::milliseconds));
-		readValueActionHistograms.push_back(Histogram::getHistogram(
-		    ROCKSDBSTORAGE_HISTOGRAM_GROUP, ROCKSDB_READVALUE_ACTION_HISTOGRAM, Histogram::Unit::milliseconds));
-		readPrefixActionHistograms.push_back(Histogram::getHistogram(
-		    ROCKSDBSTORAGE_HISTOGRAM_GROUP, ROCKSDB_READPREFIX_ACTION_HISTOGRAM, Histogram::Unit::milliseconds));
-		readRangeQueueWaitHistograms.push_back(Histogram::getHistogram(
-		    ROCKSDBSTORAGE_HISTOGRAM_GROUP, ROCKSDB_READRANGE_QUEUEWAIT_HISTOGRAM, Histogram::Unit::milliseconds));
-		readValueQueueWaitHistograms.push_back(Histogram::getHistogram(
-		    ROCKSDBSTORAGE_HISTOGRAM_GROUP, ROCKSDB_READVALUE_QUEUEWAIT_HISTOGRAM, Histogram::Unit::milliseconds));
-		readPrefixQueueWaitHistograms.push_back(Histogram::getHistogram(
-		    ROCKSDBSTORAGE_HISTOGRAM_GROUP, ROCKSDB_READPREFIX_QUEUEWAIT_HISTOGRAM, Histogram::Unit::milliseconds));
-		readRangeNewIteratorHistograms.push_back(Histogram::getHistogram(
-		    ROCKSDBSTORAGE_HISTOGRAM_GROUP, ROCKSDB_READRANGE_NEWITERATOR_HISTOGRAM, Histogram::Unit::milliseconds));
-		readValueGetHistograms.push_back(Histogram::getHistogram(
-		    ROCKSDBSTORAGE_HISTOGRAM_GROUP, ROCKSDB_READVALUE_GET_HISTOGRAM, Histogram::Unit::milliseconds));
-		readPrefixGetHistograms.push_back(Histogram::getHistogram(
-		    ROCKSDBSTORAGE_HISTOGRAM_GROUP, ROCKSDB_READPREFIX_GET_HISTOGRAM, Histogram::Unit::milliseconds));
-	}
-	commitLatencyHistogram = Histogram::getHistogram(
-	    ROCKSDBSTORAGE_HISTOGRAM_GROUP, ROCKSDB_COMMIT_LATENCY_HISTOGRAM, Histogram::Unit::milliseconds);
-	commitActionHistogram = Histogram::getHistogram(
-	    ROCKSDBSTORAGE_HISTOGRAM_GROUP, ROCKSDB_COMMIT_ACTION_HISTOGRAM, Histogram::Unit::milliseconds);
-	commitQueueWaitHistogram = Histogram::getHistogram(
-	    ROCKSDBSTORAGE_HISTOGRAM_GROUP, ROCKSDB_COMMIT_QUEUEWAIT_HISTOGRAM, Histogram::Unit::milliseconds);
-	writeHistogram =
-	    Histogram::getHistogram(ROCKSDBSTORAGE_HISTOGRAM_GROUP, ROCKSDB_WRITE_HISTOGRAM, Histogram::Unit::milliseconds);
-	deleteCompactRangeHistogram = Histogram::getHistogram(
-	    ROCKSDBSTORAGE_HISTOGRAM_GROUP, ROCKSDB_DELETE_COMPACTRANGE_HISTOGRAM, Histogram::Unit::milliseconds);
-}
-
-void RocksDBMetrics::logStats(rocksdb::DB* db) {
-	TraceEvent e(SevInfo, "ShardedRocksDBMetrics", debugID);
-	uint64_t stat;
-	for (auto& [name, ticker, cumulation] : tickerStats) {
-		stat = stats->getTickerCount(ticker);
-		e.detail(name, stat - cumulation);
-		cumulation = stat;
-	}
-	for (auto& [name, property] : intPropertyStats) {
-		stat = 0;
-		ASSERT(db->GetAggregatedIntProperty(property, &stat));
-		e.detail(name, stat);
-	}
-}
-
-void RocksDBMetrics::logMemUsage(rocksdb::DB* db) {
-	TraceEvent e(SevInfo, "ShardedRocksDBMemMetrics", debugID);
-	uint64_t stat;
-	ASSERT(db != nullptr);
-	ASSERT(db->GetAggregatedIntProperty(rocksdb::DB::Properties::kBlockCacheUsage, &stat));
-	e.detail("BlockCacheUsage", stat);
-	ASSERT(db->GetAggregatedIntProperty(rocksdb::DB::Properties::kEstimateTableReadersMem, &stat));
-	e.detail("EstimateSstReaderBytes", stat);
-	ASSERT(db->GetAggregatedIntProperty(rocksdb::DB::Properties::kCurSizeAllMemTables, &stat));
-	e.detail("AllMemtablesBytes", stat);
-	ASSERT(db->GetAggregatedIntProperty(rocksdb::DB::Properties::kBlockCachePinnedUsage, &stat));
-	e.detail("BlockCachePinnedUsage", stat);
-}
-
-void RocksDBMetrics::resetPerfContext() {
-	rocksdb::SetPerfLevel(rocksdb::PerfLevel::kEnableCount);
-	rocksdb::get_perf_context()->Reset();
-}
-
-void RocksDBMetrics::collectPerfContext(int index) {
-	for (auto& [name, metric, vals] : perfContextMetrics) {
-		vals[index] = getRocksdbPerfcontextMetric(metric);
-	}
-}
-
-void RocksDBMetrics::logPerfContext(bool ignoreZeroMetric) {
-	TraceEvent e(SevInfo, "ShardedRocksDBPerfContextMetrics", debugID);
-	e.setMaxEventLength(20000);
-	for (auto& [name, metric, vals] : perfContextMetrics) {
-		uint64_t s = 0;
-		for (auto& v : vals) {
-			s = s + v;
-		}
-		if (ignoreZeroMetric && s == 0)
-			continue;
-		e.detail(name, s);
-	}
-}
-
-uint64_t RocksDBMetrics::getRocksdbPerfcontextMetric(int metric) {
-	switch (metric) {
-	case rocksdb_user_key_comparison_count:
-		return rocksdb::get_perf_context()->user_key_comparison_count;
-	case rocksdb_block_cache_hit_count:
-		return rocksdb::get_perf_context()->block_cache_hit_count;
-	case rocksdb_block_read_count:
-		return rocksdb::get_perf_context()->block_read_count;
-	case rocksdb_block_read_byte:
-		return rocksdb::get_perf_context()->block_read_byte;
-	case rocksdb_block_read_time:
-		return rocksdb::get_perf_context()->block_read_time;
-	case rocksdb_block_checksum_time:
-		return rocksdb::get_perf_context()->block_checksum_time;
-	case rocksdb_block_decompress_time:
-		return rocksdb::get_perf_context()->block_decompress_time;
-	case rocksdb_get_read_bytes:
-		return rocksdb::get_perf_context()->get_read_bytes;
-	case rocksdb_multiget_read_bytes:
-		return rocksdb::get_perf_context()->multiget_read_bytes;
-	case rocksdb_iter_read_bytes:
-		return rocksdb::get_perf_context()->iter_read_bytes;
-	case rocksdb_internal_key_skipped_count:
-		return rocksdb::get_perf_context()->internal_key_skipped_count;
-	case rocksdb_internal_delete_skipped_count:
-		return rocksdb::get_perf_context()->internal_delete_skipped_count;
-	case rocksdb_internal_recent_skipped_count:
-		return rocksdb::get_perf_context()->internal_recent_skipped_count;
-	case rocksdb_internal_merge_count:
-		return rocksdb::get_perf_context()->internal_merge_count;
-	case rocksdb_get_snapshot_time:
-		return rocksdb::get_perf_context()->get_snapshot_time;
-	case rocksdb_get_from_memtable_time:
-		return rocksdb::get_perf_context()->get_from_memtable_time;
-	case rocksdb_get_from_memtable_count:
-		return rocksdb::get_perf_context()->get_from_memtable_count;
-	case rocksdb_get_post_process_time:
-		return rocksdb::get_perf_context()->get_post_process_time;
-	case rocksdb_get_from_output_files_time:
-		return rocksdb::get_perf_context()->get_from_output_files_time;
-	case rocksdb_seek_on_memtable_time:
-		return rocksdb::get_perf_context()->seek_on_memtable_time;
-	case rocksdb_seek_on_memtable_count:
-		return rocksdb::get_perf_context()->seek_on_memtable_count;
-	case rocksdb_next_on_memtable_count:
-		return rocksdb::get_perf_context()->next_on_memtable_count;
-	case rocksdb_prev_on_memtable_count:
-		return rocksdb::get_perf_context()->prev_on_memtable_count;
-	case rocksdb_seek_child_seek_time:
-		return rocksdb::get_perf_context()->seek_child_seek_time;
-	case rocksdb_seek_child_seek_count:
-		return rocksdb::get_perf_context()->seek_child_seek_count;
-	case rocksdb_seek_min_heap_time:
-		return rocksdb::get_perf_context()->seek_min_heap_time;
-	case rocksdb_seek_max_heap_time:
-		return rocksdb::get_perf_context()->seek_max_heap_time;
-	case rocksdb_seek_internal_seek_time:
-		return rocksdb::get_perf_context()->seek_internal_seek_time;
-	case rocksdb_find_next_user_entry_time:
-		return rocksdb::get_perf_context()->find_next_user_entry_time;
-	case rocksdb_write_wal_time:
-		return rocksdb::get_perf_context()->write_wal_time;
-	case rocksdb_write_memtable_time:
-		return rocksdb::get_perf_context()->write_memtable_time;
-	case rocksdb_write_delay_time:
-		return rocksdb::get_perf_context()->write_delay_time;
-	case rocksdb_write_pre_and_post_process_time:
-		return rocksdb::get_perf_context()->write_pre_and_post_process_time;
-	case rocksdb_db_mutex_lock_nanos:
-		return rocksdb::get_perf_context()->db_mutex_lock_nanos;
-	case rocksdb_db_condition_wait_nanos:
-		return rocksdb::get_perf_context()->db_condition_wait_nanos;
-	case rocksdb_merge_operator_time_nanos:
-		return rocksdb::get_perf_context()->merge_operator_time_nanos;
-	case rocksdb_read_index_block_nanos:
-		return rocksdb::get_perf_context()->read_index_block_nanos;
-	case rocksdb_read_filter_block_nanos:
-		return rocksdb::get_perf_context()->read_filter_block_nanos;
-	case rocksdb_new_table_block_iter_nanos:
-		return rocksdb::get_perf_context()->new_table_block_iter_nanos;
-	case rocksdb_new_table_iterator_nanos:
-		return rocksdb::get_perf_context()->new_table_iterator_nanos;
-	case rocksdb_block_seek_nanos:
-		return rocksdb::get_perf_context()->block_seek_nanos;
-	case rocksdb_find_table_nanos:
-		return rocksdb::get_perf_context()->find_table_nanos;
-	case rocksdb_bloom_memtable_hit_count:
-		return rocksdb::get_perf_context()->bloom_memtable_hit_count;
-	case rocksdb_bloom_memtable_miss_count:
-		return rocksdb::get_perf_context()->bloom_memtable_miss_count;
-	case rocksdb_bloom_sst_hit_count:
-		return rocksdb::get_perf_context()->bloom_sst_hit_count;
-	case rocksdb_bloom_sst_miss_count:
-		return rocksdb::get_perf_context()->bloom_sst_miss_count;
-	case rocksdb_key_lock_wait_time:
-		return rocksdb::get_perf_context()->key_lock_wait_time;
-	case rocksdb_key_lock_wait_count:
-		return rocksdb::get_perf_context()->key_lock_wait_count;
-	case rocksdb_env_new_sequential_file_nanos:
-		return rocksdb::get_perf_context()->env_new_sequential_file_nanos;
-	case rocksdb_env_new_random_access_file_nanos:
-		return rocksdb::get_perf_context()->env_new_random_access_file_nanos;
-	case rocksdb_env_new_writable_file_nanos:
-		return rocksdb::get_perf_context()->env_new_writable_file_nanos;
-	case rocksdb_env_reuse_writable_file_nanos:
-		return rocksdb::get_perf_context()->env_reuse_writable_file_nanos;
-	case rocksdb_env_new_random_rw_file_nanos:
-		return rocksdb::get_perf_context()->env_new_random_rw_file_nanos;
-	case rocksdb_env_new_directory_nanos:
-		return rocksdb::get_perf_context()->env_new_directory_nanos;
-	case rocksdb_env_file_exists_nanos:
-		return rocksdb::get_perf_context()->env_file_exists_nanos;
-	case rocksdb_env_get_children_nanos:
-		return rocksdb::get_perf_context()->env_get_children_nanos;
-	case rocksdb_env_get_children_file_attributes_nanos:
-		return rocksdb::get_perf_context()->env_get_children_file_attributes_nanos;
-	case rocksdb_env_delete_file_nanos:
-		return rocksdb::get_perf_context()->env_delete_file_nanos;
-	case rocksdb_env_create_dir_nanos:
-		return rocksdb::get_perf_context()->env_create_dir_nanos;
-	case rocksdb_env_create_dir_if_missing_nanos:
-		return rocksdb::get_perf_context()->env_create_dir_if_missing_nanos;
-	case rocksdb_env_delete_dir_nanos:
-		return rocksdb::get_perf_context()->env_delete_dir_nanos;
-	case rocksdb_env_get_file_size_nanos:
-		return rocksdb::get_perf_context()->env_get_file_size_nanos;
-	case rocksdb_env_get_file_modification_time_nanos:
-		return rocksdb::get_perf_context()->env_get_file_modification_time_nanos;
-	case rocksdb_env_rename_file_nanos:
-		return rocksdb::get_perf_context()->env_rename_file_nanos;
-	case rocksdb_env_link_file_nanos:
-		return rocksdb::get_perf_context()->env_link_file_nanos;
-	case rocksdb_env_lock_file_nanos:
-		return rocksdb::get_perf_context()->env_lock_file_nanos;
-	case rocksdb_env_unlock_file_nanos:
-		return rocksdb::get_perf_context()->env_unlock_file_nanos;
-	case rocksdb_env_new_logger_nanos:
-		return rocksdb::get_perf_context()->env_new_logger_nanos;
-	default:
-		break;
-	}
-	return 0;
-}
-
-ACTOR Future<Void> rocksDBAggregatedMetricsLogger(std::shared_ptr<ShardedRocksDBState> rState,
-                                                  Future<Void> openFuture,
-                                                  std::shared_ptr<RocksDBMetrics> rocksDBMetrics,
-                                                  ShardManager* shardManager) {
-	try {
-		wait(openFuture);
-		state rocksdb::DB* db = shardManager->getDb();
-		loop {
-			wait(delay(SERVER_KNOBS->ROCKSDB_METRICS_DELAY));
-			if (rState->closing) {
-				break;
-			}
-			rocksDBMetrics->logStats(db);
-			rocksDBMetrics->logMemUsage(db);
-			if (SERVER_KNOBS->ROCKSDB_PERFCONTEXT_SAMPLE_RATE != 0) {
-				rocksDBMetrics->logPerfContext(true);
-			}
-		}
-	} catch (Error& e) {
-		if (e.code() != error_code_actor_cancelled) {
-			TraceEvent(SevError, "ShardedRocksDBMetricsError").errorUnsuppressed(e);
-		}
-	}
-	return Void();
-}
-
-struct ShardedRocksDBKeyValueStore : IKeyValueStore {
-	using CF = rocksdb::ColumnFamilyHandle*;
-
-	ACTOR static Future<Void> refreshReadIteratorPools(
-	    std::shared_ptr<ShardedRocksDBState> rState,
-	    Future<Void> readyToStart,
-	    std::unordered_map<std::string, std::shared_ptr<PhysicalShard>>* physicalShards) {
-		state Reference<Histogram> histogram = Histogram::getHistogram(
-		    ROCKSDBSTORAGE_HISTOGRAM_GROUP, "TimeSpentRefreshIterators"_sr, Histogram::Unit::milliseconds);
-
-		if (SERVER_KNOBS->ROCKSDB_READ_RANGE_REUSE_ITERATORS) {
-			try {
-				wait(readyToStart);
-				loop {
-					wait(delay(SERVER_KNOBS->ROCKSDB_READ_RANGE_ITERATOR_REFRESH_TIME));
-					if (rState->closing) {
-						break;
-					}
-					double startTime = timer_monotonic();
-					for (auto& [_, shard] : *physicalShards) {
-						if (shard->initialized()) {
-							shard->refreshReadIteratorPool();
-						}
-					}
-					histogram->sample(timer_monotonic() - startTime);
-				}
-			} catch (Error& e) {
-				if (e.code() != error_code_actor_cancelled) {
-					TraceEvent(SevError, "RefreshReadIteratorPoolError").errorUnsuppressed(e);
-				}
-			}
-		}
-
-		return Void();
-	}
-
-	ACTOR static Future<Void> doRestore(ShardedRocksDBKeyValueStore* self,
-	                                    std::string shardId,
-	                                    std::vector<KeyRange> ranges,
-	                                    std::vector<CheckpointMetaData> checkpoints) {
-		for (const KeyRange& range : ranges) {
-			std::vector<DataShard*> shards = self->shardManager.getDataShardsByRange(range);
-			if (!shards.empty()) {
-				TraceEvent te(SevWarnAlways, "RestoreRangesNotEmpty", self->id);
-				te.detail("Range", range);
-				te.detail("RestoreShardID", shardId);
-				for (int i = 0; i < shards.size(); ++i) {
-					te.detail("DataShard-" + std::to_string(i), shards[i]->range);
-				}
-				te.log();
-				throw failed_to_restore_checkpoint();
-			}
-		}
-		for (const KeyRange& range : ranges) {
-			self->shardManager.addRange(range, shardId);
-		}
-		TraceEvent(SevDebug, "ShardedRocksRestoreAddRange", self->id)
-		    .detail("Ranges", describe(ranges))
-		    .detail("ShardID", shardId)
-		    .detail("Checkpoints", describe(checkpoints));
-		auto a = new Writer::RestoreAction(&self->shardManager, self->path, shardId, ranges, checkpoints);
-		auto res = a->done.getFuture();
-		self->writeThread->post(a);
-
-		try {
-			wait(res);
-		} catch (Error& e) {
-			for (const KeyRange& range : ranges) {
-				self->shardManager.removeRange(range);
-			}
-			throw;
-		}
-
-		return Void();
-	}
-
-	struct Writer : IThreadPoolReceiver {
-		const UID logId;
-		int threadIndex;
-		std::unordered_map<uint32_t, rocksdb::ColumnFamilyHandle*>* columnFamilyMap;
-		std::shared_ptr<RocksDBMetrics> rocksDBMetrics;
-		std::shared_ptr<rocksdb::RateLimiter> rateLimiter;
-		double sampleStartTime;
-
-		explicit Writer(UID logId,
-		                int threadIndex,
-		                std::unordered_map<uint32_t, rocksdb::ColumnFamilyHandle*>* columnFamilyMap,
-		                std::shared_ptr<RocksDBMetrics> rocksDBMetrics)
-		  : logId(logId), threadIndex(threadIndex), columnFamilyMap(columnFamilyMap), rocksDBMetrics(rocksDBMetrics),
-		    rateLimiter(SERVER_KNOBS->ROCKSDB_WRITE_RATE_LIMITER_BYTES_PER_SEC > 0
-		                    ? rocksdb::NewGenericRateLimiter(
-		                          SERVER_KNOBS->ROCKSDB_WRITE_RATE_LIMITER_BYTES_PER_SEC, // rate_bytes_per_sec
-		                          100 * 1000, // refill_period_us
-		                          10, // fairness
-		                          rocksdb::RateLimiter::Mode::kWritesOnly,
-		                          SERVER_KNOBS->ROCKSDB_WRITE_RATE_LIMITER_AUTO_TUNE)
-		                    : nullptr),
-		    sampleStartTime(now()) {}
-
-		~Writer() override {}
-
-		void init() override {}
-
-		void sample() {
-			if (SERVER_KNOBS->ROCKSDB_METRICS_SAMPLE_INTERVAL > 0 &&
-			    now() - sampleStartTime >= SERVER_KNOBS->ROCKSDB_METRICS_SAMPLE_INTERVAL) {
-				rocksDBMetrics->collectPerfContext(threadIndex);
-				rocksDBMetrics->resetPerfContext();
-				sampleStartTime = now();
-			}
-		}
-
-		struct OpenAction : TypedAction<Writer, OpenAction> {
-			ShardManager* shardManager;
-			ThreadReturnPromise<Void> done;
-			Optional<Future<Void>>& metrics;
-			const FlowLock* readLock;
-			const FlowLock* fetchLock;
-			std::shared_ptr<RocksDBErrorListener> errorListener;
-
-			OpenAction(ShardManager* shardManager,
-			           Optional<Future<Void>>& metrics,
-			           const FlowLock* readLock,
-			           const FlowLock* fetchLock,
-			           std::shared_ptr<RocksDBErrorListener> errorListener)
-			  : shardManager(shardManager), metrics(metrics), readLock(readLock), fetchLock(fetchLock),
-			    errorListener(errorListener) {}
-
-			double getTimeEstimate() const override { return SERVER_KNOBS->COMMIT_TIME_ESTIMATE; }
-		};
-
-		void action(OpenAction& a) {
-			auto status = a.shardManager->init();
-
-			if (!status.ok()) {
-				logRocksDBError(status, "Open");
-				a.done.sendError(statusToError(status));
-				return;
-			}
-
-			TraceEvent(SevInfo, "ShardedRocksDB").detail("Method", "Open");
-			a.done.send(Void());
-		}
-
-		struct AddShardAction : TypedAction<Writer, AddShardAction> {
-			PhysicalShard* shard;
-			ThreadReturnPromise<Void> done;
-
-			AddShardAction(PhysicalShard* shard) : shard(shard) { ASSERT(shard); }
-			double getTimeEstimate() const override { return SERVER_KNOBS->COMMIT_TIME_ESTIMATE; }
-		};
-
-		void action(AddShardAction& a) {
-			auto s = a.shard->init();
-			if (!s.ok()) {
-				a.done.sendError(statusToError(s));
-				return;
-			}
-			ASSERT(a.shard->cf);
-			(*columnFamilyMap)[a.shard->cf->GetID()] = a.shard->cf;
-			a.done.send(Void());
-		}
-
-		struct RemoveShardAction : TypedAction<Writer, RemoveShardAction> {
-			std::vector<std::shared_ptr<PhysicalShard>> shards;
-			ThreadReturnPromise<Void> done;
-
-			RemoveShardAction(std::vector<std::shared_ptr<PhysicalShard>>& shards) : shards(shards) {}
-			double getTimeEstimate() const override { return SERVER_KNOBS->COMMIT_TIME_ESTIMATE; }
-		};
-
-		void action(RemoveShardAction& a) {
-			for (auto& shard : a.shards) {
-				shard->deletePending = true;
-				columnFamilyMap->erase(shard->cf->GetID());
-			}
-			a.shards.clear();
-			a.done.send(Void());
-		}
-
-		struct CommitAction : TypedAction<Writer, CommitAction> {
-			rocksdb::DB* db;
-			std::unique_ptr<rocksdb::WriteBatch> writeBatch;
-			std::unique_ptr<std::set<PhysicalShard*>> dirtyShards;
-			const std::unordered_map<uint32_t, rocksdb::ColumnFamilyHandle*>* columnFamilyMap;
-			ThreadReturnPromise<Void> done;
-			double startTime;
-			bool getHistograms;
-			bool logShardMemUsage;
-			double getTimeEstimate() const override { return SERVER_KNOBS->COMMIT_TIME_ESTIMATE; }
-			CommitAction(rocksdb::DB* db,
-			             std::unique_ptr<rocksdb::WriteBatch> writeBatch,
-			             std::unique_ptr<std::set<PhysicalShard*>> dirtyShards,
-			             std::unordered_map<uint32_t, rocksdb::ColumnFamilyHandle*>* columnFamilyMap)
-			  : db(db), writeBatch(std::move(writeBatch)), dirtyShards(std::move(dirtyShards)),
-			    columnFamilyMap(columnFamilyMap) {
-				if (deterministicRandom()->random01() < SERVER_KNOBS->ROCKSDB_HISTOGRAMS_SAMPLE_RATE) {
-					getHistograms = true;
-					startTime = timer_monotonic();
-				} else {
-					getHistograms = false;
-				}
-			}
-		};
-
-		struct DeleteVisitor : public rocksdb::WriteBatch::Handler {
-			std::vector<std::pair<uint32_t, KeyRange>>* deletes;
-
-			DeleteVisitor(std::vector<std::pair<uint32_t, KeyRange>>* deletes) : deletes(deletes) { ASSERT(deletes); }
-
-			rocksdb::Status DeleteRangeCF(uint32_t column_family_id,
-			                              const rocksdb::Slice& begin,
-			                              const rocksdb::Slice& end) override {
-				deletes->push_back(
-				    std::make_pair(column_family_id, KeyRange(KeyRangeRef(toStringRef(begin), toStringRef(end)))));
-				return rocksdb::Status::OK();
-			}
-
-			rocksdb::Status PutCF(uint32_t column_family_id,
-			                      const rocksdb::Slice& key,
-			                      const rocksdb::Slice& value) override {
-				return rocksdb::Status::OK();
-			}
-
-			rocksdb::Status DeleteCF(uint32_t column_family_id, const rocksdb::Slice& key) override {
-				return rocksdb::Status::OK();
-			}
-
-			rocksdb::Status SingleDeleteCF(uint32_t column_family_id, const rocksdb::Slice& key) override {
-				return rocksdb::Status::OK();
-			}
-
-			rocksdb::Status MergeCF(uint32_t column_family_id,
-			                        const rocksdb::Slice& key,
-			                        const rocksdb::Slice& value) override {
-				return rocksdb::Status::OK();
-			}
-		};
-
-		rocksdb::Status doCommit(rocksdb::WriteBatch* batch,
-		                         rocksdb::DB* db,
-		                         std::vector<std::pair<uint32_t, KeyRange>>* deletes,
-		                         bool sample) {
-			if (SERVER_KNOBS->ROCKSDB_SUGGEST_COMPACT_CLEAR_RANGE) {
-				DeleteVisitor dv(deletes);
-				rocksdb::Status s = batch->Iterate(&dv);
-				if (!s.ok()) {
-					logRocksDBError(s, "CommitDeleteVisitor");
-					return s;
-				}
-
-				// If there are any range deletes, we should have added them to be deleted.
-				ASSERT(!deletes->empty() || !batch->HasDeleteRange());
-			}
-
-			rocksdb::WriteOptions options;
-			options.sync = !SERVER_KNOBS->ROCKSDB_UNSAFE_AUTO_FSYNC;
-
-			double writeBeginTime = sample ? timer_monotonic() : 0;
-			rocksdb::Status s = db->Write(options, batch);
-			if (sample) {
-				rocksDBMetrics->getWriteHistogram()->sampleSeconds(timer_monotonic() - writeBeginTime);
-			}
-			if (!s.ok()) {
-				logRocksDBError(s, "Commit");
-				return s;
-			}
-
-			return s;
-		}
-
-		void action(CommitAction& a) {
-			double commitBeginTime;
-			if (a.getHistograms) {
-				commitBeginTime = timer_monotonic();
-				rocksDBMetrics->getCommitQueueWaitHistogram()->sampleSeconds(commitBeginTime - a.startTime);
-			}
-			std::vector<std::pair<uint32_t, KeyRange>> deletes;
-			auto s = doCommit(a.writeBatch.get(), a.db, &deletes, a.getHistograms);
-			if (!s.ok()) {
-				a.done.sendError(statusToError(s));
-				return;
-			}
-
-			for (auto shard : *(a.dirtyShards)) {
-				shard->readIterPool->update();
-			}
-
-			a.done.send(Void());
-			if (SERVER_KNOBS->ROCKSDB_SUGGEST_COMPACT_CLEAR_RANGE) {
-				for (const auto& [id, range] : deletes) {
-					auto cf = columnFamilyMap->find(id);
-					ASSERT(cf != columnFamilyMap->end());
-					auto begin = toSlice(range.begin);
-					auto end = toSlice(range.end);
-
-					ASSERT(a.db->SuggestCompactRange(cf->second, &begin, &end).ok());
-				}
-			}
-
-			if (a.getHistograms) {
-				double currTime = timer_monotonic();
-				rocksDBMetrics->getCommitActionHistogram()->sampleSeconds(currTime - commitBeginTime);
-				rocksDBMetrics->getCommitLatencyHistogram()->sampleSeconds(currTime - a.startTime);
-			}
-
-			sample();
-		}
-
-		struct CloseAction : TypedAction<Writer, CloseAction> {
-			ShardManager* shardManager;
-			ThreadReturnPromise<Void> done;
-			bool deleteOnClose;
-			CloseAction(ShardManager* shardManager, bool deleteOnClose)
-			  : shardManager(shardManager), deleteOnClose(deleteOnClose) {}
-			double getTimeEstimate() const override { return SERVER_KNOBS->COMMIT_TIME_ESTIMATE; }
-		};
-
-		void action(CloseAction& a) {
-			if (a.deleteOnClose) {
-				a.shardManager->destroyAllShards();
-			} else {
-				a.shardManager->closeAllShards();
-			}
-			TraceEvent(SevInfo, "ShardedRocksDB").detail("Method", "Close");
-			a.done.send(Void());
-		}
-
-		struct CheckpointAction : TypedAction<Writer, CheckpointAction> {
-			CheckpointAction(ShardManager* shardManager, const CheckpointRequest& request)
-			  : shardManager(shardManager), request(request) {}
-
-			double getTimeEstimate() const override { return SERVER_KNOBS->COMMIT_TIME_ESTIMATE; }
-
-			ShardManager* shardManager;
-			const CheckpointRequest request;
-			ThreadReturnPromise<CheckpointMetaData> reply;
-		};
-
-		void action(CheckpointAction& a) {
-			TraceEvent(SevInfo, "ShardedRocksCheckpointBegin", logId)
-			    .detail("CheckpointID", a.request.checkpointID)
-			    .detail("Version", a.request.version)
-			    .detail("Ranges", describe(a.request.ranges))
-			    .detail("Format", static_cast<int>(a.request.format))
-			    .detail("CheckpointDir", a.request.checkpointDir);
-
-			rocksdb::Status s;
-			rocksdb::Checkpoint* checkpoint = nullptr;
-
-			PhysicalShard* ps = a.shardManager->getPhysicalShardForAllRanges(a.request.ranges);
-			if (ps == nullptr) {
-				TraceEvent(SevInfo, "ShardedRocksCheckpointInvalidPhysicalShard", logId)
-				    .detail("CheckpointID", a.request.checkpointID)
-				    .detail("Version", a.request.version)
-				    .detail("Ranges", describe(a.request.ranges))
-				    .detail("Format", static_cast<int>(a.request.format))
-				    .detail("CheckpointDir", a.request.checkpointDir);
-				a.reply.sendError(failed_to_create_checkpoint());
-				return;
-			}
-
-			TraceEvent(SevDebug, "ShardedRocksCheckpointCF", logId)
-			    .detail("CFName", ps->cf->GetName())
-			    .detail("CheckpointID", a.request.checkpointID)
-			    .detail("Version", a.request.version)
-			    .detail("Ranges", describe(a.request.ranges))
-			    .detail("PhysicalShardRanges", describe(ps->getAllRanges()))
-			    .detail("Format", static_cast<int>(a.request.format))
-			    .detail("CheckpointDir", a.request.checkpointDir);
-
-			rocksdb::PinnableSlice value;
-			rocksdb::ReadOptions readOptions = getReadOptions();
-			s = a.shardManager->getDb()->Get(
-			    readOptions, a.shardManager->getSpecialKeysShard()->cf, toSlice(persistVersion), &value);
-
-			if (!s.ok() && !s.IsNotFound()) {
-				logRocksDBError(s, "ShardedRocksCheckpointReadPersistVersion");
-				a.reply.sendError(failed_to_create_checkpoint());
-				return;
-			}
-
-			const Version version = s.IsNotFound()
-			                            ? latestVersion
-			                            : BinaryReader::fromStringRef<Version>(toStringRef(value), Unversioned());
-
-			TraceEvent(SevDebug, "ShardedRocksCheckpointVersion", logId)
-			    .detail("CheckpointVersion", a.request.version)
-			    .detail("PersistVersion", version);
-			ASSERT(a.request.version == version || a.request.version == latestVersion);
-
-			CheckpointMetaData res(ps->getAllRanges(), version, a.request.format, a.request.checkpointID);
-			s = rocksdb::Checkpoint::Create(a.shardManager->getDb(), &checkpoint);
-			if (!s.ok()) {
-				logRocksDBError(s, "CreateRocksDBCheckpoint");
-				a.reply.sendError(failed_to_create_checkpoint());
-				return;
-			}
-
-			const std::string& checkpointDir = abspath(a.request.checkpointDir);
-			platform::eraseDirectoryRecursive(checkpointDir);
-			if (a.request.format == DataMoveRocksCF) {
-				rocksdb::ExportImportFilesMetaData* pMetadata = nullptr;
-				s = checkpoint->ExportColumnFamily(ps->cf, checkpointDir, &pMetadata);
-				if (!s.ok()) {
-					logRocksDBError(s, "CheckpointExportColumnFamily");
-					a.reply.sendError(failed_to_create_checkpoint());
-					return;
-				}
-
-				populateMetaData(&res, pMetadata);
-				rocksdb::ExportImportFilesMetaData metadata = *pMetadata;
-				delete pMetadata;
-				if (SERVER_KNOBS->ROCKSDB_ENABLE_CHECKPOINT_VALIDATION) {
-					rocksdb::ImportColumnFamilyOptions importOptions;
-					importOptions.move_files = false;
-					rocksdb::ColumnFamilyHandle* handle;
-					const std::string cfName = deterministicRandom()->randomAlphaNumeric(8);
-					s = a.shardManager->getDb()->CreateColumnFamilyWithImport(
-					    getCFOptions(), cfName, importOptions, metadata, &handle);
-					if (!s.ok()) {
-						TraceEvent(SevError, "ShardedRocksCheckpointValidateImportError", logId)
-						    .detail("Status", s.ToString())
-						    .detail("CheckpointID", a.request.checkpointID)
-						    .detail("Ranges", describe(a.request.ranges))
-						    .detail("CheckDir", a.request.checkpointDir)
-						    .detail("CheckpointVersion", a.request.version)
-						    .detail("PersistVersion", version);
-						a.reply.sendError(failed_to_create_checkpoint());
-						return;
-					}
-
-					auto options = getReadOptions();
-					rocksdb::Iterator* oriIter = a.shardManager->getDb()->NewIterator(options, ps->cf);
-					rocksdb::Iterator* resIter = a.shardManager->getDb()->NewIterator(options, handle);
-					oriIter->SeekToFirst();
-					resIter->SeekToFirst();
-					while (oriIter->Valid() && resIter->Valid()) {
-						if (oriIter->key().compare(resIter->key()) != 0 ||
-						    oriIter->value().compare(resIter->value()) != 0) {
-							TraceEvent(SevError, "ShardedRocksCheckpointValidateError", logId)
-							    .detail("KeyInDB", toStringRef(oriIter->key()))
-							    .detail("ValueInDB", toStringRef(oriIter->value()))
-							    .detail("KeyInCheckpoint", toStringRef(resIter->key()))
-							    .detail("ValueInCheckpoint", toStringRef(resIter->value()));
-
-							a.reply.sendError(failed_to_create_checkpoint());
-							return;
-						}
-						oriIter->Next();
-						resIter->Next();
-					}
-					if (oriIter->Valid() || resIter->Valid()) {
-						a.reply.sendError(failed_to_create_checkpoint());
-						return;
-					}
-					delete oriIter;
-					delete resIter;
-
-					s = a.shardManager->getDb()->DropColumnFamily(handle);
-					if (!s.ok()) {
-						logRocksDBError(s, "CheckpointDropColumnFamily");
-						a.reply.sendError(failed_to_create_checkpoint());
-						return;
-					}
-					s = a.shardManager->getDb()->DestroyColumnFamilyHandle(handle);
-					if (!s.ok()) {
-						logRocksDBError(s, "CheckpointDestroyColumnFamily");
-						a.reply.sendError(failed_to_create_checkpoint());
-						return;
-					}
-					TraceEvent(SevDebug, "ShardedRocksCheckpointValidateSuccess", logId)
-					    .detail("CheckpointVersion", a.request.version)
-					    .detail("PersistVersion", version);
-				}
-			} else {
-				if (checkpoint != nullptr) {
-					delete checkpoint;
-				}
-				a.reply.sendError(not_implemented());
-				return;
-			}
-
-			res.setState(CheckpointMetaData::Complete);
-			res.dir = a.request.checkpointDir;
-			a.reply.send(res);
-
-			if (checkpoint != nullptr) {
-				delete checkpoint;
-			}
-			TraceEvent(SevInfo, "ShardedRocksCheckpointEnd", logId).detail("Checkpoint", res.toString());
-		}
-
-		struct RestoreAction : TypedAction<Writer, RestoreAction> {
-			RestoreAction(ShardManager* shardManager,
-			              const std::string& path,
-			              const std::string& shardId,
-			              const std::vector<KeyRange>& ranges,
-			              const std::vector<CheckpointMetaData>& checkpoints)
-			  : shardManager(shardManager), path(path), shardId(shardId), ranges(ranges), checkpoints(checkpoints) {}
-
-			double getTimeEstimate() const override { return SERVER_KNOBS->COMMIT_TIME_ESTIMATE; }
-
-			ShardManager* shardManager;
-			const std::string path;
-			const std::string shardId;
-			std::vector<KeyRange> ranges;
-			std::vector<CheckpointMetaData> checkpoints;
-			ThreadReturnPromise<Void> done;
-		};
-
-		void action(RestoreAction& a) {
-			TraceEvent(SevInfo, "ShardedRocksDBRestoreBegin", logId)
-			    .detail("Path", a.path)
-			    .detail("Checkpoints", describe(a.checkpoints));
-
-			ASSERT(!a.checkpoints.empty());
-
-			PhysicalShard* ps = a.shardManager->getPhysicalShardForAllRanges(a.ranges);
-			ASSERT(ps != nullptr);
-
-			const CheckpointFormat format = a.checkpoints[0].getFormat();
-			for (int i = 1; i < a.checkpoints.size(); ++i) {
-				if (a.checkpoints[i].getFormat() != format) {
-					throw invalid_checkpoint_format();
-				}
-			}
-
-			rocksdb::Status status;
-			rocksdb::WriteBatch writeBatch;
-			rocksdb::WriteOptions options;
-			options.sync = !SERVER_KNOBS->ROCKSDB_UNSAFE_AUTO_FSYNC;
-
-			if (format == DataMoveRocksCF) {
-				if (a.checkpoints.size() > 1) {
-					TraceEvent(SevError, "ShardedRocksDBRestoreMultipleCFs", logId)
-					    .detail("Path", a.path)
-					    .detail("Checkpoints", describe(a.checkpoints));
-					a.done.sendError(failed_to_restore_checkpoint());
-					return;
-				}
-				CheckpointMetaData& checkpoint = a.checkpoints.front();
-				std::sort(a.ranges.begin(), a.ranges.end(), KeyRangeRef::ArbitraryOrder());
-				std::sort(checkpoint.ranges.begin(), checkpoint.ranges.end(), KeyRangeRef::ArbitraryOrder());
-				if (a.ranges.empty() || checkpoint.ranges.empty() || a.ranges.size() > checkpoint.ranges.size() ||
-				    a.ranges.front().begin != checkpoint.ranges.front().begin) {
-					TraceEvent(SevError, "ShardedRocksDBRestoreFailed", logId)
-					    .detail("Path", a.path)
-					    .detail("Ranges", describe(a.ranges))
-					    .detail("Checkpoints", describe(a.checkpoints));
-					a.done.sendError(failed_to_restore_checkpoint());
-					return;
-				}
-
-				for (int i = 0; i < a.ranges.size(); ++i) {
-					if (a.ranges[i] != checkpoint.ranges[i] && i != a.ranges.size() - 1) {
-						TraceEvent(SevError, "ShardedRocksDBRestoreFailed", logId)
-						    .detail("Path", a.path)
-						    .detail("Ranges", describe(a.ranges))
-						    .detail("Checkpoints", describe(a.checkpoints));
-						a.done.sendError(failed_to_restore_checkpoint());
-						return;
-					}
-				}
-
-				ASSERT(!ps->initialized());
-				TraceEvent(SevDebug, "ShardedRocksRestoringCF", logId)
-				    .detail("Path", a.path)
-				    .detail("Checkpoints", describe(a.checkpoints))
-				    .detail("RocksDBCF", getRocksCF(a.checkpoints[0]).toString());
-
-				status = ps->restore(a.checkpoints[0]);
-
-				if (!status.ok()) {
-					a.done.sendError(failed_to_restore_checkpoint());
-					return;
-				} else {
-					ASSERT(ps->initialized());
-					(*columnFamilyMap)[ps->cf->GetID()] = ps->cf;
-					TraceEvent(SevInfo, "RocksDBRestoreCFSuccess", logId)
-					    .detail("Path", a.path)
-					    .detail("ColumnFaminly", ps->cf->GetName())
-					    .detail("Checkpoints", describe(a.checkpoints));
-
-					// Remove the extra data.
-					KeyRangeRef cRange(a.ranges.back().end, checkpoint.ranges.back().end);
-					if (!cRange.empty()) {
-						TraceEvent(SevInfo, "RocksDBRestoreCFRemoveExtraRange", logId)
-						    .detail("Path", a.path)
-						    .detail("Checkpoints", describe(a.checkpoints))
-						    .detail("RestoreRanges", describe(a.ranges))
-						    .detail("ClearExtraRange", cRange);
-						writeBatch.DeleteRange(ps->cf, toSlice(cRange.begin), toSlice(cRange.end));
-						status = a.shardManager->getDb()->Write(options, &writeBatch);
-						if (!status.ok()) {
-							logRocksDBError(status, "RestoreClearnExtraRanges");
-							a.done.sendError(statusToError(status));
-							return;
-						}
-					}
-				}
-			} else if (format == RocksDBKeyValues) {
-				// Make sure the files are complete for the desired ranges.
-				std::vector<KeyRange> fetchedRanges;
-				std::vector<KeyRange> intendedRanges(a.ranges.begin(), a.ranges.end());
-				std::vector<RocksDBCheckpointKeyValues> rkvs;
-				for (const auto& checkpoint : a.checkpoints) {
-					rkvs.push_back(getRocksKeyValuesCheckpoint(checkpoint));
-					ASSERT(!rkvs.back().fetchedFiles.empty());
-					for (const auto& file : rkvs.back().fetchedFiles) {
-						fetchedRanges.push_back(file.range);
-					}
-				}
-				// Verify that the collective fetchedRanges is the same as the collective intendedRanges.
-				std::sort(fetchedRanges.begin(), fetchedRanges.end(), KeyRangeRef::ArbitraryOrder());
-				std::sort(intendedRanges.begin(), intendedRanges.end(), KeyRangeRef::ArbitraryOrder());
-				int i = 0, j = 0;
-				while (i < fetchedRanges.size() && j < intendedRanges.size()) {
-					if (fetchedRanges[i].begin != intendedRanges[j].begin) {
-						break;
-					} else if (fetchedRanges[i] == intendedRanges[j]) {
-						++i;
-						++j;
-					} else if (fetchedRanges[i].contains(intendedRanges[j])) {
-						fetchedRanges[i] = KeyRangeRef(intendedRanges[j].end, fetchedRanges[i].end);
-						++j;
-					} else if (intendedRanges[j].contains(fetchedRanges[i])) {
-						intendedRanges[j] = KeyRangeRef(fetchedRanges[i].end, intendedRanges[j].end);
-						++i;
-					} else {
-						break;
-					}
-				}
-				if (i != fetchedRanges.size() || j != intendedRanges.size()) {
-					TraceEvent(SevError, "ShardedRocksDBRestoreFailed", logId)
-					    .detail("Reason", "RestoreFilesRangesMismatch")
-					    .detail("Ranges", describe(a.ranges))
-					    .setMaxFieldLength(1000)
-					    .detail("FetchedFiles", describe(rkvs));
-					a.done.sendError(failed_to_restore_checkpoint());
-					return;
-				}
-
-				if (!ps->initialized()) {
-					TraceEvent(SevDebug, "ShardedRocksRestoreInitPS", logId)
-					    .detail("Path", a.path)
-					    .detail("Checkpoints", describe(a.checkpoints))
-					    .detail("PhysicalShard", ps->toString());
-					status = ps->init();
-					if (!status.ok()) {
-						logRocksDBError(status, "RestoreInitPhysicalShard");
-						a.done.sendError(statusToError(status));
-						return;
-					}
-					(*columnFamilyMap)[ps->cf->GetID()] = ps->cf;
-				}
-				if (!status.ok()) {
-					logRocksDBError(status, "RestoreInitPhysicalShard");
-					a.done.sendError(statusToError(status));
-					return;
-				}
-
-				for (const auto& checkpoint : a.checkpoints) {
-					status = ps->restore(checkpoint);
-					if (!status.ok()) {
-						TraceEvent(SevWarnAlways, "ShardedRocksIngestFileError", logId)
-						    .detail("Error", status.ToString())
-						    .detail("Path", a.path)
-						    .detail("Checkpoint", checkpoint.toString())
-						    .detail("PhysicalShard", ps->toString());
-						break;
-					}
-				}
-
-				if (!status.ok()) {
-					TraceEvent(SevWarnAlways, "ShardedRocksRestoreError", logId).detail("Status", status.ToString());
-					for (const auto& range : a.ranges) {
-						writeBatch.DeleteRange(ps->cf, toSlice(range.begin), toSlice(range.end));
-					}
-					TraceEvent(SevInfo, "ShardedRocksRevertRestore", logId)
-					    .detail("Path", a.path)
-					    .detail("Checkpoints", describe(a.checkpoints))
-					    .detail("PhysicalShard", ps->toString())
-					    .detail("RestoreRanges", describe(a.ranges));
-
-					rocksdb::Status s = a.shardManager->getDb()->Write(options, &writeBatch);
-					if (!s.ok()) {
-						TraceEvent(SevError, "ShardedRocksRevertRestoreError", logId)
-						    .detail("Error", s.ToString())
-						    .detail("Path", a.path)
-						    .detail("PhysicalShard", ps->toString())
-						    .detail("RestoreRanges", describe(a.ranges));
-					}
-					a.done.sendError(failed_to_restore_checkpoint());
-					return;
-				}
-			} else if (format == RocksDB) {
-				a.done.sendError(not_implemented());
-				return;
-			}
-
-			TraceEvent(SevInfo, "ShardedRocksDBRestoreEnd", logId)
-			    .detail("Path", a.path)
-			    .detail("Checkpoints", describe(a.checkpoints));
-			a.done.send(Void());
-		}
-	};
-
-	struct Reader : IThreadPoolReceiver {
-		const UID logId;
-		double readValueTimeout;
-		double readValuePrefixTimeout;
-		double readRangeTimeout;
-		int threadIndex;
-		std::shared_ptr<RocksDBMetrics> rocksDBMetrics;
-		double sampleStartTime;
-
-		explicit Reader(UID logId, int threadIndex, std::shared_ptr<RocksDBMetrics> rocksDBMetrics)
-		  : logId(logId), threadIndex(threadIndex), rocksDBMetrics(rocksDBMetrics), sampleStartTime(now()) {
-			readValueTimeout = SERVER_KNOBS->ROCKSDB_READ_VALUE_TIMEOUT;
-			readValuePrefixTimeout = SERVER_KNOBS->ROCKSDB_READ_VALUE_PREFIX_TIMEOUT;
-			readRangeTimeout = SERVER_KNOBS->ROCKSDB_READ_RANGE_TIMEOUT;
-		}
-
-		void init() override {}
-
-		void sample() {
-			if (SERVER_KNOBS->ROCKSDB_METRICS_SAMPLE_INTERVAL > 0 &&
-			    now() - sampleStartTime >= SERVER_KNOBS->ROCKSDB_METRICS_SAMPLE_INTERVAL) {
-				rocksDBMetrics->collectPerfContext(threadIndex);
-				rocksDBMetrics->resetPerfContext();
-				sampleStartTime = now();
-			}
-		}
-		struct ReadValueAction : TypedAction<Reader, ReadValueAction> {
-			Key key;
-			PhysicalShard* shard;
-			Optional<UID> debugID;
-			double startTime;
-			bool getHistograms;
-			bool logShardMemUsage;
-			ThreadReturnPromise<Optional<Value>> result;
-
-			ReadValueAction(KeyRef key, PhysicalShard* shard, Optional<UID> debugID)
-			  : key(key), shard(shard), debugID(debugID), startTime(timer_monotonic()),
-			    getHistograms(
-			        (deterministicRandom()->random01() < SERVER_KNOBS->ROCKSDB_HISTOGRAMS_SAMPLE_RATE) ? true : false) {
-			}
-
-			double getTimeEstimate() const override { return SERVER_KNOBS->READ_VALUE_TIME_ESTIMATE; }
-		};
-
-		void action(ReadValueAction& a) {
-			double readBeginTime = timer_monotonic();
-			if (a.getHistograms) {
-				rocksDBMetrics->getReadValueQueueWaitHistogram(threadIndex)->sampleSeconds(readBeginTime - a.startTime);
-			}
-			Optional<TraceBatch> traceBatch;
-			if (a.debugID.present()) {
-				traceBatch = { TraceBatch{} };
-				traceBatch.get().addEvent("GetValueDebug", a.debugID.get().first(), "Reader.Before");
-			}
-			if (SERVER_KNOBS->ROCKSDB_SET_READ_TIMEOUT && readBeginTime - a.startTime > readValueTimeout) {
-				TraceEvent(SevWarn, "ShardedRocksDBError")
-				    .detail("Error", "Read value request timedout")
-				    .detail("Method", "ReadValueAction")
-				    .detail("Timeout value", readValueTimeout);
-				a.result.sendError(key_value_store_deadline_exceeded());
-				return;
-			}
-
-			rocksdb::PinnableSlice value;
-			auto options = getReadOptions();
-
-			auto db = a.shard->db;
-			if (SERVER_KNOBS->ROCKSDB_SET_READ_TIMEOUT) {
-				uint64_t deadlineMircos =
-				    db->GetEnv()->NowMicros() + (readValueTimeout - (timer_monotonic() - a.startTime)) * 1000000;
-				std::chrono::seconds deadlineSeconds(deadlineMircos / 1000000);
-				options.deadline = std::chrono::duration_cast<std::chrono::microseconds>(deadlineSeconds);
-			}
-			double dbGetBeginTime = a.getHistograms ? timer_monotonic() : 0;
-			auto s = db->Get(options, a.shard->cf, toSlice(a.key), &value);
-
-			if (a.getHistograms) {
-				rocksDBMetrics->getReadValueGetHistogram(threadIndex)
-				    ->sampleSeconds(timer_monotonic() - dbGetBeginTime);
-			}
-
-			if (a.debugID.present()) {
-				traceBatch.get().addEvent("GetValueDebug", a.debugID.get().first(), "Reader.After");
-				traceBatch.get().dump();
-			}
-			if (s.ok()) {
-				a.result.send(Value(toStringRef(value)));
-			} else if (s.IsNotFound()) {
-				a.result.send(Optional<Value>());
-			} else {
-				logRocksDBError(s, "ReadValue");
-				a.result.sendError(statusToError(s));
-			}
-
-			if (a.getHistograms) {
-				double currTime = timer_monotonic();
-				rocksDBMetrics->getReadValueActionHistogram(threadIndex)->sampleSeconds(currTime - readBeginTime);
-				rocksDBMetrics->getReadValueLatencyHistogram(threadIndex)->sampleSeconds(currTime - a.startTime);
-			}
-
-			sample();
-		}
-
-		struct ReadValuePrefixAction : TypedAction<Reader, ReadValuePrefixAction> {
-			Key key;
-			int maxLength;
-			PhysicalShard* shard;
-			Optional<UID> debugID;
-			double startTime;
-			bool getHistograms;
-			bool logShardMemUsage;
-			ThreadReturnPromise<Optional<Value>> result;
-
-			ReadValuePrefixAction(Key key, int maxLength, PhysicalShard* shard, Optional<UID> debugID)
-			  : key(key), maxLength(maxLength), shard(shard), debugID(debugID), startTime(timer_monotonic()),
-			    getHistograms((deterministicRandom()->random01() < SERVER_KNOBS->ROCKSDB_HISTOGRAMS_SAMPLE_RATE)
-			                      ? true
-			                      : false){};
-			double getTimeEstimate() const override { return SERVER_KNOBS->READ_VALUE_TIME_ESTIMATE; }
-		};
-
-		void action(ReadValuePrefixAction& a) {
-			double readBeginTime = timer_monotonic();
-			if (a.getHistograms) {
-				rocksDBMetrics->getReadPrefixQueueWaitHistogram(threadIndex)
-				    ->sampleSeconds(readBeginTime - a.startTime);
-			}
-			Optional<TraceBatch> traceBatch;
-			if (a.debugID.present()) {
-				traceBatch = { TraceBatch{} };
-				traceBatch.get().addEvent("GetValuePrefixDebug",
-				                          a.debugID.get().first(),
-				                          "Reader.Before"); //.detail("TaskID", g_network->getCurrentTask());
-			}
-			if (SERVER_KNOBS->ROCKSDB_SET_READ_TIMEOUT && readBeginTime - a.startTime > readValuePrefixTimeout) {
-				TraceEvent(SevWarn, "ShardedRocksDBError")
-				    .detail("Error", "Read value prefix request timedout")
-				    .detail("Method", "ReadValuePrefixAction")
-				    .detail("Timeout value", readValuePrefixTimeout);
-				a.result.sendError(key_value_store_deadline_exceeded());
-				return;
-			}
-
-			rocksdb::PinnableSlice value;
-			auto options = getReadOptions();
-			auto db = a.shard->db;
-			if (SERVER_KNOBS->ROCKSDB_SET_READ_TIMEOUT) {
-				uint64_t deadlineMircos =
-				    db->GetEnv()->NowMicros() + (readValuePrefixTimeout - (timer_monotonic() - a.startTime)) * 1000000;
-				std::chrono::seconds deadlineSeconds(deadlineMircos / 1000000);
-				options.deadline = std::chrono::duration_cast<std::chrono::microseconds>(deadlineSeconds);
-			}
-
-			double dbGetBeginTime = a.getHistograms ? timer_monotonic() : 0;
-			auto s = db->Get(options, a.shard->cf, toSlice(a.key), &value);
-
-			if (a.getHistograms) {
-				rocksDBMetrics->getReadPrefixGetHistogram(threadIndex)
-				    ->sampleSeconds(timer_monotonic() - dbGetBeginTime);
-			}
-
-			if (a.debugID.present()) {
-				traceBatch.get().addEvent("GetValuePrefixDebug",
-				                          a.debugID.get().first(),
-				                          "Reader.After"); //.detail("TaskID", g_network->getCurrentTask());
-				traceBatch.get().dump();
-			}
-			if (s.ok()) {
-				a.result.send(Value(StringRef(reinterpret_cast<const uint8_t*>(value.data()),
-				                              std::min(value.size(), size_t(a.maxLength)))));
-			} else if (s.IsNotFound()) {
-				a.result.send(Optional<Value>());
-			} else {
-				logRocksDBError(s, "ReadValuePrefix");
-				a.result.sendError(statusToError(s));
-			}
-			if (a.getHistograms) {
-				double currTime = timer_monotonic();
-				rocksDBMetrics->getReadPrefixActionHistogram(threadIndex)->sampleSeconds(currTime - readBeginTime);
-				rocksDBMetrics->getReadPrefixLatencyHistogram(threadIndex)->sampleSeconds(currTime - a.startTime);
-			}
-
-			sample();
-		}
-
-		struct ReadRangeAction : TypedAction<Reader, ReadRangeAction>, FastAllocated<ReadRangeAction> {
-			KeyRange keys;
-			std::vector<std::pair<PhysicalShard*, KeyRange>> shardRanges;
-			int rowLimit, byteLimit;
-			double startTime;
-			bool getHistograms;
-			bool logShardMemUsage;
-			ThreadReturnPromise<RangeResult> result;
-			ReadRangeAction(KeyRange keys, std::vector<DataShard*> shards, int rowLimit, int byteLimit)
-			  : keys(keys), rowLimit(rowLimit), byteLimit(byteLimit), startTime(timer_monotonic()),
-			    getHistograms(
-			        (deterministicRandom()->random01() < SERVER_KNOBS->ROCKSDB_HISTOGRAMS_SAMPLE_RATE) ? true : false) {
-				std::set<PhysicalShard*> usedShards;
-				for (const DataShard* shard : shards) {
-					ASSERT(shard);
-					shardRanges.emplace_back(shard->physicalShard, keys & shard->range);
-					usedShards.insert(shard->physicalShard);
-				}
-				if (usedShards.size() != shards.size()) {
-					TraceEvent("ReadRangeMetrics")
-					    .detail("NumPhysicalShards", usedShards.size())
-					    .detail("NumDataShards", shards.size());
-				}
-			}
-			double getTimeEstimate() const override { return SERVER_KNOBS->READ_RANGE_TIME_ESTIMATE; }
-		};
-
-		void action(ReadRangeAction& a) {
-			double readBeginTime = timer_monotonic();
-			if (a.getHistograms) {
-				rocksDBMetrics->getReadRangeQueueWaitHistogram(threadIndex)->sampleSeconds(readBeginTime - a.startTime);
-			}
-			if (SERVER_KNOBS->ROCKSDB_SET_READ_TIMEOUT && readBeginTime - a.startTime > readRangeTimeout) {
-				TraceEvent(SevWarn, "ShardedRocksKVSReadTimeout")
-				    .detail("Error", "Read range request timedout")
-				    .detail("Method", "ReadRangeAction")
-				    .detail("Timeout value", readRangeTimeout);
-				a.result.sendError(key_value_store_deadline_exceeded());
-				return;
-			}
-
-			int rowLimit = a.rowLimit;
-			int byteLimit = a.byteLimit;
-			RangeResult result;
-
-			if (rowLimit == 0 || byteLimit == 0) {
-				a.result.send(result);
-			}
-			if (rowLimit < 0) {
-				// Reverses the shard order so we could read range in reverse direction.
-				std::reverse(a.shardRanges.begin(), a.shardRanges.end());
-			}
-
-			// TODO: consider multi-thread reads. It's possible to read multiple shards in parallel. However, the
-			// number of rows to read needs to be calculated based on the previous read result. We may read more
-			// than we expected when parallel read is used when the previous result is not available. It's unlikely
-			// to get to performance improvement when the actual number of rows to read is very small.
-			int accumulatedBytes = 0;
-			int numShards = 0;
-			for (auto& [shard, range] : a.shardRanges) {
-				if (shard == nullptr || !shard->initialized()) {
-					TraceEvent(SevWarn, "ShardedRocksReadRangeShardNotReady", logId)
-					    .detail("Range", range)
-					    .detail("Reason", shard == nullptr ? "Not Exist" : "Not Initialized");
-					continue;
-				}
-				auto bytesRead = readRangeInDb(shard, range, rowLimit, byteLimit, &result);
-				if (bytesRead < 0) {
-					// Error reading an instance.
-					a.result.sendError(internal_error());
-					return;
-				}
-				byteLimit -= bytesRead;
-				accumulatedBytes += bytesRead;
-				++numShards;
-				if (result.size() >= abs(a.rowLimit) || accumulatedBytes >= a.byteLimit) {
-					break;
-				}
-
-				if (SERVER_KNOBS->ROCKSDB_SET_READ_TIMEOUT && timer_monotonic() - a.startTime > readRangeTimeout) {
-					TraceEvent(SevInfo, "ShardedRocksDBTimeout")
-					    .detail("Action", "ReadRange")
-					    .detail("ShardsRead", numShards)
-					    .detail("BytesRead", accumulatedBytes);
-					a.result.sendError(key_value_store_deadline_exceeded());
-					return;
-				}
-			}
-
-			result.more =
-			    (result.size() == a.rowLimit) || (result.size() == -a.rowLimit) || (accumulatedBytes >= a.byteLimit);
-			a.result.send(result);
-			if (a.getHistograms) {
-				double currTime = timer_monotonic();
-				rocksDBMetrics->getReadRangeActionHistogram(threadIndex)->sampleSeconds(currTime - readBeginTime);
-				rocksDBMetrics->getReadRangeLatencyHistogram(threadIndex)->sampleSeconds(currTime - a.startTime);
-				if (a.shardRanges.size() > 1) {
-					TraceEvent(SevInfo, "ShardedRocksDB").detail("ReadRangeShards", a.shardRanges.size());
-				}
-			}
-
-			sample();
-		}
-	};
-
-	struct Counters {
-		CounterCollection cc;
-		Counter immediateThrottle;
-		Counter failedToAcquire;
-
-		Counters()
-		  : cc("RocksDBThrottle"), immediateThrottle("ImmediateThrottle", cc), failedToAcquire("failedToAcquire", cc) {}
-	};
-
-	// Persist shard mappinng key range should not be in shardMap.
-	explicit ShardedRocksDBKeyValueStore(const std::string& path, UID id)
-	  : rState(std::make_shared<ShardedRocksDBState>()), path(path), id(id),
-	    readSemaphore(SERVER_KNOBS->ROCKSDB_READ_QUEUE_SOFT_MAX),
-	    fetchSemaphore(SERVER_KNOBS->ROCKSDB_FETCH_QUEUE_SOFT_MAX),
-	    numReadWaiters(SERVER_KNOBS->ROCKSDB_READ_QUEUE_HARD_MAX - SERVER_KNOBS->ROCKSDB_READ_QUEUE_SOFT_MAX),
-	    numFetchWaiters(SERVER_KNOBS->ROCKSDB_FETCH_QUEUE_HARD_MAX - SERVER_KNOBS->ROCKSDB_FETCH_QUEUE_SOFT_MAX),
-	    errorListener(std::make_shared<RocksDBErrorListener>()), errorFuture(errorListener->getFuture()),
-	    dbOptions(getOptions()), shardManager(path, id, dbOptions),
-	    rocksDBMetrics(std::make_shared<RocksDBMetrics>(id, dbOptions.statistics)) {
-		// In simluation, run the reader/writer threads as Coro threads (i.e. in the network thread. The storage
-		// engine is still multi-threaded as background compaction threads are still present. Reads/writes to disk
-		// will also block the network thread in a way that would be unacceptable in production but is a necessary
-		// evil here. When performing the reads in background threads in simulation, the event loop thinks there is
-		// no work to do and advances time faster than 1 sec/sec. By the time the blocking read actually finishes,
-		// simulation has advanced time by more than 5 seconds, so every read fails with a transaction_too_old
-		// error. Doing blocking IO on the main thread solves this issue. There are almost certainly better fixes,
-		// but my goal was to get a less invasive change merged first and work on a more realistic version if/when
-		// we think that would provide substantially more confidence in the correctness.
-		// TODO: Adapt the simulation framework to not advance time quickly when background reads/writes are
-		// occurring.
-		if (g_network->isSimulated()) {
-			TraceEvent(SevDebug, "ShardedRocksDB").detail("Info", "Use Coro threads in simulation.");
-			writeThread = CoroThreadPool::createThreadPool();
-			readThreads = CoroThreadPool::createThreadPool();
-		} else {
-			writeThread = createGenericThreadPool(/*stackSize=*/0, SERVER_KNOBS->ROCKSDB_WRITER_THREAD_PRIORITY);
-			readThreads = createGenericThreadPool(/*stackSize=*/0, SERVER_KNOBS->ROCKSDB_READER_THREAD_PRIORITY);
-		}
-		writeThread->addThread(new Writer(id, 0, shardManager.getColumnFamilyMap(), rocksDBMetrics), "fdb-rocksdb-wr");
-		TraceEvent("ShardedRocksDBReadThreads", id)
-		    .detail("KnobRocksDBReadParallelism", SERVER_KNOBS->ROCKSDB_READ_PARALLELISM);
-		for (unsigned i = 0; i < SERVER_KNOBS->ROCKSDB_READ_PARALLELISM; ++i) {
-			readThreads->addThread(new Reader(id, i, rocksDBMetrics), "fdb-rocksdb-re");
-		}
-	}
-
-	Future<Void> getError() const override { return errorFuture; }
-
-	ACTOR static void doClose(ShardedRocksDBKeyValueStore* self, bool deleteOnClose) {
-		self->rState->closing = true;
-		// The metrics future retains a reference to the DB, so stop it before we delete it.
-		self->metrics.reset();
-		self->refreshHolder.cancel();
-		self->cleanUpJob.cancel();
-
-		wait(self->readThreads->stop());
-		auto a = new Writer::CloseAction(&self->shardManager, deleteOnClose);
-		auto f = a->done.getFuture();
-		self->writeThread->post(a);
-		wait(f);
-		wait(self->writeThread->stop());
-		if (self->closePromise.canBeSet())
-			self->closePromise.send(Void());
-		delete self;
-	}
-
-	Future<Void> onClosed() const override { return closePromise.getFuture(); }
-
-	void dispose() override { doClose(this, true); }
-
-	void close() override { doClose(this, false); }
-
-	KeyValueStoreType getType() const override { return KeyValueStoreType(KeyValueStoreType::SSD_SHARDED_ROCKSDB); }
-
-	bool shardAware() const override { return true; }
-
-	Future<Void> init() override {
-		if (openFuture.isValid()) {
-			return openFuture;
-			// Restore durable state if KVS is open. KVS will be re-initialized during rollback. To avoid the cost
-			// of opening and closing multiple rocksdb instances, we reconcile the shard map using persist shard
-			// mapping data.
-		} else {
-			auto a = std::make_unique<Writer::OpenAction>(
-			    &shardManager, metrics, &readSemaphore, &fetchSemaphore, errorListener);
-			openFuture = a->done.getFuture();
-			this->metrics = ShardManager::shardMetricsLogger(this->rState, openFuture, &shardManager) &&
-			                rocksDBAggregatedMetricsLogger(this->rState, openFuture, rocksDBMetrics, &shardManager);
-			this->refreshHolder = refreshReadIteratorPools(this->rState, openFuture, shardManager.getAllShards());
-			this->cleanUpJob = emptyShardCleaner(this->rState, openFuture, &shardManager, writeThread);
-			writeThread->post(a.release());
-			return openFuture;
-		}
-	}
-
-	Future<Void> addRange(KeyRangeRef range, std::string id) override {
-		auto shard = shardManager.addRange(range, id);
-		if (shard->initialized()) {
-			return Void();
-		}
-		auto a = new Writer::AddShardAction(shard);
-		Future<Void> res = a->done.getFuture();
-		writeThread->post(a);
-		return res;
-	}
-
-	void set(KeyValueRef kv, const Arena*) override { shardManager.put(kv.key, kv.value); }
-
-	void clear(KeyRangeRef range, const Arena*) override {
-		if (range.singleKeyRange()) {
-			shardManager.clear(range.begin);
-		} else {
-			shardManager.clearRange(range);
-		}
-	}
-
-	// Checks and waits for few seconds if rocskdb is overloaded.
-	ACTOR Future<Void> checkRocksdbState(rocksdb::DB* db) {
-		state uint64_t estPendCompactBytes;
-		state int count = SERVER_KNOBS->ROCKSDB_CAN_COMMIT_DELAY_TIMES_ON_OVERLOAD;
-		db->GetAggregatedIntProperty(rocksdb::DB::Properties::kEstimatePendingCompactionBytes, &estPendCompactBytes);
-		while (count && estPendCompactBytes > SERVER_KNOBS->ROCKSDB_CAN_COMMIT_COMPACT_BYTES_LIMIT) {
-			wait(delay(SERVER_KNOBS->ROCKSDB_CAN_COMMIT_DELAY_ON_OVERLOAD));
-			count--;
-			db->GetAggregatedIntProperty(rocksdb::DB::Properties::kEstimatePendingCompactionBytes,
-			                             &estPendCompactBytes);
-		}
-
-		return Void();
-	}
-
-	Future<Void> canCommit() override { return checkRocksdbState(shardManager.getDb()); }
-
-	Future<Void> commit(bool) override {
-		auto a = new Writer::CommitAction(shardManager.getDb(),
-		                                  shardManager.getWriteBatch(),
-		                                  shardManager.getDirtyShards(),
-		                                  shardManager.getColumnFamilyMap());
-		auto res = a->done.getFuture();
-		writeThread->post(a);
-		return res;
-	}
-
-	void checkWaiters(const FlowLock& semaphore, int maxWaiters) {
-		if (semaphore.waiters() > maxWaiters) {
-			++counters.immediateThrottle;
-			throw server_overloaded();
-		}
-	}
-
-	// We don't throttle eager reads and reads to the FF keyspace because FDB struggles when those reads fail.
-	// Thus far, they have been low enough volume to not cause an issue.
-	static bool shouldThrottle(ReadType type, KeyRef key) {
-		return type != ReadType::EAGER && !(key.startsWith(systemKeys.begin));
-	}
-
-	ACTOR template <class Action>
-	static Future<Optional<Value>> read(Action* action, FlowLock* semaphore, IThreadPool* pool, Counter* counter) {
-		state std::unique_ptr<Action> a(action);
-		state Optional<Void> slot = wait(timeout(semaphore->take(), SERVER_KNOBS->ROCKSDB_READ_QUEUE_WAIT));
-		if (!slot.present()) {
-			++(*counter);
-			throw server_overloaded();
-		}
-
-		state FlowLock::Releaser release(*semaphore);
-
-		auto fut = a->result.getFuture();
-		pool->post(a.release());
-		Optional<Value> result = wait(fut);
-
-		return result;
-	}
-
-	Future<Optional<Value>> readValue(KeyRef key, Optional<ReadOptions> options) override {
-		auto* shard = shardManager.getDataShard(key);
-		if (shard == nullptr || !shard->physicalShard->initialized()) {
-			// TODO: read non-exist system key range should not cause an error.
-			TraceEvent(SevWarn, "ShardedRocksDB", this->id)
-			    .detail("Detail", "Read non-exist key range")
-			    .detail("ReadKey", key);
-			return Optional<Value>();
-		}
-
-		ReadType type = ReadType::NORMAL;
-		Optional<UID> debugID;
-
-		if (options.present()) {
-			type = options.get().type;
-			debugID = options.get().debugID;
-		}
-
-		if (!shouldThrottle(type, key)) {
-			auto a = new Reader::ReadValueAction(key, shard->physicalShard, debugID);
-			auto res = a->result.getFuture();
-			readThreads->post(a);
-			return res;
-		}
-
-		auto& semaphore = (type == ReadType::FETCH) ? fetchSemaphore : readSemaphore;
-		int maxWaiters = (type == ReadType::FETCH) ? numFetchWaiters : numReadWaiters;
-
-		checkWaiters(semaphore, maxWaiters);
-		auto a = std::make_unique<Reader::ReadValueAction>(key, shard->physicalShard, debugID);
-		return read(a.release(), &semaphore, readThreads.getPtr(), &counters.failedToAcquire);
-	}
-
-	Future<Optional<Value>> readValuePrefix(KeyRef key, int maxLength, Optional<ReadOptions> options) override {
-		auto* shard = shardManager.getDataShard(key);
-		if (shard == nullptr || !shard->physicalShard->initialized()) {
-			// TODO: read non-exist system key range should not cause an error.
-			TraceEvent(SevWarn, "ShardedRocksDB", this->id)
-			    .detail("Detail", "Read non-exist key range")
-			    .detail("ReadKey", key);
-			return Optional<Value>();
-		}
-
-		ReadType type = ReadType::NORMAL;
-		Optional<UID> debugID;
-
-		if (options.present()) {
-			type = options.get().type;
-			debugID = options.get().debugID;
-		}
-
-		if (!shouldThrottle(type, key)) {
-			auto a = new Reader::ReadValuePrefixAction(key, maxLength, shard->physicalShard, debugID);
-			auto res = a->result.getFuture();
-			readThreads->post(a);
-			return res;
-		}
-
-		auto& semaphore = (type == ReadType::FETCH) ? fetchSemaphore : readSemaphore;
-		int maxWaiters = (type == ReadType::FETCH) ? numFetchWaiters : numReadWaiters;
-
-		checkWaiters(semaphore, maxWaiters);
-		auto a = std::make_unique<Reader::ReadValuePrefixAction>(key, maxLength, shard->physicalShard, debugID);
-		return read(a.release(), &semaphore, readThreads.getPtr(), &counters.failedToAcquire);
-	}
-
-	ACTOR static Future<Standalone<RangeResultRef>> read(Reader::ReadRangeAction* action,
-	                                                     FlowLock* semaphore,
-	                                                     IThreadPool* pool,
-	                                                     Counter* counter) {
-		state std::unique_ptr<Reader::ReadRangeAction> a(action);
-		state Optional<Void> slot = wait(timeout(semaphore->take(), SERVER_KNOBS->ROCKSDB_READ_QUEUE_WAIT));
-		if (!slot.present()) {
-			++(*counter);
-			throw server_overloaded();
-		}
-
-		state FlowLock::Releaser release(*semaphore);
-
-		auto fut = a->result.getFuture();
-		pool->post(a.release());
-		Standalone<RangeResultRef> result = wait(fut);
-
-		return result;
-	}
-
-	Future<RangeResult> readRange(KeyRangeRef keys,
-	                              int rowLimit,
-	                              int byteLimit,
-	                              Optional<ReadOptions> options = Optional<ReadOptions>()) override {
-		TraceEvent(SevVerbose, "ShardedRocksReadRangeBegin", this->id).detail("Range", keys);
-		auto shards = shardManager.getDataShardsByRange(keys);
-
-		ReadType type = ReadType::NORMAL;
-		if (options.present()) {
-			type = options.get().type;
-		}
-
-		if (!shouldThrottle(type, keys.begin)) {
-			auto a = new Reader::ReadRangeAction(keys, shards, rowLimit, byteLimit);
-			auto res = a->result.getFuture();
-			readThreads->post(a);
-			return res;
-		}
-
-		auto& semaphore = (type == ReadType::FETCH) ? fetchSemaphore : readSemaphore;
-		int maxWaiters = (type == ReadType::FETCH) ? numFetchWaiters : numReadWaiters;
-		checkWaiters(semaphore, maxWaiters);
-
-		auto a = std::make_unique<Reader::ReadRangeAction>(keys, shards, rowLimit, byteLimit);
-		return read(a.release(), &semaphore, readThreads.getPtr(), &counters.failedToAcquire);
-	}
-
-	ACTOR static Future<Void> emptyShardCleaner(std::shared_ptr<ShardedRocksDBState> rState,
-	                                            Future<Void> openFuture,
-	                                            ShardManager* shardManager,
-	                                            Reference<IThreadPool> writeThread) {
-		state double cleanUpDelay = SERVER_KNOBS->ROCKSDB_PHYSICAL_SHARD_CLEAN_UP_DELAY;
-		state double cleanUpPeriod = cleanUpDelay * 2;
-		try {
-			wait(openFuture);
-			loop {
-				wait(delay(cleanUpPeriod));
-				if (rState->closing) {
-					break;
-				}
-				auto shards = shardManager->getPendingDeletionShards(cleanUpDelay);
-				if (shards.size() > 0) {
-					auto a = new Writer::RemoveShardAction(shards);
-					Future<Void> f = a->done.getFuture();
-					writeThread->post(a);
-					TraceEvent(SevInfo, "ShardedRocksDB").detail("DeleteEmptyShards", shards.size());
-					wait(f);
-				}
-			}
-		} catch (Error& e) {
-			if (e.code() != error_code_actor_cancelled) {
-				TraceEvent(SevError, "DeleteEmptyShardsError").errorUnsuppressed(e);
-			}
-		}
-		return Void();
-	}
-
-	StorageBytes getStorageBytes() const override {
-		uint64_t live = 0;
-		ASSERT(shardManager.getDb()->GetAggregatedIntProperty(rocksdb::DB::Properties::kLiveSstFilesSize, &live));
-
-		int64_t free;
-		int64_t total;
-		g_network->getDiskBytes(path, free, total);
-		return StorageBytes(free, total, live, free);
-	}
-
-	Future<CheckpointMetaData> checkpoint(const CheckpointRequest& request) override {
-		auto a = new Writer::CheckpointAction(&shardManager, request);
-
-		auto res = a->reply.getFuture();
-		writeThread->post(a);
-		return res;
-	}
-
-	Future<Void> restore(const std::string& shardId,
-	                     const std::vector<KeyRange>& ranges,
-	                     const std::vector<CheckpointMetaData>& checkpoints) override {
-		return doRestore(this, shardId, ranges, checkpoints);
-	}
-
-	std::vector<std::string> removeRange(KeyRangeRef range) override { return shardManager.removeRange(range); }
-
-	void persistRangeMapping(KeyRangeRef range, bool isAdd) override {
-		return shardManager.persistRangeMapping(range, isAdd);
-	}
-
-	// Used for debugging shard mapping issue.
-	std::vector<std::pair<KeyRange, std::string>> getDataMapping() { return shardManager.getDataMapping(); }
-
-	Future<EncryptionAtRestMode> encryptionMode() override {
-		return EncryptionAtRestMode(EncryptionAtRestMode::DISABLED);
-	}
-
-	CoalescedKeyRangeMap<std::string> getExistingRanges() override { return shardManager.getExistingRanges(); }
-
-	std::shared_ptr<ShardedRocksDBState> rState;
-	rocksdb::Options dbOptions;
-	ShardManager shardManager;
-	std::shared_ptr<RocksDBMetrics> rocksDBMetrics;
-	std::string path;
-	UID id;
-	Reference<IThreadPool> writeThread;
-	Reference<IThreadPool> readThreads;
-	std::shared_ptr<RocksDBErrorListener> errorListener;
-	Future<Void> errorFuture;
-	Promise<Void> closePromise;
-	Future<Void> openFuture;
-	Optional<Future<Void>> metrics;
-	FlowLock readSemaphore;
-	int numReadWaiters;
-	FlowLock fetchSemaphore;
-	int numFetchWaiters;
-	Counters counters;
-	Future<Void> refreshHolder;
-	Future<Void> cleanUpJob;
-};
-
-ACTOR Future<Void> testCheckpointRestore(IKeyValueStore* kvStore, std::vector<KeyRange> ranges) {
-	state std::string checkpointDir = "checkpoint" + deterministicRandom()->randomAlphaNumeric(5);
-	platform::eraseDirectoryRecursive(checkpointDir);
-	CheckpointRequest request(
-	    latestVersion, ranges, DataMoveRocksCF, deterministicRandom()->randomUniqueID(), checkpointDir);
-	state CheckpointMetaData checkpoint = wait(kvStore->checkpoint(request));
-	RocksDBColumnFamilyCheckpoint rocksCF = getRocksCF(checkpoint);
-
-	TraceEvent(SevDebug, "ShardedRocksCheckpointTest")
-	    .detail("Checkpoint", checkpoint.toString())
-	    .detail("ColumnFamily", rocksCF.toString());
-
-	state std::string rocksDBRestoreDir = "sharded-rocks-restore" + deterministicRandom()->randomAlphaNumeric(5);
-	platform::eraseDirectoryRecursive(rocksDBRestoreDir);
-	state IKeyValueStore* restoreKv = keyValueStoreShardedRocksDB(
-	    rocksDBRestoreDir, deterministicRandom()->randomUniqueID(), KeyValueStoreType::SSD_SHARDED_ROCKSDB);
-	wait(restoreKv->init());
-	try {
-		const std::string shardId = "restoredShard";
-		wait(restoreKv->restore(shardId, ranges, { checkpoint }));
-	} catch (Error& e) {
-		TraceEvent(SevWarnAlways, "TestRestoreCheckpointError")
-		    .errorUnsuppressed(e)
-		    .detail("Checkpoint", checkpoint.toString());
-		throw;
-	}
-
-	state int i = 0;
-	for (; i < ranges.size(); ++i) {
-		state RangeResult restoreResult = wait(restoreKv->readRange(ranges[i]));
-		RangeResult result = wait(kvStore->readRange(ranges[i]));
-		ASSERT(!restoreResult.more && !result.more);
-		ASSERT(restoreResult.size() == result.size());
-		for (int i = 0; i < result.size(); ++i) {
-			TraceEvent(SevDebug, "ReadKeyValueFromRestoredRocks")
-			    .detail("Key", result[i].key)
-			    .detail("Value", result[i].value)
-			    .detail("RestoreKey", restoreResult[i].key)
-			    .detail("RestoreValue", restoreResult[i].value);
-			ASSERT(result[i].key == restoreResult[i].key);
-			ASSERT(result[i].value == restoreResult[i].value);
-		}
-	}
-
-	Future<Void> restoreKvClose = restoreKv->onClosed();
-	restoreKv->close();
-	wait(restoreKvClose);
-	return Void();
-}
-} // namespace
-
-#endif // SSD_ROCKSDB_EXPERIMENTAL
-
-IKeyValueStore* keyValueStoreShardedRocksDB(std::string const& path,
-                                            UID logID,
-                                            KeyValueStoreType storeType,
-                                            bool checkChecksums,
-                                            bool checkIntegrity) {
-#ifdef SSD_ROCKSDB_EXPERIMENTAL
-	return new ShardedRocksDBKeyValueStore(path, logID);
-#else
-	TraceEvent(SevError, "ShardedRocksDBEngineInitFailure").detail("Reason", "Built without RocksDB");
-	ASSERT(false);
-	return nullptr;
-#endif // SSD_ROCKSDB_EXPERIMENTAL
-}
-
-#ifdef SSD_ROCKSDB_EXPERIMENTAL
-#include "flow/UnitTest.h"
-
-namespace {
-TEST_CASE("noSim/ShardedRocksDB/Initialization") {
-	state const std::string rocksDBTestDir = "sharded-rocksdb-test-db";
-	platform::eraseDirectoryRecursive(rocksDBTestDir);
-
-	state IKeyValueStore* kvStore =
-	    new ShardedRocksDBKeyValueStore(rocksDBTestDir, deterministicRandom()->randomUniqueID());
-	wait(kvStore->init());
-
-	Future<Void> closed = kvStore->onClosed();
-	kvStore->dispose();
-	wait(closed);
-	return Void();
-}
-
-TEST_CASE("noSim/ShardedRocksDB/SingleShardRead") {
-	state const std::string rocksDBTestDir = "sharded-rocksdb-test-db";
-	platform::eraseDirectoryRecursive(rocksDBTestDir);
-
-	state IKeyValueStore* kvStore =
-	    new ShardedRocksDBKeyValueStore(rocksDBTestDir, deterministicRandom()->randomUniqueID());
-	wait(kvStore->init());
-
-	KeyRangeRef range("a"_sr, "b"_sr);
-	wait(kvStore->addRange(range, "shard-1"));
-
-	kvStore->set({ "a"_sr, "foo"_sr });
-	kvStore->set({ "ac"_sr, "bar"_sr });
-	wait(kvStore->commit(false));
-
-	Optional<Value> val = wait(kvStore->readValue("a"_sr));
-	ASSERT(Optional<Value>("foo"_sr) == val);
-	{
-		Optional<Value> val = wait(kvStore->readValue("ac"_sr));
-		ASSERT(Optional<Value>("bar"_sr) == val);
-	}
-
-	Future<Void> closed = kvStore->onClosed();
-	kvStore->dispose();
-	wait(closed);
-	return Void();
-}
-
-TEST_CASE("noSim/ShardedRocksDB/RangeOps") {
-	state std::string rocksDBTestDir = "sharded-rocksdb-kvs-test-db";
-	platform::eraseDirectoryRecursive(rocksDBTestDir);
-
-	state IKeyValueStore* kvStore =
-	    new ShardedRocksDBKeyValueStore(rocksDBTestDir, deterministicRandom()->randomUniqueID());
-	wait(kvStore->init());
-
-	std::vector<Future<Void>> addRangeFutures;
-	addRangeFutures.push_back(kvStore->addRange(KeyRangeRef("0"_sr, "3"_sr), "shard-1"));
-	addRangeFutures.push_back(kvStore->addRange(KeyRangeRef("4"_sr, "7"_sr), "shard-2"));
-
-	wait(waitForAll(addRangeFutures));
-
-	kvStore->persistRangeMapping(KeyRangeRef("0"_sr, "7"_sr), true);
-
-	// write to shard 1
-	state RangeResult expectedRows;
-	for (int i = 0; i < 30; ++i) {
-		std::string key = format("%02d", i);
-		std::string value = std::to_string(i);
-		kvStore->set({ key, value });
-		expectedRows.push_back_deep(expectedRows.arena(), { key, value });
-	}
-
-	// write to shard 2
-	for (int i = 40; i < 70; ++i) {
-		std::string key = format("%02d", i);
-		std::string value = std::to_string(i);
-		kvStore->set({ key, value });
-		expectedRows.push_back_deep(expectedRows.arena(), { key, value });
-	}
-
-	wait(kvStore->commit(false));
-	Future<Void> closed = kvStore->onClosed();
-	kvStore->close();
-	wait(closed);
-	kvStore = new ShardedRocksDBKeyValueStore(rocksDBTestDir, deterministicRandom()->randomUniqueID());
-	wait(kvStore->init());
-
-	// Point read
-	state int i = 0;
-	for (i = 0; i < expectedRows.size(); ++i) {
-		Optional<Value> val = wait(kvStore->readValue(expectedRows[i].key));
-		ASSERT(val == Optional<Value>(expectedRows[i].value));
-	}
-
-	// Range read
-	// Read forward full range.
-	RangeResult result = wait(kvStore->readRange(KeyRangeRef("0"_sr, ":"_sr), 1000, 10000));
-	ASSERT_EQ(result.size(), expectedRows.size());
-	for (int i = 0; i < expectedRows.size(); ++i) {
-		ASSERT(result[i] == expectedRows[i]);
-	}
-
-	// Read backward full range.
-	{
-		RangeResult result = wait(kvStore->readRange(KeyRangeRef("0"_sr, ":"_sr), -1000, 10000));
-		ASSERT_EQ(result.size(), expectedRows.size());
-		for (int i = 0; i < expectedRows.size(); ++i) {
-			ASSERT(result[i] == expectedRows[59 - i]);
-		}
-	}
-
-	// Forward with row limit.
-	{
-		RangeResult result = wait(kvStore->readRange(KeyRangeRef("2"_sr, "6"_sr), 10, 10000));
-		ASSERT_EQ(result.size(), 10);
-		for (int i = 0; i < 10; ++i) {
-			ASSERT(result[i] == expectedRows[20 + i]);
-		}
-	}
-
-	// Add another range on shard-1.
-	wait(kvStore->addRange(KeyRangeRef("7"_sr, "9"_sr), "shard-1"));
-	kvStore->persistRangeMapping(KeyRangeRef("7"_sr, "9"_sr), true);
-
-	for (i = 70; i < 90; ++i) {
-		std::string key = format("%02d", i);
-		std::string value = std::to_string(i);
-		kvStore->set({ key, value });
-		expectedRows.push_back_deep(expectedRows.arena(), { key, value });
-	}
-
-	wait(kvStore->commit(false));
-
-	{
-		Future<Void> closed = kvStore->onClosed();
-		kvStore->close();
-		wait(closed);
-	}
-	kvStore = new ShardedRocksDBKeyValueStore(rocksDBTestDir, deterministicRandom()->randomUniqueID());
-	wait(kvStore->init());
-
-	// Read all values.
-	{
-		RangeResult result = wait(kvStore->readRange(KeyRangeRef("0"_sr, ":"_sr), 1000, 10000));
-		ASSERT_EQ(result.size(), expectedRows.size());
-		for (int i = 0; i < expectedRows.size(); ++i) {
-			ASSERT(result[i] == expectedRows[i]);
-		}
-	}
-
-	// Read partial range with row limit
-	{
-		RangeResult result = wait(kvStore->readRange(KeyRangeRef("5"_sr, ":"_sr), 35, 10000));
-		ASSERT_EQ(result.size(), 35);
-		for (int i = 0; i < result.size(); ++i) {
-			ASSERT(result[i] == expectedRows[40 + i]);
-		}
-	}
-
-	// Clear a range on a single shard.
-	kvStore->clear(KeyRangeRef("40"_sr, "45"_sr));
-	wait(kvStore->commit(false));
-
-	{
-		RangeResult result = wait(kvStore->readRange(KeyRangeRef("4"_sr, "5"_sr), 20, 10000));
-		ASSERT_EQ(result.size(), 5);
-	}
-
-	// Clear a single value.
-	kvStore->clear(KeyRangeRef("01"_sr, keyAfter("01"_sr)));
-	wait(kvStore->commit(false));
-
-	Optional<Value> val = wait(kvStore->readValue("01"_sr));
-	ASSERT(!val.present());
-
-	// Clear a range spanning on multiple shards.
-	kvStore->clear(KeyRangeRef("1"_sr, "8"_sr));
-	wait(kvStore->commit(false));
-
-	{
-		Future<Void> closed = kvStore->onClosed();
-		kvStore->close();
-		wait(closed);
-	}
-	kvStore = new ShardedRocksDBKeyValueStore(rocksDBTestDir, deterministicRandom()->randomUniqueID());
-	wait(kvStore->init());
-
-	{
-		RangeResult result = wait(kvStore->readRange(KeyRangeRef("1"_sr, "8"_sr), 1000, 10000));
-		ASSERT_EQ(result.size(), 0);
-	}
-
-	{
-		RangeResult result = wait(kvStore->readRange(KeyRangeRef("0"_sr, ":"_sr), 1000, 10000));
-		ASSERT_EQ(result.size(), 19);
-	}
-
-	{
-		Future<Void> closed = kvStore->onClosed();
-		kvStore->dispose();
-		wait(closed);
-	}
-
-	return Void();
-}
-
-TEST_CASE("noSim/ShardedRocksDB/ShardOps") {
-	state std::string rocksDBTestDir = "sharded-rocksdb-kvs-test-db";
-	platform::eraseDirectoryRecursive(rocksDBTestDir);
-
-	state ShardedRocksDBKeyValueStore* rocksdbStore =
-	    new ShardedRocksDBKeyValueStore(rocksDBTestDir, deterministicRandom()->randomUniqueID());
-	state IKeyValueStore* kvStore = rocksdbStore;
-	wait(kvStore->init());
-
-	// Add some ranges.
-	{
-		std::vector<Future<Void>> addRangeFutures;
-		addRangeFutures.push_back(kvStore->addRange(KeyRangeRef("a"_sr, "c"_sr), "shard-1"));
-		addRangeFutures.push_back(kvStore->addRange(KeyRangeRef("c"_sr, "f"_sr), "shard-2"));
-
-		wait(waitForAll(addRangeFutures));
-	}
-
-	{
-		std::vector<Future<Void>> addRangeFutures;
-		addRangeFutures.push_back(kvStore->addRange(KeyRangeRef("x"_sr, "z"_sr), "shard-1"));
-		addRangeFutures.push_back(kvStore->addRange(KeyRangeRef("l"_sr, "n"_sr), "shard-3"));
-
-		wait(waitForAll(addRangeFutures));
-	}
-
-	// Remove single range.
-	std::vector<std::string> shardsToCleanUp;
-	auto shardIds = kvStore->removeRange(KeyRangeRef("b"_sr, "c"_sr));
-	// Remove range didn't create empty shard.
-	ASSERT_EQ(shardIds.size(), 0);
-
-	// Remove range spanning on multiple shards.
-	shardIds = kvStore->removeRange(KeyRangeRef("c"_sr, "m"_sr));
-	sort(shardIds.begin(), shardIds.end());
-	int count = std::unique(shardIds.begin(), shardIds.end()) - shardIds.begin();
-	ASSERT_EQ(count, 1);
-	ASSERT(shardIds[0] == "shard-2");
-
-	// Add more ranges.
-	std::vector<Future<Void>> addRangeFutures;
-	addRangeFutures.push_back(kvStore->addRange(KeyRangeRef("b"_sr, "g"_sr), "shard-1"));
-	addRangeFutures.push_back(kvStore->addRange(KeyRangeRef("l"_sr, "m"_sr), "shard-2"));
-	addRangeFutures.push_back(kvStore->addRange(KeyRangeRef("u"_sr, "v"_sr), "shard-3"));
-
-	wait(waitForAll(addRangeFutures));
-
-	auto dataMap = rocksdbStore->getDataMapping();
-	state std::vector<std::pair<KeyRange, std::string>> mapping;
-	mapping.push_back(std::make_pair(KeyRange(KeyRangeRef("a"_sr, "b"_sr)), "shard-1"));
-	mapping.push_back(std::make_pair(KeyRange(KeyRangeRef("b"_sr, "g"_sr)), "shard-1"));
-	mapping.push_back(std::make_pair(KeyRange(KeyRangeRef("l"_sr, "m"_sr)), "shard-2"));
-	mapping.push_back(std::make_pair(KeyRange(KeyRangeRef("m"_sr, "n"_sr)), "shard-3"));
-	mapping.push_back(std::make_pair(KeyRange(KeyRangeRef("u"_sr, "v"_sr)), "shard-3"));
-	mapping.push_back(std::make_pair(KeyRange(KeyRangeRef("x"_sr, "z"_sr)), "shard-1"));
-	mapping.push_back(std::make_pair(specialKeys, DEFAULT_CF_NAME));
-
-	for (auto it = dataMap.begin(); it != dataMap.end(); ++it) {
-		std::cout << "Begin " << it->first.begin.toString() << ", End " << it->first.end.toString() << ", id "
-		          << it->second << "\n";
-	}
-	ASSERT(dataMap == mapping);
-
-	kvStore->persistRangeMapping(KeyRangeRef("a"_sr, "z"_sr), true);
-	wait(kvStore->commit(false));
-
-	// Restart.
-	Future<Void> closed = kvStore->onClosed();
-	kvStore->close();
-	wait(closed);
-
-	rocksdbStore = new ShardedRocksDBKeyValueStore(rocksDBTestDir, deterministicRandom()->randomUniqueID());
-	kvStore = rocksdbStore;
-	wait(kvStore->init());
-
-	{
-		auto dataMap = rocksdbStore->getDataMapping();
-		for (auto it = dataMap.begin(); it != dataMap.end(); ++it) {
-			std::cout << "Begin " << it->first.begin.toString() << ", End " << it->first.end.toString() << ", id "
-			          << it->second << "\n";
-		}
-		ASSERT(dataMap == mapping);
-	}
-
-	// Remove all the ranges.
-	{
-		state std::vector<std::string> shardsToCleanUp = kvStore->removeRange(KeyRangeRef("a"_sr, "z"_sr));
-		ASSERT_EQ(shardsToCleanUp.size(), 3);
-
-		// Add another range to shard-2.
-		wait(kvStore->addRange(KeyRangeRef("h"_sr, "i"_sr), "shard-2"));
-	}
-	{
-		auto dataMap = rocksdbStore->getDataMapping();
-		ASSERT_EQ(dataMap.size(), 2);
-		ASSERT(dataMap[0].second == "shard-2");
-	}
-
-	{
-		Future<Void> closed = kvStore->onClosed();
-		kvStore->dispose();
-		wait(closed);
-	}
-	return Void();
-}
-
-TEST_CASE("noSim/ShardedRocksDB/Metadata") {
-	state std::string rocksDBTestDir = "sharded-rocksdb-kvs-test-db";
-	state Key testSpecialKey = "\xff\xff/TestKey"_sr;
-	state Value testSpecialValue = "\xff\xff/TestValue"_sr;
-	platform::eraseDirectoryRecursive(rocksDBTestDir);
-
-	state ShardedRocksDBKeyValueStore* rocksdbStore =
-	    new ShardedRocksDBKeyValueStore(rocksDBTestDir, deterministicRandom()->randomUniqueID());
-	state IKeyValueStore* kvStore = rocksdbStore;
-	wait(kvStore->init());
-
-	Optional<Value> val = wait(kvStore->readValue(testSpecialKey));
-	ASSERT(!val.present());
-
-	kvStore->set(KeyValueRef(testSpecialKey, testSpecialValue));
-	wait(kvStore->commit(false));
-
-	{
-		Optional<Value> val = wait(kvStore->readValue(testSpecialKey));
-		ASSERT(val.get() == testSpecialValue);
-	}
-
-	// Add some ranges.
-	std::vector<Future<Void>> addRangeFutures;
-	addRangeFutures.push_back(kvStore->addRange(KeyRangeRef("a"_sr, "c"_sr), "shard-1"));
-	addRangeFutures.push_back(kvStore->addRange(KeyRangeRef("c"_sr, "f"_sr), "shard-2"));
-	kvStore->persistRangeMapping(KeyRangeRef("a"_sr, "f"_sr), true);
-
-	wait(waitForAll(addRangeFutures));
-	kvStore->set(KeyValueRef("a1"_sr, "foo"_sr));
-	kvStore->set(KeyValueRef("d1"_sr, "bar"_sr));
-	wait(kvStore->commit(false));
-
-	// Restart.
-	Future<Void> closed = kvStore->onClosed();
-	kvStore->close();
-	wait(closed);
-	rocksdbStore = new ShardedRocksDBKeyValueStore(rocksDBTestDir, deterministicRandom()->randomUniqueID());
-	kvStore = rocksdbStore;
-	wait(kvStore->init());
-
-	{
-		Optional<Value> val = wait(kvStore->readValue(testSpecialKey));
-		ASSERT(val.get() == testSpecialValue);
-	}
-
-	// Read value back.
-	{
-		Optional<Value> val = wait(kvStore->readValue("a1"_sr));
-		ASSERT(val == Optional<Value>("foo"_sr));
-	}
-	{
-		Optional<Value> val = wait(kvStore->readValue("d1"_sr));
-		ASSERT(val == Optional<Value>("bar"_sr));
-	}
-
-	// Remove range containing a1.
-	kvStore->persistRangeMapping(KeyRangeRef("a"_sr, "b"_sr), false);
-	auto shardIds = kvStore->removeRange(KeyRangeRef("a"_sr, "b"_sr));
-	wait(kvStore->commit(false));
-
-	// Read a1.
-	{
-		Optional<Value> val = wait(kvStore->readValue("a1"_sr));
-		ASSERT(!val.present());
-	}
-
-	// Restart.
-	{
-		Future<Void> closed = kvStore->onClosed();
-		kvStore->close();
-		wait(closed);
-	}
-	rocksdbStore = new ShardedRocksDBKeyValueStore(rocksDBTestDir, deterministicRandom()->randomUniqueID());
-	kvStore = rocksdbStore;
-	wait(kvStore->init());
-
-	// Read again.
-	{
-		Optional<Value> val = wait(kvStore->readValue("a1"_sr));
-		ASSERT(!val.present());
-	}
-	{
-		Optional<Value> val = wait(kvStore->readValue("d1"_sr));
-		ASSERT(val == Optional<Value>("bar"_sr));
-	}
-
-	auto mapping = rocksdbStore->getDataMapping();
-	ASSERT(mapping.size() == 3);
-
-	// Remove all the ranges.
-	kvStore->removeRange(KeyRangeRef("a"_sr, "f"_sr));
-	mapping = rocksdbStore->getDataMapping();
-	ASSERT(mapping.size() == 1);
-
-	// Restart.
-	{
-		Future<Void> closed = kvStore->onClosed();
-		kvStore->close();
-		wait(closed);
-	}
-	rocksdbStore = new ShardedRocksDBKeyValueStore(rocksDBTestDir, deterministicRandom()->randomUniqueID());
-	kvStore = rocksdbStore;
-	wait(kvStore->init());
-
-	// Because range metadata was not committed, ranges should be restored.
-	{
-		auto mapping = rocksdbStore->getDataMapping();
-		ASSERT(mapping.size() == 3);
-
-		// Remove ranges again.
-		kvStore->persistRangeMapping(KeyRangeRef("a"_sr, "f"_sr), false);
-		kvStore->removeRange(KeyRangeRef("a"_sr, "f"_sr));
-
-		mapping = rocksdbStore->getDataMapping();
-		ASSERT(mapping.size() == 1);
-
-		wait(kvStore->commit(false));
-	}
-
-	// Restart.
-	{
-		Future<Void> closed = kvStore->onClosed();
-		kvStore->close();
-		wait(closed);
-	}
-
-	rocksdbStore = new ShardedRocksDBKeyValueStore(rocksDBTestDir, deterministicRandom()->randomUniqueID());
-	kvStore = rocksdbStore;
-	wait(kvStore->init());
-
-	// No range available.
-	{
-		auto mapping = rocksdbStore->getDataMapping();
-		for (auto it = mapping.begin(); it != mapping.end(); ++it) {
-			std::cout << "Begin " << it->first.begin.toString() << ", End " << it->first.end.toString() << ", id "
-			          << it->second << "\n";
-		}
-		ASSERT(mapping.size() == 1);
-	}
-
-	{
-		Future<Void> closed = kvStore->onClosed();
-		kvStore->dispose();
-		wait(closed);
-	}
-
-	return Void();
-}
-
-TEST_CASE("noSim/ShardedRocksDB/CheckpointBasic") {
-	state std::string rocksDBTestDir = "sharded-rocks-checkpoint-restore";
-	state std::map<Key, Value> kvs({ { "a"_sr, "TestValueA"_sr },
-	                                 { "ab"_sr, "TestValueAB"_sr },
-	                                 { "ad"_sr, "TestValueAD"_sr },
-	                                 { "b"_sr, "TestValueB"_sr },
-	                                 { "ba"_sr, "TestValueBA"_sr },
-	                                 { "c"_sr, "TestValueC"_sr },
-	                                 { "d"_sr, "TestValueD"_sr },
-	                                 { "e"_sr, "TestValueE"_sr },
-	                                 { "h"_sr, "TestValueH"_sr },
-	                                 { "ha"_sr, "TestValueHA"_sr } });
-	platform::eraseDirectoryRecursive(rocksDBTestDir);
-	state IKeyValueStore* kvStore =
-	    new ShardedRocksDBKeyValueStore(rocksDBTestDir, deterministicRandom()->randomUniqueID());
-	wait(kvStore->init());
-
-	// Add some ranges.
-	std::vector<Future<Void>> addRangeFutures;
-	addRangeFutures.push_back(kvStore->addRange(KeyRangeRef("a"_sr, "c"_sr), "shard-1"));
-	addRangeFutures.push_back(kvStore->addRange(KeyRangeRef("c"_sr, "f"_sr), "shard-2"));
-	addRangeFutures.push_back(kvStore->addRange(KeyRangeRef("h"_sr, "k"_sr), "shard-1"));
-	kvStore->persistRangeMapping(KeyRangeRef("a"_sr, "f"_sr), true);
-	wait(waitForAll(addRangeFutures) && kvStore->commit(false));
-
-	for (const auto& [k, v] : kvs) {
-		kvStore->set(KeyValueRef(k, v));
-	}
-	wait(kvStore->commit(false));
-
-	state std::string checkpointDir = "checkpoint";
-	platform::eraseDirectoryRecursive(checkpointDir);
-
-	// Checkpoint iterator returns only the desired keyrange, i.e., ["ab", "b"].
-	CheckpointRequest request(latestVersion,
-	                          { KeyRangeRef("a"_sr, "c"_sr), KeyRangeRef("h"_sr, "k"_sr) },
-	                          DataMoveRocksCF,
-	                          deterministicRandom()->randomUniqueID(),
-	                          checkpointDir);
-	CheckpointMetaData metaData = wait(kvStore->checkpoint(request));
-
-	state Standalone<StringRef> token = BinaryWriter::toValue(KeyRangeRef("a"_sr, "k"_sr), IncludeVersion());
-	state ICheckpointReader* cpReader =
-	    newCheckpointReader(metaData, CheckpointAsKeyValues::True, deterministicRandom()->randomUniqueID());
-	ASSERT(cpReader != nullptr);
-	wait(cpReader->init(token));
-	state KeyRange testRange(KeyRangeRef("ab"_sr, "b"_sr));
-	state std::unique_ptr<ICheckpointIterator> iter0 = cpReader->getIterator(testRange);
-	state int numKeys = 0;
-	try {
-		loop {
-			RangeResult res = wait(iter0->nextBatch(CLIENT_KNOBS->REPLY_BYTE_LIMIT, CLIENT_KNOBS->REPLY_BYTE_LIMIT));
-			for (const auto& kv : res) {
-				ASSERT(testRange.contains(kv.key));
-				ASSERT(kvs[kv.key] == kv.value);
-				++numKeys;
-			}
-		}
-	} catch (Error& e) {
-		ASSERT(e.code() == error_code_end_of_stream);
-		ASSERT(numKeys == 2);
-	}
-
-	testRange = KeyRangeRef("a"_sr, "k"_sr);
-	state std::unique_ptr<ICheckpointIterator> iter1 = cpReader->getIterator(testRange);
-	try {
-		numKeys = 0;
-		loop {
-			RangeResult res = wait(iter1->nextBatch(CLIENT_KNOBS->REPLY_BYTE_LIMIT, CLIENT_KNOBS->REPLY_BYTE_LIMIT));
-			for (const auto& kv : res) {
-				ASSERT(testRange.contains(kv.key));
-				ASSERT(kvs[kv.key] == kv.value);
-				++numKeys;
-			}
-		}
-	} catch (Error& e) {
-		ASSERT(e.code() == error_code_end_of_stream);
-		ASSERT(numKeys == 7);
-	}
-
-	iter0.reset();
-	iter1.reset();
-	ASSERT(!cpReader->inUse());
-	TraceEvent(SevDebug, "ShardedRocksCheckpointReaaderTested");
-	std::vector<Future<Void>> closes;
-	closes.push_back(cpReader->close());
-	closes.push_back(kvStore->onClosed());
-	kvStore->dispose();
-	wait(waitForAll(closes));
-
-	platform::eraseDirectoryRecursive(rocksDBTestDir);
-	platform::eraseDirectoryRecursive(checkpointDir);
-
-	return Void();
-}
-
-TEST_CASE("noSim/ShardedRocksDB/CheckpointRestore") {
-	state std::string rocksDBTestDir = "sharded-rocks-checkpoint" + deterministicRandom()->randomAlphaNumeric(5);
-	state std::map<Key, Value> kvs({ { "ab"_sr, "TestValueAB"_sr },
-	                                 { "ad"_sr, "TestValueAD"_sr },
-	                                 { "b"_sr, "TestValueB"_sr },
-	                                 { "ba"_sr, "TestValueBA"_sr },
-	                                 { "c"_sr, "TestValueC"_sr },
-	                                 { "d"_sr, "TestValueD"_sr },
-	                                 { "e"_sr, "TestValueE"_sr },
-	                                 { "h"_sr, "TestValueH"_sr },
-	                                 { "ha"_sr, "TestValueHA"_sr } });
-	platform::eraseDirectoryRecursive(rocksDBTestDir);
-	state IKeyValueStore* kvStore =
-	    new ShardedRocksDBKeyValueStore(rocksDBTestDir, deterministicRandom()->randomUniqueID());
-	wait(kvStore->init());
-
-	state std::string shardId = "shard_1";
-	state std::string emptyShardId = "shard_2";
-	state KeyRangeRef rangeK(""_sr, "k"_sr);
-	state KeyRangeRef rangeKz("z1"_sr, "z3"_sr);
-	// // Add some ranges.
-	std::vector<Future<Void>> addRangeFutures;
-	addRangeFutures.push_back(kvStore->addRange(rangeK, shardId));
-	addRangeFutures.push_back(kvStore->addRange(rangeKz, emptyShardId));
-	kvStore->persistRangeMapping(rangeK, true);
-	kvStore->persistRangeMapping(rangeKz, true);
-	wait(waitForAll(addRangeFutures));
-	wait(kvStore->commit(false));
-
-	for (const auto& [k, v] : kvs) {
-		kvStore->set(KeyValueRef(k, v));
-	}
-	wait(kvStore->commit(false));
-	kvStore->clear(KeyRangeRef(""_sr, "z"_sr));
-	wait(kvStore->commit(false));
-
-	state Error err;
-	try {
-		wait(testCheckpointRestore(kvStore, { rangeK }));
-	} catch (Error& e) {
-		TraceEvent(SevError, "TestCheckpointRestoreError").errorUnsuppressed(e);
-		err = e;
-	}
-	// This will fail once RocksDB is upgraded to 8.1.
-	// ASSERT(err.code() == error_code_failed_to_restore_checkpoint);
-
-	try {
-		wait(testCheckpointRestore(kvStore, { rangeKz }));
-	} catch (Error& e) {
-		TraceEvent("TestCheckpointRestoreError").errorUnsuppressed(e);
-		err = e;
-	}
-
-	std::vector<Future<Void>> closes;
-	closes.push_back(kvStore->onClosed());
-	kvStore->close();
-	wait(waitForAll(closes));
-
-	platform::eraseDirectoryRecursive(rocksDBTestDir);
-
-	return Void();
-}
-
-TEST_CASE("noSim/ShardedRocksDB/RocksDBSstFileWriter") {
-	state std::string localFile = "rocksdb-sst-file-dump.sst";
-	state std::unique_ptr<IRocksDBSstFileWriter> sstWriter = newRocksDBSstFileWriter();
-	// Write nothing to sst file
-	sstWriter->open(localFile);
-	bool anyFileCreated = sstWriter->finish();
-	ASSERT(!anyFileCreated);
-	// Write kvs1 to sst file
-	state std::map<Key, Value> kvs1({ { "a"_sr, "1"_sr },
-	                                  { "ab"_sr, "12"_sr },
-	                                  { "ad"_sr, "14"_sr },
-	                                  { "b"_sr, "2"_sr },
-	                                  { "ba"_sr, "21"_sr },
-	                                  { "c"_sr, "3"_sr },
-	                                  { "d"_sr, "4"_sr },
-	                                  { "e"_sr, "5"_sr },
-	                                  { "h"_sr, "8"_sr },
-	                                  { "ha"_sr, "81"_sr } });
-	sstWriter = newRocksDBSstFileWriter();
-	sstWriter->open(localFile);
-	for (const auto& [key, value] : kvs1) {
-		sstWriter->write(key, value);
-	}
-	anyFileCreated = sstWriter->finish();
-	ASSERT(anyFileCreated);
-	// Write kvs2 to the same sst file where kvs2 keys are different from kvs1
-	state std::map<Key, Value> kvs2({ { "fa"_sr, "61"_sr },
-	                                  { "fab"_sr, "612"_sr },
-	                                  { "fad"_sr, "614"_sr },
-	                                  { "fb"_sr, "62"_sr },
-	                                  { "fba"_sr, "621"_sr },
-	                                  { "fc"_sr, "63"_sr },
-	                                  { "fd"_sr, "64"_sr },
-	                                  { "fe"_sr, "65"_sr },
-	                                  { "fh"_sr, "68"_sr },
-	                                  { "fha"_sr, "681"_sr } });
-	sstWriter->open(localFile);
-	for (const auto& [key, value] : kvs2) {
-		sstWriter->write(key, value);
-	}
-	anyFileCreated = sstWriter->finish();
-	ASSERT(anyFileCreated);
-	// Write kvs3 to the same sst file where kvs3 modifies values of kvs2
-	state std::map<Key, Value> kvs3({ { "fa"_sr, "1"_sr },
-	                                  { "fab"_sr, "12"_sr },
-	                                  { "fad"_sr, "14"_sr },
-	                                  { "fb"_sr, "2"_sr },
-	                                  { "fba"_sr, "21"_sr },
-	                                  { "fc"_sr, "3"_sr },
-	                                  { "fd"_sr, "4"_sr },
-	                                  { "fe"_sr, "5"_sr },
-	                                  { "fh"_sr, "8"_sr },
-	                                  { "fha"_sr, "81"_sr } });
-	sstWriter->open(localFile);
-	for (const auto& [key, value] : kvs3) {
-		sstWriter->write(key, value);
-	}
-	anyFileCreated = sstWriter->finish();
-	ASSERT(anyFileCreated);
-	// Check: sst only contains kv of kvs3
-	rocksdb::Status status;
-	rocksdb::IngestExternalFileOptions ingestOptions;
-	rocksdb::DB* db;
-	rocksdb::Options options;
-	options.create_if_missing = true;
-	status = rocksdb::DB::Open(options, "testdb", &db);
-	ASSERT(status.ok());
-	status = db->IngestExternalFile({ localFile }, ingestOptions);
-	ASSERT(status.ok());
-	std::string value;
-	for (const auto& [key, targetValue] : kvs1) {
-		status = db->Get(rocksdb::ReadOptions(), key.toString(), &value);
-		ASSERT(status.IsNotFound());
-	}
-	for (const auto& [key, targetValue] : kvs2) {
-		status = db->Get(rocksdb::ReadOptions(), key.toString(), &value);
-		ASSERT(value != targetValue.toString());
-	}
-	for (const auto& [key, targetValue] : kvs3) {
-		status = db->Get(rocksdb::ReadOptions(), key.toString(), &value);
-		ASSERT(status.ok());
-		ASSERT(value == targetValue.toString());
-	}
-	delete db;
-	return Void();
-}
-
-} // namespace
-
-#endif // SSD_ROCKSDB_EXPERIMENTAL

From 50a60c0c01dc0e828eb45542d1c979378d729271 Mon Sep 17 00:00:00 2001
From: He Liu <heliu05023@gmail.com>
Date: Thu, 18 May 2023 12:52:21 -0700
Subject: [PATCH 160/468] Fixed unit tests.

---
 fdbserver/KeyValueStoreRocksDB.actor.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/fdbserver/KeyValueStoreRocksDB.actor.cpp b/fdbserver/KeyValueStoreRocksDB.actor.cpp
index 2aaa2b020af..0f58d96a6b6 100644
--- a/fdbserver/KeyValueStoreRocksDB.actor.cpp
+++ b/fdbserver/KeyValueStoreRocksDB.actor.cpp
@@ -2506,7 +2506,7 @@ TEST_CASE("noSim/fdbserver/KeyValueStoreRocksDB/CheckpointRestore") {
 TEST_CASE("noSim/fdbserver/KeyValueStoreRocksDB/RocksDBTypes") {
 	// If the following assertion fails, update SstFileMetaData and LiveFileMetaData in RocksDBCheckpointUtils.actor.h
 	// to be the same as rocksdb::SstFileMetaData and rocksdb::LiveFileMetaData.
-	ASSERT_EQ(sizeof(rocksdb::LiveFileMetaData), 192);
+	ASSERT_EQ(sizeof(rocksdb::LiveFileMetaData), 208);
 	ASSERT_EQ(sizeof(rocksdb::ExportImportFilesMetaData), 32);
 	return Void();
 }

From 7e78d7dff42d7074dd684fb273a205c69482aa1e Mon Sep 17 00:00:00 2001
From: neethuhaneesha <nbingi@apple.com>
Date: Fri, 12 May 2023 13:06:54 -0700
Subject: [PATCH 161/468] Hex values in TSS logs and rocksb debuglogs mode knob

---
 fdbclient/ServerKnobs.cpp                |  1 +
 fdbclient/ServerKnobs.h                  |  1 +
 fdbclient/StorageServerInterface.cpp     | 16 ++++++++++++++++
 fdbserver/KeyValueStoreRocksDB.actor.cpp |  3 +++
 4 files changed, 21 insertions(+)

diff --git a/fdbclient/ServerKnobs.cpp b/fdbclient/ServerKnobs.cpp
index c619ac9aeef..73315a78d04 100644
--- a/fdbclient/ServerKnobs.cpp
+++ b/fdbclient/ServerKnobs.cpp
@@ -421,6 +421,7 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi
 	init( ROCKSDB_DISABLE_WAL_EXPERIMENTAL,                    false );
 	init( ROCKSDB_WAL_TTL_SECONDS,                                 0 );
 	init( ROCKSDB_WAL_SIZE_LIMIT_MB,                               0 );
+	init( ROCKSDB_LOG_LEVEL_DEBUG,                             false );
 	// If ROCKSDB_SINGLEKEY_DELETES_ON_CLEARRANGE is enabled, disable ROCKSDB_ENABLE_CLEAR_RANGE_EAGER_READS knob.
 	// These knobs have contrary functionality.
 	init( ROCKSDB_SINGLEKEY_DELETES_ON_CLEARRANGE,              true );
diff --git a/fdbclient/ServerKnobs.h b/fdbclient/ServerKnobs.h
index e3e7ff8e3a0..e86f9313a36 100644
--- a/fdbclient/ServerKnobs.h
+++ b/fdbclient/ServerKnobs.h
@@ -343,6 +343,7 @@ class ServerKnobs : public KnobsImpl<ServerKnobs> {
 	bool ROCKSDB_DISABLE_WAL_EXPERIMENTAL;
 	int64_t ROCKSDB_WAL_TTL_SECONDS;
 	int64_t ROCKSDB_WAL_SIZE_LIMIT_MB;
+	bool ROCKSDB_LOG_LEVEL_DEBUG;
 	bool ROCKSDB_SINGLEKEY_DELETES_ON_CLEARRANGE;
 	int ROCKSDB_SINGLEKEY_DELETES_MAX;
 	bool ROCKSDB_ENABLE_CLEAR_RANGE_EAGER_READS;
diff --git a/fdbclient/StorageServerInterface.cpp b/fdbclient/StorageServerInterface.cpp
index 71097c939e3..7b36719d644 100644
--- a/fdbclient/StorageServerInterface.cpp
+++ b/fdbclient/StorageServerInterface.cpp
@@ -149,6 +149,16 @@ static void traceKeyValuesSummary(TraceEvent& event,
 	    .detail("TSSReplySummary", tssSummaryString);
 }
 
+// convert a StringRef to Hex string
+static std::string hexStringRef(const StringRef& s) {
+	std::string result;
+	result.reserve(s.size() * 2);
+	for (int i = 0; i < s.size(); i++) {
+		result.append(format("%02x", s[i]));
+	}
+	return result;
+}
+
 static void traceKeyValuesDiff(TraceEvent& event,
                                const KeySelectorRef& begin,
                                const KeySelectorRef& end,
@@ -168,11 +178,17 @@ static void traceKeyValuesDiff(TraceEvent& event,
 			event.detail("MismatchIndex", i);
 			if (i >= ssKV.size() || i >= tssKV.size() || ssKV[i].key != tssKV[i].key) {
 				event.detail("MismatchSSKey", i < ssKV.size() ? ssKV[i].key : "missing"_sr);
+				event.detail("MismatchSSKeyHex", i < ssKV.size() ? hexStringRef(ssKV[i].key) : "missing"_sr);
 				event.detail("MismatchTSSKey", i < tssKV.size() ? tssKV[i].key : "missing"_sr);
+				event.detail("MismatchTSSKeyHex", i < tssKV.size() ? hexStringRef(tssKV[i].key) : "missing"_sr)
+				    .setMaxFieldLength(-1);
 			} else {
 				event.detail("MismatchKey", ssKV[i].key);
 				event.detail("MismatchSSValue", traceChecksumValue(ssKV[i].value));
+				event.detail("MismatchSSValueHex", hexStringRef(traceChecksumValue(ssKV[i].value)));
 				event.detail("MismatchTSSValue", traceChecksumValue(tssKV[i].value));
+				event.detail("MismatchTSSValueHex", hexStringRef(traceChecksumValue(tssKV[i].value)))
+				    .setMaxFieldLength(-1);
 			}
 			mismatchFound = true;
 			break;
diff --git a/fdbserver/KeyValueStoreRocksDB.actor.cpp b/fdbserver/KeyValueStoreRocksDB.actor.cpp
index 0f58d96a6b6..f498a036e24 100644
--- a/fdbserver/KeyValueStoreRocksDB.actor.cpp
+++ b/fdbserver/KeyValueStoreRocksDB.actor.cpp
@@ -418,6 +418,9 @@ rocksdb::Options getOptions() {
 	//    checks will be performed with ttl being first.
 	options.WAL_ttl_seconds = SERVER_KNOBS->ROCKSDB_WAL_TTL_SECONDS;
 	options.WAL_size_limit_MB = SERVER_KNOBS->ROCKSDB_WAL_SIZE_LIMIT_MB;
+	if (SERVER_KNOBS->ROCKSDB_LOG_LEVEL_DEBUG) {
+		options.info_log_level = rocksdb::InfoLogLevel::DEBUG_LEVEL;
+	}
 
 	options.statistics = rocksdb::CreateDBStatistics();
 	options.statistics->set_stats_level(rocksdb::StatsLevel(SERVER_KNOBS->ROCKSDB_STATS_LEVEL));

From e19fef295bba6187f39bad22aa3e779e165c6b76 Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyu_zhou@apple.com>
Date: Mon, 22 May 2023 08:36:04 -0700
Subject: [PATCH 162/468] Suppress AlreadyDisconnected events (#10193)

This can be too verbose to cause test failures.
---
 fdbrpc/FailureMonitor.actor.cpp | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/fdbrpc/FailureMonitor.actor.cpp b/fdbrpc/FailureMonitor.actor.cpp
index 372b3812a5c..87a79f34c6f 100644
--- a/fdbrpc/FailureMonitor.actor.cpp
+++ b/fdbrpc/FailureMonitor.actor.cpp
@@ -148,7 +148,17 @@ Future<Void> SimpleFailureMonitor::onDisconnectOrFailure(Endpoint const& endpoin
 	// If the endpoint or address is already failed, return right away
 	auto i = addressStatus.find(endpoint.getPrimaryAddress());
 	if (i == addressStatus.end() || i->second.isFailed() || failedEndpoints.count(endpoint)) {
-		TraceEvent("AlreadyDisconnected").detail("Addr", endpoint.getPrimaryAddress()).detail("Tok", endpoint.token);
+		TraceEvent event("AlreadyDisconnected");
+		if (endpoint.token.first() == 0xffffffffffffffff) {
+			// well known endpoint
+			event.suppressFor(5.0);
+		} else {
+			event.suppressFor(0.1);
+		}
+		event.detail("Addr", endpoint.getPrimaryAddress())
+		    .detail("Reason", i == addressStatus.end() || i->second.isFailed() ? "Disconnected" : "EndpointFailed")
+		    .detail("Tok", endpoint.token)
+		    .log();
 		return Void();
 	}
 

From 1d7e27a71f6e118ed82b4bfd30de52fcb009efdf Mon Sep 17 00:00:00 2001
From: w41ter <w41ter.l@gmail.com>
Date: Mon, 29 May 2023 17:30:17 +0800
Subject: [PATCH 163/468] Fix clang copy elision warning

---
 fdbclient/S3BlobStore.actor.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/fdbclient/S3BlobStore.actor.cpp b/fdbclient/S3BlobStore.actor.cpp
index f9ddf08b9e9..9c1c54704e7 100644
--- a/fdbclient/S3BlobStore.actor.cpp
+++ b/fdbclient/S3BlobStore.actor.cpp
@@ -401,7 +401,7 @@ std::string constructResourcePath(Reference<S3BlobStoreEndpoint> b, std::string
 		resource += object;
 	}
 
-	return std::move(resource);
+	return resource;
 }
 
 ACTOR Future<bool> bucketExists_impl(Reference<S3BlobStoreEndpoint> b, std::string bucket) {

From 774949b9ed8f28d180f94dfac2d0f29e8d13da2d Mon Sep 17 00:00:00 2001
From: Aaron Molitor <amolitor@apple.com>
Date: Fri, 12 May 2023 11:57:48 -0500
Subject: [PATCH 164/468] add cmake option to include RocksDB Tools with the
 Rocks DB compile

---
 cmake/CompileRocksDB.cmake | 4 ++--
 cmake/FDBComponents.cmake  | 1 +
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/cmake/CompileRocksDB.cmake b/cmake/CompileRocksDB.cmake
index 709c61dfb9d..436ac97630f 100644
--- a/cmake/CompileRocksDB.cmake
+++ b/cmake/CompileRocksDB.cmake
@@ -22,8 +22,8 @@ set(RocksDB_CMAKE_ARGS
   -DFAIL_ON_WARNINGS=OFF
   -DWITH_GFLAGS=OFF
   -DWITH_TESTS=OFF
-  -DWITH_TOOLS=OFF
-  -DWITH_CORE_TOOLS=OFF
+  -DWITH_TOOLS=${ROCKSDB_TOOLS}
+  -DWITH_CORE_TOOLS=${ROCKSDB_TOOLS}
   -DWITH_BENCHMARK_TOOLS=OFF
   -DWITH_BZ2=OFF
   -DWITH_LZ4=ON
diff --git a/cmake/FDBComponents.cmake b/cmake/FDBComponents.cmake
index 8ce7c243a7c..8c110c75cbe 100644
--- a/cmake/FDBComponents.cmake
+++ b/cmake/FDBComponents.cmake
@@ -166,6 +166,7 @@ set(PORTABLE_ROCKSDB ON CACHE BOOL "Compile RocksDB in portable mode") # Set thi
 set(ROCKSDB_SSE42 OFF CACHE BOOL "Compile RocksDB with SSE42 enabled")
 set(ROCKSDB_AVX ${USE_AVX} CACHE BOOL "Compile RocksDB with AVX enabled")
 set(ROCKSDB_AVX2 OFF CACHE BOOL "Compile RocksDB with AVX2 enabled")
+set(ROCKSDB_TOOLS OFF CACHE BOOL "Compile RocksDB tools")
 set(WITH_LIBURING OFF CACHE BOOL "Build with liburing enabled") # Set this to ON to include liburing
 # RocksDB is currently enabled by default for GCC but does not build with the latest
 # Clang.

From 1efbd08e78a89f1789702dd5114cf731037de33d Mon Sep 17 00:00:00 2001
From: Aaron Molitor <amolitor@apple.com>
Date: Tue, 30 May 2023 08:14:08 -0500
Subject: [PATCH 165/468] update links to foundationdb.org to reference GitHub

---
 bindings/go/src/fdb/doc.go               | 2 +-
 bindings/java/pom.xml.in                 | 4 ++--
 bindings/java/src/main/overview.html.in  | 2 +-
 bindings/python/README.rst               | 2 +-
 bindings/ruby/fdb.gemspec.cmake          | 2 +-
 bindings/ruby/fdb.gemspec.in             | 2 +-
 packaging/docker/samples/local/README.md | 2 +-
 tests/CMakeLists.txt                     | 2 +-
 8 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/bindings/go/src/fdb/doc.go b/bindings/go/src/fdb/doc.go
index 2ecf99f200e..9d8f25b3232 100644
--- a/bindings/go/src/fdb/doc.go
+++ b/bindings/go/src/fdb/doc.go
@@ -25,7 +25,7 @@ Package fdb provides an interface to FoundationDB databases (version 2.0 or high
 
 To build and run programs using this package, you must have an installed copy of
 the FoundationDB client libraries (version 2.0.0 or later), available for Linux,
-Windows and OS X at https://www.foundationdb.org/download/.
+Windows and OS X at https://github.com/apple/foundationdb/releases
 
 This documentation specifically applies to the FoundationDB Go binding. For more
 extensive guidance to programming with FoundationDB, as well as API
diff --git a/bindings/java/pom.xml.in b/bindings/java/pom.xml.in
index bde89709842..7ff3c896562 100644
--- a/bindings/java/pom.xml.in
+++ b/bindings/java/pom.xml.in
@@ -10,7 +10,7 @@
   <packaging>jar</packaging>
 
   <name>foundationdb-java</name>
-  <description>Java bindings for the FoundationDB database. These bindings require the FoundationDB client, which is under a different license. The client can be obtained from https://www.foundationdb.org/download/.</description>
+  <description>Java bindings for the FoundationDB database. These bindings require the FoundationDB client, which is under a different license. The client can be obtained from https://github.com/apple/foundationdb/releases</description>
   <inceptionYear>2010</inceptionYear>
   <url>https://www.foundationdb.org</url>
 
@@ -26,7 +26,7 @@
   </developers>
 
   <scm>
-    <url>http://0.0.0.0</url>
+    <url>https://github.com/apple/foundationdb</url>
   </scm>
 
   <licenses>
diff --git a/bindings/java/src/main/overview.html.in b/bindings/java/src/main/overview.html.in
index fe20448dfb1..1a0774c782c 100644
--- a/bindings/java/src/main/overview.html.in
+++ b/bindings/java/src/main/overview.html.in
@@ -7,7 +7,7 @@ FoundationDB's Java bindings rely on native libraries that are installed as part
 FoundationDB client binaries installation (see 
 <a href="/foundationdb/api-general.html#installing-client-binaries" target="_blank">
 Installing FoundationDB client binaries</a>). The JAR can be downloaded from
-<a href="https://www.foundationdb.org/download/">our website</a>
+<a href="https://central.sonatype.com/artifact/org.foundationdb/fdb-java">maven central</a>
 and then added to your classpath.<br>
 <br>
 <h1>Getting started</h1>
diff --git a/bindings/python/README.rst b/bindings/python/README.rst
index d3e2d696048..782e03896f3 100644
--- a/bindings/python/README.rst
+++ b/bindings/python/README.rst
@@ -1,3 +1,3 @@
 Complete documentation of the FoundationDB Python API can be found at https://apple.github.io/foundationdb/api-python.html.
 
-These bindings require the FoundationDB client. The client can be obtained from https://www.foundationdb.org/download/.
+These bindings require the FoundationDB client. The client can be obtained from https://github.com/apple/foundationdb/releases
diff --git a/bindings/ruby/fdb.gemspec.cmake b/bindings/ruby/fdb.gemspec.cmake
index d73b8bac9bd..bf36521fad5 100644
--- a/bindings/ruby/fdb.gemspec.cmake
+++ b/bindings/ruby/fdb.gemspec.cmake
@@ -18,5 +18,5 @@ EOF
   s.license = 'Apache-2.0'
   s.add_dependency('ffi', '~> 1.1', '>= 1.1.5')
   s.required_ruby_version = '>= 1.9.3'
-  s.requirements << 'These bindings require the FoundationDB client. The client can be obtained from https://www.foundationdb.org/download/.'
+  s.requirements << 'These bindings require the FoundationDB client. The client can be obtained from https://github.com/apple/foundationdb/releases'
 end
diff --git a/bindings/ruby/fdb.gemspec.in b/bindings/ruby/fdb.gemspec.in
index 092bc2eacda..fa61f8aba4b 100644
--- a/bindings/ruby/fdb.gemspec.in
+++ b/bindings/ruby/fdb.gemspec.in
@@ -18,5 +18,5 @@ EOF
   s.license = 'Apache-2.0'
   s.add_dependency('ffi', '~> 1.1', '>= 1.1.5')
   s.required_ruby_version = '>= 1.9.3'
-  s.requirements << 'These bindings require the FoundationDB client. The client can be obtained from https://www.foundationdb.org/download/.'
+  s.requirements << 'These bindings require the FoundationDB client. The client can be obtained from https://github.com/apple/foundationdb/releases'
 end
diff --git a/packaging/docker/samples/local/README.md b/packaging/docker/samples/local/README.md
index f7f5b3e979f..771cd08dbcb 100644
--- a/packaging/docker/samples/local/README.md
+++ b/packaging/docker/samples/local/README.md
@@ -7,7 +7,7 @@ the server process to its host machine.
 
 This depends on having the FoundationDB client installed on your host machine
 to work properly. This can be done using one of the client packages available
-on our [Download](https://www.foundationdb.org/download/) page. The startup
+from our [GitHub Releases](https://github.com/apple/foundationdb/releases). The startup
 scripts included here depend on `fdbcli` from one of those packages, and any
 client that wishes to connect will need a copy of the FoundationDB native client
 in addition to its binding of choice. Both the CLI and the native client
diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt
index e3ba968e392..3c0b6518a0d 100644
--- a/tests/CMakeLists.txt
+++ b/tests/CMakeLists.txt
@@ -29,7 +29,7 @@ if(WITH_PYTHON)
     endif()
     message(WARNING "\
   No old fdbserver binary found - using ${fdbserver_location} \
-  It is recommended to install the current stable version from https://www.foundationdb.org/download/ \
+  It is recommended to install the current stable version from https://github.com/apple/foundationdb/releases \
   Or provide a path to another fdbserver")
   endif()
 

From 2f3abb148eada5b00ec07bc176db0e8666f97cc0 Mon Sep 17 00:00:00 2001
From: w41ter <w41ter.l@gmail.com>
Date: Wed, 24 May 2023 16:09:57 +0800
Subject: [PATCH 166/468] Fix restore range loss

---
 fdbclient/FileBackupAgent.actor.cpp           |   3 +-
 .../workloads/RestoreMultiRanges.actor.cpp    | 161 ++++++++++++++++++
 tests/CMakeLists.txt                          |   1 +
 tests/slow/RestoreMultiRanges.toml            |   7 +
 4 files changed, 171 insertions(+), 1 deletion(-)
 create mode 100644 fdbserver/workloads/RestoreMultiRanges.actor.cpp
 create mode 100644 tests/slow/RestoreMultiRanges.toml

diff --git a/fdbclient/FileBackupAgent.actor.cpp b/fdbclient/FileBackupAgent.actor.cpp
index 482baaac604..6411f53f306 100644
--- a/fdbclient/FileBackupAgent.actor.cpp
+++ b/fdbclient/FileBackupAgent.actor.cpp
@@ -3045,12 +3045,13 @@ struct RestoreRangeTaskFunc : RestoreFileTaskFuncBase {
 		state Standalone<VectorRef<KeyValueRef>> blockData = wait(decodeRangeFileBlock(inFile, readOffset, readLen));
 
 		// First and last key are the range for this file
-		state KeyRange fileRange = KeyRangeRef(blockData.front().key, blockData.back().key);
+		state KeyRange fileRange;
 		state std::vector<KeyRange> originalFileRanges;
 		// If fileRange doesn't intersect restore range then we're done.
 		state int index;
 		for (index = 0; index < restoreRanges.get().size(); index++) {
 			auto& restoreRange = restoreRanges.get()[index];
+			fileRange = KeyRangeRef(blockData.front().key, blockData.back().key);
 			if (!fileRange.intersects(restoreRange))
 				continue;
 
diff --git a/fdbserver/workloads/RestoreMultiRanges.actor.cpp b/fdbserver/workloads/RestoreMultiRanges.actor.cpp
new file mode 100644
index 00000000000..2fa27162a51
--- /dev/null
+++ b/fdbserver/workloads/RestoreMultiRanges.actor.cpp
@@ -0,0 +1,161 @@
+/*
+ * RestoreMultiRanges.actor.cpp
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2022 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "fdbclient/FDBTypes.h"
+#include "fdbclient/ReadYourWrites.h"
+#include "fdbrpc/simulator.h"
+#include "fdbclient/BackupAgent.actor.h"
+#include "fdbclient/BackupContainer.h"
+#include "fdbserver/workloads/workloads.actor.h"
+#include "flow/actorcompiler.h" // This must be the last #include.
+
+struct RestoreMultiRangesWorkload : TestWorkload {
+
+	FileBackupAgent backupAgent;
+	Reference<IBackupContainer> backupContainer;
+
+	RestoreMultiRangesWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {}
+
+	static constexpr const char* NAME = "RestoreMultiRanges";
+
+	ACTOR static Future<Void> clearDatabase(Database cx) {
+		state Transaction tr(cx);
+		loop {
+			try {
+				tr.clear(normalKeys);
+				wait(tr.commit());
+				return Void();
+			} catch (Error& e) {
+				wait(tr.onError(e));
+			}
+		}
+	}
+
+	ACTOR static Future<Void> prepareDatabase(Database cx) {
+		state Transaction tr(cx);
+		loop {
+			try {
+				tr.reset();
+				tr.set("a"_sr, "a"_sr);
+				tr.set("aaaa"_sr, "aaaa"_sr);
+				tr.set("b"_sr, "b"_sr);
+				tr.set("bb"_sr, "bb"_sr);
+				tr.set("bbb"_sr, "bbb"_sr);
+				wait(tr.commit());
+				return Void();
+			} catch (Error& e) {
+				wait(tr.onError(e));
+			}
+		}
+	}
+
+	static void logTestData(const VectorRef<KeyValueRef>& data) {
+		TraceEvent("TestFailureDetail").log();
+		int index = 0;
+		for (auto& entry : data) {
+			TraceEvent("CurrentDataEntry")
+			    .detail("Index", index)
+			    .detail("Key", entry.key.toString())
+			    .detail("Value", entry.value.toString());
+			index++;
+		}
+	}
+
+	ACTOR static Future<bool> verifyDatabase(Database cx) {
+		state Transaction tr(cx);
+		state KeyRangeRef range("a"_sr, "z"_sr);
+		loop {
+			try {
+				tr.reset();
+				RangeResult kvs = wait(tr.getRange(range, 10));
+				if (kvs.size() != 4) {
+					logTestData(kvs);
+					TraceEvent(SevError, "TestFailureInfo")
+					    .detail("DataSize", kvs.size())
+					    .detail("Expect", 4)
+					    .detail("Workload", NAME);
+					return false;
+				}
+				KeyRef keys[4] = { "a"_sr, "aaaa"_sr, "bb"_sr, "bbb"_sr };
+				for (size_t i = 0; i < 4; ++i) {
+					if (kvs[i].key != keys[i]) {
+						TraceEvent(SevError, "TestFailureInfo")
+						    .detail("ExpectKey", keys[i])
+						    .detail("Got", kvs[i].key)
+						    .detail("Index", i);
+						return false;
+					}
+				}
+				return true;
+			} catch (Error& e) {
+				wait(tr.onError(e));
+			}
+		}
+	}
+
+	ACTOR static Future<Void> _start(RestoreMultiRangesWorkload* self, Database cx) {
+		wait(prepareDatabase(cx));
+
+		state std::string backupContainer = "file://simfdb/backups/";
+		state std::string tagName = "default";
+		state Standalone<VectorRef<KeyRangeRef>> backupRanges;
+		backupRanges.push_back_deep(backupRanges.arena(), KeyRangeRef("a"_sr, "z"_sr));
+		try {
+			wait(self->backupAgent.submitBackup(cx,
+			                                    StringRef(backupContainer),
+			                                    {},
+			                                    deterministicRandom()->randomInt(0, 60),
+			                                    deterministicRandom()->randomInt(0, 100),
+			                                    tagName,
+			                                    backupRanges,
+			                                    true,
+			                                    StopWhenDone::True));
+		} catch (Error& e) {
+			if (e.code() != error_code_backup_unneeded && e.code() != error_code_backup_duplicate)
+				throw;
+		}
+
+		state Reference<IBackupContainer> container;
+		wait(success(self->backupAgent.waitBackup(cx, tagName, StopWhenDone::True, &container)));
+
+		wait(clearDatabase(cx));
+
+		state Standalone<VectorRef<KeyRangeRef>> ranges;
+		ranges.push_back_deep(ranges.arena(), KeyRangeRef("a"_sr, "aaaaa"_sr));
+		ranges.push_back_deep(ranges.arena(), KeyRangeRef("bb"_sr, "bbbbb"_sr)); // Skip "b"
+		wait(success(self->backupAgent.restore(cx,
+		                                       cx,
+		                                       Key(tagName),
+		                                       Key(container->getURL()),
+		                                       {},
+		                                       ranges,
+		                                       WaitForComplete::True,
+		                                       ::invalidVersion,
+		                                       Verbose::True)));
+		return Void();
+	}
+
+	Future<Void> setup(Database const& cx) override { return Void(); }
+	Future<Void> start(Database const& cx) override { return clientId ? Void() : _start(this, cx); }
+	Future<bool> check(Database const& cx) override { return verifyDatabase(cx); }
+	void getMetrics(std::vector<PerfMetric>& m) override {}
+};
+
+WorkloadFactory<RestoreMultiRangesWorkload> RestoreMultiRangesWorkloadFactory;
diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt
index 3c0b6518a0d..56a918abe08 100644
--- a/tests/CMakeLists.txt
+++ b/tests/CMakeLists.txt
@@ -330,6 +330,7 @@ if(WITH_PYTHON)
   add_fdb_test(TEST_FILES slow/ParallelRestoreOldBackupCorrectnessCycle.toml)
   add_fdb_test(TEST_FILES slow/ParallelRestoreOldBackupCorrectnessMultiCycles.toml)
   add_fdb_test(TEST_FILES slow/ParallelRestoreOldBackupWriteDuringReadAtomicRestore.toml)
+  add_fdb_test(TEST_FILES slow/RestoreMultiRanges.toml)
   add_fdb_test(TEST_FILES ParallelRestoreOldBackupApiCorrectnessAtomicRestore.toml IGNORE)
   # Note that status tests are not deterministic.
   add_fdb_test(TEST_FILES status/invalid_proc_addresses.txt)
diff --git a/tests/slow/RestoreMultiRanges.toml b/tests/slow/RestoreMultiRanges.toml
new file mode 100644
index 00000000000..10721fd6830
--- /dev/null
+++ b/tests/slow/RestoreMultiRanges.toml
@@ -0,0 +1,7 @@
+[[test]]
+testTitle = 'RestoreMultiRanges'
+clearAfterTest = true
+simBackupAgents = 'BackupToFile'
+
+    [[test.workload]]
+    testName = 'RestoreMultiRanges'

From d7b3d881c9d677e7079b864e2ca529eebfe9d909 Mon Sep 17 00:00:00 2001
From: w41ter <w41ter.l@gmail.com>
Date: Wed, 31 May 2023 10:43:03 +0800
Subject: [PATCH 167/468] Fix test by disable tenant

---
 fdbserver/workloads/RestoreMultiRanges.actor.cpp | 11 +++++++++++
 tests/CMakeLists.txt                             |  2 +-
 tests/{slow => rare}/RestoreMultiRanges.toml     |  4 ++++
 3 files changed, 16 insertions(+), 1 deletion(-)
 rename tests/{slow => rare}/RestoreMultiRanges.toml (77%)

diff --git a/fdbserver/workloads/RestoreMultiRanges.actor.cpp b/fdbserver/workloads/RestoreMultiRanges.actor.cpp
index 2fa27162a51..b4e7e0ff5b4 100644
--- a/fdbserver/workloads/RestoreMultiRanges.actor.cpp
+++ b/fdbserver/workloads/RestoreMultiRanges.actor.cpp
@@ -79,11 +79,14 @@ struct RestoreMultiRangesWorkload : TestWorkload {
 	}
 
 	ACTOR static Future<bool> verifyDatabase(Database cx) {
+		state UID randomID = nondeterministicRandom()->randomUniqueID();
+		TraceEvent("RestoreMultiRanges_Verify").detail("UID", randomID);
 		state Transaction tr(cx);
 		state KeyRangeRef range("a"_sr, "z"_sr);
 		loop {
 			try {
 				tr.reset();
+				tr.debugTransaction(randomID);
 				RangeResult kvs = wait(tr.getRange(range, 10));
 				if (kvs.size() != 4) {
 					logTestData(kvs);
@@ -103,6 +106,7 @@ struct RestoreMultiRangesWorkload : TestWorkload {
 						return false;
 					}
 				}
+				TraceEvent("RestoreMultiRanges_VerifyPassed");
 				return true;
 			} catch (Error& e) {
 				wait(tr.onError(e));
@@ -111,12 +115,15 @@ struct RestoreMultiRangesWorkload : TestWorkload {
 	}
 
 	ACTOR static Future<Void> _start(RestoreMultiRangesWorkload* self, Database cx) {
+		TraceEvent("RestoreMultiRanges_StartBackup");
+		wait(clearDatabase(cx));
 		wait(prepareDatabase(cx));
 
 		state std::string backupContainer = "file://simfdb/backups/";
 		state std::string tagName = "default";
 		state Standalone<VectorRef<KeyRangeRef>> backupRanges;
 		backupRanges.push_back_deep(backupRanges.arena(), KeyRangeRef("a"_sr, "z"_sr));
+		TraceEvent("RestoreMultiRanges_SubmitBackup");
 		try {
 			wait(self->backupAgent.submitBackup(cx,
 			                                    StringRef(backupContainer),
@@ -132,11 +139,14 @@ struct RestoreMultiRangesWorkload : TestWorkload {
 				throw;
 		}
 
+		TraceEvent("RestoreMultiRanges_WaitBackup");
 		state Reference<IBackupContainer> container;
 		wait(success(self->backupAgent.waitBackup(cx, tagName, StopWhenDone::True, &container)));
 
+		TraceEvent("RestoreMultiRanges_ClearDatabase");
 		wait(clearDatabase(cx));
 
+		TraceEvent("RestoreMultiRanges_Restore");
 		state Standalone<VectorRef<KeyRangeRef>> ranges;
 		ranges.push_back_deep(ranges.arena(), KeyRangeRef("a"_sr, "aaaaa"_sr));
 		ranges.push_back_deep(ranges.arena(), KeyRangeRef("bb"_sr, "bbbbb"_sr)); // Skip "b"
@@ -149,6 +159,7 @@ struct RestoreMultiRangesWorkload : TestWorkload {
 		                                       WaitForComplete::True,
 		                                       ::invalidVersion,
 		                                       Verbose::True)));
+		TraceEvent("RestoreMultiRanges_Success");
 		return Void();
 	}
 
diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt
index 56a918abe08..ca37ea6661a 100644
--- a/tests/CMakeLists.txt
+++ b/tests/CMakeLists.txt
@@ -208,6 +208,7 @@ if(WITH_PYTHON)
   add_fdb_test(TEST_FILES rare/LargeApiCorrectnessStatus.toml)
   add_fdb_test(TEST_FILES rare/RYWDisable.toml)
   add_fdb_test(TEST_FILES rare/RandomReadWriteTest.toml)
+  add_fdb_test(TEST_FILES rare/RestoreMultiRanges.toml)
   add_fdb_test(TEST_FILES rare/SpecificUnitTests.toml)
   add_fdb_test(TEST_FILES rare/SwizzledLargeApiCorrectness.toml)
   add_fdb_test(TEST_FILES rare/RedwoodCorrectnessBTree.toml)
@@ -330,7 +331,6 @@ if(WITH_PYTHON)
   add_fdb_test(TEST_FILES slow/ParallelRestoreOldBackupCorrectnessCycle.toml)
   add_fdb_test(TEST_FILES slow/ParallelRestoreOldBackupCorrectnessMultiCycles.toml)
   add_fdb_test(TEST_FILES slow/ParallelRestoreOldBackupWriteDuringReadAtomicRestore.toml)
-  add_fdb_test(TEST_FILES slow/RestoreMultiRanges.toml)
   add_fdb_test(TEST_FILES ParallelRestoreOldBackupApiCorrectnessAtomicRestore.toml IGNORE)
   # Note that status tests are not deterministic.
   add_fdb_test(TEST_FILES status/invalid_proc_addresses.txt)
diff --git a/tests/slow/RestoreMultiRanges.toml b/tests/rare/RestoreMultiRanges.toml
similarity index 77%
rename from tests/slow/RestoreMultiRanges.toml
rename to tests/rare/RestoreMultiRanges.toml
index 10721fd6830..befdfa02ead 100644
--- a/tests/slow/RestoreMultiRanges.toml
+++ b/tests/rare/RestoreMultiRanges.toml
@@ -1,3 +1,6 @@
+[configuration]
+tenantModes = ['disabled']
+
 [[test]]
 testTitle = 'RestoreMultiRanges'
 clearAfterTest = true
@@ -5,3 +8,4 @@ simBackupAgents = 'BackupToFile'
 
     [[test.workload]]
     testName = 'RestoreMultiRanges'
+

From 8554747b3ee72738b012b69f492af79ebad72866 Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Tue, 6 Jun 2023 15:30:32 -0700
Subject: [PATCH 168/468] Fix workload for 7.1 branch

---
 fdbserver/CMakeLists.txt                         |  1 +
 fdbserver/workloads/RestoreMultiRanges.actor.cpp | 10 ++++------
 tests/rare/RestoreMultiRanges.toml               |  2 +-
 3 files changed, 6 insertions(+), 7 deletions(-)

diff --git a/fdbserver/CMakeLists.txt b/fdbserver/CMakeLists.txt
index d08d120dabb..dab8f0d10dc 100644
--- a/fdbserver/CMakeLists.txt
+++ b/fdbserver/CMakeLists.txt
@@ -266,6 +266,7 @@ set(FDBSERVER_SRCS
   workloads/ReportConflictingKeys.actor.cpp
   workloads/RestoreBackup.actor.cpp
   workloads/RestoreFromBlob.actor.cpp
+  workloads/RestoreMultiRanges.actor.cpp
   workloads/Rollback.actor.cpp
   workloads/RyowCorrectness.actor.cpp
   workloads/RYWDisable.actor.cpp
diff --git a/fdbserver/workloads/RestoreMultiRanges.actor.cpp b/fdbserver/workloads/RestoreMultiRanges.actor.cpp
index b4e7e0ff5b4..df5ccee8bc6 100644
--- a/fdbserver/workloads/RestoreMultiRanges.actor.cpp
+++ b/fdbserver/workloads/RestoreMultiRanges.actor.cpp
@@ -33,7 +33,7 @@ struct RestoreMultiRangesWorkload : TestWorkload {
 
 	RestoreMultiRangesWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {}
 
-	static constexpr const char* NAME = "RestoreMultiRanges";
+	std::string description() const override { return "RestoreMultiRangesWorkload"; }
 
 	ACTOR static Future<Void> clearDatabase(Database cx) {
 		state Transaction tr(cx);
@@ -93,7 +93,7 @@ struct RestoreMultiRangesWorkload : TestWorkload {
 					TraceEvent(SevError, "TestFailureInfo")
 					    .detail("DataSize", kvs.size())
 					    .detail("Expect", 4)
-					    .detail("Workload", NAME);
+					    .detail("Workload", "RestoreMultipleRanges");
 					return false;
 				}
 				KeyRef keys[4] = { "a"_sr, "aaaa"_sr, "bb"_sr, "bbb"_sr };
@@ -131,9 +131,7 @@ struct RestoreMultiRangesWorkload : TestWorkload {
 			                                    deterministicRandom()->randomInt(0, 60),
 			                                    deterministicRandom()->randomInt(0, 100),
 			                                    tagName,
-			                                    backupRanges,
-			                                    true,
-			                                    StopWhenDone::True));
+			                                    backupRanges));
 		} catch (Error& e) {
 			if (e.code() != error_code_backup_unneeded && e.code() != error_code_backup_duplicate)
 				throw;
@@ -169,4 +167,4 @@ struct RestoreMultiRangesWorkload : TestWorkload {
 	void getMetrics(std::vector<PerfMetric>& m) override {}
 };
 
-WorkloadFactory<RestoreMultiRangesWorkload> RestoreMultiRangesWorkloadFactory;
+WorkloadFactory<RestoreMultiRangesWorkload> RestoreMultiRangesWorkloadFactory("RestoreMultiRanges");
diff --git a/tests/rare/RestoreMultiRanges.toml b/tests/rare/RestoreMultiRanges.toml
index befdfa02ead..8b496004b3d 100644
--- a/tests/rare/RestoreMultiRanges.toml
+++ b/tests/rare/RestoreMultiRanges.toml
@@ -1,5 +1,5 @@
 [configuration]
-tenantModes = ['disabled']
+allowDefaultTenant = false
 
 [[test]]
 testTitle = 'RestoreMultiRanges'

From d562db2f0e8f1902b6d2765faccf1ba3fc50cf7b Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyu_zhou@apple.com>
Date: Fri, 9 Jun 2023 09:25:12 -0700
Subject: [PATCH 169/468] Add getlocation and getall fdbcli debug commands
 (#10410)

getlocation: returns the SS list for a key
getall: returns both the SS list and values on the SS for a key
---
 fdbcli/CMakeLists.txt          |  1 +
 fdbcli/DebugCommands.actor.cpp | 95 ++++++++++++++++++++++++++++++++++
 fdbcli/fdbcli.actor.cpp        | 19 ++++++-
 fdbcli/fdbcli.actor.h          |  4 ++
 fdbclient/NativeAPI.actor.h    | 14 ++++-
 5 files changed, 130 insertions(+), 3 deletions(-)
 create mode 100644 fdbcli/DebugCommands.actor.cpp

diff --git a/fdbcli/CMakeLists.txt b/fdbcli/CMakeLists.txt
index 00d37067e01..99fa7d33516 100644
--- a/fdbcli/CMakeLists.txt
+++ b/fdbcli/CMakeLists.txt
@@ -8,6 +8,7 @@ set(FDBCLI_SRCS
   ConsistencyCheckCommand.actor.cpp
   CoordinatorsCommand.actor.cpp
   DataDistributionCommand.actor.cpp
+  DebugCommands.actor.cpp
   ExcludeCommand.actor.cpp
   ExpensiveDataCheckCommand.actor.cpp
   FileConfigureCommand.actor.cpp
diff --git a/fdbcli/DebugCommands.actor.cpp b/fdbcli/DebugCommands.actor.cpp
new file mode 100644
index 00000000000..71dc09d9e1a
--- /dev/null
+++ b/fdbcli/DebugCommands.actor.cpp
@@ -0,0 +1,95 @@
+/*
+ * DebugCommands.actor.cpp
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2023 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "fdbcli/fdbcli.actor.h"
+
+#include "fdbclient/NativeAPI.actor.h"
+
+#include "flow/actorcompiler.h" // This must be the last #include.
+
+namespace fdb_cli {
+
+// The command is used to get all storage server addresses for a given key.
+ACTOR Future<bool> getLocationCommandActor(Database cx, std::vector<StringRef> tokens, Version version) {
+	if (tokens.size() != 2) {
+		printf("getlocation <KEY>\n"
+		       "fetch the storage server address for a given key.\n"
+		       "Displays the addresses of storage servers, or `not found' if location is not found.");
+		return false;
+	}
+
+	KeyRangeLocationInfo loc = wait(getKeyLocation_internal(
+	    cx, {}, tokens[1], SpanID(), Optional<UID>(), UseProvisionalProxies::False, Reverse::False, version));
+
+	if (loc.locations) {
+		printf("version is %ld\n", version);
+		printf("`%s' is at:\n", printable(tokens[1]).c_str());
+		auto locations = loc.locations->locations();
+		for (int i = 0; locations && i < locations->size(); i++) {
+			printf("  %s\n", locations->getInterface(i).address().toString().c_str());
+		}
+	} else {
+		printf("`%s': location not found\n", printable(tokens[1]).c_str());
+	}
+	return true;
+}
+// hidden commands, no help text for now
+CommandFactory getLocationCommandFactory("getlocation");
+
+// The command is used to get values from all storage servers that have the given key.
+ACTOR Future<bool> getallCommandActor(Database cx, std::vector<StringRef> tokens, Version version) {
+	if (tokens.size() != 2) {
+		printf("getall <KEY>\n"
+		       "fetch values from all storage servers that have the given key.\n"
+		       "Displays the value and the addresses of storage servers, or `not found' if key is not found.");
+		return false;
+	}
+
+	KeyRangeLocationInfo loc = wait(getKeyLocation_internal(
+	    cx, {}, tokens[1], SpanID(), Optional<UID>(), UseProvisionalProxies::False, Reverse::False, version));
+
+	if (loc.locations) {
+		printf("version is %ld\n", version);
+		printf("`%s' is at:\n", printable(tokens[1]).c_str());
+		state Reference<LocationInfo::Locations> locations = loc.locations->locations();
+		state std::vector<Future<GetValueReply>> replies;
+		for (int i = 0; locations && i < locations->size(); i++) {
+			GetValueRequest req({}, {}, tokens[1], version, {}, {}, {});
+			replies.push_back(locations->get(i, &StorageServerInterface::getValue).getReply(req));
+		}
+		wait(waitForAll(replies));
+		for (int i = 0; i < replies.size(); i++) {
+			std::string ssi = locations->getInterface(i).address().toString();
+			if (replies[i].isError()) {
+				fprintf(stderr, "ERROR: %s %s\n", ssi.c_str(), replies[i].getError().what());
+			} else {
+				Optional<Value> v = replies[i].get().value;
+				printf(" %s %s\n", ssi.c_str(), v.present() ? printable(v.get()).c_str() : "(not found)");
+			}
+		}
+	} else {
+		printf("`%s': location not found\n", printable(tokens[1]).c_str());
+	}
+	return true;
+}
+// hidden commands, no help text for now
+CommandFactory getallCommandFactory("getall");
+
+} // namespace fdb_cli
\ No newline at end of file
diff --git a/fdbcli/fdbcli.actor.cpp b/fdbcli/fdbcli.actor.cpp
index f918307cc7f..a1fafc2f214 100644
--- a/fdbcli/fdbcli.actor.cpp
+++ b/fdbcli/fdbcli.actor.cpp
@@ -1297,7 +1297,6 @@ ACTOR Future<int> cli(CLIOptions opt, LineNoise* plinenoise) {
 	}
 
 	state bool is_error = false;
-
 	state Future<Void> warn;
 	loop {
 		if (warn.isValid())
@@ -1664,6 +1663,24 @@ ACTOR Future<int> cli(CLIOptions opt, LineNoise* plinenoise) {
 					continue;
 				}
 
+				if (tokencmp(tokens[0], "getlocation")) {
+					Version _v = wait(makeInterruptable(
+					    safeThreadFutureToFuture(getTransaction(db, tenant, tr, options, intrans)->getReadVersion())));
+					bool _result = wait(makeInterruptable(getLocationCommandActor(localDb, tokens, _v)));
+					if (!_result)
+						is_error = true;
+					continue;
+				}
+
+				if (tokencmp(tokens[0], "getall")) {
+					Version _v = wait(makeInterruptable(
+					    safeThreadFutureToFuture(getTransaction(db, tenant, tr, options, intrans)->getReadVersion())));
+					bool _result = wait(makeInterruptable(getallCommandActor(localDb, tokens, _v)));
+					if (!_result)
+						is_error = true;
+					continue;
+				}
+
 				if (tokencmp(tokens[0], "getversion")) {
 					if (tokens.size() != 1) {
 						printUsage(tokens[0]);
diff --git a/fdbcli/fdbcli.actor.h b/fdbcli/fdbcli.actor.h
index 81ef4aafe69..eef4ff021e5 100644
--- a/fdbcli/fdbcli.actor.h
+++ b/fdbcli/fdbcli.actor.h
@@ -215,6 +215,10 @@ ACTOR Future<bool> versionEpochCommandActor(Reference<IDatabase> db, Database cx
 // targetversion command
 ACTOR Future<bool> targetVersionCommandActor(Reference<IDatabase> db, std::vector<StringRef> tokens);
 
+// debug commands: getlocation, getall
+ACTOR Future<bool> getLocationCommandActor(Database cx, std::vector<StringRef> tokens, Version version);
+ACTOR Future<bool> getallCommandActor(Database cx, std::vector<StringRef> tokens, Version version);
+
 } // namespace fdb_cli
 
 #include "flow/unactorcompiler.h"
diff --git a/fdbclient/NativeAPI.actor.h b/fdbclient/NativeAPI.actor.h
index 988e64aaf18..f98b5c10e95 100644
--- a/fdbclient/NativeAPI.actor.h
+++ b/fdbclient/NativeAPI.actor.h
@@ -19,8 +19,6 @@
  */
 
 #pragma once
-#include "flow/IRandom.h"
-#include "flow/Tracing.h"
 #if defined(NO_INTELLISENSE) && !defined(FDBCLIENT_NATIVEAPI_ACTOR_G_H)
 #define FDBCLIENT_NATIVEAPI_ACTOR_G_H
 #include "fdbclient/NativeAPI.actor.g.h"
@@ -30,6 +28,8 @@
 #include "flow/BooleanParam.h"
 #include "flow/flow.h"
 #include "flow/TDMetric.actor.h"
+#include "flow/IRandom.h"
+#include "flow/Tracing.h"
 #include "fdbclient/FDBTypes.h"
 #include "fdbclient/CommitProxyInterface.h"
 #include "fdbclient/ClientBooleanParams.h"
@@ -553,5 +553,15 @@ int64_t getMaxWriteKeySize(KeyRef const& key, bool hasRawAccess);
 // Returns the maximum legal size of a key that can be cleared. Keys larger than this will be assumed not to exist.
 int64_t getMaxClearKeySize(KeyRef const& key);
 
+struct KeyRangeLocationInfo;
+ACTOR Future<KeyRangeLocationInfo> getKeyLocation_internal(Database cx,
+                                                           Optional<TenantName> tenant,
+                                                           Key key,
+                                                           SpanID spanID,
+                                                           Optional<UID> debugID,
+                                                           UseProvisionalProxies useProvisionalProxies,
+                                                           Reverse isBackward,
+                                                           Version version);
+
 #include "flow/unactorcompiler.h"
 #endif

From 445fb21a46fcaeb73a0e31291e7dfa4cdd800c56 Mon Sep 17 00:00:00 2001
From: neethuhaneesha <nbingi@apple.com>
Date: Wed, 14 Jun 2023 21:39:56 -0700
Subject: [PATCH 170/468] Rocksdb max auto readahead size knob change to 64k
 (#10450)

---
 fdbclient/ServerKnobs.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/fdbclient/ServerKnobs.cpp b/fdbclient/ServerKnobs.cpp
index 73315a78d04..a0ab6bb36b1 100644
--- a/fdbclient/ServerKnobs.cpp
+++ b/fdbclient/ServerKnobs.cpp
@@ -380,7 +380,7 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi
 	init( ROCKSDB_MEMTABLE_PREFIX_BLOOM_SIZE_RATIO,              0.1 );
 	init( ROCKSDB_BLOOM_BITS_PER_KEY,                             10 );
 	init( ROCKSDB_BLOOM_WHOLE_KEY_FILTERING,                   false );
-	init( ROCKSDB_MAX_AUTO_READAHEAD_SIZE,                         0 );
+	init( ROCKSDB_MAX_AUTO_READAHEAD_SIZE,                     65536 );
 	// If rocksdb block cache size is 0, the default 8MB is used.
 	int64_t blockCacheSize = isSimulated ? 16 * 1024 * 1024 : 2147483648 /* 2GB */;
 	init( ROCKSDB_BLOCK_CACHE_SIZE,                   blockCacheSize );

From a53e8faa430c05b079e81794b8a3620b0b420467 Mon Sep 17 00:00:00 2001
From: neethuhaneesha <nbingi@apple.com>
Date: Mon, 26 Jun 2023 14:08:41 -0700
Subject: [PATCH 171/468] Avoiding read timeout checks for eager reads(write
 path) and system keys. (#10500)

---
 fdbserver/KeyValueStoreRocksDB.actor.cpp | 58 ++++++++++++++----------
 1 file changed, 33 insertions(+), 25 deletions(-)

diff --git a/fdbserver/KeyValueStoreRocksDB.actor.cpp b/fdbserver/KeyValueStoreRocksDB.actor.cpp
index f498a036e24..a43c268c905 100644
--- a/fdbserver/KeyValueStoreRocksDB.actor.cpp
+++ b/fdbserver/KeyValueStoreRocksDB.actor.cpp
@@ -1581,12 +1581,13 @@ struct RocksDBKeyValueStore : IKeyValueStore {
 
 		struct ReadValueAction : TypedAction<Reader, ReadValueAction> {
 			Key key;
+			ReadType type;
 			Optional<UID> debugID;
 			double startTime;
 			bool getHistograms;
 			ThreadReturnPromise<Optional<Value>> result;
-			ReadValueAction(KeyRef key, Optional<UID> debugID)
-			  : key(key), debugID(debugID), startTime(timer_monotonic()),
+			ReadValueAction(KeyRef key, ReadType type, Optional<UID> debugID)
+			  : key(key), type(type), debugID(debugID), startTime(timer_monotonic()),
 			    getHistograms(
 			        (deterministicRandom()->random01() < SERVER_KNOBS->ROCKSDB_HISTOGRAMS_SAMPLE_RATE) ? true : false),
 			    result(static_cast<TaskPriority>(SERVER_KNOBS->ROCKSDB_THREAD_PROMISE_PRIORITY)) {}
@@ -1611,7 +1612,7 @@ struct RocksDBKeyValueStore : IKeyValueStore {
 				traceBatch = { TraceBatch{} };
 				traceBatch.get().addEvent("GetValueDebug", a.debugID.get().first(), "Reader.Before");
 			}
-			if (readBeginTime - a.startTime > readValueTimeout) {
+			if (shouldThrottle(a.type, a.key) && readBeginTime - a.startTime > readValueTimeout) {
 				TraceEvent(SevWarn, "KVSTimeout", id)
 				    .detail("Error", "Read value request timedout")
 				    .detail("Method", "ReadValueAction")
@@ -1622,10 +1623,12 @@ struct RocksDBKeyValueStore : IKeyValueStore {
 
 			rocksdb::PinnableSlice value;
 			auto options = getReadOptions();
-			uint64_t deadlineMircos =
-			    db->GetEnv()->NowMicros() + (readValueTimeout - (readBeginTime - a.startTime)) * 1000000;
-			std::chrono::seconds deadlineSeconds(deadlineMircos / 1000000);
-			options.deadline = std::chrono::duration_cast<std::chrono::microseconds>(deadlineSeconds);
+			if (shouldThrottle(a.type, a.key)) {
+				uint64_t deadlineMircos =
+				    db->GetEnv()->NowMicros() + (readValueTimeout - (readBeginTime - a.startTime)) * 1000000;
+				std::chrono::seconds deadlineSeconds(deadlineMircos / 1000000);
+				options.deadline = std::chrono::duration_cast<std::chrono::microseconds>(deadlineSeconds);
+			}
 
 			double dbGetBeginTime = a.getHistograms ? timer_monotonic() : 0;
 			auto s = db->Get(options, cf, toSlice(a.key), &value);
@@ -1670,12 +1673,13 @@ struct RocksDBKeyValueStore : IKeyValueStore {
 		struct ReadValuePrefixAction : TypedAction<Reader, ReadValuePrefixAction> {
 			Key key;
 			int maxLength;
+			ReadType type;
 			Optional<UID> debugID;
 			double startTime;
 			bool getHistograms;
 			ThreadReturnPromise<Optional<Value>> result;
-			ReadValuePrefixAction(Key key, int maxLength, Optional<UID> debugID)
-			  : key(key), maxLength(maxLength), debugID(debugID), startTime(timer_monotonic()),
+			ReadValuePrefixAction(Key key, int maxLength, ReadType type, Optional<UID> debugID)
+			  : key(key), maxLength(maxLength), type(type), debugID(debugID), startTime(timer_monotonic()),
 			    getHistograms(
 			        (deterministicRandom()->random01() < SERVER_KNOBS->ROCKSDB_HISTOGRAMS_SAMPLE_RATE) ? true : false),
 			    result(static_cast<TaskPriority>(SERVER_KNOBS->ROCKSDB_THREAD_PROMISE_PRIORITY)) {}
@@ -1701,7 +1705,7 @@ struct RocksDBKeyValueStore : IKeyValueStore {
 				                          a.debugID.get().first(),
 				                          "Reader.Before"); //.detail("TaskID", g_network->getCurrentTask());
 			}
-			if (readBeginTime - a.startTime > readValuePrefixTimeout) {
+			if (shouldThrottle(a.type, a.key) && readBeginTime - a.startTime > readValuePrefixTimeout) {
 				TraceEvent(SevWarn, "KVSTimeout", id)
 				    .detail("Error", "Read value prefix request timedout")
 				    .detail("Method", "ReadValuePrefixAction")
@@ -1712,10 +1716,12 @@ struct RocksDBKeyValueStore : IKeyValueStore {
 
 			rocksdb::PinnableSlice value;
 			auto options = getReadOptions();
-			uint64_t deadlineMircos =
-			    db->GetEnv()->NowMicros() + (readValuePrefixTimeout - (readBeginTime - a.startTime)) * 1000000;
-			std::chrono::seconds deadlineSeconds(deadlineMircos / 1000000);
-			options.deadline = std::chrono::duration_cast<std::chrono::microseconds>(deadlineSeconds);
+			if (shouldThrottle(a.type, a.key)) {
+				uint64_t deadlineMircos =
+				    db->GetEnv()->NowMicros() + (readValuePrefixTimeout - (readBeginTime - a.startTime)) * 1000000;
+				std::chrono::seconds deadlineSeconds(deadlineMircos / 1000000);
+				options.deadline = std::chrono::duration_cast<std::chrono::microseconds>(deadlineSeconds);
+			}
 
 			double dbGetBeginTime = a.getHistograms ? timer_monotonic() : 0;
 			auto s = db->Get(options, cf, toSlice(a.key), &value);
@@ -1756,12 +1762,14 @@ struct RocksDBKeyValueStore : IKeyValueStore {
 		struct ReadRangeAction : TypedAction<Reader, ReadRangeAction>, FastAllocated<ReadRangeAction> {
 			KeyRange keys;
 			int rowLimit, byteLimit;
+			ReadType type;
 			double startTime;
 			bool getHistograms;
 			ThreadReturnPromise<RangeResult> result;
 			Counters& counters;
-			ReadRangeAction(KeyRange keys, int rowLimit, int byteLimit, Counters& counters)
-			  : keys(keys), rowLimit(rowLimit), byteLimit(byteLimit), startTime(timer_monotonic()), counters(counters),
+			ReadRangeAction(KeyRange keys, int rowLimit, int byteLimit, ReadType type, Counters& counters)
+			  : keys(keys), rowLimit(rowLimit), byteLimit(byteLimit), type(type), startTime(timer_monotonic()),
+			    counters(counters),
 			    getHistograms(deterministicRandom()->random01() < SERVER_KNOBS->ROCKSDB_HISTOGRAMS_SAMPLE_RATE),
 			    result(static_cast<TaskPriority>(SERVER_KNOBS->ROCKSDB_THREAD_PROMISE_PRIORITY)) {}
 			double getTimeEstimate() const override { return SERVER_KNOBS->READ_RANGE_TIME_ESTIMATE; }
@@ -1780,7 +1788,7 @@ struct RocksDBKeyValueStore : IKeyValueStore {
 				    threadIndex, ROCKSDB_READRANGE_QUEUEWAIT_HISTOGRAM.toString(), readBeginTime - a.startTime));
 			}
 			sharedState->readQueueLatency[threadIndex]->addMeasurement(readBeginTime - a.startTime);
-			if (readBeginTime - a.startTime > readRangeTimeout) {
+			if (shouldThrottle(a.type, a.keys.begin) && readBeginTime - a.startTime > readRangeTimeout) {
 				TraceEvent(SevWarn, "KVSTimeout", id)
 				    .detail("Error", "Read range request timedout")
 				    .detail("Method", "ReadRangeAction")
@@ -1813,7 +1821,7 @@ struct RocksDBKeyValueStore : IKeyValueStore {
 					if (result.size() >= a.rowLimit || accumulatedBytes >= a.byteLimit) {
 						break;
 					}
-					if (timer_monotonic() - a.startTime > readRangeTimeout) {
+					if (shouldThrottle(a.type, a.keys.begin) && timer_monotonic() - a.startTime > readRangeTimeout) {
 						TraceEvent(SevWarn, "KVSTimeout", id)
 						    .detail("Error", "Read range request timedout")
 						    .detail("Method", "ReadRangeAction")
@@ -1846,7 +1854,7 @@ struct RocksDBKeyValueStore : IKeyValueStore {
 					if (result.size() >= -a.rowLimit || accumulatedBytes >= a.byteLimit) {
 						break;
 					}
-					if (timer_monotonic() - a.startTime > readRangeTimeout) {
+					if (shouldThrottle(a.type, a.keys.begin) && timer_monotonic() - a.startTime > readRangeTimeout) {
 						TraceEvent(SevWarn, "KVSTimeout", id)
 						    .detail("Error", "Read range request timedout")
 						    .detail("Method", "ReadRangeAction")
@@ -2223,7 +2231,7 @@ struct RocksDBKeyValueStore : IKeyValueStore {
 
 	Future<Optional<Value>> readValue(KeyRef key, IKeyValueStore::ReadType type, Optional<UID> debugID) override {
 		if (!shouldThrottle(type, key)) {
-			auto a = new Reader::ReadValueAction(key, debugID);
+			auto a = new Reader::ReadValueAction(key, type, debugID);
 			auto res = a->result.getFuture();
 			readThreads->post(a);
 			return res;
@@ -2233,7 +2241,7 @@ struct RocksDBKeyValueStore : IKeyValueStore {
 		int maxWaiters = (type == IKeyValueStore::ReadType::FETCH) ? numFetchWaiters : numReadWaiters;
 
 		checkWaiters(semaphore, maxWaiters);
-		auto a = std::make_unique<Reader::ReadValueAction>(key, debugID);
+		auto a = std::make_unique<Reader::ReadValueAction>(key, type, debugID);
 		return read(a.release(), &semaphore, readThreads.getPtr(), &counters.failedToAcquire);
 	}
 
@@ -2242,7 +2250,7 @@ struct RocksDBKeyValueStore : IKeyValueStore {
 	                                        IKeyValueStore::ReadType type,
 	                                        Optional<UID> debugID) override {
 		if (!shouldThrottle(type, key)) {
-			auto a = new Reader::ReadValuePrefixAction(key, maxLength, debugID);
+			auto a = new Reader::ReadValuePrefixAction(key, maxLength, type, debugID);
 			auto res = a->result.getFuture();
 			readThreads->post(a);
 			return res;
@@ -2252,7 +2260,7 @@ struct RocksDBKeyValueStore : IKeyValueStore {
 		int maxWaiters = (type == IKeyValueStore::ReadType::FETCH) ? numFetchWaiters : numReadWaiters;
 
 		checkWaiters(semaphore, maxWaiters);
-		auto a = std::make_unique<Reader::ReadValuePrefixAction>(key, maxLength, debugID);
+		auto a = std::make_unique<Reader::ReadValuePrefixAction>(key, maxLength, type, debugID);
 		return read(a.release(), &semaphore, readThreads.getPtr(), &counters.failedToAcquire);
 	}
 
@@ -2281,7 +2289,7 @@ struct RocksDBKeyValueStore : IKeyValueStore {
 	                              int byteLimit,
 	                              IKeyValueStore::ReadType type) override {
 		if (!shouldThrottle(type, keys.begin)) {
-			auto a = new Reader::ReadRangeAction(keys, rowLimit, byteLimit, counters);
+			auto a = new Reader::ReadRangeAction(keys, rowLimit, byteLimit, type, counters);
 			auto res = a->result.getFuture();
 			readThreads->post(a);
 			return res;
@@ -2291,7 +2299,7 @@ struct RocksDBKeyValueStore : IKeyValueStore {
 		int maxWaiters = (type == IKeyValueStore::ReadType::FETCH) ? numFetchWaiters : numReadWaiters;
 
 		checkWaiters(semaphore, maxWaiters);
-		auto a = std::make_unique<Reader::ReadRangeAction>(keys, rowLimit, byteLimit, counters);
+		auto a = std::make_unique<Reader::ReadRangeAction>(keys, rowLimit, byteLimit, type, counters);
 		return read(a.release(), &semaphore, readThreads.getPtr(), &counters.failedToAcquire);
 	}
 

From 0ac85fa80fbf1e26d9cb17f50b62748d6af93a52 Mon Sep 17 00:00:00 2001
From: Zhe Wu <halfprice@users.noreply.github.com>
Date: Tue, 23 May 2023 14:14:33 -0700
Subject: [PATCH 172/468] Refactor healthMonitor in worker.actor.cpp

---
 fdbserver/worker.actor.cpp | 322 +++++++++++++++++++------------------
 1 file changed, 165 insertions(+), 157 deletions(-)

diff --git a/fdbserver/worker.actor.cpp b/fdbserver/worker.actor.cpp
index 15423e94b3d..57d1eb40354 100644
--- a/fdbserver/worker.actor.cpp
+++ b/fdbserver/worker.actor.cpp
@@ -1016,6 +1016,170 @@ TEST_CASE("/fdbserver/worker/addressIsRemoteLogRouter") {
 
 } // namespace
 
+// Check if the current worker is a transaction worker, and is experiencing degraded or disconnected peers. If so,
+// report degraded and disconnected peers to the cluster controller.
+void doPeerHealthCheck(Reference<AsyncVar<Optional<ClusterControllerFullInterface>> const> ccInterface,
+                       const WorkerInterface& interf,
+                       const LocalityData& locality,
+                       Reference<AsyncVar<ServerDBInfo> const> dbInfo) {
+	const auto& allPeers = FlowTransport::transport().getAllPeers();
+
+	// Check remote log router connectivity only when remote TLogs are recruited and in use.
+	bool checkRemoteLogRouterConnectivity = dbInfo->get().recoveryState == RecoveryState::ALL_LOGS_RECRUITED ||
+	                                        dbInfo->get().recoveryState == RecoveryState::FULLY_RECOVERED;
+	UpdateWorkerHealthRequest req;
+
+	enum WorkerLocation { None, Primary, Satellite, Remote };
+	WorkerLocation workerLocation = None;
+	if (addressesInDbAndPrimaryDc(interf.addresses(), dbInfo)) {
+		workerLocation = Primary;
+	} else if (addressesInDbAndRemoteDc(interf.addresses(), dbInfo)) {
+		workerLocation = Remote;
+	} else if (addressesInDbAndPrimarySatelliteDc(interf.addresses(), dbInfo)) {
+		workerLocation = Satellite;
+	}
+
+	if (workerLocation == None) {
+		// This worker doesn't need to monitor anything if it is in remote satellite.
+		return;
+	}
+
+	for (const auto& [address, peer] : allPeers) {
+		if (peer->connectFailedCount == 0 &&
+		    peer->pingLatencies.getPopulationSize() < SERVER_KNOBS->PEER_LATENCY_CHECK_MIN_POPULATION) {
+			// Ignore peers that don't have enough samples.
+			// TODO(zhewu): Currently, FlowTransport latency monitor clears ping latency samples on a
+			// regular basis, which may affect the measurement count. Currently,
+			// WORKER_HEALTH_MONITOR_INTERVAL is much smaller than the ping clearance interval, so it may be
+			// ok. If this ends to be a problem, we need to consider keep track of last ping latencies
+			// logged.
+			continue;
+		}
+
+		bool degradedPeer = false;
+		bool disconnectedPeer = false;
+
+		// If peer->lastLoggedTime == 0, we just started monitor this peer and haven't logged it once yet.
+		double lastLoggedTime = peer->lastLoggedTime <= 0.0 ? peer->lastConnectTime : peer->lastLoggedTime;
+		if ((workerLocation == Primary && addressInDbAndPrimaryDc(address, dbInfo)) ||
+		    (workerLocation == Remote && addressInDbAndRemoteDc(address, dbInfo))) {
+			// Monitors intra DC latencies between servers that in the primary or remote DC's transaction
+			// systems. Note that currently we are not monitor storage servers, since lagging in storage
+			// servers today already can trigger server exclusion by data distributor.
+
+			if (peer->connectFailedCount >= SERVER_KNOBS->PEER_DEGRADATION_CONNECTION_FAILURE_COUNT) {
+				disconnectedPeer = true;
+			} else if (peer->pingLatencies.percentile(SERVER_KNOBS->PEER_LATENCY_DEGRADATION_PERCENTILE) >
+			               SERVER_KNOBS->PEER_LATENCY_DEGRADATION_THRESHOLD ||
+			           peer->timeoutCount / (double)(peer->pingLatencies.getPopulationSize()) >
+			               SERVER_KNOBS->PEER_TIMEOUT_PERCENTAGE_DEGRADATION_THRESHOLD) {
+				degradedPeer = true;
+			}
+			if (disconnectedPeer || degradedPeer) {
+				TraceEvent("HealthMonitorDetectDegradedPeer")
+				    .detail("Peer", address)
+				    .detail("Elapsed", now() - lastLoggedTime)
+				    .detail("Disconnected", disconnectedPeer)
+				    .detail("MinLatency", peer->pingLatencies.min())
+				    .detail("MaxLatency", peer->pingLatencies.max())
+				    .detail("MeanLatency", peer->pingLatencies.mean())
+				    .detail("MedianLatency", peer->pingLatencies.median())
+				    .detail("CheckedPercentile", SERVER_KNOBS->PEER_LATENCY_DEGRADATION_PERCENTILE)
+				    .detail("CheckedPercentileLatency",
+				            peer->pingLatencies.percentile(SERVER_KNOBS->PEER_LATENCY_DEGRADATION_PERCENTILE))
+				    .detail("PingCount", peer->pingLatencies.getPopulationSize())
+				    .detail("PingTimeoutCount", peer->timeoutCount)
+				    .detail("ConnectionFailureCount", peer->connectFailedCount);
+			}
+		} else if (workerLocation == Primary && addressInDbAndPrimarySatelliteDc(address, dbInfo)) {
+			// Monitors inter DC latencies between servers in primary and primary satellite DC. Note that
+			// TLog workers in primary satellite DC are on the critical path of serving a commit.
+			if (peer->connectFailedCount >= SERVER_KNOBS->PEER_DEGRADATION_CONNECTION_FAILURE_COUNT) {
+				disconnectedPeer = true;
+			} else if (peer->pingLatencies.percentile(SERVER_KNOBS->PEER_LATENCY_DEGRADATION_PERCENTILE_SATELLITE) >
+			               SERVER_KNOBS->PEER_LATENCY_DEGRADATION_THRESHOLD_SATELLITE ||
+			           peer->timeoutCount / (double)(peer->pingLatencies.getPopulationSize()) >
+			               SERVER_KNOBS->PEER_TIMEOUT_PERCENTAGE_DEGRADATION_THRESHOLD) {
+				degradedPeer = true;
+			}
+
+			if (disconnectedPeer || degradedPeer) {
+				TraceEvent("HealthMonitorDetectDegradedPeer")
+				    .detail("Peer", address)
+				    .detail("Satellite", true)
+				    .detail("Elapsed", now() - lastLoggedTime)
+				    .detail("Disconnected", disconnectedPeer)
+				    .detail("MinLatency", peer->pingLatencies.min())
+				    .detail("MaxLatency", peer->pingLatencies.max())
+				    .detail("MeanLatency", peer->pingLatencies.mean())
+				    .detail("MedianLatency", peer->pingLatencies.median())
+				    .detail("CheckedPercentile", SERVER_KNOBS->PEER_LATENCY_DEGRADATION_PERCENTILE_SATELLITE)
+				    .detail("CheckedPercentileLatency",
+				            peer->pingLatencies.percentile(SERVER_KNOBS->PEER_LATENCY_DEGRADATION_PERCENTILE_SATELLITE))
+				    .detail("PingCount", peer->pingLatencies.getPopulationSize())
+				    .detail("PingTimeoutCount", peer->timeoutCount)
+				    .detail("ConnectionFailureCount", peer->connectFailedCount);
+			}
+		} else if (checkRemoteLogRouterConnectivity && (workerLocation == Primary || workerLocation == Satellite) &&
+		           addressIsRemoteLogRouter(address, dbInfo)) {
+			// Monitor remote log router's connectivity to the primary DCs' transaction system. We ignore
+			// latency based degradation between primary region and remote region due to that remote region
+			// may be distant from primary region.
+			if (peer->connectFailedCount >= SERVER_KNOBS->PEER_DEGRADATION_CONNECTION_FAILURE_COUNT) {
+				TraceEvent("HealthMonitorDetectDegradedPeer")
+				    .detail("WorkerLocation", workerLocation)
+				    .detail("Peer", address)
+				    .detail("RemoteLogRouter", true)
+				    .detail("Elapsed", now() - lastLoggedTime)
+				    .detail("Disconnected", true)
+				    .detail("MinLatency", peer->pingLatencies.min())
+				    .detail("MaxLatency", peer->pingLatencies.max())
+				    .detail("MeanLatency", peer->pingLatencies.mean())
+				    .detail("MedianLatency", peer->pingLatencies.median())
+				    .detail("PingCount", peer->pingLatencies.getPopulationSize())
+				    .detail("PingTimeoutCount", peer->timeoutCount)
+				    .detail("ConnectionFailureCount", peer->connectFailedCount);
+				disconnectedPeer = true;
+			}
+		}
+
+		if (disconnectedPeer) {
+			req.disconnectedPeers.push_back(address);
+		} else if (degradedPeer) {
+			req.degradedPeers.push_back(address);
+		}
+	}
+
+	if (SERVER_KNOBS->WORKER_HEALTH_REPORT_RECENT_DESTROYED_PEER) {
+		// When the worker cannot connect to a remote peer, the peer maybe erased from the list returned
+		// from getAllPeers(). Therefore, we also look through all the recent closed peers in the flow
+		// transport's health monitor. Note that all the closed peers stored here are caused by connection
+		// failure, but not normal connection close. Therefore, we report all such peers if they are also
+		// part of the transaction sub system.
+		for (const auto& address : FlowTransport::transport().healthMonitor()->getRecentClosedPeers()) {
+			if (allPeers.find(address) != allPeers.end()) {
+				// We have checked this peer in the above for loop.
+				continue;
+			}
+
+			if ((workerLocation == Primary && addressInDbAndPrimaryDc(address, dbInfo)) ||
+			    (workerLocation == Remote && addressInDbAndRemoteDc(address, dbInfo)) ||
+			    (workerLocation == Primary && addressInDbAndPrimarySatelliteDc(address, dbInfo)) ||
+			    (checkRemoteLogRouterConnectivity && (workerLocation == Primary || workerLocation == Satellite) &&
+			     addressIsRemoteLogRouter(address, dbInfo))) {
+				TraceEvent("HealthMonitorDetectRecentClosedPeer").suppressFor(30).detail("Peer", address);
+				req.disconnectedPeers.push_back(address);
+			}
+		}
+	}
+
+	if (!req.disconnectedPeers.empty() || !req.degradedPeers.empty()) {
+		// Disconnected or degraded peers are reported to the cluster controller.
+		req.address = FlowTransport::transport().getLocalAddress();
+		ccInterface->get().get().updateWorkerHealth.send(req);
+	}
+}
+
 // The actor that actively monitors the health of local and peer servers, and reports anomaly to the cluster controller.
 ACTOR Future<Void> healthMonitor(Reference<AsyncVar<Optional<ClusterControllerFullInterface>> const> ccInterface,
                                  WorkerInterface interf,
@@ -1025,163 +1189,7 @@ ACTOR Future<Void> healthMonitor(Reference<AsyncVar<Optional<ClusterControllerFu
 		Future<Void> nextHealthCheckDelay = Never();
 		if (dbInfo->get().recoveryState >= RecoveryState::ACCEPTING_COMMITS && ccInterface->get().present()) {
 			nextHealthCheckDelay = delay(SERVER_KNOBS->WORKER_HEALTH_MONITOR_INTERVAL);
-			const auto& allPeers = FlowTransport::transport().getAllPeers();
-
-			// Check remote log router connectivity only when remote TLogs are recruited and in use.
-			bool checkRemoteLogRouterConnectivity = dbInfo->get().recoveryState == RecoveryState::ALL_LOGS_RECRUITED ||
-			                                        dbInfo->get().recoveryState == RecoveryState::FULLY_RECOVERED;
-			UpdateWorkerHealthRequest req;
-
-			enum WorkerLocation { None, Primary, Satellite, Remote };
-			WorkerLocation workerLocation = None;
-			if (addressesInDbAndPrimaryDc(interf.addresses(), dbInfo)) {
-				workerLocation = Primary;
-			} else if (addressesInDbAndRemoteDc(interf.addresses(), dbInfo)) {
-				workerLocation = Remote;
-			} else if (addressesInDbAndPrimarySatelliteDc(interf.addresses(), dbInfo)) {
-				workerLocation = Satellite;
-			}
-
-			if (workerLocation != None) {
-				for (const auto& [address, peer] : allPeers) {
-					if (peer->connectFailedCount == 0 &&
-					    peer->pingLatencies.getPopulationSize() < SERVER_KNOBS->PEER_LATENCY_CHECK_MIN_POPULATION) {
-						// Ignore peers that don't have enough samples.
-						// TODO(zhewu): Currently, FlowTransport latency monitor clears ping latency samples on a
-						// regular basis, which may affect the measurement count. Currently,
-						// WORKER_HEALTH_MONITOR_INTERVAL is much smaller than the ping clearance interval, so it may be
-						// ok. If this ends to be a problem, we need to consider keep track of last ping latencies
-						// logged.
-						continue;
-					}
-					bool degradedPeer = false;
-					bool disconnectedPeer = false;
-
-					// If peer->lastLoggedTime == 0, we just started monitor this peer and haven't logged it once yet.
-					double lastLoggedTime = peer->lastLoggedTime <= 0.0 ? peer->lastConnectTime : peer->lastLoggedTime;
-					if ((workerLocation == Primary && addressInDbAndPrimaryDc(address, dbInfo)) ||
-					    (workerLocation == Remote && addressInDbAndRemoteDc(address, dbInfo))) {
-						// Monitors intra DC latencies between servers that in the primary or remote DC's transaction
-						// systems. Note that currently we are not monitor storage servers, since lagging in storage
-						// servers today already can trigger server exclusion by data distributor.
-
-						if (peer->connectFailedCount >= SERVER_KNOBS->PEER_DEGRADATION_CONNECTION_FAILURE_COUNT) {
-							disconnectedPeer = true;
-						} else if (peer->pingLatencies.percentile(SERVER_KNOBS->PEER_LATENCY_DEGRADATION_PERCENTILE) >
-						               SERVER_KNOBS->PEER_LATENCY_DEGRADATION_THRESHOLD ||
-						           peer->timeoutCount / (double)(peer->pingLatencies.getPopulationSize()) >
-						               SERVER_KNOBS->PEER_TIMEOUT_PERCENTAGE_DEGRADATION_THRESHOLD) {
-							degradedPeer = true;
-						}
-						if (disconnectedPeer || degradedPeer) {
-							TraceEvent("HealthMonitorDetectDegradedPeer")
-							    .detail("Peer", address)
-							    .detail("Elapsed", now() - lastLoggedTime)
-							    .detail("Disconnected", disconnectedPeer)
-							    .detail("MinLatency", peer->pingLatencies.min())
-							    .detail("MaxLatency", peer->pingLatencies.max())
-							    .detail("MeanLatency", peer->pingLatencies.mean())
-							    .detail("MedianLatency", peer->pingLatencies.median())
-							    .detail("CheckedPercentile", SERVER_KNOBS->PEER_LATENCY_DEGRADATION_PERCENTILE)
-							    .detail(
-							        "CheckedPercentileLatency",
-							        peer->pingLatencies.percentile(SERVER_KNOBS->PEER_LATENCY_DEGRADATION_PERCENTILE))
-							    .detail("PingCount", peer->pingLatencies.getPopulationSize())
-							    .detail("PingTimeoutCount", peer->timeoutCount)
-							    .detail("ConnectionFailureCount", peer->connectFailedCount);
-						}
-					} else if (workerLocation == Primary && addressInDbAndPrimarySatelliteDc(address, dbInfo)) {
-						// Monitors inter DC latencies between servers in primary and primary satellite DC. Note that
-						// TLog workers in primary satellite DC are on the critical path of serving a commit.
-						if (peer->connectFailedCount >= SERVER_KNOBS->PEER_DEGRADATION_CONNECTION_FAILURE_COUNT) {
-							disconnectedPeer = true;
-						} else if (peer->pingLatencies.percentile(
-						               SERVER_KNOBS->PEER_LATENCY_DEGRADATION_PERCENTILE_SATELLITE) >
-						               SERVER_KNOBS->PEER_LATENCY_DEGRADATION_THRESHOLD_SATELLITE ||
-						           peer->timeoutCount / (double)(peer->pingLatencies.getPopulationSize()) >
-						               SERVER_KNOBS->PEER_TIMEOUT_PERCENTAGE_DEGRADATION_THRESHOLD) {
-							degradedPeer = true;
-						}
-
-						if (disconnectedPeer || degradedPeer) {
-							TraceEvent("HealthMonitorDetectDegradedPeer")
-							    .detail("Peer", address)
-							    .detail("Satellite", true)
-							    .detail("Elapsed", now() - lastLoggedTime)
-							    .detail("Disconnected", disconnectedPeer)
-							    .detail("MinLatency", peer->pingLatencies.min())
-							    .detail("MaxLatency", peer->pingLatencies.max())
-							    .detail("MeanLatency", peer->pingLatencies.mean())
-							    .detail("MedianLatency", peer->pingLatencies.median())
-							    .detail("CheckedPercentile",
-							            SERVER_KNOBS->PEER_LATENCY_DEGRADATION_PERCENTILE_SATELLITE)
-							    .detail("CheckedPercentileLatency",
-							            peer->pingLatencies.percentile(
-							                SERVER_KNOBS->PEER_LATENCY_DEGRADATION_PERCENTILE_SATELLITE))
-							    .detail("PingCount", peer->pingLatencies.getPopulationSize())
-							    .detail("PingTimeoutCount", peer->timeoutCount)
-							    .detail("ConnectionFailureCount", peer->connectFailedCount);
-						}
-					} else if (checkRemoteLogRouterConnectivity &&
-					           (workerLocation == Primary || workerLocation == Satellite) &&
-					           addressIsRemoteLogRouter(address, dbInfo)) {
-						// Monitor remote log router's connectivity to the primary DCs' transaction system. We ignore
-						// latency based degradation between primary region and remote region due to that remote region
-						// may be distant from primary region.
-						if (peer->connectFailedCount >= SERVER_KNOBS->PEER_DEGRADATION_CONNECTION_FAILURE_COUNT) {
-							TraceEvent("HealthMonitorDetectDegradedPeer")
-							    .detail("WorkerLocation", workerLocation)
-							    .detail("Peer", address)
-							    .detail("RemoteLogRouter", true)
-							    .detail("Elapsed", now() - lastLoggedTime)
-							    .detail("Disconnected", true)
-							    .detail("MinLatency", peer->pingLatencies.min())
-							    .detail("MaxLatency", peer->pingLatencies.max())
-							    .detail("MeanLatency", peer->pingLatencies.mean())
-							    .detail("MedianLatency", peer->pingLatencies.median())
-							    .detail("PingCount", peer->pingLatencies.getPopulationSize())
-							    .detail("PingTimeoutCount", peer->timeoutCount)
-							    .detail("ConnectionFailureCount", peer->connectFailedCount);
-							disconnectedPeer = true;
-						}
-					}
-
-					if (disconnectedPeer) {
-						req.disconnectedPeers.push_back(address);
-					} else if (degradedPeer) {
-						req.degradedPeers.push_back(address);
-					}
-				}
-
-				if (SERVER_KNOBS->WORKER_HEALTH_REPORT_RECENT_DESTROYED_PEER) {
-					// When the worker cannot connect to a remote peer, the peer maybe erased from the list returned
-					// from getAllPeers(). Therefore, we also look through all the recent closed peers in the flow
-					// transport's health monitor. Note that all the closed peers stored here are caused by connection
-					// failure, but not normal connection close. Therefore, we report all such peers if they are also
-					// part of the transaction sub system.
-					for (const auto& address : FlowTransport::transport().healthMonitor()->getRecentClosedPeers()) {
-						if (allPeers.find(address) != allPeers.end()) {
-							// We have checked this peer in the above for loop.
-							continue;
-						}
-
-						if ((workerLocation == Primary && addressInDbAndPrimaryDc(address, dbInfo)) ||
-						    (workerLocation == Remote && addressInDbAndRemoteDc(address, dbInfo)) ||
-						    (workerLocation == Primary && addressInDbAndPrimarySatelliteDc(address, dbInfo)) ||
-						    (checkRemoteLogRouterConnectivity &&
-						     (workerLocation == Primary || workerLocation == Satellite) &&
-						     addressIsRemoteLogRouter(address, dbInfo))) {
-							TraceEvent("HealthMonitorDetectRecentClosedPeer").suppressFor(30).detail("Peer", address);
-							req.disconnectedPeers.push_back(address);
-						}
-					}
-				}
-			}
-
-			if (!req.disconnectedPeers.empty() || !req.degradedPeers.empty()) {
-				req.address = FlowTransport::transport().getLocalAddress();
-				ccInterface->get().get().updateWorkerHealth.send(req);
-			}
+			doPeerHealthCheck(ccInterface, interf, locality, dbInfo);
 		}
 		choose {
 			when(wait(nextHealthCheckDelay)) {}

From 1e4ecf33db3b242926b79452c330e6e3116570fd Mon Sep 17 00:00:00 2001
From: Zhe Wu <halfprice@users.noreply.github.com>
Date: Tue, 20 Jun 2023 12:32:29 -0700
Subject: [PATCH 173/468] Worker health monitor keeps track of recovered peer

---
 fdbclient/ServerKnobs.cpp         |  2 +-
 fdbserver/WorkerInterface.actor.h |  3 +-
 fdbserver/worker.actor.cpp        | 99 +++++++++++++++++++++++++------
 3 files changed, 84 insertions(+), 20 deletions(-)

diff --git a/fdbclient/ServerKnobs.cpp b/fdbclient/ServerKnobs.cpp
index a0ab6bb36b1..9f8145de2f3 100644
--- a/fdbclient/ServerKnobs.cpp
+++ b/fdbclient/ServerKnobs.cpp
@@ -788,7 +788,7 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi
 	init( MIN_DELAY_CC_WORST_FIT_CANDIDACY_SECONDS,             10.0 );
 	init( MAX_DELAY_CC_WORST_FIT_CANDIDACY_SECONDS,             30.0 );
 	init( DBINFO_FAILED_DELAY,                                   1.0 );
-	init( ENABLE_WORKER_HEALTH_MONITOR,                        false );
+	init( ENABLE_WORKER_HEALTH_MONITOR,                        false ); if ( randomize && BUGGIFY ) ENABLE_WORKER_HEALTH_MONITOR = true;
 	init( WORKER_HEALTH_MONITOR_INTERVAL,                       60.0 );
 	init( PEER_LATENCY_CHECK_MIN_POPULATION,                      30 );
 	init( PEER_LATENCY_DEGRADATION_PERCENTILE,                  0.50 );
diff --git a/fdbserver/WorkerInterface.actor.h b/fdbserver/WorkerInterface.actor.h
index f5cb0ac414b..10d930542b2 100644
--- a/fdbserver/WorkerInterface.actor.h
+++ b/fdbserver/WorkerInterface.actor.h
@@ -499,13 +499,14 @@ struct UpdateWorkerHealthRequest {
 	NetworkAddress address;
 	std::vector<NetworkAddress> degradedPeers;
 	std::vector<NetworkAddress> disconnectedPeers;
+	std::vector<NetworkAddress> recoveredPeers;
 
 	template <class Ar>
 	void serialize(Ar& ar) {
 		if constexpr (!is_fb_function<Ar>) {
 			ASSERT(ar.protocolVersion().isValid());
 		}
-		serializer(ar, address, degradedPeers, disconnectedPeers);
+		serializer(ar, address, degradedPeers, disconnectedPeers, recoveredPeers);
 	}
 };
 
diff --git a/fdbserver/worker.actor.cpp b/fdbserver/worker.actor.cpp
index 57d1eb40354..c04fd7d6311 100644
--- a/fdbserver/worker.actor.cpp
+++ b/fdbserver/worker.actor.cpp
@@ -1016,12 +1016,45 @@ TEST_CASE("/fdbserver/worker/addressIsRemoteLogRouter") {
 
 } // namespace
 
+// Returns true if the `peer` has enough data samples that should be checked by the health monitor.
+bool shouldCheckPeer(Reference<Peer> peer) {
+	if (peer->connectFailedCount != 0) {
+		return true;
+	}
+
+	if (peer->pingLatencies.getPopulationSize() >= SERVER_KNOBS->PEER_LATENCY_CHECK_MIN_POPULATION) {
+		// Ignore peers that don't have enough samples.
+		// TODO(zhewu): Currently, FlowTransport latency monitor clears ping latency samples on a
+		// regular basis, which may affect the measurement count. Currently,
+		// WORKER_HEALTH_MONITOR_INTERVAL is much smaller than the ping clearance interval, so it may be
+		// ok. If this ends to be a problem, we need to consider keep track of last ping latencies
+		// logged.
+		return true;
+	}
+
+	return false;
+}
+
+// Returns true if `address` is a degraded/disconnected peer in `lastReq` sent to CC.
+bool isDegradedPeer(const UpdateWorkerHealthRequest& lastReq, const NetworkAddress& address) {
+	if (std::find(lastReq.degradedPeers.begin(), lastReq.degradedPeers.end(), address) != lastReq.degradedPeers.end()) {
+		return true;
+	}
+
+	if (std::find(lastReq.disconnectedPeers.begin(), lastReq.disconnectedPeers.end(), address) !=
+	    lastReq.disconnectedPeers.end()) {
+		return true;
+	}
+
+	return false;
+}
+
 // Check if the current worker is a transaction worker, and is experiencing degraded or disconnected peers. If so,
 // report degraded and disconnected peers to the cluster controller.
-void doPeerHealthCheck(Reference<AsyncVar<Optional<ClusterControllerFullInterface>> const> ccInterface,
-                       const WorkerInterface& interf,
-                       const LocalityData& locality,
-                       Reference<AsyncVar<ServerDBInfo> const> dbInfo) {
+UpdateWorkerHealthRequest doPeerHealthCheck(const WorkerInterface& interf,
+                                            const LocalityData& locality,
+                                            Reference<AsyncVar<ServerDBInfo> const> dbInfo,
+                                            const UpdateWorkerHealthRequest& lastReq) {
 	const auto& allPeers = FlowTransport::transport().getAllPeers();
 
 	// Check remote log router connectivity only when remote TLogs are recruited and in use.
@@ -1041,18 +1074,12 @@ void doPeerHealthCheck(Reference<AsyncVar<Optional<ClusterControllerFullInterfac
 
 	if (workerLocation == None) {
 		// This worker doesn't need to monitor anything if it is in remote satellite.
-		return;
+		return req;
 	}
 
 	for (const auto& [address, peer] : allPeers) {
-		if (peer->connectFailedCount == 0 &&
-		    peer->pingLatencies.getPopulationSize() < SERVER_KNOBS->PEER_LATENCY_CHECK_MIN_POPULATION) {
-			// Ignore peers that don't have enough samples.
-			// TODO(zhewu): Currently, FlowTransport latency monitor clears ping latency samples on a
-			// regular basis, which may affect the measurement count. Currently,
-			// WORKER_HEALTH_MONITOR_INTERVAL is much smaller than the ping clearance interval, so it may be
-			// ok. If this ends to be a problem, we need to consider keep track of last ping latencies
-			// logged.
+		if (!shouldCheckPeer(peer)) {
+			req.recoveredPeers.push_back(address);
 			continue;
 		}
 
@@ -1147,6 +1174,9 @@ void doPeerHealthCheck(Reference<AsyncVar<Optional<ClusterControllerFullInterfac
 			req.disconnectedPeers.push_back(address);
 		} else if (degradedPeer) {
 			req.degradedPeers.push_back(address);
+		} else if (isDegradedPeer(lastReq, address)) {
+			TraceEvent("HealthMonitorDetectRecoveredPeer").detail("Peer", address);
+			req.recoveredPeers.push_back(address);
 		}
 	}
 
@@ -1173,11 +1203,25 @@ void doPeerHealthCheck(Reference<AsyncVar<Optional<ClusterControllerFullInterfac
 		}
 	}
 
-	if (!req.disconnectedPeers.empty() || !req.degradedPeers.empty()) {
-		// Disconnected or degraded peers are reported to the cluster controller.
-		req.address = FlowTransport::transport().getLocalAddress();
-		ccInterface->get().get().updateWorkerHealth.send(req);
+	if (g_network->isSimulated()) {
+		// Invariant check in simulation: for any peers that shouldn't be checked, we won't include it in the
+		// UpdateWorkerHealthRequest sent to CC.
+		for (const auto& [address, peer] : allPeers) {
+			if (!shouldCheckPeer(peer)) {
+				for (const auto& disconnectedPeer : req.disconnectedPeers) {
+					ASSERT(address != disconnectedPeer);
+				}
+				for (const auto& degradedPeer : req.degradedPeers) {
+					ASSERT(address != degradedPeer);
+				}
+				for (const auto& recoveredPeer : req.recoveredPeers) {
+					ASSERT(address != recoveredPeer);
+				}
+			}
+		}
 	}
+
+	return req;
 }
 
 // The actor that actively monitors the health of local and peer servers, and reports anomaly to the cluster controller.
@@ -1185,11 +1229,30 @@ ACTOR Future<Void> healthMonitor(Reference<AsyncVar<Optional<ClusterControllerFu
                                  WorkerInterface interf,
                                  LocalityData locality,
                                  Reference<AsyncVar<ServerDBInfo> const> dbInfo) {
+	state UpdateWorkerHealthRequest req;
 	loop {
 		Future<Void> nextHealthCheckDelay = Never();
 		if (dbInfo->get().recoveryState >= RecoveryState::ACCEPTING_COMMITS && ccInterface->get().present()) {
 			nextHealthCheckDelay = delay(SERVER_KNOBS->WORKER_HEALTH_MONITOR_INTERVAL);
-			doPeerHealthCheck(ccInterface, interf, locality, dbInfo);
+			req = doPeerHealthCheck(interf, locality, dbInfo, req);
+
+			if (!req.disconnectedPeers.empty() || !req.degradedPeers.empty() || !req.recoveredPeers.empty()) {
+				if (g_network->isSimulated()) {
+					// Do invarant check only in simulation.
+					for (const auto& recoveredPeer : req.recoveredPeers) {
+						for (const auto& disconnectedPeer : req.disconnectedPeers) {
+							ASSERT(recoveredPeer != disconnectedPeer);
+						}
+						for (const auto& degradedPeer : req.degradedPeers) {
+							ASSERT(recoveredPeer != degradedPeer);
+						}
+					}
+				}
+
+				// Disconnected or degraded peers are reported to the cluster controller.
+				req.address = FlowTransport::transport().getLocalAddress();
+				ccInterface->get().get().updateWorkerHealth.send(req);
+			}
 		}
 		choose {
 			when(wait(nextHealthCheckDelay)) {}

From 5fd337c7b7544474774c5426aaf826f764d5d814 Mon Sep 17 00:00:00 2001
From: Zhe Wu <halfprice@users.noreply.github.com>
Date: Tue, 20 Jun 2023 12:40:22 -0700
Subject: [PATCH 174/468] Revert logic to test testing logic

---
 fdbserver/worker.actor.cpp | 1 -
 1 file changed, 1 deletion(-)

diff --git a/fdbserver/worker.actor.cpp b/fdbserver/worker.actor.cpp
index c04fd7d6311..877947ce864 100644
--- a/fdbserver/worker.actor.cpp
+++ b/fdbserver/worker.actor.cpp
@@ -1079,7 +1079,6 @@ UpdateWorkerHealthRequest doPeerHealthCheck(const WorkerInterface& interf,
 
 	for (const auto& [address, peer] : allPeers) {
 		if (!shouldCheckPeer(peer)) {
-			req.recoveredPeers.push_back(address);
 			continue;
 		}
 

From 0331b7e1083aa56f8ca1056235dc8df891a09e06 Mon Sep 17 00:00:00 2001
From: Zhe Wu <halfprice@users.noreply.github.com>
Date: Tue, 20 Jun 2023 13:30:03 -0700
Subject: [PATCH 175/468] cleanup gray failure recovery documentation

---
 fdbserver/worker.actor.cpp | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/fdbserver/worker.actor.cpp b/fdbserver/worker.actor.cpp
index 877947ce864..4d4dc236fdf 100644
--- a/fdbserver/worker.actor.cpp
+++ b/fdbserver/worker.actor.cpp
@@ -1016,7 +1016,7 @@ TEST_CASE("/fdbserver/worker/addressIsRemoteLogRouter") {
 
 } // namespace
 
-// Returns true if the `peer` has enough data samples that should be checked by the health monitor.
+// Returns true if the `peer` has enough measurement samples that should be checked by the health monitor.
 bool shouldCheckPeer(Reference<Peer> peer) {
 	if (peer->connectFailedCount != 0) {
 		return true;
@@ -1049,8 +1049,7 @@ bool isDegradedPeer(const UpdateWorkerHealthRequest& lastReq, const NetworkAddre
 	return false;
 }
 
-// Check if the current worker is a transaction worker, and is experiencing degraded or disconnected peers. If so,
-// report degraded and disconnected peers to the cluster controller.
+// Check if the current worker is a transaction worker, and is experiencing degraded or disconnected peers.
 UpdateWorkerHealthRequest doPeerHealthCheck(const WorkerInterface& interf,
                                             const LocalityData& locality,
                                             Reference<AsyncVar<ServerDBInfo> const> dbInfo,
@@ -1185,6 +1184,8 @@ UpdateWorkerHealthRequest doPeerHealthCheck(const WorkerInterface& interf,
 		// transport's health monitor. Note that all the closed peers stored here are caused by connection
 		// failure, but not normal connection close. Therefore, we report all such peers if they are also
 		// part of the transaction sub system.
+		// Note that we don't need to calculate recovered peer in this case since all the recently closed peers are
+		// considered permanently closed peers.
 		for (const auto& address : FlowTransport::transport().healthMonitor()->getRecentClosedPeers()) {
 			if (allPeers.find(address) != allPeers.end()) {
 				// We have checked this peer in the above for loop.
@@ -1238,6 +1239,7 @@ ACTOR Future<Void> healthMonitor(Reference<AsyncVar<Optional<ClusterControllerFu
 			if (!req.disconnectedPeers.empty() || !req.degradedPeers.empty() || !req.recoveredPeers.empty()) {
 				if (g_network->isSimulated()) {
 					// Do invarant check only in simulation.
+					// Any recovered peer shouldn't appear as disconnected or degraded peer.
 					for (const auto& recoveredPeer : req.recoveredPeers) {
 						for (const auto& disconnectedPeer : req.disconnectedPeers) {
 							ASSERT(recoveredPeer != disconnectedPeer);

From 02492eb95b6f6d8c18cfe38a5ac634235629f19c Mon Sep 17 00:00:00 2001
From: Zhe Wu <halfprice@users.noreply.github.com>
Date: Wed, 21 Jun 2023 13:02:59 -0700
Subject: [PATCH 176/468] Cluster controller remove recovered peer in gray
 failure

---
 fdbserver/ClusterController.actor.cpp | 12 ++++++++++++
 fdbserver/ClusterController.actor.h   | 26 +++++++++++++++++++++++++-
 2 files changed, 37 insertions(+), 1 deletion(-)

diff --git a/fdbserver/ClusterController.actor.cpp b/fdbserver/ClusterController.actor.cpp
index f0f601c2eaa..fffcd58ea04 100644
--- a/fdbserver/ClusterController.actor.cpp
+++ b/fdbserver/ClusterController.actor.cpp
@@ -2834,6 +2834,18 @@ TEST_CASE("/fdbserver/clustercontroller/updateWorkerHealth") {
 		ASSERT_EQ(health.disconnectedPeers[badPeer3].lastRefreshTime, previousRefreshTime);
 	}
 
+	// Make badPeer1 a recovered peer, and CC should remove it from `workerAddress` bad peers.
+	{
+		wait(delay(0.001));
+		UpdateWorkerHealthRequest req;
+		req.address = workerAddress;
+		req.recoveredPeers.push_back(badPeer1);
+		data.updateWorkerHealth(req);
+		auto& health = data.workerHealth[workerAddress];
+		ASSERT(health.degradedPeers.find(badPeer1) == health.degradedPeers.end());
+		ASSERT(health.disconnectedPeers.find(badPeer1) == health.disconnectedPeers.end());
+	}
+
 	return Void();
 }
 
diff --git a/fdbserver/ClusterController.actor.h b/fdbserver/ClusterController.actor.h
index 265211ce283..3502658ab6c 100644
--- a/fdbserver/ClusterController.actor.h
+++ b/fdbserver/ClusterController.actor.h
@@ -2907,16 +2907,28 @@ class ClusterControllerData {
 		for (int i = 0; i < req.disconnectedPeers.size(); ++i) {
 			disconnectedPeersString += (i == 0 ? "" : " ") + req.disconnectedPeers[i].toString();
 		}
+		std::string recoveredPeersString;
+		for (int i = 0; i < req.recoveredPeers.size(); ++i) {
+			recoveredPeersString += (i == 0 ? "" : " ") + req.recoveredPeers[i].toString();
+		}
 		TraceEvent("ClusterControllerUpdateWorkerHealth")
 		    .detail("WorkerAddress", req.address)
 		    .detail("DegradedPeers", degradedPeersString)
-		    .detail("DisconnectedPeers", disconnectedPeersString);
+		    .detail("DisconnectedPeers", disconnectedPeersString)
+		    .detail("RecoveredPeers", recoveredPeersString);
 
 		double currentTime = now();
 
 		// Current `workerHealth` doesn't have any information about the incoming worker. Add the worker into
 		// `workerHealth`.
 		if (workerHealth.find(req.address) == workerHealth.end()) {
+			if (req.degradedPeers.empty() && req.disconnectedPeers.empty()) {
+				// This request doesn't report any new degradation. Although there may contain recovered peer, since
+				// `workerHealth` doesn't record any information on this address, those recovered peers have already
+				// been considered recovered.
+				return;
+			}
+
 			workerHealth[req.address] = {};
 			for (const auto& degradedPeer : req.degradedPeers) {
 				workerHealth[req.address].degradedPeers[degradedPeer] = { currentTime, currentTime };
@@ -2926,6 +2938,8 @@ class ClusterControllerData {
 				workerHealth[req.address].disconnectedPeers[degradedPeer] = { currentTime, currentTime };
 			}
 
+			// We can return directly here since we just created the health info for this address and there shouldn't be
+			// any recovered peers.
 			return;
 		}
 
@@ -2933,6 +2947,16 @@ class ClusterControllerData {
 
 		auto& health = workerHealth[req.address];
 
+		// Remove any recovered peers.
+		for (const auto& peer : req.recoveredPeers) {
+			TraceEvent("ClusterControllerReceivedPeerRecovering")
+			    .suppressFor(10.0)
+			    .detail("Worker", req.address)
+			    .detail("Peer", peer);
+			health.degradedPeers.erase(peer);
+			health.disconnectedPeers.erase(peer);
+		}
+
 		// Update the worker's degradedPeers.
 		for (const auto& peer : req.degradedPeers) {
 			auto it = health.degradedPeers.find(peer);

From 12c910abf21536f2aa51ca446d4a7287cacd308d Mon Sep 17 00:00:00 2001
From: neethuhaneesha <nbingi@apple.com>
Date: Thu, 13 Jul 2023 14:50:53 -0700
Subject: [PATCH 177/468] Adding backup operations trace events temporarily.

---
 fdbclient/ServerKnobs.cpp         |  3 +++
 fdbclient/ServerKnobs.h           |  1 +
 fdbserver/storageserver.actor.cpp | 28 ++++++++++++++++++++++++++++
 3 files changed, 32 insertions(+)

diff --git a/fdbclient/ServerKnobs.cpp b/fdbclient/ServerKnobs.cpp
index 9f8145de2f3..8ccb504d6f0 100644
--- a/fdbclient/ServerKnobs.cpp
+++ b/fdbclient/ServerKnobs.cpp
@@ -444,6 +444,9 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi
 	init( ROCKSDB_CAN_COMMIT_DELAY_TIMES_ON_OVERLOAD,              5 );
 	init( ROCKSDB_COMPACTION_READAHEAD_SIZE,                   32768 ); // 32 KB, performs bigger reads when doing compaction.
 	init( ROCKSDB_BLOCK_SIZE,                                  32768 ); // 32 KB, size of the block in rocksdb cache.
+	// Temporary knob to enable trace events which prints details about all the backup keys update(write/clear) operations.
+	// Caution: To be enabled only under supervision.
+	init( SS_BACKUP_KEYS_OP_LOGS,                              false ); if( randomize && BUGGIFY ) SS_BACKUP_KEYS_OP_LOGS = true;
 
 	// Leader election
 	bool longLeaderElection = randomize && BUGGIFY;
diff --git a/fdbclient/ServerKnobs.h b/fdbclient/ServerKnobs.h
index e86f9313a36..a2795d875ee 100644
--- a/fdbclient/ServerKnobs.h
+++ b/fdbclient/ServerKnobs.h
@@ -355,6 +355,7 @@ class ServerKnobs : public KnobsImpl<ServerKnobs> {
 	int ROCKSDB_STATS_LEVEL;
 	int64_t ROCKSDB_COMPACTION_READAHEAD_SIZE;
 	int64_t ROCKSDB_BLOCK_SIZE;
+	bool SS_BACKUP_KEYS_OP_LOGS;
 
 	// Leader election
 	int MAX_NOTIFICATIONS;
diff --git a/fdbserver/storageserver.actor.cpp b/fdbserver/storageserver.actor.cpp
index 0a2508d3a43..02236a8a3f6 100644
--- a/fdbserver/storageserver.actor.cpp
+++ b/fdbserver/storageserver.actor.cpp
@@ -71,6 +71,7 @@
 #include "fdbrpc/Stats.h"
 #include "flow/TDMetric.actor.h"
 #include "flow/genericactors.actor.h"
+#include "flow/crc32c.h"
 
 #include "flow/actorcompiler.h" // This must be the last #include.
 
@@ -7882,16 +7883,43 @@ void setAssignedStatus(StorageServer* self, KeyRangeRef keys, bool nowAssigned)
 	}
 }
 
+// convert a StringRef to Hex string
+static std::string hexStringRef(const StringRef& s) {
+	std::string result;
+	result.reserve(s.size() * 2);
+	for (int i = 0; i < s.size(); i++) {
+		result.append(format("%02x", s[i]));
+	}
+	return result;
+}
+
 void StorageServerDisk::clearRange(KeyRangeRef keys) {
 	storage->clear(keys);
+	if (SERVER_KNOBS->SS_BACKUP_KEYS_OP_LOGS && keys.begin.startsWith(backupLogKeys.begin)) {
+		TraceEvent("SSBackupKeyClearRange")
+		    .detail("RangeBegin", keys.begin)
+		    .detail("RangeEnd", keys.end)
+		    .detail("HexRangeBegin", hexStringRef(keys.begin))
+		    .detail("HexRangeEnd", hexStringRef(keys.end));
+	}
 	++(*kvClearRanges);
 	if (keys.singleKeyRange()) {
 		++(*kvClearSingleKey);
 	}
 }
 
+std::string checksumValue(const ValueRef& s) {
+	return s.size() > 12 ? format("(%d)%08x", s.size(), crc32c_append(0, s.begin(), s.size())) : s.toString();
+}
+
 void StorageServerDisk::writeKeyValue(KeyValueRef kv) {
 	storage->set(kv);
+	if (SERVER_KNOBS->SS_BACKUP_KEYS_OP_LOGS && kv.key.startsWith(backupLogKeys.begin)) {
+		TraceEvent("SSBackupKeyWriteKeyValue")
+		    .detail("Key", kv.key)
+		    .detail("ChecksumValue", checksumValue(kv.value))
+		    .detail("HexKey", hexStringRef(kv.key));
+	}
 	*kvCommitLogicalBytes += kv.expectedSize();
 }
 

From ba7366b482f90aec918b767fb3327c2e4dd194eb Mon Sep 17 00:00:00 2001
From: neethuhaneesha <nbingi@apple.com>
Date: Thu, 13 Jul 2023 16:04:53 -0700
Subject: [PATCH 178/468] Add more logging to backup data deletion

---
 fdbclient/BackupAgentBase.actor.cpp | 32 +++++++++++++++++++++++++++--
 fdbclient/ClientKnobs.cpp           |  1 +
 fdbclient/ClientKnobs.h             |  1 +
 fdbclient/FileBackupAgent.actor.cpp |  3 +++
 4 files changed, 35 insertions(+), 2 deletions(-)

diff --git a/fdbclient/BackupAgentBase.actor.cpp b/fdbclient/BackupAgentBase.actor.cpp
index 6033f809928..8e71406c693 100644
--- a/fdbclient/BackupAgentBase.actor.cpp
+++ b/fdbclient/BackupAgentBase.actor.cpp
@@ -22,6 +22,7 @@
 #include <time.h>
 
 #include "fdbclient/BackupAgent.actor.h"
+#include "fdbclient/Knobs.h"
 #include "fdbrpc/simulator.h"
 #include "flow/ActorCollection.h"
 #include "flow/actorcompiler.h" // has to be last include
@@ -796,6 +797,16 @@ ACTOR Future<Void> applyMutations(Database cx,
 	}
 }
 
+// convert a StringRef to Hex string
+static std::string hexStringRef(const StringRef& s) {
+	std::string result;
+	result.reserve(s.size() * 2);
+	for (int i = 0; i < s.size(); i++) {
+		result.append(format("%02x", s[i]));
+	}
+	return result;
+}
+
 ACTOR static Future<Void> _eraseLogData(Reference<ReadYourWritesTransaction> tr,
                                         Key logUidValue,
                                         Key destUidValue,
@@ -889,14 +900,31 @@ ACTOR static Future<Void> _eraseLogData(Reference<ReadYourWritesTransaction> tr,
 					uint64_t ev = bigEndian64(nextSmallestVersion);
 					uint8_t h1 = h;
 					Key vblockPrefix = StringRef(&h1, sizeof(uint8_t)).withPrefix(baLogRangePrefix);
-					tr->clear(KeyRangeRef(StringRef((uint8_t*)&bv, sizeof(uint64_t)).withPrefix(vblockPrefix),
-					                      StringRef((uint8_t*)&ev, sizeof(uint64_t)).withPrefix(vblockPrefix)));
+					KeyRange range = KeyRangeRef(StringRef((uint8_t*)&bv, sizeof(uint64_t)).withPrefix(vblockPrefix),
+					                             StringRef((uint8_t*)&ev, sizeof(uint64_t)).withPrefix(vblockPrefix));
+					tr->clear(range);
+					if (CLIENT_KNOBS->BACKUP_AGENT_VERBOSE_LOGGING) {
+						TraceEvent("EraseLogDataClearLogRanges")
+						    .detail("Range", range)
+						    .detail("Begin", 0)
+						    .detail("End", nextSmallestVersion)
+						    .detail("HexRangeBegin", hexStringRef(range.begin))
+						    .detail("HexRangeEnd", hexStringRef(range.end));
+					}
 				}
 			} else {
 				Standalone<VectorRef<KeyRangeRef>> ranges =
 				    getLogRanges(currBeginVersion, nextSmallestVersion, destUidValue);
 				for (auto& range : ranges) {
 					tr->clear(range);
+					if (CLIENT_KNOBS->BACKUP_AGENT_VERBOSE_LOGGING) {
+						TraceEvent("EraseLogDataClearLogRanges")
+						    .detail("Range", range)
+						    .detail("Begin", currBeginVersion)
+						    .detail("End", nextSmallestVersion)
+						    .detail("HexRangeBegin", hexStringRef(range.begin))
+						    .detail("HexRangeEnd", hexStringRef(range.end));
+					}
 				}
 			}
 		}
diff --git a/fdbclient/ClientKnobs.cpp b/fdbclient/ClientKnobs.cpp
index bbc315774a0..7075da0b771 100644
--- a/fdbclient/ClientKnobs.cpp
+++ b/fdbclient/ClientKnobs.cpp
@@ -195,6 +195,7 @@ void ClientKnobs::initialize(Randomize randomize) {
 	init( BACKUP_STATUS_JITTER,                   0.05 );
 	init( MIN_CLEANUP_SECONDS,                  3600.0 );
 	init( FASTRESTORE_ATOMICOP_WEIGHT,               1 ); if( randomize && BUGGIFY ) { FASTRESTORE_ATOMICOP_WEIGHT = deterministicRandom()->random01() * 200 + 1; }
+	init( BACKUP_AGENT_VERBOSE_LOGGING, false ); if (randomize && BUGGIFY) { BACKUP_AGENT_VERBOSE_LOGGING = true; }
 
 	// Configuration
 	init( DEFAULT_AUTO_COMMIT_PROXIES,               3 );
diff --git a/fdbclient/ClientKnobs.h b/fdbclient/ClientKnobs.h
index f4cf417672c..30b664f49a3 100644
--- a/fdbclient/ClientKnobs.h
+++ b/fdbclient/ClientKnobs.h
@@ -197,6 +197,7 @@ class ClientKnobs : public KnobsImpl<ClientKnobs> {
 	double BACKUP_STATUS_JITTER;
 	double MIN_CLEANUP_SECONDS;
 	int64_t FASTRESTORE_ATOMICOP_WEIGHT; // workload amplication factor for atomic op
+	bool BACKUP_AGENT_VERBOSE_LOGGING;
 
 	// Configuration
 	int32_t DEFAULT_AUTO_COMMIT_PROXIES;
diff --git a/fdbclient/FileBackupAgent.actor.cpp b/fdbclient/FileBackupAgent.actor.cpp
index 6411f53f306..ebd7571cdf4 100644
--- a/fdbclient/FileBackupAgent.actor.cpp
+++ b/fdbclient/FileBackupAgent.actor.cpp
@@ -2298,6 +2298,9 @@ struct BackupLogsDispatchTask : BackupTaskFuncBase {
 
 		tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 		tr->setOption(FDBTransactionOptions::LOCK_AWARE);
+		if (CLIENT_KNOBS->BACKUP_AGENT_VERBOSE_LOGGING) {
+			tr->debugTransaction(deterministicRandom()->randomUniqueID());
+		}
 
 		state Reference<TaskFuture> onDone = task->getDoneFuture(futureBucket);
 		state Version prevBeginVersion = Params.prevBeginVersion().get(task);

From d1d52356e4af425a062ade9185566a7492228faf Mon Sep 17 00:00:00 2001
From: neethuhaneesha <nbingi@apple.com>
Date: Thu, 13 Jul 2023 16:38:06 -0700
Subject: [PATCH 179/468] remove dependency on xerrors by replacing deprecated
 call to "xerrors

---
 bindings/go/go.mod              |  2 --
 bindings/go/src/fdb/database.go | 10 +++-------
 2 files changed, 3 insertions(+), 9 deletions(-)

diff --git a/bindings/go/go.mod b/bindings/go/go.mod
index ec5746bf99f..16e502baaf1 100644
--- a/bindings/go/go.mod
+++ b/bindings/go/go.mod
@@ -2,5 +2,3 @@ module github.com/apple/foundationdb/bindings/go
 
 // The FoundationDB go bindings currently have no external golang dependencies outside of
 // the go standard library.
-
-require golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 // indirect
diff --git a/bindings/go/src/fdb/database.go b/bindings/go/src/fdb/database.go
index e2cc60b123d..7d3be8fabe8 100644
--- a/bindings/go/src/fdb/database.go
+++ b/bindings/go/src/fdb/database.go
@@ -29,8 +29,6 @@ import "C"
 import (
 	"errors"
 	"runtime"
-
-	"golang.org/x/xerrors"
 )
 
 // Database is a handle to a FoundationDB database. Database is a lightweight
@@ -121,7 +119,7 @@ func retryable(wrapped func() (interface{}, error), onError func(Error) FutureNi
 		// Check if the error chain contains an
 		// fdb.Error
 		var ep Error
-		if xerrors.As(e, &ep) {
+		if errors.As(e, &ep) {
 			e = onError(ep).Get()
 		}
 
@@ -145,8 +143,7 @@ func retryable(wrapped func() (interface{}, error), onError func(Error) FutureNi
 // error.
 //
 // The transaction is retried if the error is or wraps a retryable Error.
-// The error is unwrapped with the xerrors.Wrapper. See https://godoc.org/golang.org/x/xerrors#Wrapper
-// for details.
+// The error is unwrapped.
 //
 // Do not return Future objects from the function provided to Transact. The
 // Transaction created by Transact may be finalized at any point after Transact
@@ -189,8 +186,7 @@ func (d Database) Transact(f func(Transaction) (interface{}, error)) (interface{
 // transaction or return the error.
 //
 // The transaction is retried if the error is or wraps a retryable Error.
-// The error is unwrapped with the xerrors.Wrapper. See https://godoc.org/golang.org/x/xerrors#Wrapper
-// for details.
+// The error is unwrapped.
 //
 // Do not return Future objects from the function provided to ReadTransact. The
 // Transaction created by ReadTransact may be finalized at any point after

From fae7712159faac7570c6e8265f71053a571a96b7 Mon Sep 17 00:00:00 2001
From: neethuhaneesha <nbingi@apple.com>
Date: Fri, 14 Jul 2023 13:29:57 -0700
Subject: [PATCH 180/468] Disabling backup keys logging in simulation to avoid
 TracingTooManyLines error.

---
 fdbclient/ClientKnobs.cpp | 2 +-
 fdbclient/ServerKnobs.cpp | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/fdbclient/ClientKnobs.cpp b/fdbclient/ClientKnobs.cpp
index 7075da0b771..a0c9628c50f 100644
--- a/fdbclient/ClientKnobs.cpp
+++ b/fdbclient/ClientKnobs.cpp
@@ -195,7 +195,7 @@ void ClientKnobs::initialize(Randomize randomize) {
 	init( BACKUP_STATUS_JITTER,                   0.05 );
 	init( MIN_CLEANUP_SECONDS,                  3600.0 );
 	init( FASTRESTORE_ATOMICOP_WEIGHT,               1 ); if( randomize && BUGGIFY ) { FASTRESTORE_ATOMICOP_WEIGHT = deterministicRandom()->random01() * 200 + 1; }
-	init( BACKUP_AGENT_VERBOSE_LOGGING, false ); if (randomize && BUGGIFY) { BACKUP_AGENT_VERBOSE_LOGGING = true; }
+	init( BACKUP_AGENT_VERBOSE_LOGGING,          false ); if (randomize && BUGGIFY) { BACKUP_AGENT_VERBOSE_LOGGING = true; }
 
 	// Configuration
 	init( DEFAULT_AUTO_COMMIT_PROXIES,               3 );
diff --git a/fdbclient/ServerKnobs.cpp b/fdbclient/ServerKnobs.cpp
index 8ccb504d6f0..78f8e4bf90d 100644
--- a/fdbclient/ServerKnobs.cpp
+++ b/fdbclient/ServerKnobs.cpp
@@ -446,7 +446,7 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi
 	init( ROCKSDB_BLOCK_SIZE,                                  32768 ); // 32 KB, size of the block in rocksdb cache.
 	// Temporary knob to enable trace events which prints details about all the backup keys update(write/clear) operations.
 	// Caution: To be enabled only under supervision.
-	init( SS_BACKUP_KEYS_OP_LOGS,                              false ); if( randomize && BUGGIFY ) SS_BACKUP_KEYS_OP_LOGS = true;
+	init( SS_BACKUP_KEYS_OP_LOGS,                              false );
 
 	// Leader election
 	bool longLeaderElection = randomize && BUGGIFY;

From 90c155e6d40a3b2d4e702db6b9f4d390a057c08d Mon Sep 17 00:00:00 2001
From: neethuhaneesha <nbingi@apple.com>
Date: Wed, 19 Jul 2023 13:48:32 -0700
Subject: [PATCH 181/468] Added backup key mutation logs

---
 fdbbackup/FileDecoder.actor.cpp      | 15 +------
 fdbclient/BackupAgentBase.actor.cpp  | 18 ++-------
 fdbclient/StorageServerInterface.cpp | 20 ++--------
 fdbserver/storageserver.actor.cpp    | 60 +++++++++++++++-------------
 flow/Arena.h                         | 15 +++++++
 5 files changed, 57 insertions(+), 71 deletions(-)

diff --git a/fdbbackup/FileDecoder.actor.cpp b/fdbbackup/FileDecoder.actor.cpp
index b1e60c1aa88..be408b90426 100644
--- a/fdbbackup/FileDecoder.actor.cpp
+++ b/fdbbackup/FileDecoder.actor.cpp
@@ -741,16 +741,6 @@ class DecodeRangeProgress {
 	int lfd = -1; // local file descriptor
 };
 
-// convert a StringRef to Hex string
-std::string hexStringRef(const StringRef& s) {
-	std::string result;
-	result.reserve(s.size() * 2);
-	for (int i = 0; i < s.size(); i++) {
-		result.append(format("%02x", s[i]));
-	}
-	return result;
-}
-
 ACTOR Future<Void> process_range_file(Reference<IBackupContainer> container,
                                       RangeFile file,
                                       UID uid,
@@ -777,8 +767,7 @@ ACTOR Future<Void> process_range_file(Reference<IBackupContainer> container,
 				    .detail("Version", file.version)
 				    .setMaxFieldLength(1000)
 				    .detail("KV", kv);
-				std::cout << file.version << " key: " << hexStringRef(kv.key) << "  value: " << hexStringRef(kv.value)
-				          << std::endl;
+				std::cout << file.version << " key: " << kv.key.toHex() << "  value: " << kv.value.toHex() << std::endl;
 			}
 		}
 	}
@@ -823,7 +812,7 @@ ACTOR Future<Void> process_file(Reference<IBackupContainer> container,
 				    .setMaxFieldLength(1000)
 				    .detail("M", m.toString());
 				std::cout << vms.version << "." << sub << " " << typeString[(int)m.type]
-				          << " param1: " << hexStringRef(m.param1) << " param2: " << hexStringRef(m.param2) << "\n";
+				          << " param1: " << m.param1.toHex() << " param2: " << m.param2.toHex() << "\n";
 			}
 		}
 	}
diff --git a/fdbclient/BackupAgentBase.actor.cpp b/fdbclient/BackupAgentBase.actor.cpp
index 8e71406c693..cd6cc19f735 100644
--- a/fdbclient/BackupAgentBase.actor.cpp
+++ b/fdbclient/BackupAgentBase.actor.cpp
@@ -797,16 +797,6 @@ ACTOR Future<Void> applyMutations(Database cx,
 	}
 }
 
-// convert a StringRef to Hex string
-static std::string hexStringRef(const StringRef& s) {
-	std::string result;
-	result.reserve(s.size() * 2);
-	for (int i = 0; i < s.size(); i++) {
-		result.append(format("%02x", s[i]));
-	}
-	return result;
-}
-
 ACTOR static Future<Void> _eraseLogData(Reference<ReadYourWritesTransaction> tr,
                                         Key logUidValue,
                                         Key destUidValue,
@@ -908,8 +898,8 @@ ACTOR static Future<Void> _eraseLogData(Reference<ReadYourWritesTransaction> tr,
 						    .detail("Range", range)
 						    .detail("Begin", 0)
 						    .detail("End", nextSmallestVersion)
-						    .detail("HexRangeBegin", hexStringRef(range.begin))
-						    .detail("HexRangeEnd", hexStringRef(range.end));
+						    .detail("HexRangeBegin", range.begin.toHex())
+						    .detail("HexRangeEnd", range.end.toHex());
 					}
 				}
 			} else {
@@ -922,8 +912,8 @@ ACTOR static Future<Void> _eraseLogData(Reference<ReadYourWritesTransaction> tr,
 						    .detail("Range", range)
 						    .detail("Begin", currBeginVersion)
 						    .detail("End", nextSmallestVersion)
-						    .detail("HexRangeBegin", hexStringRef(range.begin))
-						    .detail("HexRangeEnd", hexStringRef(range.end));
+						    .detail("HexRangeBegin", range.begin.toHex())
+						    .detail("HexRangeEnd", range.end.toHex());
 					}
 				}
 			}
diff --git a/fdbclient/StorageServerInterface.cpp b/fdbclient/StorageServerInterface.cpp
index 7b36719d644..d5579270b21 100644
--- a/fdbclient/StorageServerInterface.cpp
+++ b/fdbclient/StorageServerInterface.cpp
@@ -149,16 +149,6 @@ static void traceKeyValuesSummary(TraceEvent& event,
 	    .detail("TSSReplySummary", tssSummaryString);
 }
 
-// convert a StringRef to Hex string
-static std::string hexStringRef(const StringRef& s) {
-	std::string result;
-	result.reserve(s.size() * 2);
-	for (int i = 0; i < s.size(); i++) {
-		result.append(format("%02x", s[i]));
-	}
-	return result;
-}
-
 static void traceKeyValuesDiff(TraceEvent& event,
                                const KeySelectorRef& begin,
                                const KeySelectorRef& end,
@@ -178,17 +168,15 @@ static void traceKeyValuesDiff(TraceEvent& event,
 			event.detail("MismatchIndex", i);
 			if (i >= ssKV.size() || i >= tssKV.size() || ssKV[i].key != tssKV[i].key) {
 				event.detail("MismatchSSKey", i < ssKV.size() ? ssKV[i].key : "missing"_sr);
-				event.detail("MismatchSSKeyHex", i < ssKV.size() ? hexStringRef(ssKV[i].key) : "missing"_sr);
+				event.detail("MismatchSSKeyHex", i < ssKV.size() ? ssKV[i].key.toHex() : "missing"_sr);
 				event.detail("MismatchTSSKey", i < tssKV.size() ? tssKV[i].key : "missing"_sr);
-				event.detail("MismatchTSSKeyHex", i < tssKV.size() ? hexStringRef(tssKV[i].key) : "missing"_sr)
+				event.detail("MismatchTSSKeyHex", i < tssKV.size() ? tssKV[i].key.toHex() : "missing"_sr)
 				    .setMaxFieldLength(-1);
 			} else {
 				event.detail("MismatchKey", ssKV[i].key);
+				event.detail("MismatchSSKeyHex", ssKV[i].key.toHex());
 				event.detail("MismatchSSValue", traceChecksumValue(ssKV[i].value));
-				event.detail("MismatchSSValueHex", hexStringRef(traceChecksumValue(ssKV[i].value)));
-				event.detail("MismatchTSSValue", traceChecksumValue(tssKV[i].value));
-				event.detail("MismatchTSSValueHex", hexStringRef(traceChecksumValue(tssKV[i].value)))
-				    .setMaxFieldLength(-1);
+				event.detail("MismatchTSSValue", traceChecksumValue(tssKV[i].value)).setMaxFieldLength(-1);
 			}
 			mismatchFound = true;
 			break;
diff --git a/fdbserver/storageserver.actor.cpp b/fdbserver/storageserver.actor.cpp
index 02236a8a3f6..10f4c285bc6 100644
--- a/fdbserver/storageserver.actor.cpp
+++ b/fdbserver/storageserver.actor.cpp
@@ -71,7 +71,6 @@
 #include "fdbrpc/Stats.h"
 #include "flow/TDMetric.actor.h"
 #include "flow/genericactors.actor.h"
-#include "flow/crc32c.h"
 
 #include "flow/actorcompiler.h" // This must be the last #include.
 
@@ -5989,6 +5988,14 @@ ACTOR Future<Void> fetchKeys(StorageServer* data, AddingShard* shard) {
 
 					// Write this_block to storage
 					state KeyValueRef* kvItr = this_block.begin();
+					if (SERVER_KNOBS->SS_BACKUP_KEYS_OP_LOGS && kvItr != this_block.end() &&
+					    kvItr->key.startsWith(backupLogKeys.begin)) {
+						TraceEvent("SSBackupKeyWriteKeyValue")
+						    .detail("Key", kvItr->key)
+						    .detail("ChecksumValue", kvItr->value.getChecksum())
+						    .detail("HexKey", kvItr->key.toHex())
+						    .detail("Version", fetchVersion);
+					}
 					for (; kvItr != this_block.end(); ++kvItr) {
 						data->storage.writeKeyValue(*kvItr);
 						wait(yield());
@@ -6251,6 +6258,14 @@ ACTOR Future<Void> fetchKeys(StorageServer* data, AddingShard* shard) {
 		}
 		if (e.code() == error_code_actor_cancelled && !data->shuttingDown && shard->phase >= AddingShard::Fetching) {
 			if (shard->phase < AddingShard::FetchingCF) {
+				if (SERVER_KNOBS->SS_BACKUP_KEYS_OP_LOGS && keys.begin.startsWith(backupLogKeys.begin)) {
+					TraceEvent("SSBackupKeyClearRange")
+					    .detail("RangeBegin", keys.begin)
+					    .detail("RangeEnd", keys.end)
+					    .detail("HexRangeBegin", keys.begin.toHex())
+					    .detail("HexRangeEnd", keys.end.toHex())
+					    .detail("FetchVersion", fetchVersion);
+				}
 				data->storage.clearRange(keys);
 				++data->counters.kvSystemClearRanges;
 				data->byteSampleApplyClear(keys, invalidVersion);
@@ -6625,6 +6640,22 @@ class StorageUpdater {
 				DEBUG_MUTATION("SSUpdateMutation", ver, m, data->thisServerID).detail("FromFetch", fromFetch);
 			}
 
+			if (SERVER_KNOBS->SS_BACKUP_KEYS_OP_LOGS) {
+				if (m.type == MutationRef::SetValue && m.param1.startsWith(backupLogKeys.begin) &&
+				    m.param1.endsWith(LiteralStringRef("\x00\x00\x00\x00"))) {
+					TraceEvent("SSBackupKeyApplyMutationSet")
+					    .detail("Key", m.param1)
+					    .detail("HexKey", m.param1.toHex())
+					    .detail("Version", ver);
+				} else if (m.type == MutationRef::ClearRange && m.param1.startsWith(backupLogKeys.begin)) {
+					TraceEvent("SSBackupKeyApplyMutationClear")
+					    .detail("RangeBegin", m.param1)
+					    .detail("RangeEnd", m.param2)
+					    .detail("HexRangeBegin", m.param1.toHex())
+					    .detail("HexRangeEnd", m.param2.toHex())
+					    .detail("Version", ver);
+				}
+			}
 			splitMutation(data, data->shards, m, ver, fromFetch);
 		}
 
@@ -7883,43 +7914,16 @@ void setAssignedStatus(StorageServer* self, KeyRangeRef keys, bool nowAssigned)
 	}
 }
 
-// convert a StringRef to Hex string
-static std::string hexStringRef(const StringRef& s) {
-	std::string result;
-	result.reserve(s.size() * 2);
-	for (int i = 0; i < s.size(); i++) {
-		result.append(format("%02x", s[i]));
-	}
-	return result;
-}
-
 void StorageServerDisk::clearRange(KeyRangeRef keys) {
 	storage->clear(keys);
-	if (SERVER_KNOBS->SS_BACKUP_KEYS_OP_LOGS && keys.begin.startsWith(backupLogKeys.begin)) {
-		TraceEvent("SSBackupKeyClearRange")
-		    .detail("RangeBegin", keys.begin)
-		    .detail("RangeEnd", keys.end)
-		    .detail("HexRangeBegin", hexStringRef(keys.begin))
-		    .detail("HexRangeEnd", hexStringRef(keys.end));
-	}
 	++(*kvClearRanges);
 	if (keys.singleKeyRange()) {
 		++(*kvClearSingleKey);
 	}
 }
 
-std::string checksumValue(const ValueRef& s) {
-	return s.size() > 12 ? format("(%d)%08x", s.size(), crc32c_append(0, s.begin(), s.size())) : s.toString();
-}
-
 void StorageServerDisk::writeKeyValue(KeyValueRef kv) {
 	storage->set(kv);
-	if (SERVER_KNOBS->SS_BACKUP_KEYS_OP_LOGS && kv.key.startsWith(backupLogKeys.begin)) {
-		TraceEvent("SSBackupKeyWriteKeyValue")
-		    .detail("Key", kv.key)
-		    .detail("ChecksumValue", checksumValue(kv.value))
-		    .detail("HexKey", hexStringRef(kv.key));
-	}
 	*kvCommitLogicalBytes += kv.expectedSize();
 }
 
diff --git a/flow/Arena.h b/flow/Arena.h
index fd7af76d56f..4185e0e8523 100644
--- a/flow/Arena.h
+++ b/flow/Arena.h
@@ -41,6 +41,7 @@
 #include <set>
 #include <type_traits>
 #include <sstream>
+#include "flow/crc32c.h"
 
 // TrackIt is a zero-size class for tracking constructions, destructions, and assignments of instances
 // of a class.  Just inherit TrackIt<T> from T to enable tracking of construction and destruction of
@@ -641,6 +642,20 @@ class StringRef {
 	// True if both StringRefs reference exactly the same memory
 	bool same(const StringRef& s) const { return data == s.data && length == s.length; }
 
+	// convert a StringRef to Hex string
+	std::string toHex() const {
+		std::string result;
+		result.reserve(length * 2);
+		for (int i = 0; i < length; i++) {
+			result.append(format("%02x", data[i]));
+		}
+		return result;
+	}
+
+	std::string getChecksum() const {
+		return length > 12 ? format("(%d)%08x", length, crc32c_append(0, data, length)) : toString();
+	}
+
 private:
 	// Unimplemented; blocks conversion through std::string
 	StringRef(char*);

From 348fd48dc22cff0fafff19f1325614a2c34bc619 Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyu_zhou@apple.com>
Date: Fri, 21 Jul 2023 11:23:11 -0700
Subject: [PATCH 182/468] Debug all backup keys by inserting them into the DB
 (#10654)

Specifically, added (key, commitVersion) pairs to \xff\x02\dlog space.
---
 fdbserver/CommitProxyServer.actor.cpp | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/fdbserver/CommitProxyServer.actor.cpp b/fdbserver/CommitProxyServer.actor.cpp
index 2af27aed41e..de83562b43f 100644
--- a/fdbserver/CommitProxyServer.actor.cpp
+++ b/fdbserver/CommitProxyServer.actor.cpp
@@ -430,6 +430,18 @@ ACTOR Future<Void> addBackupMutations(ProxyCommitData* self,
 			// logRangeMutation.first) 					.detail("PartIndex", part).detail("PartIndexEndian",
 			// bigEndian32(part)).detail("PartData", backupMutation.param1);
 			//			}
+			if (SERVER_KNOBS->SS_BACKUP_KEYS_OP_LOGS) {
+				MutationRef m(backupMutation);
+				// replace \xff\x02/blog/... with \xff\x02/dlog/...
+				// replace value with commit version
+				auto param1 = backupMutation.param1.substr(4).withPrefix("\xff\x02/d"_sr);
+				m.param1 = param1;
+				// maybe add toCommit->getMutationCount(), i.e., subversion?
+				m.param2 = StringRef((uint8_t*)&commitVersion, sizeof(commitVersion));
+				auto& tags = self->tagsForKey(m.param1);
+				toCommit->addTags(tags);
+				toCommit->writeTypedMessage(m);
+			}
 		}
 	}
 	return Void();

From 7d67bbf7133a0563f269d979368fbe982c32f056 Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Fri, 21 Jul 2023 14:37:19 -0700
Subject: [PATCH 183/468] Add tags in the value for original mutations

---
 fdbserver/CommitProxyServer.actor.cpp | 19 ++++++++++++++++---
 1 file changed, 16 insertions(+), 3 deletions(-)

diff --git a/fdbserver/CommitProxyServer.actor.cpp b/fdbserver/CommitProxyServer.actor.cpp
index de83562b43f..a29a29bd38e 100644
--- a/fdbserver/CommitProxyServer.actor.cpp
+++ b/fdbserver/CommitProxyServer.actor.cpp
@@ -347,6 +347,16 @@ bool isWhitelisted(const std::vector<Standalone<StringRef>>& binPathVec, StringR
 	return std::find(binPathVec.begin(), binPathVec.end(), binPath) != binPathVec.end();
 }
 
+std::string tagsToString(std::vector<Tag> tags) {
+	std::string result;
+	std::string space;
+	for (auto& tag : tags) {
+		result += space + tag.toString();
+		space = " ";
+	}
+	return result;
+}
+
 ACTOR Future<Void> addBackupMutations(ProxyCommitData* self,
                                       const std::map<Key, MutationListRef>* logRangeMutations,
                                       LogPushData* toCommit,
@@ -435,11 +445,14 @@ ACTOR Future<Void> addBackupMutations(ProxyCommitData* self,
 				// replace \xff\x02/blog/... with \xff\x02/dlog/...
 				// replace value with commit version
 				auto param1 = backupMutation.param1.substr(4).withPrefix("\xff\x02/d"_sr);
+				auto param2 = StringRef((uint8_t*)&commitVersion, sizeof(commitVersion))
+				                  .withSuffix(StringRef(tagsToString(tags)));
 				m.param1 = param1;
 				// maybe add toCommit->getMutationCount(), i.e., subversion?
-				m.param2 = StringRef((uint8_t*)&commitVersion, sizeof(commitVersion));
-				auto& tags = self->tagsForKey(m.param1);
-				toCommit->addTags(tags);
+				m.param2 = param2;
+
+				auto& newTags = self->tagsForKey(m.param1);
+				toCommit->addTags(newTags);
 				toCommit->writeTypedMessage(m);
 			}
 		}

From c3b566429865403ee40938a0ea7e19c5bc613ff4 Mon Sep 17 00:00:00 2001
From: neethuhaneesha <nbingi@apple.com>
Date: Fri, 21 Jul 2023 18:46:17 -0700
Subject: [PATCH 184/468] Debug logging on BackupKeys. (#10656)

---
 fdbserver/storageserver.actor.cpp | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/fdbserver/storageserver.actor.cpp b/fdbserver/storageserver.actor.cpp
index 10f4c285bc6..3f905f707de 100644
--- a/fdbserver/storageserver.actor.cpp
+++ b/fdbserver/storageserver.actor.cpp
@@ -6258,14 +6258,6 @@ ACTOR Future<Void> fetchKeys(StorageServer* data, AddingShard* shard) {
 		}
 		if (e.code() == error_code_actor_cancelled && !data->shuttingDown && shard->phase >= AddingShard::Fetching) {
 			if (shard->phase < AddingShard::FetchingCF) {
-				if (SERVER_KNOBS->SS_BACKUP_KEYS_OP_LOGS && keys.begin.startsWith(backupLogKeys.begin)) {
-					TraceEvent("SSBackupKeyClearRange")
-					    .detail("RangeBegin", keys.begin)
-					    .detail("RangeEnd", keys.end)
-					    .detail("HexRangeBegin", keys.begin.toHex())
-					    .detail("HexRangeEnd", keys.end.toHex())
-					    .detail("FetchVersion", fetchVersion);
-				}
 				data->storage.clearRange(keys);
 				++data->counters.kvSystemClearRanges;
 				data->byteSampleApplyClear(keys, invalidVersion);
@@ -7920,6 +7912,14 @@ void StorageServerDisk::clearRange(KeyRangeRef keys) {
 	if (keys.singleKeyRange()) {
 		++(*kvClearSingleKey);
 	}
+	if (SERVER_KNOBS->SS_BACKUP_KEYS_OP_LOGS && keys.begin.startsWith(backupLogKeys.begin)) {
+		TraceEvent("SSBackupKeyClearRange")
+		    .detail("RangeBegin", keys.begin)
+		    .detail("RangeEnd", keys.end)
+		    .detail("HexRangeBegin", keys.begin.toHex())
+		    .detail("HexRangeEnd", keys.end.toHex())
+		    .backtrace();
+	}
 }
 
 void StorageServerDisk::writeKeyValue(KeyValueRef kv) {

From 371a49884c072e1dae55eb0b73ca9a32c765edc3 Mon Sep 17 00:00:00 2001
From: He Liu <86634338+liquid-helium@users.noreply.github.com>
Date: Sun, 23 Jul 2023 13:50:45 -0700
Subject: [PATCH 185/468] Added logs for backup keyrange data moves. (#10666)

---
 fdbserver/storageserver.actor.cpp | 29 ++++++++++++++++++++++-------
 1 file changed, 22 insertions(+), 7 deletions(-)

diff --git a/fdbserver/storageserver.actor.cpp b/fdbserver/storageserver.actor.cpp
index 3f905f707de..1b108462dc5 100644
--- a/fdbserver/storageserver.actor.cpp
+++ b/fdbserver/storageserver.actor.cpp
@@ -6234,6 +6234,11 @@ ACTOR Future<Void> fetchKeys(StorageServer* data, AddingShard* shard) {
 
 		// Wait for the transferred version (and therefore the shard data) to be committed and durable.
 		wait(data->durableVersion.whenAtLeast(feedTransferredVersion));
+		if (SERVER_KNOBS->SS_BACKUP_KEYS_OP_LOGS && keys.intersects(backupLogKeys)) {
+			TraceEvent("FetchKeyBackupKeysTransferred")
+			    .detail("Range", keys)
+			    .detail("Version", shard->transferredVersion);
+		}
 
 		ASSERT(data->shards[shard->keys.begin]->assigned() &&
 		       data->shards[shard->keys.begin]->keys ==
@@ -6257,6 +6262,12 @@ ACTOR Future<Void> fetchKeys(StorageServer* data, AddingShard* shard) {
 			data->changeFeedRemovals.erase(fetchKeysID);
 		}
 		if (e.code() == error_code_actor_cancelled && !data->shuttingDown && shard->phase >= AddingShard::Fetching) {
+			if (SERVER_KNOBS->SS_BACKUP_KEYS_OP_LOGS && keys.intersects(backupLogKeys)) {
+				TraceEvent("FetchKeyBackupKeysCancelled")
+				    .detail("Range", keys)
+				    .detail("FetchVersion", fetchVersion)
+				    .detail("LatestVersion", data->data().getLatestVersion());
+			}
 			if (shard->phase < AddingShard::FetchingCF) {
 				data->storage.clearRange(keys);
 				++data->counters.kvSystemClearRanges;
@@ -6338,7 +6349,7 @@ void ShardInfo::addMutation(Version version, bool fromFetch, MutationRef const&
 }
 
 enum ChangeServerKeysContext { CSK_UPDATE, CSK_RESTORE, CSK_ASSIGN_EMPTY };
-const char* changeServerKeysContextName[] = { "Update", "Restore" };
+const char* changeServerKeysContextName[] = { "Update", "Restore", "AssignEmpty" };
 
 void changeServerKeys(StorageServer* data,
                       const KeyRangeRef& keys,
@@ -6347,12 +6358,6 @@ void changeServerKeys(StorageServer* data,
                       ChangeServerKeysContext context) {
 	ASSERT(!keys.empty());
 
-	// TraceEvent("ChangeServerKeys", data->thisServerID)
-	//     .detail("KeyBegin", keys.begin)
-	//     .detail("KeyEnd", keys.end)
-	//     .detail("NowAssigned", nowAssigned)
-	//     .detail("Version", version)
-	//     .detail("Context", changeServerKeysContextName[(int)context]);
 	validate(data);
 
 	// TODO(alexmiller): Figure out how to selectively enable spammy data distribution events.
@@ -6361,6 +6366,16 @@ void changeServerKeys(StorageServer* data,
 	bool isDifferent = false;
 	auto existingShards = data->shards.intersectingRanges(keys);
 	for (auto it = existingShards.begin(); it != existingShards.end(); ++it) {
+		if (SERVER_KNOBS->SS_BACKUP_KEYS_OP_LOGS && keys.intersects(backupLogKeys)) {
+			TraceEvent("ChangeServerKeys", data->thisServerID)
+			    .detail("Range", keys)
+			    .detail("NowAssigned", nowAssigned)
+			    .detail("Version", version)
+			    .detail("Context", changeServerKeysContextName[(int)context])
+			    .detail("ExistingRange", KeyRangeRef(it->range()))
+			    .detail("Assigned", it->value()->assigned())
+			    .detail("Readable", it->value()->isReadable());
+		}
 		if (nowAssigned != it->value()->assigned()) {
 			isDifferent = true;
 			TraceEvent("CSKRangeDifferent", data->thisServerID)

From b1b9376d39686d00aa168d6ba6feee93c13dc7c0 Mon Sep 17 00:00:00 2001
From: neethuhaneesha <nbingi@apple.com>
Date: Tue, 25 Jul 2023 11:51:15 -0700
Subject: [PATCH 186/468] Storing the inprocess-commit deleted keysets until
 the commit is complete. (#10672)

---
 fdbserver/KeyValueStoreRocksDB.actor.cpp | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/fdbserver/KeyValueStoreRocksDB.actor.cpp b/fdbserver/KeyValueStoreRocksDB.actor.cpp
index a43c268c905..a92f60fa4a1 100644
--- a/fdbserver/KeyValueStoreRocksDB.actor.cpp
+++ b/fdbserver/KeyValueStoreRocksDB.actor.cpp
@@ -1914,6 +1914,7 @@ struct RocksDBKeyValueStore : IKeyValueStore {
 	Future<Void> openFuture;
 	std::unique_ptr<rocksdb::WriteBatch> writeBatch;
 	std::set<Key> keysSet;
+	std::set<Key> previousCommitKeysSet;
 	// maximum number of single key deletes in a commit, if ROCKSDB_SINGLEKEY_DELETES_ON_CLEARRANGE is enabled.
 	int maxDeletes;
 	Optional<Future<Void>> metrics;
@@ -2156,6 +2157,13 @@ struct RocksDBKeyValueStore : IKeyValueStore {
 						--maxDeletes;
 						it++;
 					}
+					it = previousCommitKeysSet.lower_bound(keyRange.begin);
+					while (it != previousCommitKeysSet.end() && *it < keyRange.end) {
+						writeBatch->Delete(defaultFdbCF, toSlice(*it));
+						++counters.convertedDeleteKeyReqs;
+						--maxDeletes;
+						it++;
+					}
 				}
 			} else {
 				writeBatch->DeleteRange(defaultFdbCF, toSlice(keyRange.begin), toSlice(keyRange.end));
@@ -2191,7 +2199,7 @@ struct RocksDBKeyValueStore : IKeyValueStore {
 		}
 		auto a = new Writer::CommitAction();
 		a->batchToCommit = std::move(writeBatch);
-		keysSet.clear();
+		previousCommitKeysSet = std::move(keysSet);
 		maxDeletes = SERVER_KNOBS->ROCKSDB_SINGLEKEY_DELETES_MAX;
 		auto res = a->done.getFuture();
 		writeThread->post(a);

From 97301c4d6436d881e3df6d27328e138536224027 Mon Sep 17 00:00:00 2001
From: neethuhaneesha <nbingi@apple.com>
Date: Tue, 25 Jul 2023 14:28:16 -0700
Subject: [PATCH 187/468] Deleting rocksdb keysSet after the commit completes.

---
 fdbserver/KeyValueStoreRocksDB.actor.cpp | 20 ++++++++++++--------
 1 file changed, 12 insertions(+), 8 deletions(-)

diff --git a/fdbserver/KeyValueStoreRocksDB.actor.cpp b/fdbserver/KeyValueStoreRocksDB.actor.cpp
index a92f60fa4a1..5b8d6d035b7 100644
--- a/fdbserver/KeyValueStoreRocksDB.actor.cpp
+++ b/fdbserver/KeyValueStoreRocksDB.actor.cpp
@@ -2192,20 +2192,24 @@ struct RocksDBKeyValueStore : IKeyValueStore {
 
 	Future<Void> canCommit() override { return checkRocksdbState(this); }
 
-	Future<Void> commit(bool) override {
+	ACTOR Future<Void> commitInRocksDB(RocksDBKeyValueStore* self) {
 		// If there is nothing to write, don't write.
-		if (writeBatch == nullptr) {
+		if (self->writeBatch == nullptr) {
 			return Void();
 		}
 		auto a = new Writer::CommitAction();
-		a->batchToCommit = std::move(writeBatch);
-		previousCommitKeysSet = std::move(keysSet);
-		maxDeletes = SERVER_KNOBS->ROCKSDB_SINGLEKEY_DELETES_MAX;
-		auto res = a->done.getFuture();
-		writeThread->post(a);
-		return res;
+		a->batchToCommit = std::move(self->writeBatch);
+		self->previousCommitKeysSet = std::move(self->keysSet);
+		self->maxDeletes = SERVER_KNOBS->ROCKSDB_SINGLEKEY_DELETES_MAX;
+		state Future<Void> fut = a->done.getFuture();
+		self->writeThread->post(a);
+		wait(fut);
+		self->previousCommitKeysSet.clear();
+		return Void();
 	}
 
+	Future<Void> commit(bool) override { return commitInRocksDB(this); }
+
 	void checkWaiters(const FlowLock& semaphore, int maxWaiters) {
 		if (semaphore.waiters() > maxWaiters) {
 			++counters.immediateThrottle;

From 1eb644dcc7fc6cbb5214d0bac36597b12f1b4fce Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyu_zhou@apple.com>
Date: Wed, 26 Jul 2023 17:15:36 -0700
Subject: [PATCH 188/468] Add "checkall" for checking \xff\x02/blog/ keys
 [release-7.1] (#10648)

* Add "checkall" for checking \xff\x02/blog/ keys

* Avoid GRV calls for getlocation

* Update comments

* Add non-stopping checking and remove verbose output

* Update checkall command to accept customized range

* Fix format
---
 fdbcli/DebugCommands.actor.cpp | 263 +++++++++++++++++++++++++++++++--
 fdbcli/fdbcli.actor.cpp        |  11 +-
 fdbcli/fdbcli.actor.h          |   4 +-
 3 files changed, 258 insertions(+), 20 deletions(-)

diff --git a/fdbcli/DebugCommands.actor.cpp b/fdbcli/DebugCommands.actor.cpp
index 71dc09d9e1a..7fc8634677d 100644
--- a/fdbcli/DebugCommands.actor.cpp
+++ b/fdbcli/DebugCommands.actor.cpp
@@ -20,33 +20,104 @@
 
 #include "fdbcli/fdbcli.actor.h"
 
+#include "fdbclient/FDBTypes.h"
 #include "fdbclient/NativeAPI.actor.h"
 
 #include "flow/actorcompiler.h" // This must be the last #include.
 
 namespace fdb_cli {
 
+// Gets a version at which to read from the storage servers
+ACTOR Future<Version> getVersion(Database cx) {
+	loop {
+		state Transaction tr(cx);
+		tr.setOption(FDBTransactionOptions::LOCK_AWARE);
+		try {
+			Version version = wait(tr.getReadVersion());
+			return version;
+		} catch (Error& e) {
+			wait(tr.onError(e));
+		}
+	}
+}
+
+// Get a list of storage servers that persist keys within range "kr" from the
+// first commit proxy. Returns false if there is a failure (in this case,
+// keyServersPromise will never be set).
+ACTOR Future<bool> getKeyServers(
+    Database cx,
+    Promise<std::vector<std::pair<KeyRange, std::vector<StorageServerInterface>>>> keyServersPromise,
+    KeyRangeRef kr) {
+	state std::vector<std::pair<KeyRange, std::vector<StorageServerInterface>>> keyServers;
+
+	// Try getting key server locations from the first commit proxy
+	state Future<ErrorOr<GetKeyServerLocationsReply>> keyServerLocationFuture;
+	state Key begin = kr.begin;
+	state Key end = kr.end;
+	state int limitKeyServers = 100;
+	state Span span(deterministicRandom()->randomUniqueID(), "CLI:Check"_loc);
+
+	while (begin < end) {
+		state Reference<CommitProxyInfo> commitProxyInfo =
+		    wait(cx->getCommitProxiesFuture(UseProvisionalProxies::False));
+		keyServerLocationFuture = commitProxyInfo->get(0, &CommitProxyInterface::getKeyServersLocations)
+		                              .getReplyUnlessFailedFor(GetKeyServerLocationsRequest(span.context,
+		                                                                                    Optional<TenantNameRef>(),
+		                                                                                    begin,
+		                                                                                    end,
+		                                                                                    limitKeyServers,
+		                                                                                    false,
+		                                                                                    latestVersion,
+		                                                                                    Arena()),
+		                                                       2,
+		                                                       0);
+
+		state bool keyServersInsertedForThisIteration = false;
+		choose {
+			when(ErrorOr<GetKeyServerLocationsReply> shards = wait(keyServerLocationFuture)) {
+				// Get the list of shards if one was returned.
+				if (shards.present() && !keyServersInsertedForThisIteration) {
+					keyServers.insert(keyServers.end(), shards.get().results.begin(), shards.get().results.end());
+					keyServersInsertedForThisIteration = true;
+					begin = shards.get().results.back().first.end;
+					break;
+				}
+			}
+			when(wait(cx->onProxiesChanged())) {}
+		}
+
+		if (!keyServersInsertedForThisIteration) // Retry the entire workflow
+			wait(delay(1.0));
+	}
+
+	keyServersPromise.send(keyServers);
+	return true;
+}
+
 // The command is used to get all storage server addresses for a given key.
-ACTOR Future<bool> getLocationCommandActor(Database cx, std::vector<StringRef> tokens, Version version) {
-	if (tokens.size() != 2) {
-		printf("getlocation <KEY>\n"
-		       "fetch the storage server address for a given key.\n"
+ACTOR Future<bool> getLocationCommandActor(Database cx, std::vector<StringRef> tokens) {
+	if (tokens.size() != 2 && tokens.size() != 3) {
+		printf("getlocation <KEY> [<KEY2>]\n"
+		       "fetch the storage server address for a given key or range.\n"
 		       "Displays the addresses of storage servers, or `not found' if location is not found.");
 		return false;
 	}
 
-	KeyRangeLocationInfo loc = wait(getKeyLocation_internal(
-	    cx, {}, tokens[1], SpanID(), Optional<UID>(), UseProvisionalProxies::False, Reverse::False, version));
-
-	if (loc.locations) {
-		printf("version is %ld\n", version);
-		printf("`%s' is at:\n", printable(tokens[1]).c_str());
-		auto locations = loc.locations->locations();
-		for (int i = 0; locations && i < locations->size(); i++) {
-			printf("  %s\n", locations->getInterface(i).address().toString().c_str());
+	KeyRange kr = KeyRangeRef(tokens[1], tokens.size() == 3 ? tokens[2] : keyAfter(tokens[1]));
+	// find key range locations without GRV
+	state Promise<std::vector<std::pair<KeyRange, std::vector<StorageServerInterface>>>> keyServersPromise;
+	bool found = wait(getKeyServers(cx, keyServersPromise, kr));
+	if (!found) {
+		printf("[%s, %s]locations not found\n", printable(tokens[1]).c_str(), printable(tokens[2]).c_str());
+		return false;
+	}
+	std::vector<std::pair<KeyRange, std::vector<StorageServerInterface>>> keyServers =
+	    keyServersPromise.getFuture().get();
+	for (const auto& [range, servers] : keyServers) {
+		printf("Key range: %s\n", printable(range).c_str());
+		for (const auto& server : servers) {
+			printf("  %s\n", server.address().toString().c_str());
 		}
-	} else {
-		printf("`%s': location not found\n", printable(tokens[1]).c_str());
 	}
 	return true;
 }
@@ -92,4 +163,166 @@ ACTOR Future<bool> getallCommandActor(Database cx, std::vector<StringRef> tokens
 // hidden commands, no help text for now
 CommandFactory getallCommandFactory("getall");
 
+// check that all replies are the same. Update begin to the next key to check
+bool checkResults(Version version,
+                  const std::vector<StorageServerInterface>& servers,
+                  const std::vector<Future<ErrorOr<GetKeyValuesReply>>>& replies,
+                  KeySelectorRef& begin,
+                  const KeySelectorRef& end) {
+	bool allSame = true;
+	int firstValidServer = -1;
+	for (int j = 0; j < replies.size(); j++) {
+		auto reply = replies[j].get();
+		if (!reply.present() || reply.get().error.present()) {
+			printf("Error from %s: %s\n",
+			       servers[j].address().toString().c_str(),
+			       reply.present() ? reply.get().error.get().what() : "no reply");
+			continue;
+		}
+		GetKeyValuesReply current = reply.get();
+		if (firstValidServer == -1) {
+			firstValidServer = j;
+			continue;
+		}
+		GetKeyValuesReply reference = replies[firstValidServer].get().get();
+
+		// compare the two replicas
+		if (current.data == reference.data && current.more == reference.more) {
+			continue;
+		}
+
+		allSame = false;
+		printf("#%d  %s\n", firstValidServer, servers[firstValidServer].address().toString().c_str());
+		printf("#%d  %s\n", j, servers[j].address().toString().c_str());
+		int currentI = 0, referenceI = 0;
+		while (currentI < current.data.size() || referenceI < reference.data.size()) {
+			if (currentI >= current.data.size()) {
+				printf(" #%d Unique key: %s\n", firstValidServer, printable(reference.data[referenceI].key).c_str());
+				referenceI++;
+			} else if (referenceI >= reference.data.size()) {
+				printf(" #%d Unique key: %s\n", j, printable(current.data[currentI].key).c_str());
+				currentI++;
+			} else {
+				KeyValueRef currentKV = current.data[currentI];
+				KeyValueRef referenceKV = reference.data[referenceI];
+
+				if (currentKV.key == referenceKV.key) {
+					if (currentKV.value != referenceKV.value) {
+						printf(" Value mismatch key: %s\n", printable(currentKV.key).c_str());
+						currentI++;
+						referenceI++;
+					}
+				} else if (currentKV.key < referenceKV.key) {
+					printf(" #%d Unique key: %s\n", j, printable(currentKV.key).c_str());
+					currentI++;
+				} else {
+					printf(" #%d Unique key: %s\n", firstValidServer, printable(referenceKV.key).c_str());
+					referenceI++;
+				}
+			}
+		}
+	}
+
+	if (!allSame)
+		return false;
+
+	if (firstValidServer >= 0 && replies[firstValidServer].get().get().more) {
+		const VectorRef<KeyValueRef>& result = replies[firstValidServer].get().get().data;
+		begin = firstGreaterThan(result[result.size() - 1].key);
+	} else {
+		printf("Same at version %ld\n", version);
+		begin = end; // signal that we're done
+	}
+	return true;
+}
+
+// The command is used to check the inconsistency in a keyspace, default is \xff\x02/blog/ keyspace.
+ACTOR Future<bool> checkallCommandActor(Database cx, std::vector<StringRef> tokens) {
+	// ignore tokens for now
+	state Transaction onErrorTr(cx); // This transaction exists only to access onError and its backoff behavior
+	state int i = 0;
+	state Version version;
+	state KeySelectorRef begin, end;
+	state bool checkAll = false; // do not return on first error, continue checking all keys
+	state KeyRange toCheck = backupLogKeys;
+
+	if (tokens.size() == 2 && tokens[1] == "ALL"_sr) {
+		printf("Checking all keys for corruption...\n");
+		checkAll = true;
+	}
+	if (tokens.size() == 2 && tokens[1] == "help"_sr) {
+		printf("checkall [ALL]|[<KEY> <KEY2>]\n"
+		       "Check inconsistency in a keyspace by comparing all replicas and print any corruptions.\n"
+		       "The default behavior is to stop on the first shard where corruption is found and the\n"
+		       "default keyspace is \\xff\\x02/blog/.\n"
+		       "The default keyspace can be changed by specifying a range. Note this is intended to check\n"
+		       "a small range of keys, not the entire database (consider consistencycheck for that purpose).\n");
+		return false;
+	}
+	if (tokens.size() == 3) {
+		toCheck = KeyRangeRef(tokens[1], tokens[2]);
+	}
+
+	loop {
+		printf("Start checking range: %s\n", printable(toCheck).c_str());
+		try {
+			state Promise<std::vector<std::pair<KeyRange, std::vector<StorageServerInterface>>>> keyServerPromise;
+			bool foundKeyServers = wait(getKeyServers(cx, keyServerPromise, toCheck));
+
+			if (!foundKeyServers) {
+				printf("key server locations for %s not found, retrying in 1s...\n", printable(toCheck).c_str());
+				wait(delay(1.0));
+				continue;
+			}
+
+			state std::vector<std::pair<KeyRange, std::vector<StorageServerInterface>>> keyServers =
+			    keyServerPromise.getFuture().get();
+			for (i = 0; i < keyServers.size(); i++) { // for each key range
+				state KeyRange range = keyServers[i].first;
+				range = range & toCheck;
+				if (range.empty()) {
+					continue;
+				}
+				const auto& servers = keyServers[i].second;
+				begin = firstGreaterOrEqual(range.begin);
+				end = firstGreaterOrEqual(range.end);
+				printf("Key range: %s\n", printable(range).c_str());
+				for (const auto& server : servers) {
+					printf("  %s\n", server.address().toString().c_str());
+				}
+				wait(store(version, getVersion(cx)));
+				state std::vector<Future<ErrorOr<GetKeyValuesReply>>> replies;
+				for (const auto& s : keyServers[i].second) { // for each storage server
+					GetKeyValuesRequest req;
+					req.begin = begin;
+					req.end = end;
+					req.limit = 1e4;
+					req.limitBytes = CLIENT_KNOBS->REPLY_BYTE_LIMIT;
+					req.version = version;
+					req.tags = TagSet();
+
+					replies.push_back(s.getKeyValues.getReplyUnlessFailedFor(req, 2, 0));
+				}
+				// printf("waiting for %lu replies at version: %ld\n", keyServers[i].second.size(), version);
+				wait(waitForAll(replies));
+				if (!checkResults(version, keyServers[i].second, replies, begin, end) && !checkAll) {
+					return false;
+				}
+				if (begin == end) {
+					toCheck = KeyRangeRef(range.end, toCheck.end);
+				}
+				// TODO: if there are more results, continue checking in the same shard
+			}
+		} catch (Error& e) {
+			printf("Retrying for error: %s\n", e.what());
+			wait(onErrorTr.onError(e));
+		}
+		if (toCheck.begin == toCheck.end) {
+			return true;
+		}
+		wait(delay(1.0));
+	}
+}
+
+CommandFactory checkallCommandFactory("checkall");
 } // namespace fdb_cli
\ No newline at end of file
diff --git a/fdbcli/fdbcli.actor.cpp b/fdbcli/fdbcli.actor.cpp
index a1fafc2f214..5c76c170a17 100644
--- a/fdbcli/fdbcli.actor.cpp
+++ b/fdbcli/fdbcli.actor.cpp
@@ -1664,9 +1664,7 @@ ACTOR Future<int> cli(CLIOptions opt, LineNoise* plinenoise) {
 				}
 
 				if (tokencmp(tokens[0], "getlocation")) {
-					Version _v = wait(makeInterruptable(
-					    safeThreadFutureToFuture(getTransaction(db, tenant, tr, options, intrans)->getReadVersion())));
-					bool _result = wait(makeInterruptable(getLocationCommandActor(localDb, tokens, _v)));
+					bool _result = wait(makeInterruptable(getLocationCommandActor(localDb, tokens)));
 					if (!_result)
 						is_error = true;
 					continue;
@@ -1681,6 +1679,13 @@ ACTOR Future<int> cli(CLIOptions opt, LineNoise* plinenoise) {
 					continue;
 				}
 
+				if (tokencmp(tokens[0], "checkall")) {
+					bool _result = wait(makeInterruptable(checkallCommandActor(localDb, tokens)));
+					if (!_result)
+						is_error = true;
+					continue;
+				}
+
 				if (tokencmp(tokens[0], "getversion")) {
 					if (tokens.size() != 1) {
 						printUsage(tokens[0]);
diff --git a/fdbcli/fdbcli.actor.h b/fdbcli/fdbcli.actor.h
index eef4ff021e5..33c98771297 100644
--- a/fdbcli/fdbcli.actor.h
+++ b/fdbcli/fdbcli.actor.h
@@ -216,9 +216,9 @@ ACTOR Future<bool> versionEpochCommandActor(Reference<IDatabase> db, Database cx
 ACTOR Future<bool> targetVersionCommandActor(Reference<IDatabase> db, std::vector<StringRef> tokens);
 
 // debug commands: getlocation, getall
-ACTOR Future<bool> getLocationCommandActor(Database cx, std::vector<StringRef> tokens, Version version);
+ACTOR Future<bool> getLocationCommandActor(Database cx, std::vector<StringRef> tokens);
 ACTOR Future<bool> getallCommandActor(Database cx, std::vector<StringRef> tokens, Version version);
-
+ACTOR Future<bool> checkallCommandActor(Database cx, std::vector<StringRef> tokens);
 } // namespace fdb_cli
 
 #include "flow/unactorcompiler.h"

From cde3432b87f4ed06b3878fb00638ce8b48909d93 Mon Sep 17 00:00:00 2001
From: Hao Fu <77984096+hfu94@users.noreply.github.com>
Date: Thu, 27 Jul 2023 15:53:32 -0700
Subject: [PATCH 189/468] Remove SS entries from RateKeeper once it is down
 (#10688)

---
 fdbserver/Ratekeeper.actor.cpp | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/fdbserver/Ratekeeper.actor.cpp b/fdbserver/Ratekeeper.actor.cpp
index 25475c52882..f78e0120b1f 100644
--- a/fdbserver/Ratekeeper.actor.cpp
+++ b/fdbserver/Ratekeeper.actor.cpp
@@ -175,6 +175,7 @@ class RatekeeperImpl {
 		} catch (...) {
 			// including cancellation
 			self->storageQueueInfo.erase(myQueueInfo);
+			self->healthMetrics.storageStats.erase(ssi.id());
 			throw;
 		}
 	}
@@ -221,8 +222,10 @@ class RatekeeperImpl {
 						a = Future<Void>();
 						a = splitError(trackStorageServerQueueInfo(self, change.second.get()), err);
 					}
-				} else
+				} else {
 					actors.erase(change.first);
+					self->healthMetrics.storageStats.erase(change.first);
+				}
 			}
 			when(wait(err.getFuture())) {}
 		}

From f8a043d74c70eb024d74d071d40851a1df97fba9 Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyu_zhou@apple.com>
Date: Fri, 28 Jul 2023 08:45:44 -0700
Subject: [PATCH 190/468] Add 7.1.34, 7.1.35 release notes (#10696)

---
 .../source/release-notes/release-notes-710.rst    | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/documentation/sphinx/source/release-notes/release-notes-710.rst b/documentation/sphinx/source/release-notes/release-notes-710.rst
index 71954a0e939..53476cdefc3 100644
--- a/documentation/sphinx/source/release-notes/release-notes-710.rst
+++ b/documentation/sphinx/source/release-notes/release-notes-710.rst
@@ -4,6 +4,21 @@
 Release Notes
 #############
 
+7.1.35
+======
+* Same as 7.1.34 release with AVX enabled.
+
+7.1.34
+======
+* Released with AVX disabled.
+* Fixed a high GRV latency issue when many storage servers are recruited.  `(PR #10688) <https://github.com/apple/foundationdb/pull/10688>`_
+* Fixed a single key deletion bug when ROCKSDB_SINGLEKEY_DELETES_ON_CLEARRANGE is enabled. `(PR #10672) <https://github.com/apple/foundationdb/pull/10672>`_ and `(PR #10676) <https://github.com/apple/foundationdb/pull/10676>`_
+* Added degraded and disconnected peer recovery to gray failure detection. `(PR #10565) <https://github.com/apple/foundationdb/pull/10565>`_
+* Fixed RocksDB engine to avoid read timeout checks for eager reads and system keys. `(PR #10500) <https://github.com/apple/foundationdb/pull/10500>`_
+* Fixed backup to work with s3 compatible service.  `(PR #10369) <https://github.com/apple/foundationdb/pull/10369>`_
+* Fixed data loss during multiple range restores. `(PR #10424) <https://github.com/apple/foundationdb/pull/10424>`_
+* Updated RocksDB to version 8.1.1. `(PR #10268) <https://github.com/apple/foundationdb/pull/10268>`_
+
 7.1.33
 ======
 * Same as 7.1.32 release with AVX enabled.

From fa3a6ddf341a14ddd47434d06763175364738882 Mon Sep 17 00:00:00 2001
From: FoundationDB CI <foundationdb_ci@apple.com>
Date: Mon, 31 Jul 2023 10:41:34 +0000
Subject: [PATCH 191/468] disable AVX for 7.1.34 release

---
 cmake/ConfigureCompiler.cmake | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cmake/ConfigureCompiler.cmake b/cmake/ConfigureCompiler.cmake
index e52d8665ee8..7e50469ba6a 100644
--- a/cmake/ConfigureCompiler.cmake
+++ b/cmake/ConfigureCompiler.cmake
@@ -253,7 +253,7 @@ else()
       set(USE_AVX512F OFF)
     endif()
   endif()
-  set(USE_AVX ON CACHE BOOL "Enable AVX instructions")
+  set(USE_AVX OFF CACHE BOOL "Enable AVX instructions")
   if (USE_AVX)
     if (CMAKE_HOST_SYSTEM_PROCESSOR MATCHES "^x86")
       add_compile_options(-mavx)

From 335cdd51d43b28dd88119276d8a192b53044e761 Mon Sep 17 00:00:00 2001
From: FoundationDB CI <foundationdb_ci@apple.com>
Date: Mon, 31 Jul 2023 10:41:48 +0000
Subject: [PATCH 192/468] enable AVX and update version for 7.1.35 release

---
 CMakeLists.txt                | 2 +-
 cmake/ConfigureCompiler.cmake | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 0d274e2c248..67ed468e8f6 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -23,7 +23,7 @@ else()
 endif()
 
 project(foundationdb
-  VERSION 7.1.34
+  VERSION 7.1.35
   DESCRIPTION "FoundationDB is a scalable, fault-tolerant, ordered key-value store with full ACID transactions."
   HOMEPAGE_URL "http://www.foundationdb.org/"
   LANGUAGES C CXX ASM)
diff --git a/cmake/ConfigureCompiler.cmake b/cmake/ConfigureCompiler.cmake
index 7e50469ba6a..e52d8665ee8 100644
--- a/cmake/ConfigureCompiler.cmake
+++ b/cmake/ConfigureCompiler.cmake
@@ -253,7 +253,7 @@ else()
       set(USE_AVX512F OFF)
     endif()
   endif()
-  set(USE_AVX OFF CACHE BOOL "Enable AVX instructions")
+  set(USE_AVX ON CACHE BOOL "Enable AVX instructions")
   if (USE_AVX)
     if (CMAKE_HOST_SYSTEM_PROCESSOR MATCHES "^x86")
       add_compile_options(-mavx)

From 9152ac9bd3ed608adc3821702f6b5c76bda78a75 Mon Sep 17 00:00:00 2001
From: "Johannes M. Scheuermann" <jscheuermann@apple.com>
Date: Mon, 7 Aug 2023 11:02:02 +0200
Subject: [PATCH 193/468] Fix the go tuple tests for new go version

---
 bindings/go/src/fdb/tuple/tuple_test.go | 22 +++++++++++++++++++---
 1 file changed, 19 insertions(+), 3 deletions(-)

diff --git a/bindings/go/src/fdb/tuple/tuple_test.go b/bindings/go/src/fdb/tuple/tuple_test.go
index 91c74a09c7c..7e2bfcfd51d 100644
--- a/bindings/go/src/fdb/tuple/tuple_test.go
+++ b/bindings/go/src/fdb/tuple/tuple_test.go
@@ -11,6 +11,10 @@ import (
 )
 
 var update = flag.Bool("update", false, "update .golden files")
+// Since go 1.20 math/rand uses automatically a random seed: https://tip.golang.org/doc/go1.20.
+// To enfore the old behaviour we intialize the random generator with a hard-coded seed.
+// TODO: Rethink how useful the random generator in those test cases is.
+var randomGenerator *rand.Rand
 
 func loadGolden(t *testing.T) (golden map[string][]byte) {
 	f, err := os.Open("testdata/tuples.golden")
@@ -44,13 +48,25 @@ var testUUID = UUID{
 	0x11, 0x00, 0xAA, 0xBB, 0xCC, 0xDD, 0xEE, 0xFF,
 }
 
+// Helper method to get the deterministic random number generator seeded with the same number.
+// This has changed in go 1.20: https://pkg.go.dev/math/rand@go1.20.7#Seed which would break the current
+// test setup. Once we remove the "random" test set, we can remove this function again.
+func getRandomGenerator() *rand.Rand {
+	if randomGenerator != nil {
+		return randomGenerator
+	}
+	randomGenerator = rand.New(rand.Source(rand.NewSource(1)))
+
+	return randomGenerator
+}
+
 func genBytes() interface{}     { return []byte("namespace") }
 func genBytesNil() interface{}  { return []byte{0xFF, 0x00, 0xFF} }
 func genString() interface{}    { return "namespace" }
 func genStringNil() interface{} { return "nam\x00es\xFFpace" }
-func genInt() interface{}       { return rand.Int63() }
-func genFloat() interface{}     { return float32(rand.NormFloat64()) }
-func genDouble() interface{}    { return rand.NormFloat64() }
+func genInt() interface{}       { return getRandomGenerator().Int63() }
+func genFloat() interface{}     { return float32(getRandomGenerator().NormFloat64()) }
+func genDouble() interface{}    { return getRandomGenerator().NormFloat64() }
 
 func mktuple(gen func() interface{}, count int) Tuple {
 	tt := make(Tuple, count)

From 89d9e34c13d3639b2959fc3731a89f89d60994cc Mon Sep 17 00:00:00 2001
From: Evan Tschannen <evan.tschannen@snowflake.com>
Date: Wed, 2 Aug 2023 09:07:07 -0700
Subject: [PATCH 194/468] fix: when requests are dropped from the queue the
 txnRequestOut becomes inconsistent with txnRequestIn

---
 fdbserver/GrvProxyServer.actor.cpp | 1 +
 1 file changed, 1 insertion(+)

diff --git a/fdbserver/GrvProxyServer.actor.cpp b/fdbserver/GrvProxyServer.actor.cpp
index fc3d9b5d233..ddbdb3d68a2 100644
--- a/fdbserver/GrvProxyServer.actor.cpp
+++ b/fdbserver/GrvProxyServer.actor.cpp
@@ -401,6 +401,7 @@ void proxyGRVThresholdExceeded(const GetReadVersionRequest* req, GrvProxyStats*
 // Drop a GetReadVersion request from a queue, by responding an error to the request.
 void dropRequestFromQueue(Deque<GetReadVersionRequest>* queue, GrvProxyStats* stats) {
 	proxyGRVThresholdExceeded(&queue->front(), stats);
+	++stats->txnRequestOut;
 	queue->pop_front();
 }
 

From d8a4ded109eecf69fcaf19549089015b028e41d6 Mon Sep 17 00:00:00 2001
From: Zhe Wang <zhe.wang@wustl.edu>
Date: Tue, 8 Aug 2023 11:19:54 -0700
Subject: [PATCH 195/468] Add consistency check for rocksdb only

---
 fdbclient/ServerKnobs.cpp                     |  1 +
 fdbclient/ServerKnobs.h                       |  3 +
 .../workloads/ConsistencyCheck.actor.cpp      | 77 +++++++++++++++++++
 3 files changed, 81 insertions(+)

diff --git a/fdbclient/ServerKnobs.cpp b/fdbclient/ServerKnobs.cpp
index 78f8e4bf90d..d2d827bb127 100644
--- a/fdbclient/ServerKnobs.cpp
+++ b/fdbclient/ServerKnobs.cpp
@@ -802,6 +802,7 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi
 	init( PEER_DEGRADATION_CONNECTION_FAILURE_COUNT,               5 );
 	init( WORKER_HEALTH_REPORT_RECENT_DESTROYED_PEER,           true );
 	init( STORAGE_SERVER_REBOOT_ON_IO_TIMEOUT,                 false ); if ( randomize && BUGGIFY ) STORAGE_SERVER_REBOOT_ON_IO_TIMEOUT = true;
+	init( CONSISTENCY_CHECK_SPECIFIC_ENGINE,                       0 ); // disable by default
 
 	// Test harness
 	init( WORKER_POLL_DELAY,                                     1.0 );
diff --git a/fdbclient/ServerKnobs.h b/fdbclient/ServerKnobs.h
index a2795d875ee..a00aab10071 100644
--- a/fdbclient/ServerKnobs.h
+++ b/fdbclient/ServerKnobs.h
@@ -740,6 +740,9 @@ class ServerKnobs : public KnobsImpl<ServerKnobs> {
 	                                          // Enabling this can reduce toil of manually restarting the SS.
 	                                          // Enable with caution: If io_timeout is caused by disk failure, we won't
 	                                          // want to restart the SS, which increases risk of data corruption.
+	int CONSISTENCY_CHECK_SPECIFIC_ENGINE; // When set, consistency check only check data corruption for a
+	                                       // shard which is in at least one SS with the given storage engine.
+	                                       // 1: rocksdb; 2: sqlite; Other values: disable specific check;
 
 	// Test harness
 	double WORKER_POLL_DELAY;
diff --git a/fdbserver/workloads/ConsistencyCheck.actor.cpp b/fdbserver/workloads/ConsistencyCheck.actor.cpp
index ca4181e7d82..ccddfe3665c 100644
--- a/fdbserver/workloads/ConsistencyCheck.actor.cpp
+++ b/fdbserver/workloads/ConsistencyCheck.actor.cpp
@@ -1131,6 +1131,38 @@ struct ConsistencyCheckWorkload : TestWorkload {
 		return std::make_pair(estimatedBytes, metrics);
 	}
 
+	ACTOR Future<std::unordered_map<UID, KeyValueStoreType>> getStorageType(
+	    std::vector<StorageServerInterface> storageServers) {
+		state std::vector<Future<ErrorOr<KeyValueStoreType>>> storageTypeFutures;
+		state std::unordered_map<UID, KeyValueStoreType> res;
+
+		try {
+			for (int i = 0; i < storageServers.size(); i++) {
+				ReplyPromise<KeyValueStoreType> typeReply;
+				storageTypeFutures.push_back(
+				    storageServers[i].getKeyValueStoreType.getReplyUnlessFailedFor(typeReply, 2, 0));
+			}
+			wait(waitForAll(storageTypeFutures));
+
+			for (int i = 0; i < storageServers.size(); i++) {
+				ErrorOr<KeyValueStoreType> reply = storageTypeFutures[i].get();
+				if (!reply.present()) {
+					TraceEvent(SevWarn, "ConsistencyCheck_FailedToGetStorageType")
+					    .error(reply.getError())
+					    .detail("StorageServer", storageServers[i].id())
+					    .detail("IsTSS", storageServers[i].isTss() ? "True" : "False");
+				} else {
+					res[storageServers[i].id()] = reply.get();
+				}
+			}
+		} catch (Error& e) {
+			TraceEvent("ConsistencyCheck_ErrorGetStorageType").errorUnsuppressed(e);
+			res.clear();
+		}
+
+		return res;
+	}
+
 	// Comparison function used to compare map elements by value
 	template <class K, class T>
 	static bool compareByValue(std::pair<K, T> a, std::pair<K, T> b) {
@@ -1183,6 +1215,8 @@ struct ConsistencyCheckWorkload : TestWorkload {
 		state double rateLimiterStartTime = now();
 		state int64_t bytesReadInthisRound = 0;
 		state bool testResult = true;
+		state int64_t numCompleteShards = 0;
+		state int64_t numSkippedShards = 0;
 
 		state double dbSize = 100e12;
 		if (g_network->isSimulated()) {
@@ -1293,6 +1327,40 @@ struct ConsistencyCheckWorkload : TestWorkload {
 				}
 			}
 
+			if (SERVER_KNOBS->CONSISTENCY_CHECK_SPECIFIC_ENGINE == 1 ||
+			    SERVER_KNOBS->CONSISTENCY_CHECK_SPECIFIC_ENGINE == 2) {
+				state KeyValueStoreType storageEngineTypeToCheck = SERVER_KNOBS->CONSISTENCY_CHECK_SPECIFIC_ENGINE == 1
+				                                                       ? KeyValueStoreType::StoreType::SSD_ROCKSDB_V1
+				                                                       : KeyValueStoreType::StoreType::SSD_BTREE_V2;
+				std::unordered_map<UID, KeyValueStoreType> storageTypeMapping =
+				    wait(self->getStorageType(storageServerInterfaces));
+
+				bool anySpecificEngine = false;
+				for (int i = 0; i < storageServers.size(); i++) {
+					auto ssStorageType = storageTypeMapping.find(storageServers[i]);
+					if (ssStorageType != storageTypeMapping.end()) {
+						if (ssStorageType->second == storageEngineTypeToCheck) {
+							anySpecificEngine = true;
+							break;
+						}
+					} else {
+						// We do not get the type for this SS
+					}
+				}
+				if (!anySpecificEngine) {
+					TraceEvent(SevInfo, "ConsistencyCheck_ShardSkipped")
+					    .suppressFor(1.0)
+					    .detail("Reason", "No SSD_ROCKSDB_V1 engine has the shard")
+					    .detail("Range", range.toString())
+					    .detail("ShardSeqNumber", i)
+					    .detail("ShardCount", ranges.size())
+					    .detail("NumSkippedShards", numSkippedShards)
+					    .detail("NumCompletedShards", numCompleteShards);
+					numSkippedShards++;
+					continue;
+				}
+			}
+
 			std::pair<std::vector<int64_t>, StorageMetrics> estimatedBytesAndMetrics =
 			    wait(self->getStorageSizeEstimate(storageServerInterfaces, range));
 			state std::vector<int64_t> estimatedBytes = estimatedBytesAndMetrics.first;
@@ -1736,6 +1804,15 @@ struct ConsistencyCheckWorkload : TestWorkload {
 				    .detail("Range", range)
 				    .detail("BytesRead", bytesReadInRange);
 			}
+
+			numCompleteShards++;
+			TraceEvent(SevInfo, "ConsistencyCheck_ShardComplete")
+			    .suppressFor(1.0)
+			    .detail("Range", range.toString())
+			    .detail("ShardCount", ranges.size())
+			    .detail("NumCompletedShards", numCompleteShards)
+			    .detail("BytesReadInThisRound", bytesReadInthisRound)
+			    .detail("NumSkippedShards", numSkippedShards);
 		}
 
 		// SOMEDAY: when background data distribution is implemented, include this test

From bb68c0de24b05f79137ef0d080a0bc815760fab9 Mon Sep 17 00:00:00 2001
From: Zhe Wang <zhe.wang@wustl.edu>
Date: Tue, 8 Aug 2023 18:35:32 -0700
Subject: [PATCH 196/468] address comments

---
 fdbclient/ServerKnobs.h                        |  3 +++
 fdbserver/workloads/ConsistencyCheck.actor.cpp | 13 +++++--------
 2 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/fdbclient/ServerKnobs.h b/fdbclient/ServerKnobs.h
index ac65d9e80b5..7715673f27f 100644
--- a/fdbclient/ServerKnobs.h
+++ b/fdbclient/ServerKnobs.h
@@ -744,6 +744,9 @@ class ServerKnobs : public KnobsImpl<ServerKnobs> {
 	                                       // shard which is in at least one SS with the rocksdb engine.
 	bool CONSISTENCY_CHECK_SQLITE_ENGINE; // When set, consistency check only check data corruption for a
 	                                      // shard which is in at least one SS with the sqlite engine.
+	                                      // When both CONSISTENCY_CHECK_ROCKSDB_ENGINE and
+	                                      // CONSISTENCY_CHECK_SQLITE_ENGINE are set, consistency check only checks for
+	                                      // the rocksdb engine.
 
 	// Test harness
 	double WORKER_POLL_DELAY;
diff --git a/fdbserver/workloads/ConsistencyCheck.actor.cpp b/fdbserver/workloads/ConsistencyCheck.actor.cpp
index fa7f0542d3a..7ce188a2901 100644
--- a/fdbserver/workloads/ConsistencyCheck.actor.cpp
+++ b/fdbserver/workloads/ConsistencyCheck.actor.cpp
@@ -1331,17 +1331,14 @@ struct ConsistencyCheckWorkload : TestWorkload {
 				std::unordered_map<UID, KeyValueStoreType> storageTypeMapping =
 				    wait(self->getStorageType(storageServerInterfaces));
 
+				KeyValueStoreType storageEngineToCheck = SERVER_KNOBS->CONSISTENCY_CHECK_ROCKSDB_ENGINE
+				                                             ? KeyValueStoreType::StoreType::SSD_ROCKSDB_V1
+				                                             : KeyValueStoreType::StoreType::SSD_BTREE_V2;
 				bool anySpecifiedEngine = false;
 				for (int j = 0; j < storageServers.size(); j++) {
 					auto ssStorageType = storageTypeMapping.find(storageServers[j]);
 					if (ssStorageType != storageTypeMapping.end()) {
-						if (SERVER_KNOBS->CONSISTENCY_CHECK_ROCKSDB_ENGINE &&
-						    ssStorageType->second == KeyValueStoreType::StoreType::SSD_ROCKSDB_V1) {
-							anySpecifiedEngine = true;
-							break;
-						}
-						if (SERVER_KNOBS->CONSISTENCY_CHECK_SQLITE_ENGINE &&
-						    ssStorageType->second == KeyValueStoreType::StoreType::SSD_BTREE_V2) {
+						if (ssStorageType->second == storageEngineToCheck) {
 							anySpecifiedEngine = true;
 							break;
 						}
@@ -1350,7 +1347,7 @@ struct ConsistencyCheckWorkload : TestWorkload {
 				if (!anySpecifiedEngine) {
 					TraceEvent(SevInfo, "ConsistencyCheck_ShardSkipped")
 					    .suppressFor(1.0)
-					    .detail("Reason", "No SSD_ROCKSDB_V1 engine has the shard")
+					    .detail("Reason", "No specified storage engine has the shard")
 					    .detail("Range", range.toString())
 					    .detail("ShardSeqNumber", i)
 					    .detail("ShardCount", ranges.size())

From 8c2a8386e84e376fb117945deb3c5754a5227ff9 Mon Sep 17 00:00:00 2001
From: w41ter <w41ter.l@gmail.com>
Date: Wed, 9 Aug 2023 11:07:04 +0000
Subject: [PATCH 197/468] Avoid to send request to empty resource

---
 fdbclient/S3BlobStore.actor.cpp | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/fdbclient/S3BlobStore.actor.cpp b/fdbclient/S3BlobStore.actor.cpp
index 9c1c54704e7..612b3e23c82 100644
--- a/fdbclient/S3BlobStore.actor.cpp
+++ b/fdbclient/S3BlobStore.actor.cpp
@@ -782,6 +782,11 @@ ACTOR Future<Reference<HTTP::Response>> doRequest_impl(Reference<S3BlobStoreEndp
 	headers["Host"] = bstore->host;
 	headers["Accept"] = "application/xml";
 
+	// Avoid to send request with an empty resouce.
+	if (resource.empty()) {
+		resource = "/";
+	}
+
 	// Merge extraHeaders into headers
 	for (const auto& [k, v] : bstore->extraHeaders) {
 		std::string& fieldValue = headers[k];

From 447c11245b37a9c1cfa26a415827cb3affe5e60e Mon Sep 17 00:00:00 2001
From: Zhe Wu <halfprice@users.noreply.github.com>
Date: Tue, 8 Aug 2023 15:14:41 -0700
Subject: [PATCH 198/468] Simulate connection failure in simulation (#10718)

Co-authored-by: Jingyu Zhou <jingyu_zhou@apple.com>
---
 fdbrpc/sim2.actor.cpp                  | 53 +++++++++++++++++++++++++-
 fdbrpc/simulator.h                     |  2 +
 fdbserver/workloads/ClogTlog.actor.cpp | 20 ++++++++--
 3 files changed, 70 insertions(+), 5 deletions(-)

diff --git a/fdbrpc/sim2.actor.cpp b/fdbrpc/sim2.actor.cpp
index e63a2568286..c725de9b18f 100644
--- a/fdbrpc/sim2.actor.cpp
+++ b/fdbrpc/sim2.actor.cpp
@@ -150,6 +150,16 @@ struct SimClogging {
 		return t - tnow;
 	}
 
+	bool disconnected(const IPAddress& from, const IPAddress& to) {
+		auto pair = std::make_pair(from, to);
+		if (g_simulator->speedUpSimulation || disconnectPairUntil.find(pair) == disconnectPairUntil.end()) {
+			return false;
+		}
+
+		double disconnectUntil = disconnectPairUntil[pair];
+		return now() < disconnectUntil;
+	}
+
 	void clogPairFor(const IPAddress& from, const IPAddress& to, double t) {
 		auto& u = clogPairUntil[std::make_pair(from, to)];
 		u = std::max(u, now() + t);
@@ -183,11 +193,22 @@ struct SimClogging {
 		return i->second;
 	}
 
+	void disconnectPairFor(const IPAddress& from, const IPAddress& to, double t) {
+		auto& u = disconnectPairUntil[std::make_pair(from, to)];
+		u = std::max(u, now() + t);
+	}
+
+	void reconnectPair(const IPAddress& from, const IPAddress& to) {
+		disconnectPairUntil.erase(std::make_pair(from, to));
+	}
+
 private:
 	std::map<IPAddress, double> clogSendUntil, clogRecvUntil;
 	std::map<std::pair<IPAddress, IPAddress>, double> clogPairUntil;
 	std::map<std::pair<IPAddress, IPAddress>, double> clogPairLatency;
 	std::map<std::pair<NetworkAddress, NetworkAddress>, double> clogProcessPairUntil;
+	std::map<std::pair<IPAddress, IPAddress>, double> disconnectPairUntil;
+
 	double halfLatency() const {
 		double a = deterministicRandom()->random01();
 		const double pFast = 0.999;
@@ -235,6 +256,14 @@ struct Sim2Conn final : IConnection, ReferenceCounted<Sim2Conn> {
 		                               peerProcess->childs.end(),
 		                               [&](ISimulator::ProcessInfo* child) { return child && child == process; });
 
+		if (g_clogging.disconnected(process->address.ip, peerProcess->address.ip)) {
+			TraceEvent("SimulatedDisconnection")
+			    .detail("Phase", "Connect")
+			    .detail("Address", process->address)
+			    .detail("Peer", peerProcess->address);
+			throw connection_failed();
+		}
+
 		TraceEvent("Sim2Connection")
 		    .detail("SendBufSize", sendBufSize)
 		    .detail("Latency", latency)
@@ -372,16 +401,26 @@ struct Sim2Conn final : IConnection, ReferenceCounted<Sim2Conn> {
 			while (self->sentBytes.get() == self->receivedBytes.get())
 				wait(self->sentBytes.onChange());
 			ASSERT(g_simulator.getCurrentProcess() == self->peerProcess);
+
+			// Simulated network disconnection. Make sure to only throw connection_failed() on the sender process.
+			if (g_clogging.disconnected(self->peerProcess->address.ip, self->process->address.ip)) {
+				TraceEvent("SimulatedDisconnection")
+				    .detail("Phase", "DataTransfer")
+				    .detail("Sender", self->peerProcess->address)
+				    .detail("Receiver", self->process->address);
+				throw connection_failed();
+			}
+
 			state int64_t pos =
 			    deterministicRandom()->random01() < .5
 			        ? self->sentBytes.get()
 			        : deterministicRandom()->randomInt64(self->receivedBytes.get(), self->sentBytes.get() + 1);
 			wait(delay(g_clogging.getSendDelay(
-			    self->process->address, self->peerProcess->address, self->isStableConnection())));
+			    self->peerProcess->address, self->process->address, self->isStableConnection())));
 			wait(g_simulator.onProcess(self->process));
 			ASSERT(g_simulator.getCurrentProcess() == self->process);
 			wait(delay(g_clogging.getRecvDelay(
-			    self->process->address, self->peerProcess->address, self->isStableConnection())));
+			    self->peerProcess->address, self->process->address, self->isStableConnection())));
 			ASSERT(g_simulator.getCurrentProcess() == self->process);
 			if (self->stopReceive.isReady()) {
 				wait(Future<Void>(Never()));
@@ -2074,6 +2113,16 @@ class Sim2 final : public ISimulator, public INetworkConnections {
 		g_clogging.unclogPair(from, to);
 	}
 
+	void disconnectPair(const IPAddress& from, const IPAddress& to, double seconds) override {
+		TraceEvent("DisconnectPair").detail("From", from).detail("To", to).detail("Seconds", seconds);
+		g_clogging.disconnectPairFor(from, to, seconds);
+	}
+
+	void reconnectPair(const IPAddress& from, const IPAddress& to) override {
+		TraceEvent("ReconnectPair").detail("From", from).detail("To", to);
+		g_clogging.reconnectPair(from, to);
+	}
+		
 	std::vector<ProcessInfo*> getAllProcesses() const override {
 		std::vector<ProcessInfo*> processes;
 		for (auto& c : machines) {
diff --git a/fdbrpc/simulator.h b/fdbrpc/simulator.h
index deceec32ebc..19ad57ddf7f 100644
--- a/fdbrpc/simulator.h
+++ b/fdbrpc/simulator.h
@@ -403,6 +403,8 @@ class ISimulator : public INetwork {
 	virtual void clogInterface(const IPAddress& ip, double seconds, ClogMode mode = ClogDefault) = 0;
 	virtual void clogPair(const IPAddress& from, const IPAddress& to, double seconds) = 0;
 	virtual void unclogPair(const IPAddress& from, const IPAddress& to) = 0;
+	virtual void disconnectPair(const IPAddress& from, const IPAddress& to, double seconds) = 0;
+	virtual void reconnectPair(const IPAddress& from, const IPAddress& to) = 0;
 	virtual std::vector<ProcessInfo*> getAllProcesses() const = 0;
 	virtual ProcessInfo* getProcessByAddress(NetworkAddress const& address) = 0;
 	virtual MachineInfo* getMachineByNetworkAddress(NetworkAddress const& address) = 0;
diff --git a/fdbserver/workloads/ClogTlog.actor.cpp b/fdbserver/workloads/ClogTlog.actor.cpp
index 7e8ebe01daf..46980a4415c 100644
--- a/fdbserver/workloads/ClogTlog.actor.cpp
+++ b/fdbserver/workloads/ClogTlog.actor.cpp
@@ -37,6 +37,7 @@ struct ClogTlogWorkload : TestWorkload {
 	bool enabled;
 	double testDuration;
 	bool clogged = false;
+	bool useDisconnection = false;
 	Optional<NetworkAddress> tlog; // the tlog to be clogged with all other processes except the CC
 	std::vector<std::pair<IPAddress, IPAddress>> cloggedPairs;
 
@@ -103,8 +104,13 @@ struct ClogTlogWorkload : TestWorkload {
 		// clog pairs
 		for (const auto& ip : ips) {
 			if (ip != tlog.get().ip && ip != cc) {
-				g_simulator.clogPair(ip, tlog.get().ip, seconds);
-				g_simulator.clogPair(tlog.get().ip, ip, seconds);
+				if (useDisconnection) {
+					g_simulator.disconnectPair(ip, tlog.get().ip, seconds);
+					g_simulator.disconnectPair(tlog.get().ip, ip, seconds);
+				} else {
+					g_simulator.clogPair(ip, tlog.get().ip, seconds);
+					g_simulator.clogPair(tlog.get().ip, ip, seconds);
+				}
 				cloggedPairs.emplace_back(ip, tlog.get().ip);
 				cloggedPairs.emplace_back(tlog.get().ip, ip);
 			}
@@ -115,7 +121,11 @@ struct ClogTlogWorkload : TestWorkload {
 	void unclogAll() {
 		// unclog previously clogged connections
 		for (const auto& pair : cloggedPairs) {
-			g_simulator.unclogPair(pair.first, pair.second);
+			if (useDisconnection) {
+				g_simulator.reconnectPair(pair.first, pair.second);
+			} else {
+				g_simulator.unclogPair(pair.first, pair.second);
+			}
 		}
 		cloggedPairs.clear();
 	}
@@ -145,6 +155,10 @@ struct ClogTlogWorkload : TestWorkload {
 	}
 
 	ACTOR Future<Void> clogClient(ClogTlogWorkload* self, Database cx) {
+		if (deterministicRandom()->coinflip()) {
+			self->useDisconnection = true;
+		}
+
 		// Let cycle workload issue some transactions.
 		wait(delay(20.0));
 

From fb84ff872b6d706b972863da5fdbab1ff76b064c Mon Sep 17 00:00:00 2001
From: Zhe Wu <halfprice@users.noreply.github.com>
Date: Thu, 10 Aug 2023 13:59:47 -0700
Subject: [PATCH 199/468] Fix merge conflict

---
 fdbrpc/sim2.actor.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/fdbrpc/sim2.actor.cpp b/fdbrpc/sim2.actor.cpp
index c725de9b18f..e21a304077e 100644
--- a/fdbrpc/sim2.actor.cpp
+++ b/fdbrpc/sim2.actor.cpp
@@ -152,7 +152,7 @@ struct SimClogging {
 
 	bool disconnected(const IPAddress& from, const IPAddress& to) {
 		auto pair = std::make_pair(from, to);
-		if (g_simulator->speedUpSimulation || disconnectPairUntil.find(pair) == disconnectPairUntil.end()) {
+		if (g_simulator.speedUpSimulation || disconnectPairUntil.find(pair) == disconnectPairUntil.end()) {
 			return false;
 		}
 

From b74db7f344b4772b6278a118df26b64e2b434621 Mon Sep 17 00:00:00 2001
From: Zhe Wu <halfprice@users.noreply.github.com>
Date: Thu, 10 Aug 2023 14:04:47 -0700
Subject: [PATCH 200/468] Clear format error

---
 fdbrpc/sim2.actor.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/fdbrpc/sim2.actor.cpp b/fdbrpc/sim2.actor.cpp
index e21a304077e..bb8d97a7538 100644
--- a/fdbrpc/sim2.actor.cpp
+++ b/fdbrpc/sim2.actor.cpp
@@ -2122,7 +2122,7 @@ class Sim2 final : public ISimulator, public INetworkConnections {
 		TraceEvent("ReconnectPair").detail("From", from).detail("To", to);
 		g_clogging.reconnectPair(from, to);
 	}
-		
+
 	std::vector<ProcessInfo*> getAllProcesses() const override {
 		std::vector<ProcessInfo*> processes;
 		for (auto& c : machines) {

From de564b77276d2721766e8053f7c6cdda0a8c0a7b Mon Sep 17 00:00:00 2001
From: FoundationDB CI <foundationdb_ci@apple.com>
Date: Fri, 11 Aug 2023 11:05:12 +0000
Subject: [PATCH 201/468] update version after 7.1.35 release

---
 CMakeLists.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 67ed468e8f6..78079683851 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -23,7 +23,7 @@ else()
 endif()
 
 project(foundationdb
-  VERSION 7.1.35
+  VERSION 7.1.36
   DESCRIPTION "FoundationDB is a scalable, fault-tolerant, ordered key-value store with full ACID transactions."
   HOMEPAGE_URL "http://www.foundationdb.org/"
   LANGUAGES C CXX ASM)

From 59651dee6ec7234344cf8c661d22430d91ed055d Mon Sep 17 00:00:00 2001
From: FoundationDB CI <foundationdb_ci@apple.com>
Date: Fri, 11 Aug 2023 11:06:44 +0000
Subject: [PATCH 202/468] disable AVX for 7.1.36 release

---
 cmake/ConfigureCompiler.cmake | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cmake/ConfigureCompiler.cmake b/cmake/ConfigureCompiler.cmake
index e52d8665ee8..7e50469ba6a 100644
--- a/cmake/ConfigureCompiler.cmake
+++ b/cmake/ConfigureCompiler.cmake
@@ -253,7 +253,7 @@ else()
       set(USE_AVX512F OFF)
     endif()
   endif()
-  set(USE_AVX ON CACHE BOOL "Enable AVX instructions")
+  set(USE_AVX OFF CACHE BOOL "Enable AVX instructions")
   if (USE_AVX)
     if (CMAKE_HOST_SYSTEM_PROCESSOR MATCHES "^x86")
       add_compile_options(-mavx)

From f2742562ee282a5ca36385ad1072eb1c93fdc429 Mon Sep 17 00:00:00 2001
From: FoundationDB CI <foundationdb_ci@apple.com>
Date: Fri, 11 Aug 2023 11:06:56 +0000
Subject: [PATCH 203/468] enable AVX and update version for 7.1.37 release

---
 CMakeLists.txt                | 2 +-
 cmake/ConfigureCompiler.cmake | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 78079683851..9026012a277 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -23,7 +23,7 @@ else()
 endif()
 
 project(foundationdb
-  VERSION 7.1.36
+  VERSION 7.1.37
   DESCRIPTION "FoundationDB is a scalable, fault-tolerant, ordered key-value store with full ACID transactions."
   HOMEPAGE_URL "http://www.foundationdb.org/"
   LANGUAGES C CXX ASM)
diff --git a/cmake/ConfigureCompiler.cmake b/cmake/ConfigureCompiler.cmake
index 7e50469ba6a..e52d8665ee8 100644
--- a/cmake/ConfigureCompiler.cmake
+++ b/cmake/ConfigureCompiler.cmake
@@ -253,7 +253,7 @@ else()
       set(USE_AVX512F OFF)
     endif()
   endif()
-  set(USE_AVX OFF CACHE BOOL "Enable AVX instructions")
+  set(USE_AVX ON CACHE BOOL "Enable AVX instructions")
   if (USE_AVX)
     if (CMAKE_HOST_SYSTEM_PROCESSOR MATCHES "^x86")
       add_compile_options(-mavx)

From b538fd95b8312e7f2b210c90996070c2995c8d0c Mon Sep 17 00:00:00 2001
From: FoundationDB CI <foundationdb_ci@apple.com>
Date: Fri, 11 Aug 2023 21:47:31 +0000
Subject: [PATCH 204/468] update version after 7.1.37 release

---
 CMakeLists.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 9026012a277..0c689139fe0 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -23,7 +23,7 @@ else()
 endif()
 
 project(foundationdb
-  VERSION 7.1.37
+  VERSION 7.1.38
   DESCRIPTION "FoundationDB is a scalable, fault-tolerant, ordered key-value store with full ACID transactions."
   HOMEPAGE_URL "http://www.foundationdb.org/"
   LANGUAGES C CXX ASM)

From d8639c39f4ad0a6c7c515461875dc166f270fc31 Mon Sep 17 00:00:00 2001
From: walter <w41ter.l@gmail.com>
Date: Wed, 16 Aug 2023 01:18:21 +0800
Subject: [PATCH 205/468] Fix compatibility issue of s3 backup [release-7.1]
 (#10774)

* Fix compatibility issue of s3 backup

1. For v4 signature, http request path should be encoded, currently only parameters are encoded.
2. For v2 signature, http request date header should be in form of either the http date header or x-amz-date, currently in form of ISO 8601
3. Empty response header value cause http_bad_response error.
4. Skip verifying MD5 for range get request and if http response code 206 is returned.

* Add missing header file

---------

Co-authored-by: Zhanwei Wang <wang@hashdata.cn>
---
 fdbclient/ClientKnobs.cpp       |  1 +
 fdbclient/ClientKnobs.h         |  1 +
 fdbclient/S3BlobStore.actor.cpp | 94 ++++++++++++++++++++++-----------
 fdbrpc/HTTP.actor.cpp           | 47 +++++++++++------
 4 files changed, 96 insertions(+), 47 deletions(-)

diff --git a/fdbclient/ClientKnobs.cpp b/fdbclient/ClientKnobs.cpp
index a0c9628c50f..13f2d9d482b 100644
--- a/fdbclient/ClientKnobs.cpp
+++ b/fdbclient/ClientKnobs.cpp
@@ -208,6 +208,7 @@ void ClientKnobs::initialize(Randomize randomize) {
 	init( IS_ACCEPTABLE_DELAY,                     1.5 );
 
 	init( HTTP_REQUEST_AWS_V4_HEADER,             true );
+	init( HTTP_RESPONSE_SKIP_VERIFY_CHECKSUM_FOR_PARTIAL_CONTENT, false );
 	init( BLOBSTORE_ENCRYPTION_TYPE,                "" );
 	init( BLOBSTORE_CONNECT_TRIES,                  10 );
 	init( BLOBSTORE_CONNECT_TIMEOUT,                10 );
diff --git a/fdbclient/ClientKnobs.h b/fdbclient/ClientKnobs.h
index 30b664f49a3..aa001974cac 100644
--- a/fdbclient/ClientKnobs.h
+++ b/fdbclient/ClientKnobs.h
@@ -220,6 +220,7 @@ class ClientKnobs : public KnobsImpl<ClientKnobs> {
 	double CSI_STATUS_DELAY;
 
 	bool HTTP_REQUEST_AWS_V4_HEADER; // setting this knob to true will enable AWS V4 style header.
+	bool HTTP_RESPONSE_SKIP_VERIFY_CHECKSUM_FOR_PARTIAL_CONTENT; // skip verify md5 checksum for 206 response
 	std::string BLOBSTORE_ENCRYPTION_TYPE;
 	int BLOBSTORE_CONNECT_TRIES;
 	int BLOBSTORE_CONNECT_TIMEOUT;
diff --git a/fdbclient/S3BlobStore.actor.cpp b/fdbclient/S3BlobStore.actor.cpp
index 612b3e23c82..446c8792f86 100644
--- a/fdbclient/S3BlobStore.actor.cpp
+++ b/fdbclient/S3BlobStore.actor.cpp
@@ -378,9 +378,9 @@ std::string S3BlobStoreEndpoint::getResourceURL(std::string resource, std::strin
 			params.append("&");
 		}
 		params.append("header=");
-		params.append(HTTP::urlEncode(k));
+		params.append(k);
 		params.append(":");
-		params.append(HTTP::urlEncode(v));
+		params.append(v);
 	}
 
 	if (!params.empty())
@@ -766,6 +766,42 @@ void S3BlobStoreEndpoint::returnConnection(ReusableConnection& rconn) {
 	rconn.conn = Reference<IConnection>();
 }
 
+std::string awsCanonicalURI(const std::string& resource, std::vector<std::string>& queryParameters, bool isV4) {
+	StringRef resourceRef(resource);
+	resourceRef.eat("/");
+	std::string canonicalURI("/" + resourceRef.toString());
+	size_t q = canonicalURI.find_last_of('?');
+	if (q != canonicalURI.npos)
+		canonicalURI.resize(q);
+	if (isV4) {
+		canonicalURI = HTTP::awsV4URIEncode(canonicalURI, false);
+	} else {
+		canonicalURI = HTTP::urlEncode(canonicalURI);
+	}
+
+	// Create the canonical query string
+	std::string queryString;
+	q = resource.find_last_of('?');
+	if (q != queryString.npos)
+		queryString = resource.substr(q + 1);
+
+	StringRef qStr(queryString);
+	StringRef queryParameter;
+	while ((queryParameter = qStr.eat("&")) != StringRef()) {
+		StringRef param = queryParameter.eat("=");
+		StringRef value = queryParameter.eat();
+
+		if (isV4) {
+			queryParameters.push_back(HTTP::awsV4URIEncode(param.toString(), true) + "=" +
+			                          HTTP::awsV4URIEncode(value.toString(), true));
+		} else {
+			queryParameters.push_back(HTTP::urlEncode(param.toString()) + "=" + HTTP::urlEncode(value.toString()));
+		}
+	}
+
+	return canonicalURI;
+}
+
 // Do a request, get a Response.
 // Request content is provided as UnsentPacketQueue *pContent which will be depleted as bytes are sent but the queue
 // itself must live for the life of this actor and be destroyed by the caller
@@ -814,6 +850,7 @@ ACTOR Future<Reference<HTTP::Response>> doRequest_impl(Reference<S3BlobStoreEndp
 		state Optional<NetworkAddress> remoteAddress;
 		state bool connectionEstablished = false;
 		state Reference<HTTP::Response> r;
+		state std::string canonicalURI = resource;
 
 		try {
 			// Start connecting
@@ -850,15 +887,23 @@ ACTOR Future<Reference<HTTP::Response>> doRequest_impl(Reference<S3BlobStoreEndp
 				bstore->setAuthHeaders(verb, resource, headers);
 			}
 
+			std::vector<std::string> queryParameters;
+			canonicalURI = awsCanonicalURI(resource, queryParameters, CLIENT_KNOBS->HTTP_REQUEST_AWS_V4_HEADER);
+			if (!queryParameters.empty()) {
+				canonicalURI += "?";
+				canonicalURI += boost::algorithm::join(queryParameters, "&");
+			}
+
 			if (bstore->useProxy && bstore->knobs.secure_connection == 0) {
 				// Has to be in absolute-form.
-				resource = "http://" + bstore->host + ":" + bstore->service + resource;
+				canonicalURI = "http://" + bstore->host + ":" + bstore->service + canonicalURI;
 			}
+
 			remoteAddress = rconn.conn->getPeerAddress();
 			wait(bstore->requestRate->getAllowance(1));
 			Reference<HTTP::Response> _r = wait(timeoutError(HTTP::doRequest(rconn.conn,
 			                                                                 verb,
-			                                                                 resource,
+			                                                                 canonicalURI,
 			                                                                 headers,
 			                                                                 &contentCopy,
 			                                                                 contentLen,
@@ -1000,9 +1045,9 @@ ACTOR Future<Void> listObjectsStream_impl(Reference<S3BlobStoreEndpoint> bstore,
 
 	resource.append("/?max-keys=1000");
 	if (prefix.present())
-		resource.append("&prefix=").append(HTTP::urlEncode(prefix.get()));
+		resource.append("&prefix=").append(prefix.get());
 	if (delimiter.present())
-		resource.append("&delimiter=").append(HTTP::urlEncode(std::string(1, delimiter.get())));
+		resource.append("&delimiter=").append(std::string(1, delimiter.get()));
 	resource.append("&marker=");
 	state std::string lastFile;
 	state bool more = true;
@@ -1014,7 +1059,7 @@ ACTOR Future<Void> listObjectsStream_impl(Reference<S3BlobStoreEndpoint> bstore,
 		state FlowLock::Releaser listReleaser(bstore->concurrentLists, 1);
 
 		HTTP::Headers headers;
-		state std::string fullResource = resource + HTTP::urlEncode(lastFile);
+		state std::string fullResource = resource + lastFile;
 		lastFile.clear();
 		Reference<HTTP::Response> r = wait(bstore->doRequest("GET", fullResource, headers, nullptr, 0, { 200 }));
 		listReleaser.release();
@@ -1194,7 +1239,7 @@ ACTOR Future<std::vector<std::string>> listBuckets_impl(Reference<S3BlobStoreEnd
 		state FlowLock::Releaser listReleaser(bstore->concurrentLists, 1);
 
 		HTTP::Headers headers;
-		state std::string fullResource = resource + HTTP::urlEncode(lastName);
+		state std::string fullResource = resource + lastName;
 		Reference<HTTP::Response> r = wait(bstore->doRequest("GET", fullResource, headers, nullptr, 0, { 200 }));
 		listReleaser.release();
 
@@ -1359,29 +1404,15 @@ void S3BlobStoreEndpoint::setV4AuthHeaders(std::string const& verb,
 
 	// ************* TASK 1: CREATE A CANONICAL REQUEST *************
 	// Create Create canonical URI--the part of the URI from domain to query string (use '/' if no path)
-	StringRef resourceRef(resource);
-	resourceRef.eat("/");
-	std::string canonicalURI("/" + resourceRef.toString());
-	size_t q = canonicalURI.find_last_of('?');
-	if (q != canonicalURI.npos)
-		canonicalURI.resize(q);
-	canonicalURI = HTTP::awsV4URIEncode(canonicalURI, false);
-	// Create the canonical query string
-	std::string queryString;
-	q = resource.find_last_of('?');
-	if (q != queryString.npos)
-		queryString = resource.substr(q + 1);
 	std::vector<std::string> queryParameters;
-	StringRef qStr(queryString);
-	StringRef queryParameter;
-	while ((queryParameter = qStr.eat("&")) != StringRef()) {
-		StringRef param = queryParameter.eat("=");
-		StringRef value = queryParameter.eat();
-		queryParameters.push_back(HTTP::awsV4URIEncode(param.toString(), true) + "=" +
-		                          HTTP::awsV4URIEncode(value.toString(), true));
+	std::string canonicalURI = awsCanonicalURI(resource, queryParameters, true);
+
+	std::string canonicalQueryString;
+	if (!queryParameters.empty()) {
+		std::sort(queryParameters.begin(), queryParameters.end());
+		canonicalQueryString = boost::algorithm::join(queryParameters, "&");
 	}
-	std::sort(queryParameters.begin(), queryParameters.end());
-	std::string canonicalQueryString = boost::algorithm::join(queryParameters, "&");
+
 	using namespace boost::algorithm;
 	// Create the canonical headers and signed headers
 	ASSERT(!headers["Host"].empty());
@@ -1436,11 +1467,10 @@ void S3BlobStoreEndpoint::setAuthHeaders(std::string const& verb, std::string co
 
 	std::string& date = headers["Date"];
 
-	char dateBuf[20];
+	char dateBuf[64];
 	time_t ts;
 	time(&ts);
-	// ISO 8601 format YYYYMMDD'T'HHMMSS'Z'
-	strftime(dateBuf, 20, "%Y%m%dT%H%M%SZ", gmtime(&ts));
+	strftime(dateBuf, 64, "%a, %d %b %Y %H:%M:%S GMT", gmtime(&ts));
 	date = dateBuf;
 
 	std::string msg;
diff --git a/fdbrpc/HTTP.actor.cpp b/fdbrpc/HTTP.actor.cpp
index cd2ed8f8a04..4a03c3a9239 100644
--- a/fdbrpc/HTTP.actor.cpp
+++ b/fdbrpc/HTTP.actor.cpp
@@ -23,6 +23,7 @@
 #include <cctype>
 
 #include "contrib/fmt-8.1.1/include/fmt/format.h"
+#include "fdbclient/Knobs.h"
 #include "fdbrpc/md5/md5.h"
 #include "fdbrpc/libb64/encode.h"
 #include "flow/Knobs.h"
@@ -184,6 +185,8 @@ ACTOR Future<Void> read_http_response_headers(Reference<IConnection> conn,
 		int nameEnd = -1, valueStart = -1, valueEnd = -1;
 		int len = -1;
 
+		std::string name, value;
+
 		// Read header of the form "Name: Value\n"
 		// Note that multi line header values are not supported here.
 		// Format string breaks down as follows:
@@ -193,20 +196,28 @@ ACTOR Future<Void> read_http_response_headers(Reference<IConnection> conn,
 		//   %*1[\r]       Exactly one \r
 		//   %*1[\n]       Exactly one \n
 		//   %n            Save final end position
-		if (sscanf(buf->c_str() + *pos,
-		           "%*[^:]%n:%*[ \t]%n%*[^\r]%n%*1[\r]%*1[\n]%n",
-		           &nameEnd,
-		           &valueStart,
-		           &valueEnd,
-		           &len) >= 0 &&
-		    len > 0) {
-			const std::string name(buf->substr(*pos, nameEnd));
-			const std::string value(buf->substr(*pos + valueStart, valueEnd - valueStart));
-			(*headers)[name] = value;
+		if (sscanf(buf->c_str() + *pos, "%*[^:]%n:%*[ \t]%n", &nameEnd, &valueStart) >= 0 && valueStart > 0) {
+			// found a header name
+			name = std::string(buf->substr(*pos, nameEnd));
+			*pos += valueStart;
+		} else {
+			// Malformed header line (at least according to this simple parsing)
+			throw http_bad_response();
+		}
+
+		if (sscanf(buf->c_str() + *pos, "%*[^\r]%n%*1[\r]%*1[\n]%n", &valueEnd, &len) >= 0 && len > 0) {
+			// found a header value
+			value = std::string(buf->substr(*pos, valueEnd));
+			*pos += len;
+		} else if (sscanf(buf->c_str() + *pos, "%*1[\r]%*1[\n]%n", &len) >= 0 && len > 0) {
+			// empty header value
 			*pos += len;
-			len = -1;
-		} else // Malformed header line (at least according to this simple parsing)
+		} else {
+			// Malformed header line (at least according to this simple parsing)
 			throw http_bad_response();
+		}
+
+		(*headers)[name] = value;
 	}
 }
 
@@ -217,7 +228,7 @@ ACTOR Future<Void> read_http_response(Reference<HTTP::Response> r, Reference<ICo
 	state std::string buf;
 	state size_t pos = 0;
 
-	// Read HTTP reponse code and version line
+	// Read HTTP response code and version line
 	size_t lineLen = wait(read_delimited_into_string(conn, "\r\n", &buf, pos));
 
 	int reachedEnd = -1;
@@ -319,9 +330,15 @@ ACTOR Future<Void> read_http_response(Reference<HTTP::Response> r, Reference<ICo
 	}
 
 	// If there is actual response content, check the MD5 sum against the Content-MD5 response header
-	if (r->content.size() > 0)
-		if (!r->verifyMD5(false)) // false arg means do not fail if the Content-MD5 header is missing.
+	if (r->content.size() > 0) {
+		if (r->code == 206 && CLIENT_KNOBS->HTTP_RESPONSE_SKIP_VERIFY_CHECKSUM_FOR_PARTIAL_CONTENT) {
+			return Void();
+		}
+
+		if (!r->verifyMD5(false)) { // false arg means do not fail if the Content-MD5 header is missing.
 			throw http_bad_response();
+		}
+	}
 
 	return Void();
 }

From 6eb3604a213d6fd82cd7da564fe57e6dd83afee0 Mon Sep 17 00:00:00 2001
From: FoundationDB CI <foundationdb_ci@apple.com>
Date: Wed, 16 Aug 2023 00:59:01 +0000
Subject: [PATCH 206/468] Release notes for 7.1.36/7.1.37.

---
 .../sphinx/source/release-notes/release-notes-710.rst  | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/documentation/sphinx/source/release-notes/release-notes-710.rst b/documentation/sphinx/source/release-notes/release-notes-710.rst
index 53476cdefc3..b5d40d3be45 100644
--- a/documentation/sphinx/source/release-notes/release-notes-710.rst
+++ b/documentation/sphinx/source/release-notes/release-notes-710.rst
@@ -4,6 +4,16 @@
 Release Notes
 #############
 
+7.1.37
+======
+* Same as 7.1.36 release with AVX enabled.
+
+7.1.36
+======
+* Released with AVX disabled.
+* Added consistency check for rocksdb only `(PR #10751) <https://github.com/apple/foundationdb/pull/10751>`_
+* Fixed grv queue stats when requests are dropped `(PR #10753) <https://github.com/apple/foundationdb/pull/10753>`_
+
 7.1.35
 ======
 * Same as 7.1.34 release with AVX enabled.

From 671e0bb0f0e43961d894d67fa16feb1f5204d037 Mon Sep 17 00:00:00 2001
From: Josh Slocum <josh.slocum@snowflake.com>
Date: Fri, 22 Apr 2022 10:43:21 -0500
Subject: [PATCH 207/468] remove BUILD_ALWAYS from awssdk.cmake

---
 cmake/awssdk.cmake | 1 -
 1 file changed, 1 deletion(-)

diff --git a/cmake/awssdk.cmake b/cmake/awssdk.cmake
index c3b9e95a4e3..88cb7c78e93 100644
--- a/cmake/awssdk.cmake
+++ b/cmake/awssdk.cmake
@@ -27,7 +27,6 @@ ExternalProject_Add(awssdk_project
                     -DCMAKE_EXE_LINKER_FLAGS=${AWSSDK_COMPILER_FLAGS}
                     -DCMAKE_CXX_FLAGS=${AWSSDK_LINK_FLAGS}
   TEST_COMMAND      ""
-  BUILD_ALWAYS      TRUE
   # the sdk build produces a ton of artifacts, with their own dependency tree, so there is a very specific dependency order they must be linked in
   BUILD_BYPRODUCTS  "${CMAKE_CURRENT_BINARY_DIR}/awssdk-build/install/lib64/libaws-cpp-sdk-core.a"
                     "${CMAKE_CURRENT_BINARY_DIR}/awssdk-build/install/lib64/libaws-crt-cpp.a"

From 87e057070a69ad1e9eb88143423dd2b1b3e78520 Mon Sep 17 00:00:00 2001
From: Marian Dvorsky <marian.dvorsky@snowflake.com>
Date: Wed, 24 Aug 2022 20:37:48 +0200
Subject: [PATCH 208/468] Upgrade awssdk to v1.9.331 (#7982)

---
 cmake/awssdk.cmake | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/cmake/awssdk.cmake b/cmake/awssdk.cmake
index 88cb7c78e93..0fef54338db 100644
--- a/cmake/awssdk.cmake
+++ b/cmake/awssdk.cmake
@@ -11,7 +11,7 @@ endif()
 include(ExternalProject)
 ExternalProject_Add(awssdk_project
   GIT_REPOSITORY    https://github.com/aws/aws-sdk-cpp.git
-  GIT_TAG           2af3ce543c322cb259471b3b090829464f825972 # v1.9.200
+  GIT_TAG           e4b4b310d8631bc7e9a797b6ac03a73c6f210bf6 # v1.9.331
   SOURCE_DIR        "${CMAKE_CURRENT_BINARY_DIR}/awssdk-src"
   BINARY_DIR        "${CMAKE_CURRENT_BINARY_DIR}/awssdk-build"
   GIT_CONFIG        advice.detachedHead=false
@@ -35,6 +35,7 @@ ExternalProject_Add(awssdk_project
                     "${CMAKE_CURRENT_BINARY_DIR}/awssdk-build/install/lib64/libaws-c-event-stream.a"
                     "${CMAKE_CURRENT_BINARY_DIR}/awssdk-build/install/lib64/libaws-c-http.a"
                     "${CMAKE_CURRENT_BINARY_DIR}/awssdk-build/install/lib64/libaws-c-mqtt.a"
+                    "${CMAKE_CURRENT_BINARY_DIR}/awssdk-build/install/lib64/libaws-c-sdkutils.a"
                     "${CMAKE_CURRENT_BINARY_DIR}/awssdk-build/install/lib64/libaws-c-io.a"
                     "${CMAKE_CURRENT_BINARY_DIR}/awssdk-build/install/lib64/libaws-checksums.a"
                     "${CMAKE_CURRENT_BINARY_DIR}/awssdk-build/install/lib64/libaws-c-compression.a"
@@ -75,6 +76,10 @@ add_library(awssdk_c_io STATIC IMPORTED)
 add_dependencies(awssdk_c_io awssdk_project)
 set_target_properties(awssdk_c_io PROPERTIES IMPORTED_LOCATION "${CMAKE_CURRENT_BINARY_DIR}/awssdk-build/install/lib64/libaws-c-io.a")
 
+add_library(awssdk_c_sdkutils STATIC IMPORTED)
+add_dependencies(awssdk_c_sdkutils awssdk_project)
+set_target_properties(awssdk_c_sdkutils PROPERTIES IMPORTED_LOCATION "${CMAKE_CURRENT_BINARY_DIR}/awssdk-build/install/lib64/libaws-c-sdkutils.a")
+
 add_library(awssdk_checksums STATIC IMPORTED)
 add_dependencies(awssdk_checksums awssdk_project)
 set_target_properties(awssdk_checksums PROPERTIES IMPORTED_LOCATION "${CMAKE_CURRENT_BINARY_DIR}/awssdk-build/install/lib64/libaws-checksums.a")
@@ -94,4 +99,4 @@ set_target_properties(awssdk_c_common PROPERTIES IMPORTED_LOCATION "${CMAKE_CURR
 # link them all together in one interface target
 add_library(awssdk_target INTERFACE)
 target_include_directories(awssdk_target SYSTEM INTERFACE ${CMAKE_CURRENT_BINARY_DIR}/awssdk-build/install/include)
-target_link_libraries(awssdk_target INTERFACE awssdk_core awssdk_crt awssdk_c_s3 awssdk_c_auth awssdk_c_eventstream awssdk_c_http awssdk_c_mqtt awssdk_c_io awssdk_checksums awssdk_c_compression awssdk_c_cal awssdk_c_common curl)
\ No newline at end of file
+target_link_libraries(awssdk_target INTERFACE awssdk_core awssdk_crt awssdk_c_s3 awssdk_c_auth awssdk_c_eventstream awssdk_c_http awssdk_c_mqtt awssdk_c_sdkutils awssdk_c_io awssdk_checksums awssdk_c_compression awssdk_c_cal awssdk_c_common curl)

From b1cf96e2bf294fe8cf0338e682ba4b675565a298 Mon Sep 17 00:00:00 2001
From: Andrew Noyes <andrew.noyes@snowflake.com>
Date: Tue, 20 Sep 2022 11:23:18 -0700
Subject: [PATCH 209/468] Statically link libcurl for the aws sdk (#8226)

---
 cmake/awssdk.cmake | 22 +++++++++++++++-------
 1 file changed, 15 insertions(+), 7 deletions(-)

diff --git a/cmake/awssdk.cmake b/cmake/awssdk.cmake
index 0fef54338db..e22a47bdd9d 100644
--- a/cmake/awssdk.cmake
+++ b/cmake/awssdk.cmake
@@ -2,10 +2,8 @@ project(awssdk-download NONE)
 
 # Compile the sdk with clang and libc++, since otherwise we get libc++ vs libstdc++ link errors when compiling fdb with clang
 set(AWSSDK_COMPILER_FLAGS "")
-set(AWSSDK_LINK_FLAGS "")
-if(APPLE OR CLANG OR USE_LIBCXX)
+if(APPLE OR USE_LIBCXX)
   set(AWSSDK_COMPILER_FLAGS -stdlib=libc++ -nostdlib++)
-  set(AWSSDK_LINK_FLAGS -stdlib=libc++ -lc++abi)
 endif()
 
 include(ExternalProject)
@@ -21,11 +19,11 @@ ExternalProject_Add(awssdk_project
                     -DSIMPLE_INSTALL=ON
                     -DCMAKE_INSTALL_PREFIX=install # need to specify an install prefix so it doesn't install in /usr/lib - FIXME: use absolute path
                     -DBYO_CRYPTO=ON                # we have our own crypto libraries that conflict if we let aws sdk build and link its own
-
+                    -DBUILD_CURL=ON
+                    -DBUILD_ZLIB=ON
                     
                     -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}
-                    -DCMAKE_EXE_LINKER_FLAGS=${AWSSDK_COMPILER_FLAGS}
-                    -DCMAKE_CXX_FLAGS=${AWSSDK_LINK_FLAGS}
+                    -DCMAKE_CXX_FLAGS=${AWSSDK_COMPILER_FLAGS}
   TEST_COMMAND      ""
   # the sdk build produces a ton of artifacts, with their own dependency tree, so there is a very specific dependency order they must be linked in
   BUILD_BYPRODUCTS  "${CMAKE_CURRENT_BINARY_DIR}/awssdk-build/install/lib64/libaws-cpp-sdk-core.a"
@@ -41,6 +39,8 @@ ExternalProject_Add(awssdk_project
                     "${CMAKE_CURRENT_BINARY_DIR}/awssdk-build/install/lib64/libaws-c-compression.a"
                     "${CMAKE_CURRENT_BINARY_DIR}/awssdk-build/install/lib64/libaws-c-cal.a"
                     "${CMAKE_CURRENT_BINARY_DIR}/awssdk-build/install/lib64/libaws-c-common.a"
+                    "${CMAKE_CURRENT_BINARY_DIR}/awssdk-build/install/external-install/curl/lib/libcurl.a"
+                    "${CMAKE_CURRENT_BINARY_DIR}/awssdk-build/install/external-install/zlib/lib/libz.a"
 )
 
 add_library(awssdk_core STATIC IMPORTED)
@@ -96,7 +96,15 @@ add_library(awssdk_c_common STATIC IMPORTED)
 add_dependencies(awssdk_c_common awssdk_project)
 set_target_properties(awssdk_c_common PROPERTIES IMPORTED_LOCATION "${CMAKE_CURRENT_BINARY_DIR}/awssdk-build/install/lib64/libaws-c-common.a")
 
+add_library(curl STATIC IMPORTED)
+add_dependencies(curl awssdk_project)
+set_property(TARGET curl PROPERTY IMPORTED_LOCATION "${CMAKE_CURRENT_BINARY_DIR}/awssdk-build/install/external-install/curl/lib/libcurl.a")
+
+add_library(zlib STATIC IMPORTED)
+add_dependencies(zlib awssdk_project)
+set_property(TARGET zlib PROPERTY IMPORTED_LOCATION "${CMAKE_CURRENT_BINARY_DIR}/awssdk-build/install/external-install/zlib/lib/libz.a")
+
 # link them all together in one interface target
 add_library(awssdk_target INTERFACE)
 target_include_directories(awssdk_target SYSTEM INTERFACE ${CMAKE_CURRENT_BINARY_DIR}/awssdk-build/install/include)
-target_link_libraries(awssdk_target INTERFACE awssdk_core awssdk_crt awssdk_c_s3 awssdk_c_auth awssdk_c_eventstream awssdk_c_http awssdk_c_mqtt awssdk_c_sdkutils awssdk_c_io awssdk_checksums awssdk_c_compression awssdk_c_cal awssdk_c_common curl)
+target_link_libraries(awssdk_target INTERFACE awssdk_core awssdk_crt awssdk_c_s3 awssdk_c_auth awssdk_c_eventstream awssdk_c_http awssdk_c_mqtt awssdk_c_sdkutils awssdk_c_io awssdk_checksums awssdk_c_compression awssdk_c_cal awssdk_c_common zlib curl)

From f650033462951a7f6a4cfea8d8259782dca4bbb3 Mon Sep 17 00:00:00 2001
From: Andrew Noyes <andrew.noyes@snowflake.com>
Date: Wed, 21 Sep 2022 14:20:36 -0700
Subject: [PATCH 210/468] Swap the order of curl and zlib for awssdk (#8268)

According to
https://ftp.gnu.org/old-gnu/Manuals/ld-2.9.1/html_node/ld_3.html,
dependencies should appear after. Since curl depends on zlib, put zlib
after curl.
---
 cmake/awssdk.cmake | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cmake/awssdk.cmake b/cmake/awssdk.cmake
index e22a47bdd9d..b5ca185b3b7 100644
--- a/cmake/awssdk.cmake
+++ b/cmake/awssdk.cmake
@@ -107,4 +107,4 @@ set_property(TARGET zlib PROPERTY IMPORTED_LOCATION "${CMAKE_CURRENT_BINARY_DIR}
 # link them all together in one interface target
 add_library(awssdk_target INTERFACE)
 target_include_directories(awssdk_target SYSTEM INTERFACE ${CMAKE_CURRENT_BINARY_DIR}/awssdk-build/install/include)
-target_link_libraries(awssdk_target INTERFACE awssdk_core awssdk_crt awssdk_c_s3 awssdk_c_auth awssdk_c_eventstream awssdk_c_http awssdk_c_mqtt awssdk_c_sdkutils awssdk_c_io awssdk_checksums awssdk_c_compression awssdk_c_cal awssdk_c_common zlib curl)
+target_link_libraries(awssdk_target INTERFACE awssdk_core awssdk_crt awssdk_c_s3 awssdk_c_auth awssdk_c_eventstream awssdk_c_http awssdk_c_mqtt awssdk_c_sdkutils awssdk_c_io awssdk_checksums awssdk_c_compression awssdk_c_cal awssdk_c_common curl zlib)

From a9c18d496093f6d563df8679100c2882e9cdf4a2 Mon Sep 17 00:00:00 2001
From: Andrew Noyes <andrew.noyes@snowflake.com>
Date: Wed, 28 Sep 2022 11:43:21 -0700
Subject: [PATCH 211/468] Quote aws sdk cxx flags

Cmake 3.24 seems to require this
---
 cmake/awssdk.cmake | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cmake/awssdk.cmake b/cmake/awssdk.cmake
index b5ca185b3b7..ab62f9b6d6e 100644
--- a/cmake/awssdk.cmake
+++ b/cmake/awssdk.cmake
@@ -3,7 +3,7 @@ project(awssdk-download NONE)
 # Compile the sdk with clang and libc++, since otherwise we get libc++ vs libstdc++ link errors when compiling fdb with clang
 set(AWSSDK_COMPILER_FLAGS "")
 if(APPLE OR USE_LIBCXX)
-  set(AWSSDK_COMPILER_FLAGS -stdlib=libc++ -nostdlib++)
+  set(AWSSDK_COMPILER_FLAGS "-stdlib=libc++ -nostdlib++")
 endif()
 
 include(ExternalProject)

From f98932610fc9dbf1fd205f17c25150dfd8dbf057 Mon Sep 17 00:00:00 2001
From: Zhe Wu <halfprice@users.noreply.github.com>
Date: Wed, 16 Aug 2023 22:37:21 -0700
Subject: [PATCH 212/468] Cherry pick perpetual_storage_wiggle_engine option
 into 7.1

---
 fdbcli/fdbcli.actor.cpp                       |   1 +
 fdbclient/DatabaseConfiguration.cpp           |   9 +-
 fdbclient/DatabaseConfiguration.h             |   1 +
 fdbclient/FDBTypes.h                          |  19 ++
 fdbclient/ManagementAPI.actor.cpp             |  18 ++
 fdbserver/CMakeLists.txt                      |   1 +
 fdbserver/DDTeamCollection.actor.cpp          |  44 +++-
 .../workloads/ConsistencyCheck.actor.cpp      |  30 ++-
 ...alWiggleStorageMigrationWorkload.actor.cpp | 226 ++++++++++++++++++
 flow/error_definitions.h                      |   1 +
 tests/CMakeLists.txt                          |   1 +
 .../rare/PerpetualWiggleStorageMigration.toml |  13 +
 12 files changed, 353 insertions(+), 11 deletions(-)
 create mode 100644 fdbserver/workloads/PerpetualWiggleStorageMigrationWorkload.actor.cpp
 create mode 100644 tests/rare/PerpetualWiggleStorageMigration.toml

diff --git a/fdbcli/fdbcli.actor.cpp b/fdbcli/fdbcli.actor.cpp
index 5c76c170a17..88f2fb50a22 100644
--- a/fdbcli/fdbcli.actor.cpp
+++ b/fdbcli/fdbcli.actor.cpp
@@ -798,6 +798,7 @@ void configureGenerator(const char* text, const char* line, std::vector<std::str
 		                   "resolvers=",
 		                   "perpetual_storage_wiggle=",
 		                   "perpetual_storage_wiggle_locality=",
+		                   "perpetual_storage_wiggle_engine=",
 		                   "storage_migration_type=",
 		                   "tenant_mode=",
 		                   "blob_granules_enabled=",
diff --git a/fdbclient/DatabaseConfiguration.cpp b/fdbclient/DatabaseConfiguration.cpp
index bd9554018fe..72b89f44abf 100644
--- a/fdbclient/DatabaseConfiguration.cpp
+++ b/fdbclient/DatabaseConfiguration.cpp
@@ -35,7 +35,8 @@ void DatabaseConfiguration::resetInternal() {
 	commitProxyCount = grvProxyCount = resolverCount = desiredTLogCount = tLogWriteAntiQuorum = tLogReplicationFactor =
 	    storageTeamSize = desiredLogRouterCount = -1;
 	tLogVersion = TLogVersion::DEFAULT;
-	tLogDataStoreType = storageServerStoreType = testingStorageServerStoreType = KeyValueStoreType::END;
+	tLogDataStoreType = storageServerStoreType = testingStorageServerStoreType = perpetualStoreType =
+	    KeyValueStoreType::END;
 	desiredTSSCount = 0;
 	tLogSpillType = TLogSpillType::DEFAULT;
 	autoCommitProxyCount = CLIENT_KNOBS->DEFAULT_AUTO_COMMIT_PROXIES;
@@ -405,6 +406,9 @@ StatusObject DatabaseConfiguration::toJSON(bool noPolicies) const {
 	result["backup_worker_enabled"] = (int32_t)backupWorkerEnabled;
 	result["perpetual_storage_wiggle"] = perpetualStorageWiggleSpeed;
 	result["perpetual_storage_wiggle_locality"] = perpetualStorageWiggleLocality;
+	if (perpetualStoreType.storeType() != KeyValueStoreType::END) {
+		result["perpetual_storage_wiggle_engine"] = perpetualStoreType.toString();
+	}
 	result["storage_migration_type"] = storageMigrationType.toString();
 	result["blob_granules_enabled"] = (int32_t)blobGranulesEnabled;
 	result["tenant_mode"] = tenantMode.toString();
@@ -628,6 +632,9 @@ bool DatabaseConfiguration::setInternal(KeyRef key, ValueRef value) {
 			return false;
 		}
 		perpetualStorageWiggleLocality = value.toString();
+	} else if (ck == LiteralStringRef("perpetual_storage_wiggle_engine")) {
+		parse((&type), value);
+		perpetualStoreType = (KeyValueStoreType::StoreType)type;
 	} else if (ck == LiteralStringRef("storage_migration_type")) {
 		parse((&type), value);
 		storageMigrationType = (StorageMigrationType::MigrationType)type;
diff --git a/fdbclient/DatabaseConfiguration.h b/fdbclient/DatabaseConfiguration.h
index 363b48e4b63..923a2d0ff41 100644
--- a/fdbclient/DatabaseConfiguration.h
+++ b/fdbclient/DatabaseConfiguration.h
@@ -248,6 +248,7 @@ struct DatabaseConfiguration {
 	// Perpetual Storage Setting
 	int32_t perpetualStorageWiggleSpeed;
 	std::string perpetualStorageWiggleLocality;
+	KeyValueStoreType perpetualStoreType;
 
 	// Storage Migration Type
 	StorageMigrationType storageMigrationType;
diff --git a/fdbclient/FDBTypes.h b/fdbclient/FDBTypes.h
index c494762743e..c8473dfd393 100644
--- a/fdbclient/FDBTypes.h
+++ b/fdbclient/FDBTypes.h
@@ -892,6 +892,25 @@ struct KeyValueStoreType {
 		}
 	}
 
+	// Convert a string to a KeyValueStoreType
+	// This is a many-to-one mapping as there are aliases for some storage engines
+	static KeyValueStoreType fromString(const std::string& str) {
+		static std::map<std::string, StoreType> names = { { "ssd-1", SSD_BTREE_V1 },
+			                                              { "ssd-2", SSD_BTREE_V2 },
+			                                              { "ssd", SSD_BTREE_V2 },
+			                                              { "redwood", SSD_REDWOOD_V1 },
+			                                              { "ssd-redwood-1", SSD_REDWOOD_V1 },
+			                                              { "ssd-redwood-1-experimental", SSD_REDWOOD_V1 },
+			                                              { "ssd-rocksdb-v1", SSD_ROCKSDB_V1 },
+			                                              { "memory", MEMORY },
+			                                              { "memory-radixtree-beta", MEMORY_RADIXTREE } };
+		auto it = names.find(str);
+		if (it == names.end()) {
+			throw unknown_storage_engine();
+		}
+		return it->second;
+	}
+
 private:
 	uint32_t type;
 };
diff --git a/fdbclient/ManagementAPI.actor.cpp b/fdbclient/ManagementAPI.actor.cpp
index dd0ce41d7be..2c36d2bc28e 100644
--- a/fdbclient/ManagementAPI.actor.cpp
+++ b/fdbclient/ManagementAPI.actor.cpp
@@ -217,6 +217,24 @@ std::map<std::string, std::string> configForToken(std::string const& mode) {
 				p = end + 1;
 			}
 		}
+
+		if (key == "perpetual_storage_wiggle_engine") {
+			StringRef s = value;
+
+			// Parse as engine_name[:p=v]... to handle future storage engine params
+			Value engine = s.eat(":");
+			std::map<Key, Value> params;
+			while (!s.empty()) {
+				params[s.eat("=")] = s.eat(":");
+			}
+
+			try {
+				out[p + key] = format("%d", KeyValueStoreType::fromString(engine.toString()).storeType());
+			} catch (Error& e) {
+				printf("Error: Invalid value for %s (%s): %s\n", key.c_str(), value.c_str(), e.what());
+			}
+			return out;
+		}
 		return out;
 	}
 
diff --git a/fdbserver/CMakeLists.txt b/fdbserver/CMakeLists.txt
index dab8f0d10dc..679c950c9a0 100644
--- a/fdbserver/CMakeLists.txt
+++ b/fdbserver/CMakeLists.txt
@@ -250,6 +250,7 @@ set(FDBSERVER_SRCS
   workloads/MutationLogReaderCorrectness.actor.cpp
   workloads/ParallelRestore.actor.cpp
   workloads/Performance.actor.cpp
+  workloads/PerpetualWiggleStorageMigrationWorkload.actor.cpp
   workloads/PhysicalShardMove.actor.cpp
   workloads/Ping.actor.cpp
   workloads/PopulateTPCC.actor.cpp
diff --git a/fdbserver/DDTeamCollection.actor.cpp b/fdbserver/DDTeamCollection.actor.cpp
index 97c67492eea..9f8f39b1b7d 100644
--- a/fdbserver/DDTeamCollection.actor.cpp
+++ b/fdbserver/DDTeamCollection.actor.cpp
@@ -39,6 +39,19 @@ auto get(MapContainer& m, K const& k) -> decltype(m.at(k)) {
 	return it->second;
 }
 
+void ParsePerpetualStorageWiggleLocality(const std::string& localityKeyValue,
+                                         Optional<Value>* localityKey,
+                                         Optional<Value>* localityValue) {
+	// parsing format is like "datahall:0"
+	ASSERT(isValidPerpetualStorageWiggleLocality(localityKeyValue));
+
+	// get key and value from perpetual_storage_wiggle_locality.
+	int split = localityKeyValue.find(':');
+	*localityKey = Optional<Value>(ValueRef((uint8_t*)localityKeyValue.c_str(), split));
+	*localityValue =
+	    Optional<Value>(ValueRef((uint8_t*)localityKeyValue.c_str() + split + 1, localityKeyValue.size() - split - 1));
+}
+
 } // namespace
 
 class DDTeamCollectionImpl {
@@ -2307,6 +2320,24 @@ class DDTeamCollectionImpl {
 			state InitializeStorageRequest isr;
 			isr.storeType = recruitTss ? self->configuration.testingStorageServerStoreType
 			                           : self->configuration.storageServerStoreType;
+
+			// Check if perpetual storage wiggle is enabled and perpetualStoreType is set. If so, we use
+			// perpetualStoreType for all new SSes that match perpetualStorageWiggleLocality.
+			// Note that this only applies to regular storage servers, not TSS.
+			if (!recruitTss && self->configuration.storageMigrationType == StorageMigrationType::GRADUAL &&
+			    self->configuration.perpetualStoreType.storeType() != KeyValueStoreType::END) {
+				if (self->configuration.perpetualStorageWiggleLocality == "0") {
+					isr.storeType = self->configuration.perpetualStoreType;
+				} else {
+					Optional<Value> localityKey;
+					Optional<Value> localityValue;
+					ParsePerpetualStorageWiggleLocality(
+					    self->configuration.perpetualStorageWiggleLocality, &localityKey, &localityValue);
+					if (candidateWorker.worker.locality.get(localityKey.get()) == localityValue) {
+						isr.storeType = self->configuration.perpetualStoreType;
+					}
+				}
+			}
 			isr.seedTag = invalidTag;
 			isr.reqId = deterministicRandom()->randomUniqueID();
 			isr.interfaceId = interfaceId;
@@ -2351,7 +2382,8 @@ class DDTeamCollectionImpl {
 			    .detail("Interf", interfaceId)
 			    .detail("Addr", candidateWorker.worker.address())
 			    .detail("TSS", recruitTss ? "true" : "false")
-			    .detail("RecruitingStream", self->recruitingStream.get());
+			    .detail("RecruitingStream", self->recruitingStream.get())
+			    .detail("StoreType", isr.storeType);
 
 			Future<ErrorOr<InitializeStorageReply>> fRecruit =
 			    doRecruit
@@ -2838,14 +2870,8 @@ class DDTeamCollectionImpl {
 		// NOTE: because normal \xff/conf change through `changeConfig` now will cause DD throw `movekeys_conflict()`
 		// then recruit a new DD, we only need to read current configuration once
 		if (teamCollection->configuration.perpetualStorageWiggleLocality != "0") {
-			// parsing format is like "datahall:0"
-			std::string& localityKeyValue = teamCollection->configuration.perpetualStorageWiggleLocality;
-			ASSERT(isValidPerpetualStorageWiggleLocality(localityKeyValue));
-			// get key and value from perpetual_storage_wiggle_locality.
-			int split = localityKeyValue.find(':');
-			localityKey = Optional<Value>(ValueRef((uint8_t*)localityKeyValue.c_str(), split));
-			localityValue = Optional<Value>(
-			    ValueRef((uint8_t*)localityKeyValue.c_str() + split + 1, localityKeyValue.size() - split - 1));
+			ParsePerpetualStorageWiggleLocality(
+			    teamCollection->configuration.perpetualStorageWiggleLocality, &localityKey, &localityValue);
 		}
 
 		loop {
diff --git a/fdbserver/workloads/ConsistencyCheck.actor.cpp b/fdbserver/workloads/ConsistencyCheck.actor.cpp
index 7ce188a2901..6b024359d52 100644
--- a/fdbserver/workloads/ConsistencyCheck.actor.cpp
+++ b/fdbserver/workloads/ConsistencyCheck.actor.cpp
@@ -1865,6 +1865,33 @@ struct ConsistencyCheckWorkload : TestWorkload {
 			if (!keyValueStoreType.present()) {
 				TraceEvent("ConsistencyCheck_ServerUnavailable").detail("ServerID", storageServers[i].id());
 				self->testFailure("Storage server unavailable");
+			} else if (configuration.perpetualStoreType.storeType() != KeyValueStoreType::END) {
+				// Perpetual storage wiggle is used to migrate storage. Check that the matched storage servers are
+				// correctly migrated.
+				if (wiggleLocalityKeyValue == "0" ||
+				    (storageServers[i].locality.get(wiggleLocalityKey).present() &&
+				     storageServers[i].locality.get(wiggleLocalityKey).get().toString() == wiggleLocalityValue)) {
+					if (keyValueStoreType.get() != configuration.perpetualStoreType) {
+						TraceEvent("ConsistencyCheck_WrongKeyValueStoreType")
+						    .detail("ServerID", storageServers[i].id())
+						    .detail("StoreType", keyValueStoreType.get().toString())
+						    .detail("DesiredType", configuration.perpetualStoreType.toString())
+						    .detail("IsPerpetualStoreType", true);
+						self->testFailure("Storage server has wrong key-value store type");
+						return true;
+					}
+				} else if ((!storageServers[i].isTss() &&
+				            keyValueStoreType.get() != configuration.storageServerStoreType) ||
+				           (storageServers[i].isTss() &&
+				            keyValueStoreType.get() != configuration.testingStorageServerStoreType)) {
+					TraceEvent("ConsistencyCheck_WrongKeyValueStoreType")
+					    .detail("ServerID", storageServers[i].id())
+					    .detail("StoreType", keyValueStoreType.get().toString())
+					    .detail("DesiredType", configuration.perpetualStoreType.toString())
+					    .detail("IsPerpetualStoreType", false);
+					self->testFailure("Storage server has wrong key-value store type");
+					return true;
+				}
 			} else if (((!storageServers[i].isTss() &&
 			             keyValueStoreType.get() != configuration.storageServerStoreType) ||
 			            (storageServers[i].isTss() &&
@@ -1875,7 +1902,8 @@ struct ConsistencyCheckWorkload : TestWorkload {
 				TraceEvent("ConsistencyCheck_WrongKeyValueStoreType")
 				    .detail("ServerID", storageServers[i].id())
 				    .detail("StoreType", keyValueStoreType.get().toString())
-				    .detail("DesiredType", configuration.storageServerStoreType.toString());
+				    .detail("DesiredType", configuration.storageServerStoreType.toString())
+				    .detail("IsPerpetualStoreType", false);
 				self->testFailure("Storage server has wrong key-value store type");
 				return true;
 			}
diff --git a/fdbserver/workloads/PerpetualWiggleStorageMigrationWorkload.actor.cpp b/fdbserver/workloads/PerpetualWiggleStorageMigrationWorkload.actor.cpp
new file mode 100644
index 00000000000..37d9738f4a7
--- /dev/null
+++ b/fdbserver/workloads/PerpetualWiggleStorageMigrationWorkload.actor.cpp
@@ -0,0 +1,226 @@
+/*
+ * PerpetualWiggleStorageMigrationWorkload.actor.cpp
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2022 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "fdbserver/DDTeamCollection.h"
+#include "fdbclient/FDBOptions.g.h"
+#include "fdbclient/ManagementAPI.actor.h"
+#include "fdbserver/MoveKeys.actor.h"
+#include "fdbclient/StorageServerInterface.h"
+#include "fdbserver/workloads/workloads.actor.h"
+#include "fdbclient/VersionedMap.h"
+#include "fdbclient/ReadYourWrites.h"
+#include "fdbrpc/simulator.h"
+
+#include "flow/actorcompiler.h" // This must be the last #include.
+
+namespace {
+ACTOR Future<bool> IssueConfigurationChange(Database cx, std::string config, bool force) {
+	printf("Issuing configuration change: %s\n", config.c_str());
+	state ConfigurationResult res = wait(ManagementAPI::changeConfig(cx.getReference(), config, force));
+	if (res != ConfigurationResult::SUCCESS) {
+		return false;
+	}
+	wait(delay(5.0)); // wait for read window
+	return true;
+}
+} // namespace
+
+struct PerpetualWiggleStorageMigrationWorkload : public TestWorkload {
+
+	static constexpr auto NAME = "PerpetualWiggleStorageMigrationWorkload";
+
+	PerpetualWiggleStorageMigrationWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {}
+    
+    std::string description() const override { return "PerpetualWiggleStorageMigrationWorkload"; }
+
+	Future<Void> setup(Database const& cx) override { return Void(); }
+	Future<Void> start(Database const& cx) override {
+		if (clientId == 0) {
+			return _start(this, cx);
+		}
+		return Void();
+	};
+
+	Future<bool> check(Database const& cx) override { return true; };
+
+	ACTOR static Future<Void> _start(PerpetualWiggleStorageMigrationWorkload* self, Database cx) {
+		state std::vector<StorageServerInterface> storageServers = wait(getStorageServers(cx));
+		// The test should have enough storage servers to exclude.
+		ASSERT(storageServers.size() > 3);
+
+		// Pick a storage process to exclude and later include. This process should always use storage engine from
+		// `storage_engine` configuration.
+		state StorageServerInterface ssToExcludeInclude =
+		    storageServers[deterministicRandom()->randomInt(0, storageServers.size())];
+		state ISimulator::ProcessInfo* p = g_simulator.getProcessByAddress(ssToExcludeInclude.address());
+		while (!p->isReliable()) {
+			ssToExcludeInclude = storageServers[deterministicRandom()->randomInt(0, storageServers.size())];
+			p = g_simulator.getProcessByAddress(ssToExcludeInclude.address());
+		}
+
+		TraceEvent("Test_PickedProcessToExcludeInclude")
+		    .detail("ProcessID", ssToExcludeInclude.locality.processId())
+		    .detail("Address", ssToExcludeInclude.address());
+
+		// Pick a storage process to migrate to storage engine specified in `perpetual_storage_wiggle_engine`.
+		state StorageServerInterface ssToWiggle =
+		    storageServers[deterministicRandom()->randomInt(0, storageServers.size())];
+		while (ssToExcludeInclude.locality.processId() == ssToWiggle.locality.processId()) {
+			ssToWiggle = storageServers[deterministicRandom()->randomInt(0, storageServers.size())];
+		}
+		TraceEvent("Test_PickedProcessToMigrate")
+		    .detail("ProcessID", ssToWiggle.locality.processId())
+		    .detail("Address", ssToWiggle.address());
+
+		// Issue a configuration change to ONLY migrate `ssToWiggle` using perpetual wiggle.
+		std::string migrationLocality =
+		    LocalityData::keyProcessId.toString() + ":" + ssToWiggle.locality.processId().get().toString();
+		bool change =
+		    wait(IssueConfigurationChange(cx,
+		                                  "perpetual_storage_wiggle_engine=ssd-rocksdb-v1 perpetual_storage_wiggle=1 "
+		                                  "storage_migration_type=gradual perpetual_storage_wiggle_locality=" +
+		                                      migrationLocality,
+		                                  true));
+		TraceEvent("Test_ConfigChangeDone").detail("Success", change);
+		ASSERT(change);
+
+		// Now, let's exclude `ssToExcludeInclude` process and include it again. The new SS created on this process
+		// should always uses `storage_engine` config, which is `ssd-2`.
+		state std::vector<AddressExclusion> servers;
+		servers.push_back(AddressExclusion(ssToExcludeInclude.address().ip, ssToExcludeInclude.address().port));
+
+		// Since we have enough storage servers and there won't be any failure, let's use exclude failed to make sure
+		// the exclude process can succeed.
+		wait(excludeServers(cx, servers, true));
+		TraceEvent("Test_DoneExcludeServer").log();
+
+		try {
+			std::set<NetworkAddress> inProgress = wait(checkForExcludingServers(cx, servers, true));
+			ASSERT(inProgress.empty());
+		} catch (Error& e) {
+			if (e.code() == error_code_timed_out) {
+				// it might never be excluded from serverList
+				TraceEvent(SevError, "Test_WaitingForExclusionTakeTooLong").log();
+			}
+			throw e;
+		}
+
+		TraceEvent("Test_CheckingExcludeServerDone").log();
+
+		// Include all the processes the cluster knows.
+		wait(includeServers(cx, std::vector<AddressExclusion>(1)));
+		TraceEvent("Test_IncludeServerDone").log();
+
+		wait(validateDatabase(cx, ssToExcludeInclude, ssToWiggle));
+		return Void();
+	}
+
+	ACTOR static Future<Void> validateDatabase(Database cx,
+	                                           StorageServerInterface ssToExcludeInclude,
+	                                           StorageServerInterface ssToWiggle) {
+		// Wait until `ssToExcludeInclude` to be recruited as storage server again.
+		state int missingTargetCount = 0;
+		loop {
+			std::vector<StorageServerInterface> allStorageServers = wait(getStorageServers(cx));
+			bool foundTarget = false;
+			for (auto& ss : allStorageServers) {
+				if (ss.address() == ssToExcludeInclude.address()) {
+					foundTarget = true;
+					break;
+				}
+			}
+			if (foundTarget) {
+				break;
+			}
+			++missingTargetCount;
+			if (missingTargetCount > 5) {
+				// Sometimes, the excluded storage process may not be recruited as storage server again (depending on
+				// the process class). So we don't wait indefinitely here.
+				break;
+			}
+			wait(delay(20));
+		}
+
+		// Wait until wiggle process to migrate to new storage engine.
+		state int missingWiggleStorageCount = 0;
+		state std::vector<StorageServerInterface> allSSes;
+		state bool doneCheckingWiggleStorage = false;
+		state bool containWiggleStorage = false;
+		loop {
+			std::vector<StorageServerInterface> SSes = wait(getStorageServers(cx));
+			allSSes = SSes;
+
+			state int i = 0;
+			containWiggleStorage = false;
+			doneCheckingWiggleStorage = false;
+			for (i = 0; i < allSSes.size(); ++i) {
+				state StorageServerInterface ssInterface = allSSes[i];
+				state ReplyPromise<KeyValueStoreType> typeReply;
+				ErrorOr<KeyValueStoreType> keyValueStoreType =
+				    wait(ssInterface.getKeyValueStoreType.getReplyUnlessFailedFor(typeReply, 2, 0));
+				if (keyValueStoreType.present()) {
+					TraceEvent(SevDebug, "Test_KvStorageType")
+					    .detail("StorageServer", ssInterface.address())
+					    .detail("StorageType", keyValueStoreType.get().toString());
+
+					if (ssInterface.address() == ssToExcludeInclude.address()) {
+						// If `ssToExcludeInclude` exists, it must remain using `storage_engine` type.
+						ASSERT(keyValueStoreType.get().toString() == "ssd-2");
+					}
+					if (ssInterface.address() == ssToWiggle.address()) {
+						// If `ssToWiggle` exists, we wait until it is migrate to `perpetual_storage_wiggle_engine`.
+						containWiggleStorage = true;
+						if (keyValueStoreType.get().toString() == "ssd-rocksdb-v1") {
+							doneCheckingWiggleStorage = true;
+						}
+					}
+				} else {
+					TraceEvent(SevDebug, "Test_KvStorageType")
+					    .detail("StorageServer", ssInterface.address())
+					    .detail("StorageType", "Unknown")
+					    .detail("Error", keyValueStoreType.getError().name());
+				}
+			}
+			if (doneCheckingWiggleStorage) {
+				break;
+			}
+			if (!containWiggleStorage) {
+				++missingWiggleStorageCount;
+				if (missingWiggleStorageCount > 5) {
+					TraceEvent("Test_TimeoutWaitingForWiggleStorageToShowUp").log();
+					break;
+				}
+			}
+			wait(delay(20));
+		}
+
+		if (!doneCheckingWiggleStorage) {
+			// If we fail to validate that the wiggle storage has been migrated to new storage engine, sometimes it is
+			// because after exclusion, the process may not be recruited as storage server, so we must not see it as a
+			// storage engine in the last check.
+			ASSERT(!containWiggleStorage);
+		}
+		return Void();
+	}
+
+	void getMetrics(std::vector<PerfMetric>& m) override { return; }
+};
+
+WorkloadFactory<PerpetualWiggleStorageMigrationWorkload> PerpetualWiggleStorageMigrationWorkloadFactory("PerpetualWiggleStorageMigration");
\ No newline at end of file
diff --git a/flow/error_definitions.h b/flow/error_definitions.h
index 7d6c09cfdfc..5fb0cafefaf 100755
--- a/flow/error_definitions.h
+++ b/flow/error_definitions.h
@@ -95,6 +95,7 @@ ERROR( page_encoding_not_supported, 1071, "Page encoding type is not supported o
 ERROR( page_decoding_failed, 1072, "Page content decoding failed" )
 ERROR( unexpected_encoding_type, 1073, "Page content decoding failed" )
 ERROR( encryption_key_not_found, 1074, "Encryption key not found" )
+ERROR( unknown_storage_engine, 1082, "Storage engine type is not recognized." )
 
 ERROR( broken_promise, 1100, "Broken promise" )
 ERROR( operation_cancelled, 1101, "Asynchronous operation cancelled" )
diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt
index ca37ea6661a..42861e17e59 100644
--- a/tests/CMakeLists.txt
+++ b/tests/CMakeLists.txt
@@ -206,6 +206,7 @@ if(WITH_PYTHON)
   add_fdb_test(TEST_FILES rare/InventoryTestHeavyWrites.toml)
   add_fdb_test(TEST_FILES rare/LargeApiCorrectness.toml)
   add_fdb_test(TEST_FILES rare/LargeApiCorrectnessStatus.toml)
+  add_fdb_test(TEST_FILES rare/PerpetualWiggleStorageMigration.toml)
   add_fdb_test(TEST_FILES rare/RYWDisable.toml)
   add_fdb_test(TEST_FILES rare/RandomReadWriteTest.toml)
   add_fdb_test(TEST_FILES rare/RestoreMultiRanges.toml)
diff --git a/tests/rare/PerpetualWiggleStorageMigration.toml b/tests/rare/PerpetualWiggleStorageMigration.toml
new file mode 100644
index 00000000000..f7ca30968b5
--- /dev/null
+++ b/tests/rare/PerpetualWiggleStorageMigration.toml
@@ -0,0 +1,13 @@
+[configuration]
+config = 'triple'
+storageEngineType = 0
+processesPerMachine = 2
+coordinators = 3
+machineCount = 45
+
+[[test]]
+testTitle = 'PerpetualWiggleStorageMigration'
+useDB = true
+
+    [[test.workload]]
+    testName = 'PerpetualWiggleStorageMigration'
\ No newline at end of file

From da5a911207144be00a74537eb6c86bef4abd6131 Mon Sep 17 00:00:00 2001
From: Zhe Wu <halfprice@users.noreply.github.com>
Date: Wed, 16 Aug 2023 23:02:21 -0700
Subject: [PATCH 213/468] Make sure to run PerpetualWiggleStorageMigration test
 only when RocksDB is enabled

---
 tests/CMakeLists.txt | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt
index 42861e17e59..960c9dcbec8 100644
--- a/tests/CMakeLists.txt
+++ b/tests/CMakeLists.txt
@@ -186,9 +186,11 @@ if(WITH_PYTHON)
   if (WITH_ROCKSDB_EXPERIMENTAL)
     add_fdb_test(TEST_FILES noSim/KeyValueStoreRocksDBTest.toml)
     add_fdb_test(TEST_FILES fast/PhysicalShardMove.toml)
+    add_fdb_test(TEST_FILES rare/PerpetualWiggleStorageMigration.toml)
   else()
     add_fdb_test(TEST_FILES noSim/KeyValueStoreRocksDBTest.toml IGNORE)
     add_fdb_test(TEST_FILES fast/PhysicalShardMove.toml IGNORE)
+    add_fdb_test(TEST_FILES rare/PerpetualWiggleStorageMigration.toml IGNORE)
   endif()
   add_fdb_test(TEST_FILES rare/CheckRelocation.toml)
   add_fdb_test(TEST_FILES rare/ClogTlog.toml)
@@ -206,7 +208,6 @@ if(WITH_PYTHON)
   add_fdb_test(TEST_FILES rare/InventoryTestHeavyWrites.toml)
   add_fdb_test(TEST_FILES rare/LargeApiCorrectness.toml)
   add_fdb_test(TEST_FILES rare/LargeApiCorrectnessStatus.toml)
-  add_fdb_test(TEST_FILES rare/PerpetualWiggleStorageMigration.toml)
   add_fdb_test(TEST_FILES rare/RYWDisable.toml)
   add_fdb_test(TEST_FILES rare/RandomReadWriteTest.toml)
   add_fdb_test(TEST_FILES rare/RestoreMultiRanges.toml)

From 644164db6597668952afb60e121291dce416b9c5 Mon Sep 17 00:00:00 2001
From: Zhe Wu <halfprice@users.noreply.github.com>
Date: Wed, 16 Aug 2023 23:06:22 -0700
Subject: [PATCH 214/468] Apply format

---
 .../PerpetualWiggleStorageMigrationWorkload.actor.cpp      | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/fdbserver/workloads/PerpetualWiggleStorageMigrationWorkload.actor.cpp b/fdbserver/workloads/PerpetualWiggleStorageMigrationWorkload.actor.cpp
index 37d9738f4a7..98a32daede2 100644
--- a/fdbserver/workloads/PerpetualWiggleStorageMigrationWorkload.actor.cpp
+++ b/fdbserver/workloads/PerpetualWiggleStorageMigrationWorkload.actor.cpp
@@ -47,8 +47,8 @@ struct PerpetualWiggleStorageMigrationWorkload : public TestWorkload {
 	static constexpr auto NAME = "PerpetualWiggleStorageMigrationWorkload";
 
 	PerpetualWiggleStorageMigrationWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {}
-    
-    std::string description() const override { return "PerpetualWiggleStorageMigrationWorkload"; }
+
+	std::string description() const override { return "PerpetualWiggleStorageMigrationWorkload"; }
 
 	Future<Void> setup(Database const& cx) override { return Void(); }
 	Future<Void> start(Database const& cx) override {
@@ -223,4 +223,5 @@ struct PerpetualWiggleStorageMigrationWorkload : public TestWorkload {
 	void getMetrics(std::vector<PerfMetric>& m) override { return; }
 };
 
-WorkloadFactory<PerpetualWiggleStorageMigrationWorkload> PerpetualWiggleStorageMigrationWorkloadFactory("PerpetualWiggleStorageMigration");
\ No newline at end of file
+WorkloadFactory<PerpetualWiggleStorageMigrationWorkload> PerpetualWiggleStorageMigrationWorkloadFactory(
+    "PerpetualWiggleStorageMigration");
\ No newline at end of file

From 14dae3926ff380d1a6c3dd7d426ed067535973f6 Mon Sep 17 00:00:00 2001
From: Hao Fu <77984096+hfu94@users.noreply.github.com>
Date: Sat, 19 Aug 2023 17:36:06 -0700
Subject: [PATCH 215/468] Fix inaccurate PPW configure return code (#10795)

---
 fdbcli/ConfigureCommand.actor.cpp | 1 -
 1 file changed, 1 deletion(-)

diff --git a/fdbcli/ConfigureCommand.actor.cpp b/fdbcli/ConfigureCommand.actor.cpp
index 16e98db58f3..598fc3a838a 100644
--- a/fdbcli/ConfigureCommand.actor.cpp
+++ b/fdbcli/ConfigureCommand.actor.cpp
@@ -253,7 +253,6 @@ ACTOR Future<bool> configureCommandActor(Reference<IDatabase> db,
 		fprintf(stderr,
 		        "Type `configure perpetual_storage_wiggle=1' to enable the perpetual wiggle, or `configure "
 		        "storage_migration_type=gradual' to set the gradual migration type.\n");
-		ret = false;
 		break;
 	case ConfigurationResult::SUCCESS_WARN_ROCKSDB_EXPERIMENTAL:
 		printf("Configuration changed\n");

From 7c80debe9986644a7dbb86fbef672569e6234fd9 Mon Sep 17 00:00:00 2001
From: Zhe Wu <halfprice@users.noreply.github.com>
Date: Tue, 22 Aug 2023 11:15:50 -0700
Subject: [PATCH 216/468] Cherry-pick
 https://github.com/apple/foundationdb/pull/10579 to release 7.1 branch

---
 fdbserver/ClusterController.actor.cpp  | 153 ++++++++++++++++++++++---
 fdbserver/ClusterController.actor.h    |  92 +--------------
 fdbserver/ClusterRecovery.actor.cpp    |   4 +
 fdbserver/ClusterRecovery.actor.h      |   4 +
 fdbserver/OldTLogServer_6_0.actor.cpp  |   3 +-
 fdbserver/OldTLogServer_6_2.actor.cpp  |   3 +-
 fdbserver/TLogServer.actor.cpp         |  22 +++-
 fdbserver/WorkerInterface.actor.h      |   9 +-
 fdbserver/worker.actor.cpp             |  66 +++++++++--
 fdbserver/workloads/ClogTlog.actor.cpp |  16 ++-
 tests/rare/ClogTlog.toml               |   6 +-
 11 files changed, 250 insertions(+), 128 deletions(-)

diff --git a/fdbserver/ClusterController.actor.cpp b/fdbserver/ClusterController.actor.cpp
index fffcd58ea04..49daf4ca0cb 100644
--- a/fdbserver/ClusterController.actor.cpp
+++ b/fdbserver/ClusterController.actor.cpp
@@ -194,11 +194,120 @@ struct EncryptKeyProxySingleton : Singleton<EncryptKeyProxyInterface> {
 	}
 };
 
+bool ClusterControllerData::transactionSystemContainsDegradedServers() {
+	const ServerDBInfo& dbi = db.serverInfo->get();
+	const Reference<ClusterRecoveryData> recoveryData = db.recoveryData;
+	auto transactionWorkerInList = [&dbi, &recoveryData](const std::unordered_set<NetworkAddress>& serverList,
+	                                                     bool skipSatellite,
+	                                                     bool skipRemote) -> bool {
+		for (const auto& server : serverList) {
+			if (dbi.master.addresses().contains(server)) {
+				return true;
+			}
+
+			// Getting the current log system config to check if any TLogs are degraded. recoveryData->logSystem has the
+			// most up to date log system and we should use it whenever available.
+			auto logSystemConfig = (recoveryData.isValid() && recoveryData->logSystem.isValid())
+			                           ? recoveryData->logSystem->getLogSystemConfig()
+			                           : dbi.logSystemConfig;
+			for (const auto& logSet : logSystemConfig.tLogs) {
+				if (skipSatellite && logSet.locality == tagLocalitySatellite) {
+					continue;
+				}
+
+				if (skipRemote && !logSet.isLocal) {
+					continue;
+				}
+
+				if (!logSet.isLocal) {
+					// Only check log routers in the remote region.
+					for (const auto& logRouter : logSet.logRouters) {
+						if (logRouter.present() && logRouter.interf().addresses().contains(server)) {
+							return true;
+						}
+					}
+				} else {
+					for (const auto& tlog : logSet.tLogs) {
+						if (tlog.present() && tlog.interf().addresses().contains(server)) {
+							return true;
+						}
+					}
+				}
+			}
+
+			if (recoveryData.isValid() && recoveryData->recoveryState < RecoveryState::ACCEPTING_COMMITS) {
+				// During recovery, TLogs may not be able to pull data from previous generation TLogs due to gray
+				// failures. In this case, we rely on the latest recruitment information and see if any newly recruited
+				// TLogs are degraded.
+				for (const auto& tlog : recoveryData->primaryRecruitment.tLogs) {
+					if (tlog.addresses().contains(server)) {
+						return true;
+					}
+				}
+
+				if (!skipSatellite) {
+					for (const auto& satelliteLog : recoveryData->primaryRecruitment.satelliteTLogs) {
+						if (satelliteLog.addresses().contains(server)) {
+							return true;
+						}
+					}
+				}
+			}
+
+			for (const auto& proxy : dbi.client.grvProxies) {
+				if (proxy.addresses().contains(server)) {
+					return true;
+				}
+			}
+
+			for (const auto& proxy : dbi.client.commitProxies) {
+				if (proxy.addresses().contains(server)) {
+					return true;
+				}
+			}
+
+			for (const auto& resolver : dbi.resolvers) {
+				if (resolver.addresses().contains(server)) {
+					return true;
+				}
+			}
+		}
+		return false;
+	};
+
+	// Check if transaction system contains degraded/disconnected servers. For satellite and remote regions, we only
+	// check for disconnection since the latency between prmary and satellite is across WAN and may not be very
+	// stable.
+	return transactionWorkerInList(degradationInfo.degradedServers, /*skipSatellite=*/true, /*skipRemote=*/true) ||
+	       transactionWorkerInList(degradationInfo.disconnectedServers,
+	                               /*skipSatellite=*/false,
+	                               /*skipRemote=*/!SERVER_KNOBS->CC_ENABLE_REMOTE_LOG_ROUTER_MONITORING);
+}
+
+bool ClusterControllerData::remoteTransactionSystemContainsDegradedServers() {
+	if (db.config.usableRegions <= 1) {
+		return false;
+	}
+
+	for (const auto& excludedServer : degradationInfo.degradedServers) {
+		if (addressInDbAndRemoteDc(excludedServer, db.serverInfo)) {
+			return true;
+		}
+	}
+
+	for (const auto& excludedServer : degradationInfo.disconnectedServers) {
+		if (addressInDbAndRemoteDc(excludedServer, db.serverInfo)) {
+			return true;
+		}
+	}
+
+	return false;
+}
+
 ACTOR Future<Void> clusterWatchDatabase(ClusterControllerData* cluster,
                                         ClusterControllerData::DBInfo* db,
                                         ServerCoordinators coordinators) {
 	state MasterInterface iMaster;
-	state Reference<ClusterRecoveryData> recoveryData;
 	state PromiseStream<Future<Void>> addActor;
 	state Future<Void> recoveryCore;
 
@@ -248,18 +357,18 @@ ACTOR Future<Void> clusterWatchDatabase(ClusterControllerData* cluster,
 			                              // the "first" recovery after more than a second of normal operation
 
 			TraceEvent("CCWDB", cluster->id).detail("Watching", iMaster.id());
-			recoveryData = makeReference<ClusterRecoveryData>(cluster,
-			                                                  db->serverInfo,
-			                                                  db->serverInfo->get().master,
-			                                                  db->serverInfo->get().masterLifetime,
-			                                                  coordinators,
-			                                                  db->serverInfo->get().clusterInterface,
-			                                                  LiteralStringRef(""),
-			                                                  addActor,
-			                                                  db->forceRecovery);
-
-			collection = actorCollection(recoveryData->addActor.getFuture());
-			recoveryCore = clusterRecoveryCore(recoveryData);
+			db->recoveryData = makeReference<ClusterRecoveryData>(cluster,
+			                                                      db->serverInfo,
+			                                                      db->serverInfo->get().master,
+			                                                      db->serverInfo->get().masterLifetime,
+			                                                      coordinators,
+			                                                      db->serverInfo->get().clusterInterface,
+			                                                      LiteralStringRef(""),
+			                                                      addActor,
+			                                                      db->forceRecovery);
+
+			collection = actorCollection(db->recoveryData->addActor.getFuture());
+			recoveryCore = clusterRecoveryCore(db->recoveryData);
 
 			// Master failure detection is pretty sensitive, but if we are in the middle of a very long recovery we
 			// really don't want to have to start over
@@ -279,10 +388,10 @@ ACTOR Future<Void> clusterWatchDatabase(ClusterControllerData* cluster,
 				when(wait(db->serverInfo->onChange())) {}
 				when(BackupWorkerDoneRequest req =
 				         waitNext(db->serverInfo->get().clusterInterface.notifyBackupWorkerDone.getFuture())) {
-					if (recoveryData->logSystem.isValid() && recoveryData->logSystem->removeBackupWorker(req)) {
-						recoveryData->registrationTrigger.trigger();
+					if (db->recoveryData->logSystem.isValid() && db->recoveryData->logSystem->removeBackupWorker(req)) {
+						db->recoveryData->registrationTrigger.trigger();
 					}
-					++recoveryData->backupWorkerDoneRequests;
+					++db->recoveryData->backupWorkerDoneRequests;
 					req.reply.send(Void());
 					TraceEvent(SevDebug, "BackupWorkerDoneRequest", cluster->id).log();
 				}
@@ -297,7 +406,7 @@ ACTOR Future<Void> clusterWatchDatabase(ClusterControllerData* cluster,
 			}
 
 			recoveryCore.cancel();
-			wait(cleanupRecoveryActorCollection(recoveryData, /*exThrown=*/false));
+			wait(cleanupRecoveryActorCollection(db->recoveryData, /*exThrown=*/false));
 			ASSERT(addActor.isEmpty());
 
 			wait(spinDelay);
@@ -311,7 +420,7 @@ ACTOR Future<Void> clusterWatchDatabase(ClusterControllerData* cluster,
 				wait(delay(0.0));
 
 			recoveryCore.cancel();
-			wait(cleanupRecoveryActorCollection(recoveryData, true /* exThrown */));
+			wait(cleanupRecoveryActorCollection(db->recoveryData, true /* exThrown */));
 			ASSERT(addActor.isEmpty());
 
 			TEST(err.code() == error_code_tlog_failed); // Terminated due to tLog failure
@@ -2445,11 +2554,13 @@ ACTOR Future<Void> workerHealthMonitor(ClusterControllerData* self) {
 
 			// Compare `self->degradationInfo` with `self->excludedDegradedServers` and remove those that have
 			// recovered.
+			bool hasRecoveredServer = false;
 			for (auto it = self->excludedDegradedServers.begin(); it != self->excludedDegradedServers.end();) {
 				if (self->degradationInfo.degradedServers.find(*it) == self->degradationInfo.degradedServers.end() &&
 				    self->degradationInfo.disconnectedServers.find(*it) ==
 				        self->degradationInfo.disconnectedServers.end()) {
 					self->excludedDegradedServers.erase(it++);
+					hasRecoveredServer = true;
 				} else {
 					++it;
 				}
@@ -2508,6 +2619,12 @@ ACTOR Future<Void> workerHealthMonitor(ClusterControllerData* self) {
 				}
 			}
 
+			if (hasRecoveredServer) {
+				// The best transaction system might have changed after a server is recovered. Check outstanding request
+				// and check if a better transaction system exists.
+				checkOutstandingRequests(self);
+			}
+
 			wait(delay(SERVER_KNOBS->CC_WORKER_HEALTH_CHECKING_INTERVAL));
 		} catch (Error& e) {
 			TraceEvent(SevWarnAlways, "ClusterControllerHealthMonitorError").error(e);
diff --git a/fdbserver/ClusterController.actor.h b/fdbserver/ClusterController.actor.h
index 3502658ab6c..01815195486 100644
--- a/fdbserver/ClusterController.actor.h
+++ b/fdbserver/ClusterController.actor.h
@@ -117,6 +117,8 @@ struct RecruitRemoteWorkersInfo : ReferenceCounted<RecruitRemoteWorkersInfo> {
 	RecruitRemoteWorkersInfo(RecruitRemoteFromConfigurationRequest const& req) : req(req) {}
 };
 
+struct ClusterRecoveryData;
+
 class ClusterControllerData {
 public:
 	struct DBInfo {
@@ -138,6 +140,7 @@ class ClusterControllerData {
 		Future<Void> clientCounter;
 		int clientCount;
 		AsyncVar<bool> blobGranulesEnabled;
+		Reference<ClusterRecoveryData> recoveryData;
 
 		DBInfo()
 		  : clientInfo(new AsyncVar<ClientDBInfo>()), serverInfo(new AsyncVar<ServerDBInfo>()),
@@ -3126,92 +3129,11 @@ class ClusterControllerData {
 	}
 
 	// Whether the transaction system (in primary DC if in HA setting) contains degraded servers.
-	bool transactionSystemContainsDegradedServers() {
-		const ServerDBInfo& dbi = db.serverInfo->get();
-		auto transactionWorkerInList =
-		    [&dbi](const std::unordered_set<NetworkAddress>& serverList, bool skipSatellite, bool skipRemote) -> bool {
-			for (const auto& server : serverList) {
-				if (dbi.master.addresses().contains(server)) {
-					return true;
-				}
-
-				for (const auto& logSet : dbi.logSystemConfig.tLogs) {
-					if (skipSatellite && logSet.locality == tagLocalitySatellite) {
-						continue;
-					}
-
-					if (skipRemote && !logSet.isLocal) {
-						continue;
-					}
-
-					if (!logSet.isLocal) {
-						// Only check log routers in the remote region.
-						for (const auto& logRouter : logSet.logRouters) {
-							if (logRouter.present() && logRouter.interf().addresses().contains(server)) {
-								return true;
-							}
-						}
-					} else {
-						for (const auto& tlog : logSet.tLogs) {
-							if (tlog.present() && tlog.interf().addresses().contains(server)) {
-								return true;
-							}
-						}
-					}
-				}
-
-				for (const auto& proxy : dbi.client.grvProxies) {
-					if (proxy.addresses().contains(server)) {
-						return true;
-					}
-				}
-
-				for (const auto& proxy : dbi.client.commitProxies) {
-					if (proxy.addresses().contains(server)) {
-						return true;
-					}
-				}
-
-				for (const auto& resolver : dbi.resolvers) {
-					if (resolver.addresses().contains(server)) {
-						return true;
-					}
-				}
-			}
-
-			return false;
-		};
-
-		// Check if transaction system contains degraded/disconnected servers. For satellite and remote regions, we only
-		// check for disconnection since the latency between prmary and satellite is across WAN and may not be very
-		// stable.
-		return transactionWorkerInList(degradationInfo.degradedServers, /*skipSatellite=*/true, /*skipRemote=*/true) ||
-		       transactionWorkerInList(degradationInfo.disconnectedServers,
-		                               /*skipSatellite=*/false,
-		                               /*skipRemote=*/!SERVER_KNOBS->CC_ENABLE_REMOTE_LOG_ROUTER_MONITORING);
-	}
+	bool transactionSystemContainsDegradedServers();
 
 	// Whether transaction system in the remote DC, e.g. log router and tlogs in the remote DC, contains degraded
 	// servers.
-	bool remoteTransactionSystemContainsDegradedServers() {
-		if (db.config.usableRegions <= 1) {
-			return false;
-		}
-
-		for (const auto& excludedServer : degradationInfo.degradedServers) {
-			if (addressInDbAndRemoteDc(excludedServer, db.serverInfo)) {
-				return true;
-			}
-		}
-
-		for (const auto& excludedServer : degradationInfo.disconnectedServers) {
-			if (addressInDbAndRemoteDc(excludedServer, db.serverInfo)) {
-				return true;
-			}
-		}
-
-		return false;
-	}
+	bool remoteTransactionSystemContainsDegradedServers();
 
 	// Returns true if remote DC is healthy and can failover to.
 	bool remoteDCIsHealthy() {
@@ -3247,10 +3169,6 @@ class ClusterControllerData {
 			return false;
 		}
 
-		if (db.serverInfo->get().recoveryState < RecoveryState::ACCEPTING_COMMITS) {
-			return false;
-		}
-
 		// Do not trigger recovery if the cluster controller is excluded, since the master will change
 		// anyways once the cluster controller is moved
 		if (id_worker[clusterControllerProcessId].priorityInfo.isExcluded) {
diff --git a/fdbserver/ClusterRecovery.actor.cpp b/fdbserver/ClusterRecovery.actor.cpp
index da676149639..2946c4f8824 100644
--- a/fdbserver/ClusterRecovery.actor.cpp
+++ b/fdbserver/ClusterRecovery.actor.cpp
@@ -979,6 +979,10 @@ ACTOR Future<std::vector<Standalone<CommitTransactionRef>>> recruitEverything(
 	}
 	self->backupWorkers.swap(recruits.backupWorkers);
 
+	// Store recruitment result, which may be used to check the current being recruited transaction system in gray
+	// failure detection.
+	self->primaryRecruitment = recruits;
+
 	TraceEvent(getRecoveryEventName(ClusterRecoveryEventType::CLUSTER_RECOVERY_STATE_EVENT_NAME).c_str(), self->dbgid)
 	    .detail("StatusCode", RecoveryStatus::initializing_transaction_servers)
 	    .detail("Status", RecoveryStatus::names[RecoveryStatus::initializing_transaction_servers])
diff --git a/fdbserver/ClusterRecovery.actor.h b/fdbserver/ClusterRecovery.actor.h
index d0deef911f1..9359a142322 100644
--- a/fdbserver/ClusterRecovery.actor.h
+++ b/fdbserver/ClusterRecovery.actor.h
@@ -48,6 +48,8 @@
 
 #include "flow/actorcompiler.h" // This must be the last #include.
 
+class ClusterControllerData;
+
 typedef enum {
 	CLUSTER_RECOVERY_STATE_EVENT_NAME,
 	CLUSTER_RECOVERY_COMMIT_TLOG_EVENT_NAME,
@@ -193,6 +195,8 @@ struct ClusterRecoveryData : NonCopyable, ReferenceCounted<ClusterRecoveryData>
 	std::map<Optional<Value>, int8_t> dcId_locality;
 	std::vector<Tag> allTags;
 
+	RecruitFromConfigurationReply primaryRecruitment;
+
 	int8_t getNextLocality() {
 		int8_t maxLocality = -1;
 		for (auto it : dcId_locality) {
diff --git a/fdbserver/OldTLogServer_6_0.actor.cpp b/fdbserver/OldTLogServer_6_0.actor.cpp
index 9c4f3ef611e..1389399b552 100644
--- a/fdbserver/OldTLogServer_6_0.actor.cpp
+++ b/fdbserver/OldTLogServer_6_0.actor.cpp
@@ -2809,7 +2809,8 @@ ACTOR Future<Void> tLog(IKeyValueStore* persistentData,
                         Promise<Void> recovered,
                         std::string folder,
                         Reference<AsyncVar<bool>> degraded,
-                        Reference<AsyncVar<UID>> activeSharedTLog) {
+                        Reference<AsyncVar<UID>> activeSharedTLog,
+                        Reference<AsyncVar<bool>> enablePrimaryTxnSystemHealthCheck) {
 	state TLogData self(tlogId, workerID, persistentData, persistentQueue, db, degraded, folder);
 	state Future<Void> error = actorCollection(self.sharedActors.getFuture());
 
diff --git a/fdbserver/OldTLogServer_6_2.actor.cpp b/fdbserver/OldTLogServer_6_2.actor.cpp
index f591d95f5e1..23a60865b6d 100644
--- a/fdbserver/OldTLogServer_6_2.actor.cpp
+++ b/fdbserver/OldTLogServer_6_2.actor.cpp
@@ -3300,7 +3300,8 @@ ACTOR Future<Void> tLog(IKeyValueStore* persistentData,
                         Promise<Void> recovered,
                         std::string folder,
                         Reference<AsyncVar<bool>> degraded,
-                        Reference<AsyncVar<UID>> activeSharedTLog) {
+                        Reference<AsyncVar<UID>> activeSharedTLog,
+                        Reference<AsyncVar<bool>> enablePrimaryTxnSystemHealthCheck) {
 	state TLogData self(tlogId, workerID, persistentData, persistentQueue, db, degraded, folder);
 	state Future<Void> error = actorCollection(self.sharedActors.getFuture());
 
diff --git a/fdbserver/TLogServer.actor.cpp b/fdbserver/TLogServer.actor.cpp
index a4193ea90f1..fbe04c1a801 100644
--- a/fdbserver/TLogServer.actor.cpp
+++ b/fdbserver/TLogServer.actor.cpp
@@ -373,13 +373,17 @@ struct TLogData : NonCopyable {
 
 	Reference<Histogram> commitLatencyDist;
 
+	// Controls whether the health monitoring running in this TLog force checking any other processes are degraded.
+	Reference<AsyncVar<bool>> enablePrimaryTxnSystemHealthCheck;
+
 	TLogData(UID dbgid,
 	         UID workerID,
 	         IKeyValueStore* persistentData,
 	         IDiskQueue* persistentQueue,
 	         Reference<AsyncVar<ServerDBInfo> const> dbInfo,
 	         Reference<AsyncVar<bool>> degraded,
-	         std::string folder)
+	         std::string folder,
+	         Reference<AsyncVar<bool>> enablePrimaryTxnSystemHealthCheck)
 	  : dbgid(dbgid), workerID(workerID), persistentData(persistentData), rawPersistentQueue(persistentQueue),
 	    persistentQueue(new TLogQueue(persistentQueue, dbgid)), diskQueueCommitBytes(0),
 	    largeDiskQueueCommitBytes(false), dbInfo(dbInfo), queueCommitEnd(0), queueCommitBegin(0),
@@ -389,7 +393,8 @@ struct TLogData : NonCopyable {
 	    concurrentLogRouterReads(SERVER_KNOBS->CONCURRENT_LOG_ROUTER_READS), ignorePopDeadline(0), dataFolder(folder),
 	    degraded(degraded), commitLatencyDist(Histogram::getHistogram(LiteralStringRef("tLog"),
 	                                                                  LiteralStringRef("commit"),
-	                                                                  Histogram::Unit::microseconds)) {
+	                                                                  Histogram::Unit::microseconds)),
+	    enablePrimaryTxnSystemHealthCheck(enablePrimaryTxnSystemHealthCheck) {
 		cx = openDBOnServer(dbInfo, TaskPriority::DefaultEndpoint, LockAware::True);
 	}
 };
@@ -3548,13 +3553,20 @@ ACTOR Future<Void> tLogStart(TLogData* self, InitializeTLogRequest req, Locality
 					logData->logRouterPopToVersion = recoverAt;
 					std::vector<Tag> tags;
 					tags.push_back(logData->remoteTag);
+					// Force gray failure monitoring during recovery.
+					self->enablePrimaryTxnSystemHealthCheck->set(true);
 					wait(pullAsyncData(self, logData, tags, logData->unrecoveredBefore, recoverAt, true) ||
 					     logData->removed || logData->stopCommit.onTrigger());
+					self->enablePrimaryTxnSystemHealthCheck->set(false);
 				} else if (!req.recoverTags.empty()) {
 					ASSERT(logData->unrecoveredBefore > req.knownCommittedVersion);
+
+					// Force gray failure monitoring during recovery.
+					self->enablePrimaryTxnSystemHealthCheck->set(true);
 					wait(pullAsyncData(
 					         self, logData, req.recoverTags, req.knownCommittedVersion + 1, recoverAt, false) ||
 					     logData->removed || logData->stopCommit.onTrigger());
+					self->enablePrimaryTxnSystemHealthCheck->set(false);
 				}
 				pulledRecoveryVersions = true;
 				logData->knownCommittedVersion = recoverAt;
@@ -3651,8 +3663,10 @@ ACTOR Future<Void> tLog(IKeyValueStore* persistentData,
                         Promise<Void> recovered,
                         std::string folder,
                         Reference<AsyncVar<bool>> degraded,
-                        Reference<AsyncVar<UID>> activeSharedTLog) {
-	state TLogData self(tlogId, workerID, persistentData, persistentQueue, db, degraded, folder);
+                        Reference<AsyncVar<UID>> activeSharedTLog,
+                        Reference<AsyncVar<bool>> enablePrimaryTxnSystemHealthCheck) {
+	state TLogData self(
+	    tlogId, workerID, persistentData, persistentQueue, db, degraded, folder, enablePrimaryTxnSystemHealthCheck);
 	state Future<Void> error = actorCollection(self.sharedActors.getFuture());
 
 	TraceEvent("SharedTlog", tlogId).log();
diff --git a/fdbserver/WorkerInterface.actor.h b/fdbserver/WorkerInterface.actor.h
index 10d930542b2..44032d805eb 100644
--- a/fdbserver/WorkerInterface.actor.h
+++ b/fdbserver/WorkerInterface.actor.h
@@ -1132,7 +1132,8 @@ ACTOR Future<Void> tLog(IKeyValueStore* persistentData,
                         Promise<Void> recovered,
                         std::string folder,
                         Reference<AsyncVar<bool>> degraded,
-                        Reference<AsyncVar<UID>> activeSharedTLog);
+                        Reference<AsyncVar<UID>> activeSharedTLog,
+                        Reference<AsyncVar<bool>> enablePrimaryTxnSystemHealthCheck);
 ACTOR Future<Void> resolver(ResolverInterface resolver,
                             InitializeResolverRequest initReq,
                             Reference<AsyncVar<ServerDBInfo> const> db);
@@ -1181,7 +1182,8 @@ ACTOR Future<Void> tLog(IKeyValueStore* persistentData,
                         Promise<Void> recovered,
                         std::string folder,
                         Reference<AsyncVar<bool>> degraded,
-                        Reference<AsyncVar<UID>> activeSharedTLog);
+                        Reference<AsyncVar<UID>> activeSharedTLog,
+                        Reference<AsyncVar<bool>> enablePrimaryTxnSystemHealthCheck);
 }
 namespace oldTLog_6_2 {
 ACTOR Future<Void> tLog(IKeyValueStore* persistentData,
@@ -1196,7 +1198,8 @@ ACTOR Future<Void> tLog(IKeyValueStore* persistentData,
                         Promise<Void> recovered,
                         std::string folder,
                         Reference<AsyncVar<bool>> degraded,
-                        Reference<AsyncVar<UID>> activeSharedTLog);
+                        Reference<AsyncVar<UID>> activeSharedTLog,
+                        Reference<AsyncVar<bool>> enablePrimaryTxnSystemHealthCheck);
 }
 
 typedef decltype(&tLog) TLogFn;
diff --git a/fdbserver/worker.actor.cpp b/fdbserver/worker.actor.cpp
index 4d4dc236fdf..472c343bd66 100644
--- a/fdbserver/worker.actor.cpp
+++ b/fdbserver/worker.actor.cpp
@@ -1053,7 +1053,8 @@ bool isDegradedPeer(const UpdateWorkerHealthRequest& lastReq, const NetworkAddre
 UpdateWorkerHealthRequest doPeerHealthCheck(const WorkerInterface& interf,
                                             const LocalityData& locality,
                                             Reference<AsyncVar<ServerDBInfo> const> dbInfo,
-                                            const UpdateWorkerHealthRequest& lastReq) {
+                                            const UpdateWorkerHealthRequest& lastReq,
+                                            Reference<AsyncVar<bool>> enablePrimaryTxnSystemHealthCheck) {
 	const auto& allPeers = FlowTransport::transport().getAllPeers();
 
 	// Check remote log router connectivity only when remote TLogs are recruited and in use.
@@ -1071,8 +1072,8 @@ UpdateWorkerHealthRequest doPeerHealthCheck(const WorkerInterface& interf,
 		workerLocation = Satellite;
 	}
 
-	if (workerLocation == None) {
-		// This worker doesn't need to monitor anything if it is in remote satellite.
+	if (workerLocation == None && !enablePrimaryTxnSystemHealthCheck->get()) {
+		// This worker doesn't need to monitor anything if it is not in transaction system or in remote satellite.
 		return req;
 	}
 
@@ -1086,6 +1087,22 @@ UpdateWorkerHealthRequest doPeerHealthCheck(const WorkerInterface& interf,
 
 		// If peer->lastLoggedTime == 0, we just started monitor this peer and haven't logged it once yet.
 		double lastLoggedTime = peer->lastLoggedTime <= 0.0 ? peer->lastConnectTime : peer->lastLoggedTime;
+
+		TraceEvent(SevDebug, "PeerHealthMonitor")
+		    .detail("Peer", address)
+		    .detail("Force", enablePrimaryTxnSystemHealthCheck->get())
+		    .detail("Elapsed", now() - lastLoggedTime)
+		    .detail("Disconnected", disconnectedPeer)
+		    .detail("MinLatency", peer->pingLatencies.min())
+		    .detail("MaxLatency", peer->pingLatencies.max())
+		    .detail("MeanLatency", peer->pingLatencies.mean())
+		    .detail("MedianLatency", peer->pingLatencies.median())
+		    .detail("CheckedPercentile", SERVER_KNOBS->PEER_LATENCY_DEGRADATION_PERCENTILE)
+		    .detail("CheckedPercentileLatency",
+		            peer->pingLatencies.percentile(SERVER_KNOBS->PEER_LATENCY_DEGRADATION_PERCENTILE))
+		    .detail("PingCount", peer->pingLatencies.getPopulationSize())
+		    .detail("PingTimeoutCount", peer->timeoutCount)
+		    .detail("ConnectionFailureCount", peer->connectFailedCount);
 		if ((workerLocation == Primary && addressInDbAndPrimaryDc(address, dbInfo)) ||
 		    (workerLocation == Remote && addressInDbAndRemoteDc(address, dbInfo))) {
 			// Monitors intra DC latencies between servers that in the primary or remote DC's transaction
@@ -1166,6 +1183,26 @@ UpdateWorkerHealthRequest doPeerHealthCheck(const WorkerInterface& interf,
 				    .detail("ConnectionFailureCount", peer->connectFailedCount);
 				disconnectedPeer = true;
 			}
+		} else if (enablePrimaryTxnSystemHealthCheck->get() &&
+		           (addressInDbAndPrimaryDc(address, dbInfo) || addressInDbAndPrimarySatelliteDc(address, dbInfo))) {
+			// For force checking, we only detect connection timeout. Currently this should only be used during recovery
+			// and only used in TLogs.
+			if (peer->connectFailedCount >= SERVER_KNOBS->PEER_DEGRADATION_CONNECTION_FAILURE_COUNT) {
+				TraceEvent("HealthMonitorDetectDegradedPeer")
+				    .detail("WorkerLocation", workerLocation)
+				    .detail("Peer", address)
+				    .detail("ExtensiveConnectivityCheck", true)
+				    .detail("Elapsed", now() - lastLoggedTime)
+				    .detail("Disconnected", true)
+				    .detail("MinLatency", peer->pingLatencies.min())
+				    .detail("MaxLatency", peer->pingLatencies.max())
+				    .detail("MeanLatency", peer->pingLatencies.mean())
+				    .detail("MedianLatency", peer->pingLatencies.median())
+				    .detail("PingCount", peer->pingLatencies.getPopulationSize())
+				    .detail("PingTimeoutCount", peer->timeoutCount)
+				    .detail("ConnectionFailureCount", peer->connectFailedCount);
+				disconnectedPeer = true;
+			}
 		}
 
 		if (disconnectedPeer) {
@@ -1228,13 +1265,16 @@ UpdateWorkerHealthRequest doPeerHealthCheck(const WorkerInterface& interf,
 ACTOR Future<Void> healthMonitor(Reference<AsyncVar<Optional<ClusterControllerFullInterface>> const> ccInterface,
                                  WorkerInterface interf,
                                  LocalityData locality,
-                                 Reference<AsyncVar<ServerDBInfo> const> dbInfo) {
+                                 Reference<AsyncVar<ServerDBInfo> const> dbInfo,
+                                 Reference<AsyncVar<bool>> enablePrimaryTxnSystemHealthCheck) {
 	state UpdateWorkerHealthRequest req;
 	loop {
 		Future<Void> nextHealthCheckDelay = Never();
-		if (dbInfo->get().recoveryState >= RecoveryState::ACCEPTING_COMMITS && ccInterface->get().present()) {
+		if ((dbInfo->get().recoveryState >= RecoveryState::ACCEPTING_COMMITS ||
+		     enablePrimaryTxnSystemHealthCheck->get()) &&
+		    ccInterface->get().present()) {
 			nextHealthCheckDelay = delay(SERVER_KNOBS->WORKER_HEALTH_MONITOR_INTERVAL);
-			req = doPeerHealthCheck(interf, locality, dbInfo, req);
+			req = doPeerHealthCheck(interf, locality, dbInfo, req, enablePrimaryTxnSystemHealthCheck);
 
 			if (!req.disconnectedPeers.empty() || !req.degradedPeers.empty() || !req.recoveredPeers.empty()) {
 				if (g_network->isSimulated()) {
@@ -1259,6 +1299,7 @@ ACTOR Future<Void> healthMonitor(Reference<AsyncVar<Optional<ClusterControllerFu
 			when(wait(nextHealthCheckDelay)) {}
 			when(wait(ccInterface->onChange())) {}
 			when(wait(dbInfo->onChange())) {}
+			when(wait(enablePrimaryTxnSystemHealthCheck->onChange())) {}
 		}
 	}
 }
@@ -1787,6 +1828,10 @@ ACTOR Future<Void> workerServer(Reference<IClusterConnectionRecord> connRecord,
 
 	state Reference<AsyncVar<std::set<std::string>>> issues(new AsyncVar<std::set<std::string>>());
 
+	// When set to true, the health monitor running in this worker starts monitor other transaction process in this
+	// cluster.
+	state Reference<AsyncVar<bool>> enablePrimaryTxnSystemHealthCheck = makeReference<AsyncVar<bool>>(false);
+
 	if (FLOW_KNOBS->ENABLE_CHAOS_FEATURES) {
 		TraceEvent(SevInfo, "ChaosFeaturesEnabled");
 		chaosMetricsActor = chaosMetricsLogger();
@@ -1985,7 +2030,8 @@ ACTOR Future<Void> workerServer(Reference<IClusterConnectionRecord> connRecord,
 				           recovery,
 				           folder,
 				           degraded,
-				           activeSharedTLog);
+				           activeSharedTLog,
+				           enablePrimaryTxnSystemHealthCheck);
 				recoveries.push_back(recovery.getFuture());
 				activeSharedTLog->set(s.storeID);
 
@@ -2060,7 +2106,8 @@ ACTOR Future<Void> workerServer(Reference<IClusterConnectionRecord> connRecord,
 		}
 
 		if (SERVER_KNOBS->ENABLE_WORKER_HEALTH_MONITOR) {
-			errorForwarders.add(healthMonitor(ccInterface, interf, locality, dbInfo));
+			errorForwarders.add(
+			    healthMonitor(ccInterface, interf, locality, dbInfo, enablePrimaryTxnSystemHealthCheck));
 		}
 
 		TraceEvent("RecoveriesComplete", interf.id());
@@ -2377,7 +2424,8 @@ ACTOR Future<Void> workerServer(Reference<IClusterConnectionRecord> connRecord,
 					                               Promise<Void>(),
 					                               folder,
 					                               degraded,
-					                               activeSharedTLog);
+					                               activeSharedTLog,
+					                               enablePrimaryTxnSystemHealthCheck);
 					tLogCore = handleIOErrors(tLogCore, data, logId);
 					tLogCore = handleIOErrors(tLogCore, queue, logId);
 					errorForwarders.add(forwardError(errors, Role::SHARED_TRANSACTION_LOG, logId, tLogCore));
diff --git a/fdbserver/workloads/ClogTlog.actor.cpp b/fdbserver/workloads/ClogTlog.actor.cpp
index 46980a4415c..188d358f8bc 100644
--- a/fdbserver/workloads/ClogTlog.actor.cpp
+++ b/fdbserver/workloads/ClogTlog.actor.cpp
@@ -167,7 +167,7 @@ struct ClogTlogWorkload : TestWorkload {
 		}
 
 		double startTime = now();
-		state double workloadEnd = now() + self->testDuration;
+		state double workloadEnd = now() + self->testDuration - 10;
 		TraceEvent("ClogTlog").detail("StartTime", startTime).detail("EndTime", workloadEnd);
 
 		// Clog and wait for recovery to happen
@@ -176,13 +176,21 @@ struct ClogTlogWorkload : TestWorkload {
 			wait(self->dbInfo->onChange());
 		}
 
-		// start exclusion and wait for fully recovery
-		state Future<Void> excludeLog = excludeFailedLog(self, cx);
+		state bool useGrayFailureToRecover = false;
+		if (deterministicRandom()->coinflip() && self->useDisconnection) {
+			// Use gray failure instead of exclusion to recover the cluster.
+			TraceEvent("ClogTlogUseGrayFailreToRecover").log();
+			useGrayFailureToRecover = true;
+		}
+
+		// start exclusion and wait for fully recovery. When using gray failure, the cluster should recover by itself
+		// eventually.
+		state Future<Void> excludeLog = useGrayFailureToRecover ? Never() : excludeFailedLog(self, cx);
 		state Future<Void> onChange = self->dbInfo->onChange();
 		loop choose {
 			when(wait(onChange)) {
 				if (self->dbInfo->get().recoveryState == RecoveryState::FULLY_RECOVERED) {
-					TraceEvent("ClogDoneFullyRecovered");
+					TraceEvent("ClogDoneFullyRecovered").log();
 					self->unclogAll();
 					return Void();
 				}
diff --git a/tests/rare/ClogTlog.toml b/tests/rare/ClogTlog.toml
index d5b5dc8096a..6e961420ca2 100644
--- a/tests/rare/ClogTlog.toml
+++ b/tests/rare/ClogTlog.toml
@@ -9,6 +9,10 @@ desiredTLogCount = 6
 [[knobs]]
 enable_worker_health_monitor = true
 cc_enable_worker_health_monitor = true
+cc_worker_health_checking_interval = 15
+cc_min_degradation_interval = 30
+cc_health_trigger_recovery = true
+peer_latency_degradation_threshold = 1
 
 [[test]]
 testTitle = 'ClogTlog'
@@ -22,4 +26,4 @@ testTitle = 'ClogTlog'
 
     [[test.workload]]
     testName = 'ClogTlog'
-    testDuration = 150.0
+    testDuration = 250.0

From 17230becdec6051dab61d7eebf94eb204f2d9cd6 Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyu_zhou@apple.com>
Date: Thu, 24 Aug 2023 15:36:42 -0700
Subject: [PATCH 217/468] Add a knob RESOLVE_PREFER_IPV4_ADDR to prefer IPv4
 addresses (#10826)

cherrypick #10822
---
 flow/Knobs.cpp   | 1 +
 flow/Knobs.h     | 1 +
 flow/network.cpp | 3 ++-
 flow/network.h   | 7 +++++++
 4 files changed, 11 insertions(+), 1 deletion(-)

diff --git a/flow/Knobs.cpp b/flow/Knobs.cpp
index 203fa460dec..1adbbe95874 100644
--- a/flow/Knobs.cpp
+++ b/flow/Knobs.cpp
@@ -198,6 +198,7 @@ void FlowKnobs::initialize(Randomize randomize, IsSimulated isSimulated) {
 	init( MIN_PACKET_BUFFER_FREE_BYTES,                        256 );
 	init( FLOW_TCP_NODELAY,                                      1 );
 	init( FLOW_TCP_QUICKACK,                                     0 );
+	init( RESOLVE_PREFER_IPV4_ADDR,                          false );  // Default to prefer IPv6 addresses. Set to true to prefer IPv4 addresses.
 
 	//Sim2
 	init( MIN_OPEN_TIME,                                    0.0002 );
diff --git a/flow/Knobs.h b/flow/Knobs.h
index b5a81fc325d..848e953b624 100644
--- a/flow/Knobs.h
+++ b/flow/Knobs.h
@@ -266,6 +266,7 @@ class FlowKnobs : public KnobsImpl<FlowKnobs> {
 	int MIN_PACKET_BUFFER_FREE_BYTES;
 	int FLOW_TCP_NODELAY;
 	int FLOW_TCP_QUICKACK;
+	bool RESOLVE_PREFER_IPV4_ADDR;
 
 	// Sim2
 	// FIMXE: more parameters could be factored out
diff --git a/flow/network.cpp b/flow/network.cpp
index a13f0657614..9609f7c530e 100644
--- a/flow/network.cpp
+++ b/flow/network.cpp
@@ -360,7 +360,8 @@ TEST_CASE("/flow/network/ipV6Preferred") {
 		addresses.push_back(NetworkAddress::parse(s));
 	}
 	// Confirm IPv6 is always preferred.
-	ASSERT(INetworkConnections::pickOneAddress(addresses).toString() == ipv6);
+	ASSERT((INetworkConnections::pickOneAddress(addresses).toString() == ipv6) ==
+	       !FLOW_KNOBS->RESOLVE_PREFER_IPV4_ADDR);
 
 	return Void();
 }
diff --git a/flow/network.h b/flow/network.h
index 186b91d8fc6..f89da5fd29e 100644
--- a/flow/network.h
+++ b/flow/network.h
@@ -740,13 +740,20 @@ class INetworkConnections {
 	// If a DNS name can be resolved to both and IPv4 and IPv6 addresses, we want IPv6 addresses when running the
 	// clusters on IPv6.
 	// This function takes a vector of addresses and return a random one, preferring IPv6 over IPv4.
+	// To prefer IPv4 addresses instead, set knob RESOLVE_PREFER_IPV4_ADDR to true.
 	static NetworkAddress pickOneAddress(const std::vector<NetworkAddress>& addresses) {
 		std::vector<NetworkAddress> ipV6Addresses;
+		std::vector<NetworkAddress> ipV4Addresses;
 		for (const NetworkAddress& addr : addresses) {
 			if (addr.isV6()) {
 				ipV6Addresses.push_back(addr);
+			} else {
+				ipV4Addresses.push_back(addr);
 			}
 		}
+		if (ipV4Addresses.size() > 0 && FLOW_KNOBS->RESOLVE_PREFER_IPV4_ADDR) {
+			return ipV4Addresses[deterministicRandom()->randomInt(0, ipV4Addresses.size())];
+		}
 		if (ipV6Addresses.size() > 0) {
 			return ipV6Addresses[deterministicRandom()->randomInt(0, ipV6Addresses.size())];
 		}

From ba1da20029bb5c7a1602e56b56e51e0b780209d1 Mon Sep 17 00:00:00 2001
From: Zhe Wu <halfprice@users.noreply.github.com>
Date: Tue, 29 Aug 2023 14:49:39 -0700
Subject: [PATCH 218/468] Add a knob to guard the gray failure detection during
 TLog recovery

---
 fdbclient/ServerKnobs.cpp             |  1 +
 fdbclient/ServerKnobs.h               |  2 ++
 fdbserver/ClusterController.actor.cpp |  3 ++-
 fdbserver/TLogServer.actor.cpp        | 13 +++++++++----
 4 files changed, 14 insertions(+), 5 deletions(-)

diff --git a/fdbclient/ServerKnobs.cpp b/fdbclient/ServerKnobs.cpp
index 8c33b8993b4..f0c7ba3c804 100644
--- a/fdbclient/ServerKnobs.cpp
+++ b/fdbclient/ServerKnobs.cpp
@@ -801,6 +801,7 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi
 	init( PEER_TIMEOUT_PERCENTAGE_DEGRADATION_THRESHOLD,         0.1 );
 	init( PEER_DEGRADATION_CONNECTION_FAILURE_COUNT,               5 );
 	init( WORKER_HEALTH_REPORT_RECENT_DESTROYED_PEER,           true );
+	init( GRAY_FAILURE_ENABLE_TLOG_RECOVERY_MONITORING,         true );
 	init( STORAGE_SERVER_REBOOT_ON_IO_TIMEOUT,                 false ); if ( randomize && BUGGIFY ) STORAGE_SERVER_REBOOT_ON_IO_TIMEOUT = true;
 	init( CONSISTENCY_CHECK_ROCKSDB_ENGINE,                    false );
 	init( CONSISTENCY_CHECK_SQLITE_ENGINE,                     false );
diff --git a/fdbclient/ServerKnobs.h b/fdbclient/ServerKnobs.h
index 7715673f27f..d22aa0eb0da 100644
--- a/fdbclient/ServerKnobs.h
+++ b/fdbclient/ServerKnobs.h
@@ -734,6 +734,8 @@ class ServerKnobs : public KnobsImpl<ServerKnobs> {
 	bool WORKER_HEALTH_REPORT_RECENT_DESTROYED_PEER; // When enabled, the worker's health monitor also report any recent
 	                                                 // destroyed peers who are part of the transaction system to
 	                                                 // cluster controller.
+	bool GRAY_FAILURE_ENABLE_TLOG_RECOVERY_MONITORING; // When enabled, health monitor will try to detect any gray
+	                                                   // failure during tlog recovery during the recovery process.
 	bool STORAGE_SERVER_REBOOT_ON_IO_TIMEOUT; // When enabled, storage server's worker will crash on io_timeout error;
 	                                          // this allows fdbmonitor to restart the worker and recreate the same SS.
 	                                          // When SS can be temporarily throttled by infrastructure, e.g, k8s,
diff --git a/fdbserver/ClusterController.actor.cpp b/fdbserver/ClusterController.actor.cpp
index 49daf4ca0cb..6dedda5ce8b 100644
--- a/fdbserver/ClusterController.actor.cpp
+++ b/fdbserver/ClusterController.actor.cpp
@@ -235,7 +235,8 @@ bool ClusterControllerData::transactionSystemContainsDegradedServers() {
 				}
 			}
 
-			if (recoveryData.isValid() && recoveryData->recoveryState < RecoveryState::ACCEPTING_COMMITS) {
+			if (SERVER_KNOBS->GRAY_FAILURE_ENABLE_TLOG_RECOVERY_MONITORING && recoveryData.isValid() &&
+			    recoveryData->recoveryState < RecoveryState::ACCEPTING_COMMITS) {
 				// During recovery, TLogs may not be able to pull data from previous generation TLogs due to gray
 				// failures. In this case, we rely on the latest recruitment information and see if any newly recruited
 				// TLogs are degraded.
diff --git a/fdbserver/TLogServer.actor.cpp b/fdbserver/TLogServer.actor.cpp
index fbe04c1a801..76fa84a9016 100644
--- a/fdbserver/TLogServer.actor.cpp
+++ b/fdbserver/TLogServer.actor.cpp
@@ -3553,16 +3553,21 @@ ACTOR Future<Void> tLogStart(TLogData* self, InitializeTLogRequest req, Locality
 					logData->logRouterPopToVersion = recoverAt;
 					std::vector<Tag> tags;
 					tags.push_back(logData->remoteTag);
-					// Force gray failure monitoring during recovery.
-					self->enablePrimaryTxnSystemHealthCheck->set(true);
+
+					if (SERVER_KNOBS->GRAY_FAILURE_ENABLE_TLOG_RECOVERY_MONITORING) {
+						// Force gray failure monitoring during recovery.
+						self->enablePrimaryTxnSystemHealthCheck->set(true);
+					}
 					wait(pullAsyncData(self, logData, tags, logData->unrecoveredBefore, recoverAt, true) ||
 					     logData->removed || logData->stopCommit.onTrigger());
 					self->enablePrimaryTxnSystemHealthCheck->set(false);
 				} else if (!req.recoverTags.empty()) {
 					ASSERT(logData->unrecoveredBefore > req.knownCommittedVersion);
 
-					// Force gray failure monitoring during recovery.
-					self->enablePrimaryTxnSystemHealthCheck->set(true);
+					if (SERVER_KNOBS->GRAY_FAILURE_ENABLE_TLOG_RECOVERY_MONITORING) {
+						// Force gray failure monitoring during recovery.
+						self->enablePrimaryTxnSystemHealthCheck->set(true);
+					}
 					wait(pullAsyncData(
 					         self, logData, req.recoverTags, req.knownCommittedVersion + 1, recoverAt, false) ||
 					     logData->removed || logData->stopCommit.onTrigger());

From cb3b247ae830eb488d9f6f29eec293abc9e0663a Mon Sep 17 00:00:00 2001
From: neethuhaneesha <nbingi@apple.com>
Date: Wed, 6 Sep 2023 12:21:34 -0700
Subject: [PATCH 219/468] Rocksdb delete log options.

---
 fdbclient/ServerKnobs.cpp                | 2 ++
 fdbclient/ServerKnobs.h                  | 2 ++
 fdbserver/KeyValueStoreRocksDB.actor.cpp | 2 ++
 3 files changed, 6 insertions(+)

diff --git a/fdbclient/ServerKnobs.cpp b/fdbclient/ServerKnobs.cpp
index f0c7ba3c804..fe5a266dcbd 100644
--- a/fdbclient/ServerKnobs.cpp
+++ b/fdbclient/ServerKnobs.cpp
@@ -444,6 +444,8 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi
 	init( ROCKSDB_CAN_COMMIT_DELAY_TIMES_ON_OVERLOAD,              5 );
 	init( ROCKSDB_COMPACTION_READAHEAD_SIZE,                   32768 ); // 32 KB, performs bigger reads when doing compaction.
 	init( ROCKSDB_BLOCK_SIZE,                                  32768 ); // 32 KB, size of the block in rocksdb cache.
+	init( ROCKSDB_MAX_LOG_FILE_SIZE,                        10485760 ); // 10MB.
+	init( ROCKSDB_KEEP_LOG_FILE_NUM,                             200 ); // Keeps 2GB log per storage server.
 	// Temporary knob to enable trace events which prints details about all the backup keys update(write/clear) operations.
 	// Caution: To be enabled only under supervision.
 	init( SS_BACKUP_KEYS_OP_LOGS,                              false );
diff --git a/fdbclient/ServerKnobs.h b/fdbclient/ServerKnobs.h
index d22aa0eb0da..53c7c29b13c 100644
--- a/fdbclient/ServerKnobs.h
+++ b/fdbclient/ServerKnobs.h
@@ -355,6 +355,8 @@ class ServerKnobs : public KnobsImpl<ServerKnobs> {
 	int ROCKSDB_STATS_LEVEL;
 	int64_t ROCKSDB_COMPACTION_READAHEAD_SIZE;
 	int64_t ROCKSDB_BLOCK_SIZE;
+	int ROCKSDB_MAX_LOG_FILE_SIZE;
+	int ROCKSDB_KEEP_LOG_FILE_NUM;
 	bool SS_BACKUP_KEYS_OP_LOGS;
 
 	// Leader election
diff --git a/fdbserver/KeyValueStoreRocksDB.actor.cpp b/fdbserver/KeyValueStoreRocksDB.actor.cpp
index 5b8d6d035b7..491c7ba237a 100644
--- a/fdbserver/KeyValueStoreRocksDB.actor.cpp
+++ b/fdbserver/KeyValueStoreRocksDB.actor.cpp
@@ -425,6 +425,8 @@ rocksdb::Options getOptions() {
 	options.statistics = rocksdb::CreateDBStatistics();
 	options.statistics->set_stats_level(rocksdb::StatsLevel(SERVER_KNOBS->ROCKSDB_STATS_LEVEL));
 
+	options.max_log_file_size = SERVER_KNOBS->ROCKSDB_MAX_LOG_FILE_SIZE;
+	options.keep_log_file_num = SERVER_KNOBS->ROCKSDB_KEEP_LOG_FILE_NUM;
 	options.db_log_dir = SERVER_KNOBS->LOG_DIRECTORY;
 	return options;
 }

From 37c7184aa95ada3b729811505e74fb2287d5a1ca Mon Sep 17 00:00:00 2001
From: neethuhaneesha <nbingi@apple.com>
Date: Wed, 6 Sep 2023 21:15:59 -0700
Subject: [PATCH 220/468] Perpetual wiggle pause on balance of the cluster
 (#10865)

---
 fdbclient/ServerKnobs.cpp            |  6 +-
 fdbclient/ServerKnobs.h              | 11 +++-
 fdbserver/DDTeamCollection.actor.cpp | 93 ++++++++++++++++++++++------
 fdbserver/DDTeamCollection.h         |  8 ++-
 fdbserver/DataDistribution.actor.cpp |  6 +-
 5 files changed, 97 insertions(+), 27 deletions(-)

diff --git a/fdbclient/ServerKnobs.cpp b/fdbclient/ServerKnobs.cpp
index fe5a266dcbd..f64e98fa846 100644
--- a/fdbclient/ServerKnobs.cpp
+++ b/fdbclient/ServerKnobs.cpp
@@ -233,7 +233,11 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi
 	init( ALL_DATA_REMOVED_DELAY,                                1.0 );
 	init( INITIAL_FAILURE_REACTION_DELAY,                       30.0 ); if( randomize && BUGGIFY ) INITIAL_FAILURE_REACTION_DELAY = 0.0;
 	init( CHECK_TEAM_DELAY,                                     30.0 );
-	init( PERPETUAL_WIGGLE_DELAY,                               50.0 );
+	// This is a safety knob to avoid busy spinning and the case a small cluster don't have enough space when excluding and including too fast. The basic idea is let PW wait for the re-included storage to take on data before wiggling the next one.
+	// This knob's ideal value would vary by cluster based on its size and disk type. In the meanwhile, the wiggle will also wait until the storage load is almost (85%) balanced.
+	init( PERPETUAL_WIGGLE_DELAY,                                 60 );
+	init( PERPETUAL_WIGGLE_SMALL_LOAD_RATIO,                      10 );
+	init( PERPETUAL_WIGGLE_MIN_BYTES_BALANCE_RATIO,             0.85 );
 	init( PERPETUAL_WIGGLE_DISABLE_REMOVER,                     true );
 	init( LOG_ON_COMPLETION_DELAY,         DD_QUEUE_LOGGING_INTERVAL );
 	init( BEST_TEAM_MAX_TEAM_TRIES,                               10 );
diff --git a/fdbclient/ServerKnobs.h b/fdbclient/ServerKnobs.h
index 53c7c29b13c..f64854db587 100644
--- a/fdbclient/ServerKnobs.h
+++ b/fdbclient/ServerKnobs.h
@@ -177,9 +177,14 @@ class ServerKnobs : public KnobsImpl<ServerKnobs> {
 	double METRIC_DELAY;
 	double ALL_DATA_REMOVED_DELAY;
 	double INITIAL_FAILURE_REACTION_DELAY;
-	double CHECK_TEAM_DELAY;
-	double PERPETUAL_WIGGLE_DELAY;
-	bool PERPETUAL_WIGGLE_DISABLE_REMOVER;
+	double CHECK_TEAM_DELAY; // Perpetual wiggle check cluster team healthy
+	double PERPETUAL_WIGGLE_SMALL_LOAD_RATIO; // If the average load of storage server is less than this ratio * average
+	                                          // shard bytes, the perpetual wiggle won't consider the available space
+	                                          // load balance in the cluster
+	double PERPETUAL_WIGGLE_MIN_BYTES_BALANCE_RATIO; // target min : average space load balance ratio after re-include
+	                                                 // before perpetual wiggle will start the next wiggle
+	double PERPETUAL_WIGGLE_DELAY; // The min interval between the last wiggle finish and the next wiggle start
+	bool PERPETUAL_WIGGLE_DISABLE_REMOVER; // Whether the start of perpetual wiggle replace team remover
 	double LOG_ON_COMPLETION_DELAY;
 	int BEST_TEAM_MAX_TEAM_TRIES;
 	int BEST_TEAM_OPTION_COUNT;
diff --git a/fdbserver/DDTeamCollection.actor.cpp b/fdbserver/DDTeamCollection.actor.cpp
index 9f8f39b1b7d..e24da21f19e 100644
--- a/fdbserver/DDTeamCollection.actor.cpp
+++ b/fdbserver/DDTeamCollection.actor.cpp
@@ -1983,6 +1983,36 @@ class DDTeamCollectionImpl {
 		return Void();
 	}
 
+	ACTOR static Future<Void> perpetualStorageWiggleRest(DDTeamCollection* self) {
+		state bool takeRest = true;
+		state Promise<int64_t> avgShardBytes;
+		while (takeRest) {
+			// a minimal delay to avoid excluding and including SS too fast
+			wait(delay(SERVER_KNOBS->PERPETUAL_WIGGLE_DELAY));
+
+			avgShardBytes.reset();
+			self->getAverageShardBytes.send(avgShardBytes);
+			int64_t avgBytes = wait(avgShardBytes.getFuture());
+			double ratio = self->loadBytesBalanceRatio(avgBytes * SERVER_KNOBS->PERPETUAL_WIGGLE_SMALL_LOAD_RATIO);
+			bool imbalance = ratio < SERVER_KNOBS->PERPETUAL_WIGGLE_MIN_BYTES_BALANCE_RATIO;
+
+			// there must not have other teams to place wiggled data
+			takeRest = self->server_info.size() <= self->configuration.storageTeamSize ||
+			           self->machine_info.size() < self->configuration.storageTeamSize || imbalance;
+
+			// log the extra delay and change the wiggler state
+			if (takeRest && self->configuration.storageMigrationType == StorageMigrationType::GRADUAL) {
+				TraceEvent(SevWarn, "PerpetualStorageWiggleSleep", self->distributorId)
+				    .suppressFor(SERVER_KNOBS->PERPETUAL_WIGGLE_DELAY * 4)
+				    .detail("BytesBalanceRatio", ratio)
+				    .detail("ServerSize", self->server_info.size())
+				    .detail("MachineSize", self->machine_info.size())
+				    .detail("StorageTeamSize", self->configuration.storageTeamSize);
+			}
+		}
+		return Void();
+	}
+
 	ACTOR static Future<Void> perpetualStorageWiggleIterator(DDTeamCollection* teamCollection,
 	                                                         AsyncVar<bool>* stopSignal,
 	                                                         FutureStream<Void> finishStorageWiggleSignal) {
@@ -1990,22 +2020,9 @@ class DDTeamCollectionImpl {
 			choose {
 				when(wait(stopSignal->onChange())) {}
 				when(waitNext(finishStorageWiggleSignal)) {
-					state bool takeRest = true; // delay to avoid delete and update ServerList too frequently
-					while (takeRest) {
-						wait(delayJittered(SERVER_KNOBS->PERPETUAL_WIGGLE_DELAY));
-						// there must not have other teams to place wiggled data
-						takeRest =
-						    teamCollection->server_info.size() <= teamCollection->configuration.storageTeamSize ||
-						    teamCollection->machine_info.size() < teamCollection->configuration.storageTeamSize;
-						if (takeRest &&
-						    teamCollection->configuration.storageMigrationType == StorageMigrationType::GRADUAL) {
-							TraceEvent(SevWarn, "PerpetualWiggleSleep", teamCollection->distributorId)
-							    .suppressFor(SERVER_KNOBS->PERPETUAL_WIGGLE_DELAY * 4)
-							    .detail("ServerSize", teamCollection->server_info.size())
-							    .detail("MachineSize", teamCollection->machine_info.size())
-							    .detail("StorageTeamSize", teamCollection->configuration.storageTeamSize);
-						}
-					}
+					// delay to avoid delete and update ServerList too frequently, which could result busy loop or over
+					// utilize the disk of other active SS
+					wait(perpetualStorageWiggleRest(teamCollection));
 					wait(updateNextWigglingStorageID(teamCollection));
 				}
 			}
@@ -3474,6 +3491,38 @@ Future<Void> DDTeamCollection::keyValueStoreTypeTracker(TCServerInfo* server) {
 	return DDTeamCollectionImpl::keyValueStoreTypeTracker(this, server);
 }
 
+double DDTeamCollection::loadBytesBalanceRatio(int64_t smallLoadThreshold) const {
+	double minLoadBytes = std::numeric_limits<double>::max();
+	double totalLoadBytes = 0;
+	int count = 0;
+	for (auto& [id, s] : server_info) {
+		// If a healthy SS don't have storage metrics, skip this round
+		if (server_status.get(s->getId()).isUnhealthy() || !s->metricsPresent()) {
+			TraceEvent(SevDebug, "LoadBytesBalanceRatioNoMetrics").detail("Server", id);
+			return 0;
+		}
+
+		double load = s->loadBytes();
+		totalLoadBytes += load;
+		++count;
+		minLoadBytes = std::min(minLoadBytes, load);
+	}
+
+	TraceEvent(SevDebug, "LoadBytesBalanceRatioMetrics")
+	    .detail("TotalLoad", totalLoadBytes)
+	    .detail("MinLoadBytes", minLoadBytes)
+	    .detail("SmallLoadThreshold", smallLoadThreshold)
+	    .detail("Count", count);
+
+	// avoid division-by-zero
+	double avgLoad = totalLoadBytes / count;
+	if (totalLoadBytes == 0 || avgLoad < smallLoadThreshold) {
+		return 1;
+	}
+
+	return minLoadBytes / avgLoad;
+}
+
 Future<Void> DDTeamCollection::storageServerFailureTracker(TCServerInfo* server,
                                                            Database cx,
                                                            ServerStatus* status,
@@ -3613,7 +3662,8 @@ DDTeamCollection::DDTeamCollection(Database const& cx,
                                    Reference<AsyncVar<bool>> processingWiggle,
                                    PromiseStream<GetMetricsRequest> getShardMetrics,
                                    Promise<UID> removeFailedServer,
-                                   PromiseStream<Promise<int>> getUnhealthyRelocationCount)
+                                   PromiseStream<Promise<int>> getUnhealthyRelocationCount,
+                                   PromiseStream<Promise<int64_t>> getAverageShardBytes)
   : doBuildTeams(true), lastBuildTeamsFailed(false), teamBuilder(Void()), lock(lock), output(output),
     unhealthyServers(0), storageWiggler(makeReference<StorageWiggler>(this)), processingWiggle(processingWiggle),
     shardsAffectedByTeamFailure(shardsAffectedByTeamFailure),
@@ -3627,7 +3677,8 @@ DDTeamCollection::DDTeamCollection(Database const& cx,
     checkInvalidLocalities(Void()), wrongStoreTypeRemover(Void()), clearHealthyZoneFuture(true),
     medianAvailableSpace(SERVER_KNOBS->MIN_AVAILABLE_SPACE_RATIO), lastMedianAvailableSpaceUpdate(0),
     lowestUtilizationTeam(0), highestUtilizationTeam(0), getShardMetrics(getShardMetrics),
-    getUnhealthyRelocationCount(getUnhealthyRelocationCount), removeFailedServer(removeFailedServer),
+    getUnhealthyRelocationCount(getUnhealthyRelocationCount), getAverageShardBytes(getAverageShardBytes),
+    removeFailedServer(removeFailedServer),
     ddTrackerStartingEventHolder(makeReference<EventCacheHolder>("DDTrackerStarting")),
     teamCollectionInfoEventHolder(makeReference<EventCacheHolder>("TeamCollectionInfo")),
     storageServerRecruitmentEventHolder(
@@ -5200,7 +5251,8 @@ class DDTeamCollectionUnitTest {
 		                                                           makeReference<AsyncVar<bool>>(false),
 		                                                           PromiseStream<GetMetricsRequest>(),
 		                                                           Promise<UID>(),
-		                                                           PromiseStream<Promise<int>>()));
+		                                                           PromiseStream<Promise<int>>(),
+		                                                           PromiseStream<Promise<int64_t>>()));
 
 		for (int id = 1; id <= processCount; ++id) {
 			UID uid(id, 0);
@@ -5244,7 +5296,8 @@ class DDTeamCollectionUnitTest {
 		                                                           makeReference<AsyncVar<bool>>(false),
 		                                                           PromiseStream<GetMetricsRequest>(),
 		                                                           Promise<UID>(),
-		                                                           PromiseStream<Promise<int>>()));
+		                                                           PromiseStream<Promise<int>>(),
+		                                                           PromiseStream<Promise<int64_t>>()));
 
 		for (int id = 1; id <= processCount; id++) {
 			UID uid(id, 0);
diff --git a/fdbserver/DDTeamCollection.h b/fdbserver/DDTeamCollection.h
index 529d2ead938..e14d8e2310f 100644
--- a/fdbserver/DDTeamCollection.h
+++ b/fdbserver/DDTeamCollection.h
@@ -255,6 +255,7 @@ class DDTeamCollection : public ReferenceCounted<DDTeamCollection> {
 
 	PromiseStream<GetMetricsRequest> getShardMetrics;
 	PromiseStream<Promise<int>> getUnhealthyRelocationCount;
+	PromiseStream<Promise<int64_t>> getAverageShardBytes;
 	Promise<UID> removeFailedServer;
 
 	// WIGGLING if an address is under storage wiggling.
@@ -442,6 +443,10 @@ class DDTeamCollection : public ReferenceCounted<DDTeamCollection> {
 
 	Future<Void> waitForAllDataRemoved(Database cx, UID serverID, Version addedVersion) const;
 
+	// calculate minLoadBytes / avgLoadBytes among servers. An unhealthy server's load is considered as 0. If the
+	// average load of each storage server is less than smallLoadThreshold, return 1 always.
+	double loadBytesBalanceRatio(int64_t smallLoadThreshold) const;
+
 	// Create a transaction updating `perpetualStorageWiggleIDPrefix` to the next serverID according to a sorted
 	// wiggle_pq maintained by the wiggler.
 	Future<Void> updateNextWigglingStorageID();
@@ -630,7 +635,8 @@ class DDTeamCollection : public ReferenceCounted<DDTeamCollection> {
 	                 Reference<AsyncVar<bool>> processingWiggle,
 	                 PromiseStream<GetMetricsRequest> getShardMetrics,
 	                 Promise<UID> removeFailedServer,
-	                 PromiseStream<Promise<int>> getUnhealthyRelocationCount);
+	                 PromiseStream<Promise<int>> getUnhealthyRelocationCount,
+	                 PromiseStream<Promise<int64_t>> getAverageShardBytes);
 
 	~DDTeamCollection();
 
diff --git a/fdbserver/DataDistribution.actor.cpp b/fdbserver/DataDistribution.actor.cpp
index 4d7cd8a507d..b9012293554 100644
--- a/fdbserver/DataDistribution.actor.cpp
+++ b/fdbserver/DataDistribution.actor.cpp
@@ -780,7 +780,8 @@ ACTOR Future<Void> dataDistribution(Reference<DataDistributorData> self,
 			    processingWiggle,
 			    getShardMetrics,
 			    removeFailedServer,
-			    getUnhealthyRelocationCount);
+			    getUnhealthyRelocationCount,
+			    getAverageShardBytes);
 			teamCollectionsPtrs.push_back(primaryTeamCollection.getPtr());
 			auto recruitStorage = IAsyncListener<RequestStream<RecruitStorageRequest>>::create(
 			    self->dbInfo, [](auto const& info) { return info.clusterInterface.recruitStorage; });
@@ -801,7 +802,8 @@ ACTOR Future<Void> dataDistribution(Reference<DataDistributorData> self,
 				                                    processingWiggle,
 				                                    getShardMetrics,
 				                                    removeFailedServer,
-				                                    getUnhealthyRelocationCount);
+				                                    getUnhealthyRelocationCount,
+				                                    getAverageShardBytes);
 				teamCollectionsPtrs.push_back(remoteTeamCollection.getPtr());
 				remoteTeamCollection->teamCollections = teamCollectionsPtrs;
 				actors.push_back(reportErrorsExcept(

From 96f60e134dd8dd0252851453aa2fa96bf4079885 Mon Sep 17 00:00:00 2001
From: neethuhaneesha <nbingi@apple.com>
Date: Thu, 7 Sep 2023 17:18:58 -0700
Subject: [PATCH 221/468] Locality check on perpetualStorageWiggleIDPrefix when
 DD restarts (#10864)

---
 fdbserver/DDTeamCollection.actor.cpp | 22 +++++++++++++++++++++-
 1 file changed, 21 insertions(+), 1 deletion(-)

diff --git a/fdbserver/DDTeamCollection.actor.cpp b/fdbserver/DDTeamCollection.actor.cpp
index e24da21f19e..7c81ae37bf0 100644
--- a/fdbserver/DDTeamCollection.actor.cpp
+++ b/fdbserver/DDTeamCollection.actor.cpp
@@ -2934,7 +2934,27 @@ class DDTeamCollectionImpl {
 		if (res.size() > 0) {
 			// SOMEDAY: support wiggle multiple SS at once
 			ASSERT(!self->wigglingId.present()); // only single process wiggle is allowed
-			self->wigglingId = res.begin()->first;
+
+			Optional<Value> localityKey;
+			Optional<Value> localityValue;
+			if (self->configuration.perpetualStorageWiggleLocality != "0") {
+				ParsePerpetualStorageWiggleLocality(
+				    self->configuration.perpetualStorageWiggleLocality, &localityKey, &localityValue);
+			}
+
+			// if perpetual_storage_wiggle_locality has value and not 0(disabled).
+			if (localityKey.present()) {
+				if (self->server_info.count(res.begin()->first)) {
+					auto server = self->server_info.at(res.begin()->first);
+
+					// Update the wigglingId only if it matches the locality.
+					if (server->getLastKnownInterface().locality.get(localityKey.get()) == localityValue) {
+						self->wigglingId = res.begin()->first;
+					}
+				}
+			} else {
+				self->wigglingId = res.begin()->first;
+			}
 		}
 		return Void();
 	}

From 13712cc0b9b3dea55d642d814971a3f5d78c7c93 Mon Sep 17 00:00:00 2001
From: Zhe Wu <halfprice@users.noreply.github.com>
Date: Thu, 24 Aug 2023 13:43:58 -0700
Subject: [PATCH 222/468] Add option to set perpetual_storage_wiggle_engine to
 none

---
 fdbclient/DatabaseConfiguration.cpp           |  4 ++--
 fdbclient/FDBTypes.cpp                        |  2 +-
 fdbclient/FDBTypes.h                          |  9 +++++++-
 fdbserver/DDTeamCollection.actor.cpp          |  2 +-
 .../workloads/ConsistencyCheck.actor.cpp      |  2 +-
 ...alWiggleStorageMigrationWorkload.actor.cpp | 23 ++++++++++++++++---
 6 files changed, 33 insertions(+), 9 deletions(-)

diff --git a/fdbclient/DatabaseConfiguration.cpp b/fdbclient/DatabaseConfiguration.cpp
index 72b89f44abf..e2050f6ae71 100644
--- a/fdbclient/DatabaseConfiguration.cpp
+++ b/fdbclient/DatabaseConfiguration.cpp
@@ -35,8 +35,8 @@ void DatabaseConfiguration::resetInternal() {
 	commitProxyCount = grvProxyCount = resolverCount = desiredTLogCount = tLogWriteAntiQuorum = tLogReplicationFactor =
 	    storageTeamSize = desiredLogRouterCount = -1;
 	tLogVersion = TLogVersion::DEFAULT;
-	tLogDataStoreType = storageServerStoreType = testingStorageServerStoreType = perpetualStoreType =
-	    KeyValueStoreType::END;
+	tLogDataStoreType = storageServerStoreType = testingStorageServerStoreType = KeyValueStoreType::END;
+	perpetualStoreType = KeyValueStoreType::NONE;
 	desiredTSSCount = 0;
 	tLogSpillType = TLogSpillType::DEFAULT;
 	autoCommitProxyCount = CLIENT_KNOBS->DEFAULT_AUTO_COMMIT_PROXIES;
diff --git a/fdbclient/FDBTypes.cpp b/fdbclient/FDBTypes.cpp
index ff268075bd8..a975dbe2f04 100644
--- a/fdbclient/FDBTypes.cpp
+++ b/fdbclient/FDBTypes.cpp
@@ -75,4 +75,4 @@ std::string describe(const std::string& s) {
 
 std::string describe(UID const& item) {
 	return item.shortString();
-}
+}
\ No newline at end of file
diff --git a/fdbclient/FDBTypes.h b/fdbclient/FDBTypes.h
index c8473dfd393..c1367f81274 100644
--- a/fdbclient/FDBTypes.h
+++ b/fdbclient/FDBTypes.h
@@ -857,6 +857,7 @@ struct KeyValueStoreType {
 		SSD_REDWOOD_V1 = 3,
 		MEMORY_RADIXTREE = 4,
 		SSD_ROCKSDB_V1 = 5,
+		NONE = 7,
 		END
 	};
 
@@ -887,6 +888,8 @@ struct KeyValueStoreType {
 			return "memory";
 		case MEMORY_RADIXTREE:
 			return "memory-radixtree-beta";
+		case NONE:
+			return "none";
 		default:
 			return "unknown";
 		}
@@ -903,7 +906,8 @@ struct KeyValueStoreType {
 			                                              { "ssd-redwood-1-experimental", SSD_REDWOOD_V1 },
 			                                              { "ssd-rocksdb-v1", SSD_ROCKSDB_V1 },
 			                                              { "memory", MEMORY },
-			                                              { "memory-radixtree-beta", MEMORY_RADIXTREE } };
+			                                              { "memory-radixtree-beta", MEMORY_RADIXTREE },
+														  { "none", NONE } };
 		auto it = names.find(str);
 		if (it == names.end()) {
 			throw unknown_storage_engine();
@@ -911,6 +915,9 @@ struct KeyValueStoreType {
 		return it->second;
 	}
 
+	// Whether the storage type is a valid storage type.
+	bool isValid() const { return type != NONE && type != END; }
+
 private:
 	uint32_t type;
 };
diff --git a/fdbserver/DDTeamCollection.actor.cpp b/fdbserver/DDTeamCollection.actor.cpp
index 7c81ae37bf0..d709e53ad85 100644
--- a/fdbserver/DDTeamCollection.actor.cpp
+++ b/fdbserver/DDTeamCollection.actor.cpp
@@ -2342,7 +2342,7 @@ class DDTeamCollectionImpl {
 			// perpetualStoreType for all new SSes that match perpetualStorageWiggleLocality.
 			// Note that this only applies to regular storage servers, not TSS.
 			if (!recruitTss && self->configuration.storageMigrationType == StorageMigrationType::GRADUAL &&
-			    self->configuration.perpetualStoreType.storeType() != KeyValueStoreType::END) {
+			    self->configuration.perpetualStoreType.isValid()) {
 				if (self->configuration.perpetualStorageWiggleLocality == "0") {
 					isr.storeType = self->configuration.perpetualStoreType;
 				} else {
diff --git a/fdbserver/workloads/ConsistencyCheck.actor.cpp b/fdbserver/workloads/ConsistencyCheck.actor.cpp
index 6b024359d52..7a36132f2b6 100644
--- a/fdbserver/workloads/ConsistencyCheck.actor.cpp
+++ b/fdbserver/workloads/ConsistencyCheck.actor.cpp
@@ -1865,7 +1865,7 @@ struct ConsistencyCheckWorkload : TestWorkload {
 			if (!keyValueStoreType.present()) {
 				TraceEvent("ConsistencyCheck_ServerUnavailable").detail("ServerID", storageServers[i].id());
 				self->testFailure("Storage server unavailable");
-			} else if (configuration.perpetualStoreType.storeType() != KeyValueStoreType::END) {
+			} else if (configuration.perpetualStoreType.isValid()) {
 				// Perpetual storage wiggle is used to migrate storage. Check that the matched storage servers are
 				// correctly migrated.
 				if (wiggleLocalityKeyValue == "0" ||
diff --git a/fdbserver/workloads/PerpetualWiggleStorageMigrationWorkload.actor.cpp b/fdbserver/workloads/PerpetualWiggleStorageMigrationWorkload.actor.cpp
index 98a32daede2..1a2edfcfdf9 100644
--- a/fdbserver/workloads/PerpetualWiggleStorageMigrationWorkload.actor.cpp
+++ b/fdbserver/workloads/PerpetualWiggleStorageMigrationWorkload.actor.cpp
@@ -101,6 +101,23 @@ struct PerpetualWiggleStorageMigrationWorkload : public TestWorkload {
 		TraceEvent("Test_ConfigChangeDone").detail("Success", change);
 		ASSERT(change);
 
+		wait(excludeIncludeServer(cx, ssToExcludeInclude));
+
+		wait(validateDatabase(cx, ssToExcludeInclude, ssToWiggle, "ssd-rocksdb-v1"));
+
+		if (deterministicRandom()->coinflip()) {
+			TraceEvent("Test_ClearPerpetualStorageWiggleEngine").log();
+			bool change = wait(IssueConfigurationChange(cx, "perpetual_storage_wiggle_engine=none", true));
+			TraceEvent("Test_ClearPerpetualStorageWiggleEngineDone").detail("Success", change);
+			ASSERT(change);
+
+			wait(excludeIncludeServer(cx, ssToWiggle));
+			wait(validateDatabase(cx, ssToExcludeInclude, ssToWiggle, "ssd-2"));
+		}
+		return Void();
+	}
+
+	ACTOR static Future<Void> excludeIncludeServer(Database cx, StorageServerInterface ssToExcludeInclude) {
 		// Now, let's exclude `ssToExcludeInclude` process and include it again. The new SS created on this process
 		// should always uses `storage_engine` config, which is `ssd-2`.
 		state std::vector<AddressExclusion> servers;
@@ -128,13 +145,13 @@ struct PerpetualWiggleStorageMigrationWorkload : public TestWorkload {
 		wait(includeServers(cx, std::vector<AddressExclusion>(1)));
 		TraceEvent("Test_IncludeServerDone").log();
 
-		wait(validateDatabase(cx, ssToExcludeInclude, ssToWiggle));
 		return Void();
 	}
 
 	ACTOR static Future<Void> validateDatabase(Database cx,
 	                                           StorageServerInterface ssToExcludeInclude,
-	                                           StorageServerInterface ssToWiggle) {
+	                                           StorageServerInterface ssToWiggle,
+	                                           std::string wiggleStorageType) {
 		// Wait until `ssToExcludeInclude` to be recruited as storage server again.
 		state int missingTargetCount = 0;
 		loop {
@@ -187,7 +204,7 @@ struct PerpetualWiggleStorageMigrationWorkload : public TestWorkload {
 					if (ssInterface.address() == ssToWiggle.address()) {
 						// If `ssToWiggle` exists, we wait until it is migrate to `perpetual_storage_wiggle_engine`.
 						containWiggleStorage = true;
-						if (keyValueStoreType.get().toString() == "ssd-rocksdb-v1") {
+						if (keyValueStoreType.get().toString() == wiggleStorageType) {
 							doneCheckingWiggleStorage = true;
 						}
 					}

From 7ab81461b17b6e9a57b2653d1188e23e66e4c5cf Mon Sep 17 00:00:00 2001
From: Zhe Wu <halfprice@users.noreply.github.com>
Date: Tue, 29 Aug 2023 13:58:23 -0700
Subject: [PATCH 223/468] Fix StatusWorkload after adding
 perpetual_storage_wiggle_engine

---
 .../sphinx/source/mr-status-json-schemas.rst.inc   | 14 ++++++++++++++
 fdbclient/Schemas.cpp                              | 14 ++++++++++++++
 2 files changed, 28 insertions(+)

diff --git a/documentation/sphinx/source/mr-status-json-schemas.rst.inc b/documentation/sphinx/source/mr-status-json-schemas.rst.inc
index ac9ed77da39..8f288547c8b 100644
--- a/documentation/sphinx/source/mr-status-json-schemas.rst.inc
+++ b/documentation/sphinx/source/mr-status-json-schemas.rst.inc
@@ -773,6 +773,20 @@
          "backup_worker_enabled":1,
          "perpetual_storage_wiggle": 0,
          "perpetual_storage_wiggle_locality":"0",
+         "storage_engine":{
+         "$enum":[
+             "ssd",
+             "ssd-1",
+             "ssd-2",
+             "ssd-redwood-1",
+             "ssd-rocksdb-v1",
+             "ssd-sharded-rocksdb",
+             "memory",
+             "memory-1",
+             "memory-2",
+             "memory-radixtree-beta",
+             "none"
+         ]},
          "storage_migration_type":{
          "$enum":[
              "disabled",
diff --git a/fdbclient/Schemas.cpp b/fdbclient/Schemas.cpp
index 9b8ff098dc7..89cb8f0cc43 100644
--- a/fdbclient/Schemas.cpp
+++ b/fdbclient/Schemas.cpp
@@ -806,6 +806,20 @@ const KeyRef JSONSchemas::statusSchema = LiteralStringRef(R"statusSchema(
          "backup_worker_enabled":1,
          "perpetual_storage_wiggle":0,
          "perpetual_storage_wiggle_locality":"0",
+         "perpetual_storage_wiggle_engine":{
+         "$enum":[
+             "ssd",
+             "ssd-1",
+             "ssd-2",
+             "ssd-redwood-1",
+             "ssd-rocksdb-v1",
+             "ssd-sharded-rocksdb",
+             "memory",
+             "memory-1",
+             "memory-2",
+             "memory-radixtree-beta",
+             "none"
+         ]},
          "storage_migration_type": {
              "$enum":[
              "disabled",

From 32ce0bb01081434240752decdefff84e4d739494 Mon Sep 17 00:00:00 2001
From: Zhe Wu <halfprice@users.noreply.github.com>
Date: Tue, 29 Aug 2023 14:32:21 -0700
Subject: [PATCH 224/468] Add test documents

---
 .../PerpetualWiggleStorageMigrationWorkload.actor.cpp      | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/fdbserver/workloads/PerpetualWiggleStorageMigrationWorkload.actor.cpp b/fdbserver/workloads/PerpetualWiggleStorageMigrationWorkload.actor.cpp
index 1a2edfcfdf9..8b9b82afbef 100644
--- a/fdbserver/workloads/PerpetualWiggleStorageMigrationWorkload.actor.cpp
+++ b/fdbserver/workloads/PerpetualWiggleStorageMigrationWorkload.actor.cpp
@@ -103,16 +103,19 @@ struct PerpetualWiggleStorageMigrationWorkload : public TestWorkload {
 
 		wait(excludeIncludeServer(cx, ssToExcludeInclude));
 
-		wait(validateDatabase(cx, ssToExcludeInclude, ssToWiggle, "ssd-rocksdb-v1"));
+		wait(validateDatabase(cx, ssToExcludeInclude, ssToWiggle, /*wiggleStorageType=*/"ssd-rocksdb-v1"));
 
+		// We probablistically validate that resetting perpetual_storage_wiggle_engine to none works as expected.
 		if (deterministicRandom()->coinflip()) {
 			TraceEvent("Test_ClearPerpetualStorageWiggleEngine").log();
 			bool change = wait(IssueConfigurationChange(cx, "perpetual_storage_wiggle_engine=none", true));
 			TraceEvent("Test_ClearPerpetualStorageWiggleEngineDone").detail("Success", change);
 			ASSERT(change);
 
+			// Next, we run exclude and then include `ssToWiggle`. Because perpetual_storage_wiggle_engine is set to
+			// none, the engine for `ssToWiggle` should be `storage_engine`.
 			wait(excludeIncludeServer(cx, ssToWiggle));
-			wait(validateDatabase(cx, ssToExcludeInclude, ssToWiggle, "ssd-2"));
+			wait(validateDatabase(cx, ssToExcludeInclude, ssToWiggle, /*wiggleStorageType=*/"ssd-2"));
 		}
 		return Void();
 	}

From ad603a2c26d17d6611e03ae951345391236a9df5 Mon Sep 17 00:00:00 2001
From: Zhe Wu <halfprice@users.noreply.github.com>
Date: Tue, 5 Sep 2023 11:10:53 -0700
Subject: [PATCH 225/468] Fix status json schemas
 perpetual_storage_wiggle_engine

---
 documentation/sphinx/source/mr-status-json-schemas.rst.inc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/documentation/sphinx/source/mr-status-json-schemas.rst.inc b/documentation/sphinx/source/mr-status-json-schemas.rst.inc
index 8f288547c8b..2c2a5378196 100644
--- a/documentation/sphinx/source/mr-status-json-schemas.rst.inc
+++ b/documentation/sphinx/source/mr-status-json-schemas.rst.inc
@@ -773,7 +773,7 @@
          "backup_worker_enabled":1,
          "perpetual_storage_wiggle": 0,
          "perpetual_storage_wiggle_locality":"0",
-         "storage_engine":{
+         "perpetual_storage_wiggle_engine":{
          "$enum":[
              "ssd",
              "ssd-1",

From 61770bfb6934d329cf4c0b329f7fb456f64e4794 Mon Sep 17 00:00:00 2001
From: Zhe Wu <halfprice@users.noreply.github.com>
Date: Thu, 7 Sep 2023 13:34:32 -0700
Subject: [PATCH 226/468] Remove ssd-sharded-rocksdb from schemas since it is
 only introduced in 7.3

---
 documentation/sphinx/source/mr-status-json-schemas.rst.inc | 1 -
 fdbclient/Schemas.cpp                                      | 1 -
 2 files changed, 2 deletions(-)

diff --git a/documentation/sphinx/source/mr-status-json-schemas.rst.inc b/documentation/sphinx/source/mr-status-json-schemas.rst.inc
index 2c2a5378196..015548cbf15 100644
--- a/documentation/sphinx/source/mr-status-json-schemas.rst.inc
+++ b/documentation/sphinx/source/mr-status-json-schemas.rst.inc
@@ -780,7 +780,6 @@
              "ssd-2",
              "ssd-redwood-1",
              "ssd-rocksdb-v1",
-             "ssd-sharded-rocksdb",
              "memory",
              "memory-1",
              "memory-2",
diff --git a/fdbclient/Schemas.cpp b/fdbclient/Schemas.cpp
index 89cb8f0cc43..278fb9bf04c 100644
--- a/fdbclient/Schemas.cpp
+++ b/fdbclient/Schemas.cpp
@@ -813,7 +813,6 @@ const KeyRef JSONSchemas::statusSchema = LiteralStringRef(R"statusSchema(
              "ssd-2",
              "ssd-redwood-1",
              "ssd-rocksdb-v1",
-             "ssd-sharded-rocksdb",
              "memory",
              "memory-1",
              "memory-2",

From 4bb8bb02a7c74763c0eca895b1d39fc6be10c9ef Mon Sep 17 00:00:00 2001
From: Zhe Wu <halfprice@users.noreply.github.com>
Date: Thu, 7 Sep 2023 14:15:16 -0700
Subject: [PATCH 227/468] Apply format

---
 fdbclient/FDBTypes.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/fdbclient/FDBTypes.h b/fdbclient/FDBTypes.h
index c1367f81274..34246a4a25f 100644
--- a/fdbclient/FDBTypes.h
+++ b/fdbclient/FDBTypes.h
@@ -907,7 +907,7 @@ struct KeyValueStoreType {
 			                                              { "ssd-rocksdb-v1", SSD_ROCKSDB_V1 },
 			                                              { "memory", MEMORY },
 			                                              { "memory-radixtree-beta", MEMORY_RADIXTREE },
-														  { "none", NONE } };
+			                                              { "none", NONE } };
 		auto it = names.find(str);
 		if (it == names.end()) {
 			throw unknown_storage_engine();

From 21f46cec2fa01ac24a9c2809bd8e66ac42706ed4 Mon Sep 17 00:00:00 2001
From: Zhe Wu <halfprice@users.noreply.github.com>
Date: Thu, 7 Sep 2023 21:07:05 -0700
Subject: [PATCH 228/468] Fix test failure: we need to specifically specify
 that the option name, perpetual_storage_wiggle_engine, will be shown in the
 configuration

---
 fdbserver/SimulatedCluster.actor.cpp | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/fdbserver/SimulatedCluster.actor.cpp b/fdbserver/SimulatedCluster.actor.cpp
index 3d70a8f42af..6ded84160a4 100644
--- a/fdbserver/SimulatedCluster.actor.cpp
+++ b/fdbserver/SimulatedCluster.actor.cpp
@@ -1866,7 +1866,8 @@ void setupSimulatedSystem(std::vector<Future<Void>>* systemActors,
 		if (kv.second.type() == json_spirit::int_type) {
 			startingConfigString += kv.first + ":=" + format("%d", kv.second.get_int());
 		} else if (kv.second.type() == json_spirit::str_type) {
-			if ("storage_migration_type" == kv.first || "tenant_mode" == kv.first) {
+			if ("storage_migration_type" == kv.first || "tenant_mode" == kv.first ||
+			    "perpetual_storage_wiggle_engine" == kv.first) {
 				startingConfigString += kv.first + "=" + kv.second.get_str();
 			} else {
 				startingConfigString += kv.second.get_str();

From f1c441e576faa59c64ee7dd557b3e753cbccc174 Mon Sep 17 00:00:00 2001
From: Hao Fu <77984096+hfu94@users.noreply.github.com>
Date: Fri, 8 Sep 2023 10:15:30 -0700
Subject: [PATCH 229/468] Add proxy to backup agent via global var(release-7.1)
 (#10875)

* Add proxy to backup agent via global var

backup agent itself does not have proxy info.
This changes adds the proxy via a global var.

* resolve comments

* remove redundant var

* Add proxy parameter in cmd
---
 fdbbackup/backup.actor.cpp          |  2 ++
 fdbclient/BackupAgent.actor.h       |  2 ++
 fdbclient/FileBackupAgent.actor.cpp | 32 ++++++++++++++++++++---------
 3 files changed, 26 insertions(+), 10 deletions(-)

diff --git a/fdbbackup/backup.actor.cpp b/fdbbackup/backup.actor.cpp
index c3bd7e869bf..6b986a510a9 100644
--- a/fdbbackup/backup.actor.cpp
+++ b/fdbbackup/backup.actor.cpp
@@ -222,6 +222,7 @@ CSimpleOpt::SOption g_rgAgentOptions[] = {
 	{ OPT_HELP, "--help", SO_NONE },
 	{ OPT_DEVHELP, "--dev-help", SO_NONE },
 	{ OPT_BLOB_CREDENTIALS, "--blob-credentials", SO_REQ_SEP },
+	{ OPT_PROXY, "--proxy", SO_REQ_SEP },
 #ifndef TLS_DISABLED
 	TLS_OPTION_FLAGS
 #endif
@@ -4121,6 +4122,7 @@ int main(int argc, char* argv[]) {
 		case ProgramExe::AGENT:
 			if (!initCluster())
 				return FDB_EXIT_ERROR;
+			fileBackupAgentProxy = proxy;
 			f = stopAfter(runAgent(db));
 			break;
 		case ProgramExe::BACKUP:
diff --git a/fdbclient/BackupAgent.actor.h b/fdbclient/BackupAgent.actor.h
index 43cd8ee811d..0aee518be79 100644
--- a/fdbclient/BackupAgent.actor.h
+++ b/fdbclient/BackupAgent.actor.h
@@ -56,6 +56,8 @@ FDB_DECLARE_BOOLEAN_PARAM(DeleteData);
 FDB_DECLARE_BOOLEAN_PARAM(SetValidation);
 FDB_DECLARE_BOOLEAN_PARAM(PartialBackup);
 
+extern Optional<std::string> fileBackupAgentProxy;
+
 class BackupAgentBase : NonCopyable {
 public:
 	// Time formatter for anything backup or restore related
diff --git a/fdbclient/FileBackupAgent.actor.cpp b/fdbclient/FileBackupAgent.actor.cpp
index ebd7571cdf4..2dfb89744b7 100644
--- a/fdbclient/FileBackupAgent.actor.cpp
+++ b/fdbclient/FileBackupAgent.actor.cpp
@@ -46,6 +46,8 @@
 FDB_DEFINE_BOOLEAN_PARAM(IncrementalBackupOnly);
 FDB_DEFINE_BOOLEAN_PARAM(OnlyApplyMutationLogs);
 
+Optional<std::string> fileBackupAgentProxy = Optional<std::string>();
+
 #define SevFRTestInfo SevVerbose
 // #define SevFRTestInfo SevInfo
 
@@ -584,6 +586,11 @@ struct RangeFileWriter {
 	Key lastValue;
 };
 
+static Reference<IBackupContainer> getBackupContainerWithProxy(Reference<IBackupContainer> _bc) {
+	Reference<IBackupContainer> bc = IBackupContainer::openContainer(_bc->getURL(), fileBackupAgentProxy, {});
+	return bc;
+}
+
 Standalone<VectorRef<KeyValueRef>> decodeRangeFileBlock(const Standalone<StringRef>& buf) {
 	Standalone<VectorRef<KeyValueRef>> results({}, buf.arena());
 	StringRefReader reader(buf, restore_corrupted_data());
@@ -1223,11 +1230,11 @@ struct BackupRangeTaskFunc : BackupTaskFuncBase {
 
 		// Don't need to check keepRunning(task) here because we will do that while finishing each output file, but if
 		// bc is false then clearly the backup is no longer in progress
-		state Reference<IBackupContainer> bc = wait(backup.backupContainer().getD(cx));
-		if (!bc) {
+		Reference<IBackupContainer> _bc = wait(backup.backupContainer().getD(cx));
+		if (!_bc) {
 			return Void();
 		}
-
+		state Reference<IBackupContainer> bc = getBackupContainerWithProxy(_bc);
 		state bool done = false;
 		state int64_t nrKeys = 0;
 
@@ -2018,7 +2025,7 @@ struct BackupLogRangeTaskFunc : BackupTaskFuncBase {
 				if (!bc) {
 					// Backup container must be present if we're still here
 					Reference<IBackupContainer> _bc = wait(config.backupContainer().getOrThrow(tr));
-					bc = _bc;
+					bc = getBackupContainerWithProxy(_bc);
 				}
 
 				Version currentVersion = tr->getReadVersion().get();
@@ -2545,7 +2552,8 @@ struct BackupSnapshotManifest : BackupTaskFuncBase {
 
 				if (!bc) {
 					// Backup container must be present if we're still here
-					wait(store(bc, config.backupContainer().getOrThrow(tr)));
+					Reference<IBackupContainer> _bc = wait(config.backupContainer().getOrThrow(tr));
+					bc = getBackupContainerWithProxy(_bc);
 				}
 
 				BackupConfig::RangeFileMapT::PairsType rangeresults =
@@ -3028,7 +3036,8 @@ struct RestoreRangeTaskFunc : RestoreFileTaskFuncBase {
 				tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 				tr->setOption(FDBTransactionOptions::LOCK_AWARE);
 
-				bc = restore.sourceContainer().getOrThrow(tr);
+				Reference<IBackupContainer> _bc = wait(restore.sourceContainer().getOrThrow(tr));
+				bc = getBackupContainerWithProxy(_bc);
 				restoreRanges = restore.getRestoreRangesOrDefault(tr);
 				addPrefix = restore.addPrefix().getD(tr);
 				removePrefix = restore.removePrefix().getD(tr);
@@ -3439,7 +3448,7 @@ struct RestoreLogDataTaskFunc : RestoreFileTaskFuncBase {
 				tr->setOption(FDBTransactionOptions::LOCK_AWARE);
 
 				Reference<IBackupContainer> _bc = wait(restore.sourceContainer().getOrThrow(tr));
-				bc = _bc;
+				bc = getBackupContainerWithProxy(_bc);
 
 				wait(store(ranges, restore.getRestoreRangesOrDefault(tr)));
 
@@ -4107,7 +4116,7 @@ struct StartFullRestoreTaskFunc : RestoreTaskFuncBase {
 				restore.fileBlockCount().clear(tr);
 				restore.fileCount().clear(tr);
 				Reference<IBackupContainer> _bc = wait(restore.sourceContainer().getOrThrow(tr));
-				bc = _bc;
+				bc = getBackupContainerWithProxy(_bc);
 
 				wait(tr->commit());
 				break;
@@ -4517,7 +4526,7 @@ class FileBackupAgentImpl {
 					if (pContainer != nullptr) {
 						Reference<IBackupContainer> c =
 						    wait(config.backupContainer().getOrThrow(tr, Snapshot::False, backup_invalid_info()));
-						*pContainer = c;
+						*pContainer = fileBackup::getBackupContainerWithProxy(c);
 					}
 
 					if (pUID != nullptr) {
@@ -5003,6 +5012,7 @@ class FileBackupAgentImpl {
 						wait(
 						    store(latestRestorable, getTimestampedVersion(tr, config.getLatestRestorableVersion(tr))) &&
 						    store(bc, config.backupContainer().getOrThrow(tr)));
+						bc = fileBackup::getBackupContainerWithProxy(bc);
 
 						doc.setKey("Restorable", latestRestorable.present());
 
@@ -5146,6 +5156,7 @@ class FileBackupAgentImpl {
 					wait(store(latestRestorableVersion, config.getLatestRestorableVersion(tr)) &&
 					     store(bc, config.backupContainer().getOrThrow(tr)) &&
 					     store(recentReadVersion, tr->getReadVersion()));
+					bc = fileBackup::getBackupContainerWithProxy(bc);
 
 					bool snapshotProgress = false;
 
@@ -5525,7 +5536,8 @@ class FileBackupAgentImpl {
 			}
 		}
 
-		Reference<IBackupContainer> bc = wait(backupConfig.backupContainer().getOrThrow(cx));
+		Reference<IBackupContainer> _bc = wait(backupConfig.backupContainer().getOrThrow(cx));
+		Reference<IBackupContainer> bc = fileBackup::getBackupContainerWithProxy(_bc);
 
 		if (fastRestore) {
 			TraceEvent("AtomicParallelRestoreStartRestore").log();

From ee8e14b80028163f8fcdb4c077ecbdc10999ee5c Mon Sep 17 00:00:00 2001
From: neethuhaneesha <nbingi@apple.com>
Date: Mon, 11 Sep 2023 12:55:40 -0700
Subject: [PATCH 230/468] Release notes for 7.1.38/7.1.39 (#10887)

---
 .../source/release-notes/release-notes-710.rst  | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)

diff --git a/documentation/sphinx/source/release-notes/release-notes-710.rst b/documentation/sphinx/source/release-notes/release-notes-710.rst
index b5d40d3be45..8400241ac10 100644
--- a/documentation/sphinx/source/release-notes/release-notes-710.rst
+++ b/documentation/sphinx/source/release-notes/release-notes-710.rst
@@ -4,6 +4,23 @@
 Release Notes
 #############
 
+7.1.39
+======
+* Same as 7.1.38 release with AVX enabled.
+
+7.1.38
+======
+* Released with AVX disabled.
+* Added locality check on reading perpetualStorageWiggleIDPrefix key when DD restarts. `(PR #10864) <https://github.com/apple/foundationdb/pull/10864>`_
+* Added perpetual wiggle wait based on data balance of the cluster. `(PR #10865) <https://github.com/apple/foundationdb/pull/10865>`_
+* Added rocksdb options to delete old rocksdb logs. `(PR #10872) <https://github.com/apple/foundationdb/pull/10872>`_
+* Added knob to guard the gray failure rejection during TLog recovery. `(PR #10852) <https://github.com/apple/foundationdb/pull/10852>`_
+* Added knob RESOLVE_PREFER_IPV4_ADDR to prefer IPv4 addresses. `(PR #10826) <https://github.com/apple/foundationdb/pull/10826>`_
+* Added perpetual_storage_wiggle_engine config to support storage migration with perpetual wiggle. `(PR #10790) <https://github.com/apple/foundationdb/pull/10790>`_
+* Fixed the return code for perpetual wiggle configure command. `(PR #10795) <https://github.com/apple/foundationdb/pull/10795>`_
+* Fixed a compatibility issue of s3 backup. `(PR #10774) <https://github.com/apple/foundationdb/pull/10774>`_
+* Added proxy to backup agent via global variable. `(PR #10875) <https://github.com/apple/foundationdb/pull/10875>`_
+
 7.1.37
 ======
 * Same as 7.1.36 release with AVX enabled.

From f606ece0d13e9382452ac8466cca503b9256181d Mon Sep 17 00:00:00 2001
From: FoundationDB CI <foundationdb_ci@apple.com>
Date: Tue, 12 Sep 2023 19:03:16 +0000
Subject: [PATCH 231/468] disable AVX for 7.1.38 release

---
 cmake/ConfigureCompiler.cmake | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cmake/ConfigureCompiler.cmake b/cmake/ConfigureCompiler.cmake
index e52d8665ee8..7e50469ba6a 100644
--- a/cmake/ConfigureCompiler.cmake
+++ b/cmake/ConfigureCompiler.cmake
@@ -253,7 +253,7 @@ else()
       set(USE_AVX512F OFF)
     endif()
   endif()
-  set(USE_AVX ON CACHE BOOL "Enable AVX instructions")
+  set(USE_AVX OFF CACHE BOOL "Enable AVX instructions")
   if (USE_AVX)
     if (CMAKE_HOST_SYSTEM_PROCESSOR MATCHES "^x86")
       add_compile_options(-mavx)

From 728ae0062027e51ece0d6f403783bb7569969666 Mon Sep 17 00:00:00 2001
From: FoundationDB CI <foundationdb_ci@apple.com>
Date: Tue, 12 Sep 2023 19:04:21 +0000
Subject: [PATCH 232/468] enable AVX and update version for 7.1.39 release

---
 CMakeLists.txt                | 2 +-
 cmake/ConfigureCompiler.cmake | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 0c689139fe0..ff94ad3602a 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -23,7 +23,7 @@ else()
 endif()
 
 project(foundationdb
-  VERSION 7.1.38
+  VERSION 7.1.39
   DESCRIPTION "FoundationDB is a scalable, fault-tolerant, ordered key-value store with full ACID transactions."
   HOMEPAGE_URL "http://www.foundationdb.org/"
   LANGUAGES C CXX ASM)
diff --git a/cmake/ConfigureCompiler.cmake b/cmake/ConfigureCompiler.cmake
index 7e50469ba6a..e52d8665ee8 100644
--- a/cmake/ConfigureCompiler.cmake
+++ b/cmake/ConfigureCompiler.cmake
@@ -253,7 +253,7 @@ else()
       set(USE_AVX512F OFF)
     endif()
   endif()
-  set(USE_AVX OFF CACHE BOOL "Enable AVX instructions")
+  set(USE_AVX ON CACHE BOOL "Enable AVX instructions")
   if (USE_AVX)
     if (CMAKE_HOST_SYSTEM_PROCESSOR MATCHES "^x86")
       add_compile_options(-mavx)

From 12a8f87f3d9e3e9ee6f2baff57067714e4470056 Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyu_zhou@apple.com>
Date: Tue, 12 Sep 2023 15:30:18 -0700
Subject: [PATCH 233/468] Fix an assertion failure in tests (#10893)

The buggified value 120 triggers assertion failure, so restore it to the default.

Reproduction seed: -f ./tests/rare/ClogTlog.toml -s 537536800 -b on
commit: 163774cfd at release-7.1
---
 tests/rare/ClogTlog.toml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tests/rare/ClogTlog.toml b/tests/rare/ClogTlog.toml
index 6e961420ca2..1d8e6561643 100644
--- a/tests/rare/ClogTlog.toml
+++ b/tests/rare/ClogTlog.toml
@@ -13,6 +13,7 @@ cc_worker_health_checking_interval = 15
 cc_min_degradation_interval = 30
 cc_health_trigger_recovery = true
 peer_latency_degradation_threshold = 1
+peek_tracker_expiration_time = 600
 
 [[test]]
 testTitle = 'ClogTlog'

From 6ef0063070f1caf4993fa9b046fd6e6638289b22 Mon Sep 17 00:00:00 2001
From: FoundationDB CI <foundationdb_ci@apple.com>
Date: Wed, 13 Sep 2023 17:29:51 +0000
Subject: [PATCH 234/468] update version after 7.1.39 release

---
 CMakeLists.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index ff94ad3602a..ae9fe81e9ac 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -23,7 +23,7 @@ else()
 endif()
 
 project(foundationdb
-  VERSION 7.1.39
+  VERSION 7.1.40
   DESCRIPTION "FoundationDB is a scalable, fault-tolerant, ordered key-value store with full ACID transactions."
   HOMEPAGE_URL "http://www.foundationdb.org/"
   LANGUAGES C CXX ASM)

From 33bcf72f5cdd8da3626fb9d3187350666f37518a Mon Sep 17 00:00:00 2001
From: "Johannes M. Scheuermann" <jscheuermann@apple.com>
Date: Thu, 14 Sep 2023 13:58:03 +0200
Subject: [PATCH 235/468] Correct the way how the backup agent makes use of the
 proxy and add proxy to trace events

---
 fdbbackup/backup.actor.cpp                    |  1 +
 fdbclient/BackupContainer.actor.cpp           | 11 ++++++++++-
 fdbclient/BackupContainerFileSystem.actor.cpp | 11 ++++++++++-
 fdbclient/S3BlobStore.actor.cpp               |  8 +++++---
 4 files changed, 26 insertions(+), 5 deletions(-)

diff --git a/fdbbackup/backup.actor.cpp b/fdbbackup/backup.actor.cpp
index 6b986a510a9..a8dd6a8f4e4 100644
--- a/fdbbackup/backup.actor.cpp
+++ b/fdbbackup/backup.actor.cpp
@@ -4076,6 +4076,7 @@ int main(int argc, char* argv[]) {
 		    .detail("CommandLine", commandLine)
 		    .setMaxFieldLength(0)
 		    .detail("MemoryLimit", memLimit)
+		    .detail("Proxy", proxy.orDefault(""))
 		    .trackLatest("ProgramStart");
 
 		// Ordinarily, this is done when the network is run. However, network thread should be set before TraceEvents
diff --git a/fdbclient/BackupContainer.actor.cpp b/fdbclient/BackupContainer.actor.cpp
index 416d15c5484..1c3a8906d9f 100644
--- a/fdbclient/BackupContainer.actor.cpp
+++ b/fdbclient/BackupContainer.actor.cpp
@@ -270,11 +270,20 @@ Reference<IBackupContainer> IBackupContainer::openContainer(const std::string& u
 			r = makeReference<BackupContainerLocalDirectory>(url, encryptionKeyFileName);
 		} else if (u.startsWith("blobstore://"_sr)) {
 			std::string resource;
+			Optional<std::string> blobstoreProxy;
+
+			// If no proxy is passed down to the openContainer method, try to fallback to the
+			// fileBackupAgentProxy which is a global variable and will be set for the backup_agent.
+			if (proxy.present()) {
+				blobstoreProxy = proxy.get();
+			} else if (fileBackupAgentProxy.present()) {
+				blobstoreProxy = fileBackupAgentProxy.get();
+			}
 
 			// The URL parameters contain blobstore endpoint tunables as well as possible backup-specific options.
 			S3BlobStoreEndpoint::ParametersT backupParams;
 			Reference<S3BlobStoreEndpoint> bstore =
-			    S3BlobStoreEndpoint::fromString(url, proxy, &resource, &lastOpenError, &backupParams);
+			    S3BlobStoreEndpoint::fromString(url, blobstoreProxy, &resource, &lastOpenError, &backupParams);
 
 			if (resource.empty())
 				throw backup_invalid_url();
diff --git a/fdbclient/BackupContainerFileSystem.actor.cpp b/fdbclient/BackupContainerFileSystem.actor.cpp
index 634cba1c15e..1a1e1fcedbf 100644
--- a/fdbclient/BackupContainerFileSystem.actor.cpp
+++ b/fdbclient/BackupContainerFileSystem.actor.cpp
@@ -1516,11 +1516,20 @@ Reference<BackupContainerFileSystem> BackupContainerFileSystem::openContainerFS(
 			r = makeReference<BackupContainerLocalDirectory>(url, encryptionKeyFileName);
 		} else if (u.startsWith("blobstore://"_sr)) {
 			std::string resource;
+			Optional<std::string> blobstoreProxy;
+
+			// If no proxy is passed down to the openContainer method, try to fallback to the
+			// fileBackupAgentProxy which is a global variable and will be set for the backup_agent.
+			if (proxy.present()) {
+				blobstoreProxy = proxy.get();
+			} else if (fileBackupAgentProxy.present()) {
+				blobstoreProxy = fileBackupAgentProxy.get();
+			}
 
 			// The URL parameters contain blobstore endpoint tunables as well as possible backup-specific options.
 			S3BlobStoreEndpoint::ParametersT backupParams;
 			Reference<S3BlobStoreEndpoint> bstore =
-			    S3BlobStoreEndpoint::fromString(url, proxy, &resource, &lastOpenError, &backupParams);
+			    S3BlobStoreEndpoint::fromString(url, blobstoreProxy, &resource, &lastOpenError, &backupParams);
 
 			if (resource.empty())
 				throw backup_invalid_url();
diff --git a/fdbclient/S3BlobStore.actor.cpp b/fdbclient/S3BlobStore.actor.cpp
index 446c8792f86..808ac78c9d8 100644
--- a/fdbclient/S3BlobStore.actor.cpp
+++ b/fdbclient/S3BlobStore.actor.cpp
@@ -714,7 +714,8 @@ ACTOR Future<S3BlobStoreEndpoint::ReusableConnection> connect_impl(Reference<S3B
 			TraceEvent("S3BlobStoreEndpointReusingConnected")
 			    .suppressFor(60)
 			    .detail("RemoteEndpoint", rconn.conn->getPeerAddress())
-			    .detail("ExpiresIn", rconn.expirationTime - now());
+			    .detail("ExpiresIn", rconn.expirationTime - now())
+			    .detail("Proxy", b->proxyHost.orDefault(""));
 			return rconn;
 		}
 	}
@@ -747,7 +748,8 @@ ACTOR Future<S3BlobStoreEndpoint::ReusableConnection> connect_impl(Reference<S3B
 	TraceEvent("S3BlobStoreEndpointNewConnection")
 	    .suppressFor(60)
 	    .detail("RemoteEndpoint", conn->getPeerAddress())
-	    .detail("ExpiresIn", b->knobs.max_connection_life);
+	    .detail("ExpiresIn", b->knobs.max_connection_life)
+	    .detail("Proxy", b->proxyHost.orDefault(""));
 
 	if (b->lookupKey || b->lookupSecret || b->knobs.sdk_auth)
 		wait(b->updateSecret());
@@ -960,7 +962,7 @@ ACTOR Future<Reference<HTTP::Response>> doRequest_impl(Reference<S3BlobStoreEndp
 		else
 			event.detail("RemoteHost", bstore->host);
 
-		event.detail("Verb", verb).detail("Resource", resource).detail("ThisTry", thisTry);
+		event.detail("Verb", verb).detail("Resource", resource).detail("ThisTry", thisTry).detail("Proxy", bstore->proxyHost.orDefault(""));
 
 		// If r is not valid or not code 429 then increment the try count.  429's will not count against the attempt
 		// limit.

From 159f691cf6ece5b36a63a5c2b7f4de70f6e2542c Mon Sep 17 00:00:00 2001
From: "Johannes M. Scheuermann" <jscheuermann@apple.com>
Date: Fri, 15 Sep 2023 09:25:20 +0200
Subject: [PATCH 236/468] Correct format

---
 fdbclient/S3BlobStore.actor.cpp | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/fdbclient/S3BlobStore.actor.cpp b/fdbclient/S3BlobStore.actor.cpp
index 808ac78c9d8..eec2c6f3a87 100644
--- a/fdbclient/S3BlobStore.actor.cpp
+++ b/fdbclient/S3BlobStore.actor.cpp
@@ -962,7 +962,10 @@ ACTOR Future<Reference<HTTP::Response>> doRequest_impl(Reference<S3BlobStoreEndp
 		else
 			event.detail("RemoteHost", bstore->host);
 
-		event.detail("Verb", verb).detail("Resource", resource).detail("ThisTry", thisTry).detail("Proxy", bstore->proxyHost.orDefault(""));
+		event.detail("Verb", verb)
+		    .detail("Resource", resource)
+		    .detail("ThisTry", thisTry)
+		    .detail("Proxy", bstore->proxyHost.orDefault(""));
 
 		// If r is not valid or not code 429 then increment the try count.  429's will not count against the attempt
 		// limit.

From 6fb29f812396ec41dc83f3bf429656f0a5741ad6 Mon Sep 17 00:00:00 2001
From: Zhe Wang <zhe.wang@wustl.edu>
Date: Thu, 14 Sep 2023 22:30:33 -0700
Subject: [PATCH 237/468] support dd getTeam to check longest storage queue of
 a team

---
 fdbclient/ServerKnobs.cpp                 |  1 +
 fdbclient/ServerKnobs.h                   |  1 +
 fdbclient/StorageServerInterface.h        |  3 ++-
 fdbserver/DataDistribution.actor.h        |  3 +++
 fdbserver/DataDistributionQueue.actor.cpp | 32 ++++++++++++++++++++++
 fdbserver/StorageMetrics.actor.h          |  6 ++++-
 fdbserver/TCInfo.actor.cpp                | 33 +++++++++++++++++++++++
 fdbserver/TCInfo.h                        | 10 +++++++
 fdbserver/storageserver.actor.cpp         |  9 +++++--
 9 files changed, 94 insertions(+), 4 deletions(-)

diff --git a/fdbclient/ServerKnobs.cpp b/fdbclient/ServerKnobs.cpp
index f64e98fa846..9190be26ced 100644
--- a/fdbclient/ServerKnobs.cpp
+++ b/fdbclient/ServerKnobs.cpp
@@ -162,6 +162,7 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi
 	init( PRIORITY_TEAM_1_LEFT,                                  800 );
 	init( PRIORITY_TEAM_FAILED,                                  805 );
 	init( PRIORITY_TEAM_0_LEFT,                                  809 );
+	init( PRIORITY_STORAGE_QUEUE_AWARE_REDISTRIBUTE,             940 );
 	init( PRIORITY_SPLIT_SHARD,                                  950 ); if( randomize && BUGGIFY ) PRIORITY_SPLIT_SHARD = 350;
 
 	// Data distribution
diff --git a/fdbclient/ServerKnobs.h b/fdbclient/ServerKnobs.h
index f64854db587..eee02a8a899 100644
--- a/fdbclient/ServerKnobs.h
+++ b/fdbclient/ServerKnobs.h
@@ -162,6 +162,7 @@ class ServerKnobs : public KnobsImpl<ServerKnobs> {
 	int PRIORITY_TEAM_FAILED; // Priority when a server in the team is excluded as failed
 	int PRIORITY_TEAM_0_LEFT;
 	int PRIORITY_SPLIT_SHARD;
+	int PRIORITY_STORAGE_QUEUE_AWARE_REDISTRIBUTE;
 
 	// Data distribution
 	double RETRY_RELOCATESHARD_DELAY;
diff --git a/fdbclient/StorageServerInterface.h b/fdbclient/StorageServerInterface.h
index 221c2bf3641..ab8ed9494b2 100644
--- a/fdbclient/StorageServerInterface.h
+++ b/fdbclient/StorageServerInterface.h
@@ -1007,12 +1007,13 @@ struct GetStorageMetricsReply {
 	double bytesInputRate;
 	int64_t versionLag;
 	double lastUpdate;
+	int64_t bytesDurable, bytesInput;
 
 	GetStorageMetricsReply() : bytesInputRate(0) {}
 
 	template <class Ar>
 	void serialize(Ar& ar) {
-		serializer(ar, load, available, capacity, bytesInputRate, versionLag, lastUpdate);
+		serializer(ar, load, available, capacity, bytesInputRate, versionLag, lastUpdate, bytesDurable, bytesInput);
 	}
 };
 
diff --git a/fdbserver/DataDistribution.actor.h b/fdbserver/DataDistribution.actor.h
index e2428c961b3..e66140379f2 100644
--- a/fdbserver/DataDistribution.actor.h
+++ b/fdbserver/DataDistribution.actor.h
@@ -46,6 +46,9 @@ struct IDataDistributionTeam {
 	virtual std::vector<UID> const& getServerIDs() const = 0;
 	virtual void addDataInFlightToTeam(int64_t delta) = 0;
 	virtual int64_t getDataInFlightToTeam() const = 0;
+	virtual void incrementStorageQueueAwareShardToTeam(int64_t delta) = 0;
+	virtual int64_t getStorageQueueAwareShardPerServerNumMax() const = 0;
+	virtual int64_t getLongestStorageQueueSize() const = 0;
 	virtual int64_t getLoadBytes(bool includeInFlight = true, double inflightPenalty = 1.0) const = 0;
 	virtual int64_t getMinAvailableSpace(bool includeInFlight = true) const = 0;
 	virtual double getMinAvailableSpaceRatio(bool includeInFlight = true) const = 0;
diff --git a/fdbserver/DataDistributionQueue.actor.cpp b/fdbserver/DataDistributionQueue.actor.cpp
index 016f40d1fcf..02e15f61e6f 100644
--- a/fdbserver/DataDistributionQueue.actor.cpp
+++ b/fdbserver/DataDistributionQueue.actor.cpp
@@ -79,6 +79,10 @@ struct RelocateData {
 		return priority == SERVER_KNOBS->PRIORITY_SPLIT_SHARD || priority == SERVER_KNOBS->PRIORITY_MERGE_SHARD;
 	}
 
+	static bool isStorageQueueAwarePriority(int priority) {
+		return priority == SERVER_KNOBS->PRIORITY_STORAGE_QUEUE_AWARE_REDISTRIBUTE;
+	}
+
 	bool operator>(const RelocateData& rhs) const {
 		return priority != rhs.priority
 		           ? priority > rhs.priority
@@ -176,6 +180,28 @@ class ParallelTCInfo final : public ReferenceCounted<ParallelTCInfo>, public IDa
 		});
 	}
 
+	void incrementStorageQueueAwareShardToTeam(int64_t delta) override {
+		for (auto& team : teams) {
+			team->incrementStorageQueueAwareShardToTeam(delta);
+		}
+	}
+
+	int64_t getLongestStorageQueueSize() const override {
+		int64_t queueSize = 0;
+		for (const auto& team : teams) {
+			queueSize = std::max(queueSize, team->getLongestStorageQueueSize());
+		}
+		return queueSize;
+	}
+
+	int64_t getStorageQueueAwareShardPerServerNumMax() const override {
+		int64_t shardNumMax = 0;
+		for (const auto& team : teams) {
+			shardNumMax = std::max(shardNumMax, team->getStorageQueueAwareShardPerServerNumMax());
+		}
+		return shardNumMax;
+	}
+
 	int64_t getMinAvailableSpace(bool includeInFlight = true) const override {
 		int64_t result = std::numeric_limits<int64_t>::max();
 		for (const auto& team : teams) {
@@ -1255,6 +1281,8 @@ ACTOR Future<Void> dataDistributionRelocator(DDQueueData* self, RelocateData rd,
 
 			// FIXME: do not add data in flight to servers that were already in the src.
 			healthyDestinations.addDataInFlightToTeam(+metrics.bytes);
+			if (RelocateData::isStorageQueueAwarePriority(rd.priority))
+				healthyDestinations.incrementStorageQueueAwareShardToTeam(+1);
 
 			launchDest(rd, bestTeams, self->destBusymap);
 
@@ -1359,6 +1387,8 @@ ACTOR Future<Void> dataDistributionRelocator(DDQueueData* self, RelocateData rd,
 				}
 
 				healthyDestinations.addDataInFlightToTeam(-metrics.bytes);
+				if (RelocateData::isStorageQueueAwarePriority(rd.priority))
+					healthyDestinations.incrementStorageQueueAwareShardToTeam(-1);
 
 				// onFinished.send( rs );
 				if (!error.code()) {
@@ -1392,6 +1422,8 @@ ACTOR Future<Void> dataDistributionRelocator(DDQueueData* self, RelocateData rd,
 			} else {
 				TEST(true); // move to removed server
 				healthyDestinations.addDataInFlightToTeam(-metrics.bytes);
+				if (RelocateData::isStorageQueueAwarePriority(rd.priority))
+					healthyDestinations.incrementStorageQueueAwareShardToTeam(-1);
 				if (!signalledTransferComplete) {
 					// Update destination busyness map to allow further movement to them.
 					// signalling transferComplete calls completeDest() in complete(), so
diff --git a/fdbserver/StorageMetrics.actor.h b/fdbserver/StorageMetrics.actor.h
index b30337fbcea..cdc9fde82f6 100644
--- a/fdbserver/StorageMetrics.actor.h
+++ b/fdbserver/StorageMetrics.actor.h
@@ -435,7 +435,9 @@ struct StorageServerMetrics {
 	                       StorageBytes sb,
 	                       double bytesInputRate,
 	                       int64_t versionLag,
-	                       double lastUpdate) const {
+	                       double lastUpdate,
+	                       int64_t bytesDurable,
+	                       int64_t bytesInput) const {
 		GetStorageMetricsReply rep;
 
 		// SOMEDAY: make bytes dynamic with hard disk space
@@ -464,6 +466,8 @@ struct StorageServerMetrics {
 
 		rep.versionLag = versionLag;
 		rep.lastUpdate = lastUpdate;
+		rep.bytesDurable = bytesDurable;
+		rep.bytesInput = bytesInput;
 
 		req.reply.send(rep);
 	}
diff --git a/fdbserver/TCInfo.actor.cpp b/fdbserver/TCInfo.actor.cpp
index 341ee94f9d8..f308bfc4ffc 100644
--- a/fdbserver/TCInfo.actor.cpp
+++ b/fdbserver/TCInfo.actor.cpp
@@ -221,6 +221,10 @@ int64_t TCServerInfo::loadBytes() const {
 	return getMetrics().load.bytes;
 }
 
+int64_t TCServerInfo::getStorageQueueSize() const {
+	return getMetrics().bytesInput - getMetrics().bytesDurable;
+}
+
 void TCServerInfo::removeTeam(Reference<TCTeamInfo> team) {
 	for (int t = 0; t < teams.size(); t++) {
 		if (teams[t] == team) {
@@ -356,6 +360,35 @@ int64_t TCTeamInfo::getDataInFlightToTeam() const {
 	return dataInFlight;
 }
 
+void TCTeamInfo::incrementStorageQueueAwareShardToTeam(int64_t delta) {
+	for (int i = 0; i < servers.size(); i++)
+		servers[i]->incrementStorageQueueAwareShardToServer(delta);
+}
+
+int64_t TCTeamInfo::getStorageQueueAwareShardPerServerNumMax() const {
+	int64_t storageQueueAwareShardNumMax = 0;
+	for (auto const& server : servers) {
+		int64_t shardNum = server->getStorageQueueAwareShardNum();
+		if (shardNum > storageQueueAwareShardNumMax) {
+			storageQueueAwareShardNumMax = shardNum;
+		}
+	}
+	return storageQueueAwareShardNumMax;
+}
+
+int64_t TCTeamInfo::getLongestStorageQueueSize() const {
+	int64_t longestQueueSize = 0;
+	for (const auto& server : servers) {
+		if (server->metricsPresent()) {
+			int64_t storageQueueSize = server->getStorageQueueSize();
+			if (storageQueueSize > longestQueueSize) {
+				longestQueueSize = storageQueueSize;
+			}
+		}
+	}
+	return longestQueueSize;
+}
+
 int64_t TCTeamInfo::getLoadBytes(bool includeInFlight, double inflightPenalty) const {
 	int64_t physicalBytes = getLoadAverage();
 	double minAvailableSpaceRatio = getMinAvailableSpaceRatio(includeInFlight);
diff --git a/fdbserver/TCInfo.h b/fdbserver/TCInfo.h
index bac6f521219..10fd91f3ab5 100644
--- a/fdbserver/TCInfo.h
+++ b/fdbserver/TCInfo.h
@@ -45,6 +45,7 @@ class TCServerInfo : public ReferenceCounted<TCServerInfo> {
 	KeyValueStoreType storeType; // Storage engine type
 
 	int64_t dataInFlightToServer;
+	int64_t storageQueueAwareShardNum;
 	std::vector<Reference<TCTeamInfo>> teams;
 	ErrorOr<GetStorageMetricsReply> metrics;
 
@@ -84,6 +85,9 @@ class TCServerInfo : public ReferenceCounted<TCServerInfo> {
 	Future<Void> updateStoreType();
 	KeyValueStoreType getStoreType() const { return storeType; }
 	int64_t getDataInFlightToServer() const { return dataInFlightToServer; }
+	void incrementStorageQueueAwareShardToServer(int64_t bytes) { storageQueueAwareShardNum += bytes; }
+	int64_t getStorageQueueAwareShardNum() const { return storageQueueAwareShardNum; }
+	int64_t getStorageQueueSize() const;
 	void incrementDataInFlightToServer(int64_t bytes) { dataInFlightToServer += bytes; }
 	void cancel();
 	std::vector<Reference<TCTeamInfo>> const& getTeams() const { return teams; }
@@ -193,6 +197,12 @@ class TCTeamInfo final : public ReferenceCounted<TCTeamInfo>, public IDataDistri
 
 	int64_t getDataInFlightToTeam() const override;
 
+	void incrementStorageQueueAwareShardToTeam(int64_t delta) override;
+
+	int64_t getStorageQueueAwareShardPerServerNumMax() const override;
+
+	int64_t getLongestStorageQueueSize() const override;
+
 	int64_t getLoadBytes(bool includeInFlight = true, double inflightPenalty = 1.0) const override;
 
 	int64_t getMinAvailableSpace(bool includeInFlight = true) const override;
diff --git a/fdbserver/storageserver.actor.cpp b/fdbserver/storageserver.actor.cpp
index 1b108462dc5..9ab6733399c 100644
--- a/fdbserver/storageserver.actor.cpp
+++ b/fdbserver/storageserver.actor.cpp
@@ -8669,8 +8669,13 @@ ACTOR Future<Void> metricsCore(StorageServer* self, StorageServerInterface ssi)
 			}
 			when(GetStorageMetricsRequest req = waitNext(ssi.getStorageMetrics.getFuture())) {
 				StorageBytes sb = self->storage.getStorageBytes();
-				self->metrics.getStorageMetrics(
-				    req, sb, self->counters.bytesInput.getRate(), self->versionLag, self->lastUpdate);
+				self->metrics.getStorageMetrics(req,
+				                                sb,
+				                                self->counters.bytesInput.getRate(),
+				                                self->versionLag,
+				                                self->lastUpdate,
+				                                self->counters.bytesDurable.getValue(),
+				                                self->counters.bytesInput.getValue());
 			}
 			when(ReadHotSubRangeRequest req = waitNext(ssi.getReadHotRanges.getFuture())) {
 				if (!self->isReadable(req.keys)) {

From dd56c5d4f271f0f321129cd559da003c166dec58 Mon Sep 17 00:00:00 2001
From: Zhe Wang <zhe.wang@wustl.edu>
Date: Fri, 15 Sep 2023 10:29:29 -0700
Subject: [PATCH 238/468] getLongestStorageQueueSize should notify users if a
 server of the team falls behind

---
 fdbserver/DataDistributionQueue.actor.cpp | 10 +++++++---
 fdbserver/TCInfo.actor.cpp                |  2 ++
 2 files changed, 9 insertions(+), 3 deletions(-)

diff --git a/fdbserver/DataDistributionQueue.actor.cpp b/fdbserver/DataDistributionQueue.actor.cpp
index 02e15f61e6f..6d05a9899ed 100644
--- a/fdbserver/DataDistributionQueue.actor.cpp
+++ b/fdbserver/DataDistributionQueue.actor.cpp
@@ -187,11 +187,15 @@ class ParallelTCInfo final : public ReferenceCounted<ParallelTCInfo>, public IDa
 	}
 
 	int64_t getLongestStorageQueueSize() const override {
-		int64_t queueSize = 0;
+		int64_t maxQueueSize = 0;
 		for (const auto& team : teams) {
-			queueSize = std::max(queueSize, team->getLongestStorageQueueSize());
+			int64_t queueSize = team->getLongestStorageQueueSize();
+			if (queueSize == -1) {
+				return -1;
+			}
+			maxQueueSize = std::max(maxQueueSize, queueSize);
 		}
-		return queueSize;
+		return maxQueueSize;
 	}
 
 	int64_t getStorageQueueAwareShardPerServerNumMax() const override {
diff --git a/fdbserver/TCInfo.actor.cpp b/fdbserver/TCInfo.actor.cpp
index f308bfc4ffc..de8d6454886 100644
--- a/fdbserver/TCInfo.actor.cpp
+++ b/fdbserver/TCInfo.actor.cpp
@@ -384,6 +384,8 @@ int64_t TCTeamInfo::getLongestStorageQueueSize() const {
 			if (storageQueueSize > longestQueueSize) {
 				longestQueueSize = storageQueueSize;
 			}
+		} else if (server->ssVersionTooFarBehind.get() == true) {
+			return -1;
 		}
 	}
 	return longestQueueSize;

From 3c69a94712e76e9cb9fb9e8f1aa93fb178ce9cdf Mon Sep 17 00:00:00 2001
From: Zhe Wang <zhe.wang@wustl.edu>
Date: Fri, 15 Sep 2023 11:42:05 -0700
Subject: [PATCH 239/468] should delay to observe storage queue when decrement
 storage queue aware shard counter

---
 fdbclient/ServerKnobs.cpp                 |  1 +
 fdbclient/ServerKnobs.h                   |  1 +
 fdbserver/DDTeamCollection.actor.cpp      |  2 ++
 fdbserver/DataDistribution.actor.h        |  3 ++-
 fdbserver/DataDistributionQueue.actor.cpp | 16 ++++++++++-----
 fdbserver/TCInfo.actor.cpp                | 25 +++++++++++++++++++++--
 fdbserver/TCInfo.h                        |  9 ++++++--
 7 files changed, 47 insertions(+), 10 deletions(-)

diff --git a/fdbclient/ServerKnobs.cpp b/fdbclient/ServerKnobs.cpp
index 9190be26ced..38cd9835fd4 100644
--- a/fdbclient/ServerKnobs.cpp
+++ b/fdbclient/ServerKnobs.cpp
@@ -290,6 +290,7 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi
 	init( DD_STORAGE_WIGGLE_PAUSE_THRESHOLD,                      10 ); if( randomize && BUGGIFY ) DD_STORAGE_WIGGLE_PAUSE_THRESHOLD = 1000;
 	init( DD_STORAGE_WIGGLE_STUCK_THRESHOLD,                      20 );
 	init( DD_BUILD_EXTRA_TEAMS_OVERRIDE,                          10 ); if( randomize && BUGGIFY ) DD_BUILD_EXTRA_TEAMS_OVERRIDE = 2;
+	init( DD_DECREMENT_STORAGE_QUEUE_AWARE_SHARD_DELAY_TIME,   300.0 );
 
 	// TeamRemover
 	init( TR_FLAG_DISABLE_MACHINE_TEAM_REMOVER,                false ); if( randomize && BUGGIFY ) TR_FLAG_DISABLE_MACHINE_TEAM_REMOVER = deterministicRandom()->random01() < 0.1 ? true : false; // false by default. disable the consistency check when it's true
diff --git a/fdbclient/ServerKnobs.h b/fdbclient/ServerKnobs.h
index eee02a8a899..8acc96d5107 100644
--- a/fdbclient/ServerKnobs.h
+++ b/fdbclient/ServerKnobs.h
@@ -232,6 +232,7 @@ class ServerKnobs : public KnobsImpl<ServerKnobs> {
 	int DD_TEAM_ZERO_SERVER_LEFT_LOG_DELAY;
 	int DD_STORAGE_WIGGLE_PAUSE_THRESHOLD; // How many unhealthy relocations are ongoing will pause storage wiggle
 	int DD_STORAGE_WIGGLE_STUCK_THRESHOLD; // How many times bestTeamStuck accumulate will pause storage wiggle
+	double DD_DECREMENT_STORAGE_QUEUE_AWARE_SHARD_DELAY_TIME;
 
 	// TeamRemover to remove redundant teams
 	bool TR_FLAG_DISABLE_MACHINE_TEAM_REMOVER; // disable the machineTeamRemover actor
diff --git a/fdbserver/DDTeamCollection.actor.cpp b/fdbserver/DDTeamCollection.actor.cpp
index d709e53ad85..e867424d7f4 100644
--- a/fdbserver/DDTeamCollection.actor.cpp
+++ b/fdbserver/DDTeamCollection.actor.cpp
@@ -1020,6 +1020,8 @@ class DDTeamCollectionImpl {
 		state ServerStatus status(false, false, false, server->getLastKnownInterface().locality);
 		state bool lastIsUnhealthy = false;
 		state Future<Void> metricsTracker = server->serverMetricsPolling();
+		state Future<Void> decrementMovingInStorageQueueAwareShardHandler =
+		    server->decrementMovingInStorageQueueAwareShardHandler();
 
 		state Future<std::pair<StorageServerInterface, ProcessClass>> interfaceChanged = server->onInterfaceChanged;
 
diff --git a/fdbserver/DataDistribution.actor.h b/fdbserver/DataDistribution.actor.h
index e66140379f2..6c9ed7dc787 100644
--- a/fdbserver/DataDistribution.actor.h
+++ b/fdbserver/DataDistribution.actor.h
@@ -46,7 +46,8 @@ struct IDataDistributionTeam {
 	virtual std::vector<UID> const& getServerIDs() const = 0;
 	virtual void addDataInFlightToTeam(int64_t delta) = 0;
 	virtual int64_t getDataInFlightToTeam() const = 0;
-	virtual void incrementStorageQueueAwareShardToTeam(int64_t delta) = 0;
+	virtual void incrementMovingStorageQueueAwareShardToTeam() = 0;
+	virtual void decrementMovingStorageQueueAwareShardToTeam() = 0;
 	virtual int64_t getStorageQueueAwareShardPerServerNumMax() const = 0;
 	virtual int64_t getLongestStorageQueueSize() const = 0;
 	virtual int64_t getLoadBytes(bool includeInFlight = true, double inflightPenalty = 1.0) const = 0;
diff --git a/fdbserver/DataDistributionQueue.actor.cpp b/fdbserver/DataDistributionQueue.actor.cpp
index 6d05a9899ed..dc64cdf1276 100644
--- a/fdbserver/DataDistributionQueue.actor.cpp
+++ b/fdbserver/DataDistributionQueue.actor.cpp
@@ -180,9 +180,15 @@ class ParallelTCInfo final : public ReferenceCounted<ParallelTCInfo>, public IDa
 		});
 	}
 
-	void incrementStorageQueueAwareShardToTeam(int64_t delta) override {
+	void incrementMovingStorageQueueAwareShardToTeam() override {
 		for (auto& team : teams) {
-			team->incrementStorageQueueAwareShardToTeam(delta);
+			team->incrementMovingStorageQueueAwareShardToTeam();
+		}
+	}
+
+	void decrementMovingStorageQueueAwareShardToTeam() override {
+		for (auto& team : teams) {
+			team->decrementMovingStorageQueueAwareShardToTeam();
 		}
 	}
 
@@ -1286,7 +1292,7 @@ ACTOR Future<Void> dataDistributionRelocator(DDQueueData* self, RelocateData rd,
 			// FIXME: do not add data in flight to servers that were already in the src.
 			healthyDestinations.addDataInFlightToTeam(+metrics.bytes);
 			if (RelocateData::isStorageQueueAwarePriority(rd.priority))
-				healthyDestinations.incrementStorageQueueAwareShardToTeam(+1);
+				healthyDestinations.incrementMovingStorageQueueAwareShardToTeam();
 
 			launchDest(rd, bestTeams, self->destBusymap);
 
@@ -1392,7 +1398,7 @@ ACTOR Future<Void> dataDistributionRelocator(DDQueueData* self, RelocateData rd,
 
 				healthyDestinations.addDataInFlightToTeam(-metrics.bytes);
 				if (RelocateData::isStorageQueueAwarePriority(rd.priority))
-					healthyDestinations.incrementStorageQueueAwareShardToTeam(-1);
+					healthyDestinations.decrementMovingStorageQueueAwareShardToTeam();
 
 				// onFinished.send( rs );
 				if (!error.code()) {
@@ -1427,7 +1433,7 @@ ACTOR Future<Void> dataDistributionRelocator(DDQueueData* self, RelocateData rd,
 				TEST(true); // move to removed server
 				healthyDestinations.addDataInFlightToTeam(-metrics.bytes);
 				if (RelocateData::isStorageQueueAwarePriority(rd.priority))
-					healthyDestinations.incrementStorageQueueAwareShardToTeam(-1);
+					healthyDestinations.decrementMovingStorageQueueAwareShardToTeam();
 				if (!signalledTransferComplete) {
 					// Update destination busyness map to allow further movement to them.
 					// signalling transferComplete calls completeDest() in complete(), so
diff --git a/fdbserver/TCInfo.actor.cpp b/fdbserver/TCInfo.actor.cpp
index de8d6454886..f79d8a62f26 100644
--- a/fdbserver/TCInfo.actor.cpp
+++ b/fdbserver/TCInfo.actor.cpp
@@ -110,6 +110,18 @@ class TCServerInfoImpl {
 			lastUpdate = now();
 		}
 	}
+
+	ACTOR static Future<Void> decrementMovingInStorageQueueAwareShardHandler(TCServerInfo* server) {
+		loop {
+			choose {
+				when(waitNext(server->decrementMovingInStorageQueueAwareShard.getFuture())) {
+					wait(delay(SERVER_KNOBS->DD_DECREMENT_STORAGE_QUEUE_AWARE_SHARD_DELAY_TIME));
+					wait(updateServerMetrics(server)); // Keep metrics updated before decrement the counter
+					server->storageQueueAwareShardNum--;
+				}
+			}
+		}
+	}
 };
 
 class TCTeamInfoImpl {
@@ -172,6 +184,10 @@ Future<Void> TCServerInfo::serverMetricsPolling() {
 	return TCServerInfoImpl::serverMetricsPolling(this);
 }
 
+Future<Void> TCServerInfo::decrementMovingInStorageQueueAwareShardHandler() {
+	return TCServerInfoImpl::decrementMovingInStorageQueueAwareShardHandler(this);
+}
+
 void TCServerInfo::updateInDesiredDC(std::vector<Optional<Key>> const& includedDCs) {
 	inDesiredDC =
 	    (includedDCs.empty() ||
@@ -360,9 +376,14 @@ int64_t TCTeamInfo::getDataInFlightToTeam() const {
 	return dataInFlight;
 }
 
-void TCTeamInfo::incrementStorageQueueAwareShardToTeam(int64_t delta) {
+void TCTeamInfo::incrementMovingStorageQueueAwareShardToTeam() {
+	for (int i = 0; i < servers.size(); i++)
+		servers[i]->incrementMovingStorageQueueAwareShardToServer();
+}
+
+void TCTeamInfo::decrementMovingStorageQueueAwareShardToTeam() {
 	for (int i = 0; i < servers.size(); i++)
-		servers[i]->incrementStorageQueueAwareShardToServer(delta);
+		servers[i]->decrementMovingStorageQueueAwareShardToServer();
 }
 
 int64_t TCTeamInfo::getStorageQueueAwareShardPerServerNumMax() const {
diff --git a/fdbserver/TCInfo.h b/fdbserver/TCInfo.h
index 10fd91f3ab5..c431361e659 100644
--- a/fdbserver/TCInfo.h
+++ b/fdbserver/TCInfo.h
@@ -67,6 +67,7 @@ class TCServerInfo : public ReferenceCounted<TCServerInfo> {
 	Promise<Void> updated;
 	AsyncVar<bool> wrongStoreTypeToRemove;
 	AsyncVar<bool> ssVersionTooFarBehind;
+	PromiseStream<Void> decrementMovingInStorageQueueAwareShard;
 
 	TCServerInfo(StorageServerInterface ssi,
 	             DDTeamCollection* collection,
@@ -85,7 +86,8 @@ class TCServerInfo : public ReferenceCounted<TCServerInfo> {
 	Future<Void> updateStoreType();
 	KeyValueStoreType getStoreType() const { return storeType; }
 	int64_t getDataInFlightToServer() const { return dataInFlightToServer; }
-	void incrementStorageQueueAwareShardToServer(int64_t bytes) { storageQueueAwareShardNum += bytes; }
+	void incrementMovingStorageQueueAwareShardToServer() { storageQueueAwareShardNum++; }
+	void decrementMovingStorageQueueAwareShardToServer() { decrementMovingInStorageQueueAwareShard.send(Void()); }
 	int64_t getStorageQueueAwareShardNum() const { return storageQueueAwareShardNum; }
 	int64_t getStorageQueueSize() const;
 	void incrementDataInFlightToServer(int64_t bytes) { dataInFlightToServer += bytes; }
@@ -110,6 +112,7 @@ class TCServerInfo : public ReferenceCounted<TCServerInfo> {
 	Future<Void> updateServerMetrics();
 	static Future<Void> updateServerMetrics(Reference<TCServerInfo> server);
 	Future<Void> serverMetricsPolling();
+	Future<Void> decrementMovingInStorageQueueAwareShardHandler();
 
 	~TCServerInfo();
 };
@@ -197,7 +200,9 @@ class TCTeamInfo final : public ReferenceCounted<TCTeamInfo>, public IDataDistri
 
 	int64_t getDataInFlightToTeam() const override;
 
-	void incrementStorageQueueAwareShardToTeam(int64_t delta) override;
+	void incrementMovingStorageQueueAwareShardToTeam() override;
+
+	void decrementMovingStorageQueueAwareShardToTeam() override;
 
 	int64_t getStorageQueueAwareShardPerServerNumMax() const override;
 

From 075a43555e56279708e036bd80480cc51f165fb7 Mon Sep 17 00:00:00 2001
From: Zhe Wang <zhe.wang@wustl.edu>
Date: Fri, 15 Sep 2023 11:53:09 -0700
Subject: [PATCH 240/468] change func/var names

---
 fdbserver/DataDistribution.actor.h        | 2 +-
 fdbserver/DataDistributionQueue.actor.cpp | 4 ++--
 fdbserver/TCInfo.actor.cpp                | 6 +++---
 fdbserver/TCInfo.h                        | 8 ++++----
 4 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/fdbserver/DataDistribution.actor.h b/fdbserver/DataDistribution.actor.h
index 6c9ed7dc787..6722e5980d6 100644
--- a/fdbserver/DataDistribution.actor.h
+++ b/fdbserver/DataDistribution.actor.h
@@ -48,7 +48,7 @@ struct IDataDistributionTeam {
 	virtual int64_t getDataInFlightToTeam() const = 0;
 	virtual void incrementMovingStorageQueueAwareShardToTeam() = 0;
 	virtual void decrementMovingStorageQueueAwareShardToTeam() = 0;
-	virtual int64_t getStorageQueueAwareShardPerServerNumMax() const = 0;
+	virtual int64_t getMovingInStorageQueueAwareShardCounterMax() const = 0;
 	virtual int64_t getLongestStorageQueueSize() const = 0;
 	virtual int64_t getLoadBytes(bool includeInFlight = true, double inflightPenalty = 1.0) const = 0;
 	virtual int64_t getMinAvailableSpace(bool includeInFlight = true) const = 0;
diff --git a/fdbserver/DataDistributionQueue.actor.cpp b/fdbserver/DataDistributionQueue.actor.cpp
index dc64cdf1276..4d1ff0cfa6c 100644
--- a/fdbserver/DataDistributionQueue.actor.cpp
+++ b/fdbserver/DataDistributionQueue.actor.cpp
@@ -204,10 +204,10 @@ class ParallelTCInfo final : public ReferenceCounted<ParallelTCInfo>, public IDa
 		return maxQueueSize;
 	}
 
-	int64_t getStorageQueueAwareShardPerServerNumMax() const override {
+	int64_t getMovingInStorageQueueAwareShardCounterMax() const override {
 		int64_t shardNumMax = 0;
 		for (const auto& team : teams) {
-			shardNumMax = std::max(shardNumMax, team->getStorageQueueAwareShardPerServerNumMax());
+			shardNumMax = std::max(shardNumMax, team->getMovingInStorageQueueAwareShardCounterMax());
 		}
 		return shardNumMax;
 	}
diff --git a/fdbserver/TCInfo.actor.cpp b/fdbserver/TCInfo.actor.cpp
index f79d8a62f26..afd05229346 100644
--- a/fdbserver/TCInfo.actor.cpp
+++ b/fdbserver/TCInfo.actor.cpp
@@ -117,7 +117,7 @@ class TCServerInfoImpl {
 				when(waitNext(server->decrementMovingInStorageQueueAwareShard.getFuture())) {
 					wait(delay(SERVER_KNOBS->DD_DECREMENT_STORAGE_QUEUE_AWARE_SHARD_DELAY_TIME));
 					wait(updateServerMetrics(server)); // Keep metrics updated before decrement the counter
-					server->storageQueueAwareShardNum--;
+					server->movingInStorageQueueAwareShardCounter--;
 				}
 			}
 		}
@@ -386,10 +386,10 @@ void TCTeamInfo::decrementMovingStorageQueueAwareShardToTeam() {
 		servers[i]->decrementMovingStorageQueueAwareShardToServer();
 }
 
-int64_t TCTeamInfo::getStorageQueueAwareShardPerServerNumMax() const {
+int64_t TCTeamInfo::getMovingInStorageQueueAwareShardCounterMax() const {
 	int64_t storageQueueAwareShardNumMax = 0;
 	for (auto const& server : servers) {
-		int64_t shardNum = server->getStorageQueueAwareShardNum();
+		int64_t shardNum = server->getMovingInStorageQueueAwareShardCounter();
 		if (shardNum > storageQueueAwareShardNumMax) {
 			storageQueueAwareShardNumMax = shardNum;
 		}
diff --git a/fdbserver/TCInfo.h b/fdbserver/TCInfo.h
index c431361e659..ab8939d4e2a 100644
--- a/fdbserver/TCInfo.h
+++ b/fdbserver/TCInfo.h
@@ -45,7 +45,7 @@ class TCServerInfo : public ReferenceCounted<TCServerInfo> {
 	KeyValueStoreType storeType; // Storage engine type
 
 	int64_t dataInFlightToServer;
-	int64_t storageQueueAwareShardNum;
+	int64_t movingInStorageQueueAwareShardCounter;
 	std::vector<Reference<TCTeamInfo>> teams;
 	ErrorOr<GetStorageMetricsReply> metrics;
 
@@ -86,9 +86,9 @@ class TCServerInfo : public ReferenceCounted<TCServerInfo> {
 	Future<Void> updateStoreType();
 	KeyValueStoreType getStoreType() const { return storeType; }
 	int64_t getDataInFlightToServer() const { return dataInFlightToServer; }
-	void incrementMovingStorageQueueAwareShardToServer() { storageQueueAwareShardNum++; }
+	void incrementMovingStorageQueueAwareShardToServer() { movingInStorageQueueAwareShardCounter++; }
 	void decrementMovingStorageQueueAwareShardToServer() { decrementMovingInStorageQueueAwareShard.send(Void()); }
-	int64_t getStorageQueueAwareShardNum() const { return storageQueueAwareShardNum; }
+	int64_t getMovingInStorageQueueAwareShardCounter() const { return movingInStorageQueueAwareShardCounter; }
 	int64_t getStorageQueueSize() const;
 	void incrementDataInFlightToServer(int64_t bytes) { dataInFlightToServer += bytes; }
 	void cancel();
@@ -204,7 +204,7 @@ class TCTeamInfo final : public ReferenceCounted<TCTeamInfo>, public IDataDistri
 
 	void decrementMovingStorageQueueAwareShardToTeam() override;
 
-	int64_t getStorageQueueAwareShardPerServerNumMax() const override;
+	int64_t getMovingInStorageQueueAwareShardCounterMax() const override;
 
 	int64_t getLongestStorageQueueSize() const override;
 

From d670560323db1bb0662bb6f92b01eef0930ba221 Mon Sep 17 00:00:00 2001
From: Zhe Wang <zhe.wang@wustl.edu>
Date: Fri, 15 Sep 2023 16:24:19 -0700
Subject: [PATCH 241/468] address comments and add inflight constraints by
 factor the data move parallelism

---
 fdbclient/ServerKnobs.cpp                 |  2 +-
 fdbclient/ServerKnobs.h                   |  2 +-
 fdbserver/DDTeamCollection.actor.cpp      |  2 -
 fdbserver/DataDistribution.actor.h        |  3 --
 fdbserver/DataDistributionQueue.actor.cpp | 47 ++++++++---------------
 fdbserver/TCInfo.actor.cpp                | 37 ------------------
 fdbserver/TCInfo.h                        | 12 ------
 7 files changed, 18 insertions(+), 87 deletions(-)

diff --git a/fdbclient/ServerKnobs.cpp b/fdbclient/ServerKnobs.cpp
index 38cd9835fd4..40383152e9f 100644
--- a/fdbclient/ServerKnobs.cpp
+++ b/fdbclient/ServerKnobs.cpp
@@ -135,6 +135,7 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi
 	init( DD_QUEUE_LOGGING_INTERVAL,                             5.0 );
 	init( RELOCATION_PARALLELISM_PER_SOURCE_SERVER,                2 ); if( randomize && BUGGIFY ) RELOCATION_PARALLELISM_PER_SOURCE_SERVER = 1;
 	init( RELOCATION_PARALLELISM_PER_DEST_SERVER,                 10 ); if( randomize && BUGGIFY ) RELOCATION_PARALLELISM_PER_DEST_SERVER = 1; // Note: if this is smaller than FETCH_KEYS_PARALLELISM, this will artificially reduce performance. The current default of 10 is probably too high but is set conservatively for now.
+	init( RELOCATION_PARALLELISM_FACTOR_HOT_SHARD,               0.2 );
 	init( DD_QUEUE_MAX_KEY_SERVERS,                              100 ); if( randomize && BUGGIFY ) DD_QUEUE_MAX_KEY_SERVERS = 1;
 	init( DD_REBALANCE_PARALLELISM,                               50 );
 	init( DD_REBALANCE_RESET_AMOUNT,                              30 );
@@ -290,7 +291,6 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi
 	init( DD_STORAGE_WIGGLE_PAUSE_THRESHOLD,                      10 ); if( randomize && BUGGIFY ) DD_STORAGE_WIGGLE_PAUSE_THRESHOLD = 1000;
 	init( DD_STORAGE_WIGGLE_STUCK_THRESHOLD,                      20 );
 	init( DD_BUILD_EXTRA_TEAMS_OVERRIDE,                          10 ); if( randomize && BUGGIFY ) DD_BUILD_EXTRA_TEAMS_OVERRIDE = 2;
-	init( DD_DECREMENT_STORAGE_QUEUE_AWARE_SHARD_DELAY_TIME,   300.0 );
 
 	// TeamRemover
 	init( TR_FLAG_DISABLE_MACHINE_TEAM_REMOVER,                false ); if( randomize && BUGGIFY ) TR_FLAG_DISABLE_MACHINE_TEAM_REMOVER = deterministicRandom()->random01() < 0.1 ? true : false; // false by default. disable the consistency check when it's true
diff --git a/fdbclient/ServerKnobs.h b/fdbclient/ServerKnobs.h
index 8acc96d5107..b5355012ba0 100644
--- a/fdbclient/ServerKnobs.h
+++ b/fdbclient/ServerKnobs.h
@@ -128,6 +128,7 @@ class ServerKnobs : public KnobsImpl<ServerKnobs> {
 	double DD_QUEUE_LOGGING_INTERVAL;
 	double RELOCATION_PARALLELISM_PER_SOURCE_SERVER;
 	double RELOCATION_PARALLELISM_PER_DEST_SERVER;
+	double RELOCATION_PARALLELISM_FACTOR_HOT_SHARD;
 	int DD_QUEUE_MAX_KEY_SERVERS;
 	int DD_REBALANCE_PARALLELISM;
 	int DD_REBALANCE_RESET_AMOUNT;
@@ -232,7 +233,6 @@ class ServerKnobs : public KnobsImpl<ServerKnobs> {
 	int DD_TEAM_ZERO_SERVER_LEFT_LOG_DELAY;
 	int DD_STORAGE_WIGGLE_PAUSE_THRESHOLD; // How many unhealthy relocations are ongoing will pause storage wiggle
 	int DD_STORAGE_WIGGLE_STUCK_THRESHOLD; // How many times bestTeamStuck accumulate will pause storage wiggle
-	double DD_DECREMENT_STORAGE_QUEUE_AWARE_SHARD_DELAY_TIME;
 
 	// TeamRemover to remove redundant teams
 	bool TR_FLAG_DISABLE_MACHINE_TEAM_REMOVER; // disable the machineTeamRemover actor
diff --git a/fdbserver/DDTeamCollection.actor.cpp b/fdbserver/DDTeamCollection.actor.cpp
index e867424d7f4..d709e53ad85 100644
--- a/fdbserver/DDTeamCollection.actor.cpp
+++ b/fdbserver/DDTeamCollection.actor.cpp
@@ -1020,8 +1020,6 @@ class DDTeamCollectionImpl {
 		state ServerStatus status(false, false, false, server->getLastKnownInterface().locality);
 		state bool lastIsUnhealthy = false;
 		state Future<Void> metricsTracker = server->serverMetricsPolling();
-		state Future<Void> decrementMovingInStorageQueueAwareShardHandler =
-		    server->decrementMovingInStorageQueueAwareShardHandler();
 
 		state Future<std::pair<StorageServerInterface, ProcessClass>> interfaceChanged = server->onInterfaceChanged;
 
diff --git a/fdbserver/DataDistribution.actor.h b/fdbserver/DataDistribution.actor.h
index 6722e5980d6..06ca4ca2172 100644
--- a/fdbserver/DataDistribution.actor.h
+++ b/fdbserver/DataDistribution.actor.h
@@ -46,9 +46,6 @@ struct IDataDistributionTeam {
 	virtual std::vector<UID> const& getServerIDs() const = 0;
 	virtual void addDataInFlightToTeam(int64_t delta) = 0;
 	virtual int64_t getDataInFlightToTeam() const = 0;
-	virtual void incrementMovingStorageQueueAwareShardToTeam() = 0;
-	virtual void decrementMovingStorageQueueAwareShardToTeam() = 0;
-	virtual int64_t getMovingInStorageQueueAwareShardCounterMax() const = 0;
 	virtual int64_t getLongestStorageQueueSize() const = 0;
 	virtual int64_t getLoadBytes(bool includeInFlight = true, double inflightPenalty = 1.0) const = 0;
 	virtual int64_t getMinAvailableSpace(bool includeInFlight = true) const = 0;
diff --git a/fdbserver/DataDistributionQueue.actor.cpp b/fdbserver/DataDistributionQueue.actor.cpp
index 4d1ff0cfa6c..e37222ab141 100644
--- a/fdbserver/DataDistributionQueue.actor.cpp
+++ b/fdbserver/DataDistributionQueue.actor.cpp
@@ -180,18 +180,6 @@ class ParallelTCInfo final : public ReferenceCounted<ParallelTCInfo>, public IDa
 		});
 	}
 
-	void incrementMovingStorageQueueAwareShardToTeam() override {
-		for (auto& team : teams) {
-			team->incrementMovingStorageQueueAwareShardToTeam();
-		}
-	}
-
-	void decrementMovingStorageQueueAwareShardToTeam() override {
-		for (auto& team : teams) {
-			team->decrementMovingStorageQueueAwareShardToTeam();
-		}
-	}
-
 	int64_t getLongestStorageQueueSize() const override {
 		int64_t maxQueueSize = 0;
 		for (const auto& team : teams) {
@@ -204,14 +192,6 @@ class ParallelTCInfo final : public ReferenceCounted<ParallelTCInfo>, public IDa
 		return maxQueueSize;
 	}
 
-	int64_t getMovingInStorageQueueAwareShardCounterMax() const override {
-		int64_t shardNumMax = 0;
-		for (const auto& team : teams) {
-			shardNumMax = std::max(shardNumMax, team->getMovingInStorageQueueAwareShardCounterMax());
-		}
-		return shardNumMax;
-	}
-
 	int64_t getMinAvailableSpace(bool includeInFlight = true) const override {
 		int64_t result = std::numeric_limits<int64_t>::max();
 		for (const auto& team : teams) {
@@ -334,13 +314,22 @@ int getSrcWorkFactor(RelocateData const& relocation, int singleRegionTeamSize) {
 		return WORK_FULL_UTILIZATION / SERVER_KNOBS->RELOCATION_PARALLELISM_PER_SOURCE_SERVER;
 	else if (relocation.healthPriority == SERVER_KNOBS->PRIORITY_TEAM_2_LEFT)
 		return WORK_FULL_UTILIZATION / 2 / SERVER_KNOBS->RELOCATION_PARALLELISM_PER_SOURCE_SERVER;
+	else if (relocation.priority == SERVER_KNOBS->PRIORITY_STORAGE_QUEUE_AWARE_REDISTRIBUTE)
+		return std::max(1.0,
+		                WORK_FULL_UTILIZATION / singleRegionTeamSize /
+		                    SERVER_KNOBS->RELOCATION_PARALLELISM_PER_SOURCE_SERVER /
+		                    SERVER_KNOBS->RELOCATION_PARALLELISM_FACTOR_HOT_SHARD);
 	else // for now we assume that any message at a lower priority can best be assumed to have a full team left for work
 		return WORK_FULL_UTILIZATION / singleRegionTeamSize / SERVER_KNOBS->RELOCATION_PARALLELISM_PER_SOURCE_SERVER;
 }
 
-int getDestWorkFactor() {
+int getDestWorkFactor(bool forRedistributedHotShard) {
 	// Work of moving a shard is even across destination servers
-	return WORK_FULL_UTILIZATION / SERVER_KNOBS->RELOCATION_PARALLELISM_PER_DEST_SERVER;
+	int res = WORK_FULL_UTILIZATION / SERVER_KNOBS->RELOCATION_PARALLELISM_PER_DEST_SERVER;
+	if (forRedistributedHotShard) {
+		res = std::max(1.0, res / SERVER_KNOBS->RELOCATION_PARALLELISM_FACTOR_HOT_SHARD);
+	}
+	return res;
 }
 
 // Data movement's resource control: Do not overload servers used for the RelocateData
@@ -390,7 +379,7 @@ bool canLaunchDest(const std::vector<std::pair<Reference<IDataDistributionTeam>,
 	if (SERVER_KNOBS->RELOCATION_PARALLELISM_PER_DEST_SERVER <= 0) {
 		return true;
 	}
-	int workFactor = getDestWorkFactor();
+	int workFactor = getDestWorkFactor(priority == SERVER_KNOBS->PRIORITY_STORAGE_QUEUE_AWARE_REDISTRIBUTE);
 	for (auto& team : candidateTeams) {
 		for (UID id : team.first->getServerIDs()) {
 			if (!busymapDest[id].canLaunch(priority, workFactor)) {
@@ -413,7 +402,8 @@ void launchDest(RelocateData& relocation,
                 const std::vector<std::pair<Reference<IDataDistributionTeam>, bool>>& candidateTeams,
                 std::map<UID, Busyness>& destBusymap) {
 	ASSERT(relocation.completeDests.empty());
-	int destWorkFactor = getDestWorkFactor();
+	int destWorkFactor =
+	    getDestWorkFactor(relocation.priority == SERVER_KNOBS->PRIORITY_STORAGE_QUEUE_AWARE_REDISTRIBUTE);
 	for (auto& team : candidateTeams) {
 		for (UID id : team.first->getServerIDs()) {
 			relocation.completeDests.push_back(id);
@@ -423,7 +413,8 @@ void launchDest(RelocateData& relocation,
 }
 
 void completeDest(const RelocateData& relocation, std::map<UID, Busyness>& destBusymap) {
-	int destWorkFactor = getDestWorkFactor();
+	int destWorkFactor =
+	    getDestWorkFactor(relocation.priority == SERVER_KNOBS->PRIORITY_STORAGE_QUEUE_AWARE_REDISTRIBUTE);
 	for (UID id : relocation.completeDests) {
 		destBusymap[id].removeWork(relocation.priority, destWorkFactor);
 	}
@@ -1291,8 +1282,6 @@ ACTOR Future<Void> dataDistributionRelocator(DDQueueData* self, RelocateData rd,
 
 			// FIXME: do not add data in flight to servers that were already in the src.
 			healthyDestinations.addDataInFlightToTeam(+metrics.bytes);
-			if (RelocateData::isStorageQueueAwarePriority(rd.priority))
-				healthyDestinations.incrementMovingStorageQueueAwareShardToTeam();
 
 			launchDest(rd, bestTeams, self->destBusymap);
 
@@ -1397,8 +1386,6 @@ ACTOR Future<Void> dataDistributionRelocator(DDQueueData* self, RelocateData rd,
 				}
 
 				healthyDestinations.addDataInFlightToTeam(-metrics.bytes);
-				if (RelocateData::isStorageQueueAwarePriority(rd.priority))
-					healthyDestinations.decrementMovingStorageQueueAwareShardToTeam();
 
 				// onFinished.send( rs );
 				if (!error.code()) {
@@ -1432,8 +1419,6 @@ ACTOR Future<Void> dataDistributionRelocator(DDQueueData* self, RelocateData rd,
 			} else {
 				TEST(true); // move to removed server
 				healthyDestinations.addDataInFlightToTeam(-metrics.bytes);
-				if (RelocateData::isStorageQueueAwarePriority(rd.priority))
-					healthyDestinations.decrementMovingStorageQueueAwareShardToTeam();
 				if (!signalledTransferComplete) {
 					// Update destination busyness map to allow further movement to them.
 					// signalling transferComplete calls completeDest() in complete(), so
diff --git a/fdbserver/TCInfo.actor.cpp b/fdbserver/TCInfo.actor.cpp
index afd05229346..77bdd42b95b 100644
--- a/fdbserver/TCInfo.actor.cpp
+++ b/fdbserver/TCInfo.actor.cpp
@@ -110,18 +110,6 @@ class TCServerInfoImpl {
 			lastUpdate = now();
 		}
 	}
-
-	ACTOR static Future<Void> decrementMovingInStorageQueueAwareShardHandler(TCServerInfo* server) {
-		loop {
-			choose {
-				when(waitNext(server->decrementMovingInStorageQueueAwareShard.getFuture())) {
-					wait(delay(SERVER_KNOBS->DD_DECREMENT_STORAGE_QUEUE_AWARE_SHARD_DELAY_TIME));
-					wait(updateServerMetrics(server)); // Keep metrics updated before decrement the counter
-					server->movingInStorageQueueAwareShardCounter--;
-				}
-			}
-		}
-	}
 };
 
 class TCTeamInfoImpl {
@@ -184,10 +172,6 @@ Future<Void> TCServerInfo::serverMetricsPolling() {
 	return TCServerInfoImpl::serverMetricsPolling(this);
 }
 
-Future<Void> TCServerInfo::decrementMovingInStorageQueueAwareShardHandler() {
-	return TCServerInfoImpl::decrementMovingInStorageQueueAwareShardHandler(this);
-}
-
 void TCServerInfo::updateInDesiredDC(std::vector<Optional<Key>> const& includedDCs) {
 	inDesiredDC =
 	    (includedDCs.empty() ||
@@ -376,27 +360,6 @@ int64_t TCTeamInfo::getDataInFlightToTeam() const {
 	return dataInFlight;
 }
 
-void TCTeamInfo::incrementMovingStorageQueueAwareShardToTeam() {
-	for (int i = 0; i < servers.size(); i++)
-		servers[i]->incrementMovingStorageQueueAwareShardToServer();
-}
-
-void TCTeamInfo::decrementMovingStorageQueueAwareShardToTeam() {
-	for (int i = 0; i < servers.size(); i++)
-		servers[i]->decrementMovingStorageQueueAwareShardToServer();
-}
-
-int64_t TCTeamInfo::getMovingInStorageQueueAwareShardCounterMax() const {
-	int64_t storageQueueAwareShardNumMax = 0;
-	for (auto const& server : servers) {
-		int64_t shardNum = server->getMovingInStorageQueueAwareShardCounter();
-		if (shardNum > storageQueueAwareShardNumMax) {
-			storageQueueAwareShardNumMax = shardNum;
-		}
-	}
-	return storageQueueAwareShardNumMax;
-}
-
 int64_t TCTeamInfo::getLongestStorageQueueSize() const {
 	int64_t longestQueueSize = 0;
 	for (const auto& server : servers) {
diff --git a/fdbserver/TCInfo.h b/fdbserver/TCInfo.h
index ab8939d4e2a..123717c42cf 100644
--- a/fdbserver/TCInfo.h
+++ b/fdbserver/TCInfo.h
@@ -45,7 +45,6 @@ class TCServerInfo : public ReferenceCounted<TCServerInfo> {
 	KeyValueStoreType storeType; // Storage engine type
 
 	int64_t dataInFlightToServer;
-	int64_t movingInStorageQueueAwareShardCounter;
 	std::vector<Reference<TCTeamInfo>> teams;
 	ErrorOr<GetStorageMetricsReply> metrics;
 
@@ -67,7 +66,6 @@ class TCServerInfo : public ReferenceCounted<TCServerInfo> {
 	Promise<Void> updated;
 	AsyncVar<bool> wrongStoreTypeToRemove;
 	AsyncVar<bool> ssVersionTooFarBehind;
-	PromiseStream<Void> decrementMovingInStorageQueueAwareShard;
 
 	TCServerInfo(StorageServerInterface ssi,
 	             DDTeamCollection* collection,
@@ -86,9 +84,6 @@ class TCServerInfo : public ReferenceCounted<TCServerInfo> {
 	Future<Void> updateStoreType();
 	KeyValueStoreType getStoreType() const { return storeType; }
 	int64_t getDataInFlightToServer() const { return dataInFlightToServer; }
-	void incrementMovingStorageQueueAwareShardToServer() { movingInStorageQueueAwareShardCounter++; }
-	void decrementMovingStorageQueueAwareShardToServer() { decrementMovingInStorageQueueAwareShard.send(Void()); }
-	int64_t getMovingInStorageQueueAwareShardCounter() const { return movingInStorageQueueAwareShardCounter; }
 	int64_t getStorageQueueSize() const;
 	void incrementDataInFlightToServer(int64_t bytes) { dataInFlightToServer += bytes; }
 	void cancel();
@@ -112,7 +107,6 @@ class TCServerInfo : public ReferenceCounted<TCServerInfo> {
 	Future<Void> updateServerMetrics();
 	static Future<Void> updateServerMetrics(Reference<TCServerInfo> server);
 	Future<Void> serverMetricsPolling();
-	Future<Void> decrementMovingInStorageQueueAwareShardHandler();
 
 	~TCServerInfo();
 };
@@ -200,12 +194,6 @@ class TCTeamInfo final : public ReferenceCounted<TCTeamInfo>, public IDataDistri
 
 	int64_t getDataInFlightToTeam() const override;
 
-	void incrementMovingStorageQueueAwareShardToTeam() override;
-
-	void decrementMovingStorageQueueAwareShardToTeam() override;
-
-	int64_t getMovingInStorageQueueAwareShardCounterMax() const override;
-
 	int64_t getLongestStorageQueueSize() const override;
 
 	int64_t getLoadBytes(bool includeInFlight = true, double inflightPenalty = 1.0) const override;

From da6613efd0902aea46c48ee8562ceb5209cfdc00 Mon Sep 17 00:00:00 2001
From: Zhe Wang <zhe.wang@wustl.edu>
Date: Fri, 15 Sep 2023 16:41:58 -0700
Subject: [PATCH 242/468] ENABLE_STORAGE_QUEUE_AWARE_TEAM_SELECTION

---
 fdbclient/ServerKnobs.cpp                 |  4 +--
 fdbclient/ServerKnobs.h                   |  4 +--
 fdbserver/DDTeamCollection.actor.cpp      | 34 +++++++++++++++++++++--
 fdbserver/DataDistribution.actor.h        | 12 ++++----
 fdbserver/DataDistributionQueue.actor.cpp | 28 ++++++-------------
 5 files changed, 51 insertions(+), 31 deletions(-)

diff --git a/fdbclient/ServerKnobs.cpp b/fdbclient/ServerKnobs.cpp
index 40383152e9f..7067718dc2c 100644
--- a/fdbclient/ServerKnobs.cpp
+++ b/fdbclient/ServerKnobs.cpp
@@ -135,7 +135,6 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi
 	init( DD_QUEUE_LOGGING_INTERVAL,                             5.0 );
 	init( RELOCATION_PARALLELISM_PER_SOURCE_SERVER,                2 ); if( randomize && BUGGIFY ) RELOCATION_PARALLELISM_PER_SOURCE_SERVER = 1;
 	init( RELOCATION_PARALLELISM_PER_DEST_SERVER,                 10 ); if( randomize && BUGGIFY ) RELOCATION_PARALLELISM_PER_DEST_SERVER = 1; // Note: if this is smaller than FETCH_KEYS_PARALLELISM, this will artificially reduce performance. The current default of 10 is probably too high but is set conservatively for now.
-	init( RELOCATION_PARALLELISM_FACTOR_HOT_SHARD,               0.2 );
 	init( DD_QUEUE_MAX_KEY_SERVERS,                              100 ); if( randomize && BUGGIFY ) DD_QUEUE_MAX_KEY_SERVERS = 1;
 	init( DD_REBALANCE_PARALLELISM,                               50 );
 	init( DD_REBALANCE_RESET_AMOUNT,                              30 );
@@ -163,7 +162,6 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi
 	init( PRIORITY_TEAM_1_LEFT,                                  800 );
 	init( PRIORITY_TEAM_FAILED,                                  805 );
 	init( PRIORITY_TEAM_0_LEFT,                                  809 );
-	init( PRIORITY_STORAGE_QUEUE_AWARE_REDISTRIBUTE,             940 );
 	init( PRIORITY_SPLIT_SHARD,                                  950 ); if( randomize && BUGGIFY ) PRIORITY_SPLIT_SHARD = 350;
 
 	// Data distribution
@@ -291,6 +289,8 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi
 	init( DD_STORAGE_WIGGLE_PAUSE_THRESHOLD,                      10 ); if( randomize && BUGGIFY ) DD_STORAGE_WIGGLE_PAUSE_THRESHOLD = 1000;
 	init( DD_STORAGE_WIGGLE_STUCK_THRESHOLD,                      20 );
 	init( DD_BUILD_EXTRA_TEAMS_OVERRIDE,                          10 ); if( randomize && BUGGIFY ) DD_BUILD_EXTRA_TEAMS_OVERRIDE = 2;
+	init( ENABLE_STORAGE_QUEUE_AWARE_TEAM_SELECTION,            true );
+	init( DD_TARGET_STORAGE_QUEUE_SIZE,                         TARGET_BYTES_PER_STORAGE_SERVER/3 );
 
 	// TeamRemover
 	init( TR_FLAG_DISABLE_MACHINE_TEAM_REMOVER,                false ); if( randomize && BUGGIFY ) TR_FLAG_DISABLE_MACHINE_TEAM_REMOVER = deterministicRandom()->random01() < 0.1 ? true : false; // false by default. disable the consistency check when it's true
diff --git a/fdbclient/ServerKnobs.h b/fdbclient/ServerKnobs.h
index b5355012ba0..1d23ee2e908 100644
--- a/fdbclient/ServerKnobs.h
+++ b/fdbclient/ServerKnobs.h
@@ -128,7 +128,6 @@ class ServerKnobs : public KnobsImpl<ServerKnobs> {
 	double DD_QUEUE_LOGGING_INTERVAL;
 	double RELOCATION_PARALLELISM_PER_SOURCE_SERVER;
 	double RELOCATION_PARALLELISM_PER_DEST_SERVER;
-	double RELOCATION_PARALLELISM_FACTOR_HOT_SHARD;
 	int DD_QUEUE_MAX_KEY_SERVERS;
 	int DD_REBALANCE_PARALLELISM;
 	int DD_REBALANCE_RESET_AMOUNT;
@@ -163,7 +162,6 @@ class ServerKnobs : public KnobsImpl<ServerKnobs> {
 	int PRIORITY_TEAM_FAILED; // Priority when a server in the team is excluded as failed
 	int PRIORITY_TEAM_0_LEFT;
 	int PRIORITY_SPLIT_SHARD;
-	int PRIORITY_STORAGE_QUEUE_AWARE_REDISTRIBUTE;
 
 	// Data distribution
 	double RETRY_RELOCATESHARD_DELAY;
@@ -233,6 +231,8 @@ class ServerKnobs : public KnobsImpl<ServerKnobs> {
 	int DD_TEAM_ZERO_SERVER_LEFT_LOG_DELAY;
 	int DD_STORAGE_WIGGLE_PAUSE_THRESHOLD; // How many unhealthy relocations are ongoing will pause storage wiggle
 	int DD_STORAGE_WIGGLE_STUCK_THRESHOLD; // How many times bestTeamStuck accumulate will pause storage wiggle
+	bool ENABLE_STORAGE_QUEUE_AWARE_TEAM_SELECTION; // experimental!
+	int64_t DD_TARGET_STORAGE_QUEUE_SIZE;
 
 	// TeamRemover to remove redundant teams
 	bool TR_FLAG_DISABLE_MACHINE_TEAM_REMOVER; // disable the machineTeamRemover actor
diff --git a/fdbserver/DDTeamCollection.actor.cpp b/fdbserver/DDTeamCollection.actor.cpp
index d709e53ad85..b19dcd68b31 100644
--- a/fdbserver/DDTeamCollection.actor.cpp
+++ b/fdbserver/DDTeamCollection.actor.cpp
@@ -265,6 +265,14 @@ class DDTeamCollectionImpl {
 					    (!req.preferLowerUtilization ||
 					     self->teams[currentIndex]->hasHealthyAvailableSpace(self->medianAvailableSpace))) {
 						int64_t loadBytes = self->teams[currentIndex]->getLoadBytes(true, req.inflightPenalty);
+						if (req.storageQueueAware) {
+							int64_t storageQueueSize = self->teams[currentIndex]->getLongestStorageQueueSize();
+							if (storageQueueSize == -1) {
+								continue; // this SS may not healthy, skip
+							} else if (storageQueueSize > SERVER_KNOBS->DD_TARGET_STORAGE_QUEUE_SIZE) {
+								continue; // this SS storage queue is too long, skip
+							}
+						}
 						if ((!bestOption.present() || (req.preferLowerUtilization && loadBytes < bestLoadBytes) ||
 						     (!req.preferLowerUtilization && loadBytes > bestLoadBytes)) &&
 						    (!req.teamMustHaveShards ||
@@ -308,6 +316,15 @@ class DDTeamCollectionImpl {
 					            self->shardsAffectedByTeamFailure->hasShards(
 					                ShardsAffectedByTeamFailure::Team(dest->getServerIDs(), self->primary)));
 
+					if (req.storageQueueAware) {
+						int64_t storageQueueSize = dest->getLongestStorageQueueSize();
+						if (storageQueueSize == -1) {
+							ok = false; // this SS may not healthy, skip
+						} else if (storageQueueSize > SERVER_KNOBS->DD_TARGET_STORAGE_QUEUE_SIZE) {
+							ok = false; // this SS storage queue is too long, skip
+						}
+					}
+
 					if (ok)
 						randomTeams.push_back(dest);
 					else
@@ -326,6 +343,14 @@ class DDTeamCollectionImpl {
 
 				for (int i = 0; i < randomTeams.size(); i++) {
 					int64_t loadBytes = randomTeams[i]->getLoadBytes(true, req.inflightPenalty);
+					if (req.storageQueueAware) {
+						int64_t storageQueueSize = randomTeams[i]->getLongestStorageQueueSize();
+						if (storageQueueSize == -1) {
+							continue; // this SS may not healthy, skip
+						} else if (storageQueueSize > SERVER_KNOBS->DD_TARGET_STORAGE_QUEUE_SIZE) {
+							continue; // this SS storage queue is too long, skip
+						}
+					}
 					if (!bestOption.present() || (req.preferLowerUtilization && loadBytes < bestLoadBytes) ||
 					    (!req.preferLowerUtilization && loadBytes > bestLoadBytes)) {
 
@@ -374,11 +399,16 @@ class DDTeamCollectionImpl {
 			// 	self->traceAllInfo(true);
 			// }
 
-			req.reply.send(std::make_pair(bestOption, foundSrc));
+			if (req.storageQueueAware && !bestOption.present()) {
+				req.storageQueueAware = false;
+				wait(getTeam(self, req)); // re-run getTeam without storageQueueAware
+			} else {
+				req.reply.send(std::make_pair(bestOption, foundSrc));
+			}
 
 			return Void();
 		} catch (Error& e) {
-			if (e.code() != error_code_actor_cancelled)
+			if (e.code() != error_code_actor_cancelled && req.reply.canBeSet())
 				req.reply.sendError(e);
 			throw;
 		}
diff --git a/fdbserver/DataDistribution.actor.h b/fdbserver/DataDistribution.actor.h
index 06ca4ca2172..a53489a4421 100644
--- a/fdbserver/DataDistribution.actor.h
+++ b/fdbserver/DataDistribution.actor.h
@@ -80,6 +80,7 @@ struct IDataDistributionTeam {
 struct GetTeamRequest {
 	bool wantsNewServers;
 	bool wantsTrueBest;
+	bool storageQueueAware;
 	bool preferLowerUtilization;
 	bool teamMustHaveShards;
 	double inflightPenalty;
@@ -93,15 +94,16 @@ struct GetTeamRequest {
 	               bool preferLowerUtilization,
 	               bool teamMustHaveShards,
 	               double inflightPenalty = 1.0)
-	  : wantsNewServers(wantsNewServers), wantsTrueBest(wantsTrueBest), preferLowerUtilization(preferLowerUtilization),
-	    teamMustHaveShards(teamMustHaveShards), inflightPenalty(inflightPenalty) {}
+	  : wantsNewServers(wantsNewServers), wantsTrueBest(wantsTrueBest), storageQueueAware(false),
+	    preferLowerUtilization(preferLowerUtilization), teamMustHaveShards(teamMustHaveShards),
+	    inflightPenalty(inflightPenalty) {}
 
 	std::string getDesc() const {
 		std::stringstream ss;
 
-		ss << "WantsNewServers:" << wantsNewServers << " WantsTrueBest:" << wantsTrueBest
-		   << " PreferLowerUtilization:" << preferLowerUtilization << " teamMustHaveShards:" << teamMustHaveShards
-		   << " inflightPenalty:" << inflightPenalty << ";";
+		ss << "WantsNewServers:" << wantsNewServers << " WantsTrueBest:" << wantsTrueBest << "StorageQueueAware"
+		   << storageQueueAware << " PreferLowerUtilization:" << preferLowerUtilization
+		   << " teamMustHaveShards:" << teamMustHaveShards << " inflightPenalty:" << inflightPenalty << ";";
 		ss << "CompleteSources:";
 		for (const auto& cs : completeSources) {
 			ss << cs.toString() << ",";
diff --git a/fdbserver/DataDistributionQueue.actor.cpp b/fdbserver/DataDistributionQueue.actor.cpp
index e37222ab141..030e0dd2d53 100644
--- a/fdbserver/DataDistributionQueue.actor.cpp
+++ b/fdbserver/DataDistributionQueue.actor.cpp
@@ -79,10 +79,6 @@ struct RelocateData {
 		return priority == SERVER_KNOBS->PRIORITY_SPLIT_SHARD || priority == SERVER_KNOBS->PRIORITY_MERGE_SHARD;
 	}
 
-	static bool isStorageQueueAwarePriority(int priority) {
-		return priority == SERVER_KNOBS->PRIORITY_STORAGE_QUEUE_AWARE_REDISTRIBUTE;
-	}
-
 	bool operator>(const RelocateData& rhs) const {
 		return priority != rhs.priority
 		           ? priority > rhs.priority
@@ -314,22 +310,13 @@ int getSrcWorkFactor(RelocateData const& relocation, int singleRegionTeamSize) {
 		return WORK_FULL_UTILIZATION / SERVER_KNOBS->RELOCATION_PARALLELISM_PER_SOURCE_SERVER;
 	else if (relocation.healthPriority == SERVER_KNOBS->PRIORITY_TEAM_2_LEFT)
 		return WORK_FULL_UTILIZATION / 2 / SERVER_KNOBS->RELOCATION_PARALLELISM_PER_SOURCE_SERVER;
-	else if (relocation.priority == SERVER_KNOBS->PRIORITY_STORAGE_QUEUE_AWARE_REDISTRIBUTE)
-		return std::max(1.0,
-		                WORK_FULL_UTILIZATION / singleRegionTeamSize /
-		                    SERVER_KNOBS->RELOCATION_PARALLELISM_PER_SOURCE_SERVER /
-		                    SERVER_KNOBS->RELOCATION_PARALLELISM_FACTOR_HOT_SHARD);
 	else // for now we assume that any message at a lower priority can best be assumed to have a full team left for work
 		return WORK_FULL_UTILIZATION / singleRegionTeamSize / SERVER_KNOBS->RELOCATION_PARALLELISM_PER_SOURCE_SERVER;
 }
 
-int getDestWorkFactor(bool forRedistributedHotShard) {
+int getDestWorkFactor() {
 	// Work of moving a shard is even across destination servers
-	int res = WORK_FULL_UTILIZATION / SERVER_KNOBS->RELOCATION_PARALLELISM_PER_DEST_SERVER;
-	if (forRedistributedHotShard) {
-		res = std::max(1.0, res / SERVER_KNOBS->RELOCATION_PARALLELISM_FACTOR_HOT_SHARD);
-	}
-	return res;
+	return WORK_FULL_UTILIZATION / SERVER_KNOBS->RELOCATION_PARALLELISM_PER_DEST_SERVER;
 }
 
 // Data movement's resource control: Do not overload servers used for the RelocateData
@@ -379,7 +366,7 @@ bool canLaunchDest(const std::vector<std::pair<Reference<IDataDistributionTeam>,
 	if (SERVER_KNOBS->RELOCATION_PARALLELISM_PER_DEST_SERVER <= 0) {
 		return true;
 	}
-	int workFactor = getDestWorkFactor(priority == SERVER_KNOBS->PRIORITY_STORAGE_QUEUE_AWARE_REDISTRIBUTE);
+	int workFactor = getDestWorkFactor();
 	for (auto& team : candidateTeams) {
 		for (UID id : team.first->getServerIDs()) {
 			if (!busymapDest[id].canLaunch(priority, workFactor)) {
@@ -402,8 +389,7 @@ void launchDest(RelocateData& relocation,
                 const std::vector<std::pair<Reference<IDataDistributionTeam>, bool>>& candidateTeams,
                 std::map<UID, Busyness>& destBusymap) {
 	ASSERT(relocation.completeDests.empty());
-	int destWorkFactor =
-	    getDestWorkFactor(relocation.priority == SERVER_KNOBS->PRIORITY_STORAGE_QUEUE_AWARE_REDISTRIBUTE);
+	int destWorkFactor = getDestWorkFactor();
 	for (auto& team : candidateTeams) {
 		for (UID id : team.first->getServerIDs()) {
 			relocation.completeDests.push_back(id);
@@ -413,8 +399,7 @@ void launchDest(RelocateData& relocation,
 }
 
 void completeDest(const RelocateData& relocation, std::map<UID, Busyness>& destBusymap) {
-	int destWorkFactor =
-	    getDestWorkFactor(relocation.priority == SERVER_KNOBS->PRIORITY_STORAGE_QUEUE_AWARE_REDISTRIBUTE);
+	int destWorkFactor = getDestWorkFactor();
 	for (UID id : relocation.completeDests) {
 		destBusymap[id].removeWork(relocation.priority, destWorkFactor);
 	}
@@ -1155,6 +1140,9 @@ ACTOR Future<Void> dataDistributionRelocator(DDQueueData* self, RelocateData rd,
 					                          inflightPenalty);
 					req.src = rd.src;
 					req.completeSources = rd.completeSources;
+					if (SERVER_KNOBS->ENABLE_STORAGE_QUEUE_AWARE_TEAM_SELECTION) {
+						req.storageQueueAware = true;
+					}
 					// bestTeam.second = false if the bestTeam in the teamCollection (in the DC) does not have any
 					// server that hosts the relocateData. This is possible, for example, in a fearless configuration
 					// when the remote DC is just brought up.

From ff603d0d514bed9af2948eaa4aa0838a9625fd5e Mon Sep 17 00:00:00 2001
From: Zhe Wang <zhe.wang@wustl.edu>
Date: Mon, 18 Sep 2023 10:49:01 -0700
Subject: [PATCH 243/468] address comments

---
 fdbclient/ServerKnobs.cpp                 | 2 +-
 fdbserver/DDTeamCollection.actor.cpp      | 8 --------
 fdbserver/DataDistributionQueue.actor.cpp | 4 +---
 fdbserver/TCInfo.actor.cpp                | 5 +----
 4 files changed, 3 insertions(+), 16 deletions(-)

diff --git a/fdbclient/ServerKnobs.cpp b/fdbclient/ServerKnobs.cpp
index 7067718dc2c..ae0235f8b26 100644
--- a/fdbclient/ServerKnobs.cpp
+++ b/fdbclient/ServerKnobs.cpp
@@ -289,7 +289,7 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi
 	init( DD_STORAGE_WIGGLE_PAUSE_THRESHOLD,                      10 ); if( randomize && BUGGIFY ) DD_STORAGE_WIGGLE_PAUSE_THRESHOLD = 1000;
 	init( DD_STORAGE_WIGGLE_STUCK_THRESHOLD,                      20 );
 	init( DD_BUILD_EXTRA_TEAMS_OVERRIDE,                          10 ); if( randomize && BUGGIFY ) DD_BUILD_EXTRA_TEAMS_OVERRIDE = 2;
-	init( ENABLE_STORAGE_QUEUE_AWARE_TEAM_SELECTION,            true );
+	init( ENABLE_STORAGE_QUEUE_AWARE_TEAM_SELECTION,           false );
 	init( DD_TARGET_STORAGE_QUEUE_SIZE,                         TARGET_BYTES_PER_STORAGE_SERVER/3 );
 
 	// TeamRemover
diff --git a/fdbserver/DDTeamCollection.actor.cpp b/fdbserver/DDTeamCollection.actor.cpp
index b19dcd68b31..feb898387b2 100644
--- a/fdbserver/DDTeamCollection.actor.cpp
+++ b/fdbserver/DDTeamCollection.actor.cpp
@@ -343,14 +343,6 @@ class DDTeamCollectionImpl {
 
 				for (int i = 0; i < randomTeams.size(); i++) {
 					int64_t loadBytes = randomTeams[i]->getLoadBytes(true, req.inflightPenalty);
-					if (req.storageQueueAware) {
-						int64_t storageQueueSize = randomTeams[i]->getLongestStorageQueueSize();
-						if (storageQueueSize == -1) {
-							continue; // this SS may not healthy, skip
-						} else if (storageQueueSize > SERVER_KNOBS->DD_TARGET_STORAGE_QUEUE_SIZE) {
-							continue; // this SS storage queue is too long, skip
-						}
-					}
 					if (!bestOption.present() || (req.preferLowerUtilization && loadBytes < bestLoadBytes) ||
 					    (!req.preferLowerUtilization && loadBytes > bestLoadBytes)) {
 
diff --git a/fdbserver/DataDistributionQueue.actor.cpp b/fdbserver/DataDistributionQueue.actor.cpp
index 030e0dd2d53..00e1b152ed4 100644
--- a/fdbserver/DataDistributionQueue.actor.cpp
+++ b/fdbserver/DataDistributionQueue.actor.cpp
@@ -1140,9 +1140,7 @@ ACTOR Future<Void> dataDistributionRelocator(DDQueueData* self, RelocateData rd,
 					                          inflightPenalty);
 					req.src = rd.src;
 					req.completeSources = rd.completeSources;
-					if (SERVER_KNOBS->ENABLE_STORAGE_QUEUE_AWARE_TEAM_SELECTION) {
-						req.storageQueueAware = true;
-					}
+					req.storageQueueAware = SERVER_KNOBS->ENABLE_STORAGE_QUEUE_AWARE_TEAM_SELECTION;
 					// bestTeam.second = false if the bestTeam in the teamCollection (in the DC) does not have any
 					// server that hosts the relocateData. This is possible, for example, in a fearless configuration
 					// when the remote DC is just brought up.
diff --git a/fdbserver/TCInfo.actor.cpp b/fdbserver/TCInfo.actor.cpp
index 77bdd42b95b..b80b7836ad3 100644
--- a/fdbserver/TCInfo.actor.cpp
+++ b/fdbserver/TCInfo.actor.cpp
@@ -364,10 +364,7 @@ int64_t TCTeamInfo::getLongestStorageQueueSize() const {
 	int64_t longestQueueSize = 0;
 	for (const auto& server : servers) {
 		if (server->metricsPresent()) {
-			int64_t storageQueueSize = server->getStorageQueueSize();
-			if (storageQueueSize > longestQueueSize) {
-				longestQueueSize = storageQueueSize;
-			}
+			longestQueueSize = std::max(longestQueueSize, server->getStorageQueueSize());
 		} else if (server->ssVersionTooFarBehind.get() == true) {
 			return -1;
 		}

From d54af8e124dc32f7316b9c0ee714d8f332857e29 Mon Sep 17 00:00:00 2001
From: Zhe Wang <zhe.wang@wustl.edu>
Date: Mon, 18 Sep 2023 22:04:39 -0700
Subject: [PATCH 244/468] address comments

---
 fdbclient/ServerKnobs.cpp                 |  4 ++--
 fdbserver/DDTeamCollection.actor.cpp      | 13 +++++++------
 fdbserver/DataDistribution.actor.h        |  2 +-
 fdbserver/DataDistributionQueue.actor.cpp | 10 +++++-----
 fdbserver/TCInfo.actor.cpp                |  6 +++---
 fdbserver/TCInfo.h                        |  2 +-
 6 files changed, 19 insertions(+), 18 deletions(-)

diff --git a/fdbclient/ServerKnobs.cpp b/fdbclient/ServerKnobs.cpp
index ae0235f8b26..7987479a92c 100644
--- a/fdbclient/ServerKnobs.cpp
+++ b/fdbclient/ServerKnobs.cpp
@@ -289,8 +289,8 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi
 	init( DD_STORAGE_WIGGLE_PAUSE_THRESHOLD,                      10 ); if( randomize && BUGGIFY ) DD_STORAGE_WIGGLE_PAUSE_THRESHOLD = 1000;
 	init( DD_STORAGE_WIGGLE_STUCK_THRESHOLD,                      20 );
 	init( DD_BUILD_EXTRA_TEAMS_OVERRIDE,                          10 ); if( randomize && BUGGIFY ) DD_BUILD_EXTRA_TEAMS_OVERRIDE = 2;
-	init( ENABLE_STORAGE_QUEUE_AWARE_TEAM_SELECTION,           false );
-	init( DD_TARGET_STORAGE_QUEUE_SIZE,                         TARGET_BYTES_PER_STORAGE_SERVER/3 );
+	init( ENABLE_STORAGE_QUEUE_AWARE_TEAM_SELECTION,           false ); if( randomize && BUGGIFY ) ENABLE_STORAGE_QUEUE_AWARE_TEAM_SELECTION = true;
+	init( DD_TARGET_STORAGE_QUEUE_SIZE, TARGET_BYTES_PER_STORAGE_SERVER/3 ); if( randomize && BUGGIFY ) DD_TARGET_STORAGE_QUEUE_SIZE = TARGET_BYTES_PER_STORAGE_SERVER/10;
 
 	// TeamRemover
 	init( TR_FLAG_DISABLE_MACHINE_TEAM_REMOVER,                false ); if( randomize && BUGGIFY ) TR_FLAG_DISABLE_MACHINE_TEAM_REMOVER = deterministicRandom()->random01() < 0.1 ? true : false; // false by default. disable the consistency check when it's true
diff --git a/fdbserver/DDTeamCollection.actor.cpp b/fdbserver/DDTeamCollection.actor.cpp
index feb898387b2..41ecc119799 100644
--- a/fdbserver/DDTeamCollection.actor.cpp
+++ b/fdbserver/DDTeamCollection.actor.cpp
@@ -266,10 +266,11 @@ class DDTeamCollectionImpl {
 					     self->teams[currentIndex]->hasHealthyAvailableSpace(self->medianAvailableSpace))) {
 						int64_t loadBytes = self->teams[currentIndex]->getLoadBytes(true, req.inflightPenalty);
 						if (req.storageQueueAware) {
-							int64_t storageQueueSize = self->teams[currentIndex]->getLongestStorageQueueSize();
-							if (storageQueueSize == -1) {
+							Optional<int64_t> storageQueueSize =
+							    self->teams[currentIndex]->getLongestStorageQueueSize();
+							if (!storageQueueSize.present()) {
 								continue; // this SS may not healthy, skip
-							} else if (storageQueueSize > SERVER_KNOBS->DD_TARGET_STORAGE_QUEUE_SIZE) {
+							} else if (storageQueueSize.get() > SERVER_KNOBS->DD_TARGET_STORAGE_QUEUE_SIZE) {
 								continue; // this SS storage queue is too long, skip
 							}
 						}
@@ -317,10 +318,10 @@ class DDTeamCollectionImpl {
 					                ShardsAffectedByTeamFailure::Team(dest->getServerIDs(), self->primary)));
 
 					if (req.storageQueueAware) {
-						int64_t storageQueueSize = dest->getLongestStorageQueueSize();
-						if (storageQueueSize == -1) {
+						Optional<int64_t> storageQueueSize = dest->getLongestStorageQueueSize();
+						if (!storageQueueSize.present()) {
 							ok = false; // this SS may not healthy, skip
-						} else if (storageQueueSize > SERVER_KNOBS->DD_TARGET_STORAGE_QUEUE_SIZE) {
+						} else if (storageQueueSize.get() > SERVER_KNOBS->DD_TARGET_STORAGE_QUEUE_SIZE) {
 							ok = false; // this SS storage queue is too long, skip
 						}
 					}
diff --git a/fdbserver/DataDistribution.actor.h b/fdbserver/DataDistribution.actor.h
index a53489a4421..f44c1219662 100644
--- a/fdbserver/DataDistribution.actor.h
+++ b/fdbserver/DataDistribution.actor.h
@@ -46,7 +46,7 @@ struct IDataDistributionTeam {
 	virtual std::vector<UID> const& getServerIDs() const = 0;
 	virtual void addDataInFlightToTeam(int64_t delta) = 0;
 	virtual int64_t getDataInFlightToTeam() const = 0;
-	virtual int64_t getLongestStorageQueueSize() const = 0;
+	virtual Optional<int64_t> getLongestStorageQueueSize() const = 0;
 	virtual int64_t getLoadBytes(bool includeInFlight = true, double inflightPenalty = 1.0) const = 0;
 	virtual int64_t getMinAvailableSpace(bool includeInFlight = true) const = 0;
 	virtual double getMinAvailableSpaceRatio(bool includeInFlight = true) const = 0;
diff --git a/fdbserver/DataDistributionQueue.actor.cpp b/fdbserver/DataDistributionQueue.actor.cpp
index 00e1b152ed4..af07ebd481e 100644
--- a/fdbserver/DataDistributionQueue.actor.cpp
+++ b/fdbserver/DataDistributionQueue.actor.cpp
@@ -176,14 +176,14 @@ class ParallelTCInfo final : public ReferenceCounted<ParallelTCInfo>, public IDa
 		});
 	}
 
-	int64_t getLongestStorageQueueSize() const override {
+	Optional<int64_t> getLongestStorageQueueSize() const override {
 		int64_t maxQueueSize = 0;
 		for (const auto& team : teams) {
-			int64_t queueSize = team->getLongestStorageQueueSize();
-			if (queueSize == -1) {
-				return -1;
+			Optional<int64_t> queueSize = team->getLongestStorageQueueSize();
+			if (!queueSize.present()) {
+				return Optional<int64_t>();
 			}
-			maxQueueSize = std::max(maxQueueSize, queueSize);
+			maxQueueSize = std::max(maxQueueSize, queueSize.get());
 		}
 		return maxQueueSize;
 	}
diff --git a/fdbserver/TCInfo.actor.cpp b/fdbserver/TCInfo.actor.cpp
index b80b7836ad3..fce320e5604 100644
--- a/fdbserver/TCInfo.actor.cpp
+++ b/fdbserver/TCInfo.actor.cpp
@@ -360,13 +360,13 @@ int64_t TCTeamInfo::getDataInFlightToTeam() const {
 	return dataInFlight;
 }
 
-int64_t TCTeamInfo::getLongestStorageQueueSize() const {
+Optional<int64_t> TCTeamInfo::getLongestStorageQueueSize() const {
 	int64_t longestQueueSize = 0;
 	for (const auto& server : servers) {
 		if (server->metricsPresent()) {
 			longestQueueSize = std::max(longestQueueSize, server->getStorageQueueSize());
-		} else if (server->ssVersionTooFarBehind.get() == true) {
-			return -1;
+		} else {
+			return Optional<int64_t>();
 		}
 	}
 	return longestQueueSize;
diff --git a/fdbserver/TCInfo.h b/fdbserver/TCInfo.h
index 123717c42cf..222695e6504 100644
--- a/fdbserver/TCInfo.h
+++ b/fdbserver/TCInfo.h
@@ -194,7 +194,7 @@ class TCTeamInfo final : public ReferenceCounted<TCTeamInfo>, public IDataDistri
 
 	int64_t getDataInFlightToTeam() const override;
 
-	int64_t getLongestStorageQueueSize() const override;
+	Optional<int64_t> getLongestStorageQueueSize() const override;
 
 	int64_t getLoadBytes(bool includeInFlight = true, double inflightPenalty = 1.0) const override;
 

From 5d7baf9afd65e635fb8b9b309ab76343bb571cc1 Mon Sep 17 00:00:00 2001
From: Zhe Wang <zhe.wang@wustl.edu>
Date: Tue, 19 Sep 2023 13:14:02 -0700
Subject: [PATCH 245/468] address comments

---
 fdbserver/DataDistribution.actor.h | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/fdbserver/DataDistribution.actor.h b/fdbserver/DataDistribution.actor.h
index f44c1219662..28028a75e4c 100644
--- a/fdbserver/DataDistribution.actor.h
+++ b/fdbserver/DataDistribution.actor.h
@@ -101,8 +101,8 @@ struct GetTeamRequest {
 	std::string getDesc() const {
 		std::stringstream ss;
 
-		ss << "WantsNewServers:" << wantsNewServers << " WantsTrueBest:" << wantsTrueBest << "StorageQueueAware"
-		   << storageQueueAware << " PreferLowerUtilization:" << preferLowerUtilization
+		ss << "WantsNewServers:" << wantsNewServers << " WantsTrueBest:" << wantsTrueBest
+		   << " StorageQueueAware:" << storageQueueAware << " PreferLowerUtilization:" << preferLowerUtilization
 		   << " teamMustHaveShards:" << teamMustHaveShards << " inflightPenalty:" << inflightPenalty << ";";
 		ss << "CompleteSources:";
 		for (const auto& cs : completeSources) {

From 5f180a3b41697fc7a305ce4aecb4e2d6eb9b24e6 Mon Sep 17 00:00:00 2001
From: neethuhaneesha <nbingi@apple.com>
Date: Tue, 19 Sep 2023 09:09:37 -0700
Subject: [PATCH 246/468] Erase storageWiggleID from disk if not used

---
 fdbserver/DDTeamCollection.actor.cpp | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/fdbserver/DDTeamCollection.actor.cpp b/fdbserver/DDTeamCollection.actor.cpp
index 41ecc119799..8dcbfa4e986 100644
--- a/fdbserver/DDTeamCollection.actor.cpp
+++ b/fdbserver/DDTeamCollection.actor.cpp
@@ -2952,8 +2952,11 @@ class DDTeamCollectionImpl {
 
 	// read the current map of `perpetualStorageWiggleIDPrefix`, then restore wigglingId.
 	ACTOR static Future<Void> readStorageWiggleMap(DDTeamCollection* self) {
+		state KeyBackedObjectMap<UID, StorageWiggleValue, decltype(IncludeVersion())> metadataMap(
+		    perpetualStorageWiggleIDPrefix.withSuffix(self->primary ? "primary/"_sr : "remote/"_sr), IncludeVersion());
 		state std::vector<std::pair<UID, StorageWiggleValue>> res =
 		    wait(readStorageWiggleValues(self->cx, self->primary, false));
+
 		if (res.size() > 0) {
 			// SOMEDAY: support wiggle multiple SS at once
 			ASSERT(!self->wigglingId.present()); // only single process wiggle is allowed
@@ -2973,6 +2976,8 @@ class DDTeamCollectionImpl {
 					// Update the wigglingId only if it matches the locality.
 					if (server->getLastKnownInterface().locality.get(localityKey.get()) == localityValue) {
 						self->wigglingId = res.begin()->first;
+					} else {
+						wait(self->eraseStorageWiggleMap(&metadataMap, res.begin()->first));
 					}
 				}
 			} else {

From 3f31504f0e11fee14ee70248336229a9cc460eea Mon Sep 17 00:00:00 2001
From: FoundationDB CI <foundationdb_ci@apple.com>
Date: Thu, 21 Sep 2023 04:14:47 +0000
Subject: [PATCH 247/468] disable AVX for 7.1.40 release

---
 cmake/ConfigureCompiler.cmake | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cmake/ConfigureCompiler.cmake b/cmake/ConfigureCompiler.cmake
index e52d8665ee8..7e50469ba6a 100644
--- a/cmake/ConfigureCompiler.cmake
+++ b/cmake/ConfigureCompiler.cmake
@@ -253,7 +253,7 @@ else()
       set(USE_AVX512F OFF)
     endif()
   endif()
-  set(USE_AVX ON CACHE BOOL "Enable AVX instructions")
+  set(USE_AVX OFF CACHE BOOL "Enable AVX instructions")
   if (USE_AVX)
     if (CMAKE_HOST_SYSTEM_PROCESSOR MATCHES "^x86")
       add_compile_options(-mavx)

From da6380912f715a693de740c4c2454d96b09ddb15 Mon Sep 17 00:00:00 2001
From: FoundationDB CI <foundationdb_ci@apple.com>
Date: Thu, 21 Sep 2023 04:15:00 +0000
Subject: [PATCH 248/468] enable AVX and update version for 7.1.41 release

---
 CMakeLists.txt                | 2 +-
 cmake/ConfigureCompiler.cmake | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index ae9fe81e9ac..a966a400fe3 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -23,7 +23,7 @@ else()
 endif()
 
 project(foundationdb
-  VERSION 7.1.40
+  VERSION 7.1.41
   DESCRIPTION "FoundationDB is a scalable, fault-tolerant, ordered key-value store with full ACID transactions."
   HOMEPAGE_URL "http://www.foundationdb.org/"
   LANGUAGES C CXX ASM)
diff --git a/cmake/ConfigureCompiler.cmake b/cmake/ConfigureCompiler.cmake
index 7e50469ba6a..e52d8665ee8 100644
--- a/cmake/ConfigureCompiler.cmake
+++ b/cmake/ConfigureCompiler.cmake
@@ -253,7 +253,7 @@ else()
       set(USE_AVX512F OFF)
     endif()
   endif()
-  set(USE_AVX OFF CACHE BOOL "Enable AVX instructions")
+  set(USE_AVX ON CACHE BOOL "Enable AVX instructions")
   if (USE_AVX)
     if (CMAKE_HOST_SYSTEM_PROCESSOR MATCHES "^x86")
       add_compile_options(-mavx)

From 477abdbc27a92b05e1688374b65ae792fa3654b4 Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyu_zhou@apple.com>
Date: Thu, 21 Sep 2023 08:56:05 -0700
Subject: [PATCH 249/468] Add 7.1.40, 7.1.41 release notes (#10917)

---
 .../source/release-notes/release-notes-710.rst       | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/documentation/sphinx/source/release-notes/release-notes-710.rst b/documentation/sphinx/source/release-notes/release-notes-710.rst
index 8400241ac10..32781050fd6 100644
--- a/documentation/sphinx/source/release-notes/release-notes-710.rst
+++ b/documentation/sphinx/source/release-notes/release-notes-710.rst
@@ -4,6 +4,18 @@
 Release Notes
 #############
 
+7.1.41
+======
+* Same as 7.1.40 release with AVX enabled.
+
+7.1.40
+======
+* Released with AVX disabled.
+* Removed storageWiggleID from storage metadata if the storage server is not wiggling. `(PR #10913) <https://github.com/apple/foundationdb/pull/10913>`_
+* Augmented storage team selection to be aware of storage queue sizes. `(PR #10905) <https://github.com/apple/foundationdb/pull/10905>`_
+* Fixed the proxy setting for backup agents. `(PR #10903) <https://github.com/apple/foundationdb/pull/10903>`_
+* Added an option to set perpetual_storage_wiggle_engine to none. `(PR #10881) <https://github.com/apple/foundationdb/pull/10881>`_
+
 7.1.39
 ======
 * Same as 7.1.38 release with AVX enabled.

From c74e192b69a4a5a7a38a45e13129617457a17d70 Mon Sep 17 00:00:00 2001
From: FoundationDB CI <foundationdb_ci@apple.com>
Date: Thu, 21 Sep 2023 18:55:31 +0000
Subject: [PATCH 250/468] update version after 7.1.41 release

---
 CMakeLists.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index a966a400fe3..d2a58f73a23 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -23,7 +23,7 @@ else()
 endif()
 
 project(foundationdb
-  VERSION 7.1.41
+  VERSION 7.1.42
   DESCRIPTION "FoundationDB is a scalable, fault-tolerant, ordered key-value store with full ACID transactions."
   HOMEPAGE_URL "http://www.foundationdb.org/"
   LANGUAGES C CXX ASM)

From dd289e6ac6ba9473cb9431eccc34323e5ccd5d8c Mon Sep 17 00:00:00 2001
From: Zhe Wang <zhe.wang@wustl.edu>
Date: Mon, 18 Sep 2023 17:25:53 -0700
Subject: [PATCH 251/468] add hot shard split data move priority, constraint
 data move parallelism for this priority

---
 fdbclient/ServerKnobs.cpp                 |  2 ++
 fdbclient/ServerKnobs.h                   |  2 ++
 fdbserver/DataDistributionQueue.actor.cpp | 27 ++++++++++++++++-------
 3 files changed, 23 insertions(+), 8 deletions(-)

diff --git a/fdbclient/ServerKnobs.cpp b/fdbclient/ServerKnobs.cpp
index 7987479a92c..b93556363db 100644
--- a/fdbclient/ServerKnobs.cpp
+++ b/fdbclient/ServerKnobs.cpp
@@ -135,6 +135,7 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi
 	init( DD_QUEUE_LOGGING_INTERVAL,                             5.0 );
 	init( RELOCATION_PARALLELISM_PER_SOURCE_SERVER,                2 ); if( randomize && BUGGIFY ) RELOCATION_PARALLELISM_PER_SOURCE_SERVER = 1;
 	init( RELOCATION_PARALLELISM_PER_DEST_SERVER,                 10 ); if( randomize && BUGGIFY ) RELOCATION_PARALLELISM_PER_DEST_SERVER = 1; // Note: if this is smaller than FETCH_KEYS_PARALLELISM, this will artificially reduce performance. The current default of 10 is probably too high but is set conservatively for now.
+	init( RELOCATION_WORK_AMPLIFIER_FOR_LONG_SSQUEUE_MOVE,       5.0 );
 	init( DD_QUEUE_MAX_KEY_SERVERS,                              100 ); if( randomize && BUGGIFY ) DD_QUEUE_MAX_KEY_SERVERS = 1;
 	init( DD_REBALANCE_PARALLELISM,                               50 );
 	init( DD_REBALANCE_RESET_AMOUNT,                              30 );
@@ -162,6 +163,7 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi
 	init( PRIORITY_TEAM_1_LEFT,                                  800 );
 	init( PRIORITY_TEAM_FAILED,                                  805 );
 	init( PRIORITY_TEAM_0_LEFT,                                  809 );
+	init( PRIORITY_TEAM_STORAGE_QUEUE_TOO_LONG,                  940 );
 	init( PRIORITY_SPLIT_SHARD,                                  950 ); if( randomize && BUGGIFY ) PRIORITY_SPLIT_SHARD = 350;
 
 	// Data distribution
diff --git a/fdbclient/ServerKnobs.h b/fdbclient/ServerKnobs.h
index 1d23ee2e908..86455e122d5 100644
--- a/fdbclient/ServerKnobs.h
+++ b/fdbclient/ServerKnobs.h
@@ -128,6 +128,7 @@ class ServerKnobs : public KnobsImpl<ServerKnobs> {
 	double DD_QUEUE_LOGGING_INTERVAL;
 	double RELOCATION_PARALLELISM_PER_SOURCE_SERVER;
 	double RELOCATION_PARALLELISM_PER_DEST_SERVER;
+	double RELOCATION_WORK_AMPLIFIER_FOR_LONG_SSQUEUE_MOVE;
 	int DD_QUEUE_MAX_KEY_SERVERS;
 	int DD_REBALANCE_PARALLELISM;
 	int DD_REBALANCE_RESET_AMOUNT;
@@ -161,6 +162,7 @@ class ServerKnobs : public KnobsImpl<ServerKnobs> {
 	int PRIORITY_TEAM_1_LEFT;
 	int PRIORITY_TEAM_FAILED; // Priority when a server in the team is excluded as failed
 	int PRIORITY_TEAM_0_LEFT;
+	int PRIORITY_TEAM_STORAGE_QUEUE_TOO_LONG; // Priority when a server has a long storage queue
 	int PRIORITY_SPLIT_SHARD;
 
 	// Data distribution
diff --git a/fdbserver/DataDistributionQueue.actor.cpp b/fdbserver/DataDistributionQueue.actor.cpp
index af07ebd481e..375391c8107 100644
--- a/fdbserver/DataDistributionQueue.actor.cpp
+++ b/fdbserver/DataDistributionQueue.actor.cpp
@@ -305,18 +305,29 @@ struct Busyness {
 
 // find the "workFactor" for this, were it launched now
 int getSrcWorkFactor(RelocateData const& relocation, int singleRegionTeamSize) {
+	int res = 0;
 	if (relocation.healthPriority == SERVER_KNOBS->PRIORITY_TEAM_1_LEFT ||
 	    relocation.healthPriority == SERVER_KNOBS->PRIORITY_TEAM_0_LEFT)
-		return WORK_FULL_UTILIZATION / SERVER_KNOBS->RELOCATION_PARALLELISM_PER_SOURCE_SERVER;
+		res = WORK_FULL_UTILIZATION / SERVER_KNOBS->RELOCATION_PARALLELISM_PER_SOURCE_SERVER;
 	else if (relocation.healthPriority == SERVER_KNOBS->PRIORITY_TEAM_2_LEFT)
-		return WORK_FULL_UTILIZATION / 2 / SERVER_KNOBS->RELOCATION_PARALLELISM_PER_SOURCE_SERVER;
+		res = WORK_FULL_UTILIZATION / 2 / SERVER_KNOBS->RELOCATION_PARALLELISM_PER_SOURCE_SERVER;
 	else // for now we assume that any message at a lower priority can best be assumed to have a full team left for work
-		return WORK_FULL_UTILIZATION / singleRegionTeamSize / SERVER_KNOBS->RELOCATION_PARALLELISM_PER_SOURCE_SERVER;
+		res = WORK_FULL_UTILIZATION / singleRegionTeamSize / SERVER_KNOBS->RELOCATION_PARALLELISM_PER_SOURCE_SERVER;
+	if (relocation.priority == SERVER_KNOBS->PRIORITY_TEAM_STORAGE_QUEUE_TOO_LONG) {
+		res = std::min(static_cast<int>(res * SERVER_KNOBS->RELOCATION_WORK_AMPLIFIER_FOR_LONG_SSQUEUE_MOVE),
+		               WORK_FULL_UTILIZATION);
+	}
+	return res;
 }
 
-int getDestWorkFactor() {
+int getDestWorkFactor(int priority) {
 	// Work of moving a shard is even across destination servers
-	return WORK_FULL_UTILIZATION / SERVER_KNOBS->RELOCATION_PARALLELISM_PER_DEST_SERVER;
+	int res = WORK_FULL_UTILIZATION / SERVER_KNOBS->RELOCATION_PARALLELISM_PER_DEST_SERVER;
+	if (priority == SERVER_KNOBS->PRIORITY_TEAM_STORAGE_QUEUE_TOO_LONG) {
+		res = std::min(static_cast<int>(res * SERVER_KNOBS->RELOCATION_WORK_AMPLIFIER_FOR_LONG_SSQUEUE_MOVE),
+		               WORK_FULL_UTILIZATION);
+	}
+	return res;
 }
 
 // Data movement's resource control: Do not overload servers used for the RelocateData
@@ -366,7 +377,7 @@ bool canLaunchDest(const std::vector<std::pair<Reference<IDataDistributionTeam>,
 	if (SERVER_KNOBS->RELOCATION_PARALLELISM_PER_DEST_SERVER <= 0) {
 		return true;
 	}
-	int workFactor = getDestWorkFactor();
+	int workFactor = getDestWorkFactor(priority);
 	for (auto& team : candidateTeams) {
 		for (UID id : team.first->getServerIDs()) {
 			if (!busymapDest[id].canLaunch(priority, workFactor)) {
@@ -389,7 +400,7 @@ void launchDest(RelocateData& relocation,
                 const std::vector<std::pair<Reference<IDataDistributionTeam>, bool>>& candidateTeams,
                 std::map<UID, Busyness>& destBusymap) {
 	ASSERT(relocation.completeDests.empty());
-	int destWorkFactor = getDestWorkFactor();
+	int destWorkFactor = getDestWorkFactor(relocation.priority);
 	for (auto& team : candidateTeams) {
 		for (UID id : team.first->getServerIDs()) {
 			relocation.completeDests.push_back(id);
@@ -399,7 +410,7 @@ void launchDest(RelocateData& relocation,
 }
 
 void completeDest(const RelocateData& relocation, std::map<UID, Busyness>& destBusymap) {
-	int destWorkFactor = getDestWorkFactor();
+	int destWorkFactor = getDestWorkFactor(relocation.priority);
 	for (UID id : relocation.completeDests) {
 		destBusymap[id].removeWork(relocation.priority, destWorkFactor);
 	}

From 1ecd31ed5457c8f2b6d9a7971acbb9ecc51ca561 Mon Sep 17 00:00:00 2001
From: Zhe Wang <zhe.wang@wustl.edu>
Date: Mon, 18 Sep 2023 21:37:04 -0700
Subject: [PATCH 252/468] from managementAPI to ddtracker

---
 fdbclient/ClusterInterface.h                  |   5 +-
 fdbclient/ManagementAPI.actor.cpp             |  19 ++++
 fdbclient/ManagementAPI.actor.h               |   4 +
 fdbserver/CMakeLists.txt                      |   1 +
 fdbserver/ClusterController.actor.cpp         |  23 ++++
 fdbserver/DataDistribution.actor.cpp          |  20 +++-
 fdbserver/DataDistribution.actor.h            |   1 +
 fdbserver/DataDistributionTracker.actor.cpp   |   8 ++
 fdbserver/DataDistributorInterface.h          |   5 +-
 fdbserver/tester.actor.cpp                    |   2 +-
 .../workloads/ManualShardSplit.actor.cpp      | 100 ++++++++++++++++++
 fdbserver/workloads/workloads.actor.h         |   4 +-
 tests/CMakeLists.txt                          |   1 +
 tests/fast/ManualShardSplit.toml              |   6 ++
 14 files changed, 194 insertions(+), 5 deletions(-)
 create mode 100644 fdbserver/workloads/ManualShardSplit.actor.cpp
 create mode 100644 tests/fast/ManualShardSplit.toml

diff --git a/fdbclient/ClusterInterface.h b/fdbclient/ClusterInterface.h
index 14935f1700a..3f90faf2904 100644
--- a/fdbclient/ClusterInterface.h
+++ b/fdbclient/ClusterInterface.h
@@ -264,6 +264,7 @@ struct MoveShardRequest {
 	constexpr static FileIdentifier file_identifier = 2799592;
 
 	KeyRange shard;
+	std::vector<Key> splitPoints;
 	std::vector<NetworkAddress> addresses;
 	ReplyPromise<Void> reply;
 
@@ -271,9 +272,11 @@ struct MoveShardRequest {
 	MoveShardRequest(KeyRange shard, std::vector<NetworkAddress> addresses)
 	  : shard{ std::move(shard) }, addresses{ std::move(addresses) } {}
 
+	MoveShardRequest(KeyRange shard, std::vector<Key> splitPoints) : shard(shard), splitPoints(splitPoints) {}
+
 	template <class Ar>
 	void serialize(Ar& ar) {
-		serializer(ar, shard, addresses, reply);
+		serializer(ar, shard, addresses, reply, splitPoints);
 	}
 };
 
diff --git a/fdbclient/ManagementAPI.actor.cpp b/fdbclient/ManagementAPI.actor.cpp
index 2c36d2bc28e..6bdbd3fe216 100644
--- a/fdbclient/ManagementAPI.actor.cpp
+++ b/fdbclient/ManagementAPI.actor.cpp
@@ -2342,6 +2342,25 @@ ACTOR Future<Void> forceRecovery(Reference<IClusterConnectionRecord> clusterFile
 	}
 }
 
+ACTOR Future<Void> moveShard(Reference<IClusterConnectionRecord> clusterFile,
+                             KeyRange range,
+                             std::vector<Key> splitPoints) {
+	state Reference<AsyncVar<Optional<ClusterInterface>>> clusterInterface(new AsyncVar<Optional<ClusterInterface>>);
+	state Future<Void> leaderMon = monitorLeader<ClusterInterface>(clusterFile, clusterInterface);
+
+	loop {
+		choose {
+			when(wait(clusterInterface->get().present()
+			              ? brokenPromiseToNever(
+			                    clusterInterface->get().get().moveShard.getReply(MoveShardRequest(range, splitPoints)))
+			              : Never())) {
+				return Void();
+			}
+			when(wait(clusterInterface->onChange())) {}
+		}
+	}
+}
+
 ACTOR Future<Void> waitForPrimaryDC(Database cx, StringRef dcId) {
 	state ReadYourWritesTransaction tr(cx);
 
diff --git a/fdbclient/ManagementAPI.actor.h b/fdbclient/ManagementAPI.actor.h
index f9d204bdb39..cffda4caded 100644
--- a/fdbclient/ManagementAPI.actor.h
+++ b/fdbclient/ManagementAPI.actor.h
@@ -148,6 +148,10 @@ ACTOR Future<int> setDDMode(Database cx, int mode);
 
 ACTOR Future<Void> forceRecovery(Reference<IClusterConnectionRecord> clusterFile, Standalone<StringRef> dcId);
 
+ACTOR Future<Void> moveShard(Reference<IClusterConnectionRecord> clusterFile,
+                             KeyRange range,
+                             std::vector<Key> splitPoints);
+
 ACTOR Future<Void> printHealthyZone(Database cx);
 ACTOR Future<Void> setDDIgnoreRebalanceSwitch(Database cx, bool ignoreRebalance);
 ACTOR Future<bool> clearHealthyZone(Database cx, bool printWarning = false, bool clearSSFailureZoneString = false);
diff --git a/fdbserver/CMakeLists.txt b/fdbserver/CMakeLists.txt
index 679c950c9a0..d01ed15ae46 100644
--- a/fdbserver/CMakeLists.txt
+++ b/fdbserver/CMakeLists.txt
@@ -241,6 +241,7 @@ set(FDBSERVER_SRCS
   workloads/LogMetrics.actor.cpp
   workloads/LowLatency.actor.cpp
   workloads/MachineAttrition.actor.cpp
+  workloads/ManualShardSplit.actor.cpp
   workloads/Mako.actor.cpp
   workloads/MemoryKeyValueStore.cpp
   workloads/MemoryKeyValueStore.h
diff --git a/fdbserver/ClusterController.actor.cpp b/fdbserver/ClusterController.actor.cpp
index 6dedda5ce8b..43c8caa9838 100644
--- a/fdbserver/ClusterController.actor.cpp
+++ b/fdbserver/ClusterController.actor.cpp
@@ -2034,6 +2034,28 @@ ACTOR Future<Void> handleForcedRecoveries(ClusterControllerData* self, ClusterCo
 	}
 }
 
+ACTOR Future<Void> triggerMoveShards(ClusterControllerData* self, MoveShardRequest req) {
+	try {
+		while (self->db.serverInfo->get().recoveryState < RecoveryState::ACCEPTING_COMMITS ||
+		       !self->db.serverInfo->get().distributor.present()) {
+			wait(self->db.serverInfo->onChange());
+		}
+		DistributorSplitRangeRequest fReq(req.shard, req.splitPoints);
+		SplitShardReply rep = wait(self->db.serverInfo->get().distributor.get().distributorSplitRange.getReply(fReq));
+		req.reply.send(Void());
+	} catch (Error& e) {
+		req.reply.sendError(e);
+	}
+	return Void();
+}
+
+ACTOR Future<Void> handleMoveShards(ClusterControllerData* self, ClusterControllerFullInterface interf) {
+	loop {
+		state MoveShardRequest req = waitNext(interf.clientInterface.moveShard.getFuture());
+		self->addActor.send(triggerMoveShards(self, req));
+	}
+}
+
 struct SingletonRecruitThrottler {
 	double lastRecruitStart;
 
@@ -2665,6 +2687,7 @@ ACTOR Future<Void> clusterControllerCore(ClusterControllerFullInterface interf,
 	self.addActor.send(updatedChangedDatacenters(&self));
 	self.addActor.send(updateDatacenterVersionDifference(&self));
 	self.addActor.send(handleForcedRecoveries(&self, interf));
+	self.addActor.send(handleMoveShards(&self, interf));
 	self.addActor.send(monitorDataDistributor(&self));
 	self.addActor.send(monitorRatekeeper(&self));
 	self.addActor.send(monitorBlobManager(&self));
diff --git a/fdbserver/DataDistribution.actor.cpp b/fdbserver/DataDistribution.actor.cpp
index b9012293554..0ea2ea133e2 100644
--- a/fdbserver/DataDistribution.actor.cpp
+++ b/fdbserver/DataDistribution.actor.cpp
@@ -516,6 +516,7 @@ ACTOR Future<Void> monitorBatchLimitedTime(Reference<AsyncVar<ServerDBInfo> cons
 // Runs the data distribution algorithm for FDB, including the DD Queue, DD tracker, and DD team collection
 ACTOR Future<Void> dataDistribution(Reference<DataDistributorData> self,
                                     PromiseStream<GetMetricsListRequest> getShardMetricsList,
+                                    PromiseStream<DistributorSplitRangeRequest> manualShardSplit,
                                     const DDEnabledState* ddEnabledState) {
 	state double lastLimited = 0;
 	self->addActor.send(monitorBatchLimitedTime(self->dbInfo, &lastLimited));
@@ -734,6 +735,7 @@ ACTOR Future<Void> dataDistribution(Reference<DataDistributorData> self,
 			                                                            shardsAffectedByTeamFailure,
 			                                                            getShardMetrics,
 			                                                            getShardMetricsList,
+			                                                            manualShardSplit,
 			                                                            getAverageShardBytes.getFuture(),
 			                                                            readyToStart,
 			                                                            anyZeroHealthyTeams,
@@ -1254,10 +1256,23 @@ ACTOR Future<Void> ddGetMetrics(GetDataDistributorMetricsRequest req,
 	return Void();
 }
 
+ACTOR Future<Void> ddSplitRange(DistributorSplitRangeRequest req,
+                                PromiseStream<DistributorSplitRangeRequest> manualShardSplit) {
+	ErrorOr<SplitShardReply> result = wait(errorOr(
+	    brokenPromiseToNever(manualShardSplit.getReply(DistributorSplitRangeRequest(req.range, req.splitPoints)))));
+	if (result.isError()) {
+		req.reply.sendError(result.getError());
+	} else {
+		req.reply.send(result.get());
+	}
+	return Void();
+}
+
 ACTOR Future<Void> dataDistributor(DataDistributorInterface di, Reference<AsyncVar<ServerDBInfo> const> db) {
 	state Reference<DataDistributorData> self(new DataDistributorData(db, di.id()));
 	state Future<Void> collection = actorCollection(self->addActor.getFuture());
 	state PromiseStream<GetMetricsListRequest> getShardMetricsList;
+	state PromiseStream<DistributorSplitRangeRequest> manualShardSplit;
 	state Database cx = openDBOnServer(db, TaskPriority::DefaultDelay, LockAware::True);
 	state ActorCollection actors(false);
 	state DDEnabledState ddEnabledState;
@@ -1269,7 +1284,7 @@ ACTOR Future<Void> dataDistributor(DataDistributorInterface di, Reference<AsyncV
 		self->addActor.send(waitFailureServer(di.waitFailure.getFuture()));
 		self->addActor.send(cacheServerWatcher(&cx));
 		state Future<Void> distributor =
-		    reportErrorsExcept(dataDistribution(self, getShardMetricsList, &ddEnabledState),
+		    reportErrorsExcept(dataDistribution(self, getShardMetricsList, manualShardSplit, &ddEnabledState),
 		                       "DataDistribution",
 		                       di.id(),
 		                       &normalDataDistributorErrors());
@@ -1294,6 +1309,9 @@ ACTOR Future<Void> dataDistributor(DataDistributorInterface di, Reference<AsyncV
 			         waitNext(di.distributorExclCheckReq.getFuture())) {
 				actors.add(ddExclusionSafetyCheck(exclCheckReq, self, cx));
 			}
+			when(DistributorSplitRangeRequest splitRangeReq = waitNext(di.distributorSplitRange.getFuture())) {
+				actors.add(ddSplitRange(splitRangeReq, manualShardSplit));
+			}
 		}
 	} catch (Error& err) {
 		if (normalDataDistributorErrors().count(err.code()) == 0) {
diff --git a/fdbserver/DataDistribution.actor.h b/fdbserver/DataDistribution.actor.h
index 28028a75e4c..25419c8425a 100644
--- a/fdbserver/DataDistribution.actor.h
+++ b/fdbserver/DataDistribution.actor.h
@@ -253,6 +253,7 @@ ACTOR Future<Void> dataDistributionTracker(Reference<InitialDataDistribution> in
                                            Reference<ShardsAffectedByTeamFailure> shardsAffectedByTeamFailure,
                                            PromiseStream<GetMetricsRequest> getShardMetrics,
                                            PromiseStream<GetMetricsListRequest> getShardMetricsList,
+                                           PromiseStream<DistributorSplitRangeRequest> manualShardSplit,
                                            FutureStream<Promise<int64_t>> getAverageShardBytes,
                                            Promise<Void> readyToStart,
                                            Reference<AsyncVar<bool>> zeroHealthyTeams,
diff --git a/fdbserver/DataDistributionTracker.actor.cpp b/fdbserver/DataDistributionTracker.actor.cpp
index ebdace2df90..b1b43459f6d 100644
--- a/fdbserver/DataDistributionTracker.actor.cpp
+++ b/fdbserver/DataDistributionTracker.actor.cpp
@@ -927,6 +927,7 @@ ACTOR Future<Void> dataDistributionTracker(Reference<InitialDataDistribution> in
                                            Reference<ShardsAffectedByTeamFailure> shardsAffectedByTeamFailure,
                                            PromiseStream<GetMetricsRequest> getShardMetrics,
                                            PromiseStream<GetMetricsListRequest> getShardMetricsList,
+                                           PromiseStream<DistributorSplitRangeRequest> manualShardSplit,
                                            FutureStream<Promise<int64_t>> getAverageShardBytes,
                                            Promise<Void> readyToStart,
                                            Reference<AsyncVar<bool>> anyZeroHealthyTeams,
@@ -967,6 +968,13 @@ ACTOR Future<Void> dataDistributionTracker(Reference<InitialDataDistribution> in
 			when(GetMetricsListRequest req = waitNext(getShardMetricsList.getFuture())) {
 				self.sizeChanges.add(fetchShardMetricsList(&self, req));
 			}
+			when(DistributorSplitRangeRequest req = waitNext(manualShardSplit.getFuture())) {
+				TraceEvent(SevInfo, "DDSplitRange", self.distributorId)
+				    .detail("Range", req.range)
+				    .detail("SplitPoints", describe(req.splitPoints));
+				req.reply.send(SplitShardReply());
+				// TODO
+			}
 			when(wait(self.sizeChanges.getResult())) {}
 		}
 	} catch (Error& e) {
diff --git a/fdbserver/DataDistributorInterface.h b/fdbserver/DataDistributorInterface.h
index b858278f988..9e047e6f1bc 100644
--- a/fdbserver/DataDistributorInterface.h
+++ b/fdbserver/DataDistributorInterface.h
@@ -154,14 +154,17 @@ struct DistributorExclusionSafetyCheckRequest {
 struct DistributorSplitRangeRequest {
 	constexpr static FileIdentifier file_identifier = 1384441;
 	std::vector<Key> splitPoints;
+	KeyRange range;
 	ReplyPromise<SplitShardReply> reply;
 
 	DistributorSplitRangeRequest() {}
 	explicit DistributorSplitRangeRequest(std::vector<Key> splitPoints) : splitPoints{ std::move(splitPoints) } {}
+	explicit DistributorSplitRangeRequest(KeyRange range, std::vector<Key> splitPoints)
+	  : splitPoints(splitPoints), range(range) {}
 
 	template <class Ar>
 	void serialize(Ar& ar) {
-		serializer(ar, splitPoints, reply);
+		serializer(ar, splitPoints, reply, range);
 	}
 };
 
diff --git a/fdbserver/tester.actor.cpp b/fdbserver/tester.actor.cpp
index 56128e54324..b1dad749727 100644
--- a/fdbserver/tester.actor.cpp
+++ b/fdbserver/tester.actor.cpp
@@ -352,7 +352,7 @@ Reference<TestWorkload> getWorkloadIface(WorkloadRequest work,
 	wcx.dbInfo = dbInfo;
 	wcx.options = options;
 	wcx.sharedRandomNumber = work.sharedRandomNumber;
-
+	std::cout << "Create: " << testName.toString() << "\n";
 	auto workload = IWorkloadFactory::create(testName.toString(), wcx);
 
 	auto unconsumedOptions = checkAllOptionsConsumed(workload ? workload->options : VectorRef<KeyValueRef>());
diff --git a/fdbserver/workloads/ManualShardSplit.actor.cpp b/fdbserver/workloads/ManualShardSplit.actor.cpp
new file mode 100644
index 00000000000..c523793195b
--- /dev/null
+++ b/fdbserver/workloads/ManualShardSplit.actor.cpp
@@ -0,0 +1,100 @@
+/*
+ * ManualShardSplit.actor.cpp
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2022 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "fdbclient/ManagementAPI.actor.h"
+#include "fdbclient/NativeAPI.actor.h"
+#include "fdbserver/Knobs.h"
+#include "fdbserver/QuietDatabase.h"
+#include "fdbrpc/simulator.h"
+#include "fdbserver/workloads/workloads.actor.h"
+#include "flow/Error.h"
+#include "flow/IRandom.h"
+#include "flow/flow.h"
+#include <cstdint>
+#include <limits>
+#include "flow/actorcompiler.h" // This must be the last #include.
+
+struct ManualShardSplitWorkload : TestWorkload {
+	static constexpr auto NAME = "ManualShardSplitWorkload";
+	const bool enabled;
+
+	ManualShardSplitWorkload(WorkloadContext const& wcx) : TestWorkload(wcx), enabled(!clientId) {}
+
+	std::string description() const override { return "ManualShardSplit"; }
+
+	Future<Void> setup(Database const& cx) override { return Void(); }
+
+	Future<Void> start(Database const& cx) override {
+		if (!enabled) {
+			return Void();
+		}
+		return _start(this, cx);
+	}
+
+	ACTOR Future<Void> _start(ManualShardSplitWorkload* self, Database cx) {
+		TraceEvent("ManualShardSplitTestBegin");
+		state std::map<Key, Value> kvs({ { "TestKeyA"_sr, "TestValueA"_sr },
+		                                 { "TestKeyB"_sr, "TestValueB"_sr },
+		                                 { "TestKeyC"_sr, "TestValueC"_sr },
+		                                 { "TestKeyD"_sr, "TestValueD"_sr },
+		                                 { "TestKeyE"_sr, "TestValueE"_sr },
+		                                 { "TestKeyF"_sr, "TestValueF"_sr } });
+
+		Version ver = wait(self->populateData(self, cx, &kvs));
+		KeyRange range = KeyRangeRef("TestKeyA"_sr, "TestKeyD"_sr);
+		std::vector<Key> splitPoints = { "TestKeyB"_sr };
+		wait(moveShard(cx->getConnectionRecord(), range, splitPoints));
+		return Void();
+	}
+
+	ACTOR Future<Version> populateData(ManualShardSplitWorkload* self, Database cx, std::map<Key, Value>* kvs) {
+		state Reference<ReadYourWritesTransaction> tr = makeReference<ReadYourWritesTransaction>(cx);
+		state Version version;
+		state UID debugID;
+
+		loop {
+			debugID = deterministicRandom()->randomUniqueID();
+			try {
+				tr->debugTransaction(debugID);
+				for (const auto& [key, value] : *kvs) {
+					tr->set(key, value);
+				}
+				wait(tr->commit());
+				version = tr->getCommittedVersion();
+				break;
+			} catch (Error& e) {
+				TraceEvent("TestCommitError").errorUnsuppressed(e);
+				wait(tr->onError(e));
+			}
+		}
+
+		TraceEvent("PopulateTestDataDone")
+		    .detail("CommitVersion", tr->getCommittedVersion())
+		    .detail("DebugID", debugID);
+
+		return version;
+	}
+
+	Future<bool> check(Database const& cx) override { return true; }
+
+	void getMetrics(std::vector<PerfMetric>& m) override {}
+};
+
+WorkloadFactory<ManualShardSplitWorkload> ManualShardSplitWorkloadFactory("ManualShardSplit");
\ No newline at end of file
diff --git a/fdbserver/workloads/workloads.actor.h b/fdbserver/workloads/workloads.actor.h
index c7c0557b682..3c612456b9e 100644
--- a/fdbserver/workloads/workloads.actor.h
+++ b/fdbserver/workloads/workloads.actor.h
@@ -107,8 +107,10 @@ struct KVWorkload : TestWorkload {
 struct IWorkloadFactory : ReferenceCounted<IWorkloadFactory> {
 	static Reference<TestWorkload> create(std::string const& name, WorkloadContext const& wcx) {
 		auto it = factories().find(name);
-		if (it == factories().end())
+		if (it == factories().end()) {
+			std::cout << "factory does not find the name\n";
 			return {}; // or throw?
+		}
 		return it->second->create(wcx);
 	}
 	static std::map<std::string, Reference<IWorkloadFactory>>& factories() {
diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt
index 960c9dcbec8..e476fe49c3f 100644
--- a/tests/CMakeLists.txt
+++ b/tests/CMakeLists.txt
@@ -155,6 +155,7 @@ if(WITH_PYTHON)
   add_fdb_test(TEST_FILES fast/LowLatency.toml)
   # TODO: Fix failures and reenable this test:
   add_fdb_test(TEST_FILES fast/LowLatencySingleClog.toml IGNORE)
+  add_fdb_test(TEST_FILES fast/ManualShardSplit.toml)
   add_fdb_test(TEST_FILES fast/MemoryLifetime.toml)
   add_fdb_test(TEST_FILES fast/MoveKeysCycle.toml)
   add_fdb_test(TEST_FILES fast/MutationLogReaderCorrectness.toml)
diff --git a/tests/fast/ManualShardSplit.toml b/tests/fast/ManualShardSplit.toml
new file mode 100644
index 00000000000..d3ea5d9d039
--- /dev/null
+++ b/tests/fast/ManualShardSplit.toml
@@ -0,0 +1,6 @@
+[[test]]
+testTitle = 'ManualShardSplit'
+useDB = true
+
+    [[test.workload]]
+    testName = 'ManualShardSplit'
\ No newline at end of file

From fbf646e278da24af1890e863da2369ea856bb39a Mon Sep 17 00:00:00 2001
From: Zhe Wang <zhe.wang@wustl.edu>
Date: Tue, 19 Sep 2023 14:47:49 -0700
Subject: [PATCH 253/468] from DDTracker to ShardSplitter

---
 fdbserver/DataDistribution.actor.h            |   2 +
 fdbserver/DataDistributionTracker.actor.cpp   | 101 ++++++++++++++----
 .../workloads/ManualShardSplit.actor.cpp      |   6 +-
 3 files changed, 88 insertions(+), 21 deletions(-)

diff --git a/fdbserver/DataDistribution.actor.h b/fdbserver/DataDistribution.actor.h
index 25419c8425a..e590dfd3d86 100644
--- a/fdbserver/DataDistribution.actor.h
+++ b/fdbserver/DataDistribution.actor.h
@@ -245,6 +245,8 @@ struct ShardTrackedData {
 	Future<Void> trackShard;
 	Future<Void> trackBytes;
 	Reference<AsyncVar<Optional<ShardMetrics>>> stats;
+	Reference<AsyncVar<bool>> shouldManualSplit;
+	std::shared_ptr<std::vector<Key>> manualSplitPoints;
 };
 
 ACTOR Future<Void> dataDistributionTracker(Reference<InitialDataDistribution> initData,
diff --git a/fdbserver/DataDistributionTracker.actor.cpp b/fdbserver/DataDistributionTracker.actor.cpp
index b1b43459f6d..11dc10b61b3 100644
--- a/fdbserver/DataDistributionTracker.actor.cpp
+++ b/fdbserver/DataDistributionTracker.actor.cpp
@@ -477,21 +477,42 @@ struct HasBeenTrueFor : ReferenceCounted<HasBeenTrueFor> {
 ACTOR Future<Void> shardSplitter(DataDistributionTracker* self,
                                  KeyRange keys,
                                  Reference<AsyncVar<Optional<ShardMetrics>>> shardSize,
-                                 ShardSizeBounds shardBounds) {
+                                 ShardSizeBounds shardBounds,
+                                 std::vector<Key> manualSplitKeys) {
 	state StorageMetrics metrics = shardSize->get().get().metrics;
 	state BandwidthStatus bandwidthStatus = getBandwidthStatus(metrics);
+	state bool doManualSplit = !manualSplitKeys.empty();
 
 	// Split
 	TEST(true); // shard to be split
 
-	StorageMetrics splitMetrics;
-	splitMetrics.bytes = shardBounds.max.bytes / 2;
-	splitMetrics.bytesPerKSecond =
-	    keys.begin >= keyServersKeys.begin ? splitMetrics.infinity : SERVER_KNOBS->SHARD_SPLIT_BYTES_PER_KSEC;
-	splitMetrics.iosPerKSecond = splitMetrics.infinity;
-	splitMetrics.bytesReadPerKSecond = splitMetrics.infinity; // Don't split by readBandwidth
-
-	state Standalone<VectorRef<KeyRef>> splitKeys = wait(getSplitKeys(self, keys, splitMetrics, metrics));
+	state Standalone<VectorRef<KeyRef>> splitKeys;
+	if (doManualSplit) {
+		splitKeys.push_back_deep(splitKeys.arena(), keys.begin);
+		for (const auto& manualSplitKey : manualSplitKeys) {
+			splitKeys.push_back_deep(splitKeys.arena(), manualSplitKey);
+		}
+		splitKeys.push_back_deep(splitKeys.arena(), keys.end);
+		TraceEvent e(SevInfo, "ManualShardSplit", self->distributorId);
+		e.setMaxEventLength(20000);
+		e.detail("Begin", keys.begin);
+		e.detail("End", keys.end);
+		int i = 0;
+		for (const auto& splitKey : splitKeys) {
+			e.detail("SplitPoint" + std::to_string(i), printable(splitKey).c_str());
+			i++;
+		}
+	} else {
+		StorageMetrics splitMetrics;
+		splitMetrics.bytes = shardBounds.max.bytes / 2;
+		splitMetrics.bytesPerKSecond =
+		    keys.begin >= keyServersKeys.begin ? splitMetrics.infinity : SERVER_KNOBS->SHARD_SPLIT_BYTES_PER_KSEC;
+		splitMetrics.iosPerKSecond = splitMetrics.infinity;
+		splitMetrics.bytesReadPerKSecond = splitMetrics.infinity; // Don't split by readBandwidth
+
+		Standalone<VectorRef<KeyRef>> splitKeys_ = wait(getSplitKeys(self, keys, splitMetrics, metrics));
+		splitKeys = splitKeys_;
+	}
 	// fprintf(stderr, "split keys:\n");
 	// for( int i = 0; i < splitKeys.size(); i++ ) {
 	//	fprintf(stderr, "   %s\n", printable(splitKeys[i]).c_str());
@@ -513,6 +534,8 @@ ACTOR Future<Void> shardSplitter(DataDistributionTracker* self,
 	}
 
 	if (numShards > 1) {
+		int priority =
+		    doManualSplit ? SERVER_KNOBS->PRIORITY_TEAM_STORAGE_QUEUE_TOO_LONG : SERVER_KNOBS->PRIORITY_SPLIT_SHARD;
 		int skipRange = deterministicRandom()->randomInt(0, numShards);
 		// The queue can't deal with RelocateShard requests which split an existing shard into three pieces, so
 		// we have to send the unskipped ranges in this order (nibbling in from the edges of the old range)
@@ -525,12 +548,18 @@ ACTOR Future<Void> shardSplitter(DataDistributionTracker* self,
 		for (int i = 0; i < skipRange; i++) {
 			KeyRangeRef r(splitKeys[i], splitKeys[i + 1]);
 			self->shardsAffectedByTeamFailure->defineShard(r);
-			self->output.send(RelocateShard(r, SERVER_KNOBS->PRIORITY_SPLIT_SHARD));
+			self->output.send(RelocateShard(r, priority));
 		}
 		for (int i = numShards - 1; i > skipRange; i--) {
 			KeyRangeRef r(splitKeys[i], splitKeys[i + 1]);
 			self->shardsAffectedByTeamFailure->defineShard(r);
-			self->output.send(RelocateShard(r, SERVER_KNOBS->PRIORITY_SPLIT_SHARD));
+			self->output.send(RelocateShard(r, priority));
+		}
+		if (doManualSplit) {
+			// If manual shard split, we move all shards out of the team
+			KeyRangeRef r(splitKeys[skipRange], splitKeys[skipRange + 1]);
+			self->shardsAffectedByTeamFailure->defineShard(r);
+			self->output.send(RelocateShard(r, priority));
 		}
 
 		self->sizeChanges.add(changeSizes(self, keys, shardSize->get().get().metrics.bytes));
@@ -685,8 +714,11 @@ Future<Void> shardMerger(DataDistributionTracker* self,
 ACTOR Future<Void> shardEvaluator(DataDistributionTracker* self,
                                   KeyRange keys,
                                   Reference<AsyncVar<Optional<ShardMetrics>>> shardSize,
-                                  Reference<HasBeenTrueFor> wantsToMerge) {
-	Future<Void> onChange = shardSize->onChange() || yieldedFuture(self->maxShardSize->onChange());
+                                  Reference<HasBeenTrueFor> wantsToMerge,
+                                  Reference<AsyncVar<bool>> shouldManualSplit,
+                                  std::shared_ptr<std::vector<Key>> manualSplitPoints) {
+	Future<Void> onChange =
+	    shardSize->onChange() || yieldedFuture(self->maxShardSize->onChange()) || shouldManualSplit->onChange();
 
 	// There are the bounds inside of which we are happy with the shard size.
 	// getShardSizeBounds() will allways have shardBounds.min.bytes == 0 for shards that start at allKeys.begin,
@@ -699,6 +731,15 @@ ACTOR Future<Void> shardEvaluator(DataDistributionTracker* self,
 	                   (bandwidthStatus == BandwidthStatusHigh && keys.begin < keyServersKeys.begin);
 	bool shouldMerge = stats.bytes < shardBounds.min.bytes && bandwidthStatus == BandwidthStatusLow;
 
+	std::vector<Key> manualSplitKeys;
+	if (shouldManualSplit->get() == true) {
+		// Force to split
+		shouldSplit = true;
+		shouldMerge = false;
+		manualSplitKeys.insert(manualSplitKeys.end(), manualSplitPoints->begin(), manualSplitPoints->end());
+		manualSplitPoints->clear();
+	}
+
 	// Every invocation must set this or clear it
 	if (shouldMerge && !self->anyZeroHealthyTeams->get()) {
 		auto whenLongEnough = wantsToMerge->set(shardSize->get().get().lastLowBandwidthStartTime);
@@ -730,7 +771,7 @@ ACTOR Future<Void> shardEvaluator(DataDistributionTracker* self,
 		onChange = onChange || shardMerger(self, keys, shardSize);
 	}
 	if (shouldSplit) {
-		onChange = onChange || shardSplitter(self, keys, shardSize, shardBounds);
+		onChange = onChange || shardSplitter(self, keys, shardSize, shardBounds, manualSplitKeys);
 	}
 
 	wait(onChange);
@@ -739,7 +780,9 @@ ACTOR Future<Void> shardEvaluator(DataDistributionTracker* self,
 
 ACTOR Future<Void> shardTracker(DataDistributionTracker::SafeAccessor self,
                                 KeyRange keys,
-                                Reference<AsyncVar<Optional<ShardMetrics>>> shardSize) {
+                                Reference<AsyncVar<Optional<ShardMetrics>>> shardSize,
+                                Reference<AsyncVar<bool>> shouldManualSplit,
+                                std::shared_ptr<std::vector<Key>> manualSplitPoints) {
 	wait(yieldedFuture(self()->readyToStart.getFuture()));
 
 	if (!shardSize->get().present())
@@ -765,7 +808,7 @@ ACTOR Future<Void> shardTracker(DataDistributionTracker::SafeAccessor self,
 	try {
 		loop {
 			// Use the current known size to check for (and start) splits and merges.
-			wait(shardEvaluator(self(), keys, shardSize, wantsToMerge));
+			wait(shardEvaluator(self(), keys, shardSize, wantsToMerge, shouldManualSplit, manualSplitPoints));
 
 			// We could have a lot of actors being released from the previous wait at the same time. Immediately calling
 			// delay(0) mitigates the resulting SlowTask
@@ -790,6 +833,8 @@ void restartShardTrackers(DataDistributionTracker* self, KeyRangeRef keys, Optio
 		}
 
 		auto shardMetrics = makeReference<AsyncVar<Optional<ShardMetrics>>>();
+		auto shouldManualSplit = makeReference<AsyncVar<bool>>(false);
+		auto manualSplitPoints = std::make_shared<std::vector<Key>>();
 
 		// For the case where the new tracker will take over at the boundaries of current shard(s)
 		//  we can use the old size if it is available. This will be the case when merging shards.
@@ -805,7 +850,10 @@ void restartShardTrackers(DataDistributionTracker* self, KeyRangeRef keys, Optio
 
 		ShardTrackedData data;
 		data.stats = shardMetrics;
-		data.trackShard = shardTracker(DataDistributionTracker::SafeAccessor(self), ranges[i], shardMetrics);
+		data.shouldManualSplit = shouldManualSplit;
+		data.manualSplitPoints = manualSplitPoints;
+		data.trackShard = shardTracker(
+		    DataDistributionTracker::SafeAccessor(self), ranges[i], shardMetrics, shouldManualSplit, manualSplitPoints);
 		data.trackBytes = trackShardMetrics(DataDistributionTracker::SafeAccessor(self), ranges[i], shardMetrics);
 		self->shards.insert(ranges[i], data);
 	}
@@ -972,8 +1020,25 @@ ACTOR Future<Void> dataDistributionTracker(Reference<InitialDataDistribution> in
 				TraceEvent(SevInfo, "DDSplitRange", self.distributorId)
 				    .detail("Range", req.range)
 				    .detail("SplitPoints", describe(req.splitPoints));
+				for (auto it : self.shards.intersectingRanges(req.range)) {
+					bool triggerManualSplitShard = false;
+					if (it->range().begin < req.range.begin && req.range.begin < it->range().end) {
+						triggerManualSplitShard = true;
+						it->value().manualSplitPoints->push_back(req.range.begin);
+					}
+					if (it->range().begin < req.range.end && req.range.end < it->range().end) {
+						triggerManualSplitShard = true;
+						it->value().manualSplitPoints->push_back(req.range.end);
+					}
+					if (triggerManualSplitShard) {
+						// Trigger split for this shard
+						it->value().shouldManualSplit->set(true);
+					} else {
+						// Trigger data move for this shard
+						
+					}
+				}
 				req.reply.send(SplitShardReply());
-				// TODO
 			}
 			when(wait(self.sizeChanges.getResult())) {}
 		}
diff --git a/fdbserver/workloads/ManualShardSplit.actor.cpp b/fdbserver/workloads/ManualShardSplit.actor.cpp
index c523793195b..635ae745fb6 100644
--- a/fdbserver/workloads/ManualShardSplit.actor.cpp
+++ b/fdbserver/workloads/ManualShardSplit.actor.cpp
@@ -58,9 +58,9 @@ struct ManualShardSplitWorkload : TestWorkload {
 		                                 { "TestKeyF"_sr, "TestValueF"_sr } });
 
 		Version ver = wait(self->populateData(self, cx, &kvs));
-		KeyRange range = KeyRangeRef("TestKeyA"_sr, "TestKeyD"_sr);
-		std::vector<Key> splitPoints = { "TestKeyB"_sr };
-		wait(moveShard(cx->getConnectionRecord(), range, splitPoints));
+		state std::vector<Key> splitPoints = {};
+		wait(moveShard(cx->getConnectionRecord(), KeyRangeRef("TestKeyA"_sr, "TestKeyD"_sr), splitPoints));
+		wait(moveShard(cx->getConnectionRecord(), KeyRangeRef("TestKeyB"_sr, "TestKeyC"_sr), splitPoints));
 		return Void();
 	}
 

From 3b3aee2c3d8329f5fab0d6fc2c217a29085915be Mon Sep 17 00:00:00 2001
From: Zhe Wang <zhe.wang@wustl.edu>
Date: Tue, 19 Sep 2023 15:06:25 -0700
Subject: [PATCH 254/468] add shard move

---
 fdbserver/DataDistributionQueue.actor.cpp   | 3 +++
 fdbserver/DataDistributionTracker.actor.cpp | 3 ++-
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/fdbserver/DataDistributionQueue.actor.cpp b/fdbserver/DataDistributionQueue.actor.cpp
index 375391c8107..033e3b73d19 100644
--- a/fdbserver/DataDistributionQueue.actor.cpp
+++ b/fdbserver/DataDistributionQueue.actor.cpp
@@ -1796,6 +1796,9 @@ ACTOR Future<Void> dataDistributionQueue(Database cx,
 
 			choose {
 				when(RelocateShard rs = waitNext(self.input)) {
+					if (rs.priority == SERVER_KNOBS->PRIORITY_TEAM_STORAGE_QUEUE_TOO_LONG) {
+						TraceEvent(SevInfo, "ShardRelocatorByManualSplit", self.distributorId).detail("Range", rs.keys);
+					}
 					bool wasEmpty = serversToLaunchFrom.empty();
 					self.queueRelocation(rs, serversToLaunchFrom);
 					if (wasEmpty && !serversToLaunchFrom.empty())
diff --git a/fdbserver/DataDistributionTracker.actor.cpp b/fdbserver/DataDistributionTracker.actor.cpp
index 11dc10b61b3..8521f9c6474 100644
--- a/fdbserver/DataDistributionTracker.actor.cpp
+++ b/fdbserver/DataDistributionTracker.actor.cpp
@@ -1035,7 +1035,8 @@ ACTOR Future<Void> dataDistributionTracker(Reference<InitialDataDistribution> in
 						it->value().shouldManualSplit->set(true);
 					} else {
 						// Trigger data move for this shard
-						
+						self.output.send(
+						    RelocateShard(it->range(), SERVER_KNOBS->PRIORITY_TEAM_STORAGE_QUEUE_TOO_LONG));
 					}
 				}
 				req.reply.send(SplitShardReply());

From 81aab1c061ee189e7cd3371dac3fa3ea96e19fd7 Mon Sep 17 00:00:00 2001
From: Zhe Wang <zhe.wang@wustl.edu>
Date: Tue, 19 Sep 2023 15:22:21 -0700
Subject: [PATCH 255/468] increase manual split priority

---
 fdbclient/ServerKnobs.cpp                 | 2 +-
 fdbserver/DataDistributionQueue.actor.cpp | 4 ++++
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/fdbclient/ServerKnobs.cpp b/fdbclient/ServerKnobs.cpp
index b93556363db..ec75f19bf3c 100644
--- a/fdbclient/ServerKnobs.cpp
+++ b/fdbclient/ServerKnobs.cpp
@@ -163,8 +163,8 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi
 	init( PRIORITY_TEAM_1_LEFT,                                  800 );
 	init( PRIORITY_TEAM_FAILED,                                  805 );
 	init( PRIORITY_TEAM_0_LEFT,                                  809 );
-	init( PRIORITY_TEAM_STORAGE_QUEUE_TOO_LONG,                  940 );
 	init( PRIORITY_SPLIT_SHARD,                                  950 ); if( randomize && BUGGIFY ) PRIORITY_SPLIT_SHARD = 350;
+	init( PRIORITY_TEAM_STORAGE_QUEUE_TOO_LONG,                  960 );
 
 	// Data distribution
 	init( RETRY_RELOCATESHARD_DELAY,                             0.1 );
diff --git a/fdbserver/DataDistributionQueue.actor.cpp b/fdbserver/DataDistributionQueue.actor.cpp
index 033e3b73d19..ad1794c3851 100644
--- a/fdbserver/DataDistributionQueue.actor.cpp
+++ b/fdbserver/DataDistributionQueue.actor.cpp
@@ -1099,6 +1099,10 @@ ACTOR Future<Void> dataDistributionRelocator(DDQueueData* self, RelocateData rd,
 	state double startTime = now();
 	state std::vector<UID> destIds;
 
+	if (rd.priority == SERVER_KNOBS->PRIORITY_TEAM_STORAGE_QUEUE_TOO_LONG) {
+		TraceEvent(SevInfo, "DDRelocatorForManualShardSplit").detail("Range", rd.keys);
+	}
+
 	try {
 		if (now() - self->lastInterval < 1.0) {
 			relocateShardInterval.severity = SevDebug;

From 6cc3b02ee3522069f9a08f7ec4e37cef88a6d96c Mon Sep 17 00:00:00 2001
From: Zhe Wang <zhe.wang@wustl.edu>
Date: Wed, 20 Sep 2023 10:48:47 -0700
Subject: [PATCH 256/468] add redistribute fdbcli command

---
 fdbcli/CMakeLists.txt                |  1 +
 fdbcli/RedistributeCommand.actor.cpp | 58 ++++++++++++++++++++++++++++
 fdbcli/fdbcli.actor.cpp              |  7 ++++
 fdbcli/fdbcli.actor.h                |  3 ++
 4 files changed, 69 insertions(+)
 create mode 100644 fdbcli/RedistributeCommand.actor.cpp

diff --git a/fdbcli/CMakeLists.txt b/fdbcli/CMakeLists.txt
index 99fa7d33516..ae2e306f53c 100644
--- a/fdbcli/CMakeLists.txt
+++ b/fdbcli/CMakeLists.txt
@@ -20,6 +20,7 @@ set(FDBCLI_SRCS
   LockCommand.actor.cpp
   ChangeFeedCommand.actor.cpp
   MaintenanceCommand.actor.cpp
+  RedistributeCommand.actor.cpp
   ProfileCommand.actor.cpp
   SetClassCommand.actor.cpp
   SnapshotCommand.actor.cpp
diff --git a/fdbcli/RedistributeCommand.actor.cpp b/fdbcli/RedistributeCommand.actor.cpp
new file mode 100644
index 00000000000..00f8427c1a4
--- /dev/null
+++ b/fdbcli/RedistributeCommand.actor.cpp
@@ -0,0 +1,58 @@
+/*
+ * redistribute.actor.cpp
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2022 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <cinttypes>
+
+#include "fdbcli/fdbcli.actor.h"
+
+#include "fdbclient/FDBTypes.h"
+#include "fdbclient/ManagementAPI.actor.h"
+#include "fdbclient/IClientApi.h"
+
+#include "fdbclient/Knobs.h"
+#include "flow/Arena.h"
+#include "flow/FastRef.h"
+#include "flow/ThreadHelper.actor.h"
+#include "flow/actorcompiler.h" // This must be the last #include.
+
+namespace fdb_cli {
+
+ACTOR Future<bool> redistributeCommandActor(Reference<IClusterConnectionRecord> clusterFile,
+                                            std::vector<StringRef> tokens) {
+	state bool result = true;
+	if (tokens.size() == 3) {
+		Key begin = tokens[1];
+		Key end = tokens[2];
+		wait(moveShard(clusterFile, KeyRangeRef(begin, end), {}));
+	} else {
+		printUsage(tokens[0]);
+		result = false;
+	}
+	return result;
+}
+
+CommandFactory RedistributeFactory("redistribute",
+                                   CommandHelp("redistribute [Begin] [End]",
+                                               "Redistribute data within the range<Begin, End> among the cluster.\n",
+                                               "Given an input range, say <b, d>, and we suppose the current FDB"
+                                               "internal shard boundary is [a, c), [c, d), [d, e),"
+                                               "our splitting algorithm splits the input range into [b, c) and [c, d)."
+                                               "Then the two ranges will be redistributed among the cluster.\n"));
+} // namespace fdb_cli
diff --git a/fdbcli/fdbcli.actor.cpp b/fdbcli/fdbcli.actor.cpp
index 88f2fb50a22..f44667c149a 100644
--- a/fdbcli/fdbcli.actor.cpp
+++ b/fdbcli/fdbcli.actor.cpp
@@ -1743,6 +1743,13 @@ ACTOR Future<int> cli(CLIOptions opt, LineNoise* plinenoise) {
 					continue;
 				}
 
+				if (tokencmp(tokens[0], "redistribute")) {
+					bool _result = wait(makeInterruptable(redistributeCommandActor(ccf, tokens)));
+					if (!_result)
+						is_error = true;
+					continue;
+				}
+
 				if (tokencmp(tokens[0], "consistencycheck")) {
 					getTransaction(db, managementTenant, tr, options, intrans);
 					bool _result = wait(makeInterruptable(consistencyCheckCommandActor(tr, tokens, intrans)));
diff --git a/fdbcli/fdbcli.actor.h b/fdbcli/fdbcli.actor.h
index 33c98771297..087fbc98d60 100644
--- a/fdbcli/fdbcli.actor.h
+++ b/fdbcli/fdbcli.actor.h
@@ -185,6 +185,9 @@ ACTOR Future<bool> clearHealthyZone(Reference<IDatabase> db,
                                     bool printWarning = false,
                                     bool clearSSFailureZoneString = false);
 ACTOR Future<bool> maintenanceCommandActor(Reference<IDatabase> db, std::vector<StringRef> tokens);
+// redistributed command
+ACTOR Future<bool> redistributeCommandActor(Reference<IClusterConnectionRecord> clusterFile,
+                                            std::vector<StringRef> tokens);
 // profile command
 ACTOR Future<bool> profileCommandActor(Database db,
                                        Reference<ITransaction> tr,

From b05a116558dfe343b2e937623b80dce9330c7013 Mon Sep 17 00:00:00 2001
From: Zhe Wang <zhe.wang@wustl.edu>
Date: Wed, 20 Sep 2023 12:40:34 -0700
Subject: [PATCH 257/468] code cleanup & no logical change

---
 fdbcli/RedistributeCommand.actor.cpp          | 14 ++------
 fdbclient/ClusterInterface.h                  |  5 ++-
 fdbclient/ManagementAPI.actor.cpp             | 12 +++----
 fdbclient/ManagementAPI.actor.h               |  4 +--
 fdbserver/ClusterController.actor.cpp         |  2 +-
 fdbserver/DataDistribution.actor.cpp          |  4 +--
 fdbserver/DataDistributionTracker.actor.cpp   | 32 ++++++++++++-------
 fdbserver/DataDistributorInterface.h          |  5 +--
 fdbserver/tester.actor.cpp                    |  2 +-
 .../workloads/ManualShardSplit.actor.cpp      | 13 ++------
 fdbserver/workloads/workloads.actor.h         |  1 -
 11 files changed, 39 insertions(+), 55 deletions(-)

diff --git a/fdbcli/RedistributeCommand.actor.cpp b/fdbcli/RedistributeCommand.actor.cpp
index 00f8427c1a4..986414db3b2 100644
--- a/fdbcli/RedistributeCommand.actor.cpp
+++ b/fdbcli/RedistributeCommand.actor.cpp
@@ -1,5 +1,5 @@
 /*
- * redistribute.actor.cpp
+ * RedistributeCommand.actor.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -18,18 +18,8 @@
  * limitations under the License.
  */
 
-#include <cinttypes>
-
 #include "fdbcli/fdbcli.actor.h"
-
-#include "fdbclient/FDBTypes.h"
 #include "fdbclient/ManagementAPI.actor.h"
-#include "fdbclient/IClientApi.h"
-
-#include "fdbclient/Knobs.h"
-#include "flow/Arena.h"
-#include "flow/FastRef.h"
-#include "flow/ThreadHelper.actor.h"
 #include "flow/actorcompiler.h" // This must be the last #include.
 
 namespace fdb_cli {
@@ -40,7 +30,7 @@ ACTOR Future<bool> redistributeCommandActor(Reference<IClusterConnectionRecord>
 	if (tokens.size() == 3) {
 		Key begin = tokens[1];
 		Key end = tokens[2];
-		wait(moveShard(clusterFile, KeyRangeRef(begin, end), {}));
+		wait(moveShard(clusterFile, KeyRangeRef(begin, end)));
 	} else {
 		printUsage(tokens[0]);
 		result = false;
diff --git a/fdbclient/ClusterInterface.h b/fdbclient/ClusterInterface.h
index 3f90faf2904..4459743daee 100644
--- a/fdbclient/ClusterInterface.h
+++ b/fdbclient/ClusterInterface.h
@@ -264,7 +264,6 @@ struct MoveShardRequest {
 	constexpr static FileIdentifier file_identifier = 2799592;
 
 	KeyRange shard;
-	std::vector<Key> splitPoints;
 	std::vector<NetworkAddress> addresses;
 	ReplyPromise<Void> reply;
 
@@ -272,11 +271,11 @@ struct MoveShardRequest {
 	MoveShardRequest(KeyRange shard, std::vector<NetworkAddress> addresses)
 	  : shard{ std::move(shard) }, addresses{ std::move(addresses) } {}
 
-	MoveShardRequest(KeyRange shard, std::vector<Key> splitPoints) : shard(shard), splitPoints(splitPoints) {}
+	MoveShardRequest(KeyRange shard) : shard(shard) {}
 
 	template <class Ar>
 	void serialize(Ar& ar) {
-		serializer(ar, shard, addresses, reply, splitPoints);
+		serializer(ar, shard, addresses, reply);
 	}
 };
 
diff --git a/fdbclient/ManagementAPI.actor.cpp b/fdbclient/ManagementAPI.actor.cpp
index 6bdbd3fe216..e2ec60056c2 100644
--- a/fdbclient/ManagementAPI.actor.cpp
+++ b/fdbclient/ManagementAPI.actor.cpp
@@ -2342,18 +2342,16 @@ ACTOR Future<Void> forceRecovery(Reference<IClusterConnectionRecord> clusterFile
 	}
 }
 
-ACTOR Future<Void> moveShard(Reference<IClusterConnectionRecord> clusterFile,
-                             KeyRange range,
-                             std::vector<Key> splitPoints) {
+ACTOR Future<Void> moveShard(Reference<IClusterConnectionRecord> clusterFile, KeyRange range) {
 	state Reference<AsyncVar<Optional<ClusterInterface>>> clusterInterface(new AsyncVar<Optional<ClusterInterface>>);
 	state Future<Void> leaderMon = monitorLeader<ClusterInterface>(clusterFile, clusterInterface);
 
 	loop {
 		choose {
-			when(wait(clusterInterface->get().present()
-			              ? brokenPromiseToNever(
-			                    clusterInterface->get().get().moveShard.getReply(MoveShardRequest(range, splitPoints)))
-			              : Never())) {
+			when(wait(
+			    clusterInterface->get().present()
+			        ? brokenPromiseToNever(clusterInterface->get().get().moveShard.getReply(MoveShardRequest(range)))
+			        : Never())) {
 				return Void();
 			}
 			when(wait(clusterInterface->onChange())) {}
diff --git a/fdbclient/ManagementAPI.actor.h b/fdbclient/ManagementAPI.actor.h
index cffda4caded..7b456a41779 100644
--- a/fdbclient/ManagementAPI.actor.h
+++ b/fdbclient/ManagementAPI.actor.h
@@ -148,9 +148,7 @@ ACTOR Future<int> setDDMode(Database cx, int mode);
 
 ACTOR Future<Void> forceRecovery(Reference<IClusterConnectionRecord> clusterFile, Standalone<StringRef> dcId);
 
-ACTOR Future<Void> moveShard(Reference<IClusterConnectionRecord> clusterFile,
-                             KeyRange range,
-                             std::vector<Key> splitPoints);
+ACTOR Future<Void> moveShard(Reference<IClusterConnectionRecord> clusterFile, KeyRange range);
 
 ACTOR Future<Void> printHealthyZone(Database cx);
 ACTOR Future<Void> setDDIgnoreRebalanceSwitch(Database cx, bool ignoreRebalance);
diff --git a/fdbserver/ClusterController.actor.cpp b/fdbserver/ClusterController.actor.cpp
index 43c8caa9838..9fc1ec74a8c 100644
--- a/fdbserver/ClusterController.actor.cpp
+++ b/fdbserver/ClusterController.actor.cpp
@@ -2040,7 +2040,7 @@ ACTOR Future<Void> triggerMoveShards(ClusterControllerData* self, MoveShardReque
 		       !self->db.serverInfo->get().distributor.present()) {
 			wait(self->db.serverInfo->onChange());
 		}
-		DistributorSplitRangeRequest fReq(req.shard, req.splitPoints);
+		DistributorSplitRangeRequest fReq({ req.shard.begin, req.shard.end });
 		SplitShardReply rep = wait(self->db.serverInfo->get().distributor.get().distributorSplitRange.getReply(fReq));
 		req.reply.send(Void());
 	} catch (Error& e) {
diff --git a/fdbserver/DataDistribution.actor.cpp b/fdbserver/DataDistribution.actor.cpp
index 0ea2ea133e2..d32252620c7 100644
--- a/fdbserver/DataDistribution.actor.cpp
+++ b/fdbserver/DataDistribution.actor.cpp
@@ -1258,8 +1258,8 @@ ACTOR Future<Void> ddGetMetrics(GetDataDistributorMetricsRequest req,
 
 ACTOR Future<Void> ddSplitRange(DistributorSplitRangeRequest req,
                                 PromiseStream<DistributorSplitRangeRequest> manualShardSplit) {
-	ErrorOr<SplitShardReply> result = wait(errorOr(
-	    brokenPromiseToNever(manualShardSplit.getReply(DistributorSplitRangeRequest(req.range, req.splitPoints)))));
+	ErrorOr<SplitShardReply> result =
+	    wait(errorOr(brokenPromiseToNever(manualShardSplit.getReply(DistributorSplitRangeRequest(req.splitPoints)))));
 	if (result.isError()) {
 		req.reply.sendError(result.getError());
 	} else {
diff --git a/fdbserver/DataDistributionTracker.actor.cpp b/fdbserver/DataDistributionTracker.actor.cpp
index 8521f9c6474..ee96f905d2b 100644
--- a/fdbserver/DataDistributionTracker.actor.cpp
+++ b/fdbserver/DataDistributionTracker.actor.cpp
@@ -1017,18 +1017,28 @@ ACTOR Future<Void> dataDistributionTracker(Reference<InitialDataDistribution> in
 				self.sizeChanges.add(fetchShardMetricsList(&self, req));
 			}
 			when(DistributorSplitRangeRequest req = waitNext(manualShardSplit.getFuture())) {
-				TraceEvent(SevInfo, "DDSplitRange", self.distributorId)
-				    .detail("Range", req.range)
-				    .detail("SplitPoints", describe(req.splitPoints));
-				for (auto it : self.shards.intersectingRanges(req.range)) {
+				if (req.splitPoints.size() != 2) {
+					TraceEvent(SevWarn, "DDSplitRangeWrongInputSize", self.distributorId)
+					    .detail("SplitPoints", req.splitPoints);
+					req.reply.sendError(not_implemented());
+					continue;
+				} else if (req.splitPoints[0] == req.splitPoints[1]) {
+					TraceEvent(SevWarn, "DDSplitRangeEmptyInputRange", self.distributorId)
+					    .detail("SplitPoints", req.splitPoints);
+					req.reply.sendError(not_implemented());
+					continue;
+				}
+				TraceEvent(SevInfo, "DDSplitRangeStart", self.distributorId).detail("SplitPoints", req.splitPoints);
+				KeyRange inputRange = req.splitPoints[0] < req.splitPoints[1]
+				                          ? KeyRangeRef(req.splitPoints[0], req.splitPoints[1])
+				                          : KeyRangeRef(req.splitPoints[1], req.splitPoints[0]);
+				for (auto it : self.shards.intersectingRanges(inputRange)) {
 					bool triggerManualSplitShard = false;
-					if (it->range().begin < req.range.begin && req.range.begin < it->range().end) {
-						triggerManualSplitShard = true;
-						it->value().manualSplitPoints->push_back(req.range.begin);
-					}
-					if (it->range().begin < req.range.end && req.range.end < it->range().end) {
-						triggerManualSplitShard = true;
-						it->value().manualSplitPoints->push_back(req.range.end);
+					for (const auto& splitPoint : req.splitPoints) {
+						if (it->range().begin < splitPoint && splitPoint < it->range().end) {
+							triggerManualSplitShard = true;
+							it->value().manualSplitPoints->push_back(splitPoint);
+						}
 					}
 					if (triggerManualSplitShard) {
 						// Trigger split for this shard
diff --git a/fdbserver/DataDistributorInterface.h b/fdbserver/DataDistributorInterface.h
index 9e047e6f1bc..b858278f988 100644
--- a/fdbserver/DataDistributorInterface.h
+++ b/fdbserver/DataDistributorInterface.h
@@ -154,17 +154,14 @@ struct DistributorExclusionSafetyCheckRequest {
 struct DistributorSplitRangeRequest {
 	constexpr static FileIdentifier file_identifier = 1384441;
 	std::vector<Key> splitPoints;
-	KeyRange range;
 	ReplyPromise<SplitShardReply> reply;
 
 	DistributorSplitRangeRequest() {}
 	explicit DistributorSplitRangeRequest(std::vector<Key> splitPoints) : splitPoints{ std::move(splitPoints) } {}
-	explicit DistributorSplitRangeRequest(KeyRange range, std::vector<Key> splitPoints)
-	  : splitPoints(splitPoints), range(range) {}
 
 	template <class Ar>
 	void serialize(Ar& ar) {
-		serializer(ar, splitPoints, reply, range);
+		serializer(ar, splitPoints, reply);
 	}
 };
 
diff --git a/fdbserver/tester.actor.cpp b/fdbserver/tester.actor.cpp
index b1dad749727..56128e54324 100644
--- a/fdbserver/tester.actor.cpp
+++ b/fdbserver/tester.actor.cpp
@@ -352,7 +352,7 @@ Reference<TestWorkload> getWorkloadIface(WorkloadRequest work,
 	wcx.dbInfo = dbInfo;
 	wcx.options = options;
 	wcx.sharedRandomNumber = work.sharedRandomNumber;
-	std::cout << "Create: " << testName.toString() << "\n";
+
 	auto workload = IWorkloadFactory::create(testName.toString(), wcx);
 
 	auto unconsumedOptions = checkAllOptionsConsumed(workload ? workload->options : VectorRef<KeyValueRef>());
diff --git a/fdbserver/workloads/ManualShardSplit.actor.cpp b/fdbserver/workloads/ManualShardSplit.actor.cpp
index 635ae745fb6..ad85015e889 100644
--- a/fdbserver/workloads/ManualShardSplit.actor.cpp
+++ b/fdbserver/workloads/ManualShardSplit.actor.cpp
@@ -20,15 +20,9 @@
 
 #include "fdbclient/ManagementAPI.actor.h"
 #include "fdbclient/NativeAPI.actor.h"
-#include "fdbserver/Knobs.h"
-#include "fdbserver/QuietDatabase.h"
-#include "fdbrpc/simulator.h"
 #include "fdbserver/workloads/workloads.actor.h"
 #include "flow/Error.h"
 #include "flow/IRandom.h"
-#include "flow/flow.h"
-#include <cstdint>
-#include <limits>
 #include "flow/actorcompiler.h" // This must be the last #include.
 
 struct ManualShardSplitWorkload : TestWorkload {
@@ -58,9 +52,8 @@ struct ManualShardSplitWorkload : TestWorkload {
 		                                 { "TestKeyF"_sr, "TestValueF"_sr } });
 
 		Version ver = wait(self->populateData(self, cx, &kvs));
-		state std::vector<Key> splitPoints = {};
-		wait(moveShard(cx->getConnectionRecord(), KeyRangeRef("TestKeyA"_sr, "TestKeyD"_sr), splitPoints));
-		wait(moveShard(cx->getConnectionRecord(), KeyRangeRef("TestKeyB"_sr, "TestKeyC"_sr), splitPoints));
+		wait(moveShard(cx->getConnectionRecord(), KeyRangeRef("TestKeyA"_sr, "TestKeyD"_sr)));
+		wait(moveShard(cx->getConnectionRecord(), KeyRangeRef("TestKeyB"_sr, "TestKeyC"_sr)));
 		return Void();
 	}
 
@@ -97,4 +90,4 @@ struct ManualShardSplitWorkload : TestWorkload {
 	void getMetrics(std::vector<PerfMetric>& m) override {}
 };
 
-WorkloadFactory<ManualShardSplitWorkload> ManualShardSplitWorkloadFactory("ManualShardSplit");
\ No newline at end of file
+WorkloadFactory<ManualShardSplitWorkload> ManualShardSplitWorkloadFactory("ManualShardSplit");
diff --git a/fdbserver/workloads/workloads.actor.h b/fdbserver/workloads/workloads.actor.h
index 3c612456b9e..76d5c6d32d9 100644
--- a/fdbserver/workloads/workloads.actor.h
+++ b/fdbserver/workloads/workloads.actor.h
@@ -108,7 +108,6 @@ struct IWorkloadFactory : ReferenceCounted<IWorkloadFactory> {
 	static Reference<TestWorkload> create(std::string const& name, WorkloadContext const& wcx) {
 		auto it = factories().find(name);
 		if (it == factories().end()) {
-			std::cout << "factory does not find the name\n";
 			return {}; // or throw?
 		}
 		return it->second->create(wcx);

From f8d61e6965af129a3441191f480f5f20cb0c3c1b Mon Sep 17 00:00:00 2001
From: Zhe Wang <zhe.wang@wustl.edu>
Date: Wed, 20 Sep 2023 20:11:57 -0700
Subject: [PATCH 258/468] add more trace events

---
 fdbserver/DataDistributionQueue.actor.cpp | 34 +++++++++++++++++++++--
 1 file changed, 32 insertions(+), 2 deletions(-)

diff --git a/fdbserver/DataDistributionQueue.actor.cpp b/fdbserver/DataDistributionQueue.actor.cpp
index ad1794c3851..fc2429d1a8d 100644
--- a/fdbserver/DataDistributionQueue.actor.cpp
+++ b/fdbserver/DataDistributionQueue.actor.cpp
@@ -960,6 +960,10 @@ struct DDQueueData {
 		for (; it != combined.end(); it++) {
 			RelocateData rd(*it);
 
+			if (rd.priority == SERVER_KNOBS->PRIORITY_TEAM_STORAGE_QUEUE_TOO_LONG) {
+				TraceEvent(SevInfo, "ManualShardSplitLaunchCombinedWork").detail("Range", rd.keys);
+			}
+
 			// Check if there is an inflight shard that is overlapped with the queued relocateShard (rd)
 			bool overlappingInFlight = false;
 			auto intersectingInFlight = inFlight.intersectingRanges(rd.keys);
@@ -978,6 +982,9 @@ struct DDQueueData {
 
 			if (overlappingInFlight) {
 				// logRelocation( rd, "SkippingOverlappingInFlight" );
+				if (rd.priority == SERVER_KNOBS->PRIORITY_TEAM_STORAGE_QUEUE_TOO_LONG) {
+					TraceEvent(SevWarn, "ManualShardSplitSkipForOverlappingInFlight").detail("Range", rd.keys);
+				}
 				continue;
 			}
 
@@ -997,6 +1004,9 @@ struct DDQueueData {
 			// FIXME: we need spare capacity even when we're just going to be cancelling work via TEAM_HEALTHY
 			if (!canLaunchSrc(rd, teamSize, singleRegionTeamSize, busymap, cancellableRelocations)) {
 				// logRelocation( rd, "SkippingQueuedRelocation" );
+				if (rd.priority == SERVER_KNOBS->PRIORITY_TEAM_STORAGE_QUEUE_TOO_LONG) {
+					TraceEvent(SevWarn, "ManualShardSplitSkipForCannotLaunchSrc").detail("Range", rd.keys);
+				}
 				continue;
 			}
 
@@ -1100,7 +1110,7 @@ ACTOR Future<Void> dataDistributionRelocator(DDQueueData* self, RelocateData rd,
 	state std::vector<UID> destIds;
 
 	if (rd.priority == SERVER_KNOBS->PRIORITY_TEAM_STORAGE_QUEUE_TOO_LONG) {
-		TraceEvent(SevInfo, "DDRelocatorForManualShardSplit").detail("Range", rd.keys);
+		TraceEvent(SevInfo, "ManualShardSplitDDRelocatorStart").detail("Range", rd.keys);
 	}
 
 	try {
@@ -1413,11 +1423,19 @@ ACTOR Future<Void> dataDistributionRelocator(DDQueueData* self, RelocateData rd,
 					self->bytesWritten += metrics.bytes;
 					self->shardsAffectedByTeamFailure->finishMove(rd.keys);
 					relocationComplete.send(rd);
+					if (rd.priority == SERVER_KNOBS->PRIORITY_TEAM_STORAGE_QUEUE_TOO_LONG) {
+						TraceEvent(SevInfo, "ManualShardSplitDDRelocatorComplete").detail("Range", rd.keys);
+					}
 					return Void();
 				} else {
 					throw error;
 				}
 			} else {
+				if (rd.priority == SERVER_KNOBS->PRIORITY_TEAM_STORAGE_QUEUE_TOO_LONG) {
+					TraceEvent(SevInfo, "ManualShardSplitDDRelocatorError")
+					    .errorUnsuppressed(error)
+					    .detail("Range", rd.keys);
+				}
 				TEST(true); // move to removed server
 				healthyDestinations.addDataInFlightToTeam(-metrics.bytes);
 				if (!signalledTransferComplete) {
@@ -1431,6 +1449,9 @@ ACTOR Future<Void> dataDistributionRelocator(DDQueueData* self, RelocateData rd,
 			}
 		}
 	} catch (Error& e) {
+		if (rd.priority == SERVER_KNOBS->PRIORITY_TEAM_STORAGE_QUEUE_TOO_LONG) {
+			TraceEvent(SevInfo, "ManualShardSplitDDRelocatorError").errorUnsuppressed(e).detail("Range", rd.keys);
+		}
 		TraceEvent(relocateShardInterval.end(), distributorId)
 		    .errorUnsuppressed(e)
 		    .detail("Duration", now() - startTime);
@@ -1801,7 +1822,7 @@ ACTOR Future<Void> dataDistributionQueue(Database cx,
 			choose {
 				when(RelocateShard rs = waitNext(self.input)) {
 					if (rs.priority == SERVER_KNOBS->PRIORITY_TEAM_STORAGE_QUEUE_TOO_LONG) {
-						TraceEvent(SevInfo, "ShardRelocatorByManualSplit", self.distributorId).detail("Range", rs.keys);
+						TraceEvent(SevInfo, "ManualShardSplitStart", self.distributorId).detail("Range", rs.keys);
 					}
 					bool wasEmpty = serversToLaunchFrom.empty();
 					self.queueRelocation(rs, serversToLaunchFrom);
@@ -1814,17 +1835,26 @@ ACTOR Future<Void> dataDistributionQueue(Database cx,
 					launchQueuedWorkTimeout = Never();
 				}
 				when(RelocateData results = waitNext(self.fetchSourceServersComplete.getFuture())) {
+					if (results.priority == SERVER_KNOBS->PRIORITY_TEAM_STORAGE_QUEUE_TOO_LONG) {
+						TraceEvent(SevInfo, "ManualShardSplitFetchSourceComplete").detail("Range", results.keys);
+					}
 					// This when is triggered by queueRelocation() which is triggered by sending self.input
 					self.completeSourceFetch(results);
 					launchData = results;
 				}
 				when(RelocateData done = waitNext(self.dataTransferComplete.getFuture())) {
+					if (done.priority == SERVER_KNOBS->PRIORITY_TEAM_STORAGE_QUEUE_TOO_LONG) {
+						TraceEvent(SevInfo, "ManualShardSplitDataTransferComplete").detail("Range", done.keys);
+					}
 					complete(done, self.busymap, self.destBusymap);
 					if (serversToLaunchFrom.empty() && !done.src.empty())
 						launchQueuedWorkTimeout = delay(0, TaskPriority::DataDistributionLaunch);
 					serversToLaunchFrom.insert(done.src.begin(), done.src.end());
 				}
 				when(RelocateData done = waitNext(self.relocationComplete.getFuture())) {
+					if (done.priority == SERVER_KNOBS->PRIORITY_TEAM_STORAGE_QUEUE_TOO_LONG) {
+						TraceEvent(SevInfo, "ManualShardSplitRelocationComplete").detail("Range", done.keys);
+					}
 					self.activeRelocations--;
 					self.finishRelocation(done.priority, done.healthPriority);
 					self.fetchKeysComplete.erase(done);

From 40d44bc8195b4510d61dff4036c1a109c9d91e66 Mon Sep 17 00:00:00 2001
From: Zhe Wang <zhe.wang@wustl.edu>
Date: Thu, 21 Sep 2023 11:48:30 -0700
Subject: [PATCH 259/468] address comments

---
 fdbclient/ServerKnobs.cpp                   |  4 +--
 fdbclient/ServerKnobs.h                     |  4 +--
 fdbserver/ClusterController.actor.cpp       | 11 +++-----
 fdbserver/DataDistributionQueue.actor.cpp   | 30 ++++++++++-----------
 fdbserver/DataDistributionTracker.actor.cpp |  6 ++---
 fdbserver/workloads/workloads.actor.h       |  3 +--
 6 files changed, 25 insertions(+), 33 deletions(-)

diff --git a/fdbclient/ServerKnobs.cpp b/fdbclient/ServerKnobs.cpp
index ec75f19bf3c..952b22f0e48 100644
--- a/fdbclient/ServerKnobs.cpp
+++ b/fdbclient/ServerKnobs.cpp
@@ -135,7 +135,7 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi
 	init( DD_QUEUE_LOGGING_INTERVAL,                             5.0 );
 	init( RELOCATION_PARALLELISM_PER_SOURCE_SERVER,                2 ); if( randomize && BUGGIFY ) RELOCATION_PARALLELISM_PER_SOURCE_SERVER = 1;
 	init( RELOCATION_PARALLELISM_PER_DEST_SERVER,                 10 ); if( randomize && BUGGIFY ) RELOCATION_PARALLELISM_PER_DEST_SERVER = 1; // Note: if this is smaller than FETCH_KEYS_PARALLELISM, this will artificially reduce performance. The current default of 10 is probably too high but is set conservatively for now.
-	init( RELOCATION_WORK_AMPLIFIER_FOR_LONG_SSQUEUE_MOVE,       5.0 );
+	init( RELOCATION_WORK_AMPLIFIER_FOR_MANUAL_SHARD_SPLIT,      5.0 );
 	init( DD_QUEUE_MAX_KEY_SERVERS,                              100 ); if( randomize && BUGGIFY ) DD_QUEUE_MAX_KEY_SERVERS = 1;
 	init( DD_REBALANCE_PARALLELISM,                               50 );
 	init( DD_REBALANCE_RESET_AMOUNT,                              30 );
@@ -164,7 +164,7 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi
 	init( PRIORITY_TEAM_FAILED,                                  805 );
 	init( PRIORITY_TEAM_0_LEFT,                                  809 );
 	init( PRIORITY_SPLIT_SHARD,                                  950 ); if( randomize && BUGGIFY ) PRIORITY_SPLIT_SHARD = 350;
-	init( PRIORITY_TEAM_STORAGE_QUEUE_TOO_LONG,                  960 );
+	init( PRIORITY_MANUAL_SHARD_SPLIT,                           960 );
 
 	// Data distribution
 	init( RETRY_RELOCATESHARD_DELAY,                             0.1 );
diff --git a/fdbclient/ServerKnobs.h b/fdbclient/ServerKnobs.h
index 86455e122d5..0bf67f221e8 100644
--- a/fdbclient/ServerKnobs.h
+++ b/fdbclient/ServerKnobs.h
@@ -128,7 +128,7 @@ class ServerKnobs : public KnobsImpl<ServerKnobs> {
 	double DD_QUEUE_LOGGING_INTERVAL;
 	double RELOCATION_PARALLELISM_PER_SOURCE_SERVER;
 	double RELOCATION_PARALLELISM_PER_DEST_SERVER;
-	double RELOCATION_WORK_AMPLIFIER_FOR_LONG_SSQUEUE_MOVE;
+	double RELOCATION_WORK_AMPLIFIER_FOR_MANUAL_SHARD_SPLIT;
 	int DD_QUEUE_MAX_KEY_SERVERS;
 	int DD_REBALANCE_PARALLELISM;
 	int DD_REBALANCE_RESET_AMOUNT;
@@ -162,7 +162,7 @@ class ServerKnobs : public KnobsImpl<ServerKnobs> {
 	int PRIORITY_TEAM_1_LEFT;
 	int PRIORITY_TEAM_FAILED; // Priority when a server in the team is excluded as failed
 	int PRIORITY_TEAM_0_LEFT;
-	int PRIORITY_TEAM_STORAGE_QUEUE_TOO_LONG; // Priority when a server has a long storage queue
+	int PRIORITY_MANUAL_SHARD_SPLIT; // Priority when a server has a long storage queue
 	int PRIORITY_SPLIT_SHARD;
 
 	// Data distribution
diff --git a/fdbserver/ClusterController.actor.cpp b/fdbserver/ClusterController.actor.cpp
index 9fc1ec74a8c..3b387dec652 100644
--- a/fdbserver/ClusterController.actor.cpp
+++ b/fdbserver/ClusterController.actor.cpp
@@ -2049,13 +2049,6 @@ ACTOR Future<Void> triggerMoveShards(ClusterControllerData* self, MoveShardReque
 	return Void();
 }
 
-ACTOR Future<Void> handleMoveShards(ClusterControllerData* self, ClusterControllerFullInterface interf) {
-	loop {
-		state MoveShardRequest req = waitNext(interf.clientInterface.moveShard.getFuture());
-		self->addActor.send(triggerMoveShards(self, req));
-	}
-}
-
 struct SingletonRecruitThrottler {
 	double lastRecruitStart;
 
@@ -2687,7 +2680,6 @@ ACTOR Future<Void> clusterControllerCore(ClusterControllerFullInterface interf,
 	self.addActor.send(updatedChangedDatacenters(&self));
 	self.addActor.send(updateDatacenterVersionDifference(&self));
 	self.addActor.send(handleForcedRecoveries(&self, interf));
-	self.addActor.send(handleMoveShards(&self, interf));
 	self.addActor.send(monitorDataDistributor(&self));
 	self.addActor.send(monitorRatekeeper(&self));
 	self.addActor.send(monitorBlobManager(&self));
@@ -2784,6 +2776,9 @@ ACTOR Future<Void> clusterControllerCore(ClusterControllerFullInterface interf,
 		when(GetServerDBInfoRequest req = waitNext(interf.getServerDBInfo.getFuture())) {
 			self.addActor.send(clusterGetServerInfo(&self.db, req.knownServerInfoID, req.reply));
 		}
+		when(MoveShardRequest req = waitNext(interf.clientInterface.moveShard.getFuture())) {
+			self.addActor.send(triggerMoveShards(&self, req));
+		}
 		when(wait(leaderFail)) {
 			// We are no longer the leader if this has changed.
 			endRole(Role::CLUSTER_CONTROLLER, interf.id(), "Leader Replaced", true);
diff --git a/fdbserver/DataDistributionQueue.actor.cpp b/fdbserver/DataDistributionQueue.actor.cpp
index fc2429d1a8d..cfd0e4af332 100644
--- a/fdbserver/DataDistributionQueue.actor.cpp
+++ b/fdbserver/DataDistributionQueue.actor.cpp
@@ -313,8 +313,8 @@ int getSrcWorkFactor(RelocateData const& relocation, int singleRegionTeamSize) {
 		res = WORK_FULL_UTILIZATION / 2 / SERVER_KNOBS->RELOCATION_PARALLELISM_PER_SOURCE_SERVER;
 	else // for now we assume that any message at a lower priority can best be assumed to have a full team left for work
 		res = WORK_FULL_UTILIZATION / singleRegionTeamSize / SERVER_KNOBS->RELOCATION_PARALLELISM_PER_SOURCE_SERVER;
-	if (relocation.priority == SERVER_KNOBS->PRIORITY_TEAM_STORAGE_QUEUE_TOO_LONG) {
-		res = std::min(static_cast<int>(res * SERVER_KNOBS->RELOCATION_WORK_AMPLIFIER_FOR_LONG_SSQUEUE_MOVE),
+	if (relocation.priority == SERVER_KNOBS->PRIORITY_MANUAL_SHARD_SPLIT) {
+		res = std::min(static_cast<int>(res * SERVER_KNOBS->RELOCATION_WORK_AMPLIFIER_FOR_MANUAL_SHARD_SPLIT),
 		               WORK_FULL_UTILIZATION);
 	}
 	return res;
@@ -323,8 +323,8 @@ int getSrcWorkFactor(RelocateData const& relocation, int singleRegionTeamSize) {
 int getDestWorkFactor(int priority) {
 	// Work of moving a shard is even across destination servers
 	int res = WORK_FULL_UTILIZATION / SERVER_KNOBS->RELOCATION_PARALLELISM_PER_DEST_SERVER;
-	if (priority == SERVER_KNOBS->PRIORITY_TEAM_STORAGE_QUEUE_TOO_LONG) {
-		res = std::min(static_cast<int>(res * SERVER_KNOBS->RELOCATION_WORK_AMPLIFIER_FOR_LONG_SSQUEUE_MOVE),
+	if (priority == SERVER_KNOBS->PRIORITY_MANUAL_SHARD_SPLIT) {
+		res = std::min(static_cast<int>(res * SERVER_KNOBS->RELOCATION_WORK_AMPLIFIER_FOR_MANUAL_SHARD_SPLIT),
 		               WORK_FULL_UTILIZATION);
 	}
 	return res;
@@ -960,7 +960,7 @@ struct DDQueueData {
 		for (; it != combined.end(); it++) {
 			RelocateData rd(*it);
 
-			if (rd.priority == SERVER_KNOBS->PRIORITY_TEAM_STORAGE_QUEUE_TOO_LONG) {
+			if (rd.priority == SERVER_KNOBS->PRIORITY_MANUAL_SHARD_SPLIT) {
 				TraceEvent(SevInfo, "ManualShardSplitLaunchCombinedWork").detail("Range", rd.keys);
 			}
 
@@ -982,7 +982,7 @@ struct DDQueueData {
 
 			if (overlappingInFlight) {
 				// logRelocation( rd, "SkippingOverlappingInFlight" );
-				if (rd.priority == SERVER_KNOBS->PRIORITY_TEAM_STORAGE_QUEUE_TOO_LONG) {
+				if (rd.priority == SERVER_KNOBS->PRIORITY_MANUAL_SHARD_SPLIT) {
 					TraceEvent(SevWarn, "ManualShardSplitSkipForOverlappingInFlight").detail("Range", rd.keys);
 				}
 				continue;
@@ -1004,7 +1004,7 @@ struct DDQueueData {
 			// FIXME: we need spare capacity even when we're just going to be cancelling work via TEAM_HEALTHY
 			if (!canLaunchSrc(rd, teamSize, singleRegionTeamSize, busymap, cancellableRelocations)) {
 				// logRelocation( rd, "SkippingQueuedRelocation" );
-				if (rd.priority == SERVER_KNOBS->PRIORITY_TEAM_STORAGE_QUEUE_TOO_LONG) {
+				if (rd.priority == SERVER_KNOBS->PRIORITY_MANUAL_SHARD_SPLIT) {
 					TraceEvent(SevWarn, "ManualShardSplitSkipForCannotLaunchSrc").detail("Range", rd.keys);
 				}
 				continue;
@@ -1109,7 +1109,7 @@ ACTOR Future<Void> dataDistributionRelocator(DDQueueData* self, RelocateData rd,
 	state double startTime = now();
 	state std::vector<UID> destIds;
 
-	if (rd.priority == SERVER_KNOBS->PRIORITY_TEAM_STORAGE_QUEUE_TOO_LONG) {
+	if (rd.priority == SERVER_KNOBS->PRIORITY_MANUAL_SHARD_SPLIT) {
 		TraceEvent(SevInfo, "ManualShardSplitDDRelocatorStart").detail("Range", rd.keys);
 	}
 
@@ -1423,7 +1423,7 @@ ACTOR Future<Void> dataDistributionRelocator(DDQueueData* self, RelocateData rd,
 					self->bytesWritten += metrics.bytes;
 					self->shardsAffectedByTeamFailure->finishMove(rd.keys);
 					relocationComplete.send(rd);
-					if (rd.priority == SERVER_KNOBS->PRIORITY_TEAM_STORAGE_QUEUE_TOO_LONG) {
+					if (rd.priority == SERVER_KNOBS->PRIORITY_MANUAL_SHARD_SPLIT) {
 						TraceEvent(SevInfo, "ManualShardSplitDDRelocatorComplete").detail("Range", rd.keys);
 					}
 					return Void();
@@ -1431,7 +1431,7 @@ ACTOR Future<Void> dataDistributionRelocator(DDQueueData* self, RelocateData rd,
 					throw error;
 				}
 			} else {
-				if (rd.priority == SERVER_KNOBS->PRIORITY_TEAM_STORAGE_QUEUE_TOO_LONG) {
+				if (rd.priority == SERVER_KNOBS->PRIORITY_MANUAL_SHARD_SPLIT) {
 					TraceEvent(SevInfo, "ManualShardSplitDDRelocatorError")
 					    .errorUnsuppressed(error)
 					    .detail("Range", rd.keys);
@@ -1449,7 +1449,7 @@ ACTOR Future<Void> dataDistributionRelocator(DDQueueData* self, RelocateData rd,
 			}
 		}
 	} catch (Error& e) {
-		if (rd.priority == SERVER_KNOBS->PRIORITY_TEAM_STORAGE_QUEUE_TOO_LONG) {
+		if (rd.priority == SERVER_KNOBS->PRIORITY_MANUAL_SHARD_SPLIT) {
 			TraceEvent(SevInfo, "ManualShardSplitDDRelocatorError").errorUnsuppressed(e).detail("Range", rd.keys);
 		}
 		TraceEvent(relocateShardInterval.end(), distributorId)
@@ -1821,7 +1821,7 @@ ACTOR Future<Void> dataDistributionQueue(Database cx,
 
 			choose {
 				when(RelocateShard rs = waitNext(self.input)) {
-					if (rs.priority == SERVER_KNOBS->PRIORITY_TEAM_STORAGE_QUEUE_TOO_LONG) {
+					if (rs.priority == SERVER_KNOBS->PRIORITY_MANUAL_SHARD_SPLIT) {
 						TraceEvent(SevInfo, "ManualShardSplitStart", self.distributorId).detail("Range", rs.keys);
 					}
 					bool wasEmpty = serversToLaunchFrom.empty();
@@ -1835,7 +1835,7 @@ ACTOR Future<Void> dataDistributionQueue(Database cx,
 					launchQueuedWorkTimeout = Never();
 				}
 				when(RelocateData results = waitNext(self.fetchSourceServersComplete.getFuture())) {
-					if (results.priority == SERVER_KNOBS->PRIORITY_TEAM_STORAGE_QUEUE_TOO_LONG) {
+					if (results.priority == SERVER_KNOBS->PRIORITY_MANUAL_SHARD_SPLIT) {
 						TraceEvent(SevInfo, "ManualShardSplitFetchSourceComplete").detail("Range", results.keys);
 					}
 					// This when is triggered by queueRelocation() which is triggered by sending self.input
@@ -1843,7 +1843,7 @@ ACTOR Future<Void> dataDistributionQueue(Database cx,
 					launchData = results;
 				}
 				when(RelocateData done = waitNext(self.dataTransferComplete.getFuture())) {
-					if (done.priority == SERVER_KNOBS->PRIORITY_TEAM_STORAGE_QUEUE_TOO_LONG) {
+					if (done.priority == SERVER_KNOBS->PRIORITY_MANUAL_SHARD_SPLIT) {
 						TraceEvent(SevInfo, "ManualShardSplitDataTransferComplete").detail("Range", done.keys);
 					}
 					complete(done, self.busymap, self.destBusymap);
@@ -1852,7 +1852,7 @@ ACTOR Future<Void> dataDistributionQueue(Database cx,
 					serversToLaunchFrom.insert(done.src.begin(), done.src.end());
 				}
 				when(RelocateData done = waitNext(self.relocationComplete.getFuture())) {
-					if (done.priority == SERVER_KNOBS->PRIORITY_TEAM_STORAGE_QUEUE_TOO_LONG) {
+					if (done.priority == SERVER_KNOBS->PRIORITY_MANUAL_SHARD_SPLIT) {
 						TraceEvent(SevInfo, "ManualShardSplitRelocationComplete").detail("Range", done.keys);
 					}
 					self.activeRelocations--;
diff --git a/fdbserver/DataDistributionTracker.actor.cpp b/fdbserver/DataDistributionTracker.actor.cpp
index ee96f905d2b..5aedc1c25ca 100644
--- a/fdbserver/DataDistributionTracker.actor.cpp
+++ b/fdbserver/DataDistributionTracker.actor.cpp
@@ -534,8 +534,7 @@ ACTOR Future<Void> shardSplitter(DataDistributionTracker* self,
 	}
 
 	if (numShards > 1) {
-		int priority =
-		    doManualSplit ? SERVER_KNOBS->PRIORITY_TEAM_STORAGE_QUEUE_TOO_LONG : SERVER_KNOBS->PRIORITY_SPLIT_SHARD;
+		int priority = doManualSplit ? SERVER_KNOBS->PRIORITY_MANUAL_SHARD_SPLIT : SERVER_KNOBS->PRIORITY_SPLIT_SHARD;
 		int skipRange = deterministicRandom()->randomInt(0, numShards);
 		// The queue can't deal with RelocateShard requests which split an existing shard into three pieces, so
 		// we have to send the unskipped ranges in this order (nibbling in from the edges of the old range)
@@ -1045,8 +1044,7 @@ ACTOR Future<Void> dataDistributionTracker(Reference<InitialDataDistribution> in
 						it->value().shouldManualSplit->set(true);
 					} else {
 						// Trigger data move for this shard
-						self.output.send(
-						    RelocateShard(it->range(), SERVER_KNOBS->PRIORITY_TEAM_STORAGE_QUEUE_TOO_LONG));
+						self.output.send(RelocateShard(it->range(), SERVER_KNOBS->PRIORITY_MANUAL_SHARD_SPLIT));
 					}
 				}
 				req.reply.send(SplitShardReply());
diff --git a/fdbserver/workloads/workloads.actor.h b/fdbserver/workloads/workloads.actor.h
index 76d5c6d32d9..c7c0557b682 100644
--- a/fdbserver/workloads/workloads.actor.h
+++ b/fdbserver/workloads/workloads.actor.h
@@ -107,9 +107,8 @@ struct KVWorkload : TestWorkload {
 struct IWorkloadFactory : ReferenceCounted<IWorkloadFactory> {
 	static Reference<TestWorkload> create(std::string const& name, WorkloadContext const& wcx) {
 		auto it = factories().find(name);
-		if (it == factories().end()) {
+		if (it == factories().end())
 			return {}; // or throw?
-		}
 		return it->second->create(wcx);
 	}
 	static std::map<std::string, Reference<IWorkloadFactory>>& factories() {

From c3d4ab1b29616bf5a9469dfb58abfffef4986403 Mon Sep 17 00:00:00 2001
From: "Johannes M. Scheuermann" <jscheuermann@apple.com>
Date: Fri, 22 Sep 2023 08:13:09 +0200
Subject: [PATCH 260/468] Update CGO LDflags for new MacOS linker

---
 bindings/go/src/fdb/fdb_darwin.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bindings/go/src/fdb/fdb_darwin.go b/bindings/go/src/fdb/fdb_darwin.go
index c4157af70be..9050615ba23 100644
--- a/bindings/go/src/fdb/fdb_darwin.go
+++ b/bindings/go/src/fdb/fdb_darwin.go
@@ -1,5 +1,5 @@
 package fdb
 
 //#cgo CFLAGS: -I/usr/local/include/
-//#cgo LDFLAGS: -L/usr/local/lib/
+//#cgo LDFLAGS: -Wl,-rpath,/usr/local/lib
 import "C"

From 7e9e5d7961c3d0765dc107db6af548ab259e60fc Mon Sep 17 00:00:00 2001
From: "Johannes M. Scheuermann" <jscheuermann@apple.com>
Date: Fri, 22 Sep 2023 08:48:47 +0200
Subject: [PATCH 261/468] Add the linker option back again

---
 bindings/go/src/fdb/fdb_darwin.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bindings/go/src/fdb/fdb_darwin.go b/bindings/go/src/fdb/fdb_darwin.go
index 9050615ba23..83a6daff05e 100644
--- a/bindings/go/src/fdb/fdb_darwin.go
+++ b/bindings/go/src/fdb/fdb_darwin.go
@@ -1,5 +1,5 @@
 package fdb
 
 //#cgo CFLAGS: -I/usr/local/include/
-//#cgo LDFLAGS: -Wl,-rpath,/usr/local/lib
+//#cgo LDFLAGS: -L/usr/local/lib/ -Wl,-rpath,/usr/local/lib
 import "C"

From 38d57f7260b00dbcbc5cb1a1c2ce912632a8fdef Mon Sep 17 00:00:00 2001
From: Zhe Wang <zhe.wang@wustl.edu>
Date: Fri, 22 Sep 2023 09:37:00 -0700
Subject: [PATCH 262/468] address comments

---
 fdbcli/fdbcli.actor.cpp                   |  3 ++-
 fdbclient/ServerKnobs.cpp                 |  2 +-
 fdbclient/ServerKnobs.h                   |  2 +-
 fdbserver/DataDistributionQueue.actor.cpp | 22 ++++++++--------------
 4 files changed, 12 insertions(+), 17 deletions(-)

diff --git a/fdbcli/fdbcli.actor.cpp b/fdbcli/fdbcli.actor.cpp
index f44667c149a..305330872db 100644
--- a/fdbcli/fdbcli.actor.cpp
+++ b/fdbcli/fdbcli.actor.cpp
@@ -1745,8 +1745,9 @@ ACTOR Future<int> cli(CLIOptions opt, LineNoise* plinenoise) {
 
 				if (tokencmp(tokens[0], "redistribute")) {
 					bool _result = wait(makeInterruptable(redistributeCommandActor(ccf, tokens)));
-					if (!_result)
+					if (!_result) {
 						is_error = true;
+					}
 					continue;
 				}
 
diff --git a/fdbclient/ServerKnobs.cpp b/fdbclient/ServerKnobs.cpp
index 952b22f0e48..be967e22809 100644
--- a/fdbclient/ServerKnobs.cpp
+++ b/fdbclient/ServerKnobs.cpp
@@ -135,7 +135,7 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi
 	init( DD_QUEUE_LOGGING_INTERVAL,                             5.0 );
 	init( RELOCATION_PARALLELISM_PER_SOURCE_SERVER,                2 ); if( randomize && BUGGIFY ) RELOCATION_PARALLELISM_PER_SOURCE_SERVER = 1;
 	init( RELOCATION_PARALLELISM_PER_DEST_SERVER,                 10 ); if( randomize && BUGGIFY ) RELOCATION_PARALLELISM_PER_DEST_SERVER = 1; // Note: if this is smaller than FETCH_KEYS_PARALLELISM, this will artificially reduce performance. The current default of 10 is probably too high but is set conservatively for now.
-	init( RELOCATION_WORK_AMPLIFIER_FOR_MANUAL_SHARD_SPLIT,      5.0 );
+	init( MANUAL_SPLIT_RELOCATION_PARALLELISM_PER_DEST_SERVER,     2 ); if( randomize && BUGGIFY ) MANUAL_SPLIT_RELOCATION_PARALLELISM_PER_DEST_SERVER = 1;
 	init( DD_QUEUE_MAX_KEY_SERVERS,                              100 ); if( randomize && BUGGIFY ) DD_QUEUE_MAX_KEY_SERVERS = 1;
 	init( DD_REBALANCE_PARALLELISM,                               50 );
 	init( DD_REBALANCE_RESET_AMOUNT,                              30 );
diff --git a/fdbclient/ServerKnobs.h b/fdbclient/ServerKnobs.h
index 0bf67f221e8..60db5673602 100644
--- a/fdbclient/ServerKnobs.h
+++ b/fdbclient/ServerKnobs.h
@@ -128,7 +128,7 @@ class ServerKnobs : public KnobsImpl<ServerKnobs> {
 	double DD_QUEUE_LOGGING_INTERVAL;
 	double RELOCATION_PARALLELISM_PER_SOURCE_SERVER;
 	double RELOCATION_PARALLELISM_PER_DEST_SERVER;
-	double RELOCATION_WORK_AMPLIFIER_FOR_MANUAL_SHARD_SPLIT;
+	double MANUAL_SPLIT_RELOCATION_PARALLELISM_PER_DEST_SERVER;
 	int DD_QUEUE_MAX_KEY_SERVERS;
 	int DD_REBALANCE_PARALLELISM;
 	int DD_REBALANCE_RESET_AMOUNT;
diff --git a/fdbserver/DataDistributionQueue.actor.cpp b/fdbserver/DataDistributionQueue.actor.cpp
index cfd0e4af332..211ec483200 100644
--- a/fdbserver/DataDistributionQueue.actor.cpp
+++ b/fdbserver/DataDistributionQueue.actor.cpp
@@ -305,29 +305,23 @@ struct Busyness {
 
 // find the "workFactor" for this, were it launched now
 int getSrcWorkFactor(RelocateData const& relocation, int singleRegionTeamSize) {
-	int res = 0;
 	if (relocation.healthPriority == SERVER_KNOBS->PRIORITY_TEAM_1_LEFT ||
 	    relocation.healthPriority == SERVER_KNOBS->PRIORITY_TEAM_0_LEFT)
-		res = WORK_FULL_UTILIZATION / SERVER_KNOBS->RELOCATION_PARALLELISM_PER_SOURCE_SERVER;
+		return WORK_FULL_UTILIZATION / SERVER_KNOBS->RELOCATION_PARALLELISM_PER_SOURCE_SERVER;
 	else if (relocation.healthPriority == SERVER_KNOBS->PRIORITY_TEAM_2_LEFT)
-		res = WORK_FULL_UTILIZATION / 2 / SERVER_KNOBS->RELOCATION_PARALLELISM_PER_SOURCE_SERVER;
+		return WORK_FULL_UTILIZATION / 2 / SERVER_KNOBS->RELOCATION_PARALLELISM_PER_SOURCE_SERVER;
 	else // for now we assume that any message at a lower priority can best be assumed to have a full team left for work
-		res = WORK_FULL_UTILIZATION / singleRegionTeamSize / SERVER_KNOBS->RELOCATION_PARALLELISM_PER_SOURCE_SERVER;
-	if (relocation.priority == SERVER_KNOBS->PRIORITY_MANUAL_SHARD_SPLIT) {
-		res = std::min(static_cast<int>(res * SERVER_KNOBS->RELOCATION_WORK_AMPLIFIER_FOR_MANUAL_SHARD_SPLIT),
-		               WORK_FULL_UTILIZATION);
-	}
-	return res;
+		return WORK_FULL_UTILIZATION / singleRegionTeamSize / SERVER_KNOBS->RELOCATION_PARALLELISM_PER_SOURCE_SERVER;
 }
 
 int getDestWorkFactor(int priority) {
-	// Work of moving a shard is even across destination servers
-	int res = WORK_FULL_UTILIZATION / SERVER_KNOBS->RELOCATION_PARALLELISM_PER_DEST_SERVER;
 	if (priority == SERVER_KNOBS->PRIORITY_MANUAL_SHARD_SPLIT) {
-		res = std::min(static_cast<int>(res * SERVER_KNOBS->RELOCATION_WORK_AMPLIFIER_FOR_MANUAL_SHARD_SPLIT),
-		               WORK_FULL_UTILIZATION);
+		// We do not want many shards from manual split are moved to the same dest
+		return WORK_FULL_UTILIZATION / SERVER_KNOBS->MANUAL_SPLIT_RELOCATION_PARALLELISM_PER_DEST_SERVER;
+	} else {
+		// Work of moving a shard is even across destination servers
+		return WORK_FULL_UTILIZATION / SERVER_KNOBS->RELOCATION_PARALLELISM_PER_DEST_SERVER;
 	}
-	return res;
 }
 
 // Data movement's resource control: Do not overload servers used for the RelocateData

From 8e70903bd1f1ae252a647bc59d487687a98c07db Mon Sep 17 00:00:00 2001
From: Zhe Wang <zhe.wang@wustl.edu>
Date: Fri, 22 Sep 2023 14:07:01 -0700
Subject: [PATCH 263/468] address comments

---
 fdbserver/DataDistributionTracker.actor.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/fdbserver/DataDistributionTracker.actor.cpp b/fdbserver/DataDistributionTracker.actor.cpp
index 5aedc1c25ca..7a7cd34e2af 100644
--- a/fdbserver/DataDistributionTracker.actor.cpp
+++ b/fdbserver/DataDistributionTracker.actor.cpp
@@ -735,7 +735,7 @@ ACTOR Future<Void> shardEvaluator(DataDistributionTracker* self,
 		// Force to split
 		shouldSplit = true;
 		shouldMerge = false;
-		manualSplitKeys.insert(manualSplitKeys.end(), manualSplitPoints->begin(), manualSplitPoints->end());
+		manualSplitKeys = *manualSplitPoints;
 		manualSplitPoints->clear();
 	}
 

From 599dfa36090260cb635aba8adfb74487e856787c Mon Sep 17 00:00:00 2001
From: Zhe Wang <zhe.wang@wustl.edu>
Date: Fri, 22 Sep 2023 17:39:20 -0700
Subject: [PATCH 264/468] fix issues for corner cases

---
 fdbcli/RedistributeCommand.actor.cpp          | 21 +++++++-----
 fdbclient/ManagementAPI.actor.cpp             | 17 +++-------
 fdbclient/ManagementAPI.actor.h               |  2 +-
 fdbserver/ClusterController.actor.cpp         |  3 ++
 fdbserver/DataDistribution.actor.cpp          | 26 +++++++++-----
 .../workloads/ManualShardSplit.actor.cpp      | 34 +++++++++++++++++--
 flow/error_definitions.h                      |  1 +
 7 files changed, 72 insertions(+), 32 deletions(-)

diff --git a/fdbcli/RedistributeCommand.actor.cpp b/fdbcli/RedistributeCommand.actor.cpp
index 986414db3b2..03dd5e85ec3 100644
--- a/fdbcli/RedistributeCommand.actor.cpp
+++ b/fdbcli/RedistributeCommand.actor.cpp
@@ -30,7 +30,11 @@ ACTOR Future<bool> redistributeCommandActor(Reference<IClusterConnectionRecord>
 	if (tokens.size() == 3) {
 		Key begin = tokens[1];
 		Key end = tokens[2];
-		wait(moveShard(clusterFile, KeyRangeRef(begin, end)));
+		if (begin == end) {
+			printUsage(tokens[0]);
+			result = false;
+		}
+		wait(moveShard(clusterFile, KeyRangeRef(begin, end), /*timeoutSeconds=*/30));
 	} else {
 		printUsage(tokens[0]);
 		result = false;
@@ -38,11 +42,12 @@ ACTOR Future<bool> redistributeCommandActor(Reference<IClusterConnectionRecord>
 	return result;
 }
 
-CommandFactory RedistributeFactory("redistribute",
-                                   CommandHelp("redistribute [Begin] [End]",
-                                               "Redistribute data within the range<Begin, End> among the cluster.\n",
-                                               "Given an input range, say <b, d>, and we suppose the current FDB"
-                                               "internal shard boundary is [a, c), [c, d), [d, e),"
-                                               "our splitting algorithm splits the input range into [b, c) and [c, d)."
-                                               "Then the two ranges will be redistributed among the cluster.\n"));
+CommandFactory RedistributeFactory(
+    "redistribute",
+    CommandHelp("redistribute [Begin] [End]",
+                "Redistribute data of the range<Begin, End> among the cluster, where [Begin]!=[End]\n",
+                "Given an input range, say <b, d>, where b!=d and we suppose the current FDB"
+                "internal shard boundary is [a, c), [c, d), [d, e),"
+                "our splitting algorithm splits the input range into [b, c) and [c, d)."
+                "Then the two ranges will be redistributed among the cluster.\n"));
 } // namespace fdb_cli
diff --git a/fdbclient/ManagementAPI.actor.cpp b/fdbclient/ManagementAPI.actor.cpp
index e2ec60056c2..b3883390bd4 100644
--- a/fdbclient/ManagementAPI.actor.cpp
+++ b/fdbclient/ManagementAPI.actor.cpp
@@ -2342,21 +2342,14 @@ ACTOR Future<Void> forceRecovery(Reference<IClusterConnectionRecord> clusterFile
 	}
 }
 
-ACTOR Future<Void> moveShard(Reference<IClusterConnectionRecord> clusterFile, KeyRange range) {
+ACTOR Future<Void> moveShard(Reference<IClusterConnectionRecord> clusterFile, KeyRange range, double timeoutSeconds) {
 	state Reference<AsyncVar<Optional<ClusterInterface>>> clusterInterface(new AsyncVar<Optional<ClusterInterface>>);
 	state Future<Void> leaderMon = monitorLeader<ClusterInterface>(clusterFile, clusterInterface);
-
-	loop {
-		choose {
-			when(wait(
-			    clusterInterface->get().present()
-			        ? brokenPromiseToNever(clusterInterface->get().get().moveShard.getReply(MoveShardRequest(range)))
-			        : Never())) {
-				return Void();
-			}
-			when(wait(clusterInterface->onChange())) {}
-		}
+	while (!clusterInterface->get().present()) {
+		wait(clusterInterface->onChange());
 	}
+	wait(timeoutError(clusterInterface->get().get().moveShard.getReply(MoveShardRequest(range)), timeoutSeconds));
+	return Void();
 }
 
 ACTOR Future<Void> waitForPrimaryDC(Database cx, StringRef dcId) {
diff --git a/fdbclient/ManagementAPI.actor.h b/fdbclient/ManagementAPI.actor.h
index 7b456a41779..7adddc0eeab 100644
--- a/fdbclient/ManagementAPI.actor.h
+++ b/fdbclient/ManagementAPI.actor.h
@@ -148,7 +148,7 @@ ACTOR Future<int> setDDMode(Database cx, int mode);
 
 ACTOR Future<Void> forceRecovery(Reference<IClusterConnectionRecord> clusterFile, Standalone<StringRef> dcId);
 
-ACTOR Future<Void> moveShard(Reference<IClusterConnectionRecord> clusterFile, KeyRange range);
+ACTOR Future<Void> moveShard(Reference<IClusterConnectionRecord> clusterFile, KeyRange range, double timeoutSeconds);
 
 ACTOR Future<Void> printHealthyZone(Database cx);
 ACTOR Future<Void> setDDIgnoreRebalanceSwitch(Database cx, bool ignoreRebalance);
diff --git a/fdbserver/ClusterController.actor.cpp b/fdbserver/ClusterController.actor.cpp
index 3b387dec652..b01978c8b8b 100644
--- a/fdbserver/ClusterController.actor.cpp
+++ b/fdbserver/ClusterController.actor.cpp
@@ -2044,6 +2044,9 @@ ACTOR Future<Void> triggerMoveShards(ClusterControllerData* self, MoveShardReque
 		SplitShardReply rep = wait(self->db.serverInfo->get().distributor.get().distributorSplitRange.getReply(fReq));
 		req.reply.send(Void());
 	} catch (Error& e) {
+		if (e.code() == error_code_actor_cancelled) {
+			throw e;
+		}
 		req.reply.sendError(e);
 	}
 	return Void();
diff --git a/fdbserver/DataDistribution.actor.cpp b/fdbserver/DataDistribution.actor.cpp
index d32252620c7..0377ffb00b5 100644
--- a/fdbserver/DataDistribution.actor.cpp
+++ b/fdbserver/DataDistribution.actor.cpp
@@ -483,13 +483,15 @@ struct DataDistributorData : NonCopyable, ReferenceCounted<DataDistributorData>
 	Reference<EventCacheHolder> movingDataEventHolder;
 	Reference<EventCacheHolder> totalDataInFlightEventHolder;
 	Reference<EventCacheHolder> totalDataInFlightRemoteEventHolder;
+	bool initialized;
 
 	DataDistributorData(Reference<AsyncVar<ServerDBInfo> const> const& db, UID id)
 	  : dbInfo(db), ddId(id), teamCollection(nullptr),
 	    initialDDEventHolder(makeReference<EventCacheHolder>("InitialDD")),
 	    movingDataEventHolder(makeReference<EventCacheHolder>("MovingData")),
 	    totalDataInFlightEventHolder(makeReference<EventCacheHolder>("TotalDataInFlight")),
-	    totalDataInFlightRemoteEventHolder(makeReference<EventCacheHolder>("TotalDataInFlightRemote")) {}
+	    totalDataInFlightRemoteEventHolder(makeReference<EventCacheHolder>("TotalDataInFlightRemote")),
+	    initialized(false) {}
 };
 
 ACTOR Future<Void> monitorBatchLimitedTime(Reference<AsyncVar<ServerDBInfo> const> db, double* lastLimited) {
@@ -825,7 +827,7 @@ ACTOR Future<Void> dataDistribution(Reference<DataDistributorData> self,
 
 			actors.push_back(DDTeamCollection::printSnapshotTeamsInfo(primaryTeamCollection));
 			actors.push_back(yieldPromiseStream(output.getFuture(), input));
-
+			self->initialized = true;
 			wait(waitForAll(actors));
 			return Void();
 		} catch (Error& e) {
@@ -1258,12 +1260,14 @@ ACTOR Future<Void> ddGetMetrics(GetDataDistributorMetricsRequest req,
 
 ACTOR Future<Void> ddSplitRange(DistributorSplitRangeRequest req,
                                 PromiseStream<DistributorSplitRangeRequest> manualShardSplit) {
-	ErrorOr<SplitShardReply> result =
-	    wait(errorOr(brokenPromiseToNever(manualShardSplit.getReply(DistributorSplitRangeRequest(req.splitPoints)))));
-	if (result.isError()) {
-		req.reply.sendError(result.getError());
-	} else {
-		req.reply.send(result.get());
+	try {
+		SplitShardReply res = wait(manualShardSplit.getReply(DistributorSplitRangeRequest(req.splitPoints)));
+		req.reply.send(res);
+	} catch (Error& e) {
+		if (e.code() == error_code_actor_cancelled) {
+			throw e;
+		}
+		req.reply.sendError(e);
 	}
 	return Void();
 }
@@ -1310,7 +1314,11 @@ ACTOR Future<Void> dataDistributor(DataDistributorInterface di, Reference<AsyncV
 				actors.add(ddExclusionSafetyCheck(exclCheckReq, self, cx));
 			}
 			when(DistributorSplitRangeRequest splitRangeReq = waitNext(di.distributorSplitRange.getFuture())) {
-				actors.add(ddSplitRange(splitRangeReq, manualShardSplit));
+				if (self->initialized) {
+					actors.add(ddSplitRange(splitRangeReq, manualShardSplit));
+				} else {
+					splitRangeReq.reply.sendError(dd_not_initialized());
+				}
 			}
 		}
 	} catch (Error& err) {
diff --git a/fdbserver/workloads/ManualShardSplit.actor.cpp b/fdbserver/workloads/ManualShardSplit.actor.cpp
index ad85015e889..e0f9a2d06de 100644
--- a/fdbserver/workloads/ManualShardSplit.actor.cpp
+++ b/fdbserver/workloads/ManualShardSplit.actor.cpp
@@ -52,8 +52,38 @@ struct ManualShardSplitWorkload : TestWorkload {
 		                                 { "TestKeyF"_sr, "TestValueF"_sr } });
 
 		Version ver = wait(self->populateData(self, cx, &kvs));
-		wait(moveShard(cx->getConnectionRecord(), KeyRangeRef("TestKeyA"_sr, "TestKeyD"_sr)));
-		wait(moveShard(cx->getConnectionRecord(), KeyRangeRef("TestKeyB"_sr, "TestKeyC"_sr)));
+		TraceEvent("ManualShardSplitPopulateDataDone");
+		loop {
+			try {
+				wait(moveShard(
+				    cx->getConnectionRecord(), KeyRangeRef("TestKeyA"_sr, "TestKeyD"_sr), /*timeoutSeconds=*/30));
+				break;
+			} catch (Error& e) {
+				if (e.code() == error_code_dd_not_initialized || e.code() == error_code_timed_out ||
+				    e.code() == error_code_broken_promise) {
+					wait(delay(1.0));
+					continue;
+				} else {
+					throw e;
+				}
+			}
+		}
+		loop {
+			try {
+				wait(moveShard(
+				    cx->getConnectionRecord(), KeyRangeRef("TestKeyB"_sr, "TestKeyC"_sr), /*timeoutSeconds=*/30));
+				break;
+			} catch (Error& e) {
+				if (e.code() == error_code_dd_not_initialized || e.code() == error_code_timed_out ||
+				    e.code() == error_code_broken_promise) {
+					wait(delay(1.0));
+					continue;
+				} else {
+					throw e;
+				}
+			}
+		}
+		TraceEvent("ManualShardSplitTestComplete");
 		return Void();
 	}
 
diff --git a/flow/error_definitions.h b/flow/error_definitions.h
index 5fb0cafefaf..ed717956ec1 100755
--- a/flow/error_definitions.h
+++ b/flow/error_definitions.h
@@ -95,6 +95,7 @@ ERROR( page_encoding_not_supported, 1071, "Page encoding type is not supported o
 ERROR( page_decoding_failed, 1072, "Page content decoding failed" )
 ERROR( unexpected_encoding_type, 1073, "Page content decoding failed" )
 ERROR( encryption_key_not_found, 1074, "Encryption key not found" )
+ERROR( dd_not_initialized, 1075, "Data distribution has not initialized")
 ERROR( unknown_storage_engine, 1082, "Storage engine type is not recognized." )
 
 ERROR( broken_promise, 1100, "Broken promise" )

From 229b23727d50a9c7e280fe3d21239a1b2af2f577 Mon Sep 17 00:00:00 2001
From: Zhe Wang <zhe.wang@wustl.edu>
Date: Sat, 23 Sep 2023 13:58:18 -0700
Subject: [PATCH 265/468] add more trace events

---
 fdbcli/RedistributeCommand.actor.cpp          |  2 +-
 fdbclient/ManagementAPI.actor.cpp             | 15 +++++++++++----
 fdbclient/ManagementAPI.actor.h               |  2 +-
 fdbserver/ClusterController.actor.cpp         |  5 +++++
 fdbserver/DataDistribution.actor.cpp          | 17 ++++++++++++++---
 fdbserver/DataDistributionQueue.actor.cpp     | 17 +++++++++++------
 fdbserver/DataDistributionTracker.actor.cpp   | 19 +++++++++++++------
 .../workloads/ManualShardSplit.actor.cpp      | 10 ++++------
 flow/error_definitions.h                      |  2 +-
 9 files changed, 61 insertions(+), 28 deletions(-)

diff --git a/fdbcli/RedistributeCommand.actor.cpp b/fdbcli/RedistributeCommand.actor.cpp
index 03dd5e85ec3..ee782493402 100644
--- a/fdbcli/RedistributeCommand.actor.cpp
+++ b/fdbcli/RedistributeCommand.actor.cpp
@@ -34,7 +34,7 @@ ACTOR Future<bool> redistributeCommandActor(Reference<IClusterConnectionRecord>
 			printUsage(tokens[0]);
 			result = false;
 		}
-		wait(moveShard(clusterFile, KeyRangeRef(begin, end), /*timeoutSeconds=*/30));
+		wait(redistribute(clusterFile, KeyRangeRef(begin, end), /*timeoutSeconds=*/30));
 	} else {
 		printUsage(tokens[0]);
 		result = false;
diff --git a/fdbclient/ManagementAPI.actor.cpp b/fdbclient/ManagementAPI.actor.cpp
index b3883390bd4..06ab0d6ab40 100644
--- a/fdbclient/ManagementAPI.actor.cpp
+++ b/fdbclient/ManagementAPI.actor.cpp
@@ -2342,13 +2342,20 @@ ACTOR Future<Void> forceRecovery(Reference<IClusterConnectionRecord> clusterFile
 	}
 }
 
-ACTOR Future<Void> moveShard(Reference<IClusterConnectionRecord> clusterFile, KeyRange range, double timeoutSeconds) {
+ACTOR Future<Void> redistribute(Reference<IClusterConnectionRecord> clusterFile,
+                                KeyRange range,
+                                double timeoutSeconds) {
 	state Reference<AsyncVar<Optional<ClusterInterface>>> clusterInterface(new AsyncVar<Optional<ClusterInterface>>);
 	state Future<Void> leaderMon = monitorLeader<ClusterInterface>(clusterFile, clusterInterface);
-	while (!clusterInterface->get().present()) {
-		wait(clusterInterface->onChange());
+	try {
+		while (!clusterInterface->get().present()) {
+			wait(clusterInterface->onChange());
+		}
+		wait(timeoutError(clusterInterface->get().get().moveShard.getReply(MoveShardRequest(range)), timeoutSeconds));
+	} catch (Error& e) {
+		TraceEvent(SevWarn, "RedistributeFailedToTrigger").errorUnsuppressed(e).detail("InputRange", range);
+		throw manual_shard_split_failed();
 	}
-	wait(timeoutError(clusterInterface->get().get().moveShard.getReply(MoveShardRequest(range)), timeoutSeconds));
 	return Void();
 }
 
diff --git a/fdbclient/ManagementAPI.actor.h b/fdbclient/ManagementAPI.actor.h
index 7adddc0eeab..5261ec1b9bb 100644
--- a/fdbclient/ManagementAPI.actor.h
+++ b/fdbclient/ManagementAPI.actor.h
@@ -148,7 +148,7 @@ ACTOR Future<int> setDDMode(Database cx, int mode);
 
 ACTOR Future<Void> forceRecovery(Reference<IClusterConnectionRecord> clusterFile, Standalone<StringRef> dcId);
 
-ACTOR Future<Void> moveShard(Reference<IClusterConnectionRecord> clusterFile, KeyRange range, double timeoutSeconds);
+ACTOR Future<Void> redistribute(Reference<IClusterConnectionRecord> clusterFile, KeyRange range, double timeoutSeconds);
 
 ACTOR Future<Void> printHealthyZone(Database cx);
 ACTOR Future<Void> setDDIgnoreRebalanceSwitch(Database cx, bool ignoreRebalance);
diff --git a/fdbserver/ClusterController.actor.cpp b/fdbserver/ClusterController.actor.cpp
index b01978c8b8b..74d8a5c9373 100644
--- a/fdbserver/ClusterController.actor.cpp
+++ b/fdbserver/ClusterController.actor.cpp
@@ -2040,13 +2040,18 @@ ACTOR Future<Void> triggerMoveShards(ClusterControllerData* self, MoveShardReque
 		       !self->db.serverInfo->get().distributor.present()) {
 			wait(self->db.serverInfo->onChange());
 		}
+		TraceEvent(SevInfo, "ManualShardSplitCCReceived", self->id).detail("InputRange", req.shard);
 		DistributorSplitRangeRequest fReq({ req.shard.begin, req.shard.end });
 		SplitShardReply rep = wait(self->db.serverInfo->get().distributor.get().distributorSplitRange.getReply(fReq));
 		req.reply.send(Void());
+		TraceEvent(SevInfo, "ManualShardSplitCCTriggered", self->id).detail("InputRange", req.shard);
 	} catch (Error& e) {
 		if (e.code() == error_code_actor_cancelled) {
 			throw e;
 		}
+		TraceEvent(SevWarn, "ManualShardSplitCCFailedToTrigger", self->id)
+		    .errorUnsuppressed(e)
+		    .detail("InputRange", req.shard);
 		req.reply.sendError(e);
 	}
 	return Void();
diff --git a/fdbserver/DataDistribution.actor.cpp b/fdbserver/DataDistribution.actor.cpp
index 0377ffb00b5..1a67d426999 100644
--- a/fdbserver/DataDistribution.actor.cpp
+++ b/fdbserver/DataDistribution.actor.cpp
@@ -828,6 +828,7 @@ ACTOR Future<Void> dataDistribution(Reference<DataDistributorData> self,
 			actors.push_back(DDTeamCollection::printSnapshotTeamsInfo(primaryTeamCollection));
 			actors.push_back(yieldPromiseStream(output.getFuture(), input));
 			self->initialized = true;
+
 			wait(waitForAll(actors));
 			return Void();
 		} catch (Error& e) {
@@ -1259,14 +1260,21 @@ ACTOR Future<Void> ddGetMetrics(GetDataDistributorMetricsRequest req,
 }
 
 ACTOR Future<Void> ddSplitRange(DistributorSplitRangeRequest req,
-                                PromiseStream<DistributorSplitRangeRequest> manualShardSplit) {
+                                PromiseStream<DistributorSplitRangeRequest> manualShardSplit,
+                                UID ddId) {
 	try {
+		TraceEvent(SevInfo, "ManualShardSplitDDReceived", ddId).detail("InputSplitPoints", req.splitPoints);
 		SplitShardReply res = wait(manualShardSplit.getReply(DistributorSplitRangeRequest(req.splitPoints)));
 		req.reply.send(res);
+		TraceEvent(SevInfo, "ManualShardSplitDDTriggered", ddId).detail("InputSplitPoints", req.splitPoints);
 	} catch (Error& e) {
 		if (e.code() == error_code_actor_cancelled) {
 			throw e;
 		}
+		TraceEvent(SevWarn, "ManualShardSplitDDFailedToTrigger", ddId)
+		    .errorUnsuppressed(e)
+		    .detail("Reason", "DDSplitRangeError")
+		    .detail("InputSplitPoints", req.splitPoints);
 		req.reply.sendError(e);
 	}
 	return Void();
@@ -1315,9 +1323,12 @@ ACTOR Future<Void> dataDistributor(DataDistributorInterface di, Reference<AsyncV
 			}
 			when(DistributorSplitRangeRequest splitRangeReq = waitNext(di.distributorSplitRange.getFuture())) {
 				if (self->initialized) {
-					actors.add(ddSplitRange(splitRangeReq, manualShardSplit));
+					actors.add(ddSplitRange(splitRangeReq, manualShardSplit, di.id()));
 				} else {
-					splitRangeReq.reply.sendError(dd_not_initialized());
+					splitRangeReq.reply.sendError(manual_shard_split_failed());
+					TraceEvent(SevWarn, "ManualShardSplitDDFailedToTrigger", di.id())
+					    .detail("Reason", "DDNotInitialized")
+					    .detail("InputSplitPoints", splitRangeReq.splitPoints);
 				}
 			}
 		}
diff --git a/fdbserver/DataDistributionQueue.actor.cpp b/fdbserver/DataDistributionQueue.actor.cpp
index 211ec483200..d5211e8aca0 100644
--- a/fdbserver/DataDistributionQueue.actor.cpp
+++ b/fdbserver/DataDistributionQueue.actor.cpp
@@ -977,7 +977,9 @@ struct DDQueueData {
 			if (overlappingInFlight) {
 				// logRelocation( rd, "SkippingOverlappingInFlight" );
 				if (rd.priority == SERVER_KNOBS->PRIORITY_MANUAL_SHARD_SPLIT) {
-					TraceEvent(SevWarn, "ManualShardSplitSkipForOverlappingInFlight").detail("Range", rd.keys);
+					TraceEvent(SevWarn, "ManualShardSplitDDQueueSkipRange")
+					    .detail("Range", rd.keys)
+					    .detail("Reason", "OverlappingInFlight");
 				}
 				continue;
 			}
@@ -999,7 +1001,9 @@ struct DDQueueData {
 			if (!canLaunchSrc(rd, teamSize, singleRegionTeamSize, busymap, cancellableRelocations)) {
 				// logRelocation( rd, "SkippingQueuedRelocation" );
 				if (rd.priority == SERVER_KNOBS->PRIORITY_MANUAL_SHARD_SPLIT) {
-					TraceEvent(SevWarn, "ManualShardSplitSkipForCannotLaunchSrc").detail("Range", rd.keys);
+					TraceEvent(SevWarn, "ManualShardSplitDDQueueSkipRange")
+					    .detail("Range", rd.keys)
+					    .detail("Reason", "CannotLaunchSrc");
 				}
 				continue;
 			}
@@ -1816,7 +1820,8 @@ ACTOR Future<Void> dataDistributionQueue(Database cx,
 			choose {
 				when(RelocateShard rs = waitNext(self.input)) {
 					if (rs.priority == SERVER_KNOBS->PRIORITY_MANUAL_SHARD_SPLIT) {
-						TraceEvent(SevInfo, "ManualShardSplitStart", self.distributorId).detail("Range", rs.keys);
+						TraceEvent(SevInfo, "ManualShardSplitDDQueueStart", self.distributorId)
+						    .detail("Range", rs.keys);
 					}
 					bool wasEmpty = serversToLaunchFrom.empty();
 					self.queueRelocation(rs, serversToLaunchFrom);
@@ -1830,7 +1835,7 @@ ACTOR Future<Void> dataDistributionQueue(Database cx,
 				}
 				when(RelocateData results = waitNext(self.fetchSourceServersComplete.getFuture())) {
 					if (results.priority == SERVER_KNOBS->PRIORITY_MANUAL_SHARD_SPLIT) {
-						TraceEvent(SevInfo, "ManualShardSplitFetchSourceComplete").detail("Range", results.keys);
+						TraceEvent(SevInfo, "ManualShardSplitDDQueueFetchSourceComplete").detail("Range", results.keys);
 					}
 					// This when is triggered by queueRelocation() which is triggered by sending self.input
 					self.completeSourceFetch(results);
@@ -1838,7 +1843,7 @@ ACTOR Future<Void> dataDistributionQueue(Database cx,
 				}
 				when(RelocateData done = waitNext(self.dataTransferComplete.getFuture())) {
 					if (done.priority == SERVER_KNOBS->PRIORITY_MANUAL_SHARD_SPLIT) {
-						TraceEvent(SevInfo, "ManualShardSplitDataTransferComplete").detail("Range", done.keys);
+						TraceEvent(SevInfo, "ManualShardSplitDDQueueDataTransferComplete").detail("Range", done.keys);
 					}
 					complete(done, self.busymap, self.destBusymap);
 					if (serversToLaunchFrom.empty() && !done.src.empty())
@@ -1847,7 +1852,7 @@ ACTOR Future<Void> dataDistributionQueue(Database cx,
 				}
 				when(RelocateData done = waitNext(self.relocationComplete.getFuture())) {
 					if (done.priority == SERVER_KNOBS->PRIORITY_MANUAL_SHARD_SPLIT) {
-						TraceEvent(SevInfo, "ManualShardSplitRelocationComplete").detail("Range", done.keys);
+						TraceEvent(SevInfo, "ManualShardSplitDDQueueRelocationComplete").detail("Range", done.keys);
 					}
 					self.activeRelocations--;
 					self.finishRelocation(done.priority, done.healthPriority);
diff --git a/fdbserver/DataDistributionTracker.actor.cpp b/fdbserver/DataDistributionTracker.actor.cpp
index 7a7cd34e2af..dd5b98c5151 100644
--- a/fdbserver/DataDistributionTracker.actor.cpp
+++ b/fdbserver/DataDistributionTracker.actor.cpp
@@ -493,7 +493,7 @@ ACTOR Future<Void> shardSplitter(DataDistributionTracker* self,
 			splitKeys.push_back_deep(splitKeys.arena(), manualSplitKey);
 		}
 		splitKeys.push_back_deep(splitKeys.arena(), keys.end);
-		TraceEvent e(SevInfo, "ManualShardSplit", self->distributorId);
+		TraceEvent e(SevInfo, "ManualShardSplitShardSplitterStart", self->distributorId);
 		e.setMaxEventLength(20000);
 		e.detail("Begin", keys.begin);
 		e.detail("End", keys.end);
@@ -1017,17 +1017,20 @@ ACTOR Future<Void> dataDistributionTracker(Reference<InitialDataDistribution> in
 			}
 			when(DistributorSplitRangeRequest req = waitNext(manualShardSplit.getFuture())) {
 				if (req.splitPoints.size() != 2) {
-					TraceEvent(SevWarn, "DDSplitRangeWrongInputSize", self.distributorId)
+					TraceEvent(SevWarn, "ManualShardSplitDDFailedToTrigger", self.distributorId)
+					    .detail("Reason", "Wrong number of split points")
 					    .detail("SplitPoints", req.splitPoints);
-					req.reply.sendError(not_implemented());
+					req.reply.sendError(manual_shard_split_failed());
 					continue;
 				} else if (req.splitPoints[0] == req.splitPoints[1]) {
-					TraceEvent(SevWarn, "DDSplitRangeEmptyInputRange", self.distributorId)
+					TraceEvent(SevWarn, "ManualShardSplitDDFailedToTrigger", self.distributorId)
+					    .detail("Reason", "Two split points are same")
 					    .detail("SplitPoints", req.splitPoints);
-					req.reply.sendError(not_implemented());
+					req.reply.sendError(manual_shard_split_failed());
 					continue;
 				}
-				TraceEvent(SevInfo, "DDSplitRangeStart", self.distributorId).detail("SplitPoints", req.splitPoints);
+				TraceEvent(SevInfo, "ManualShardSplitDDStart", self.distributorId)
+				    .detail("SplitPoints", req.splitPoints);
 				KeyRange inputRange = req.splitPoints[0] < req.splitPoints[1]
 				                          ? KeyRangeRef(req.splitPoints[0], req.splitPoints[1])
 				                          : KeyRangeRef(req.splitPoints[1], req.splitPoints[0]);
@@ -1042,9 +1045,13 @@ ACTOR Future<Void> dataDistributionTracker(Reference<InitialDataDistribution> in
 					if (triggerManualSplitShard) {
 						// Trigger split for this shard
 						it->value().shouldManualSplit->set(true);
+						TraceEvent(SevInfo, "ManualShardSplitDDTriggerShardSplit", self.distributorId)
+						    .detail("Shard", it->range());
 					} else {
 						// Trigger data move for this shard
 						self.output.send(RelocateShard(it->range(), SERVER_KNOBS->PRIORITY_MANUAL_SHARD_SPLIT));
+						TraceEvent(SevInfo, "ManualShardSplitDDTriggerShardMove", self.distributorId)
+						    .detail("Shard", it->range());
 					}
 				}
 				req.reply.send(SplitShardReply());
diff --git a/fdbserver/workloads/ManualShardSplit.actor.cpp b/fdbserver/workloads/ManualShardSplit.actor.cpp
index e0f9a2d06de..024fb4c0b7f 100644
--- a/fdbserver/workloads/ManualShardSplit.actor.cpp
+++ b/fdbserver/workloads/ManualShardSplit.actor.cpp
@@ -55,12 +55,11 @@ struct ManualShardSplitWorkload : TestWorkload {
 		TraceEvent("ManualShardSplitPopulateDataDone");
 		loop {
 			try {
-				wait(moveShard(
+				wait(redistribute(
 				    cx->getConnectionRecord(), KeyRangeRef("TestKeyA"_sr, "TestKeyD"_sr), /*timeoutSeconds=*/30));
 				break;
 			} catch (Error& e) {
-				if (e.code() == error_code_dd_not_initialized || e.code() == error_code_timed_out ||
-				    e.code() == error_code_broken_promise) {
+				if (e.code() == error_code_manual_shard_split_failed) {
 					wait(delay(1.0));
 					continue;
 				} else {
@@ -70,12 +69,11 @@ struct ManualShardSplitWorkload : TestWorkload {
 		}
 		loop {
 			try {
-				wait(moveShard(
+				wait(redistribute(
 				    cx->getConnectionRecord(), KeyRangeRef("TestKeyB"_sr, "TestKeyC"_sr), /*timeoutSeconds=*/30));
 				break;
 			} catch (Error& e) {
-				if (e.code() == error_code_dd_not_initialized || e.code() == error_code_timed_out ||
-				    e.code() == error_code_broken_promise) {
+				if (e.code() == error_code_manual_shard_split_failed) {
 					wait(delay(1.0));
 					continue;
 				} else {
diff --git a/flow/error_definitions.h b/flow/error_definitions.h
index ed717956ec1..6de15721c5b 100755
--- a/flow/error_definitions.h
+++ b/flow/error_definitions.h
@@ -95,8 +95,8 @@ ERROR( page_encoding_not_supported, 1071, "Page encoding type is not supported o
 ERROR( page_decoding_failed, 1072, "Page content decoding failed" )
 ERROR( unexpected_encoding_type, 1073, "Page content decoding failed" )
 ERROR( encryption_key_not_found, 1074, "Encryption key not found" )
-ERROR( dd_not_initialized, 1075, "Data distribution has not initialized")
 ERROR( unknown_storage_engine, 1082, "Storage engine type is not recognized." )
+ERROR( manual_shard_split_failed, 1083, "Manual shard split is failed to trigger")
 
 ERROR( broken_promise, 1100, "Broken promise" )
 ERROR( operation_cancelled, 1101, "Asynchronous operation cancelled" )

From fc4c7cb1c9fe43907283389300c4a1863fde1f58 Mon Sep 17 00:00:00 2001
From: Zhe Wang <zhe.wang@wustl.edu>
Date: Tue, 26 Sep 2023 14:28:24 -0700
Subject: [PATCH 266/468] avoid duplicated split points in manual shard split

---
 fdbserver/DataDistribution.actor.h            |  4 +-
 fdbserver/DataDistributionTracker.actor.cpp   | 77 +++++++++++--------
 .../workloads/ManualShardSplit.actor.cpp      | 58 +++++++-------
 3 files changed, 78 insertions(+), 61 deletions(-)

diff --git a/fdbserver/DataDistribution.actor.h b/fdbserver/DataDistribution.actor.h
index e590dfd3d86..fffcd3853b5 100644
--- a/fdbserver/DataDistribution.actor.h
+++ b/fdbserver/DataDistribution.actor.h
@@ -245,8 +245,8 @@ struct ShardTrackedData {
 	Future<Void> trackShard;
 	Future<Void> trackBytes;
 	Reference<AsyncVar<Optional<ShardMetrics>>> stats;
-	Reference<AsyncVar<bool>> shouldManualSplit;
-	std::shared_ptr<std::vector<Key>> manualSplitPoints;
+	Reference<AsyncVar<Void>> manualSplitTrigger;
+	std::shared_ptr<std::set<Key>> manualSplitPoints;
 };
 
 ACTOR Future<Void> dataDistributionTracker(Reference<InitialDataDistribution> initData,
diff --git a/fdbserver/DataDistributionTracker.actor.cpp b/fdbserver/DataDistributionTracker.actor.cpp
index dd5b98c5151..7be7ea40038 100644
--- a/fdbserver/DataDistributionTracker.actor.cpp
+++ b/fdbserver/DataDistributionTracker.actor.cpp
@@ -478,30 +478,38 @@ ACTOR Future<Void> shardSplitter(DataDistributionTracker* self,
                                  KeyRange keys,
                                  Reference<AsyncVar<Optional<ShardMetrics>>> shardSize,
                                  ShardSizeBounds shardBounds,
-                                 std::vector<Key> manualSplitKeys) {
+                                 std::shared_ptr<std::set<Key>> manualSplitPoints) {
 	state StorageMetrics metrics = shardSize->get().get().metrics;
 	state BandwidthStatus bandwidthStatus = getBandwidthStatus(metrics);
-	state bool doManualSplit = !manualSplitKeys.empty();
+	state bool doManualSplit = !manualSplitPoints->empty();
 
 	// Split
 	TEST(true); // shard to be split
 
 	state Standalone<VectorRef<KeyRef>> splitKeys;
 	if (doManualSplit) {
-		splitKeys.push_back_deep(splitKeys.arena(), keys.begin);
-		for (const auto& manualSplitKey : manualSplitKeys) {
-			splitKeys.push_back_deep(splitKeys.arena(), manualSplitKey);
+		// Check if all split points are with in the shard
+		for (const auto& manualSplitKey : *manualSplitPoints) {
+			ASSERT_WE_THINK(manualSplitKey < keys.end && manualSplitKey > keys.begin);
 		}
-		splitKeys.push_back_deep(splitKeys.arena(), keys.end);
-		TraceEvent e(SevInfo, "ManualShardSplitShardSplitterStart", self->distributorId);
-		e.setMaxEventLength(20000);
-		e.detail("Begin", keys.begin);
-		e.detail("End", keys.end);
-		int i = 0;
-		for (const auto& splitKey : splitKeys) {
-			e.detail("SplitPoint" + std::to_string(i), printable(splitKey).c_str());
-			i++;
+		// Add two ends to the split points
+		manualSplitPoints->insert(keys.begin);
+		manualSplitPoints->insert(keys.end);
+		// Sort split points and add to splitKeys
+		std::vector<Key> manualSplitPointList;
+		for (const auto& manualSplitKey : *manualSplitPoints) {
+			manualSplitPointList.push_back(manualSplitKey);
 		}
+		std::sort(manualSplitPointList.begin(), manualSplitPointList.end(), std::less<Key>());
+		for (const auto& manualSplitPoint : manualSplitPointList) {
+			splitKeys.push_back_deep(splitKeys.arena(), manualSplitPoint);
+		}
+		TraceEvent(SevInfo, "ManualShardSplitReadSplitPointInShard", self->distributorId)
+		    .detail("Points", describe(*manualSplitPoints))
+		    .detail("Size", manualSplitPoints->size())
+		    .detail("Shard", keys);
+		// Cleanup
+		manualSplitPoints->clear();
 	} else {
 		StorageMetrics splitMetrics;
 		splitMetrics.bytes = shardBounds.max.bytes / 2;
@@ -714,10 +722,10 @@ ACTOR Future<Void> shardEvaluator(DataDistributionTracker* self,
                                   KeyRange keys,
                                   Reference<AsyncVar<Optional<ShardMetrics>>> shardSize,
                                   Reference<HasBeenTrueFor> wantsToMerge,
-                                  Reference<AsyncVar<bool>> shouldManualSplit,
-                                  std::shared_ptr<std::vector<Key>> manualSplitPoints) {
+                                  Reference<AsyncVar<Void>> manualSplitTrigger,
+                                  std::shared_ptr<std::set<Key>> manualSplitPoints) {
 	Future<Void> onChange =
-	    shardSize->onChange() || yieldedFuture(self->maxShardSize->onChange()) || shouldManualSplit->onChange();
+	    shardSize->onChange() || yieldedFuture(self->maxShardSize->onChange()) || manualSplitTrigger->onChange();
 
 	// There are the bounds inside of which we are happy with the shard size.
 	// getShardSizeBounds() will allways have shardBounds.min.bytes == 0 for shards that start at allKeys.begin,
@@ -730,13 +738,10 @@ ACTOR Future<Void> shardEvaluator(DataDistributionTracker* self,
 	                   (bandwidthStatus == BandwidthStatusHigh && keys.begin < keyServersKeys.begin);
 	bool shouldMerge = stats.bytes < shardBounds.min.bytes && bandwidthStatus == BandwidthStatusLow;
 
-	std::vector<Key> manualSplitKeys;
-	if (shouldManualSplit->get() == true) {
-		// Force to split
+	if (!manualSplitPoints->empty()) {
+		// Have points to split, then force to split
 		shouldSplit = true;
 		shouldMerge = false;
-		manualSplitKeys = *manualSplitPoints;
-		manualSplitPoints->clear();
 	}
 
 	// Every invocation must set this or clear it
@@ -770,7 +775,7 @@ ACTOR Future<Void> shardEvaluator(DataDistributionTracker* self,
 		onChange = onChange || shardMerger(self, keys, shardSize);
 	}
 	if (shouldSplit) {
-		onChange = onChange || shardSplitter(self, keys, shardSize, shardBounds, manualSplitKeys);
+		onChange = onChange || shardSplitter(self, keys, shardSize, shardBounds, manualSplitPoints);
 	}
 
 	wait(onChange);
@@ -780,8 +785,8 @@ ACTOR Future<Void> shardEvaluator(DataDistributionTracker* self,
 ACTOR Future<Void> shardTracker(DataDistributionTracker::SafeAccessor self,
                                 KeyRange keys,
                                 Reference<AsyncVar<Optional<ShardMetrics>>> shardSize,
-                                Reference<AsyncVar<bool>> shouldManualSplit,
-                                std::shared_ptr<std::vector<Key>> manualSplitPoints) {
+                                Reference<AsyncVar<Void>> manualSplitTrigger,
+                                std::shared_ptr<std::set<Key>> manualSplitPoints) {
 	wait(yieldedFuture(self()->readyToStart.getFuture()));
 
 	if (!shardSize->get().present())
@@ -807,7 +812,7 @@ ACTOR Future<Void> shardTracker(DataDistributionTracker::SafeAccessor self,
 	try {
 		loop {
 			// Use the current known size to check for (and start) splits and merges.
-			wait(shardEvaluator(self(), keys, shardSize, wantsToMerge, shouldManualSplit, manualSplitPoints));
+			wait(shardEvaluator(self(), keys, shardSize, wantsToMerge, manualSplitTrigger, manualSplitPoints));
 
 			// We could have a lot of actors being released from the previous wait at the same time. Immediately calling
 			// delay(0) mitigates the resulting SlowTask
@@ -832,8 +837,8 @@ void restartShardTrackers(DataDistributionTracker* self, KeyRangeRef keys, Optio
 		}
 
 		auto shardMetrics = makeReference<AsyncVar<Optional<ShardMetrics>>>();
-		auto shouldManualSplit = makeReference<AsyncVar<bool>>(false);
-		auto manualSplitPoints = std::make_shared<std::vector<Key>>();
+		auto manualSplitTrigger = makeReference<AsyncVar<Void>>();
+		auto manualSplitPoints = std::make_shared<std::set<Key>>();
 
 		// For the case where the new tracker will take over at the boundaries of current shard(s)
 		//  we can use the old size if it is available. This will be the case when merging shards.
@@ -849,10 +854,13 @@ void restartShardTrackers(DataDistributionTracker* self, KeyRangeRef keys, Optio
 
 		ShardTrackedData data;
 		data.stats = shardMetrics;
-		data.shouldManualSplit = shouldManualSplit;
+		data.manualSplitTrigger = manualSplitTrigger;
 		data.manualSplitPoints = manualSplitPoints;
-		data.trackShard = shardTracker(
-		    DataDistributionTracker::SafeAccessor(self), ranges[i], shardMetrics, shouldManualSplit, manualSplitPoints);
+		data.trackShard = shardTracker(DataDistributionTracker::SafeAccessor(self),
+		                               ranges[i],
+		                               shardMetrics,
+		                               manualSplitTrigger,
+		                               manualSplitPoints);
 		data.trackBytes = trackShardMetrics(DataDistributionTracker::SafeAccessor(self), ranges[i], shardMetrics);
 		self->shards.insert(ranges[i], data);
 	}
@@ -1039,12 +1047,15 @@ ACTOR Future<Void> dataDistributionTracker(Reference<InitialDataDistribution> in
 					for (const auto& splitPoint : req.splitPoints) {
 						if (it->range().begin < splitPoint && splitPoint < it->range().end) {
 							triggerManualSplitShard = true;
-							it->value().manualSplitPoints->push_back(splitPoint);
+							it->value().manualSplitPoints->insert(splitPoint);
+							TraceEvent(SevInfo, "ManualShardSplitAddSplitPointToShard", self.distributorId)
+							    .detail("Point", splitPoint)
+							    .detail("Shard", it->range());
 						}
 					}
 					if (triggerManualSplitShard) {
 						// Trigger split for this shard
-						it->value().shouldManualSplit->set(true);
+						it->value().manualSplitTrigger->trigger();
 						TraceEvent(SevInfo, "ManualShardSplitDDTriggerShardSplit", self.distributorId)
 						    .detail("Shard", it->range());
 					} else {
diff --git a/fdbserver/workloads/ManualShardSplit.actor.cpp b/fdbserver/workloads/ManualShardSplit.actor.cpp
index 024fb4c0b7f..d6a0b5837ba 100644
--- a/fdbserver/workloads/ManualShardSplit.actor.cpp
+++ b/fdbserver/workloads/ManualShardSplit.actor.cpp
@@ -50,36 +50,42 @@ struct ManualShardSplitWorkload : TestWorkload {
 		                                 { "TestKeyD"_sr, "TestValueD"_sr },
 		                                 { "TestKeyE"_sr, "TestValueE"_sr },
 		                                 { "TestKeyF"_sr, "TestValueF"_sr } });
-
 		Version ver = wait(self->populateData(self, cx, &kvs));
 		TraceEvent("ManualShardSplitPopulateDataDone");
-		loop {
-			try {
-				wait(redistribute(
-				    cx->getConnectionRecord(), KeyRangeRef("TestKeyA"_sr, "TestKeyD"_sr), /*timeoutSeconds=*/30));
-				break;
-			} catch (Error& e) {
-				if (e.code() == error_code_manual_shard_split_failed) {
-					wait(delay(1.0));
-					continue;
-				} else {
-					throw e;
-				}
-			}
-		}
-		loop {
-			try {
-				wait(redistribute(
-				    cx->getConnectionRecord(), KeyRangeRef("TestKeyB"_sr, "TestKeyC"_sr), /*timeoutSeconds=*/30));
-				break;
-			} catch (Error& e) {
-				if (e.code() == error_code_manual_shard_split_failed) {
-					wait(delay(1.0));
-					continue;
-				} else {
-					throw e;
+		state std::vector<KeyRangeRef> ranges = {
+			KeyRangeRef("TestKeyA"_sr, "TestKeyB"_sr), KeyRangeRef("TestKeyB"_sr, "TestKeyC"_sr),
+			KeyRangeRef("TestKeyC"_sr, "TestKeyD"_sr), KeyRangeRef("TestKeyD"_sr, "TestKeyE"_sr),
+			KeyRangeRef("TestKeyE"_sr, "TestKeyF"_sr), KeyRangeRef("TestKeyA"_sr, "TestKeyC"_sr),
+			KeyRangeRef("TestKeyA"_sr, "TestKeyD"_sr), KeyRangeRef("TestKeyA"_sr, "TestKeyE"_sr),
+			KeyRangeRef("TestKeyA"_sr, "TestKeyF"_sr), KeyRangeRef("TestKeyB"_sr, "TestKeyD"_sr),
+			KeyRangeRef("TestKeyB"_sr, "TestKeyE"_sr), KeyRangeRef("TestKeyB"_sr, "TestKeyF"_sr),
+			KeyRangeRef("TestKeyC"_sr, "TestKeyE"_sr), KeyRangeRef("TestKeyC"_sr, "TestKeyF"_sr),
+			KeyRangeRef("TestKeyD"_sr, "TestKeyF"_sr), KeyRangeRef("TestKeyE"_sr, "TestKeyE"_sr),
+			KeyRangeRef("TestKeyF"_sr, "TestKeyF"_sr),
+		};
+		state int count = 0;
+		while (count < 100) {
+			state KeyRange selectRange = deterministicRandom()->randomChoice(ranges);
+			loop {
+				try {
+					wait(redistribute(cx->getConnectionRecord(), selectRange, 30));
+					if (selectRange.empty()) {
+						throw internal_error(); // empty is not expected to be complete
+					}
+					break;
+				} catch (Error& e) {
+					if (e.code() == error_code_manual_shard_split_failed) {
+						if (selectRange.empty()) {
+							break; // expected for the empty range
+						}
+						wait(delay(1.0));
+						continue;
+					} else {
+						throw e;
+					}
 				}
 			}
+			count++;
 		}
 		TraceEvent("ManualShardSplitTestComplete");
 		return Void();

From 89f88ff35a654ac59e424f20036b767660c1f74e Mon Sep 17 00:00:00 2001
From: Zhe Wang <zhe.wang@wustl.edu>
Date: Thu, 28 Sep 2023 11:32:26 -0700
Subject: [PATCH 267/468] Actively use storage-queue-aware getTeam for manual
 shard split

---
 fdbclient/ServerKnobs.cpp                     |  1 +
 fdbclient/ServerKnobs.h                       |  1 +
 fdbserver/DDTeamCollection.actor.cpp          | 45 ++++++++++++++++++-
 fdbserver/DataDistribution.actor.h            |  4 +-
 fdbserver/DataDistributionQueue.actor.cpp     | 43 ++++++++++++++----
 .../workloads/ManualShardSplit.actor.cpp      | 40 ++++++++++++++---
 6 files changed, 118 insertions(+), 16 deletions(-)

diff --git a/fdbclient/ServerKnobs.cpp b/fdbclient/ServerKnobs.cpp
index be967e22809..fe145a54842 100644
--- a/fdbclient/ServerKnobs.cpp
+++ b/fdbclient/ServerKnobs.cpp
@@ -293,6 +293,7 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi
 	init( DD_BUILD_EXTRA_TEAMS_OVERRIDE,                          10 ); if( randomize && BUGGIFY ) DD_BUILD_EXTRA_TEAMS_OVERRIDE = 2;
 	init( ENABLE_STORAGE_QUEUE_AWARE_TEAM_SELECTION,           false ); if( randomize && BUGGIFY ) ENABLE_STORAGE_QUEUE_AWARE_TEAM_SELECTION = true;
 	init( DD_TARGET_STORAGE_QUEUE_SIZE, TARGET_BYTES_PER_STORAGE_SERVER/3 ); if( randomize && BUGGIFY ) DD_TARGET_STORAGE_QUEUE_SIZE = TARGET_BYTES_PER_STORAGE_SERVER/10;
+	init( TRACE_STORAGE_QUEUE_AWARE_GET_TEAM_FOR_MANUAL_SPLIT_ONLY, true ); if (isSimulated) TRACE_STORAGE_QUEUE_AWARE_GET_TEAM_FOR_MANUAL_SPLIT_ONLY = false;
 
 	// TeamRemover
 	init( TR_FLAG_DISABLE_MACHINE_TEAM_REMOVER,                false ); if( randomize && BUGGIFY ) TR_FLAG_DISABLE_MACHINE_TEAM_REMOVER = deterministicRandom()->random01() < 0.1 ? true : false; // false by default. disable the consistency check when it's true
diff --git a/fdbclient/ServerKnobs.h b/fdbclient/ServerKnobs.h
index 60db5673602..4002d88bd93 100644
--- a/fdbclient/ServerKnobs.h
+++ b/fdbclient/ServerKnobs.h
@@ -235,6 +235,7 @@ class ServerKnobs : public KnobsImpl<ServerKnobs> {
 	int DD_STORAGE_WIGGLE_STUCK_THRESHOLD; // How many times bestTeamStuck accumulate will pause storage wiggle
 	bool ENABLE_STORAGE_QUEUE_AWARE_TEAM_SELECTION; // experimental!
 	int64_t DD_TARGET_STORAGE_QUEUE_SIZE;
+	bool TRACE_STORAGE_QUEUE_AWARE_GET_TEAM_FOR_MANUAL_SPLIT_ONLY;
 
 	// TeamRemover to remove redundant teams
 	bool TR_FLAG_DISABLE_MACHINE_TEAM_REMOVER; // disable the machineTeamRemover actor
diff --git a/fdbserver/DDTeamCollection.actor.cpp b/fdbserver/DDTeamCollection.actor.cpp
index 8dcbfa4e986..18eca4c5094 100644
--- a/fdbserver/DDTeamCollection.actor.cpp
+++ b/fdbserver/DDTeamCollection.actor.cpp
@@ -192,7 +192,8 @@ class DDTeamCollectionImpl {
 					TraceEvent(SevWarn, "DDTeamMedianAvailableSpaceTooSmall", self->distributorId)
 					    .detail("MedianAvailableSpaceRatio", self->medianAvailableSpace)
 					    .detail("TargetAvailableSpaceRatio", SERVER_KNOBS->TARGET_AVAILABLE_SPACE_RATIO)
-					    .detail("Primary", self->primary);
+					    .detail("Primary", self->primary)
+					    .detail("RelocationId", req.relocationId);
 					self->printDetailedTeamsInfo.trigger();
 				}
 			}
@@ -216,6 +217,14 @@ class DDTeamCollectionImpl {
 			// The situation happens rarely. We may want to eliminate this situation someday
 			if (!self->teams.size()) {
 				req.reply.send(std::make_pair(Optional<Reference<IDataDistributionTeam>>(), foundSrc));
+				if (req.storageQueueAware && req.traceStorageQueueAware) {
+					TraceEvent(SevWarn, "StorageQueueAwareGetTeam", self->distributorId)
+					    .detail("Request", req.getDesc())
+					    .detail("Src", describe(req.src))
+					    .detail("Succeed", false)
+					    .detail("Reason", "Teams size 0")
+					    .detail("RelocationID", req.relocationId);
+				}
 				return Void();
 			}
 
@@ -224,6 +233,8 @@ class DDTeamCollectionImpl {
 			Optional<Reference<IDataDistributionTeam>> bestOption;
 			std::vector<Reference<TCTeamInfo>> randomTeams;
 			const std::set<UID> completeSources(req.completeSources.begin(), req.completeSources.end());
+			state int numSkippedSSFailedGetQueueLength = 0;
+			state int numSkippedSSQueueTooLong = 0;
 
 			// Note: this block does not apply any filters from the request
 			if (!req.wantsNewServers) {
@@ -244,6 +255,14 @@ class DDTeamCollectionImpl {
 						if (found && teamList[j]->isHealthy()) {
 							bestOption = teamList[j];
 							req.reply.send(std::make_pair(bestOption, foundSrc));
+							if (req.storageQueueAware && req.traceStorageQueueAware) {
+								TraceEvent(SevInfo, "StorageQueueAwareGetTeam", self->distributorId)
+								    .detail("Request", req.getDesc())
+								    .detail("Src", describe(req.src))
+								    .detail("Succeed", false)
+								    .detail("Reason", "Not want new server")
+								    .detail("RelocationID", req.relocationId);
+							}
 							return Void();
 						}
 					}
@@ -269,8 +288,10 @@ class DDTeamCollectionImpl {
 							Optional<int64_t> storageQueueSize =
 							    self->teams[currentIndex]->getLongestStorageQueueSize();
 							if (!storageQueueSize.present()) {
+								numSkippedSSFailedGetQueueLength++;
 								continue; // this SS may not healthy, skip
 							} else if (storageQueueSize.get() > SERVER_KNOBS->DD_TARGET_STORAGE_QUEUE_SIZE) {
+								numSkippedSSQueueTooLong++;
 								continue; // this SS storage queue is too long, skip
 							}
 						}
@@ -320,8 +341,10 @@ class DDTeamCollectionImpl {
 					if (req.storageQueueAware) {
 						Optional<int64_t> storageQueueSize = dest->getLongestStorageQueueSize();
 						if (!storageQueueSize.present()) {
+							numSkippedSSFailedGetQueueLength++;
 							ok = false; // this SS may not healthy, skip
 						} else if (storageQueueSize.get() > SERVER_KNOBS->DD_TARGET_STORAGE_QUEUE_SIZE) {
+							numSkippedSSQueueTooLong++;
 							ok = false; // this SS storage queue is too long, skip
 						}
 					}
@@ -382,6 +405,16 @@ class DDTeamCollectionImpl {
 						if (found) {
 							bestOption = teamList[j];
 							req.reply.send(std::make_pair(bestOption, foundSrc));
+							if (req.storageQueueAware && req.traceStorageQueueAware) {
+								TraceEvent(SevInfo, "StorageQueueAwareGetTeam", self->distributorId)
+								    .detail("Request", req.getDesc())
+								    .detail("Src", describe(req.src))
+								    .detail("Succeed", false)
+								    .detail("Reason", "zero health team")
+								    .detail("NumSkippedSSFailedGetQueueLength", numSkippedSSFailedGetQueueLength)
+								    .detail("NumSkippedSSQueueTooLong", numSkippedSSQueueTooLong)
+								    .detail("RelocationID", req.relocationId);
+							}
 							return Void();
 						}
 					}
@@ -392,6 +425,16 @@ class DDTeamCollectionImpl {
 			// 	self->traceAllInfo(true);
 			// }
 
+			if (req.storageQueueAware && req.traceStorageQueueAware) {
+				TraceEvent(bestOption.present() ? SevInfo : SevWarn, "StorageQueueAwareGetTeam", self->distributorId)
+				    .detail("Request", req.getDesc())
+				    .detail("Src", describe(req.src))
+				    .detail("Succeed", bestOption.present())
+				    .detail("NumSkippedSSFailedGetQueueLength", numSkippedSSFailedGetQueueLength)
+				    .detail("NumSkippedSSQueueTooLong", numSkippedSSQueueTooLong)
+				    .detail("RelocationID", req.relocationId);
+			}
+
 			if (req.storageQueueAware && !bestOption.present()) {
 				req.storageQueueAware = false;
 				wait(getTeam(self, req)); // re-run getTeam without storageQueueAware
diff --git a/fdbserver/DataDistribution.actor.h b/fdbserver/DataDistribution.actor.h
index fffcd3853b5..0776ffd90a2 100644
--- a/fdbserver/DataDistribution.actor.h
+++ b/fdbserver/DataDistribution.actor.h
@@ -84,6 +84,8 @@ struct GetTeamRequest {
 	bool preferLowerUtilization;
 	bool teamMustHaveShards;
 	double inflightPenalty;
+	UID relocationId;
+	bool traceStorageQueueAware;
 	std::vector<UID> completeSources;
 	std::vector<UID> src;
 	Promise<std::pair<Optional<Reference<IDataDistributionTeam>>, bool>> reply;
@@ -96,7 +98,7 @@ struct GetTeamRequest {
 	               double inflightPenalty = 1.0)
 	  : wantsNewServers(wantsNewServers), wantsTrueBest(wantsTrueBest), storageQueueAware(false),
 	    preferLowerUtilization(preferLowerUtilization), teamMustHaveShards(teamMustHaveShards),
-	    inflightPenalty(inflightPenalty) {}
+	    inflightPenalty(inflightPenalty), traceStorageQueueAware(false) {}
 
 	std::string getDesc() const {
 		std::stringstream ss;
diff --git a/fdbserver/DataDistributionQueue.actor.cpp b/fdbserver/DataDistributionQueue.actor.cpp
index d5211e8aca0..5c61e0bd7b0 100644
--- a/fdbserver/DataDistributionQueue.actor.cpp
+++ b/fdbserver/DataDistributionQueue.actor.cpp
@@ -63,7 +63,8 @@ struct RelocateData {
 	    wantsNewServers(rs.priority == SERVER_KNOBS->PRIORITY_REBALANCE_OVERUTILIZED_TEAM ||
 	                    rs.priority == SERVER_KNOBS->PRIORITY_REBALANCE_UNDERUTILIZED_TEAM ||
 	                    rs.priority == SERVER_KNOBS->PRIORITY_SPLIT_SHARD ||
-	                    rs.priority == SERVER_KNOBS->PRIORITY_TEAM_REDUNDANT),
+	                    rs.priority == SERVER_KNOBS->PRIORITY_TEAM_REDUNDANT ||
+	                    rs.priority == SERVER_KNOBS->PRIORITY_MANUAL_SHARD_SPLIT),
 	    cancellable(true), interval("QueuedRelocation") {}
 
 	static bool isHealthPriority(int priority) {
@@ -1108,7 +1109,9 @@ ACTOR Future<Void> dataDistributionRelocator(DDQueueData* self, RelocateData rd,
 	state std::vector<UID> destIds;
 
 	if (rd.priority == SERVER_KNOBS->PRIORITY_MANUAL_SHARD_SPLIT) {
-		TraceEvent(SevInfo, "ManualShardSplitDDRelocatorStart").detail("Range", rd.keys);
+		TraceEvent(SevInfo, "ManualShardSplitDDRelocatorStart", distributorId)
+		    .detail("Range", rd.keys)
+		    .detail("RelocationID", relocateShardInterval.pairID);
 	}
 
 	try {
@@ -1164,6 +1167,16 @@ ACTOR Future<Void> dataDistributionRelocator(DDQueueData* self, RelocateData rd,
 					req.src = rd.src;
 					req.completeSources = rd.completeSources;
 					req.storageQueueAware = SERVER_KNOBS->ENABLE_STORAGE_QUEUE_AWARE_TEAM_SELECTION;
+					req.relocationId = relocateShardInterval.pairID;
+					if (SERVER_KNOBS->TRACE_STORAGE_QUEUE_AWARE_GET_TEAM_FOR_MANUAL_SPLIT_ONLY) {
+						if (rd.priority == SERVER_KNOBS->PRIORITY_MANUAL_SHARD_SPLIT) {
+							req.traceStorageQueueAware = true;
+						} else {
+							req.traceStorageQueueAware = false;
+						}
+					} else {
+						req.traceStorageQueueAware = true;
+					}
 					// bestTeam.second = false if the bestTeam in the teamCollection (in the DC) does not have any
 					// server that hosts the relocateData. This is possible, for example, in a fearless configuration
 					// when the remote DC is just brought up.
@@ -1217,7 +1230,8 @@ ACTOR Future<Void> dataDistributionRelocator(DDQueueData* self, RelocateData rd,
 					    .detail("TeamCollectionId", tciIndex)
 					    .detail("AnyDestOverloaded", anyDestOverloaded)
 					    .detail("NumOfTeamCollections", self->teamCollections.size())
-					    .detail("Servers", destServersString(bestTeams));
+					    .detail("Servers", destServersString(bestTeams))
+					    .detail("RelocationID", relocateShardInterval.pairID);
 					wait(delay(SERVER_KNOBS->DEST_OVERLOADED_DELAY, TaskPriority::DataDistributionLaunch));
 				} else {
 					TEST(true); // did not find a healthy destination team on the first attempt
@@ -1228,7 +1242,8 @@ ACTOR Future<Void> dataDistributionRelocator(DDQueueData* self, RelocateData rd,
 					    .detail("DestOverloadedCount", destOverloadedCount)
 					    .detail("TeamCollectionId", tciIndex)
 					    .detail("AnyDestOverloaded", anyDestOverloaded)
-					    .detail("NumOfTeamCollections", self->teamCollections.size());
+					    .detail("NumOfTeamCollections", self->teamCollections.size())
+					    .detail("RelocationID", relocateShardInterval.pairID);
 					wait(delay(SERVER_KNOBS->BEST_TEAM_STUCK_DELAY, TaskPriority::DataDistributionLaunch));
 				}
 
@@ -1284,7 +1299,8 @@ ACTOR Future<Void> dataDistributionRelocator(DDQueueData* self, RelocateData rd,
 				TraceEvent(SevWarn, "IncorrectDestTeamSize")
 				    .suppressFor(1.0)
 				    .detail("ExpectedTeamSize", self->teamSize)
-				    .detail("DestTeamSize", totalIds);
+				    .detail("DestTeamSize", totalIds)
+				    .detail("RelocationID", relocateShardInterval.pairID);
 			}
 
 			self->shardsAffectedByTeamFailure->moveShard(rd.keys, destinationTeams);
@@ -1422,7 +1438,12 @@ ACTOR Future<Void> dataDistributionRelocator(DDQueueData* self, RelocateData rd,
 					self->shardsAffectedByTeamFailure->finishMove(rd.keys);
 					relocationComplete.send(rd);
 					if (rd.priority == SERVER_KNOBS->PRIORITY_MANUAL_SHARD_SPLIT) {
-						TraceEvent(SevInfo, "ManualShardSplitDDRelocatorComplete").detail("Range", rd.keys);
+						TraceEvent(SevInfo, "ManualShardSplitDDRelocatorComplete", distributorId)
+						    .detail("Range", rd.keys)
+						    .detail("Dest", describe(destIds))
+						    .detail("Src", describe(rd.src))
+						    .detail("Duration", now() - startTime)
+						    .detail("RelocationID", relocateShardInterval.pairID);
 					}
 					return Void();
 				} else {
@@ -1430,9 +1451,10 @@ ACTOR Future<Void> dataDistributionRelocator(DDQueueData* self, RelocateData rd,
 				}
 			} else {
 				if (rd.priority == SERVER_KNOBS->PRIORITY_MANUAL_SHARD_SPLIT) {
-					TraceEvent(SevInfo, "ManualShardSplitDDRelocatorError")
+					TraceEvent(SevInfo, "ManualShardSplitDDRelocatorError", distributorId)
 					    .errorUnsuppressed(error)
-					    .detail("Range", rd.keys);
+					    .detail("Range", rd.keys)
+					    .detail("RelocationID", relocateShardInterval.pairID);
 				}
 				TEST(true); // move to removed server
 				healthyDestinations.addDataInFlightToTeam(-metrics.bytes);
@@ -1448,7 +1470,10 @@ ACTOR Future<Void> dataDistributionRelocator(DDQueueData* self, RelocateData rd,
 		}
 	} catch (Error& e) {
 		if (rd.priority == SERVER_KNOBS->PRIORITY_MANUAL_SHARD_SPLIT) {
-			TraceEvent(SevInfo, "ManualShardSplitDDRelocatorError").errorUnsuppressed(e).detail("Range", rd.keys);
+			TraceEvent(SevInfo, "ManualShardSplitDDRelocatorError", distributorId)
+			    .errorUnsuppressed(e)
+			    .detail("Range", rd.keys)
+			    .detail("RelocationID", relocateShardInterval.pairID);
 		}
 		TraceEvent(relocateShardInterval.end(), distributorId)
 		    .errorUnsuppressed(e)
diff --git a/fdbserver/workloads/ManualShardSplit.actor.cpp b/fdbserver/workloads/ManualShardSplit.actor.cpp
index d6a0b5837ba..f7cc954c18e 100644
--- a/fdbserver/workloads/ManualShardSplit.actor.cpp
+++ b/fdbserver/workloads/ManualShardSplit.actor.cpp
@@ -64,18 +64,18 @@ struct ManualShardSplitWorkload : TestWorkload {
 			KeyRangeRef("TestKeyF"_sr, "TestKeyF"_sr),
 		};
 		state int count = 0;
-		while (count < 100) {
-			state KeyRange selectRange = deterministicRandom()->randomChoice(ranges);
+		while (count < 25) {
+			state KeyRange selectRange1 = deterministicRandom()->randomChoice(ranges);
 			loop {
 				try {
-					wait(redistribute(cx->getConnectionRecord(), selectRange, 30));
-					if (selectRange.empty()) {
+					wait(redistribute(cx->getConnectionRecord(), selectRange1, 30));
+					if (selectRange1.empty()) {
 						throw internal_error(); // empty is not expected to be complete
 					}
 					break;
 				} catch (Error& e) {
 					if (e.code() == error_code_manual_shard_split_failed) {
-						if (selectRange.empty()) {
+						if (selectRange1.empty()) {
 							break; // expected for the empty range
 						}
 						wait(delay(1.0));
@@ -86,6 +86,36 @@ struct ManualShardSplitWorkload : TestWorkload {
 				}
 			}
 			count++;
+			wait(delay(20.0));
+		}
+		state int i = 0;
+		for (; i < 5; i++) {
+			state int j = 0;
+			for (; j < 5; j++) {
+				state Key beginKey = std::to_string(i) + std::to_string(j);
+				state Key endKey = std::to_string(i) + std::to_string(j) + "a";
+				state KeyRange selectRange2 = KeyRangeRef(beginKey, endKey);
+				loop {
+					try {
+						wait(redistribute(cx->getConnectionRecord(), selectRange2, 30));
+						if (selectRange2.empty()) {
+							throw internal_error(); // empty is not expected to be complete
+						}
+						break;
+					} catch (Error& e) {
+						if (e.code() == error_code_manual_shard_split_failed) {
+							if (selectRange2.empty()) {
+								break; // expected for the empty range
+							}
+							wait(delay(1.0));
+							continue;
+						} else {
+							throw e;
+						}
+					}
+				}
+				wait(delay(20.0));
+			}
 		}
 		TraceEvent("ManualShardSplitTestComplete");
 		return Void();

From 3a29a5ec428efa086734bab8b03c87ce9026d5ce Mon Sep 17 00:00:00 2001
From: hao fu <hfu5@apple.com>
Date: Thu, 28 Sep 2023 17:28:14 -0700
Subject: [PATCH 268/468] Add Knobs to control retry delays for BlobStore

We learnt that a new connection needs to be made for each HTTP
request through proxy to AWS S3, thus it would fail when try
to re-use the connections and see retryable errors.

Meanwhile, delay between each retry grow 2x each time it failed,
if the connection pool has a larger size, the delay can be long.

As a result, this change adds Knobs to cap the max delay of
retryable errors, including one for general retryable errors,
and the other only for connection failure error.

This change also adds more logging for debugging.
---
 documentation/sphinx/source/backups.rst |  4 ++++
 fdbclient/ClientKnobs.cpp               |  3 +++
 fdbclient/ClientKnobs.h                 |  2 ++
 fdbclient/S3BlobStore.actor.cpp         | 26 +++++++++++++++++++++++--
 fdbclient/S3BlobStore.h                 |  5 ++++-
 fdbrpc/HTTP.actor.cpp                   | 12 ++++++++++++
 6 files changed, 49 insertions(+), 3 deletions(-)

diff --git a/documentation/sphinx/source/backups.rst b/documentation/sphinx/source/backups.rst
index 793103340b6..00ea0d1cc97 100644
--- a/documentation/sphinx/source/backups.rst
+++ b/documentation/sphinx/source/backups.rst
@@ -152,6 +152,10 @@ Here is a complete list of valid parameters:
 
  *max_recv_bytes_per_second* (or *rbps*) - Max receive bytes per second for all requests combined.
 
+ *max_delay_retryable_error (or *dre*) - Max seconds to delay before retry again when seeing an retryable error.
+
+ *max_delay_connection_failed (or *dcf*) - Max seconds to delay before retry again when seeing an connection failure.
+
  *header* - Add an additional HTTP header to each blob store REST API request.  Can be specified multiple times.  Format is *header=<FieldName>:<FieldValue>* where both strings are non-empty.
  
   **Example**: The URL parameter *header=x-amz-storage-class:REDUCED_REDUNDANCY* would send the HTTP header required to use the reduced redundancy storage option in the S3 API.
diff --git a/fdbclient/ClientKnobs.cpp b/fdbclient/ClientKnobs.cpp
index 13f2d9d482b..f0552923a39 100644
--- a/fdbclient/ClientKnobs.cpp
+++ b/fdbclient/ClientKnobs.cpp
@@ -233,6 +233,9 @@ void ClientKnobs::initialize(Randomize randomize) {
 	init( BLOBSTORE_MAX_SEND_BYTES_PER_SECOND,      1e9 );
 	init( BLOBSTORE_MAX_RECV_BYTES_PER_SECOND,      1e9 );
 
+	init( BLOBSTORE_MAX_DELAY_RETRYABLE_ERROR,      60  );
+	init( BLOBSTORE_MAX_DELAY_CONNECTION_FAILED,    10  );
+
 	init( BLOBSTORE_LIST_REQUESTS_PER_SECOND,       200 );
 	init( BLOBSTORE_WRITE_REQUESTS_PER_SECOND,       50 );
 	init( BLOBSTORE_READ_REQUESTS_PER_SECOND,       100 );
diff --git a/fdbclient/ClientKnobs.h b/fdbclient/ClientKnobs.h
index aa001974cac..a6a6b62a75f 100644
--- a/fdbclient/ClientKnobs.h
+++ b/fdbclient/ClientKnobs.h
@@ -244,6 +244,8 @@ class ClientKnobs : public KnobsImpl<ClientKnobs> {
 	int BLOBSTORE_READ_CACHE_BLOCKS_PER_FILE;
 	int BLOBSTORE_MAX_SEND_BYTES_PER_SECOND;
 	int BLOBSTORE_MAX_RECV_BYTES_PER_SECOND;
+	int BLOBSTORE_MAX_DELAY_RETRYABLE_ERROR;
+	int BLOBSTORE_MAX_DELAY_CONNECTION_FAILED;
 
 	int CONSISTENCY_CHECK_RATE_LIMIT_MAX;
 	int CONSISTENCY_CHECK_ONE_ROUND_TARGET_COMPLETION_TIME;
diff --git a/fdbclient/S3BlobStore.actor.cpp b/fdbclient/S3BlobStore.actor.cpp
index eec2c6f3a87..1717790c3e0 100644
--- a/fdbclient/S3BlobStore.actor.cpp
+++ b/fdbclient/S3BlobStore.actor.cpp
@@ -85,6 +85,8 @@ S3BlobStoreEndpoint::BlobKnobs::BlobKnobs() {
 	read_cache_blocks_per_file = CLIENT_KNOBS->BLOBSTORE_READ_CACHE_BLOCKS_PER_FILE;
 	max_send_bytes_per_second = CLIENT_KNOBS->BLOBSTORE_MAX_SEND_BYTES_PER_SECOND;
 	max_recv_bytes_per_second = CLIENT_KNOBS->BLOBSTORE_MAX_RECV_BYTES_PER_SECOND;
+	max_delay_retryable_error = CLIENT_KNOBS->BLOBSTORE_MAX_DELAY_RETRYABLE_ERROR;
+	max_delay_connection_failed = CLIENT_KNOBS->BLOBSTORE_MAX_DELAY_CONNECTION_FAILED;
 	sdk_auth = false;
 }
 
@@ -122,6 +124,8 @@ bool S3BlobStoreEndpoint::BlobKnobs::set(StringRef name, int value) {
 	TRY_PARAM(read_cache_blocks_per_file, rcb);
 	TRY_PARAM(max_send_bytes_per_second, sbps);
 	TRY_PARAM(max_recv_bytes_per_second, rbps);
+	TRY_PARAM(max_delay_retryable_error, dre);
+	TRY_PARAM(max_delay_connection_failed, dcf);
 	TRY_PARAM(sdk_auth, sa);
 #undef TRY_PARAM
 	return false;
@@ -159,6 +163,8 @@ std::string S3BlobStoreEndpoint::BlobKnobs::getURLParameters() const {
 	_CHECK_PARAM(read_cache_blocks_per_file, rcb);
 	_CHECK_PARAM(max_send_bytes_per_second, sbps);
 	_CHECK_PARAM(max_recv_bytes_per_second, rbps);
+	_CHECK_PARAM(max_delay_retryable_error, dre);
+	_CHECK_PARAM(max_delay_connection_failed, dcf);
 #undef _CHECK_PARAM
 	return r;
 }
@@ -720,6 +726,12 @@ ACTOR Future<S3BlobStoreEndpoint::ReusableConnection> connect_impl(Reference<S3B
 		}
 	}
 	std::string host = b->host, service = b->service;
+	TraceEvent(SevDebug, "S3BlobStoreEndpointBuildingNewConnection")
+	    .detail("UseProxy", b->useProxy)
+	    .detail("TLS", b->knobs.secure_connection == 1)
+	    .detail("Host", host)
+	    .detail("Service", service)
+	    .log();
 	if (service.empty()) {
 		if (b->useProxy) {
 			fprintf(stderr, "ERROR: Port can't be empty when using HTTP proxy.\n");
@@ -745,7 +757,7 @@ ACTOR Future<S3BlobStoreEndpoint::ReusableConnection> connect_impl(Reference<S3B
 	}
 	wait(conn->connectHandshake());
 
-	TraceEvent("S3BlobStoreEndpointNewConnection")
+	TraceEvent("S3BlobStoreEndpointNewConnectionSuccess")
 	    .suppressFor(60)
 	    .detail("RemoteEndpoint", conn->getPeerAddress())
 	    .detail("ExpiresIn", b->knobs.max_connection_life)
@@ -922,6 +934,7 @@ ACTOR Future<Reference<HTTP::Response>> doRequest_impl(Reference<S3BlobStoreEndp
 			rconn.conn.clear();
 
 		} catch (Error& e) {
+			TraceEvent("S3BlobStoreDoRequestError").errorUnsuppressed(e);
 			if (e.code() == error_code_actor_cancelled)
 				throw;
 			err = e;
@@ -946,9 +959,13 @@ ACTOR Future<Reference<HTTP::Response>> doRequest_impl(Reference<S3BlobStoreEndp
 		TraceEvent event(SevWarn,
 		                 retryable ? "S3BlobStoreEndpointRequestFailedRetryable" : "S3BlobStoreEndpointRequestFailed");
 
+		bool connectionFailed = false;
 		// Attach err to trace event if present, otherwise extract some stuff from the response
 		if (err.present()) {
 			event.errorUnsuppressed(err.get());
+			if (err.get().code() == error_code_connection_failed) {
+				connectionFailed = true;
+			}
 		}
 		event.suppressFor(60);
 		if (!err.present()) {
@@ -965,6 +982,7 @@ ACTOR Future<Reference<HTTP::Response>> doRequest_impl(Reference<S3BlobStoreEndp
 		event.detail("Verb", verb)
 		    .detail("Resource", resource)
 		    .detail("ThisTry", thisTry)
+		    .detail("URI", canonicalURI)
 		    .detail("Proxy", bstore->proxyHost.orDefault(""));
 
 		// If r is not valid or not code 429 then increment the try count.  429's will not count against the attempt
@@ -974,8 +992,12 @@ ACTOR Future<Reference<HTTP::Response>> doRequest_impl(Reference<S3BlobStoreEndp
 
 		// We will wait delay seconds before the next retry, start with nextRetryDelay.
 		double delay = nextRetryDelay;
+		// conenctionFailed is treated specially as we know proxy to AWS can only serve 1 request per connection
+		// so there is no point of waiting too long, instead retry more aggressively
+		double limit =
+		    connectionFailed ? bstore->knobs.max_delay_connection_failed : bstore->knobs.max_delay_retryable_error;
 		// Double but limit the *next* nextRetryDelay.
-		nextRetryDelay = std::min(nextRetryDelay * 2, 60.0);
+		nextRetryDelay = std::min(nextRetryDelay * 2, limit);
 
 		if (retryable) {
 			// If r is valid then obey the Retry-After response header if present.
diff --git a/fdbclient/S3BlobStore.h b/fdbclient/S3BlobStore.h
index 86e0e54e32d..7e837009866 100644
--- a/fdbclient/S3BlobStore.h
+++ b/fdbclient/S3BlobStore.h
@@ -59,7 +59,7 @@ class S3BlobStoreEndpoint : public ReferenceCounted<S3BlobStoreEndpoint> {
 		    delete_requests_per_second, multipart_max_part_size, multipart_min_part_size, concurrent_requests,
 		    concurrent_uploads, concurrent_lists, concurrent_reads_per_file, concurrent_writes_per_file,
 		    read_block_size, read_ahead_blocks, read_cache_blocks_per_file, max_send_bytes_per_second,
-		    max_recv_bytes_per_second, sdk_auth;
+		    max_recv_bytes_per_second, max_delay_retryable_error, max_delay_connection_failed, sdk_auth;
 		bool set(StringRef name, int value);
 		std::string getURLParameters() const;
 		static std::vector<std::string> getKnobDescriptions() {
@@ -92,6 +92,9 @@ class S3BlobStoreEndpoint : public ReferenceCounted<S3BlobStoreEndpoint> {
 				"max_send_bytes_per_second (or sbps)   Max send bytes per second for all requests combined.",
 				"max_recv_bytes_per_second (or rbps)   Max receive bytes per second for all requests combined (NOT YET "
 				"USED).",
+				"max_delay_retryable_error (or dre)    Max seconds to delay before retry when see a retryable error.",
+				"max_delay_connection_failed (or dcf)  Max seconds to delay before retry when see a connection "
+				"failure.",
 				"sdk_auth (or sa)                      Use AWS SDK to resolve credentials. Only valid if "
 				"BUILD_AWS_BACKUP is enabled."
 			};
diff --git a/fdbrpc/HTTP.actor.cpp b/fdbrpc/HTTP.actor.cpp
index 4a03c3a9239..ffb86022686 100644
--- a/fdbrpc/HTTP.actor.cpp
+++ b/fdbrpc/HTTP.actor.cpp
@@ -422,7 +422,14 @@ ACTOR Future<Reference<HTTP::Response>> doRequest(Reference<IConnection> conn,
 			// If we already got a response, before finishing sending the request, then close the connection,
 			// set the Connection header to "close" as a hint to the caller that this connection can't be used
 			// again, and break out of the send loop.
+			// If there is an error from the future such a connection is closed, this would be run too
 			if (responseReading.isReady()) {
+				TraceEvent("HTTPRequestConectionClosing")
+				    .detail("Ready", responseReading.isReady())
+				    .detail("Error", responseReading.isError())
+				    .detail("QueueEmpty", pContent->empty())
+				    .detail("Verb", verb)
+				    .log();
 				conn->close();
 				r->headers["Connection"] = "close";
 				earlyResponse = true;
@@ -560,6 +567,7 @@ ACTOR Future<Void> sendProxyConnectRequest(Reference<IConnection> conn,
 			                                           requestTimeout));
 			r = _r;
 		} catch (Error& e) {
+			TraceEvent("ProxyRequestFailed").errorUnsuppressed(e);
 			if (e.code() == error_code_actor_cancelled)
 				throw;
 			err = e;
@@ -568,6 +576,10 @@ ACTOR Future<Void> sendProxyConnectRequest(Reference<IConnection> conn,
 		// If err is not present then r is valid.
 		// If r->code is in successCodes then record the successful request and return r.
 		if (!err.present() && r->code == 200) {
+			TraceEvent(SevDebug, "ProxyRequestSuccess")
+			    .detail("RemoteHost", remoteHost)
+			    .detail("RemoteService", remoteService)
+			    .log();
 			return Void();
 		}
 

From 928a1a4009cd3aae7fea13d2d2ee9affc9751567 Mon Sep 17 00:00:00 2001
From: hao fu <hfu5@apple.com>
Date: Fri, 29 Sep 2023 14:49:32 -0700
Subject: [PATCH 269/468] Add more log entry

---
 fdbclient/S3BlobStore.actor.cpp | 1 +
 1 file changed, 1 insertion(+)

diff --git a/fdbclient/S3BlobStore.actor.cpp b/fdbclient/S3BlobStore.actor.cpp
index 1717790c3e0..914dcfb3f5f 100644
--- a/fdbclient/S3BlobStore.actor.cpp
+++ b/fdbclient/S3BlobStore.actor.cpp
@@ -983,6 +983,7 @@ ACTOR Future<Reference<HTTP::Response>> doRequest_impl(Reference<S3BlobStoreEndp
 		    .detail("Resource", resource)
 		    .detail("ThisTry", thisTry)
 		    .detail("URI", canonicalURI)
+		    .detail("ConnectionFailed", connectionFailed)
 		    .detail("Proxy", bstore->proxyHost.orDefault(""));
 
 		// If r is not valid or not code 429 then increment the try count.  429's will not count against the attempt

From 536c6ac189e48da83d8bbcd001bd2767b26d2d23 Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Mon, 2 Oct 2023 09:53:30 -0700
Subject: [PATCH 270/468] Fix an invalid memory access

Found by Valgrind
---
 fdbclient/S3BlobStore.actor.cpp | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/fdbclient/S3BlobStore.actor.cpp b/fdbclient/S3BlobStore.actor.cpp
index 914dcfb3f5f..401c82a48fa 100644
--- a/fdbclient/S3BlobStore.actor.cpp
+++ b/fdbclient/S3BlobStore.actor.cpp
@@ -302,10 +302,11 @@ Reference<S3BlobStoreEndpoint> S3BlobStoreEndpoint::fromString(const std::string
 			}
 
 			// The parameter is known to S3BlobStoreEndpoint so it must be numeric and valid.
-			char* valueEnd;
-			int ivalue = strtol(value.toString().c_str(), &valueEnd, 10);
-			if (*valueEnd || (ivalue == 0 && value.toString() != "0"))
-				throw format("%s is not a valid value for %s", value.toString().c_str(), name.toString().c_str());
+			char* valueEnd = nullptr;
+			std::string s = value.toString();
+			int ivalue = strtol(s.c_str(), &valueEnd, 10);
+			if (*valueEnd || (ivalue == 0 && s != "0"))
+				throw format("%s is not a valid value for %s", s.c_str(), name.toString().c_str());
 
 			// It should not be possible for this set to fail now since the dummy set above had to have worked.
 			ASSERT(knobs.set(name, ivalue));

From a693f2512411e06abde59702d1b0f5795a05d3d1 Mon Sep 17 00:00:00 2001
From: w41ter <w41ter.l@gmail.com>
Date: Thu, 10 Aug 2023 09:09:51 +0000
Subject: [PATCH 271/468] Fix guess region from s3 URL

---
 fdbclient/S3BlobStore.actor.cpp | 15 +++++++++++++--
 1 file changed, 13 insertions(+), 2 deletions(-)

diff --git a/fdbclient/S3BlobStore.actor.cpp b/fdbclient/S3BlobStore.actor.cpp
index 401c82a48fa..558153e0c27 100644
--- a/fdbclient/S3BlobStore.actor.cpp
+++ b/fdbclient/S3BlobStore.actor.cpp
@@ -177,13 +177,12 @@ std::string guessRegionFromDomain(std::string domain) {
 		const char* service = knownServices[i];
 
 		std::size_t p = domain.find(service);
-
 		if (p == std::string::npos || (p >= 1 && domain[p - 1] != '.')) {
 			// eg. 127.0.0.1, example.com, s3-service.example.com, mys3.example.com
 			continue;
 		}
 
-		StringRef h(domain.c_str() + p);
+		StringRef h = StringRef(domain).substr(p);
 
 		if (!h.startsWith(LiteralStringRef("oss-"))) {
 			h.eat(service); // ignore s3 service
@@ -1838,3 +1837,15 @@ TEST_CASE("/backup/s3/v4headers") {
 
 	return Void();
 }
+
+TEST_CASE("/backup/s3/guess_region") {
+	std::string url = "blobstore://s3.us-west-2.amazonaws.com/resource_name?bucket=bucket_name&sa=1";
+
+	std::string resource;
+	std::string error;
+	S3BlobStoreEndpoint::ParametersT parameters;
+	Reference<S3BlobStoreEndpoint> s3 = S3BlobStoreEndpoint::fromString(url, {}, &resource, &error, &parameters);
+	ASSERT(s3->getRegion() == "us-west-2");
+
+	return Void();
+}

From 42b6a22e90f078821a2868de56c211f98f57ec70 Mon Sep 17 00:00:00 2001
From: neethuhaneesha <nbingi@apple.com>
Date: Tue, 3 Oct 2023 12:38:26 -0700
Subject: [PATCH 272/468] Reducing canCommit wait to avoid bigger spikes on
 pendinng compaction bytes (#10945)

---
 fdbclient/ServerKnobs.cpp | 4 ++--
 fdbclient/ServerKnobs.h   | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/fdbclient/ServerKnobs.cpp b/fdbclient/ServerKnobs.cpp
index fe145a54842..953963f38a1 100644
--- a/fdbclient/ServerKnobs.cpp
+++ b/fdbclient/ServerKnobs.cpp
@@ -449,8 +449,8 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi
 	// Can commit will delay ROCKSDB_CAN_COMMIT_DELAY_ON_OVERLOAD seconds for
 	// ROCKSDB_CAN_COMMIT_DELAY_TIMES_ON_OVERLOAD times, if rocksdb overloaded.
 	// Set ROCKSDB_CAN_COMMIT_DELAY_TIMES_ON_OVERLOAD to 0, to disable
-	init( ROCKSDB_CAN_COMMIT_DELAY_ON_OVERLOAD,                    1 );
-	init( ROCKSDB_CAN_COMMIT_DELAY_TIMES_ON_OVERLOAD,              5 );
+	init( ROCKSDB_CAN_COMMIT_DELAY_ON_OVERLOAD,                  0.2 );
+	init( ROCKSDB_CAN_COMMIT_DELAY_TIMES_ON_OVERLOAD,             20 );
 	init( ROCKSDB_COMPACTION_READAHEAD_SIZE,                   32768 ); // 32 KB, performs bigger reads when doing compaction.
 	init( ROCKSDB_BLOCK_SIZE,                                  32768 ); // 32 KB, size of the block in rocksdb cache.
 	init( ROCKSDB_MAX_LOG_FILE_SIZE,                        10485760 ); // 10MB.
diff --git a/fdbclient/ServerKnobs.h b/fdbclient/ServerKnobs.h
index 4002d88bd93..047239d7b94 100644
--- a/fdbclient/ServerKnobs.h
+++ b/fdbclient/ServerKnobs.h
@@ -348,7 +348,7 @@ class ServerKnobs : public KnobsImpl<ServerKnobs> {
 	int64_t ROCKSDB_HARD_PENDING_COMPACT_BYTES_LIMIT;
 	int64_t ROCKSDB_CAN_COMMIT_COMPACT_BYTES_LIMIT;
 	bool ROCKSDB_PARANOID_FILE_CHECKS;
-	int ROCKSDB_CAN_COMMIT_DELAY_ON_OVERLOAD;
+	double ROCKSDB_CAN_COMMIT_DELAY_ON_OVERLOAD;
 	int ROCKSDB_CAN_COMMIT_DELAY_TIMES_ON_OVERLOAD;
 	bool ROCKSDB_DISABLE_WAL_EXPERIMENTAL;
 	int64_t ROCKSDB_WAL_TTL_SECONDS;

From 297f3f7707d6d8856c951442d8294e4b4440f356 Mon Sep 17 00:00:00 2001
From: Evan Tschannen <evan.tschannen@snowflake.com>
Date: Fri, 14 Jul 2023 14:12:22 -0700
Subject: [PATCH 273/468] fix: Non-storage processes were not being checked for
 locality exclusions fix: Data distribution when not detect a newly added
 process was locality excluded fix: RemoveServerSafely did not wait for
 processes to be excluded before killing them when excluding localities

---
 fdbclient/DatabaseConfiguration.cpp           |   5 +-
 fdbclient/DatabaseConfiguration.h             |   2 +-
 fdbserver/ClusterController.actor.cpp         |  11 +-
 fdbserver/ClusterController.actor.h           |  44 ++++--
 fdbserver/Status.actor.cpp                    |  14 +-
 .../fdbserver/ExclusionTracker.actor.h        | 144 ++++++++++++++++++
 .../workloads/ConsistencyCheck.actor.cpp      |   4 +-
 .../workloads/RemoveServersSafely.actor.cpp   |  14 +-
 8 files changed, 200 insertions(+), 38 deletions(-)
 create mode 100644 fdbserver/include/fdbserver/ExclusionTracker.actor.h

diff --git a/fdbclient/DatabaseConfiguration.cpp b/fdbclient/DatabaseConfiguration.cpp
index e2050f6ae71..a60fe3d307d 100644
--- a/fdbclient/DatabaseConfiguration.cpp
+++ b/fdbclient/DatabaseConfiguration.cpp
@@ -707,7 +707,7 @@ Optional<ValueRef> DatabaseConfiguration::get(KeyRef key) const {
 	}
 }
 
-bool DatabaseConfiguration::isExcludedServer(NetworkAddressList a) const {
+bool DatabaseConfiguration::isExcludedServer(NetworkAddressList a, const LocalityData& locality) const {
 	return get(encodeExcludedServersKey(AddressExclusion(a.address.ip, a.address.port))).present() ||
 	       get(encodeExcludedServersKey(AddressExclusion(a.address.ip))).present() ||
 	       get(encodeFailedServersKey(AddressExclusion(a.address.ip, a.address.port))).present() ||
@@ -718,7 +718,8 @@ bool DatabaseConfiguration::isExcludedServer(NetworkAddressList a) const {
 	         get(encodeExcludedServersKey(AddressExclusion(a.secondaryAddress.get().ip))).present() ||
 	         get(encodeFailedServersKey(AddressExclusion(a.secondaryAddress.get().ip, a.secondaryAddress.get().port)))
 	             .present() ||
-	         get(encodeFailedServersKey(AddressExclusion(a.secondaryAddress.get().ip))).present()));
+	         get(encodeFailedServersKey(AddressExclusion(a.secondaryAddress.get().ip))).present())) ||
+	       isExcludedLocality(locality);
 }
 std::set<AddressExclusion> DatabaseConfiguration::getExcludedServers() const {
 	const_cast<DatabaseConfiguration*>(this)->makeConfigurationImmutable();
diff --git a/fdbclient/DatabaseConfiguration.h b/fdbclient/DatabaseConfiguration.h
index 923a2d0ff41..409aa54814a 100644
--- a/fdbclient/DatabaseConfiguration.h
+++ b/fdbclient/DatabaseConfiguration.h
@@ -258,7 +258,7 @@ struct DatabaseConfiguration {
 	TenantMode tenantMode;
 
 	// Excluded servers (no state should be here)
-	bool isExcludedServer(NetworkAddressList) const;
+	bool isExcludedServer(NetworkAddressList, const LocalityData& locality) const;
 	bool isExcludedLocality(const LocalityData& locality) const;
 	bool isMachineExcluded(const LocalityData& locality) const;
 	std::set<AddressExclusion> getExcludedServers() const;
diff --git a/fdbserver/ClusterController.actor.cpp b/fdbserver/ClusterController.actor.cpp
index 74d8a5c9373..9d53bc63346 100644
--- a/fdbserver/ClusterController.actor.cpp
+++ b/fdbserver/ClusterController.actor.cpp
@@ -626,7 +626,7 @@ ProcessClass::Fitness findBestFitnessForSingleton(const ClusterControllerData* s
 	// If the process has been marked as excluded, we take the max with ExcludeFit to ensure its fit
 	// is at least as bad as ExcludeFit. This assists with successfully offboarding such processes
 	// and removing them from the cluster.
-	if (self->db.config.isExcludedServer(worker.interf.addresses())) {
+	if (self->db.config.isExcludedServer(worker.interf.addresses(), worker.interf.locality)) {
 		bestFitness = std::max(bestFitness, ProcessClass::ExcludeFit);
 	}
 	return bestFitness;
@@ -1079,8 +1079,8 @@ void clusterRegisterMaster(ClusterControllerData* self, RegisterMasterRequest co
 			self->gotFullyRecoveredConfig = true;
 			db->fullyRecoveredConfig = req.configuration.get();
 			for (auto& it : self->id_worker) {
-				bool isExcludedFromConfig =
-				    db->fullyRecoveredConfig.isExcludedServer(it.second.details.interf.addresses());
+				bool isExcludedFromConfig = db->fullyRecoveredConfig.isExcludedServer(
+				    it.second.details.interf.addresses(), it.second.details.interf.locality);
 				if (it.second.priorityInfo.isExcluded != isExcludedFromConfig) {
 					it.second.priorityInfo.isExcluded = isExcludedFromConfig;
 					if (!it.second.reply.isSet()) {
@@ -1281,7 +1281,7 @@ ACTOR Future<Void> registerWorker(RegisterWorkerRequest req,
 		}
 
 		if (self->gotFullyRecoveredConfig) {
-			newPriorityInfo.isExcluded = self->db.fullyRecoveredConfig.isExcludedServer(w.addresses());
+			newPriorityInfo.isExcluded = self->db.fullyRecoveredConfig.isExcludedServer(w.addresses(), w.locality);
 		}
 	}
 
@@ -2741,7 +2741,8 @@ ACTOR Future<Void> clusterControllerCore(ClusterControllerFullInterface interf,
 
 			for (auto const& [id, worker] : self.id_worker) {
 				if ((req.flags & GetWorkersRequest::NON_EXCLUDED_PROCESSES_ONLY) &&
-				    self.db.config.isExcludedServer(worker.details.interf.addresses())) {
+				    self.db.config.isExcludedServer(worker.details.interf.addresses(),
+				                                    worker.details.interf.locality)) {
 					continue;
 				}
 
diff --git a/fdbserver/ClusterController.actor.h b/fdbserver/ClusterController.actor.h
index 01815195486..7e326dadc84 100644
--- a/fdbserver/ClusterController.actor.h
+++ b/fdbserver/ClusterController.actor.h
@@ -368,7 +368,8 @@ class ClusterControllerData {
 
 		for (auto& it : id_worker) {
 			auto fitness = it.second.details.processClass.machineClassFitness(ProcessClass::Storage);
-			if (workerAvailable(it.second, false) && !conf.isExcludedServer(it.second.details.interf.addresses()) &&
+			if (workerAvailable(it.second, false) && it.second.details.recoveredDiskFiles &&
+			    !conf.isExcludedServer(it.second.details.interf.addresses(), it.second.details.interf.locality) &&
 			    !isExcludedDegradedServer(it.second.details.interf.addresses()) &&
 			    fitness != ProcessClass::NeverAssign &&
 			    (!dcId.present() || it.second.details.interf.locality.dcId() == dcId.get())) {
@@ -600,7 +601,12 @@ class ClusterControllerData {
 				logWorkerUnavailable(SevInfo, id, "complex", "Worker is not available", worker_details, fitness, dcIds);
 				continue;
 			}
-			if (conf.isExcludedServer(worker_details.interf.addresses())) {
+			if (!worker_details.recoveredDiskFiles) {
+				logWorkerUnavailable(
+				    SevInfo, id, "complex", "Worker disk file recovery unfinished", worker_details, fitness, dcIds);
+				continue;
+			}
+			if (conf.isExcludedServer(worker_details.interf.addresses(), worker_details.interf.locality)) {
 				logWorkerUnavailable(SevInfo,
 				                     id,
 				                     "complex",
@@ -845,7 +851,12 @@ class ClusterControllerData {
 				logWorkerUnavailable(SevInfo, id, "simple", "Worker is not available", worker_details, fitness, dcIds);
 				continue;
 			}
-			if (conf.isExcludedServer(worker_details.interf.addresses())) {
+			if (!worker_details.recoveredDiskFiles) {
+				logWorkerUnavailable(
+				    SevInfo, id, "simple", "Worker disk file recovery unfinished", worker_details, fitness, dcIds);
+				continue;
+			}
+			if (conf.isExcludedServer(worker_details.interf.addresses(), worker_details.interf.locality)) {
 				logWorkerUnavailable(SevInfo,
 				                     id,
 				                     "simple",
@@ -992,7 +1003,12 @@ class ClusterControllerData {
 				    SevInfo, id, "deprecated", "Worker is not available", worker_details, fitness, dcIds);
 				continue;
 			}
-			if (conf.isExcludedServer(worker_details.interf.addresses())) {
+			if (!worker_details.recoveredDiskFiles) {
+				logWorkerUnavailable(
+				    SevInfo, id, "deprecated", "Worker disk file recovery unfinished", worker_details, fitness, dcIds);
+				continue;
+			}
+			if (conf.isExcludedServer(worker_details.interf.addresses(), worker_details.interf.locality)) {
 				logWorkerUnavailable(SevInfo,
 				                     id,
 				                     "deprecated",
@@ -1435,7 +1451,7 @@ class ClusterControllerData {
 
 		for (auto& it : id_worker) {
 			auto fitness = it.second.details.processClass.machineClassFitness(role);
-			if (conf.isExcludedServer(it.second.details.interf.addresses()) ||
+			if (conf.isExcludedServer(it.second.details.interf.addresses(), it.second.details.interf.locality) ||
 			    isExcludedDegradedServer(it.second.details.interf.addresses())) {
 				fitness = std::max(fitness, ProcessClass::ExcludeFit);
 			}
@@ -1482,7 +1498,7 @@ class ClusterControllerData {
 		for (auto& it : id_worker) {
 			auto fitness = it.second.details.processClass.machineClassFitness(role);
 			if (workerAvailable(it.second, checkStable) &&
-			    !conf.isExcludedServer(it.second.details.interf.addresses()) &&
+			    !conf.isExcludedServer(it.second.details.interf.addresses(), it.second.details.interf.locality) &&
 			    !isExcludedDegradedServer(it.second.details.interf.addresses()) &&
 			    it.second.details.interf.locality.dcId() == dcId &&
 			    (!minWorker.present() ||
@@ -1619,7 +1635,7 @@ class ClusterControllerData {
 		std::set<Optional<Standalone<StringRef>>> result;
 		for (auto& it : id_worker)
 			if (workerAvailable(it.second, checkStable) &&
-			    !conf.isExcludedServer(it.second.details.interf.addresses()) &&
+			    !conf.isExcludedServer(it.second.details.interf.addresses(), it.second.details.interf.locality) &&
 			    !isExcludedDegradedServer(it.second.details.interf.addresses()))
 				result.insert(it.second.details.interf.locality.dcId());
 		return result;
@@ -2132,8 +2148,9 @@ class ClusterControllerData {
 		for (auto& it : first) {
 			auto w = id_worker.find(it.locality.processId());
 			ASSERT(w != id_worker.end());
-			ASSERT(!conf.isExcludedServer(w->second.details.interf.addresses()));
-			firstDetails.push_back(w->second.details);
+			auto const& [_, workerInfo] = *w;
+			ASSERT(!conf.isExcludedServer(workerInfo.details.interf.addresses(), workerInfo.details.interf.locality));
+			firstDetails.push_back(workerInfo.details);
 			//TraceEvent("CompareAddressesFirst").detail(description.c_str(), w->second.details.interf.address());
 		}
 		RoleFitness firstFitness(firstDetails, role, firstUsed);
@@ -2142,8 +2159,9 @@ class ClusterControllerData {
 		for (auto& it : second) {
 			auto w = id_worker.find(it.locality.processId());
 			ASSERT(w != id_worker.end());
-			ASSERT(!conf.isExcludedServer(w->second.details.interf.addresses()));
-			secondDetails.push_back(w->second.details);
+			auto const& [_, workerInfo] = *w;
+			ASSERT(!conf.isExcludedServer(workerInfo.details.interf.addresses(), workerInfo.details.interf.locality));
+			secondDetails.push_back(workerInfo.details);
 			//TraceEvent("CompareAddressesSecond").detail(description.c_str(), w->second.details.interf.address());
 		}
 		RoleFitness secondFitness(secondDetails, role, secondUsed);
@@ -2489,7 +2507,7 @@ class ClusterControllerData {
 		// still need master for recovery.
 		ProcessClass::Fitness oldMasterFit =
 		    masterWorker->second.details.processClass.machineClassFitness(ProcessClass::Master);
-		if (db.config.isExcludedServer(dbi.master.addresses())) {
+		if (db.config.isExcludedServer(dbi.master.addresses(), dbi.master.locality)) {
 			oldMasterFit = std::max(oldMasterFit, ProcessClass::ExcludeFit);
 		}
 
@@ -2500,7 +2518,7 @@ class ClusterControllerData {
 		WorkerFitnessInfo mworker = getWorkerForRoleInDatacenter(
 		    clusterControllerDcId, ProcessClass::Master, ProcessClass::NeverAssign, db.config, id_used, {}, true);
 		auto newMasterFit = mworker.worker.processClass.machineClassFitness(ProcessClass::Master);
-		if (db.config.isExcludedServer(mworker.worker.interf.addresses())) {
+		if (db.config.isExcludedServer(mworker.worker.interf.addresses(), mworker.worker.interf.locality)) {
 			newMasterFit = std::max(newMasterFit, ProcessClass::ExcludeFit);
 		}
 
diff --git a/fdbserver/Status.actor.cpp b/fdbserver/Status.actor.cpp
index cf63e88d6c8..4c784f6051a 100644
--- a/fdbserver/Status.actor.cpp
+++ b/fdbserver/Status.actor.cpp
@@ -362,10 +362,10 @@ static JsonBuilderObject machineStatusFetcher(WorkerEvents mMetrics,
 			// FIXME: this will not catch if the secondary address of the process was excluded
 			NetworkAddressList tempList;
 			tempList.address = it->first;
-			bool excludedServer = false;
-			bool excludedLocality = false;
-			if (configuration.present() && configuration.get().isExcludedServer(tempList))
-				excludedServer = true;
+			bool excludedServer = true;
+			bool excludedLocality = true;
+			if (configuration.present() && !configuration.get().isExcludedServer(tempList, LocalityData()))
+				excludedServer = false;
 			if (locality.count(it->first) && configuration.present() &&
 			    configuration.get().isMachineExcluded(locality[it->first]))
 				excludedLocality = true;
@@ -1059,8 +1059,8 @@ ACTOR static Future<JsonBuilderObject> processStatusFetcher(
 			statusObj["roles"] = roles.getStatusForAddress(address);
 
 			if (configuration.present()) {
-				statusObj["excluded"] = configuration.get().isExcludedServer(workerItr->interf.addresses()) ||
-				                        configuration.get().isExcludedLocality(workerItr->interf.locality);
+				statusObj["excluded"] =
+				    configuration.get().isExcludedServer(workerItr->interf.addresses(), workerItr->interf.locality);
 			}
 
 			statusObj["class_type"] = workerItr->processClass.toString();
@@ -2019,7 +2019,7 @@ static int getExtraTLogEligibleZones(const std::vector<WorkerDetails>& workers,
 	std::map<Key, std::set<StringRef>> dcId_zone;
 	for (auto const& worker : workers) {
 		if (worker.processClass.machineClassFitness(ProcessClass::TLog) < ProcessClass::NeverAssign &&
-		    !configuration.isExcludedServer(worker.interf.addresses())) {
+		    !configuration.isExcludedServer(worker.interf.addresses(), worker.interf.locality)) {
 			allZones.insert(worker.interf.locality.zoneId().get());
 			if (worker.interf.locality.dcId().present()) {
 				dcId_zone[worker.interf.locality.dcId().get()].insert(worker.interf.locality.zoneId().get());
diff --git a/fdbserver/include/fdbserver/ExclusionTracker.actor.h b/fdbserver/include/fdbserver/ExclusionTracker.actor.h
new file mode 100644
index 00000000000..745301f011b
--- /dev/null
+++ b/fdbserver/include/fdbserver/ExclusionTracker.actor.h
@@ -0,0 +1,144 @@
+/*
+ * ExclusionTracker.actor.h
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2022 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+// When actually compiled (NO_INTELLISENSE), include the generated version of this file.  In intellisense use the source
+// version.
+#if defined(NO_INTELLISENSE) && !defined(EXCLUSION_TRACKER_ACTOR_G_H)
+#define EXCLUSION_TRACKER_ACTOR_G_H
+#include "fdbserver/ExclusionTracker.actor.g.h"
+#elif !defined(EXCLUSION_TRACKER_ACTOR_H)
+#define EXCLUSION_TRACKER_ACTOR_H
+
+#include <set>
+#include "flow/flow.h"
+#include "fdbclient/DatabaseContext.h"
+#include "fdbclient/ManagementAPI.actor.h"
+#include "flow/actorcompiler.h" // This must be the last #include.
+
+struct ExclusionTracker {
+	std::set<AddressExclusion> excluded;
+	std::set<AddressExclusion> failed;
+
+	AsyncTrigger changed;
+
+	Database db;
+	Future<Void> trackerFuture;
+
+	ExclusionTracker() {}
+	ExclusionTracker(Database db) : db(db) { trackerFuture = tracker(this); }
+
+	bool isFailedOrExcluded(NetworkAddress addr) {
+		AddressExclusion addrExclusion(addr.ip, addr.port);
+		return excluded.count(addrExclusion) || failed.count(addrExclusion);
+	}
+
+	ACTOR static Future<Void> tracker(ExclusionTracker* self) {
+		// Fetch the list of excluded servers
+		state ReadYourWritesTransaction tr(self->db);
+		loop {
+			try {
+				tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
+				tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
+				tr.setOption(FDBTransactionOptions::LOCK_AWARE);
+				state Future<RangeResult> fresultsExclude = tr.getRange(excludedServersKeys, CLIENT_KNOBS->TOO_MANY);
+				state Future<RangeResult> fresultsFailed = tr.getRange(failedServersKeys, CLIENT_KNOBS->TOO_MANY);
+				state Future<RangeResult> flocalitiesExclude =
+				    tr.getRange(excludedLocalityKeys, CLIENT_KNOBS->TOO_MANY);
+				state Future<RangeResult> flocalitiesFailed = tr.getRange(failedLocalityKeys, CLIENT_KNOBS->TOO_MANY);
+				state Future<std::vector<ProcessData>> fworkers = getWorkers(&tr.getTransaction());
+				wait(success(fresultsExclude) && success(fresultsFailed) && success(flocalitiesExclude) &&
+				     success(flocalitiesFailed));
+
+				state RangeResult excludedResults = fresultsExclude.get();
+				ASSERT(!excludedResults.more && excludedResults.size() < CLIENT_KNOBS->TOO_MANY);
+
+				state RangeResult failedResults = fresultsFailed.get();
+				ASSERT(!failedResults.more && failedResults.size() < CLIENT_KNOBS->TOO_MANY);
+
+				state RangeResult excludedLocalityResults = flocalitiesExclude.get();
+				ASSERT(!excludedLocalityResults.more && excludedLocalityResults.size() < CLIENT_KNOBS->TOO_MANY);
+
+				state RangeResult failedLocalityResults = flocalitiesFailed.get();
+				ASSERT(!failedLocalityResults.more && failedLocalityResults.size() < CLIENT_KNOBS->TOO_MANY);
+
+				state std::set<AddressExclusion> newExcluded;
+				state std::set<AddressExclusion> newFailed;
+				for (const auto& r : excludedResults) {
+					AddressExclusion addr = decodeExcludedServersKey(r.key);
+					if (addr.isValid()) {
+						newExcluded.insert(addr);
+					}
+				}
+				for (const auto& r : failedResults) {
+					AddressExclusion addr = decodeFailedServersKey(r.key);
+					if (addr.isValid()) {
+						newFailed.insert(addr);
+					}
+				}
+
+				wait(success(fworkers));
+				std::vector<ProcessData> workers = fworkers.get();
+				for (const auto& r : excludedLocalityResults) {
+					std::string locality = decodeExcludedLocalityKey(r.key);
+					std::set<AddressExclusion> localityExcludedAddresses = getAddressesByLocality(workers, locality);
+					newExcluded.insert(localityExcludedAddresses.begin(), localityExcludedAddresses.end());
+				}
+				for (const auto& r : failedLocalityResults) {
+					std::string locality = decodeFailedLocalityKey(r.key);
+					std::set<AddressExclusion> localityFailedAddresses = getAddressesByLocality(workers, locality);
+					newFailed.insert(localityFailedAddresses.begin(), localityFailedAddresses.end());
+				}
+
+				bool foundChange = false;
+				if (self->excluded != newExcluded) {
+					self->excluded = newExcluded;
+					foundChange = true;
+				}
+				if (self->failed != newFailed) {
+					self->failed = newFailed;
+					foundChange = true;
+				}
+
+				if (foundChange) {
+					self->changed.trigger();
+				}
+
+				state Future<Void> watchFuture =
+				    tr.watch(excludedServersVersionKey) || tr.watch(failedServersVersionKey) ||
+				    tr.watch(excludedLocalityVersionKey) || tr.watch(failedLocalityVersionKey);
+				wait(tr.commit());
+				if (excludedLocalityResults.size() > 0 || failedLocalityResults.size() > 0) {
+					// when there are excluded localities we need to monitor for when the worker list changes, so we
+					// must poll
+					watchFuture = watchFuture || delay(10.0);
+				}
+				wait(watchFuture);
+				tr.reset();
+			} catch (Error& e) {
+				wait(tr.onError(e));
+			}
+		}
+	}
+};
+
+#include "flow/unactorcompiler.h"
+#endif
\ No newline at end of file
diff --git a/fdbserver/workloads/ConsistencyCheck.actor.cpp b/fdbserver/workloads/ConsistencyCheck.actor.cpp
index 7a36132f2b6..f41f775da7a 100644
--- a/fdbserver/workloads/ConsistencyCheck.actor.cpp
+++ b/fdbserver/workloads/ConsistencyCheck.actor.cpp
@@ -1935,7 +1935,7 @@ struct ConsistencyCheckWorkload : TestWorkload {
 
 		for (int i = 0; i < workers.size(); i++) {
 			NetworkAddress addr = workers[i].interf.stableAddress();
-			if (!configuration.isExcludedServer(workers[i].interf.addresses()) &&
+			if (!configuration.isExcludedServer(workers[i].interf.addresses(), workers[i].interf.locality) &&
 			    (workers[i].processClass == ProcessClass::StorageClass ||
 			     workers[i].processClass == ProcessClass::UnsetClass)) {
 				bool found = false;
@@ -2152,7 +2152,7 @@ struct ConsistencyCheckWorkload : TestWorkload {
 		for (const auto& worker : workers) {
 			NetworkAddress addr = worker.interf.stableAddress();
 			bool inCCDc = worker.interf.locality.dcId() == ccDcId;
-			if (!configuration.isExcludedServer(worker.interf.addresses())) {
+			if (!configuration.isExcludedServer(worker.interf.addresses(), worker.interf.locality)) {
 				if (worker.processClass == ProcessClass::BlobWorkerClass) {
 					numBlobWorkerProcesses++;
 
diff --git a/fdbserver/workloads/RemoveServersSafely.actor.cpp b/fdbserver/workloads/RemoveServersSafely.actor.cpp
index 6be4a58ceea..5c3d7b54279 100644
--- a/fdbserver/workloads/RemoveServersSafely.actor.cpp
+++ b/fdbserver/workloads/RemoveServersSafely.actor.cpp
@@ -677,14 +677,12 @@ struct RemoveServersSafelyWorkload : TestWorkload {
 		// We need to skip at least the quorum change if there's nothing to kill, because there might not be enough
 		// servers left alive to do a coordinators auto (?)
 		if (toKill.size()) {
-			if (!excludeLocalitiesInsteadOfServers) {
-				// Wait for removal to be safe
-				TraceEvent("RemoveAndKill", functionId)
-				    .detail("Step", "Wait For Server Exclusion")
-				    .detail("Addresses", describe(toKill))
-				    .detail("ClusterAvailable", g_simulator.isAvailable());
-				wait(success(checkForExcludingServers(cx, toKillArray, true /* wait for exclusion */)));
-			}
+			// Wait for removal to be safe
+			TraceEvent("RemoveAndKill", functionId)
+			    .detail("Step", "Wait For Server Exclusion")
+			    .detail("Addresses", describe(toKill))
+			    .detail("ClusterAvailable", g_simulator->isAvailable());
+			wait(success(checkForExcludingServers(cx, toKillArray, true /* wait for exclusion */)));
 
 			TraceEvent("RemoveAndKill", functionId)
 			    .detail("Step", "coordinators auto")

From 29d8c15c779bd12006ba6b211b0e9cb23f3f8e3d Mon Sep 17 00:00:00 2001
From: Evan Tschannen <evan.tschannen@snowflake.com>
Date: Tue, 1 Aug 2023 12:03:51 -0700
Subject: [PATCH 274/468] fix: do not allow locality based excludes if they
 cannot exclude the required addresses

---
 .../workloads/RemoveServersSafely.actor.cpp   | 28 +++++++++++++++----
 1 file changed, 23 insertions(+), 5 deletions(-)

diff --git a/fdbserver/workloads/RemoveServersSafely.actor.cpp b/fdbserver/workloads/RemoveServersSafely.actor.cpp
index 5c3d7b54279..21e38d82fc1 100644
--- a/fdbserver/workloads/RemoveServersSafely.actor.cpp
+++ b/fdbserver/workloads/RemoveServersSafely.actor.cpp
@@ -634,9 +634,9 @@ struct RemoveServersSafelyWorkload : TestWorkload {
 
 		state bool excludeLocalitiesInsteadOfServers = deterministicRandom()->coinflip();
 		if (markExcludeAsFailed) {
-			if (excludeLocalitiesInsteadOfServers) {
-				state std::unordered_set<std::string> toKillLocalitiesFailed =
-				    self->getLocalitiesFromAddresses(toKillMarkFailedArray);
+			state std::unordered_set<std::string> toKillLocalitiesFailed =
+			    self->getLocalitiesFromAddresses(toKillMarkFailedArray);
+			if (excludeLocalitiesInsteadOfServers && toKillLocalitiesFailed.size() > 0) {
 				TraceEvent("RemoveAndKill", functionId)
 				    .detail("Step", "Excluding localities with failed option")
 				    .detail("FailedAddressesSize", toKillMarkFailedArray.size())
@@ -655,8 +655,8 @@ struct RemoveServersSafelyWorkload : TestWorkload {
 			}
 		}
 
-		if (excludeLocalitiesInsteadOfServers) {
-			state std::unordered_set<std::string> toKillLocalities = self->getLocalitiesFromAddresses(toKillArray);
+		state std::unordered_set<std::string> toKillLocalities = self->getLocalitiesFromAddresses(toKillArray);
+		if (excludeLocalitiesInsteadOfServers && toKillLocalities.size() > 0) {
 			TraceEvent("RemoveAndKill", functionId)
 			    .detail("Step", "Excluding localities without failed option")
 			    .detail("AddressesSize", toKillArray.size())
@@ -779,6 +779,24 @@ struct RemoveServersSafelyWorkload : TestWorkload {
 			}
 		}
 
+		for (const auto& processInfo : processes) {
+			AddressExclusion pAddr(processInfo->address.ip, processInfo->address.port);
+			if (std::find(addresses.begin(), addresses.end(), pAddr) != addresses.end()) {
+				std::map<std::string, std::string> localityData = processInfo->locality.getAllData();
+				bool found = false;
+				for (const auto& l : localityData) {
+					if (toKillLocalities.count(LocalityData::ExcludeLocalityPrefix.toString() + l.first + ":" +
+					                           l.second)) {
+						found = true;
+						break;
+					}
+				}
+				if (!found) {
+					return std::unordered_set<std::string>();
+				}
+			}
+		}
+
 		return toKillLocalities;
 	}
 

From d5690ef23ac48e1b7707b78332fb2b866251b636 Mon Sep 17 00:00:00 2001
From: Xiaoge Su <magichp@gmail.com>
Date: Tue, 3 Oct 2023 16:05:39 -0700
Subject: [PATCH 275/468] fixup! Fix build failures

---
 fdbserver/ClusterController.actor.h             | 17 +----------------
 .../workloads/RemoveServersSafely.actor.cpp     |  2 +-
 2 files changed, 2 insertions(+), 17 deletions(-)

diff --git a/fdbserver/ClusterController.actor.h b/fdbserver/ClusterController.actor.h
index 7e326dadc84..53fbfb904df 100644
--- a/fdbserver/ClusterController.actor.h
+++ b/fdbserver/ClusterController.actor.h
@@ -368,7 +368,7 @@ class ClusterControllerData {
 
 		for (auto& it : id_worker) {
 			auto fitness = it.second.details.processClass.machineClassFitness(ProcessClass::Storage);
-			if (workerAvailable(it.second, false) && it.second.details.recoveredDiskFiles &&
+			if (workerAvailable(it.second, false) &&
 			    !conf.isExcludedServer(it.second.details.interf.addresses(), it.second.details.interf.locality) &&
 			    !isExcludedDegradedServer(it.second.details.interf.addresses()) &&
 			    fitness != ProcessClass::NeverAssign &&
@@ -601,11 +601,6 @@ class ClusterControllerData {
 				logWorkerUnavailable(SevInfo, id, "complex", "Worker is not available", worker_details, fitness, dcIds);
 				continue;
 			}
-			if (!worker_details.recoveredDiskFiles) {
-				logWorkerUnavailable(
-				    SevInfo, id, "complex", "Worker disk file recovery unfinished", worker_details, fitness, dcIds);
-				continue;
-			}
 			if (conf.isExcludedServer(worker_details.interf.addresses(), worker_details.interf.locality)) {
 				logWorkerUnavailable(SevInfo,
 				                     id,
@@ -851,11 +846,6 @@ class ClusterControllerData {
 				logWorkerUnavailable(SevInfo, id, "simple", "Worker is not available", worker_details, fitness, dcIds);
 				continue;
 			}
-			if (!worker_details.recoveredDiskFiles) {
-				logWorkerUnavailable(
-				    SevInfo, id, "simple", "Worker disk file recovery unfinished", worker_details, fitness, dcIds);
-				continue;
-			}
 			if (conf.isExcludedServer(worker_details.interf.addresses(), worker_details.interf.locality)) {
 				logWorkerUnavailable(SevInfo,
 				                     id,
@@ -1003,11 +993,6 @@ class ClusterControllerData {
 				    SevInfo, id, "deprecated", "Worker is not available", worker_details, fitness, dcIds);
 				continue;
 			}
-			if (!worker_details.recoveredDiskFiles) {
-				logWorkerUnavailable(
-				    SevInfo, id, "deprecated", "Worker disk file recovery unfinished", worker_details, fitness, dcIds);
-				continue;
-			}
 			if (conf.isExcludedServer(worker_details.interf.addresses(), worker_details.interf.locality)) {
 				logWorkerUnavailable(SevInfo,
 				                     id,
diff --git a/fdbserver/workloads/RemoveServersSafely.actor.cpp b/fdbserver/workloads/RemoveServersSafely.actor.cpp
index 21e38d82fc1..bcda83779d4 100644
--- a/fdbserver/workloads/RemoveServersSafely.actor.cpp
+++ b/fdbserver/workloads/RemoveServersSafely.actor.cpp
@@ -681,7 +681,7 @@ struct RemoveServersSafelyWorkload : TestWorkload {
 			TraceEvent("RemoveAndKill", functionId)
 			    .detail("Step", "Wait For Server Exclusion")
 			    .detail("Addresses", describe(toKill))
-			    .detail("ClusterAvailable", g_simulator->isAvailable());
+			    .detail("ClusterAvailable", g_simulator.isAvailable());
 			wait(success(checkForExcludingServers(cx, toKillArray, true /* wait for exclusion */)));
 
 			TraceEvent("RemoveAndKill", functionId)

From bae47e07e7e7d7e19d7409931bf0be1661db27ca Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Tue, 3 Oct 2023 15:10:49 -0700
Subject: [PATCH 276/468] Fix strtol return type and error checking

Added an overflow test as well.
---
 fdbclient/S3BlobStore.actor.cpp | 14 ++++++++++++--
 1 file changed, 12 insertions(+), 2 deletions(-)

diff --git a/fdbclient/S3BlobStore.actor.cpp b/fdbclient/S3BlobStore.actor.cpp
index 558153e0c27..892fd09f9e0 100644
--- a/fdbclient/S3BlobStore.actor.cpp
+++ b/fdbclient/S3BlobStore.actor.cpp
@@ -23,6 +23,7 @@
 #include "fdbrpc/md5/md5.h"
 #include "fdbrpc/libb64/encode.h"
 #include "fdbclient/sha1/SHA1.h"
+#include <climits>
 #include <time.h>
 #include <iomanip>
 #include <openssl/sha.h>
@@ -303,8 +304,9 @@ Reference<S3BlobStoreEndpoint> S3BlobStoreEndpoint::fromString(const std::string
 			// The parameter is known to S3BlobStoreEndpoint so it must be numeric and valid.
 			char* valueEnd = nullptr;
 			std::string s = value.toString();
-			int ivalue = strtol(s.c_str(), &valueEnd, 10);
-			if (*valueEnd || (ivalue == 0 && s != "0"))
+			long int ivalue = strtol(s.c_str(), &valueEnd, 10);
+			if (*valueEnd || (ivalue == 0 && s != "0") ||
+			    (((ivalue == LONG_MAX) || (ivalue == LONG_MIN)) && errno == ERANGE))
 				throw format("%s is not a valid value for %s", s.c_str(), name.toString().c_str());
 
 			// It should not be possible for this set to fail now since the dummy set above had to have worked.
@@ -1847,5 +1849,13 @@ TEST_CASE("/backup/s3/guess_region") {
 	Reference<S3BlobStoreEndpoint> s3 = S3BlobStoreEndpoint::fromString(url, {}, &resource, &error, &parameters);
 	ASSERT(s3->getRegion() == "us-west-2");
 
+	url = "blobstore://s3.us-west-2.amazonaws.com/resource_name?bucket=bucket_name&sc=922337203685477580700";
+	try {
+		s3 = S3BlobStoreEndpoint::fromString(url, {}, &resource, &error, &parameters);
+		ASSERT(false); // not reached
+	} catch (Error& e) {
+		// conversion of 922337203685477580700 to long int will overflow
+		ASSERT_EQ(e.code(), error_code_backup_invalid_url);
+	}
 	return Void();
 }

From d5ec2edafc881c4df064bc5e82daec6a7480612a Mon Sep 17 00:00:00 2001
From: Zhe Wang <zhe.wang@wustl.edu>
Date: Tue, 3 Oct 2023 15:26:21 -0700
Subject: [PATCH 277/468] fix-redistribute-cli

---
 fdbcli/RedistributeCommand.actor.cpp | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/fdbcli/RedistributeCommand.actor.cpp b/fdbcli/RedistributeCommand.actor.cpp
index ee782493402..d877b97ab31 100644
--- a/fdbcli/RedistributeCommand.actor.cpp
+++ b/fdbcli/RedistributeCommand.actor.cpp
@@ -30,9 +30,9 @@ ACTOR Future<bool> redistributeCommandActor(Reference<IClusterConnectionRecord>
 	if (tokens.size() == 3) {
 		Key begin = tokens[1];
 		Key end = tokens[2];
-		if (begin == end) {
+		if (begin >= end) {
 			printUsage(tokens[0]);
-			result = false;
+			return false;
 		}
 		wait(redistribute(clusterFile, KeyRangeRef(begin, end), /*timeoutSeconds=*/30));
 	} else {
@@ -45,8 +45,8 @@ ACTOR Future<bool> redistributeCommandActor(Reference<IClusterConnectionRecord>
 CommandFactory RedistributeFactory(
     "redistribute",
     CommandHelp("redistribute [Begin] [End]",
-                "Redistribute data of the range<Begin, End> among the cluster, where [Begin]!=[End]\n",
-                "Given an input range, say <b, d>, where b!=d and we suppose the current FDB"
+                "Redistribute data of the range<Begin, End> among the cluster, where [Begin]<[End]\n",
+                "Given an input range, say <b, d>, where b<d and we suppose the current FDB"
                 "internal shard boundary is [a, c), [c, d), [d, e),"
                 "our splitting algorithm splits the input range into [b, c) and [c, d)."
                 "Then the two ranges will be redistributed among the cluster.\n"));

From 059f47bc42fd1198c72586c3ce72c945a5493b84 Mon Sep 17 00:00:00 2001
From: Zhe Wang <zhe.wang@wustl.edu>
Date: Fri, 29 Sep 2023 20:55:11 -0700
Subject: [PATCH 278/468] auto hot storage server detect

---
 fdbclient/ServerKnobs.cpp                   |  9 ++++-
 fdbclient/ServerKnobs.h                     |  7 ++++
 fdbserver/DDTeamCollection.actor.cpp        | 25 ++++++++++--
 fdbserver/DDTeamCollection.h                |  4 +-
 fdbserver/DataDistribution.actor.cpp        |  8 +++-
 fdbserver/DataDistribution.actor.h          | 10 +++++
 fdbserver/DataDistributionQueue.actor.cpp   |  3 +-
 fdbserver/DataDistributionTracker.actor.cpp | 40 +++++++++++++++++++
 fdbserver/TCInfo.actor.cpp                  | 18 +++++++++
 fdbserver/TCInfo.h                          | 43 +++++++++++++++++++++
 10 files changed, 158 insertions(+), 9 deletions(-)

diff --git a/fdbclient/ServerKnobs.cpp b/fdbclient/ServerKnobs.cpp
index 953963f38a1..1c2c1428f44 100644
--- a/fdbclient/ServerKnobs.cpp
+++ b/fdbclient/ServerKnobs.cpp
@@ -163,6 +163,7 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi
 	init( PRIORITY_TEAM_1_LEFT,                                  800 );
 	init( PRIORITY_TEAM_FAILED,                                  805 );
 	init( PRIORITY_TEAM_0_LEFT,                                  809 );
+	init( PRIORITY_TEAM_STORAGE_QUEUE_TOO_LONG,                  940 ); if( randomize && BUGGIFY ) PRIORITY_TEAM_STORAGE_QUEUE_TOO_LONG = 345;
 	init( PRIORITY_SPLIT_SHARD,                                  950 ); if( randomize && BUGGIFY ) PRIORITY_SPLIT_SHARD = 350;
 	init( PRIORITY_MANUAL_SHARD_SPLIT,                           960 );
 
@@ -292,8 +293,14 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi
 	init( DD_STORAGE_WIGGLE_STUCK_THRESHOLD,                      20 );
 	init( DD_BUILD_EXTRA_TEAMS_OVERRIDE,                          10 ); if( randomize && BUGGIFY ) DD_BUILD_EXTRA_TEAMS_OVERRIDE = 2;
 	init( ENABLE_STORAGE_QUEUE_AWARE_TEAM_SELECTION,           false ); if( randomize && BUGGIFY ) ENABLE_STORAGE_QUEUE_AWARE_TEAM_SELECTION = true;
-	init( DD_TARGET_STORAGE_QUEUE_SIZE, TARGET_BYTES_PER_STORAGE_SERVER/3 ); if( randomize && BUGGIFY ) DD_TARGET_STORAGE_QUEUE_SIZE = TARGET_BYTES_PER_STORAGE_SERVER/10;
 	init( TRACE_STORAGE_QUEUE_AWARE_GET_TEAM_FOR_MANUAL_SPLIT_ONLY, true ); if (isSimulated) TRACE_STORAGE_QUEUE_AWARE_GET_TEAM_FOR_MANUAL_SPLIT_ONLY = false;
+	init( DD_TARGET_STORAGE_QUEUE_SIZE, TARGET_BYTES_PER_STORAGE_SERVER*0.35 ); if( randomize && BUGGIFY ) DD_TARGET_STORAGE_QUEUE_SIZE = TARGET_BYTES_PER_STORAGE_SERVER*0.035;
+	init( ENABLE_AUTO_SHARD_SPLIT_FOR_LONG_STORAGE_QUEUE,      false ); if( randomize && BUGGIFY ) ENABLE_AUTO_SHARD_SPLIT_FOR_LONG_STORAGE_QUEUE = true;
+	init( DD_SS_TOO_LONG_STORAGE_QUEUE_BYTES, TARGET_BYTES_PER_STORAGE_SERVER*0.5); if( randomize && BUGGIFY ) DD_SS_TOO_LONG_STORAGE_QUEUE_BYTES = TARGET_BYTES_PER_STORAGE_SERVER*0.05;
+	init( DD_SS_SHORT_STORAGE_QUEUE_BYTES, TARGET_BYTES_PER_STORAGE_SERVER*0.35); if( randomize && BUGGIFY ) DD_SS_SHORT_STORAGE_QUEUE_BYTES = TARGET_BYTES_PER_STORAGE_SERVER*0.035;
+	init( DD_STORAGE_QUEUE_TOO_LONG_DURATION,                  300.0 ); if( isSimulated ) DD_STORAGE_QUEUE_TOO_LONG_DURATION = deterministicRandom()->random01() * 10 + 1;
+	init( DD_MIN_LONG_STORAGE_QUEUE_SPLIT_INTERVAL_SEC,         60.0 ); if( isSimulated ) DD_MIN_LONG_STORAGE_QUEUE_SPLIT_INTERVAL_SEC = 5.0;
+	init( DD_MIN_SHARD_BYTES_PER_KSEC_TO_MOVE_OUT, SHARD_MIN_BYTES_PER_KSEC);
 
 	// TeamRemover
 	init( TR_FLAG_DISABLE_MACHINE_TEAM_REMOVER,                false ); if( randomize && BUGGIFY ) TR_FLAG_DISABLE_MACHINE_TEAM_REMOVER = deterministicRandom()->random01() < 0.1 ? true : false; // false by default. disable the consistency check when it's true
diff --git a/fdbclient/ServerKnobs.h b/fdbclient/ServerKnobs.h
index 047239d7b94..54ed8d7abbb 100644
--- a/fdbclient/ServerKnobs.h
+++ b/fdbclient/ServerKnobs.h
@@ -157,6 +157,7 @@ class ServerKnobs : public KnobsImpl<ServerKnobs> {
 	int PRIORITY_TEAM_REDUNDANT;
 	int PRIORITY_MERGE_SHARD;
 	int PRIORITY_POPULATE_REGION;
+	int PRIORITY_TEAM_STORAGE_QUEUE_TOO_LONG;
 	int PRIORITY_TEAM_UNHEALTHY;
 	int PRIORITY_TEAM_2_LEFT;
 	int PRIORITY_TEAM_1_LEFT;
@@ -236,6 +237,12 @@ class ServerKnobs : public KnobsImpl<ServerKnobs> {
 	bool ENABLE_STORAGE_QUEUE_AWARE_TEAM_SELECTION; // experimental!
 	int64_t DD_TARGET_STORAGE_QUEUE_SIZE;
 	bool TRACE_STORAGE_QUEUE_AWARE_GET_TEAM_FOR_MANUAL_SPLIT_ONLY;
+	bool ENABLE_AUTO_SHARD_SPLIT_FOR_LONG_STORAGE_QUEUE;
+	int64_t DD_SS_TOO_LONG_STORAGE_QUEUE_BYTES;
+	int64_t DD_SS_SHORT_STORAGE_QUEUE_BYTES;
+	double DD_STORAGE_QUEUE_TOO_LONG_DURATION;
+	double DD_MIN_LONG_STORAGE_QUEUE_SPLIT_INTERVAL_SEC;
+	int64_t DD_MIN_SHARD_BYTES_PER_KSEC_TO_MOVE_OUT;
 
 	// TeamRemover to remove redundant teams
 	bool TR_FLAG_DISABLE_MACHINE_TEAM_REMOVER; // disable the machineTeamRemover actor
diff --git a/fdbserver/DDTeamCollection.actor.cpp b/fdbserver/DDTeamCollection.actor.cpp
index 18eca4c5094..4d350b0d2a8 100644
--- a/fdbserver/DDTeamCollection.actor.cpp
+++ b/fdbserver/DDTeamCollection.actor.cpp
@@ -1442,6 +1442,20 @@ class DDTeamCollectionImpl {
 					when(wait(storageMetadataTracker || storeTypeTracker)) {}
 					when(wait(server->ssVersionTooFarBehind.onChange())) {}
 					when(wait(self->disableFailingLaggingServers.onChange())) {}
+					when(wait(server->storageQueueTooLong.onChange())) {
+						TraceEvent(SevInfo, "StorageQueueTooLongTriggerSplit", self->distributorId)
+						    .detail("SSID", server->getId());
+						std::vector<ShardsAffectedByTeamFailure::Team> teams;
+						for (const auto& team : server->getTeams()) {
+							std::vector<UID> servers;
+							for (const auto& server : team->getServers()) {
+								servers.push_back(server->getId());
+							}
+							teams.push_back(ShardsAffectedByTeamFailure::Team(servers, self->primary));
+						}
+						self->triggerSplitForStorageQueueTooLong.send(
+						    ServerTeamInfo(server->getId(), teams, self->primary));
+					}
 				}
 
 				if (recordTeamCollectionInfo) {
@@ -3754,7 +3768,8 @@ DDTeamCollection::DDTeamCollection(Database const& cx,
                                    PromiseStream<GetMetricsRequest> getShardMetrics,
                                    Promise<UID> removeFailedServer,
                                    PromiseStream<Promise<int>> getUnhealthyRelocationCount,
-                                   PromiseStream<Promise<int64_t>> getAverageShardBytes)
+                                   PromiseStream<Promise<int64_t>> getAverageShardBytes,
+                                   PromiseStream<ServerTeamInfo> triggerSplitForStorageQueueTooLong)
   : doBuildTeams(true), lastBuildTeamsFailed(false), teamBuilder(Void()), lock(lock), output(output),
     unhealthyServers(0), storageWiggler(makeReference<StorageWiggler>(this)), processingWiggle(processingWiggle),
     shardsAffectedByTeamFailure(shardsAffectedByTeamFailure),
@@ -3769,7 +3784,7 @@ DDTeamCollection::DDTeamCollection(Database const& cx,
     medianAvailableSpace(SERVER_KNOBS->MIN_AVAILABLE_SPACE_RATIO), lastMedianAvailableSpaceUpdate(0),
     lowestUtilizationTeam(0), highestUtilizationTeam(0), getShardMetrics(getShardMetrics),
     getUnhealthyRelocationCount(getUnhealthyRelocationCount), getAverageShardBytes(getAverageShardBytes),
-    removeFailedServer(removeFailedServer),
+    triggerSplitForStorageQueueTooLong(triggerSplitForStorageQueueTooLong), removeFailedServer(removeFailedServer),
     ddTrackerStartingEventHolder(makeReference<EventCacheHolder>("DDTrackerStarting")),
     teamCollectionInfoEventHolder(makeReference<EventCacheHolder>("TeamCollectionInfo")),
     storageServerRecruitmentEventHolder(
@@ -5343,7 +5358,8 @@ class DDTeamCollectionUnitTest {
 		                                                           PromiseStream<GetMetricsRequest>(),
 		                                                           Promise<UID>(),
 		                                                           PromiseStream<Promise<int>>(),
-		                                                           PromiseStream<Promise<int64_t>>()));
+		                                                           PromiseStream<Promise<int64_t>>(),
+		                                                           PromiseStream<ServerTeamInfo>()));
 
 		for (int id = 1; id <= processCount; ++id) {
 			UID uid(id, 0);
@@ -5388,7 +5404,8 @@ class DDTeamCollectionUnitTest {
 		                                                           PromiseStream<GetMetricsRequest>(),
 		                                                           Promise<UID>(),
 		                                                           PromiseStream<Promise<int>>(),
-		                                                           PromiseStream<Promise<int64_t>>()));
+		                                                           PromiseStream<Promise<int64_t>>(),
+		                                                           PromiseStream<ServerTeamInfo>()));
 
 		for (int id = 1; id <= processCount; id++) {
 			UID uid(id, 0);
diff --git a/fdbserver/DDTeamCollection.h b/fdbserver/DDTeamCollection.h
index e14d8e2310f..e9d302a9c1a 100644
--- a/fdbserver/DDTeamCollection.h
+++ b/fdbserver/DDTeamCollection.h
@@ -256,6 +256,7 @@ class DDTeamCollection : public ReferenceCounted<DDTeamCollection> {
 	PromiseStream<GetMetricsRequest> getShardMetrics;
 	PromiseStream<Promise<int>> getUnhealthyRelocationCount;
 	PromiseStream<Promise<int64_t>> getAverageShardBytes;
+	PromiseStream<ServerTeamInfo> triggerSplitForStorageQueueTooLong;
 	Promise<UID> removeFailedServer;
 
 	// WIGGLING if an address is under storage wiggling.
@@ -636,7 +637,8 @@ class DDTeamCollection : public ReferenceCounted<DDTeamCollection> {
 	                 PromiseStream<GetMetricsRequest> getShardMetrics,
 	                 Promise<UID> removeFailedServer,
 	                 PromiseStream<Promise<int>> getUnhealthyRelocationCount,
-	                 PromiseStream<Promise<int64_t>> getAverageShardBytes);
+	                 PromiseStream<Promise<int64_t>> getAverageShardBytes,
+	                 PromiseStream<ServerTeamInfo> triggerSplitForStorageQueueTooLong);
 
 	~DDTeamCollection();
 
diff --git a/fdbserver/DataDistribution.actor.cpp b/fdbserver/DataDistribution.actor.cpp
index 1a67d426999..43a05917219 100644
--- a/fdbserver/DataDistribution.actor.cpp
+++ b/fdbserver/DataDistribution.actor.cpp
@@ -670,6 +670,7 @@ ACTOR Future<Void> dataDistribution(Reference<DataDistributorData> self,
 			state PromiseStream<RelocateShard> output;
 			state PromiseStream<RelocateShard> input;
 			state PromiseStream<Promise<int64_t>> getAverageShardBytes;
+			state PromiseStream<ServerTeamInfo> triggerSplitForStorageQueueTooLong;
 			state PromiseStream<Promise<int>> getUnhealthyRelocationCount;
 			state PromiseStream<GetMetricsRequest> getShardMetrics;
 			state Reference<AsyncVar<bool>> processingUnhealthy(new AsyncVar<bool>(false));
@@ -739,6 +740,7 @@ ACTOR Future<Void> dataDistribution(Reference<DataDistributorData> self,
 			                                                            getShardMetricsList,
 			                                                            manualShardSplit,
 			                                                            getAverageShardBytes.getFuture(),
+			                                                            triggerSplitForStorageQueueTooLong.getFuture(),
 			                                                            readyToStart,
 			                                                            anyZeroHealthyTeams,
 			                                                            self->ddId,
@@ -785,7 +787,8 @@ ACTOR Future<Void> dataDistribution(Reference<DataDistributorData> self,
 			    getShardMetrics,
 			    removeFailedServer,
 			    getUnhealthyRelocationCount,
-			    getAverageShardBytes);
+			    getAverageShardBytes,
+			    triggerSplitForStorageQueueTooLong);
 			teamCollectionsPtrs.push_back(primaryTeamCollection.getPtr());
 			auto recruitStorage = IAsyncListener<RequestStream<RecruitStorageRequest>>::create(
 			    self->dbInfo, [](auto const& info) { return info.clusterInterface.recruitStorage; });
@@ -807,7 +810,8 @@ ACTOR Future<Void> dataDistribution(Reference<DataDistributorData> self,
 				                                    getShardMetrics,
 				                                    removeFailedServer,
 				                                    getUnhealthyRelocationCount,
-				                                    getAverageShardBytes);
+				                                    getAverageShardBytes,
+				                                    triggerSplitForStorageQueueTooLong);
 				teamCollectionsPtrs.push_back(remoteTeamCollection.getPtr());
 				remoteTeamCollection->teamCollections = teamCollectionsPtrs;
 				actors.push_back(reportErrorsExcept(
diff --git a/fdbserver/DataDistribution.actor.h b/fdbserver/DataDistribution.actor.h
index 0776ffd90a2..0c942821a71 100644
--- a/fdbserver/DataDistribution.actor.h
+++ b/fdbserver/DataDistribution.actor.h
@@ -208,6 +208,15 @@ class ShardsAffectedByTeamFailure : public ReferenceCounted<ShardsAffectedByTeam
 	void insert(Team team, KeyRange const& range);
 };
 
+struct ServerTeamInfo {
+	UID serverId;
+	std::vector<ShardsAffectedByTeamFailure::Team> teams;
+	bool primary;
+
+	ServerTeamInfo() {}
+	ServerTeamInfo(UID serverId, std::vector<ShardsAffectedByTeamFailure::Team> teams, bool primary)
+	  : serverId(serverId), teams(teams), primary(primary) {}
+};
 // DDShardInfo is so named to avoid link-time name collision with ShardInfo within the StorageServer
 struct DDShardInfo {
 	Key key;
@@ -259,6 +268,7 @@ ACTOR Future<Void> dataDistributionTracker(Reference<InitialDataDistribution> in
                                            PromiseStream<GetMetricsListRequest> getShardMetricsList,
                                            PromiseStream<DistributorSplitRangeRequest> manualShardSplit,
                                            FutureStream<Promise<int64_t>> getAverageShardBytes,
+                                           FutureStream<ServerTeamInfo> triggerSplitForStorageQueueTooLong,
                                            Promise<Void> readyToStart,
                                            Reference<AsyncVar<bool>> zeroHealthyTeams,
                                            UID distributorId,
diff --git a/fdbserver/DataDistributionQueue.actor.cpp b/fdbserver/DataDistributionQueue.actor.cpp
index 5c61e0bd7b0..a20389f0a19 100644
--- a/fdbserver/DataDistributionQueue.actor.cpp
+++ b/fdbserver/DataDistributionQueue.actor.cpp
@@ -64,7 +64,8 @@ struct RelocateData {
 	                    rs.priority == SERVER_KNOBS->PRIORITY_REBALANCE_UNDERUTILIZED_TEAM ||
 	                    rs.priority == SERVER_KNOBS->PRIORITY_SPLIT_SHARD ||
 	                    rs.priority == SERVER_KNOBS->PRIORITY_TEAM_REDUNDANT ||
-	                    rs.priority == SERVER_KNOBS->PRIORITY_MANUAL_SHARD_SPLIT),
+	                    rs.priority == SERVER_KNOBS->PRIORITY_MANUAL_SHARD_SPLIT ||
+	                    rs.priority == SERVER_KNOBS->PRIORITY_TEAM_STORAGE_QUEUE_TOO_LONG),
 	    cancellable(true), interval("QueuedRelocation") {}
 
 	static bool isHealthPriority(int priority) {
diff --git a/fdbserver/DataDistributionTracker.actor.cpp b/fdbserver/DataDistributionTracker.actor.cpp
index 7be7ea40038..d91a77d07be 100644
--- a/fdbserver/DataDistributionTracker.actor.cpp
+++ b/fdbserver/DataDistributionTracker.actor.cpp
@@ -984,6 +984,7 @@ ACTOR Future<Void> dataDistributionTracker(Reference<InitialDataDistribution> in
                                            PromiseStream<GetMetricsListRequest> getShardMetricsList,
                                            PromiseStream<DistributorSplitRangeRequest> manualShardSplit,
                                            FutureStream<Promise<int64_t>> getAverageShardBytes,
+                                           FutureStream<ServerTeamInfo> triggerSplitForStorageQueueTooLong,
                                            Promise<Void> readyToStart,
                                            Reference<AsyncVar<bool>> anyZeroHealthyTeams,
                                            UID distributorId,
@@ -1008,6 +1009,45 @@ ACTOR Future<Void> dataDistributionTracker(Reference<InitialDataDistribution> in
 			when(Promise<int64_t> req = waitNext(getAverageShardBytes)) {
 				req.send(self.maxShardSize->get().get() / 2);
 			}
+			when(ServerTeamInfo req = waitNext(triggerSplitForStorageQueueTooLong)) {
+				TraceEvent e("DDTrackerStorageServerQueueTooLongNotified", self.distributorId);
+				e.detail("Server", req.serverId);
+				e.detail("Teams", req.teams.size());
+				int64_t maxShardWriteTraffic = 0;
+				KeyRange shardToMove;
+				ShardsAffectedByTeamFailure::Team selectedTeam;
+				for (const auto& team : req.teams) {
+					for (auto const& shard : self.shardsAffectedByTeamFailure->getShardsFor(team)) {
+						for (auto it : self.shards.intersectingRanges(shard)) {
+							if (it->value().stats->get().present()) {
+								int64_t shardWriteTraffic = it->value().stats->get().get().metrics.bytesPerKSecond;
+								if (shardWriteTraffic > maxShardWriteTraffic) {
+									shardToMove = it->range();
+									selectedTeam = team;
+									maxShardWriteTraffic = shardWriteTraffic;
+								}
+							}
+						}
+					}
+				}
+				if (!shardToMove.empty()) {
+					e.detail("TeamSelected", selectedTeam.servers);
+					e.detail("ShardSelected", shardToMove);
+					e.detail("ShardWriteBytesPerKSec", maxShardWriteTraffic);
+					if (maxShardWriteTraffic >= SERVER_KNOBS->DD_MIN_SHARD_BYTES_PER_KSEC_TO_MOVE_OUT) {
+						RelocateShard rs;
+						rs.keys = shardToMove;
+						rs.priority = SERVER_KNOBS->PRIORITY_TEAM_STORAGE_QUEUE_TOO_LONG;
+						self.output.send(rs);
+						TraceEvent("SendRelocateToDDQueue", self.distributorId)
+						    .detail("ServerPrimary", req.primary)
+						    .detail("ServerTeam", selectedTeam.servers)
+						    .detail("KeyBegin", rs.keys.begin)
+						    .detail("KeyEnd", rs.keys.end)
+						    .detail("Priority", rs.priority);
+					}
+				}
+			}
 			when(wait(loggingTrigger)) {
 				TraceEvent("DDTrackerStats", self.distributorId)
 				    .detail("Shards", self.shards.size())
diff --git a/fdbserver/TCInfo.actor.cpp b/fdbserver/TCInfo.actor.cpp
index fce320e5604..3ca843511cb 100644
--- a/fdbserver/TCInfo.actor.cpp
+++ b/fdbserver/TCInfo.actor.cpp
@@ -92,6 +92,24 @@ class TCServerInfoImpl {
 				server->collection->removeLaggingStorageServer(server->lastKnownInterface.locality.zoneId().get());
 			}
 		}
+
+		if (SERVER_KNOBS->ENABLE_AUTO_SHARD_SPLIT_FOR_LONG_STORAGE_QUEUE) {
+			int64_t queueSize = server->getStorageQueueSize();
+			bool storageQueueKeepTooLong = server->updateAndGetStorageQueueTooLong(queueSize, server->getId());
+			double currentTime = now();
+			if (storageQueueKeepTooLong) {
+				if (!server->lastStorageQueueTooLongTriggerTime.present() ||
+				    currentTime - server->lastStorageQueueTooLongTriggerTime.get() >
+				        SERVER_KNOBS->DD_MIN_LONG_STORAGE_QUEUE_SPLIT_INTERVAL_SEC) {
+					server->storageQueueTooLong.trigger(); // will trigger split
+					TraceEvent(SevInfo, "StorageQueueTooLongNotified", server->collection->getDistributorId())
+					    .detail("SSID", server->getId())
+					    .detail("CurrentQueueSize", queueSize);
+					server->lastStorageQueueTooLongTriggerTime = currentTime;
+				}
+			}
+		}
+
 		return Void();
 	}
 
diff --git a/fdbserver/TCInfo.h b/fdbserver/TCInfo.h
index 222695e6504..9f67a3e28d2 100644
--- a/fdbserver/TCInfo.h
+++ b/fdbserver/TCInfo.h
@@ -47,6 +47,9 @@ class TCServerInfo : public ReferenceCounted<TCServerInfo> {
 	int64_t dataInFlightToServer;
 	std::vector<Reference<TCTeamInfo>> teams;
 	ErrorOr<GetStorageMetricsReply> metrics;
+	Optional<double> storageQueueTooLongStartTime;
+	Optional<double> whenRedistributeForLongStorageQueue;
+	Optional<double> lastStorageQueueTooLongTriggerTime;
 
 	GetStorageMetricsReply const& getMetrics() const { return metrics.get(); }
 
@@ -66,6 +69,7 @@ class TCServerInfo : public ReferenceCounted<TCServerInfo> {
 	Promise<Void> updated;
 	AsyncVar<bool> wrongStoreTypeToRemove;
 	AsyncVar<bool> ssVersionTooFarBehind;
+	AsyncVar<Void> storageQueueTooLong;
 
 	TCServerInfo(StorageServerInterface ssi,
 	             DDTeamCollection* collection,
@@ -108,6 +112,45 @@ class TCServerInfo : public ReferenceCounted<TCServerInfo> {
 	static Future<Void> updateServerMetrics(Reference<TCServerInfo> server);
 	Future<Void> serverMetricsPolling();
 
+	// Return if storage queue keeps too long for a while
+	bool updateAndGetStorageQueueTooLong(int64_t currentBytes, UID ssid) {
+		double currentTime = now();
+		if (currentBytes > SERVER_KNOBS->DD_SS_TOO_LONG_STORAGE_QUEUE_BYTES) {
+			if (!storageQueueTooLongStartTime.present()) {
+				storageQueueTooLongStartTime = currentTime;
+				TraceEvent(SevWarn, "TCDetectStorageQueueBecomeTooLong")
+				    .detail("SSID", ssid)
+				    .detail("StorageQueueBytes", currentBytes)
+				    .detail("Duration", currentTime - storageQueueTooLongStartTime.get());
+			} else {
+				TraceEvent(SevDebug, "TCDetectStorageQueueTooLong")
+				    .detail("SSID", ssid)
+				    .detail("StorageQueueBytes", currentBytes)
+				    .detail("Duration", currentTime - storageQueueTooLongStartTime.get());
+			}
+		} else if (currentBytes < SERVER_KNOBS->DD_SS_SHORT_STORAGE_QUEUE_BYTES) {
+			if (storageQueueTooLongStartTime.present()) {
+				storageQueueTooLongStartTime = Optional<double>();
+				TraceEvent(SevWarn, "TCDetectStorageQueueBecomeNormal")
+				    .detail("SSID", ssid)
+				    .detail("StorageQueueBytes", currentBytes);
+			}
+		} else {
+			if (storageQueueTooLongStartTime.present()) {
+				TraceEvent(SevDebug, "TCDetectStorageQueueLong")
+				    .detail("SSID", ssid)
+				    .detail("StorageQueueBytes", currentBytes)
+				    .detail("Duration", currentTime - storageQueueTooLongStartTime.get());
+			}
+		}
+		if (storageQueueTooLongStartTime.present() &&
+		    currentTime - storageQueueTooLongStartTime.get() > SERVER_KNOBS->DD_STORAGE_QUEUE_TOO_LONG_DURATION) {
+			return true;
+		} else {
+			return false;
+		}
+	}
+
 	~TCServerInfo();
 };
 

From 8a9d6fddd544f04fe6059b470472dde7f01d2623 Mon Sep 17 00:00:00 2001
From: Zhe Wang <zhe.wang@wustl.edu>
Date: Mon, 9 Oct 2023 11:18:38 -0700
Subject: [PATCH 279/468] address comments

---
 fdbserver/DataDistributionTracker.actor.cpp | 25 ++++++++++-----------
 fdbserver/TCInfo.h                          |  3 +--
 2 files changed, 13 insertions(+), 15 deletions(-)

diff --git a/fdbserver/DataDistributionTracker.actor.cpp b/fdbserver/DataDistributionTracker.actor.cpp
index d91a77d07be..faead47450d 100644
--- a/fdbserver/DataDistributionTracker.actor.cpp
+++ b/fdbserver/DataDistributionTracker.actor.cpp
@@ -1021,7 +1021,8 @@ ACTOR Future<Void> dataDistributionTracker(Reference<InitialDataDistribution> in
 						for (auto it : self.shards.intersectingRanges(shard)) {
 							if (it->value().stats->get().present()) {
 								int64_t shardWriteTraffic = it->value().stats->get().get().metrics.bytesPerKSecond;
-								if (shardWriteTraffic > maxShardWriteTraffic) {
+								if (shardWriteTraffic > maxShardWriteTraffic &&
+								    shardWriteTraffic > SERVER_KNOBS->DD_MIN_SHARD_BYTES_PER_KSEC_TO_MOVE_OUT) {
 									shardToMove = it->range();
 									selectedTeam = team;
 									maxShardWriteTraffic = shardWriteTraffic;
@@ -1034,18 +1035,16 @@ ACTOR Future<Void> dataDistributionTracker(Reference<InitialDataDistribution> in
 					e.detail("TeamSelected", selectedTeam.servers);
 					e.detail("ShardSelected", shardToMove);
 					e.detail("ShardWriteBytesPerKSec", maxShardWriteTraffic);
-					if (maxShardWriteTraffic >= SERVER_KNOBS->DD_MIN_SHARD_BYTES_PER_KSEC_TO_MOVE_OUT) {
-						RelocateShard rs;
-						rs.keys = shardToMove;
-						rs.priority = SERVER_KNOBS->PRIORITY_TEAM_STORAGE_QUEUE_TOO_LONG;
-						self.output.send(rs);
-						TraceEvent("SendRelocateToDDQueue", self.distributorId)
-						    .detail("ServerPrimary", req.primary)
-						    .detail("ServerTeam", selectedTeam.servers)
-						    .detail("KeyBegin", rs.keys.begin)
-						    .detail("KeyEnd", rs.keys.end)
-						    .detail("Priority", rs.priority);
-					}
+					RelocateShard rs;
+					rs.keys = shardToMove;
+					rs.priority = SERVER_KNOBS->PRIORITY_TEAM_STORAGE_QUEUE_TOO_LONG;
+					self.output.send(rs);
+					TraceEvent("SendRelocateToDDQueue", self.distributorId)
+					    .detail("ServerPrimary", req.primary)
+					    .detail("ServerTeam", selectedTeam.servers)
+					    .detail("KeyBegin", rs.keys.begin)
+					    .detail("KeyEnd", rs.keys.end)
+					    .detail("Priority", rs.priority);
 				}
 			}
 			when(wait(loggingTrigger)) {
diff --git a/fdbserver/TCInfo.h b/fdbserver/TCInfo.h
index 9f67a3e28d2..ed163c82a87 100644
--- a/fdbserver/TCInfo.h
+++ b/fdbserver/TCInfo.h
@@ -48,7 +48,6 @@ class TCServerInfo : public ReferenceCounted<TCServerInfo> {
 	std::vector<Reference<TCTeamInfo>> teams;
 	ErrorOr<GetStorageMetricsReply> metrics;
 	Optional<double> storageQueueTooLongStartTime;
-	Optional<double> whenRedistributeForLongStorageQueue;
 	Optional<double> lastStorageQueueTooLongTriggerTime;
 
 	GetStorageMetricsReply const& getMetrics() const { return metrics.get(); }
@@ -112,7 +111,7 @@ class TCServerInfo : public ReferenceCounted<TCServerInfo> {
 	static Future<Void> updateServerMetrics(Reference<TCServerInfo> server);
 	Future<Void> serverMetricsPolling();
 
-	// Return if storage queue keeps too long for a while
+	// Return true if storage queue keeps too long for a while
 	bool updateAndGetStorageQueueTooLong(int64_t currentBytes, UID ssid) {
 		double currentTime = now();
 		if (currentBytes > SERVER_KNOBS->DD_SS_TOO_LONG_STORAGE_QUEUE_BYTES) {

From 834cd01eb58bfcf36e0806e4700546e55dfea615 Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Wed, 11 Oct 2023 15:15:02 -0700
Subject: [PATCH 280/468] Fix trackExcludedServers to use ExclusionTracker

---
 fdbserver/CMakeLists.txt                      |   1 +
 fdbserver/DDTeamCollection.actor.cpp          | 111 +++++-------------
 .../fdbserver => }/ExclusionTracker.actor.h   |   0
 3 files changed, 28 insertions(+), 84 deletions(-)
 rename fdbserver/{include/fdbserver => }/ExclusionTracker.actor.h (100%)

diff --git a/fdbserver/CMakeLists.txt b/fdbserver/CMakeLists.txt
index d01ed15ae46..31d8c3ed8ea 100644
--- a/fdbserver/CMakeLists.txt
+++ b/fdbserver/CMakeLists.txt
@@ -41,6 +41,7 @@ set(FDBSERVER_SRCS
   DiskQueue.actor.cpp
   EncryptKeyProxy.actor.cpp
   EncryptKeyProxyInterface.h
+  ExclusionTracker.actor.h
   FDBExecHelper.actor.cpp
   FDBExecHelper.actor.h
   FDBRocksDBVersion.h
diff --git a/fdbserver/DDTeamCollection.actor.cpp b/fdbserver/DDTeamCollection.actor.cpp
index 18eca4c5094..dfb6d67a74f 100644
--- a/fdbserver/DDTeamCollection.actor.cpp
+++ b/fdbserver/DDTeamCollection.actor.cpp
@@ -19,6 +19,7 @@
  */
 
 #include "fdbserver/DDTeamCollection.h"
+#include "fdbserver/ExclusionTracker.actor.h"
 #include "flow/Trace.h"
 #include "flow/actorcompiler.h" // This must be the last #include.
 #include <climits>
@@ -1928,96 +1929,38 @@ class DDTeamCollectionImpl {
 
 	ACTOR static Future<Void> trackExcludedServers(DDTeamCollection* self) {
 		// Fetch the list of excluded servers
-		state ReadYourWritesTransaction tr(self->cx);
+		state ExclusionTracker exclusionTracker(self->cx);
 		loop {
-			try {
-				tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
-				state Future<RangeResult> fresultsExclude = tr.getRange(excludedServersKeys, CLIENT_KNOBS->TOO_MANY);
-				state Future<RangeResult> fresultsFailed = tr.getRange(failedServersKeys, CLIENT_KNOBS->TOO_MANY);
-				state Future<RangeResult> flocalitiesExclude =
-				    tr.getRange(excludedLocalityKeys, CLIENT_KNOBS->TOO_MANY);
-				state Future<RangeResult> flocalitiesFailed = tr.getRange(failedLocalityKeys, CLIENT_KNOBS->TOO_MANY);
-				state Future<std::vector<ProcessData>> fworkers = getWorkers(self->cx);
-				wait(success(fresultsExclude) && success(fresultsFailed) && success(flocalitiesExclude) &&
-				     success(flocalitiesFailed));
-
-				state RangeResult excludedResults = fresultsExclude.get();
-				ASSERT(!excludedResults.more && excludedResults.size() < CLIENT_KNOBS->TOO_MANY);
-
-				state RangeResult failedResults = fresultsFailed.get();
-				ASSERT(!failedResults.more && failedResults.size() < CLIENT_KNOBS->TOO_MANY);
-
-				state RangeResult excludedLocalityResults = flocalitiesExclude.get();
-				ASSERT(!excludedLocalityResults.more && excludedLocalityResults.size() < CLIENT_KNOBS->TOO_MANY);
-
-				state RangeResult failedLocalityResults = flocalitiesFailed.get();
-				ASSERT(!failedLocalityResults.more && failedLocalityResults.size() < CLIENT_KNOBS->TOO_MANY);
-
-				state std::set<AddressExclusion> excluded;
-				state std::set<AddressExclusion> failed;
-				for (const auto& r : excludedResults) {
-					AddressExclusion addr = decodeExcludedServersKey(r.key);
-					if (addr.isValid()) {
-						excluded.insert(addr);
-					}
-				}
-				for (const auto& r : failedResults) {
-					AddressExclusion addr = decodeFailedServersKey(r.key);
-					if (addr.isValid()) {
-						failed.insert(addr);
-					}
-				}
-
-				wait(success(fworkers));
-				std::vector<ProcessData> workers = fworkers.get();
-				for (const auto& r : excludedLocalityResults) {
-					std::string locality = decodeExcludedLocalityKey(r.key);
-					std::set<AddressExclusion> localityExcludedAddresses = getAddressesByLocality(workers, locality);
-					excluded.insert(localityExcludedAddresses.begin(), localityExcludedAddresses.end());
-				}
-				for (const auto& r : failedLocalityResults) {
-					std::string locality = decodeFailedLocalityKey(r.key);
-					std::set<AddressExclusion> localityFailedAddresses = getAddressesByLocality(workers, locality);
-					failed.insert(localityFailedAddresses.begin(), localityFailedAddresses.end());
-				}
-
-				// Reset and reassign self->excludedServers based on excluded, but we only
-				// want to trigger entries that are different
-				// Do not retrigger and double-overwrite failed or wiggling servers
-				auto old = self->excludedServers.getKeys();
-				for (const auto& o : old) {
-					if (!excluded.count(o) && !failed.count(o) &&
-					    !(self->excludedServers.count(o) &&
-					      self->excludedServers.get(o) == DDTeamCollection::Status::WIGGLING)) {
-						self->excludedServers.set(o, DDTeamCollection::Status::NONE);
-					}
+			// wait for new set of excluded servers
+			wait(exclusionTracker.changed.onTrigger());
+
+			// Reset and reassign self->excludedServers based on excluded, but we only
+			// want to trigger entries that are different
+			// Do not retrigger and double-overwrite failed or wiggling servers
+			auto old = self->excludedServers.getKeys();
+			for (const auto& o : old) {
+				if (!exclusionTracker.excluded.count(o) && !exclusionTracker.failed.count(o) &&
+				    !(self->excludedServers.count(o) &&
+				      self->excludedServers.get(o) == DDTeamCollection::Status::WIGGLING)) {
+					self->excludedServers.set(o, DDTeamCollection::Status::NONE);
 				}
-				for (const auto& n : excluded) {
-					if (!failed.count(n)) {
-						self->excludedServers.set(n, DDTeamCollection::Status::EXCLUDED);
-					}
+			}
+			for (const auto& n : exclusionTracker.excluded) {
+				if (!exclusionTracker.failed.count(n)) {
+					self->excludedServers.set(n, DDTeamCollection::Status::EXCLUDED);
 				}
+			}
 
-				for (const auto& f : failed) {
-					self->excludedServers.set(f, DDTeamCollection::Status::FAILED);
-				}
+			for (const auto& f : exclusionTracker.failed) {
+				self->excludedServers.set(f, DDTeamCollection::Status::FAILED);
+			}
 
-				TraceEvent("DDExcludedServersChanged", self->distributorId)
-				    .detail("AddressesExcluded", excludedResults.size())
-				    .detail("AddressesFailed", failedResults.size())
-				    .detail("LocalitiesExcluded", excludedLocalityResults.size())
-				    .detail("LocalitiesFailed", failedLocalityResults.size());
+			TraceEvent("DDExcludedServersChanged", self->distributorId)
+			    .detail("AddressesExcluded", exclusionTracker.excluded.size())
+			    .detail("AddressesFailed", exclusionTracker.failed.size())
+			    .detail("Primary", self->isPrimary());
 
-				self->restartRecruiting.trigger();
-				state Future<Void> watchFuture =
-				    tr.watch(excludedServersVersionKey) || tr.watch(failedServersVersionKey) ||
-				    tr.watch(excludedLocalityVersionKey) || tr.watch(failedLocalityVersionKey);
-				wait(tr.commit());
-				wait(watchFuture);
-				tr.reset();
-			} catch (Error& e) {
-				wait(tr.onError(e));
-			}
+			self->restartRecruiting.trigger();
 		}
 	}
 
diff --git a/fdbserver/include/fdbserver/ExclusionTracker.actor.h b/fdbserver/ExclusionTracker.actor.h
similarity index 100%
rename from fdbserver/include/fdbserver/ExclusionTracker.actor.h
rename to fdbserver/ExclusionTracker.actor.h

From a3dbd4baf7eef987a8b7b77c296476127707a8f3 Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyu_zhou@apple.com>
Date: Fri, 13 Oct 2023 15:32:54 -0700
Subject: [PATCH 281/468] Add 7.1.42, 7.1.43 release notes (#10981)

---
 .../source/release-notes/release-notes-710.rst     | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/documentation/sphinx/source/release-notes/release-notes-710.rst b/documentation/sphinx/source/release-notes/release-notes-710.rst
index 32781050fd6..e7fe3b12619 100644
--- a/documentation/sphinx/source/release-notes/release-notes-710.rst
+++ b/documentation/sphinx/source/release-notes/release-notes-710.rst
@@ -4,6 +4,20 @@
 Release Notes
 #############
 
+7.1.43
+======
+* Same as 7.1.42 release with AVX enabled.
+
+7.1.42
+======
+* Released with AVX disabled.
+* Added redistribute fdbcli command to manually split shards. `(PR #10909) <https://github.com/apple/foundationdb/pull/10909>`_, `(PR #10936) <https://github.com/apple/foundationdb/pull/10936>`_, `(PR #10942) <https://github.com/apple/foundationdb/pull/10942>`_, `(PR #10905) <https://github.com/apple/foundationdb/pull/10905>`_, and `(PR #10958) <https://github.com/apple/foundationdb/pull/10958>`_
+* Fixed a MacOS linking issue for go bindings. `(PR #10924) <https://github.com/apple/foundationdb/pull/10924>`_
+* Added knobs to control backup retry delays for blob stores. `(PR #10947) <https://github.com/apple/foundationdb/pull/10947>`_
+* Fixed two use-after-free bugs for backup agents. `(PR #10951) <https://github.com/apple/foundationdb/pull/10951>`_
+* Added automatic range split for hot storage queue. `(PR #10932) <https://github.com/apple/foundationdb/pull/10932>`_
+* Fixed multiple bugs related to locality based exclusions. `(PR #10976) <https://github.com/apple/foundationdb/pull/10976>`_
+
 7.1.41
 ======
 * Same as 7.1.40 release with AVX enabled.

From 9c0ba06fb653945cd5e28560fbb9f6b140b86258 Mon Sep 17 00:00:00 2001
From: Zhe Wang <zhe.wang@wustl.edu>
Date: Mon, 16 Oct 2023 10:49:46 -0700
Subject: [PATCH 282/468] add write traffic to ddMetricsGetRange

---
 fdbclient/FDBTypes.h                        | 11 +++++++----
 fdbclient/SpecialKeySpace.actor.cpp         |  1 +
 fdbserver/DataDistributionTracker.actor.cpp |  4 +++-
 3 files changed, 11 insertions(+), 5 deletions(-)

diff --git a/fdbclient/FDBTypes.h b/fdbclient/FDBTypes.h
index 34246a4a25f..266c305cdf6 100644
--- a/fdbclient/FDBTypes.h
+++ b/fdbclient/FDBTypes.h
@@ -1283,16 +1283,19 @@ struct HealthMetrics {
 
 struct DDMetricsRef {
 	int64_t shardBytes;
+	int64_t shardBytesPerKSecond;
 	KeyRef beginKey;
 
-	DDMetricsRef() : shardBytes(0) {}
-	DDMetricsRef(int64_t bytes, KeyRef begin) : shardBytes(bytes), beginKey(begin) {}
+	DDMetricsRef() : shardBytes(0), shardBytesPerKSecond(0) {}
+	DDMetricsRef(int64_t bytes, int64_t bytesPerKSecond, KeyRef begin)
+	  : shardBytes(bytes), shardBytesPerKSecond(bytesPerKSecond), beginKey(begin) {}
 	DDMetricsRef(Arena& a, const DDMetricsRef& copyFrom)
-	  : shardBytes(copyFrom.shardBytes), beginKey(a, copyFrom.beginKey) {}
+	  : shardBytes(copyFrom.shardBytes), shardBytesPerKSecond(copyFrom.shardBytesPerKSecond),
+	    beginKey(a, copyFrom.beginKey) {}
 
 	template <class Ar>
 	void serialize(Ar& ar) {
-		serializer(ar, shardBytes, beginKey);
+		serializer(ar, shardBytes, beginKey, shardBytesPerKSecond);
 	}
 };
 
diff --git a/fdbclient/SpecialKeySpace.actor.cpp b/fdbclient/SpecialKeySpace.actor.cpp
index 1f0970b668d..23e7f43eeea 100644
--- a/fdbclient/SpecialKeySpace.actor.cpp
+++ b/fdbclient/SpecialKeySpace.actor.cpp
@@ -721,6 +721,7 @@ ACTOR Future<RangeResult> ddMetricsGetRangeActor(ReadYourWritesTransaction* ryw,
 				// Use json string encoded in utf-8 to encode the values, easy for adding more fields in the future
 				json_spirit::mObject statsObj;
 				statsObj["shard_bytes"] = ddMetricsRef.shardBytes;
+				statsObj["shard_bytes_per_ksecond"] = ddMetricsRef.shardBytesPerKSecond;
 				std::string statsString =
 				    json_spirit::write_string(json_spirit::mValue(statsObj), json_spirit::Output_options::raw_utf8);
 				ValueRef bytes(result.arena(), statsString);
diff --git a/fdbserver/DataDistributionTracker.actor.cpp b/fdbserver/DataDistributionTracker.actor.cpp
index faead47450d..27bc6a15fd8 100644
--- a/fdbserver/DataDistributionTracker.actor.cpp
+++ b/fdbserver/DataDistributionTracker.actor.cpp
@@ -945,7 +945,9 @@ ACTOR Future<Void> fetchShardMetricsList_impl(DataDistributionTracker* self, Get
 					break;
 				}
 				result.push_back_deep(result.arena(),
-				                      DDMetricsRef(stats->get().get().metrics.bytes, KeyRef(t.begin().toString())));
+				                      DDMetricsRef(stats->get().get().metrics.bytes,
+				                                   stats->get().get().metrics.bytesPerKSecond,
+				                                   KeyRef(t.begin().toString())));
 				++shardNum;
 				if (shardNum >= req.shardLimit) {
 					break;

From 1083418728e71ce4deecf8a39b8a28b1230cc2eb Mon Sep 17 00:00:00 2001
From: Zhe Wang <zhe.wang@wustl.edu>
Date: Tue, 17 Oct 2023 13:24:59 -0700
Subject: [PATCH 283/468] Fix unknown key in DataDistributionMetrics test

---
 fdbclient/Schemas.cpp | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/fdbclient/Schemas.cpp b/fdbclient/Schemas.cpp
index 278fb9bf04c..bbc6f7faaff 100644
--- a/fdbclient/Schemas.cpp
+++ b/fdbclient/Schemas.cpp
@@ -1063,7 +1063,8 @@ const KeyRef JSONSchemas::latencyBandConfigurationSchema = LiteralStringRef(R"co
 
 const KeyRef JSONSchemas::dataDistributionStatsSchema = LiteralStringRef(R"""(
 {
-  "shard_bytes": 1947000
+  "shard_bytes": 1947000,
+  "shard_bytes_per_ksecond": 1000000
 }
 )""");
 

From b0de8f313b50aefaed0913d7c90f77fd931f5fec Mon Sep 17 00:00:00 2001
From: neethuhaneesha <nbingi@apple.com>
Date: Tue, 17 Oct 2023 12:06:16 -0700
Subject: [PATCH 284/468] Compaction rate limiter changes.

---
 fdbserver/KeyValueStoreRocksDB.actor.cpp | 7 +------
 1 file changed, 1 insertion(+), 6 deletions(-)

diff --git a/fdbserver/KeyValueStoreRocksDB.actor.cpp b/fdbserver/KeyValueStoreRocksDB.actor.cpp
index 491c7ba237a..78b13b05e89 100644
--- a/fdbserver/KeyValueStoreRocksDB.actor.cpp
+++ b/fdbserver/KeyValueStoreRocksDB.actor.cpp
@@ -1144,7 +1144,7 @@ struct RocksDBKeyValueStore : IKeyValueStore {
 		                          SERVER_KNOBS->ROCKSDB_WRITE_RATE_LIMITER_BYTES_PER_SEC, // rate_bytes_per_sec
 		                          100 * 1000, // refill_period_us
 		                          10, // fairness
-		                          rocksdb::RateLimiter::Mode::kWritesOnly,
+		                          rocksdb::RateLimiter::Mode::kAllIo,
 		                          SERVER_KNOBS->ROCKSDB_WRITE_RATE_LIMITER_AUTO_TUNE)
 		                    : nullptr) {
 			if (SERVER_KNOBS->ROCKSDB_PERFCONTEXT_ENABLE) {
@@ -1334,11 +1334,6 @@ struct RocksDBKeyValueStore : IKeyValueStore {
 			}
 
 			double writeBeginTime = timer_monotonic();
-			if (rateLimiter) {
-				// Controls the total write rate of compaction and flush in bytes per second.
-				// Request for batchToCommit bytes. If this request cannot be satisfied, the call is blocked.
-				rateLimiter->Request(a.batchToCommit->GetDataSize() /* bytes */, rocksdb::Env::IO_HIGH);
-			}
 			rocksdb::Status s = db->Write(options, a.batchToCommit.get());
 			readIterPool->update();
 			double currTime = timer_monotonic();

From 427fcd42998a316f7259a6ec93d83521273fe167 Mon Sep 17 00:00:00 2001
From: Zhe Wang <zhe.wang@wustl.edu>
Date: Tue, 17 Oct 2023 15:54:51 -0700
Subject: [PATCH 285/468] enable DataDistributionMetrics test

---
 .../DataDistributionMetrics.actor.cpp         | 93 +++++++++++--------
 tests/CMakeLists.txt                          |  1 +
 tests/rare/DataDistributionMetrics.toml       |  8 ++
 3 files changed, 65 insertions(+), 37 deletions(-)
 create mode 100644 tests/rare/DataDistributionMetrics.toml

diff --git a/fdbserver/workloads/DataDistributionMetrics.actor.cpp b/fdbserver/workloads/DataDistributionMetrics.actor.cpp
index 4cb59c5768a..c3f3fa2667c 100644
--- a/fdbserver/workloads/DataDistributionMetrics.actor.cpp
+++ b/fdbserver/workloads/DataDistributionMetrics.actor.cpp
@@ -30,14 +30,18 @@ struct DataDistributionMetricsWorkload : KVWorkload {
 
 	int numShards, readPerTx, writePerTx;
 	int64_t avgBytes;
+	int64_t avgBytesPerKSeconds;
+	int64_t transactionTimeLimit;
 	double testDuration;
 	std::string keyPrefix;
 	PerfIntCounter commits, errors;
 	double delayPerLoop;
 
 	DataDistributionMetricsWorkload(WorkloadContext const& wcx)
-	  : KVWorkload(wcx), numShards(0), avgBytes(0), commits("Commits"), errors("Errors") {
+	  : KVWorkload(wcx), numShards(0), avgBytes(0), avgBytesPerKSeconds(0), commits("Commits"), errors("Errors") {
 		testDuration = getOption(options, LiteralStringRef("testDuration"), 10.0);
+		// transaction time out duration(ms)
+		transactionTimeLimit = getOption(options, "transactionTimeLimit"_sr, 10000);
 		keyPrefix = getOption(options, LiteralStringRef("keyPrefix"), LiteralStringRef("DDMetrics")).toString();
 		readPerTx = getOption(options, LiteralStringRef("readPerTransaction"), 1);
 		writePerTx = getOption(options, LiteralStringRef("writePerTransaction"), 5 * readPerTx);
@@ -73,6 +77,9 @@ struct DataDistributionMetricsWorkload : KVWorkload {
 	ACTOR Future<Void> resultConsistencyCheckClient(Database cx, DataDistributionMetricsWorkload* self) {
 		state Reference<ReadYourWritesTransaction> tr = makeReference<ReadYourWritesTransaction>(cx);
 		loop {
+			tr->setOption(FDBTransactionOptions::RAW_ACCESS);
+			tr->setOption(FDBTransactionOptions::TIMEOUT,
+			              StringRef((uint8_t*)&self->transactionTimeLimit, sizeof(int64_t)));
 			try {
 				wait(delay(self->delayPerLoop));
 				int startIndex = deterministicRandom()->randomInt(0, self->nodeCount - 1);
@@ -88,7 +95,7 @@ struct DataDistributionMetricsWorkload : KVWorkload {
 				// the range. If we didn't read through the end of the range, then the second last key
 				// in the result will be the last key less than endKey. (Condition #2)
 				state KeySelector end = KeySelectorRef(endKey.withPrefix(ddStatsRange.begin, endKey.arena()), false, 2);
-				RangeResult result = wait(tr->getRange(begin, end, GetRangeLimits(CLIENT_KNOBS->SHARD_COUNT_LIMIT)));
+				RangeResult result = wait(tr->getRange(begin, end, GetRangeLimits(CLIENT_KNOBS->TOO_MANY)));
 				// Condition #1 and #2 can be broken if multiple rpc calls happened in one getRange
 				if (result.size() > 1) {
 					if (result[0].key > begin.getKey() || result[1].key <= begin.getKey()) {
@@ -100,13 +107,14 @@ struct DataDistributionMetricsWorkload : KVWorkload {
 						    .detail("SecondKey", result[1].key)
 						    .detail("BeginKeySelector", begin);
 					}
-					if (result[result.size() - 1].key < end.getKey() || result[result.size() - 2].key >= end.getKey()) {
+					if (!result.readThroughEnd && (result[result.size() - 1].key < end.getKey() ||
+					                               result[result.size() - 2].key >= end.getKey())) {
 						++self->errors;
 						TraceEvent(SevError, "TestFailure")
 						    .detail("Reason", "Result mismatches the given end selector")
 						    .detail("Size", result.size())
-						    .detail("FirstKey", result[result.size() - 1].key)
-						    .detail("SecondKey", result[result.size() - 2].key)
+						    .detail("LastKey", result[result.size() - 1].key)
+						    .detail("SecondLastKey", result[result.size() - 2].key)
 						    .detail("EndKeySelector", end);
 					}
 					// Debugging traces
@@ -123,9 +131,10 @@ struct DataDistributionMetricsWorkload : KVWorkload {
 				}
 			} catch (Error& e) {
 				// Ignore timed_out error and cross_module_read, the end key selector may read through the end
-				if (e.code() == error_code_timed_out || e.code() == error_code_special_keys_cross_module_read)
+				if (e.code() == error_code_timed_out || e.code() == error_code_transaction_timed_out) {
+					tr->reset();
 					continue;
-				TraceEvent(SevDebug, "FailedToRetrieveDDMetrics").error(e);
+				}
 				wait(tr->onError(e));
 			}
 		}
@@ -139,38 +148,47 @@ struct DataDistributionMetricsWorkload : KVWorkload {
 		// TODO : find why this not work
 		// wait(quietDatabase(cx, self->dbInfo, "PopulateTPCC"));
 		state Reference<ReadYourWritesTransaction> tr = makeReference<ReadYourWritesTransaction>(cx);
-		try {
-			state RangeResult result = wait(tr->getRange(ddStatsRange, CLIENT_KNOBS->SHARD_COUNT_LIMIT));
-			ASSERT(!result.more);
-			self->numShards = result.size();
-			if (self->numShards < 1)
-				return false;
-			state int64_t totalBytes = 0;
-			auto schema = readJSONStrictly(JSONSchemas::dataDistributionStatsSchema.toString()).get_obj();
-			for (int i = 0; i < result.size(); ++i) {
-				ASSERT(result[i].key.startsWith(ddStatsRange.begin));
-				std::string errorStr;
-				auto valueObj = readJSONStrictly(result[i].value.toString()).get_obj();
-				TEST(true); // data_distribution_stats schema validation
-				if (!schemaMatch(schema, valueObj, errorStr, SevError, true)) {
-					TraceEvent(SevError, "DataDistributionStatsSchemaValidationFailed")
-					    .detail("ErrorStr", errorStr.c_str())
-					    .detail("JSON", json_spirit::write_string(json_spirit::mValue(result[i].value.toString())));
-					return false;
+		state int i;
+		state int retries = 0;
+		loop {
+			tr->setOption(FDBTransactionOptions::RAW_ACCESS);
+			tr->setOption(FDBTransactionOptions::TIMEOUT,
+			              StringRef((uint8_t*)&self->transactionTimeLimit, sizeof(int64_t)));
+			try {
+				state RangeResult result = wait(tr->getRange(ddStatsRange, CLIENT_KNOBS->TOO_MANY));
+				ASSERT(!result.more);
+				self->numShards = result.size();
+				// There's no guarantee that #shards <= CLIENT_KNOBS->SHARD_COUNT_LIMIT all the time
+				ASSERT(self->numShards >= 1);
+				state int64_t totalBytes = 0;
+				state int64_t totalBytesShardPerKSecond = 0;
+				auto schema = readJSONStrictly(JSONSchemas::dataDistributionStatsSchema.toString()).get_obj();
+				for (i = 0; i < result.size(); ++i) {
+					ASSERT(result[i].key.startsWith(ddStatsRange.begin));
+					std::string errorStr;
+					auto valueObj = readJSONStrictly(result[i].value.toString()).get_obj();
+					if (!schemaMatch(schema, valueObj, errorStr, SevError, true)) {
+						TraceEvent(SevError, "DataDistributionStatsSchemaValidationFailed")
+						    .detail("ErrorStr", errorStr.c_str())
+						    .detail("JSON", json_spirit::write_string(json_spirit::mValue(result[i].value.toString())));
+						return false;
+					}
+					totalBytes += valueObj["shard_bytes"].get_int64();
+					totalBytesShardPerKSecond += valueObj["shard_bytes_per_ksecond"].get_int64();
 				}
-				totalBytes += valueObj["shard_bytes"].get_int64();
+				self->avgBytes = totalBytes / self->numShards;
+				self->avgBytesPerKSeconds = totalBytesShardPerKSecond / self->numShards;
+				break;
+			} catch (Error& e) {
+				if (e.code() == error_code_timed_out || e.code() == error_code_transaction_timed_out) {
+					tr->reset();
+					// The RPC call may in some corner cases get no response
+					if (++retries > 10)
+						break;
+					continue;
+				}
+				wait(tr->onError(e));
 			}
-			self->avgBytes = totalBytes / self->numShards;
-			// fetch data-distribution stats for a smaller range
-			ASSERT(result.size());
-			state int idx = deterministicRandom()->randomInt(0, result.size());
-			RangeResult res = wait(tr->getRange(
-			    KeyRangeRef(result[idx].key, idx + 1 < result.size() ? result[idx + 1].key : ddStatsRange.end), 100));
-			ASSERT_WE_THINK(res.size() == 1 && res[0] == result[idx]); // It works good now. However, not sure in any
-			                                                           // case of data-distribution, the number changes
-		} catch (Error& e) {
-			TraceEvent(SevError, "FailedToRetrieveDDMetrics").detail("Error", e.what());
-			throw;
 		}
 		return true;
 	}
@@ -193,6 +211,7 @@ struct DataDistributionMetricsWorkload : KVWorkload {
 	void getMetrics(std::vector<PerfMetric>& m) override {
 		m.emplace_back("NumShards", numShards, Averaged::True);
 		m.emplace_back("AvgBytes", avgBytes, Averaged::True);
+		m.emplace_back("AvgBytesPerKSeconds", avgBytesPerKSeconds, Averaged::True);
 		m.push_back(commits.getMetric());
 	}
 };
diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt
index e476fe49c3f..3eef8efffe9 100644
--- a/tests/CMakeLists.txt
+++ b/tests/CMakeLists.txt
@@ -204,6 +204,7 @@ if(WITH_PYTHON)
   add_fdb_test(TEST_FILES rare/CycleRollbackClogged.toml)
   add_fdb_test(TEST_FILES rare/CycleWithKills.toml)
   add_fdb_test(TEST_FILES rare/DcLag.toml)
+  add_fdb_test(TEST_FILES rare/DataDistributionMetrics.toml)
   add_fdb_test(TEST_FILES rare/FuzzTest.toml)
   add_fdb_test(TEST_FILES rare/HighContentionPrefixAllocator.toml)
   add_fdb_test(TEST_FILES rare/InventoryTestHeavyWrites.toml)
diff --git a/tests/rare/DataDistributionMetrics.toml b/tests/rare/DataDistributionMetrics.toml
new file mode 100644
index 00000000000..39aafd7f422
--- /dev/null
+++ b/tests/rare/DataDistributionMetrics.toml
@@ -0,0 +1,8 @@
+[configuration]
+buggify = false
+
+[[test]]
+testTitle = 'DataDistributionMetricsTest'
+
+    [[test.workload]]
+    testName = 'DataDistributionMetrics'
\ No newline at end of file

From 3b019f0d3ef0a39ae06b429545493d63dd70a90a Mon Sep 17 00:00:00 2001
From: neethuhaneesha <nbingi@apple.com>
Date: Wed, 18 Oct 2023 14:32:25 -0700
Subject: [PATCH 286/468] Perpetual wiggle option to have multiple SS in
 rebalance state during wiggling.

---
 fdbclient/ServerKnobs.cpp            |  1 +
 fdbclient/ServerKnobs.h              |  5 ++
 fdbserver/DDTeamCollection.actor.cpp | 72 ++++++++++++++++++++++++++--
 fdbserver/DDTeamCollection.h         |  5 ++
 4 files changed, 80 insertions(+), 3 deletions(-)

diff --git a/fdbclient/ServerKnobs.cpp b/fdbclient/ServerKnobs.cpp
index 1c2c1428f44..4404cbbdda4 100644
--- a/fdbclient/ServerKnobs.cpp
+++ b/fdbclient/ServerKnobs.cpp
@@ -241,6 +241,7 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi
 	init( PERPETUAL_WIGGLE_DELAY,                                 60 );
 	init( PERPETUAL_WIGGLE_SMALL_LOAD_RATIO,                      10 );
 	init( PERPETUAL_WIGGLE_MIN_BYTES_BALANCE_RATIO,             0.85 );
+	init( PW_MAX_SS_LESSTHAN_MIN_BYTES_BALANCE_RATIO,              0 );
 	init( PERPETUAL_WIGGLE_DISABLE_REMOVER,                     true );
 	init( LOG_ON_COMPLETION_DELAY,         DD_QUEUE_LOGGING_INTERVAL );
 	init( BEST_TEAM_MAX_TEAM_TRIES,                               10 );
diff --git a/fdbclient/ServerKnobs.h b/fdbclient/ServerKnobs.h
index 54ed8d7abbb..fd1a453e4c5 100644
--- a/fdbclient/ServerKnobs.h
+++ b/fdbclient/ServerKnobs.h
@@ -186,6 +186,11 @@ class ServerKnobs : public KnobsImpl<ServerKnobs> {
 	                                          // load balance in the cluster
 	double PERPETUAL_WIGGLE_MIN_BYTES_BALANCE_RATIO; // target min : average space load balance ratio after re-include
 	                                                 // before perpetual wiggle will start the next wiggle
+	int PW_MAX_SS_LESSTHAN_MIN_BYTES_BALANCE_RATIO; // Maximum number of storage servers that can have the load bytes
+	                                                // less than PERPETUAL_WIGGLE_MIN_BYTES_BALANCE_RATIO before
+	                                                // perpetual wiggle will start the next wiggle.
+	                                                // Used to speed up wiggling rather than waiting for every SS to get
+	                                                // balanced/filledup before starting the next wiggle.
 	double PERPETUAL_WIGGLE_DELAY; // The min interval between the last wiggle finish and the next wiggle start
 	bool PERPETUAL_WIGGLE_DISABLE_REMOVER; // Whether the start of perpetual wiggle replace team remover
 	double LOG_ON_COMPLETION_DELAY;
diff --git a/fdbserver/DDTeamCollection.actor.cpp b/fdbserver/DDTeamCollection.actor.cpp
index 6c1a010ca4a..0f91367a325 100644
--- a/fdbserver/DDTeamCollection.actor.cpp
+++ b/fdbserver/DDTeamCollection.actor.cpp
@@ -2016,8 +2016,25 @@ class DDTeamCollectionImpl {
 			avgShardBytes.reset();
 			self->getAverageShardBytes.send(avgShardBytes);
 			int64_t avgBytes = wait(avgShardBytes.getFuture());
-			double ratio = self->loadBytesBalanceRatio(avgBytes * SERVER_KNOBS->PERPETUAL_WIGGLE_SMALL_LOAD_RATIO);
-			bool imbalance = ratio < SERVER_KNOBS->PERPETUAL_WIGGLE_MIN_BYTES_BALANCE_RATIO;
+			bool imbalance;
+			int numSSToBeLoadBytesBalanced;
+			double ratio;
+
+			if (SERVER_KNOBS->PW_MAX_SS_LESSTHAN_MIN_BYTES_BALANCE_RATIO) {
+				// PW_MAX_SS_LESSTHAN_MIN_BYTES_BALANCE_RATIO: Maximum number of storage servers that can
+				// have the load bytes less than PERPETUAL_WIGGLE_MIN_BYTES_BALANCE_RATIO before perpetual
+				// wiggle will start the next wiggle.
+				// The wiggle waits until the numSSToBeLoadBytesBalanced to be less than
+				// PW_MAX_SS_LESSTHAN_MIN_BYTES_BALANCE_RATIO before starting the next wiggle. With this we can have
+				// mutiple SS that are in balancing state. Used to speed up wiggling rather than waiting for every SS to
+				// get balanced/filledup before starting the next wiggle.
+				numSSToBeLoadBytesBalanced =
+				    self->numSSToBeLoadBytesBalanced(avgBytes * SERVER_KNOBS->PERPETUAL_WIGGLE_SMALL_LOAD_RATIO);
+				imbalance = numSSToBeLoadBytesBalanced > SERVER_KNOBS->PW_MAX_SS_LESSTHAN_MIN_BYTES_BALANCE_RATIO;
+			} else {
+				ratio = self->loadBytesBalanceRatio(avgBytes * SERVER_KNOBS->PERPETUAL_WIGGLE_SMALL_LOAD_RATIO);
+				imbalance = ratio < SERVER_KNOBS->PERPETUAL_WIGGLE_MIN_BYTES_BALANCE_RATIO;
+			}
 
 			// there must not have other teams to place wiggled data
 			takeRest = self->server_info.size() <= self->configuration.storageTeamSize ||
@@ -2027,7 +2044,9 @@ class DDTeamCollectionImpl {
 			if (takeRest && self->configuration.storageMigrationType == StorageMigrationType::GRADUAL) {
 				TraceEvent(SevWarn, "PerpetualStorageWiggleSleep", self->distributorId)
 				    .suppressFor(SERVER_KNOBS->PERPETUAL_WIGGLE_DELAY * 4)
-				    .detail("BytesBalanceRatio", ratio)
+				    .detail("ImbalanceFactor",
+				            SERVER_KNOBS->PW_MAX_SS_LESSTHAN_MIN_BYTES_BALANCE_RATIO ? numSSToBeLoadBytesBalanced
+				                                                                     : ratio)
 				    .detail("ServerSize", self->server_info.size())
 				    .detail("MachineSize", self->machine_info.size())
 				    .detail("StorageTeamSize", self->configuration.storageTeamSize);
@@ -3571,6 +3590,53 @@ double DDTeamCollection::loadBytesBalanceRatio(int64_t smallLoadThreshold) const
 	return minLoadBytes / avgLoad;
 }
 
+int DDTeamCollection::numSSToBeLoadBytesBalanced(int64_t smallLoadThreshold) const {
+	double totalLoadBytes = 0;
+	int count = 0;
+	for (auto& [id, s] : server_info) {
+		// If a healthy SS don't have storage metrics, skip this round
+		if (server_status.get(s->getId()).isUnhealthy() || !s->metricsPresent()) {
+			TraceEvent(SevDebug, "NumSSToBeLoadBytesBalancedNoMetrics").detail("Server", id);
+			return INT_MAX; // return all are imbalanced
+		}
+
+		totalLoadBytes += s->loadBytes();
+		++count;
+	}
+
+	if (!count)
+		return INT_MAX;
+
+	double avgLoad = totalLoadBytes / count;
+	if (totalLoadBytes == 0 || avgLoad < smallLoadThreshold) {
+		return 0;
+	}
+
+	int numSSToBeLoadBytesBalanced = 0;
+	double balanceRatio;
+	for (auto& [id, s] : server_info) {
+		// If a healthy SS don't have storage metrics, skip this round
+		if (server_status.get(s->getId()).isUnhealthy() || !s->metricsPresent()) {
+			TraceEvent(SevDebug, "NumSSToBeLoadBytesBalancedNoMetrics").detail("Server", id);
+			return INT_MAX;
+		}
+
+		balanceRatio = s->loadBytes() / avgLoad;
+		if (balanceRatio < SERVER_KNOBS->PERPETUAL_WIGGLE_MIN_BYTES_BALANCE_RATIO) {
+			numSSToBeLoadBytesBalanced++;
+		}
+	}
+
+	TraceEvent(SevDebug, "NumSSToBeLoadBytesBalancedMetrics")
+	    .detail("NumSSToBeLoadBytesBalanced", numSSToBeLoadBytesBalanced)
+	    .detail("TotalLoad", totalLoadBytes)
+	    .detail("AvgLoad", avgLoad)
+	    .detail("SmallLoadThreshold", smallLoadThreshold)
+	    .detail("Count", count);
+
+	return numSSToBeLoadBytesBalanced;
+}
+
 Future<Void> DDTeamCollection::storageServerFailureTracker(TCServerInfo* server,
                                                            Database cx,
                                                            ServerStatus* status,
diff --git a/fdbserver/DDTeamCollection.h b/fdbserver/DDTeamCollection.h
index e9d302a9c1a..3331fafd044 100644
--- a/fdbserver/DDTeamCollection.h
+++ b/fdbserver/DDTeamCollection.h
@@ -448,6 +448,11 @@ class DDTeamCollection : public ReferenceCounted<DDTeamCollection> {
 	// average load of each storage server is less than smallLoadThreshold, return 1 always.
 	double loadBytesBalanceRatio(int64_t smallLoadThreshold) const;
 
+	// calculate number of Storage Servers that are yet need to be balanced(disk bytes). Balanced here means the Storage
+	// Serves which are having the ratio of loadBytes / avgLoadBytes less than the
+	// PERPETUAL_WIGGLE_MIN_BYTES_BALANCE_RATIO.
+	int numSSToBeLoadBytesBalanced(int64_t smallLoadThreshold) const;
+
 	// Create a transaction updating `perpetualStorageWiggleIDPrefix` to the next serverID according to a sorted
 	// wiggle_pq maintained by the wiggler.
 	Future<Void> updateNextWigglingStorageID();

From 1c2dd6cc671528ed88dfd64228331be127b90761 Mon Sep 17 00:00:00 2001
From: Johannes Scheuermann <johscheuer@users.noreply.github.com>
Date: Thu, 19 Oct 2023 23:03:55 +0200
Subject: [PATCH 287/468] Fix exclude status of machines in status json
 (#10996)

Previously, `status json` would report a machine as excluded when any of
its processes were excluded. But a machine should only be reported as
excluded when all of its processes are excluded.

I tested this change by running a small, three process test cluster. I
excluded one machine, and verified `status json` reported the overall
machine as not excluded.

Co-authored-by: Lukas Joswiak <lukas.joswiak@snowflake.com>
---
 fdbserver/Status.actor.cpp | 17 ++++++++++-------
 1 file changed, 10 insertions(+), 7 deletions(-)

diff --git a/fdbserver/Status.actor.cpp b/fdbserver/Status.actor.cpp
index 4c784f6051a..600b472848a 100644
--- a/fdbserver/Status.actor.cpp
+++ b/fdbserver/Status.actor.cpp
@@ -285,7 +285,7 @@ static JsonBuilderObject machineStatusFetcher(WorkerEvents mMetrics,
 	// map from machine networkAddress to datacenter ID
 	std::map<NetworkAddress, std::string> dcIds;
 	std::map<NetworkAddress, LocalityData> locality;
-	std::map<std::string, bool> notExcludedMap;
+	std::map<std::string, bool> excludedMap;
 	std::map<std::string, int32_t> workerContribMap;
 	std::map<std::string, JsonBuilderObject> machineJsonMap;
 
@@ -351,7 +351,7 @@ static JsonBuilderObject machineStatusFetcher(WorkerEvents mMetrics,
 				statusObj["network"] = networkObj;
 
 				if (configuration.present()) {
-					notExcludedMap[machineId] =
+					excludedMap[machineId] =
 					    true; // Will be set to false below if this or any later process is not excluded
 				}
 
@@ -359,7 +359,6 @@ static JsonBuilderObject machineStatusFetcher(WorkerEvents mMetrics,
 				machineJsonMap[machineId] = statusObj;
 			}
 
-			// FIXME: this will not catch if the secondary address of the process was excluded
 			NetworkAddressList tempList;
 			tempList.address = it->first;
 			bool excludedServer = true;
@@ -367,10 +366,14 @@ static JsonBuilderObject machineStatusFetcher(WorkerEvents mMetrics,
 			if (configuration.present() && !configuration.get().isExcludedServer(tempList, LocalityData()))
 				excludedServer = false;
 			if (locality.count(it->first) && configuration.present() &&
-			    configuration.get().isMachineExcluded(locality[it->first]))
-				excludedLocality = true;
+			    !configuration.get().isMachineExcluded(locality[it->first]))
+				excludedLocality = false;
 
-			notExcludedMap[machineId] = excludedServer || excludedLocality;
+			// If any server is not excluded, set the overall exclusion status
+			// of the machine to false.
+			if (!excludedServer && !excludedLocality) {
+				excludedMap[machineId] = false;
+			}
 			workerContribMap[machineId]++;
 		} catch (Error&) {
 			++failed;
@@ -381,7 +384,7 @@ static JsonBuilderObject machineStatusFetcher(WorkerEvents mMetrics,
 	for (auto& mapPair : machineJsonMap) {
 		auto& machineId = mapPair.first;
 		auto& jsonItem = machineJsonMap[machineId];
-		jsonItem["excluded"] = notExcludedMap[machineId];
+		jsonItem["excluded"] = excludedMap[machineId];
 		jsonItem["contributing_workers"] = workerContribMap[machineId];
 		machineMap[machineId] = jsonItem;
 	}

From 53bd794fc705ac97e6723763667670e81dc02e82 Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Mon, 16 Oct 2023 09:33:09 -0700
Subject: [PATCH 288/468] Fix an assertion failure in tests

Restore the default value for knob peek_tracker_expiration_time.

Failed seed:
   -f ./tests/slow/MoveKeysSideband.toml -s 669512178 -b on
commit: a3dbd4baf
---
 tests/slow/MoveKeysSideband.toml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/tests/slow/MoveKeysSideband.toml b/tests/slow/MoveKeysSideband.toml
index 89a7fafa385..ffe9f2516ee 100644
--- a/tests/slow/MoveKeysSideband.toml
+++ b/tests/slow/MoveKeysSideband.toml
@@ -1,3 +1,6 @@
+[[knobs]]
+peek_tracker_expiration_time = 600
+
 [[test]]
 testTitle = 'MoveKeysNew'
 

From 7ba77a3707fb040138e7d317539202edf87b9173 Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Thu, 19 Oct 2023 14:50:01 -0700
Subject: [PATCH 289/468] Fix RemoveServersSafely workload timeout error

This workload can have timeout error when using locality-based exclusion. The
sequence is:

1. RemoveServerSafely workload exclude locality by processid
2. Attrition reboots the target process, thus changing the processid, because
processid is generated for each worker process at fdbd()
3. RemoveServerSafely waits for the process exclusion, which never succeed
4. Timeout

The fix monitors processid locality changes and reissue the exclusion with the
correct locality.

To reproduce:
seed: -f ./tests/fast/SwizzledRollbackSideband.toml -s 879108103 -b on
commit: a3dbd4baf release-7.1
---
 fdbclient/ManagementAPI.actor.cpp             |  6 +--
 fdbserver/ExclusionTracker.actor.h            |  8 ++++
 .../workloads/RemoveServersSafely.actor.cpp   | 45 ++++++++++++++++++-
 3 files changed, 55 insertions(+), 4 deletions(-)

diff --git a/fdbclient/ManagementAPI.actor.cpp b/fdbclient/ManagementAPI.actor.cpp
index 06ab0d6ab40..cfd4992e94d 100644
--- a/fdbclient/ManagementAPI.actor.cpp
+++ b/fdbclient/ManagementAPI.actor.cpp
@@ -1782,15 +1782,15 @@ std::pair<std::string, std::string> decodeLocality(const std::string& locality)
 }
 
 // Returns the list of IPAddresses of the workers that match the given locality.
-// Example: locality="dcid:primary" returns all the ip addresses of the workers in the primary dc.
+// Example: locality="locality_dcid:primary" returns all the ip addresses of the workers in the primary dc.
 std::set<AddressExclusion> getAddressesByLocality(const std::vector<ProcessData>& workers,
                                                   const std::string& locality) {
 	std::pair<std::string, std::string> localityKeyValue = decodeLocality(locality);
 
 	std::set<AddressExclusion> localityAddresses;
 	for (int i = 0; i < workers.size(); i++) {
-		if (workers[i].locality.isPresent(localityKeyValue.first) &&
-		    workers[i].locality.get(localityKeyValue.first) == localityKeyValue.second) {
+		auto localityValue = workers[i].locality.get(localityKeyValue.first);
+		if (localityValue.present() && localityValue.get() == localityKeyValue.second) {
 			localityAddresses.insert(AddressExclusion(workers[i].address.ip, workers[i].address.port));
 		}
 	}
diff --git a/fdbserver/ExclusionTracker.actor.h b/fdbserver/ExclusionTracker.actor.h
index 745301f011b..d54028d7a5b 100644
--- a/fdbserver/ExclusionTracker.actor.h
+++ b/fdbserver/ExclusionTracker.actor.h
@@ -30,6 +30,7 @@
 
 #include <set>
 #include "flow/flow.h"
+#include "flow/Trace.h"
 #include "fdbclient/DatabaseContext.h"
 #include "fdbclient/ManagementAPI.actor.h"
 #include "flow/actorcompiler.h" // This must be the last #include.
@@ -101,11 +102,17 @@ struct ExclusionTracker {
 					std::string locality = decodeExcludedLocalityKey(r.key);
 					std::set<AddressExclusion> localityExcludedAddresses = getAddressesByLocality(workers, locality);
 					newExcluded.insert(localityExcludedAddresses.begin(), localityExcludedAddresses.end());
+					if (localityExcludedAddresses.empty()) {
+						TraceEvent(SevWarn, "ExclusionTrackerLocalityNotFound").detail("Locality", locality);
+					}
 				}
 				for (const auto& r : failedLocalityResults) {
 					std::string locality = decodeFailedLocalityKey(r.key);
 					std::set<AddressExclusion> localityFailedAddresses = getAddressesByLocality(workers, locality);
 					newFailed.insert(localityFailedAddresses.begin(), localityFailedAddresses.end());
+					if (localityFailedAddresses.empty()) {
+						TraceEvent(SevWarn, "ExclusionTrackerFailedLocalityNotFound").detail("Locality", locality);
+					}
 				}
 
 				bool foundChange = false;
@@ -134,6 +141,7 @@ struct ExclusionTracker {
 				wait(watchFuture);
 				tr.reset();
 			} catch (Error& e) {
+				TraceEvent("ExclusionTrackerError").error(e);
 				wait(tr.onError(e));
 			}
 		}
diff --git a/fdbserver/workloads/RemoveServersSafely.actor.cpp b/fdbserver/workloads/RemoveServersSafely.actor.cpp
index bcda83779d4..bb2e8cbbabf 100644
--- a/fdbserver/workloads/RemoveServersSafely.actor.cpp
+++ b/fdbserver/workloads/RemoveServersSafely.actor.cpp
@@ -508,6 +508,44 @@ struct RemoveServersSafelyWorkload : TestWorkload {
 		return killProcArray;
 	}
 
+	// If a process is rebooted, it's processid will change. So we need to monitor
+	// such changes and re-issue the locality-based exclusion again.
+	ACTOR static Future<Void> checkLocalityChange(RemoveServersSafelyWorkload* self,
+	                                              Database cx,
+	                                              std::vector<AddressExclusion> toKillArray,
+	                                              std::unordered_set<std::string> origKillLocalities,
+	                                              bool markExcludeAsFailed) {
+		state std::unordered_set<std::string> killLocalities = origKillLocalities;
+
+		loop {
+			wait(delay(10.0));
+			wait(self->updateProcessIds(cx));
+			std::unordered_set<std::string> toKillLocalities = self->getLocalitiesFromAddresses(toKillArray);
+			if (toKillLocalities == killLocalities) {
+				continue;
+			}
+
+			// The kill localities have changed.
+			TraceEvent("RemoveAndKill")
+			    .detail("Step", "localities changed")
+			    .detail("OrigKillLocalities", describe(origKillLocalities))
+			    .detail("KillLocalities", describe(killLocalities))
+			    .detail("ToKillLocalities", describe(toKillLocalities));
+			killLocalities = toKillLocalities;
+
+			// Include back the localities that are no longer in the kill list
+			state bool failed = true;
+			wait(includeLocalities(cx, std::vector<std::string>(), failed, true));
+			wait(includeLocalities(cx, std::vector<std::string>(), !failed, true));
+
+			// Exclude the localities that are now in the kill list
+			wait(excludeLocalities(cx, killLocalities, markExcludeAsFailed));
+			TraceEvent("RemoveAndKill")
+			    .detail("Step", "new localities excluded")
+			    .detail("Localities", describe(killLocalities));
+		}
+	}
+
 	// Attempts to exclude a set of processes, and once the exclusion is successful it kills them.
 	// If markExcludeAsFailed is true, then it is an error if we cannot complete the exclusion.
 	ACTOR static Future<Void> removeAndKill(RemoveServersSafelyWorkload* self,
@@ -682,7 +720,12 @@ struct RemoveServersSafelyWorkload : TestWorkload {
 			    .detail("Step", "Wait For Server Exclusion")
 			    .detail("Addresses", describe(toKill))
 			    .detail("ClusterAvailable", g_simulator.isAvailable());
-			wait(success(checkForExcludingServers(cx, toKillArray, true /* wait for exclusion */)));
+			if (excludeLocalitiesInsteadOfServers) {
+				wait(success(checkForExcludingServers(cx, toKillArray, true /* wait for exclusion */)) ||
+				     checkLocalityChange(self, cx, toKillArray, toKillLocalities, markExcludeAsFailed));
+			} else {
+				wait(success(checkForExcludingServers(cx, toKillArray, true /* wait for exclusion */)));
+			}
 
 			TraceEvent("RemoveAndKill", functionId)
 			    .detail("Step", "coordinators auto")

From a1423f5f5226ed624ed02978eee5b2c57dd2a42a Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Thu, 19 Oct 2023 21:01:00 -0700
Subject: [PATCH 290/468] Fix force flag for exclusion

---
 fdbserver/workloads/RemoveServersSafely.actor.cpp | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/fdbserver/workloads/RemoveServersSafely.actor.cpp b/fdbserver/workloads/RemoveServersSafely.actor.cpp
index bb2e8cbbabf..946faa55f8b 100644
--- a/fdbserver/workloads/RemoveServersSafely.actor.cpp
+++ b/fdbserver/workloads/RemoveServersSafely.actor.cpp
@@ -530,7 +530,8 @@ struct RemoveServersSafelyWorkload : TestWorkload {
 			    .detail("Step", "localities changed")
 			    .detail("OrigKillLocalities", describe(origKillLocalities))
 			    .detail("KillLocalities", describe(killLocalities))
-			    .detail("ToKillLocalities", describe(toKillLocalities));
+			    .detail("ToKillLocalities", describe(toKillLocalities))
+			    .detail("Failed", markExcludeAsFailed);
 			killLocalities = toKillLocalities;
 
 			// Include back the localities that are no longer in the kill list
@@ -722,7 +723,11 @@ struct RemoveServersSafelyWorkload : TestWorkload {
 			    .detail("ClusterAvailable", g_simulator.isAvailable());
 			if (excludeLocalitiesInsteadOfServers) {
 				wait(success(checkForExcludingServers(cx, toKillArray, true /* wait for exclusion */)) ||
-				     checkLocalityChange(self, cx, toKillArray, toKillLocalities, markExcludeAsFailed));
+				     checkLocalityChange(self,
+				                         cx,
+				                         toKillArray,
+				                         toKillLocalities,
+				                         markExcludeAsFailed && toKillLocalitiesFailed.size() > 0));
 			} else {
 				wait(success(checkForExcludingServers(cx, toKillArray, true /* wait for exclusion */)));
 			}

From 44ab9c7a5da9671f5d4ceffd4b9c2272a5e31adf Mon Sep 17 00:00:00 2001
From: "Johannes M. Scheuermann" <jscheuermann@apple.com>
Date: Thu, 19 Oct 2023 11:42:46 +0200
Subject: [PATCH 291/468] Correct the check for a machine being excluded

---
 fdbserver/Status.actor.cpp | 29 +++++++++++++++++------------
 1 file changed, 17 insertions(+), 12 deletions(-)

diff --git a/fdbserver/Status.actor.cpp b/fdbserver/Status.actor.cpp
index 600b472848a..c657f265c18 100644
--- a/fdbserver/Status.actor.cpp
+++ b/fdbserver/Status.actor.cpp
@@ -359,19 +359,24 @@ static JsonBuilderObject machineStatusFetcher(WorkerEvents mMetrics,
 				machineJsonMap[machineId] = statusObj;
 			}
 
-			NetworkAddressList tempList;
-			tempList.address = it->first;
 			bool excludedServer = true;
-			bool excludedLocality = true;
-			if (configuration.present() && !configuration.get().isExcludedServer(tempList, LocalityData()))
-				excludedServer = false;
-			if (locality.count(it->first) && configuration.present() &&
-			    !configuration.get().isMachineExcluded(locality[it->first]))
-				excludedLocality = false;
-
-			// If any server is not excluded, set the overall exclusion status
-			// of the machine to false.
-			if (!excludedServer && !excludedLocality) {
+			// If the machine is already marked as not excluded, because at least one process was found to not be
+			// excluded, we can stop checking further servers on this machine.
+			if (configuration.present() && excludedMap[machineId]) {
+				NetworkAddressList tempList;
+				tempList.address = it->first;
+				// Check if the locality data is present and if so, make use of it.
+				auto localityData = LocalityData();
+				if (locality.count(it->first)) {
+					localityData = locality[it->first];
+				}
+
+				// The isExcludedServer method already contains a check for the excluded localities.
+				excludedServer = configuration.get().isExcludedServer(tempList, localityData);
+			}
+
+			// If any server is not excluded, set the overall exclusion status of the machine to false.
+			if (!excludedServer) {
 				excludedMap[machineId] = false;
 			}
 			workerContribMap[machineId]++;

From 6170c7b396055ff45385ac4e40d63eb17d82547b Mon Sep 17 00:00:00 2001
From: "Johannes M. Scheuermann" <jscheuermann@apple.com>
Date: Wed, 18 Oct 2023 15:12:39 +0200
Subject: [PATCH 292/468] Make sure the get all excluded servers method returns
 servers that are excluded based on locality

---
 fdbclient/ManagementAPI.actor.cpp | 40 +++++++++++++++++++++++++++----
 1 file changed, 36 insertions(+), 4 deletions(-)

diff --git a/fdbclient/ManagementAPI.actor.cpp b/fdbclient/ManagementAPI.actor.cpp
index 06ab0d6ab40..37b2479c4eb 100644
--- a/fdbclient/ManagementAPI.actor.cpp
+++ b/fdbclient/ManagementAPI.actor.cpp
@@ -1692,10 +1692,35 @@ ACTOR Future<std::vector<AddressExclusion>> getExcludedFailedServerList(Transact
 
 ACTOR Future<std::vector<AddressExclusion>> getAllExcludedServers(Transaction* tr) {
 	state std::vector<AddressExclusion> exclusions;
-	std::vector<AddressExclusion> excludedServers = wait(getExcludedServerList(tr));
+	// Request all exclusion based information concurrently.
+	state Future<std::vector<AddressExclusion>> fExcludedServers = getExcludedServerList(tr);
+	state Future<std::vector<AddressExclusion>> fExcludedFailed = getExcludedFailedServerList(tr);
+	state Future<std::vector<std::string>> fExcludedLocalities = getAllExcludedLocalities(tr);
+	// Wait until all data is gathered.
+	wait(success(fExcludedServers) && success(fExcludedFailed) && success(fExcludedLocalities));
+	// Update the exclusions vector with all excluded servers.
+	auto excludedServers = fExcludedServers.get();
 	exclusions.insert(exclusions.end(), excludedServers.begin(), excludedServers.end());
-	std::vector<AddressExclusion> excludedFailed = wait(getExcludedFailedServerList(tr));
+	auto excludedFailed = fExcludedFailed.get();
 	exclusions.insert(exclusions.end(), excludedFailed.begin(), excludedFailed.end());
+
+	// We have to return all servers that are excluded, this includes servers that are excluded
+	// based on the locality. Otherwise those excluded servers might be used, even if they shouldn't.
+	state std::vector<std::string> excludedLocalities = fExcludedLocalities.get();
+
+	// Only if at least one locality was found we have to perform this check.
+	if (!excludedLocalities.empty()) {
+		// First we have to fetch all workers to match the localities of each worker against the excluded localities.
+		state std::vector<ProcessData> workers = wait(getWorkers(tr));
+		for (const auto& locality : excludedLocalities) {
+			std::set<AddressExclusion> localityAddresses = getAddressesByLocality(workers, locality);
+			if (!localityAddresses.empty()) {
+				// Add all the server ipaddresses that belong to the given localities to the exclusionSet.
+				exclusions.insert(exclusions.end(), localityAddresses.begin(), localityAddresses.end());
+			}
+		}
+	}
+
 	uniquify(exclusions);
 	return exclusions;
 }
@@ -1743,10 +1768,17 @@ ACTOR Future<std::vector<std::string>> getExcludedFailedLocalityList(Transaction
 
 ACTOR Future<std::vector<std::string>> getAllExcludedLocalities(Transaction* tr) {
 	state std::vector<std::string> exclusions;
-	std::vector<std::string> excludedLocalities = wait(getExcludedLocalityList(tr));
+	state Future<std::vector<std::string>> fExcludedLocalities = getExcludedLocalityList(tr);
+	state Future<std::vector<std::string>> fFailedLocalities = getExcludedFailedLocalityList(tr);
+
+	// Wait until all data is gathered.
+	wait(success(fExcludedLocalities) && success(fFailedLocalities));
+
+	auto excludedLocalities = fExcludedLocalities.get();
 	exclusions.insert(exclusions.end(), excludedLocalities.begin(), excludedLocalities.end());
-	std::vector<std::string> failedLocalities = wait(getExcludedFailedLocalityList(tr));
+	auto failedLocalities = fFailedLocalities.get();
 	exclusions.insert(exclusions.end(), failedLocalities.begin(), failedLocalities.end());
+
 	uniquify(exclusions);
 	return exclusions;
 }

From 7e8371fed5a12c1c689839221fd121bdfc6e85b7 Mon Sep 17 00:00:00 2001
From: "Johannes M. Scheuermann" <jscheuermann@apple.com>
Date: Fri, 20 Oct 2023 10:26:32 +0200
Subject: [PATCH 293/468] Issue get worker request concurrently

---
 fdbclient/ManagementAPI.actor.cpp | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/fdbclient/ManagementAPI.actor.cpp b/fdbclient/ManagementAPI.actor.cpp
index 37b2479c4eb..1c6a61729f2 100644
--- a/fdbclient/ManagementAPI.actor.cpp
+++ b/fdbclient/ManagementAPI.actor.cpp
@@ -1696,7 +1696,10 @@ ACTOR Future<std::vector<AddressExclusion>> getAllExcludedServers(Transaction* t
 	state Future<std::vector<AddressExclusion>> fExcludedServers = getExcludedServerList(tr);
 	state Future<std::vector<AddressExclusion>> fExcludedFailed = getExcludedFailedServerList(tr);
 	state Future<std::vector<std::string>> fExcludedLocalities = getAllExcludedLocalities(tr);
-	// Wait until all data is gathered.
+	state Future<std::vector<ProcessData>> fWorkers = getWorkers(tr);
+
+	// Wait until all data is gathered, we are not waiting here for the workers future to return
+	// instead we wait for the worker future only if we need the data.
 	wait(success(fExcludedServers) && success(fExcludedFailed) && success(fExcludedLocalities));
 	// Update the exclusions vector with all excluded servers.
 	auto excludedServers = fExcludedServers.get();
@@ -1711,7 +1714,9 @@ ACTOR Future<std::vector<AddressExclusion>> getAllExcludedServers(Transaction* t
 	// Only if at least one locality was found we have to perform this check.
 	if (!excludedLocalities.empty()) {
 		// First we have to fetch all workers to match the localities of each worker against the excluded localities.
-		state std::vector<ProcessData> workers = wait(getWorkers(tr));
+		wait(success(fWorkers));
+		state std::vector<ProcessData> workers = fWorkers.get();
+
 		for (const auto& locality : excludedLocalities) {
 			std::set<AddressExclusion> localityAddresses = getAddressesByLocality(workers, locality);
 			if (!localityAddresses.empty()) {

From 7d391b3a900fc4b17790205d12ca8dde16f2ff69 Mon Sep 17 00:00:00 2001
From: "Johannes M. Scheuermann" <jscheuermann@apple.com>
Date: Fri, 20 Oct 2023 15:33:08 +0200
Subject: [PATCH 294/468] Make sure server list is validated against the
 excluded localites

---
 fdbcli/ExcludeCommand.actor.cpp     | 48 ++++++++++++++++-------------
 fdbclient/DatabaseConfiguration.cpp | 16 +---------
 fdbclient/DatabaseConfiguration.h   |  1 -
 fdbclient/ManagementAPI.actor.cpp   | 47 ++++++++++++++++++++++++----
 fdbclient/ManagementAPI.actor.h     | 12 ++++++--
 fdbclient/SpecialKeySpace.actor.cpp | 40 ++++++++++++++++++++++--
 fdbrpc/Locality.cpp                 |  1 -
 fdbrpc/Locality.h                   |  1 -
 8 files changed, 116 insertions(+), 50 deletions(-)

diff --git a/fdbcli/ExcludeCommand.actor.cpp b/fdbcli/ExcludeCommand.actor.cpp
index db67bd8a6e8..406019f88aa 100644
--- a/fdbcli/ExcludeCommand.actor.cpp
+++ b/fdbcli/ExcludeCommand.actor.cpp
@@ -143,9 +143,8 @@ ACTOR Future<std::vector<std::string>> getExcludedLocalities(Reference<IDatabase
 }
 
 ACTOR Future<std::set<NetworkAddress>> checkForExcludingServers(Reference<IDatabase> db,
-                                                                std::vector<AddressExclusion> excl,
+                                                                std::set<AddressExclusion> exclusions,
                                                                 bool waitForAllExcluded) {
-	state std::set<AddressExclusion> exclusions(excl.begin(), excl.end());
 	state std::set<NetworkAddress> inProgressExclusion;
 	state Reference<ITransaction> tr = db->createTransaction();
 	loop {
@@ -193,7 +192,7 @@ ACTOR Future<std::set<NetworkAddress>> checkForExcludingServers(Reference<IDatab
 	return inProgressExclusion;
 }
 
-ACTOR Future<Void> checkForCoordinators(Reference<IDatabase> db, std::vector<AddressExclusion> exclusionVector) {
+ACTOR Future<Void> checkForCoordinators(Reference<IDatabase> db, std::set<AddressExclusion> exclusions) {
 
 	state bool foundCoordinator = false;
 	state std::vector<NetworkAddress> coordinatorList;
@@ -210,9 +209,10 @@ ACTOR Future<Void> checkForCoordinators(Reference<IDatabase> db, std::vector<Add
 			wait(safeThreadFutureToFuture(tr->onError(e)));
 		}
 	}
+
 	for (const auto& c : coordinatorList) {
-		if (std::count(exclusionVector.begin(), exclusionVector.end(), AddressExclusion(c.ip, c.port)) ||
-		    std::count(exclusionVector.begin(), exclusionVector.end(), AddressExclusion(c.ip))) {
+		if (exclusions.find(AddressExclusion(c.ip, c.port)) != exclusions.end() ||
+		    exclusions.find(AddressExclusion(c.ip)) != exclusions.end()) {
 			fprintf(stderr, "WARNING: %s is a coordinator!\n", c.toString().c_str());
 			foundCoordinator = true;
 		}
@@ -268,7 +268,6 @@ ACTOR Future<bool> excludeCommandActor(Reference<IDatabase> db, std::vector<Stri
 
 		return true;
 	} else {
-		state std::vector<AddressExclusion> exclusionVector;
 		state std::set<AddressExclusion> exclusionSet;
 		state std::vector<AddressExclusion> exclusionAddresses;
 		state std::unordered_set<std::string> exclusionLocalities;
@@ -277,9 +276,11 @@ ACTOR Future<bool> excludeCommandActor(Reference<IDatabase> db, std::vector<Stri
 		state bool waitForAllExcluded = true;
 		state bool markFailed = false;
 		state std::vector<ProcessData> workers;
-		bool result = wait(fdb_cli::getWorkers(db, &workers));
-		if (!result)
-			return false;
+		state std::map<std::string, StorageServerInterface> server_interfaces;
+		state Future<bool> future_workers = fdb_cli::getWorkers(db, &workers);
+		state Future<Void> future_server_interfaces = fdb_cli::getStorageServerInterfaces(db, &server_interfaces);
+		wait(success(future_workers) && success(future_server_interfaces));
+
 		for (auto t = tokens.begin() + 1; t != tokens.end(); ++t) {
 			if (*t == LiteralStringRef("FORCE")) {
 				force = true;
@@ -289,15 +290,21 @@ ACTOR Future<bool> excludeCommandActor(Reference<IDatabase> db, std::vector<Stri
 				markFailed = true;
 			} else if (t->startsWith(LocalityData::ExcludeLocalityPrefix) &&
 			           t->toString().find(':') != std::string::npos) {
-				std::set<AddressExclusion> localityAddresses = getAddressesByLocality(workers, t->toString());
-				if (localityAddresses.empty()) {
+				exclusionLocalities.insert(t->toString());
+				auto localityAddresses = getAddressesByLocality(workers, t->toString());
+				auto localityServerAddresses = getServerAddressesByLocality(server_interfaces, t->toString());
+				if (localityAddresses.empty() && localityServerAddresses.empty()) {
 					noMatchLocalities.push_back(t->toString());
-				} else {
-					// add all the server ipaddresses that belong to the given localities to the exclusionSet.
-					exclusionVector.insert(exclusionVector.end(), localityAddresses.begin(), localityAddresses.end());
+					continue;
+				}
+
+				if (!localityAddresses.empty()) {
 					exclusionSet.insert(localityAddresses.begin(), localityAddresses.end());
 				}
-				exclusionLocalities.insert(t->toString());
+
+				if (!localityServerAddresses.empty()) {
+					exclusionSet.insert(localityServerAddresses.begin(), localityServerAddresses.end());
+				}
 			} else {
 				auto a = AddressExclusion::parse(*t);
 				if (!a.isValid()) {
@@ -308,13 +315,12 @@ ACTOR Future<bool> excludeCommandActor(Reference<IDatabase> db, std::vector<Stri
 						printf("        Do not include the `:tls' suffix when naming a process\n");
 					return true;
 				}
-				exclusionVector.push_back(a);
 				exclusionSet.insert(a);
 				exclusionAddresses.push_back(a);
 			}
 		}
 
-		if (exclusionAddresses.empty() && exclusionLocalities.empty()) {
+		if (exclusionSet.empty()) {
 			fprintf(stderr, "ERROR: At least one valid network endpoint address or a locality is not provided\n");
 			return false;
 		}
@@ -332,14 +338,14 @@ ACTOR Future<bool> excludeCommandActor(Reference<IDatabase> db, std::vector<Stri
 			warn.cancel();
 
 		state std::set<NetworkAddress> notExcludedServers =
-		    wait(checkForExcludingServers(db, exclusionVector, waitForAllExcluded));
+		    wait(checkForExcludingServers(db, exclusionSet, waitForAllExcluded));
 		std::map<IPAddress, std::set<uint16_t>> workerPorts;
 		for (auto addr : workers)
 			workerPorts[addr.address.ip].insert(addr.address.port);
 
 		// Print a list of all excluded addresses that don't have a corresponding worker
 		std::set<AddressExclusion> absentExclusions;
-		for (const auto& addr : exclusionVector) {
+		for (const auto& addr : exclusionSet) {
 			auto worker = workerPorts.find(addr.ip);
 			if (worker == workerPorts.end())
 				absentExclusions.insert(addr);
@@ -347,7 +353,7 @@ ACTOR Future<bool> excludeCommandActor(Reference<IDatabase> db, std::vector<Stri
 				absentExclusions.insert(addr);
 		}
 
-		for (const auto& exclusion : exclusionVector) {
+		for (const auto& exclusion : exclusionSet) {
 			if (absentExclusions.find(exclusion) != absentExclusions.end()) {
 				if (exclusion.port == 0) {
 					fprintf(stderr,
@@ -395,7 +401,7 @@ ACTOR Future<bool> excludeCommandActor(Reference<IDatabase> db, std::vector<Stri
 			    locality.c_str());
 		}
 
-		wait(checkForCoordinators(db, exclusionVector));
+		wait(checkForCoordinators(db, exclusionSet));
 		return true;
 	}
 }
diff --git a/fdbclient/DatabaseConfiguration.cpp b/fdbclient/DatabaseConfiguration.cpp
index a60fe3d307d..f829e5a5af8 100644
--- a/fdbclient/DatabaseConfiguration.cpp
+++ b/fdbclient/DatabaseConfiguration.cpp
@@ -758,20 +758,6 @@ bool DatabaseConfiguration::isExcludedLocality(const LocalityData& locality) con
 	return false;
 }
 
-// checks if this machineid of given locality is excluded.
-bool DatabaseConfiguration::isMachineExcluded(const LocalityData& locality) const {
-	if (locality.machineId().present()) {
-		return get(encodeExcludedLocalityKey(LocalityData::ExcludeLocalityKeyMachineIdPrefix.toString() +
-		                                     locality.machineId().get().toString()))
-		           .present() ||
-		       get(encodeFailedLocalityKey(LocalityData::ExcludeLocalityKeyMachineIdPrefix.toString() +
-		                                   locality.machineId().get().toString()))
-		           .present();
-	}
-
-	return false;
-}
-
 // Gets the list of already excluded localities (with failed option)
 std::set<std::string> DatabaseConfiguration::getExcludedLocalities() const {
 	// TODO: revisit all const_cast usages
@@ -858,4 +844,4 @@ TEST_CASE("/fdbclient/databaseConfiguration/overwriteCommitProxy") {
 	ASSERT(conf1.getDesiredCommitProxies() == conf2.getDesiredCommitProxies());
 
 	return Void();
-}
\ No newline at end of file
+}
diff --git a/fdbclient/DatabaseConfiguration.h b/fdbclient/DatabaseConfiguration.h
index 409aa54814a..f41aa43cea8 100644
--- a/fdbclient/DatabaseConfiguration.h
+++ b/fdbclient/DatabaseConfiguration.h
@@ -260,7 +260,6 @@ struct DatabaseConfiguration {
 	// Excluded servers (no state should be here)
 	bool isExcludedServer(NetworkAddressList, const LocalityData& locality) const;
 	bool isExcludedLocality(const LocalityData& locality) const;
-	bool isMachineExcluded(const LocalityData& locality) const;
 	std::set<AddressExclusion> getExcludedServers() const;
 	std::set<std::string> getExcludedLocalities() const;
 
diff --git a/fdbclient/ManagementAPI.actor.cpp b/fdbclient/ManagementAPI.actor.cpp
index 0f7818772b8..dbfab7c8ed5 100644
--- a/fdbclient/ManagementAPI.actor.cpp
+++ b/fdbclient/ManagementAPI.actor.cpp
@@ -1818,21 +1818,56 @@ std::pair<std::string, std::string> decodeLocality(const std::string& locality)
 	return std::make_pair("", "");
 }
 
+// Returns the list of IPAddresses of the servers that match the given locality.
+// Example: locality="dcid:primary" returns all the ip addresses of the servers in the primary dc.
+std::set<AddressExclusion> getServerAddressesByLocality(
+    const std::map<std::string, StorageServerInterface> server_interfaces,
+    const std::string& locality) {
+	std::pair<std::string, std::string> locality_key_value = decodeLocality(locality);
+	std::set<AddressExclusion> locality_addresses;
+
+	for (auto& server : server_interfaces) {
+		auto locality_value = server.second.locality.get(locality_key_value.first);
+		if (!locality_value.present()) {
+			continue;
+		}
+
+		if (locality_value.get() != locality_key_value.second) {
+			continue;
+		}
+
+		auto primary_address = server.second.address();
+		locality_addresses.insert(AddressExclusion(primary_address.ip, primary_address.port));
+		if (server.second.secondaryAddress().present()) {
+			auto secondary_address = server.second.secondaryAddress().get();
+			locality_addresses.insert(AddressExclusion(secondary_address.ip, secondary_address.port));
+		}
+	}
+
+	return locality_addresses;
+}
+
 // Returns the list of IPAddresses of the workers that match the given locality.
 // Example: locality="locality_dcid:primary" returns all the ip addresses of the workers in the primary dc.
 std::set<AddressExclusion> getAddressesByLocality(const std::vector<ProcessData>& workers,
                                                   const std::string& locality) {
-	std::pair<std::string, std::string> localityKeyValue = decodeLocality(locality);
+	std::pair<std::string, std::string> locality_key_value = decodeLocality(locality);
+	std::set<AddressExclusion> locality_addresses;
 
-	std::set<AddressExclusion> localityAddresses;
 	for (int i = 0; i < workers.size(); i++) {
-		auto localityValue = workers[i].locality.get(localityKeyValue.first);
-		if (localityValue.present() && localityValue.get() == localityKeyValue.second) {
-			localityAddresses.insert(AddressExclusion(workers[i].address.ip, workers[i].address.port));
+		auto locality_value = workers[i].locality.get(locality_key_value.first);
+		if (!locality_value.present()) {
+			continue;
 		}
+
+		if (locality_value.get() != locality_key_value.second) {
+			continue;
+		}
+
+		locality_addresses.insert(AddressExclusion(workers[i].address.ip, workers[i].address.port));
 	}
 
-	return localityAddresses;
+	return locality_addresses;
 }
 
 ACTOR Future<Void> printHealthyZone(Database cx) {
diff --git a/fdbclient/ManagementAPI.actor.h b/fdbclient/ManagementAPI.actor.h
index 5261ec1b9bb..a9e99a2736b 100644
--- a/fdbclient/ManagementAPI.actor.h
+++ b/fdbclient/ManagementAPI.actor.h
@@ -106,11 +106,17 @@ ACTOR Future<std::vector<std::string>> getExcludedLocalityList(Transaction* tr);
 // Get the current list of failed localities.
 ACTOR Future<std::vector<std::string>> getExcludedFailedLocalityList(Transaction* tr);
 
+// Decodes the locality string to a pair of locality prefix and its value.
+// The prefix could be dcid, processid, machineid, processid.
+std::pair<std::string, std::string> decodeLocality(const std::string& locality);
+std::set<AddressExclusion> getServerAddressesByLocality(
+    const std::map<std::string, StorageServerInterface> server_interfaces,
+    const std::string& locality);
 std::set<AddressExclusion> getAddressesByLocality(const std::vector<ProcessData>& workers, const std::string& locality);
 
-// Check for the given, previously excluded servers to be evacuated (no longer used for state).  If waitForExclusion is
-// true, this actor returns once it is safe to shut down all such machines without impacting fault tolerance, until and
-// unless any of them are explicitly included with includeServers()
+// Check for the given, previously excluded servers to be evacuated (no longer used for state).  If waitForExclusion
+// is true, this actor returns once it is safe to shut down all such machines without impacting fault tolerance,
+// until and unless any of them are explicitly included with includeServers()
 ACTOR Future<std::set<NetworkAddress>> checkForExcludingServers(Database cx,
                                                                 std::vector<AddressExclusion> servers,
                                                                 bool waitForAllExcluded);
diff --git a/fdbclient/SpecialKeySpace.actor.cpp b/fdbclient/SpecialKeySpace.actor.cpp
index 23e7f43eeea..36081701f3e 100644
--- a/fdbclient/SpecialKeySpace.actor.cpp
+++ b/fdbclient/SpecialKeySpace.actor.cpp
@@ -1167,7 +1167,12 @@ ACTOR Future<RangeResult> ExclusionInProgressActor(ReadYourWritesTransaction* ry
 	tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE); // necessary?
 	tr.setOption(FDBTransactionOptions::LOCK_AWARE);
 
-	state std::vector<AddressExclusion> excl = wait((getAllExcludedServers(&tr)));
+	state Future<std::vector<AddressExclusion>> fExclusions = getAllExcludedServers(&tr);
+	state Future<std::vector<std::string>> fExcludedLocalities = getAllExcludedLocalities(&tr);
+
+	wait(success(fExclusions) && success(fExcludedLocalities));
+
+	state std::vector<AddressExclusion> excl = fExclusions.get();
 	state std::set<AddressExclusion> exclusions(excl.begin(), excl.end());
 	state std::set<NetworkAddress> inProgressExclusion;
 	// Just getting a consistent read version proves that a set of tlogs satisfying the exclusions has completed
@@ -1175,18 +1180,48 @@ ACTOR Future<RangeResult> ExclusionInProgressActor(ReadYourWritesTransaction* ry
 	state RangeResult serverList = wait(tr.getRange(serverListKeys, CLIENT_KNOBS->TOO_MANY));
 	ASSERT(!serverList.more && serverList.size() < CLIENT_KNOBS->TOO_MANY);
 
+	// We have to make use of the localities here to verify if a server is still in the server list,
+	// even if it might be missing in the workers as the server is not running anymore.
+	state std::vector<std::string> excludedLocalities = fExcludedLocalities.get();
+	// Decode the excluded localities to check if any server is excluded by locality.
+	state std::vector<std::pair<std::string, std::string>> decodedExcludedLocalities;
+	for (auto& excludedLocality : excludedLocalities) {
+		decodedExcludedLocalities.push_back(decodeLocality(excludedLocality));
+	}
+
 	for (auto& s : serverList) {
-		auto addresses = decodeServerListValue(s.value).getKeyValues.getEndpoint().addresses;
+		auto decodedServer = decodeServerListValue(s.value);
+		auto addresses = decodedServer.getKeyValues.getEndpoint().addresses;
 		if (addressExcluded(exclusions, addresses.address)) {
 			inProgressExclusion.insert(addresses.address);
 		}
+
 		if (addresses.secondaryAddress.present() && addressExcluded(exclusions, addresses.secondaryAddress.get())) {
 			inProgressExclusion.insert(addresses.secondaryAddress.get());
 		}
+
+		// Check if the server is excluded based on a locality.
+		for (auto& excludedLocality : decodedExcludedLocalities) {
+			if (!decodedServer.locality.isPresent(excludedLocality.first)) {
+				continue;
+			}
+
+			if (decodedServer.locality.get(excludedLocality.first) != excludedLocality.second) {
+				continue;
+			}
+
+			inProgressExclusion.insert(addresses.address);
+			if (addresses.secondaryAddress.present()) {
+				inProgressExclusion.insert(addresses.secondaryAddress.get());
+			}
+		}
 	}
 
 	Optional<Standalone<StringRef>> value = wait(tr.get(logsKey));
 	ASSERT(value.present());
+	// TODO(jscheuermann): The logs key range doesn't hold any information about localities. This is a limitation
+	// for locality based exclusions. The problematic edge case here is a log server that still has mutation on it
+	// but is currently not part of the worker list, e.g. because it was shutdown or is partitioned.
 	auto logs = decodeLogsValue(value.get());
 	for (auto const& log : logs.first) {
 		if (log.second == NetworkAddress() || addressExcluded(exclusions, log.second)) {
@@ -1212,6 +1247,7 @@ ACTOR Future<RangeResult> ExclusionInProgressActor(ReadYourWritesTransaction* ry
 			result.arena().dependsOn(addrKey.arena());
 		}
 	}
+
 	return result;
 }
 
diff --git a/fdbrpc/Locality.cpp b/fdbrpc/Locality.cpp
index bd4c3a94617..ba38472c0a7 100644
--- a/fdbrpc/Locality.cpp
+++ b/fdbrpc/Locality.cpp
@@ -26,7 +26,6 @@ const StringRef LocalityData::keyZoneId = LiteralStringRef("zoneid");
 const StringRef LocalityData::keyDcId = LiteralStringRef("dcid");
 const StringRef LocalityData::keyMachineId = LiteralStringRef("machineid");
 const StringRef LocalityData::keyDataHallId = LiteralStringRef("data_hall");
-const StringRef LocalityData::ExcludeLocalityKeyMachineIdPrefix = LiteralStringRef("locality_machineid:");
 const StringRef LocalityData::ExcludeLocalityPrefix = LiteralStringRef("locality_");
 
 ProcessClass::Fitness ProcessClass::machineClassFitness(ClusterRole role) const {
diff --git a/fdbrpc/Locality.h b/fdbrpc/Locality.h
index 3e8d07f6f43..2e92344ca87 100644
--- a/fdbrpc/Locality.h
+++ b/fdbrpc/Locality.h
@@ -335,7 +335,6 @@ struct LocalityData {
 	}
 
 	static const UID UNSET_ID;
-	static const StringRef ExcludeLocalityKeyMachineIdPrefix;
 	static const StringRef ExcludeLocalityPrefix;
 };
 

From b81e43647261512b47c02cac33b74431b9a05f5e Mon Sep 17 00:00:00 2001
From: "Johannes M. Scheuermann" <jscheuermann@apple.com>
Date: Tue, 24 Oct 2023 10:36:38 +0200
Subject: [PATCH 295/468] Add missing fdbcli util method

---
 fdbcli/Util.actor.cpp | 26 ++++++++++++++++++++++++++
 1 file changed, 26 insertions(+)

diff --git a/fdbcli/Util.actor.cpp b/fdbcli/Util.actor.cpp
index d40a5dcaebf..3c21a3f03ee 100644
--- a/fdbcli/Util.actor.cpp
+++ b/fdbcli/Util.actor.cpp
@@ -160,4 +160,30 @@ ACTOR Future<bool> getWorkers(Reference<IDatabase> db, std::vector<ProcessData>*
 	}
 }
 
+ACTOR Future<Void> getStorageServerInterfaces(Reference<IDatabase> db,
+                                              std::map<std::string, StorageServerInterface>* interfaces) {
+	state Reference<ITransaction> tr = db->createTransaction();
+	loop {
+		interfaces->clear();
+		try {
+			tr->setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
+			tr->setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
+			tr->setOption(FDBTransactionOptions::LOCK_AWARE);
+			state ThreadFuture<RangeResult> serverListF = tr->getRange(serverListKeys, CLIENT_KNOBS->TOO_MANY);
+			wait(success(safeThreadFutureToFuture(serverListF)));
+			ASSERT(!serverListF.get().more);
+			ASSERT_LT(serverListF.get().size(), CLIENT_KNOBS->TOO_MANY);
+			RangeResult serverList = serverListF.get();
+			// decode server interfaces
+			for (int i = 0; i < serverList.size(); i++) {
+				auto ssi = decodeServerListValue(serverList[i].value);
+				(*interfaces)[ssi.address().toString()] = ssi;
+			}
+			return Void();
+		} catch (Error& e) {
+			wait(safeThreadFutureToFuture(tr->onError(e)));
+		}
+	}
+}
+
 } // namespace fdb_cli

From 1a8f9a0eef8a788fe2925719daf775edfecaa967 Mon Sep 17 00:00:00 2001
From: "Johannes M. Scheuermann" <jscheuermann@apple.com>
Date: Tue, 24 Oct 2023 11:01:10 +0200
Subject: [PATCH 296/468] Add missing fdbcli util method

---
 fdbcli/fdbcli.actor.h | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/fdbcli/fdbcli.actor.h b/fdbcli/fdbcli.actor.h
index 087fbc98d60..a9370f0a23c 100644
--- a/fdbcli/fdbcli.actor.h
+++ b/fdbcli/fdbcli.actor.h
@@ -104,6 +104,9 @@ void printUsage(StringRef command);
 // Pre: tr failed with special_keys_api_failure error
 // Read the error message special key and return the message
 ACTOR Future<std::string> getSpecialKeysFailureErrorMessage(Reference<ITransaction> tr);
+// Using \xff\serverList/to get all server interfaces
+ACTOR Future<Void> getStorageServerInterfaces(Reference<IDatabase> db,
+                                              std::map<std::string, StorageServerInterface>* interfaces);
 // Using \xff\xff/worker_interfaces/ special key, get all worker interfaces
 ACTOR Future<Void> getWorkerInterfaces(Reference<ITransaction> tr,
                                        std::map<Key, std::pair<Value, ClientLeaderRegInterface>>* address_interface);

From 73dad4734cb1820873be459b269bf482177650d2 Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyu_zhou@apple.com>
Date: Tue, 24 Oct 2023 13:34:12 -0700
Subject: [PATCH 297/468] Update 7.1.42 release notes (#11029)

---
 .../sphinx/source/release-notes/release-notes-710.rst        | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/documentation/sphinx/source/release-notes/release-notes-710.rst b/documentation/sphinx/source/release-notes/release-notes-710.rst
index e7fe3b12619..6bdbf6b6392 100644
--- a/documentation/sphinx/source/release-notes/release-notes-710.rst
+++ b/documentation/sphinx/source/release-notes/release-notes-710.rst
@@ -16,7 +16,10 @@ Release Notes
 * Added knobs to control backup retry delays for blob stores. `(PR #10947) <https://github.com/apple/foundationdb/pull/10947>`_
 * Fixed two use-after-free bugs for backup agents. `(PR #10951) <https://github.com/apple/foundationdb/pull/10951>`_
 * Added automatic range split for hot storage queue. `(PR #10932) <https://github.com/apple/foundationdb/pull/10932>`_
-* Fixed multiple bugs related to locality based exclusions. `(PR #10976) <https://github.com/apple/foundationdb/pull/10976>`_
+* Fixed multiple bugs related to locality based exclusions. `(PR #10976) <https://github.com/apple/foundationdb/pull/10976>`_, `(PR #11008) <https://github.com/apple/foundationdb/pull/11008>`_, and `(PR #11025) <https://github.com/apple/foundationdb/pull/11025>`_
+* Fixed compaction rate limiter for RocksDB storage engine. `(PR #10988) <https://github.com/apple/foundationdb/pull/10988>`_
+* Added a perpetual wiggle option to have multiple storage servers in rebalance state during wiggling. `(PR #10995) <https://github.com/apple/foundationdb/pull/10995>`_
+* Fixed exclude status of machines in status json when not all processes are excluded. `(PR #10996) <https://github.com/apple/foundationdb/pull/10996>`_ and `(PR #11006) <https://github.com/apple/foundationdb/pull/11006>`_
 
 7.1.41
 ======

From fa0556c59d6dfbfbabf93cfda17e6f6b63276789 Mon Sep 17 00:00:00 2001
From: "Johannes M. Scheuermann" <jscheuermann@apple.com>
Date: Wed, 25 Oct 2023 12:26:34 +0200
Subject: [PATCH 298/468] Allow the exclusions of localities that are not
 matching any process

---
 fdbcli/ExcludeCommand.actor.cpp | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/fdbcli/ExcludeCommand.actor.cpp b/fdbcli/ExcludeCommand.actor.cpp
index 406019f88aa..8819089f457 100644
--- a/fdbcli/ExcludeCommand.actor.cpp
+++ b/fdbcli/ExcludeCommand.actor.cpp
@@ -320,8 +320,10 @@ ACTOR Future<bool> excludeCommandActor(Reference<IDatabase> db, std::vector<Stri
 			}
 		}
 
-		if (exclusionSet.empty()) {
-			fprintf(stderr, "ERROR: At least one valid network endpoint address or a locality is not provided\n");
+		// The validation if a locality or address has no match is done below and will result in a warning. If we abort
+		// here the provided locality and/or address will not be excluded.
+		if (exclusionAddresses.empty() && exclusionLocalities.empty()) {
+			fprintf(stderr, "ERROR: At least one valid network endpoint address or a locality must be provided\n");
 			return false;
 		}
 

From 1e413f73720e65fac4e63ce74ae8e09d06c76033 Mon Sep 17 00:00:00 2001
From: FoundationDB CI <foundationdb_ci@apple.com>
Date: Wed, 25 Oct 2023 16:35:02 +0000
Subject: [PATCH 299/468] disable AVX for 7.1.42 release

---
 cmake/ConfigureCompiler.cmake | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cmake/ConfigureCompiler.cmake b/cmake/ConfigureCompiler.cmake
index e52d8665ee8..7e50469ba6a 100644
--- a/cmake/ConfigureCompiler.cmake
+++ b/cmake/ConfigureCompiler.cmake
@@ -253,7 +253,7 @@ else()
       set(USE_AVX512F OFF)
     endif()
   endif()
-  set(USE_AVX ON CACHE BOOL "Enable AVX instructions")
+  set(USE_AVX OFF CACHE BOOL "Enable AVX instructions")
   if (USE_AVX)
     if (CMAKE_HOST_SYSTEM_PROCESSOR MATCHES "^x86")
       add_compile_options(-mavx)

From 0301045e333ac12c4d1184a4621ce6809d8211d4 Mon Sep 17 00:00:00 2001
From: FoundationDB CI <foundationdb_ci@apple.com>
Date: Wed, 25 Oct 2023 16:35:21 +0000
Subject: [PATCH 300/468] enable AVX and update version for 7.1.43 release

---
 CMakeLists.txt                | 2 +-
 cmake/ConfigureCompiler.cmake | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index d2a58f73a23..c58126050fe 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -23,7 +23,7 @@ else()
 endif()
 
 project(foundationdb
-  VERSION 7.1.42
+  VERSION 7.1.43
   DESCRIPTION "FoundationDB is a scalable, fault-tolerant, ordered key-value store with full ACID transactions."
   HOMEPAGE_URL "http://www.foundationdb.org/"
   LANGUAGES C CXX ASM)
diff --git a/cmake/ConfigureCompiler.cmake b/cmake/ConfigureCompiler.cmake
index 7e50469ba6a..e52d8665ee8 100644
--- a/cmake/ConfigureCompiler.cmake
+++ b/cmake/ConfigureCompiler.cmake
@@ -253,7 +253,7 @@ else()
       set(USE_AVX512F OFF)
     endif()
   endif()
-  set(USE_AVX OFF CACHE BOOL "Enable AVX instructions")
+  set(USE_AVX ON CACHE BOOL "Enable AVX instructions")
   if (USE_AVX)
     if (CMAKE_HOST_SYSTEM_PROCESSOR MATCHES "^x86")
       add_compile_options(-mavx)

From 12aef4242fd7cfdc56a38a01565307df5a2784a5 Mon Sep 17 00:00:00 2001
From: FoundationDB CI <foundationdb_ci@apple.com>
Date: Wed, 25 Oct 2023 22:32:40 +0000
Subject: [PATCH 301/468] update version after 7.1.43 release

---
 CMakeLists.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index c58126050fe..f7842a3ec68 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -23,7 +23,7 @@ else()
 endif()
 
 project(foundationdb
-  VERSION 7.1.43
+  VERSION 7.1.44
   DESCRIPTION "FoundationDB is a scalable, fault-tolerant, ordered key-value store with full ACID transactions."
   HOMEPAGE_URL "http://www.foundationdb.org/"
   LANGUAGES C CXX ASM)

From da6241005f9e30c621f7f2c7302a172d0d203dd6 Mon Sep 17 00:00:00 2001
From: Zhe Wang <zhe.wang@wustl.edu>
Date: Tue, 5 Dec 2023 21:00:08 -0800
Subject: [PATCH 302/468] fix-consistency-check-issues

---
 fdbclient/ClientKnobs.cpp                     |  5 ++
 fdbclient/ClientKnobs.h                       |  5 ++
 fdbserver/fdbserver.actor.cpp                 |  2 +-
 fdbserver/tester.actor.cpp                    | 17 +++--
 fdbserver/worker.actor.cpp                    |  1 +
 .../workloads/ConsistencyCheck.actor.cpp      | 66 +++++++++++++++++--
 6 files changed, 86 insertions(+), 10 deletions(-)

diff --git a/fdbclient/ClientKnobs.cpp b/fdbclient/ClientKnobs.cpp
index f0552923a39..b2535cfdf66 100644
--- a/fdbclient/ClientKnobs.cpp
+++ b/fdbclient/ClientKnobs.cpp
@@ -258,6 +258,11 @@ void ClientKnobs::initialize(Randomize randomize) {
 
 	init( CONSISTENCY_CHECK_RATE_LIMIT_MAX,        50e6 ); // Limit in per sec
 	init( CONSISTENCY_CHECK_ONE_ROUND_TARGET_COMPLETION_TIME,	7 * 24 * 60 * 60 ); // 7 days
+	init( CONSISTENCY_CHECK_CUSTOM_RANGE,          true );
+	init( CONSISTENCY_CHECK_DISTRIBUTED,           true );
+	init( CONSISTENCY_CHECK_DISTRIBUTED_WIGGLE_ROOM, 50 );
+	init( CONSISTENCY_CHECK_RANGE_BEGIN,             "" );
+	init( CONSISTENCY_CHECK_RANGE_END,       "\xff\xff" );
 
 	//fdbcli
 	init( CLI_CONNECT_PARALLELISM,                  400 );
diff --git a/fdbclient/ClientKnobs.h b/fdbclient/ClientKnobs.h
index a6a6b62a75f..811d2401349 100644
--- a/fdbclient/ClientKnobs.h
+++ b/fdbclient/ClientKnobs.h
@@ -249,6 +249,11 @@ class ClientKnobs : public KnobsImpl<ClientKnobs> {
 
 	int CONSISTENCY_CHECK_RATE_LIMIT_MAX;
 	int CONSISTENCY_CHECK_ONE_ROUND_TARGET_COMPLETION_TIME;
+	bool CONSISTENCY_CHECK_CUSTOM_RANGE;
+	bool CONSISTENCY_CHECK_DISTRIBUTED;
+	int CONSISTENCY_CHECK_DISTRIBUTED_WIGGLE_ROOM;
+	std::string CONSISTENCY_CHECK_RANGE_BEGIN;
+	std::string CONSISTENCY_CHECK_RANGE_END;
 
 	// fdbcli
 	int CLI_CONNECT_PARALLELISM;
diff --git a/fdbserver/fdbserver.actor.cpp b/fdbserver/fdbserver.actor.cpp
index 2cd200a509d..d59a1bbd7ac 100644
--- a/fdbserver/fdbserver.actor.cpp
+++ b/fdbserver/fdbserver.actor.cpp
@@ -2181,7 +2181,7 @@ int main(int argc, char* argv[]) {
 			auto m = startSystemMonitor(opts.dataFolder, opts.dcId, opts.zoneId, opts.zoneId);
 			f = stopAfter(runTests(opts.connectionFile,
 			                       TEST_TYPE_CONSISTENCY_CHECK,
-			                       TEST_HERE,
+			                       CLIENT_KNOBS->CONSISTENCY_CHECK_DISTRIBUTED ? TEST_ON_TESTERS : TEST_HERE,
 			                       1,
 			                       opts.testFile,
 			                       StringRef(),
diff --git a/fdbserver/tester.actor.cpp b/fdbserver/tester.actor.cpp
index 56128e54324..0178c29f0d7 100644
--- a/fdbserver/tester.actor.cpp
+++ b/fdbserver/tester.actor.cpp
@@ -562,7 +562,10 @@ ACTOR Future<Void> runWorkloadAsync(Database cx,
 			startReq = req;
 			if (!startResult.present()) {
 				try {
-					TraceEvent("TestStarting", workIface.id()).detail("Workload", workload->description());
+					TraceEvent("TestStarting", workIface.id())
+					    .detail("Workload", workload->description())
+					    .detail("ClientCount", workload->clientCount)
+					    .detail("ClientId", workload->clientId);
 					wait(workload->start(cx) || databaseError);
 					startResult = Void();
 				} catch (Error& e) {
@@ -949,9 +952,11 @@ ACTOR Future<Void> checkConsistency(Database cx,
 	options.push_back_deep(options.arena(),
 	                       KeyValueRef(LiteralStringRef("quiescentWaitTimeout"),
 	                                   ValueRef(options.arena(), format("%f", quiescentWaitTimeout))));
-	options.push_back_deep(options.arena(), KeyValueRef(LiteralStringRef("distributed"), LiteralStringRef("false")));
+	// options.push_back_deep(options.arena(), KeyValueRef(LiteralStringRef("distributed"), LiteralStringRef("false")));
+	options.push_back_deep(options.arena(),
+	                       KeyValueRef(LiteralStringRef("distributed"),
+	                                   StringRef(CLIENT_KNOBS->CONSISTENCY_CHECK_DISTRIBUTED ? "true" : "false")));
 	spec.options.push_back_deep(spec.options.arena(), options);
-
 	state double start = now();
 	state bool lastRun = false;
 	loop {
@@ -1659,6 +1664,7 @@ ACTOR Future<Void> runTests(Reference<AsyncVar<Optional<struct ClusterController
                             Optional<TenantName> defaultTenant) {
 	state int flags = (at == TEST_ON_SERVERS ? 0 : GetWorkersRequest::TESTER_CLASS_ONLY) |
 	                  GetWorkersRequest::NON_EXCLUDED_PROCESSES_ONLY;
+	TraceEvent("RunTests").detail("TestOnServers", at == TEST_ON_SERVERS);
 	state Future<Void> testerTimeout = delay(600.0); // wait 600 sec for testers to show up
 	state std::vector<WorkerDetails> workers;
 
@@ -1749,8 +1755,11 @@ ACTOR Future<Void> runTests(Reference<IClusterConnectionRecord> connRecord,
 		                       KeyValueRef(LiteralStringRef("testName"), LiteralStringRef("ConsistencyCheck")));
 		options.push_back_deep(options.arena(),
 		                       KeyValueRef(LiteralStringRef("performQuiescentChecks"), LiteralStringRef("false")));
+		// options.push_back_deep(options.arena(),
+		//                        KeyValueRef(LiteralStringRef("distributed"), LiteralStringRef("false")));
 		options.push_back_deep(options.arena(),
-		                       KeyValueRef(LiteralStringRef("distributed"), LiteralStringRef("false")));
+		                       KeyValueRef(LiteralStringRef("distributed"),
+		                                   StringRef(CLIENT_KNOBS->CONSISTENCY_CHECK_DISTRIBUTED ? "true" : "false")));
 		options.push_back_deep(options.arena(),
 		                       KeyValueRef(LiteralStringRef("failureIsError"), LiteralStringRef("true")));
 		options.push_back_deep(options.arena(), KeyValueRef(LiteralStringRef("indefinite"), LiteralStringRef("true")));
diff --git a/fdbserver/worker.actor.cpp b/fdbserver/worker.actor.cpp
index 472c343bd66..c89e5eabda5 100644
--- a/fdbserver/worker.actor.cpp
+++ b/fdbserver/worker.actor.cpp
@@ -3155,6 +3155,7 @@ ACTOR Future<Void> fdbd(Reference<IClusterConnectionRecord> connRecord,
 			whitelistBinPaths = ",, random_path,  /bin/snap_create.sh,,";
 		}
 		TraceEvent("StartingFDBD")
+		    .detail("ProcessClass", processClass.toString())
 		    .detail("ZoneID", localities.zoneId())
 		    .detail("MachineId", localities.machineId())
 		    .detail("DiskPath", dataFolder)
diff --git a/fdbserver/workloads/ConsistencyCheck.actor.cpp b/fdbserver/workloads/ConsistencyCheck.actor.cpp
index f41f775da7a..bec25ecc8d6 100644
--- a/fdbserver/workloads/ConsistencyCheck.actor.cpp
+++ b/fdbserver/workloads/ConsistencyCheck.actor.cpp
@@ -187,10 +187,16 @@ struct ConsistencyCheckWorkload : TestWorkload {
 	ACTOR Future<Void> _start(Database cx, ConsistencyCheckWorkload* self) {
 		loop {
 			while (self->suspendConsistencyCheck.get()) {
-				TraceEvent("ConsistencyCheck_Suspended").log();
+				TraceEvent("ConsistencyCheck_Suspended")
+				    .detail("Distributed", self->distributed)
+				    .detail("ClientCount", self->clientCount)
+				    .detail("ClientId", self->clientId);
 				wait(self->suspendConsistencyCheck.onChange());
 			}
-			TraceEvent("ConsistencyCheck_StartingOrResuming").log();
+			TraceEvent("ConsistencyCheck_StartingOrResuming")
+			    .detail("Distributed", self->distributed)
+			    .detail("ClientCount", self->clientCount)
+			    .detail("ClientId", self->clientId);
 			choose {
 				when(wait(self->runCheck(cx, self))) {
 					if (!self->indefinite)
@@ -319,7 +325,35 @@ struct ConsistencyCheckWorkload : TestWorkload {
 
 				// Get a list of key servers; verify that the TLogs and master all agree about who the key servers are
 				state Promise<std::vector<std::pair<KeyRange, std::vector<StorageServerInterface>>>> keyServerPromise;
-				bool keyServerResult = wait(self->getKeyServers(cx, self, keyServerPromise, keyServersKeys));
+				// bool keyServerResult = wait(self->getKeyServers(cx, self, keyServerPromise, keyServersKeys));
+
+				KeyRange rangeToCheck = allKeys;
+				if (CLIENT_KNOBS->CONSISTENCY_CHECK_CUSTOM_RANGE) {
+					Key rangeBegin = StringRef(CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_BEGIN);
+					if (rangeBegin > allKeys.end) {
+						rangeBegin = allKeys.end;
+					}
+					Key rangeEnd = StringRef(CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_END);
+					if (rangeEnd > allKeys.end) {
+						rangeEnd = allKeys.end;
+					}
+					if (rangeBegin < rangeEnd) {
+						rangeToCheck = KeyRangeRef(rangeBegin, rangeEnd);
+					} else if (rangeBegin > rangeEnd) {
+						rangeToCheck = KeyRangeRef(rangeEnd, rangeBegin);
+					} else {
+						rangeToCheck = allKeys;
+					}
+				}
+
+				TraceEvent("ConsistencyCheck_Config")
+				    .detail("RangeToCheck", rangeToCheck)
+				    .detail("Distributed", self->distributed)
+				    .detail("ClientCount", self->clientCount)
+				    .detail("ClientId", self->clientId);
+
+				bool keyServerResult =
+				    wait(self->getKeyServers(cx, self, keyServerPromise, rangeToCheck.withPrefix(keyServersPrefix)));
 				if (keyServerResult) {
 					state std::vector<std::pair<KeyRange, std::vector<StorageServerInterface>>> keyServers =
 					    keyServerPromise.getFuture().get();
@@ -1236,13 +1270,35 @@ struct ConsistencyCheckWorkload : TestWorkload {
 		shardOrder.reserve(ranges.size());
 		for (int k = 0; k < ranges.size(); k++)
 			shardOrder.push_back(k);
-		if (self->shuffleShards) {
+		if (self->shuffleShards && !CLIENT_KNOBS->CONSISTENCY_CHECK_DISTRIBUTED) {
 			uint32_t seed = self->sharedRandomNumber + self->repetitions;
 			DeterministicRandom sharedRandom(seed == 0 ? 1 : seed);
 			sharedRandom.randomShuffle(shardOrder);
 		}
 
-		for (; i < ranges.size(); i += increment) {
+		state int endPoint;
+		if (CLIENT_KNOBS->CONSISTENCY_CHECK_DISTRIBUTED) {
+			int batchSize = (ranges.size() / self->clientCount) + 1;
+			i = self->clientId * batchSize; // overwrite the starting point i
+			i = std::max(0, i - CLIENT_KNOBS->CONSISTENCY_CHECK_DISTRIBUTED_WIGGLE_ROOM);
+			endPoint = (self->clientId + 1) * batchSize;
+			endPoint = std::min(static_cast<int>(ranges.size()),
+			                    endPoint + CLIENT_KNOBS->CONSISTENCY_CHECK_DISTRIBUTED_WIGGLE_ROOM);
+			increment = 1;
+		} else {
+			endPoint = ranges.size();
+		}
+
+		TraceEvent("ConsistencyCheck_StartTask")
+		    .detail("ShardCount", ranges.size())
+		    .detail("ClientId", self->clientId)
+		    .detail("ClientCount", self->clientCount)
+		    .detail("StartPoint", i)
+		    .detail("EndPoint", endPoint)
+		    .detail("BeginKey", ranges[i].begin)
+		    .detail("EndKey", ranges[endPoint - 1].end);
+
+		for (; i < endPoint; i += increment) {
 			state int shard = shardOrder[i];
 
 			state KeyRangeRef range = ranges[shard];

From 28f0a5f8fe237fc767c90ff04bc65b78b5dcdf3d Mon Sep 17 00:00:00 2001
From: Zhe Wang <zhe.wang@wustl.edu>
Date: Wed, 6 Dec 2023 12:59:16 -0800
Subject: [PATCH 303/468] fix

---
 .../workloads/ConsistencyCheck.actor.cpp      | 32 +++++++++++++++----
 1 file changed, 26 insertions(+), 6 deletions(-)

diff --git a/fdbserver/workloads/ConsistencyCheck.actor.cpp b/fdbserver/workloads/ConsistencyCheck.actor.cpp
index bec25ecc8d6..6664df098b2 100644
--- a/fdbserver/workloads/ConsistencyCheck.actor.cpp
+++ b/fdbserver/workloads/ConsistencyCheck.actor.cpp
@@ -1266,6 +1266,9 @@ struct ConsistencyCheckWorkload : TestWorkload {
 			ranges.push_back(range);
 		}
 
+		state Key beginKeyToCheck = ranges[0].begin;
+		state Key endKeyToCheck = ranges[ranges.size() - 1].end;
+
 		state std::vector<int> shardOrder;
 		shardOrder.reserve(ranges.size());
 		for (int k = 0; k < ranges.size(); k++)
@@ -1278,25 +1281,29 @@ struct ConsistencyCheckWorkload : TestWorkload {
 
 		state int endPoint;
 		if (CLIENT_KNOBS->CONSISTENCY_CHECK_DISTRIBUTED) {
-			int batchSize = (ranges.size() / self->clientCount) + 1;
+			int batchSize = (shardOrder.size() / self->clientCount) + 1;
 			i = self->clientId * batchSize; // overwrite the starting point i
 			i = std::max(0, i - CLIENT_KNOBS->CONSISTENCY_CHECK_DISTRIBUTED_WIGGLE_ROOM);
 			endPoint = (self->clientId + 1) * batchSize;
-			endPoint = std::min(static_cast<int>(ranges.size()),
+			endPoint = std::min(static_cast<int>(shardOrder.size()),
 			                    endPoint + CLIENT_KNOBS->CONSISTENCY_CHECK_DISTRIBUTED_WIGGLE_ROOM);
 			increment = 1;
+			// overwrite beginKeyToCheck and endKeyToCheck
+			beginKeyToCheck = ranges[shardOrder[i]].begin;
+			endKeyToCheck = ranges[shardOrder[endPoint - 1]].end;
 		} else {
-			endPoint = ranges.size();
+			endPoint = shardOrder.size();
 		}
 
 		TraceEvent("ConsistencyCheck_StartTask")
+		    .detail("Distributed", self->distributed)
 		    .detail("ShardCount", ranges.size())
 		    .detail("ClientId", self->clientId)
 		    .detail("ClientCount", self->clientCount)
 		    .detail("StartPoint", i)
 		    .detail("EndPoint", endPoint)
-		    .detail("BeginKey", ranges[i].begin)
-		    .detail("EndKey", ranges[endPoint - 1].end);
+		    .detail("BeginKey", beginKeyToCheck)
+		    .detail("EndKey", endKeyToCheck);
 
 		for (; i < endPoint; i += increment) {
 			state int shard = shardOrder[i];
@@ -1436,7 +1443,8 @@ struct ConsistencyCheckWorkload : TestWorkload {
 
 			// The first client may need to skip the rest of the loop contents if it is just processing this shard to
 			// get a size estimate
-			if (!self->firstClient || shard % (effectiveClientCount * self->shardSampleFactor) == 0) {
+			if (CLIENT_KNOBS->CONSISTENCY_CHECK_DISTRIBUTED || !self->firstClient ||
+			    shard % (effectiveClientCount * self->shardSampleFactor) == 0) {
 				state int shardKeys = 0;
 				state int shardBytes = 0;
 				state int sampledBytes = 0;
@@ -1861,6 +1869,8 @@ struct ConsistencyCheckWorkload : TestWorkload {
 			numCompleteShards++;
 			TraceEvent(SevInfo, "ConsistencyCheck_ShardComplete")
 			    .suppressFor(1.0)
+			    .detail("ClientId", self->clientId)
+			    .detail("ClientCount", self->clientCount)
 			    .detail("Range", range.toString())
 			    .detail("ShardCount", ranges.size())
 			    .detail("NumCompletedShards", numCompleteShards)
@@ -1868,6 +1878,16 @@ struct ConsistencyCheckWorkload : TestWorkload {
 			    .detail("NumSkippedShards", numSkippedShards);
 		}
 
+		TraceEvent("ConsistencyCheck_EndTask")
+		    .detail("Distributed", self->distributed)
+		    .detail("ShardCount", ranges.size())
+		    .detail("ClientId", self->clientId)
+		    .detail("ClientCount", self->clientCount)
+		    .detail("StartPoint", i)
+		    .detail("EndPoint", endPoint)
+		    .detail("BeginKey", beginKeyToCheck)
+		    .detail("EndKey", endKeyToCheck);
+
 		// SOMEDAY: when background data distribution is implemented, include this test
 		// In a quiescent database, check that the sizes of storage servers are roughly the same
 		/*if(self->performQuiescentChecks)

From 8870df510c888dc0a965dc9156bdff0e4ba4a4ae Mon Sep 17 00:00:00 2001
From: Zhe Wang <zhe.wang@wustl.edu>
Date: Wed, 6 Dec 2023 14:16:15 -0800
Subject: [PATCH 304/468] allow real time cancel

---
 .../workloads/ConsistencyCheck.actor.cpp      | 29 +++++++++++++++++--
 1 file changed, 27 insertions(+), 2 deletions(-)

diff --git a/fdbserver/workloads/ConsistencyCheck.actor.cpp b/fdbserver/workloads/ConsistencyCheck.actor.cpp
index 6664df098b2..7437ade6bc0 100644
--- a/fdbserver/workloads/ConsistencyCheck.actor.cpp
+++ b/fdbserver/workloads/ConsistencyCheck.actor.cpp
@@ -1306,6 +1306,19 @@ struct ConsistencyCheckWorkload : TestWorkload {
 		    .detail("EndKey", endKeyToCheck);
 
 		for (; i < endPoint; i += increment) {
+			if (CLIENT_KNOBS->CONSISTENCY_CHECK_DISTRIBUTED && self->suspendConsistencyCheck.get()) {
+				TraceEvent("ConsistencyCheck_Cancelled")
+					.detail("Distributed", self->distributed)
+					.detail("ShardCount", ranges.size())
+					.detail("ClientId", self->clientId)
+					.detail("ClientCount", self->clientCount)
+					.detail("CurrentPoint", i)
+					.detail("EndPoint", endPoint)
+					.detail("BeginKey", beginKeyToCheck)
+					.detail("EndKey", endKeyToCheck)
+					.detail("CompleteEndKey", ranges[shardOrder[i]].begin);
+					break;
+			}
 			state int shard = shardOrder[i];
 
 			state KeyRangeRef range = ranges[shard];
@@ -1465,6 +1478,20 @@ struct ConsistencyCheckWorkload : TestWorkload {
 				// Read a limited number of entries at a time, repeating until all keys in the shard have been read
 				loop {
 					try {
+						if (CLIENT_KNOBS->CONSISTENCY_CHECK_DISTRIBUTED && self->suspendConsistencyCheck.get()) {
+							TraceEvent("ConsistencyCheck_DataCheckCancelled")
+								.detail("Distributed", self->distributed)
+								.detail("ShardCount", ranges.size())
+								.detail("ClientId", self->clientId)
+								.detail("ClientCount", self->clientCount)
+								.detail("CurrentPoint", i)
+								.detail("EndPoint", endPoint)
+								.detail("BeginKey", beginKeyToCheck)
+								.detail("EndKey", endKeyToCheck)
+								.detail("CompleteEndKey", lastStartSampleKey);
+								break;
+						}
+
 						lastSampleKey = lastStartSampleKey;
 
 						// Get the min version of the storage servers
@@ -1883,8 +1910,6 @@ struct ConsistencyCheckWorkload : TestWorkload {
 		    .detail("ShardCount", ranges.size())
 		    .detail("ClientId", self->clientId)
 		    .detail("ClientCount", self->clientCount)
-		    .detail("StartPoint", i)
-		    .detail("EndPoint", endPoint)
 		    .detail("BeginKey", beginKeyToCheck)
 		    .detail("EndKey", endKeyToCheck);
 

From 26b4f5414f296110c5cdf982f1f032e48c9fa83a Mon Sep 17 00:00:00 2001
From: Zhe Wang <zhe.wang@wustl.edu>
Date: Wed, 6 Dec 2023 14:17:25 -0800
Subject: [PATCH 305/468] allow real time cancel

---
 .../workloads/ConsistencyCheck.actor.cpp      | 40 +++++++++----------
 1 file changed, 20 insertions(+), 20 deletions(-)

diff --git a/fdbserver/workloads/ConsistencyCheck.actor.cpp b/fdbserver/workloads/ConsistencyCheck.actor.cpp
index 7437ade6bc0..a48ae7b4197 100644
--- a/fdbserver/workloads/ConsistencyCheck.actor.cpp
+++ b/fdbserver/workloads/ConsistencyCheck.actor.cpp
@@ -1308,16 +1308,16 @@ struct ConsistencyCheckWorkload : TestWorkload {
 		for (; i < endPoint; i += increment) {
 			if (CLIENT_KNOBS->CONSISTENCY_CHECK_DISTRIBUTED && self->suspendConsistencyCheck.get()) {
 				TraceEvent("ConsistencyCheck_Cancelled")
-					.detail("Distributed", self->distributed)
-					.detail("ShardCount", ranges.size())
-					.detail("ClientId", self->clientId)
-					.detail("ClientCount", self->clientCount)
-					.detail("CurrentPoint", i)
-					.detail("EndPoint", endPoint)
-					.detail("BeginKey", beginKeyToCheck)
-					.detail("EndKey", endKeyToCheck)
-					.detail("CompleteEndKey", ranges[shardOrder[i]].begin);
-					break;
+				    .detail("Distributed", self->distributed)
+				    .detail("ShardCount", ranges.size())
+				    .detail("ClientId", self->clientId)
+				    .detail("ClientCount", self->clientCount)
+				    .detail("CurrentPoint", i)
+				    .detail("EndPoint", endPoint)
+				    .detail("BeginKey", beginKeyToCheck)
+				    .detail("EndKey", endKeyToCheck)
+				    .detail("CompleteEndKey", ranges[shardOrder[i]].begin);
+				break;
 			}
 			state int shard = shardOrder[i];
 
@@ -1480,16 +1480,16 @@ struct ConsistencyCheckWorkload : TestWorkload {
 					try {
 						if (CLIENT_KNOBS->CONSISTENCY_CHECK_DISTRIBUTED && self->suspendConsistencyCheck.get()) {
 							TraceEvent("ConsistencyCheck_DataCheckCancelled")
-								.detail("Distributed", self->distributed)
-								.detail("ShardCount", ranges.size())
-								.detail("ClientId", self->clientId)
-								.detail("ClientCount", self->clientCount)
-								.detail("CurrentPoint", i)
-								.detail("EndPoint", endPoint)
-								.detail("BeginKey", beginKeyToCheck)
-								.detail("EndKey", endKeyToCheck)
-								.detail("CompleteEndKey", lastStartSampleKey);
-								break;
+							    .detail("Distributed", self->distributed)
+							    .detail("ShardCount", ranges.size())
+							    .detail("ClientId", self->clientId)
+							    .detail("ClientCount", self->clientCount)
+							    .detail("CurrentPoint", i)
+							    .detail("EndPoint", endPoint)
+							    .detail("BeginKey", beginKeyToCheck)
+							    .detail("EndKey", endKeyToCheck)
+							    .detail("CompleteEndKey", lastStartSampleKey);
+							break;
 						}
 
 						lastSampleKey = lastStartSampleKey;

From 60bf37b8ffdf327671405ed21efa23e31680f245 Mon Sep 17 00:00:00 2001
From: Zhe Wang <zhe.wang@wustl.edu>
Date: Wed, 6 Dec 2023 16:32:58 -0800
Subject: [PATCH 306/468] add trace length

---
 fdbserver/workloads/ConsistencyCheck.actor.cpp | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/fdbserver/workloads/ConsistencyCheck.actor.cpp b/fdbserver/workloads/ConsistencyCheck.actor.cpp
index a48ae7b4197..10c873a12da 100644
--- a/fdbserver/workloads/ConsistencyCheck.actor.cpp
+++ b/fdbserver/workloads/ConsistencyCheck.actor.cpp
@@ -347,6 +347,8 @@ struct ConsistencyCheckWorkload : TestWorkload {
 				}
 
 				TraceEvent("ConsistencyCheck_Config")
+				    .setMaxEventLength(-1)
+				    .setMaxFieldLength(-1)
 				    .detail("RangeToCheck", rangeToCheck)
 				    .detail("Distributed", self->distributed)
 				    .detail("ClientCount", self->clientCount)
@@ -1296,6 +1298,8 @@ struct ConsistencyCheckWorkload : TestWorkload {
 		}
 
 		TraceEvent("ConsistencyCheck_StartTask")
+		    .setMaxEventLength(-1)
+		    .setMaxFieldLength(-1)
 		    .detail("Distributed", self->distributed)
 		    .detail("ShardCount", ranges.size())
 		    .detail("ClientId", self->clientId)
@@ -1308,6 +1312,8 @@ struct ConsistencyCheckWorkload : TestWorkload {
 		for (; i < endPoint; i += increment) {
 			if (CLIENT_KNOBS->CONSISTENCY_CHECK_DISTRIBUTED && self->suspendConsistencyCheck.get()) {
 				TraceEvent("ConsistencyCheck_Cancelled")
+				    .setMaxEventLength(-1)
+				    .setMaxFieldLength(-1)
 				    .detail("Distributed", self->distributed)
 				    .detail("ShardCount", ranges.size())
 				    .detail("ClientId", self->clientId)
@@ -1480,6 +1486,8 @@ struct ConsistencyCheckWorkload : TestWorkload {
 					try {
 						if (CLIENT_KNOBS->CONSISTENCY_CHECK_DISTRIBUTED && self->suspendConsistencyCheck.get()) {
 							TraceEvent("ConsistencyCheck_DataCheckCancelled")
+							    .setMaxEventLength(-1)
+							    .setMaxFieldLength(-1)
 							    .detail("Distributed", self->distributed)
 							    .detail("ShardCount", ranges.size())
 							    .detail("ClientId", self->clientId)
@@ -1640,6 +1648,8 @@ struct ConsistencyCheckWorkload : TestWorkload {
 										}
 
 										TraceEvent("ConsistencyCheck_DataInconsistent")
+										    .setMaxEventLength(-1)
+										    .setMaxFieldLength(-1)
 										    .detail(format("StorageServer%d", j).c_str(), storageServers[j].toString())
 										    .detail(format("StorageServer%d", firstValidServer).c_str(),
 										            storageServers[firstValidServer].toString())
@@ -1680,6 +1690,8 @@ struct ConsistencyCheckWorkload : TestWorkload {
 								TraceEvent("ConsistencyCheck_StorageServerUnavailable")
 								    .errorUnsuppressed(e)
 								    .suppressFor(1.0)
+								    .setMaxEventLength(-1)
+								    .setMaxFieldLength(-1)
 								    .detail("StorageServer", storageServers[j])
 								    .detail("ShardBegin", printable(range.begin))
 								    .detail("ShardEnd", printable(range.end))
@@ -1896,6 +1908,8 @@ struct ConsistencyCheckWorkload : TestWorkload {
 			numCompleteShards++;
 			TraceEvent(SevInfo, "ConsistencyCheck_ShardComplete")
 			    .suppressFor(1.0)
+			    .setMaxEventLength(-1)
+			    .setMaxFieldLength(-1)
 			    .detail("ClientId", self->clientId)
 			    .detail("ClientCount", self->clientCount)
 			    .detail("Range", range.toString())
@@ -1906,6 +1920,8 @@ struct ConsistencyCheckWorkload : TestWorkload {
 		}
 
 		TraceEvent("ConsistencyCheck_EndTask")
+		    .setMaxEventLength(-1)
+		    .setMaxFieldLength(-1)
 		    .detail("Distributed", self->distributed)
 		    .detail("ShardCount", ranges.size())
 		    .detail("ClientId", self->clientId)

From a7bad66a2e7a8b984fef7df0ce6167fd47214718 Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyu_zhou@apple.com>
Date: Thu, 7 Dec 2023 14:30:52 -0800
Subject: [PATCH 307/468] Merge pull request #11091 from sbodagala/main

Address issues in code related to "fdbcli checkall" command
---
 fdbcli/DebugCommands.actor.cpp | 13 ++++++++-----
 1 file changed, 8 insertions(+), 5 deletions(-)

diff --git a/fdbcli/DebugCommands.actor.cpp b/fdbcli/DebugCommands.actor.cpp
index 7fc8634677d..c6a8665856d 100644
--- a/fdbcli/DebugCommands.actor.cpp
+++ b/fdbcli/DebugCommands.actor.cpp
@@ -192,9 +192,12 @@ bool checkResults(Version version,
 		}
 
 		allSame = false;
-		printf("#%d  %s\n", firstValidServer, servers[firstValidServer].address().toString().c_str());
-		printf("#%d  %s\n", j, servers[j].address().toString().c_str());
-		int currentI = 0, referenceI = 0;
+		printf("#%d  server: %s  key count: %lu\n",
+		       firstValidServer,
+		       servers[firstValidServer].address().toString().c_str(),
+		       reference.data.size());
+		printf("#%d  server: %s  key count: %lu\n", j, servers[j].address().toString().c_str(), current.data.size());
+		size_t currentI = 0, referenceI = 0;
 		while (currentI < current.data.size() || referenceI < reference.data.size()) {
 			if (currentI >= current.data.size()) {
 				printf(" #%d Unique key: %s\n", firstValidServer, printable(reference.data[referenceI].key).c_str());
@@ -209,9 +212,9 @@ bool checkResults(Version version,
 				if (currentKV.key == referenceKV.key) {
 					if (currentKV.value != referenceKV.value) {
 						printf(" Value mismatch key: %s\n", printable(currentKV.key).c_str());
-						currentI++;
-						referenceI++;
 					}
+					currentI++;
+					referenceI++;
 				} else if (currentKV.key < referenceKV.key) {
 					printf(" #%d Unique key: %s\n", j, printable(currentKV.key).c_str());
 					currentI++;

From 12e57170d40caafe0522fae6bc68527639284344 Mon Sep 17 00:00:00 2001
From: Zhe Wang <zhe.wang@wustl.edu>
Date: Thu, 7 Dec 2023 16:09:21 -0800
Subject: [PATCH 308/468] print ConsistencyCheck_ShardWrongPlaceSkipped

---
 fdbserver/workloads/ConsistencyCheck.actor.cpp | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/fdbserver/workloads/ConsistencyCheck.actor.cpp b/fdbserver/workloads/ConsistencyCheck.actor.cpp
index 10c873a12da..c701d6bdde7 100644
--- a/fdbserver/workloads/ConsistencyCheck.actor.cpp
+++ b/fdbserver/workloads/ConsistencyCheck.actor.cpp
@@ -1476,6 +1476,7 @@ struct ConsistencyCheckWorkload : TestWorkload {
 				state Key lastSampleKey;
 				state Key lastStartSampleKey;
 				state int64_t totalReadAmount = 0;
+				state bool completeCheck = true;
 
 				state KeySelector begin = firstGreaterOrEqual(range.begin);
 				state Transaction onErrorTr(
@@ -1483,6 +1484,7 @@ struct ConsistencyCheckWorkload : TestWorkload {
 
 				// Read a limited number of entries at a time, repeating until all keys in the shard have been read
 				loop {
+					completeCheck = true;
 					try {
 						if (CLIENT_KNOBS->CONSISTENCY_CHECK_DISTRIBUTED && self->suspendConsistencyCheck.get()) {
 							TraceEvent("ConsistencyCheck_DataCheckCancelled")
@@ -1684,6 +1686,7 @@ struct ConsistencyCheckWorkload : TestWorkload {
 
 							// If the data is not available and we aren't relocating this shard
 							else if (!isRelocating) {
+								completeCheck = false;
 								Error e =
 								    rangeResult.isError() ? rangeResult.getError() : rangeResult.get().error.get();
 
@@ -1706,6 +1709,8 @@ struct ConsistencyCheckWorkload : TestWorkload {
 									self->testFailure("Storage server unavailable");
 									return false;
 								}
+							} else {
+								completeCheck = false;
 							}
 						}
 
@@ -1785,6 +1790,16 @@ struct ConsistencyCheckWorkload : TestWorkload {
 					}
 				}
 
+				if (!completeCheck) {
+					TraceEvent(SevWarnAlways, "ConsistencyCheck_ShardNoCompleteCheck")
+					    .setMaxEventLength(-1)
+					    .setMaxFieldLength(-1)
+					    .detail("ClientId", self->clientId)
+					    .detail("ClientCount", self->clientCount)
+					    .detail("ShardBegin", printable(range.begin))
+					    .detail("ShardEnd", printable(range.end));
+				}
+
 				canSplit = canSplit && sampledBytes - splitBytes >= shardBounds.min.bytes && sampledBytes > splitBytes;
 
 				// Update the size of all storage servers containing this shard

From 64092984a3c4d4b688094d2af9abf7a155db85dc Mon Sep 17 00:00:00 2001
From: Zhe Wang <zhe.wang@wustl.edu>
Date: Thu, 7 Dec 2023 20:53:47 -0800
Subject: [PATCH 309/468] refine trace event

---
 fdbserver/workloads/ConsistencyCheck.actor.cpp | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/fdbserver/workloads/ConsistencyCheck.actor.cpp b/fdbserver/workloads/ConsistencyCheck.actor.cpp
index c701d6bdde7..8c332b83775 100644
--- a/fdbserver/workloads/ConsistencyCheck.actor.cpp
+++ b/fdbserver/workloads/ConsistencyCheck.actor.cpp
@@ -1307,7 +1307,9 @@ struct ConsistencyCheckWorkload : TestWorkload {
 		    .detail("StartPoint", i)
 		    .detail("EndPoint", endPoint)
 		    .detail("BeginKey", beginKeyToCheck)
-		    .detail("EndKey", endKeyToCheck);
+		    .detail("EndKey", endKeyToCheck)
+		    .detail("Repetitions", self->repetitions)
+		    .trackLatest("ConsistencyCheck_StartTask");
 
 		for (; i < endPoint; i += increment) {
 			if (CLIENT_KNOBS->CONSISTENCY_CHECK_DISTRIBUTED && self->suspendConsistencyCheck.get()) {
@@ -1797,7 +1799,9 @@ struct ConsistencyCheckWorkload : TestWorkload {
 					    .detail("ClientId", self->clientId)
 					    .detail("ClientCount", self->clientCount)
 					    .detail("ShardBegin", printable(range.begin))
-					    .detail("ShardEnd", printable(range.end));
+					    .detail("ShardEnd", printable(range.end))
+					    .detail("Repetitions", self->repetitions)
+					    .trackLatest("ConsistencyCheck_ShardNoCompleteCheck");
 				}
 
 				canSplit = canSplit && sampledBytes - splitBytes >= shardBounds.min.bytes && sampledBytes > splitBytes;
@@ -1942,7 +1946,9 @@ struct ConsistencyCheckWorkload : TestWorkload {
 		    .detail("ClientId", self->clientId)
 		    .detail("ClientCount", self->clientCount)
 		    .detail("BeginKey", beginKeyToCheck)
-		    .detail("EndKey", endKeyToCheck);
+		    .detail("EndKey", endKeyToCheck)
+		    .detail("Repetitions", self->repetitions)
+		    .trackLatest("ConsistencyCheck_EndTask");
 
 		// SOMEDAY: when background data distribution is implemented, include this test
 		// In a quiescent database, check that the sizes of storage servers are roughly the same

From a32690831a61b116f409bcc845a5b1e4faa13afb Mon Sep 17 00:00:00 2001
From: Zhe Wang <zhe.wang@wustl.edu>
Date: Thu, 7 Dec 2023 22:12:01 -0800
Subject: [PATCH 310/468] handle a corner case

---
 fdbserver/workloads/ConsistencyCheck.actor.cpp | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/fdbserver/workloads/ConsistencyCheck.actor.cpp b/fdbserver/workloads/ConsistencyCheck.actor.cpp
index 8c332b83775..3e7b5a7d849 100644
--- a/fdbserver/workloads/ConsistencyCheck.actor.cpp
+++ b/fdbserver/workloads/ConsistencyCheck.actor.cpp
@@ -1290,6 +1290,17 @@ struct ConsistencyCheckWorkload : TestWorkload {
 			endPoint = std::min(static_cast<int>(shardOrder.size()),
 			                    endPoint + CLIENT_KNOBS->CONSISTENCY_CHECK_DISTRIBUTED_WIGGLE_ROOM);
 			increment = 1;
+			// If the start point i is larger than the list boundary, this client needs not run
+			if (i >= endPoint) {
+				TraceEvent("ConsistencyCheck_NoTaskToRun")
+				    .detail("Distributed", self->distributed)
+				    .detail("ShardCount", ranges.size())
+				    .detail("ClientId", self->clientId)
+				    .detail("ClientCount", self->clientCount)
+				    .detail("Repetitions", self->repetitions);
+				wait(delay(600.0) || self->suspendConsistencyCheck.onChange());
+				return true;
+			}
 			// overwrite beginKeyToCheck and endKeyToCheck
 			beginKeyToCheck = ranges[shardOrder[i]].begin;
 			endKeyToCheck = ranges[shardOrder[endPoint - 1]].end;

From e89cf52f6c9a96985107492bb06029495f8f9809 Mon Sep 17 00:00:00 2001
From: FoundationDB CI <foundationdb_ci@apple.com>
Date: Sat, 9 Dec 2023 00:59:28 +0000
Subject: [PATCH 311/468] disable AVX for 7.1.44 release

---
 cmake/ConfigureCompiler.cmake | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cmake/ConfigureCompiler.cmake b/cmake/ConfigureCompiler.cmake
index e52d8665ee8..7e50469ba6a 100644
--- a/cmake/ConfigureCompiler.cmake
+++ b/cmake/ConfigureCompiler.cmake
@@ -253,7 +253,7 @@ else()
       set(USE_AVX512F OFF)
     endif()
   endif()
-  set(USE_AVX ON CACHE BOOL "Enable AVX instructions")
+  set(USE_AVX OFF CACHE BOOL "Enable AVX instructions")
   if (USE_AVX)
     if (CMAKE_HOST_SYSTEM_PROCESSOR MATCHES "^x86")
       add_compile_options(-mavx)

From 936b70f1553c1d03eef9957d7c80e7fb3d8d4dd4 Mon Sep 17 00:00:00 2001
From: FoundationDB CI <foundationdb_ci@apple.com>
Date: Sat, 9 Dec 2023 00:59:42 +0000
Subject: [PATCH 312/468] enable AVX and update version for 7.1.45 release

---
 CMakeLists.txt                | 2 +-
 cmake/ConfigureCompiler.cmake | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index f7842a3ec68..a8008f3dfb3 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -23,7 +23,7 @@ else()
 endif()
 
 project(foundationdb
-  VERSION 7.1.44
+  VERSION 7.1.45
   DESCRIPTION "FoundationDB is a scalable, fault-tolerant, ordered key-value store with full ACID transactions."
   HOMEPAGE_URL "http://www.foundationdb.org/"
   LANGUAGES C CXX ASM)
diff --git a/cmake/ConfigureCompiler.cmake b/cmake/ConfigureCompiler.cmake
index 7e50469ba6a..e52d8665ee8 100644
--- a/cmake/ConfigureCompiler.cmake
+++ b/cmake/ConfigureCompiler.cmake
@@ -253,7 +253,7 @@ else()
       set(USE_AVX512F OFF)
     endif()
   endif()
-  set(USE_AVX OFF CACHE BOOL "Enable AVX instructions")
+  set(USE_AVX ON CACHE BOOL "Enable AVX instructions")
   if (USE_AVX)
     if (CMAKE_HOST_SYSTEM_PROCESSOR MATCHES "^x86")
       add_compile_options(-mavx)

From bbf5b001994ca7d323ec2cf912be166fab1b53f9 Mon Sep 17 00:00:00 2001
From: He Liu <heliu05023@gmail.com>
Date: Sat, 9 Dec 2023 11:19:46 -0800
Subject: [PATCH 313/468] Added logs for invalid mutations.

---
 fdbserver/storageserver.actor.cpp | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/fdbserver/storageserver.actor.cpp b/fdbserver/storageserver.actor.cpp
index 9ab6733399c..9daf5aa99a6 100644
--- a/fdbserver/storageserver.actor.cpp
+++ b/fdbserver/storageserver.actor.cpp
@@ -4655,7 +4655,7 @@ Optional<MutationRef> clipMutation(MutationRef const& m, KeyRangeRef range) {
 	return Optional<MutationRef>();
 }
 
-bool convertAtomicOp(MutationRef& m, StorageServer::VersionedData const& data, UpdateEagerReadInfo* eager, Arena& ar) {
+bool convertAtomicOp(MutationRef& m, StorageServer::VersionedData const& data, UpdateEagerReadInfo* eager, Arena& ar, const UID& logId) {
 	// After this function call, m should be copied into an arena immediately (before modifying data, shards, or eager)
 	if (m.type != MutationRef::ClearRange && m.type != MutationRef::SetValue) {
 		Optional<StringRef> oldVal;
@@ -4711,6 +4711,9 @@ bool convertAtomicOp(MutationRef& m, StorageServer::VersionedData const& data, U
 				return true;
 			}
 			return false;
+		default:
+			TraceEvent(SevWarnAlways, "InvalidMutationType", logId).detail("Mutation", m);
+			return false;
 		}
 		m.type = MutationRef::SetValue;
 	}
@@ -6583,7 +6586,7 @@ void StorageServer::addMutation(Version version,
 	    nonExpanded; // need to keep non-expanded but atomic converted version of clear mutations for change feeds
 	auto& mLog = addVersionToMutationLog(version);
 
-	if (!convertAtomicOp(expanded, data(), eagerReads, mLog.arena())) {
+	if (!convertAtomicOp(expanded, data(), eagerReads, mLog.arena(), this->thisServerID)) {
 		return;
 	}
 	if (expanded.type == MutationRef::ClearRange) {

From 784eebbaee316a9be0b2a2872a4abe24ebb36d30 Mon Sep 17 00:00:00 2001
From: FoundationDB CI <foundationdb_ci@apple.com>
Date: Sat, 9 Dec 2023 19:45:46 +0000
Subject: [PATCH 314/468] update version after 7.1.45 release

---
 CMakeLists.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index a8008f3dfb3..994e9664fb9 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -23,7 +23,7 @@ else()
 endif()
 
 project(foundationdb
-  VERSION 7.1.45
+  VERSION 7.1.46
   DESCRIPTION "FoundationDB is a scalable, fault-tolerant, ordered key-value store with full ACID transactions."
   HOMEPAGE_URL "http://www.foundationdb.org/"
   LANGUAGES C CXX ASM)

From 6fca89e03fb3374348aabf956a8589f83faec9c8 Mon Sep 17 00:00:00 2001
From: He Liu <heliu05023@gmail.com>
Date: Sat, 9 Dec 2023 12:13:54 -0800
Subject: [PATCH 315/468] fmt.

---
 fdbserver/storageserver.actor.cpp | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/fdbserver/storageserver.actor.cpp b/fdbserver/storageserver.actor.cpp
index 9daf5aa99a6..b5a5bf54f68 100644
--- a/fdbserver/storageserver.actor.cpp
+++ b/fdbserver/storageserver.actor.cpp
@@ -4655,7 +4655,11 @@ Optional<MutationRef> clipMutation(MutationRef const& m, KeyRangeRef range) {
 	return Optional<MutationRef>();
 }
 
-bool convertAtomicOp(MutationRef& m, StorageServer::VersionedData const& data, UpdateEagerReadInfo* eager, Arena& ar, const UID& logId) {
+bool convertAtomicOp(MutationRef& m,
+                     StorageServer::VersionedData const& data,
+                     UpdateEagerReadInfo* eager,
+                     Arena& ar,
+                     const UID& logId) {
 	// After this function call, m should be copied into an arena immediately (before modifying data, shards, or eager)
 	if (m.type != MutationRef::ClearRange && m.type != MutationRef::SetValue) {
 		Optional<StringRef> oldVal;

From 3522ec60caca122e25445fd65495bdd921e4338a Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Sun, 10 Dec 2023 22:19:30 -0800
Subject: [PATCH 316/468] Add key ranges as filters for fdbdecode

In the filter file, now we can use key ranges like "\x01\x05-\x01\x06" as the
filter, in addition to prefix filters.
---
 fdbbackup/FileDecoder.actor.cpp | 147 +++++++++++++++++++++-----------
 1 file changed, 98 insertions(+), 49 deletions(-)

diff --git a/fdbbackup/FileDecoder.actor.cpp b/fdbbackup/FileDecoder.actor.cpp
index be408b90426..d6e58d90a9b 100644
--- a/fdbbackup/FileDecoder.actor.cpp
+++ b/fdbbackup/FileDecoder.actor.cpp
@@ -63,48 +63,49 @@ extern bool g_crashOnError;
 namespace file_converter {
 
 void printDecodeUsage() {
-	std::cout << "Decoder for FoundationDB backup mutation logs.\n"
-	             "Usage: fdbdecode  [OPTIONS]\n"
-	             "  -r, --container URL\n"
-	             "                 Backup container URL, e.g., file:///some/path/.\n"
-	             "  -i, --input    FILE\n"
-	             "                 Log file filter, only matched files are decoded.\n"
-	             "  --log          Enables trace file logging for the CLI session.\n"
-	             "  --logdir PATH  Specifes the output directory for trace files. If\n"
-	             "                 unspecified, defaults to the current directory. Has\n"
-	             "                 no effect unless --log is specified.\n"
-	             "  --loggroup     LOG_GROUP\n"
-	             "                 Sets the LogGroup field with the specified value for all\n"
-	             "                 events in the trace output (defaults to `default').\n"
-	             "  --trace-format FORMAT\n"
-	             "                 Select the format of the trace files, xml (the default) or json.\n"
-	             "                 Has no effect unless --log is specified.\n"
-	             "  --crash        Crash on serious error.\n"
-	             "  --blob-credentials FILE\n"
-	             "                 File containing blob credentials in JSON format.\n"
-	             "                 The same credential format/file fdbbackup uses.\n"
-	             "  -t, --file-type [log|range|both]\n"
-	             "                 Specifies the backup file type to decode.\n"
+	std::cout
+	    << "Decoder for FoundationDB backup mutation logs.\n"
+	       "Usage: fdbdecode  [OPTIONS]\n"
+	       "  -r, --container URL\n"
+	       "                 Backup container URL, e.g., file:///some/path/.\n"
+	       "  -i, --input    FILE\n"
+	       "                 Log file filter, only matched files are decoded.\n"
+	       "  --log          Enables trace file logging for the CLI session.\n"
+	       "  --logdir PATH  Specifes the output directory for trace files. If\n"
+	       "                 unspecified, defaults to the current directory. Has\n"
+	       "                 no effect unless --log is specified.\n"
+	       "  --loggroup     LOG_GROUP\n"
+	       "                 Sets the LogGroup field with the specified value for all\n"
+	       "                 events in the trace output (defaults to `default').\n"
+	       "  --trace-format FORMAT\n"
+	       "                 Select the format of the trace files, xml (the default) or json.\n"
+	       "                 Has no effect unless --log is specified.\n"
+	       "  --crash        Crash on serious error.\n"
+	       "  --blob-credentials FILE\n"
+	       "                 File containing blob credentials in JSON format.\n"
+	       "                 The same credential format/file fdbbackup uses.\n"
+	       "  -t, --file-type [log|range|both]\n"
+	       "                 Specifies the backup file type to decode.\n"
 #ifndef TLS_DISABLED
 	    TLS_HELP
 #endif
-	             "  --build-flags  Print build information and exit.\n"
-	             "  --list-only    Print file list and exit.\n"
-	             "  --validate-filters Validate the default RangeMap filtering logic with a slower one.\n"
-	             "  -k KEY_PREFIX  Use a single prefix for filtering mutations.\n"
-	             "  --filters PREFIX_FILTER_FILE\n"
-	             "                 A file containing a list of prefix filters in HEX format separated by \";\",\n"
-	             "                 e.g., \"\\x05\\x01;\\x15\\x2b\"\n"
-	             "  --hex-prefix   HEX_PREFIX\n"
-	             "                 The prefix specified in HEX format, e.g., --hex-prefix \"\\\\x05\\\\x01\".\n"
-	             "  --begin-version-filter BEGIN_VERSION\n"
-	             "                 The version range's begin version (inclusive) for filtering.\n"
-	             "  --end-version-filter END_VERSION\n"
-	             "                 The version range's end version (exclusive) for filtering.\n"
-	             "  --knob-KNOBNAME KNOBVALUE\n"
-	             "                 Changes a knob value. KNOBNAME should be lowercase.\n"
-	             "  -s, --save     Save a copy of downloaded files (default: not saving).\n"
-	             "\n";
+	       "  --build-flags  Print build information and exit.\n"
+	       "  --list-only    Print file list and exit.\n"
+	       "  --validate-filters Validate the default RangeMap filtering logic with a slower one.\n"
+	       "  -k KEY_PREFIX  Use a single prefix for filtering mutations.\n"
+	       "  --filters PREFIX_FILTER_FILE\n"
+	       "                 A file containing a list of prefix or range filters in HEX format separated by \";\",\n"
+	       "                 e.g., \"\\x05\\x01;\\x15\\x2b;\\x01\\x05-\\x01\\x06\"\n"
+	       "  --hex-prefix   HEX_PREFIX\n"
+	       "                 The prefix specified in HEX format, e.g., --hex-prefix \"\\\\x05\\\\x01\".\n"
+	       "  --begin-version-filter BEGIN_VERSION\n"
+	       "                 The version range's begin version (inclusive) for filtering.\n"
+	       "  --end-version-filter END_VERSION\n"
+	       "                 The version range's end version (exclusive) for filtering.\n"
+	       "  --knob-KNOBNAME KNOBVALUE\n"
+	       "                 Changes a knob value. KNOBNAME should be lowercase.\n"
+	       "  -s, --save     Save a copy of downloaded files (default: not saving).\n"
+	       "\n";
 	return;
 }
 
@@ -125,6 +126,7 @@ struct DecodeParams : public ReferenceCounted<DecodeParams> {
 	bool save_file_locally = false;
 	bool validate_filters = false;
 	std::vector<std::string> prefixes; // Key prefixes for filtering
+	std::vector<KeyRange> ranges; // Key ranges for filtering
 	// more efficient data structure for intersection queries than "prefixes"
 	fileBackup::RangeMapFilters filters;
 	Version beginVersionFilter = 0;
@@ -140,7 +142,12 @@ struct DecodeParams : public ReferenceCounted<DecodeParams> {
 
 	bool overlap(Version version) const { return version >= beginVersionFilter && version < endVersionFilter; }
 
-	void updateRangeMap() { filters.updateFilters(prefixes); }
+	void updateRangeMap() {
+		if (!ranges.empty()) {
+			filters = fileBackup::RangeMapFilters(ranges);
+		}
+		filters.updateFilters(prefixes);
+	}
 
 	bool matchFilters(const MutationRef& m) const {
 		bool match = filters.match(m);
@@ -166,6 +173,22 @@ struct DecodeParams : public ReferenceCounted<DecodeParams> {
 				ASSERT(false);
 			}
 		}
+		for (const auto& range : ranges) {
+			if (isSingleKeyMutation((MutationRef::Type)m.type)) {
+				if (range.contains(m.param1)) {
+					ASSERT(match);
+					return true;
+				}
+			} else if (m.type == MutationRef::ClearRange) {
+				KeyRange range2(KeyRangeRef(m.param1, m.param2));
+				if (range.intersects(range2)) {
+					ASSERT(match);
+					return true;
+				}
+			} else {
+				ASSERT(false);
+			}
+		}
 		ASSERT(!match);
 		return false;
 	}
@@ -234,6 +257,9 @@ struct DecodeParams : public ReferenceCounted<DecodeParams> {
 		if (!prefixes.empty()) {
 			s.append(", KeyPrefixes: ").append(printable(describe(prefixes)));
 		}
+		if (!ranges.empty()) {
+			s.append(", KeyRanges: ").append(printable(describe(ranges)));
+		}
 		for (const auto& [knob, value] : knobs) {
 			s.append(", KNOB-").append(knob).append(" = ").append(value);
 		}
@@ -304,9 +330,12 @@ std::string decode_hex_string(std::string line, bool& err) {
 }
 
 // Parses and returns a ";" separated HEX encoded strings. So the ";" in
-// the string should be escaped as "\;".
+// the string should be escaped as "\x3b". Filters can also be specified
+// as a "begin-end" range, where "begin" and "end" are HEX encoded strings,
+// and will be inserted into the argument "ranges".
+// So the "-" in the range should be escaped as "\x2d".
 // Sets "err" to true if there is any parsing error.
-std::vector<std::string> parsePrefixesLine(const std::string& line, bool& err) {
+std::vector<std::string> parsePrefixesLine(const std::string& line, bool& err, std::vector<KeyRange>& ranges) {
 	std::vector<std::string> results;
 	err = false;
 
@@ -316,19 +345,39 @@ std::vector<std::string> parsePrefixesLine(const std::string& line, bool& err) {
 		if (end == line.npos) {
 			end = line.size();
 		}
-		auto prefix = decode_hex_string(line.substr(p, end - p), err);
-		if (err) {
-			return results;
+		auto current = line.substr(p, end - p);
+		int dash = current.find_first_of('-');
+		if (dash != current.npos) {
+			// found a range
+			auto begin = current.substr(0, dash);
+			auto end = current.substr(dash + 1);
+			begin = decode_hex_string(begin, err);
+			if (err) {
+				std::cerr << "ERROR: parsing range begin failed. Range is: " << current << "\n";
+				return results;
+			}
+			end = decode_hex_string(end, err);
+			if (err) {
+				std::cerr << "ERROR: parsing range end failed. Range is: " << current << "\n";
+				return results;
+			}
+			ranges.emplace_back(KeyRangeRef(begin, end));
+		} else {
+			// found a single prefix
+			auto prefix = decode_hex_string(current, err);
+			if (err) {
+				return results;
+			}
+			results.push_back(prefix);
 		}
-		results.push_back(prefix);
 		p = end + 1;
 	}
 	return results;
 }
 
-std::vector<std::string> parsePrefixFile(const std::string& filename, bool& err) {
+std::vector<std::string> parsePrefixFile(const std::string& filename, bool& err, std::vector<KeyRange>& ranges) {
 	std::string line = readFileBytes(filename, 64 * 1024 * 1024);
-	return parsePrefixesLine(line, err);
+	return parsePrefixesLine(line, err, ranges);
 }
 
 int parseDecodeCommandLine(Reference<DecodeParams> param, CSimpleOpt* args) {
@@ -381,7 +430,7 @@ int parseDecodeCommandLine(Reference<DecodeParams> param, CSimpleOpt* args) {
 			break;
 
 		case OPT_FILTERS:
-			param->prefixes = parsePrefixFile(args->OptionArg(), err);
+			param->prefixes = parsePrefixFile(args->OptionArg(), err, param->ranges);
 			if (err) {
 				throw std::runtime_error("ERROR:" + std::string(args->OptionArg()) + "contains invalid prefix(es)");
 			}

From d83b03786273767e35f691b395b0bfae115eb972 Mon Sep 17 00:00:00 2001
From: Zhe Wang <zhe.wang@wustl.edu>
Date: Sun, 10 Dec 2023 23:35:44 -0800
Subject: [PATCH 317/468] retriable consistency checker

---
 fdbclient/ClientKnobs.cpp                     |   4 +-
 fdbclient/ClientKnobs.h                       |   4 +-
 fdbserver/tester.actor.cpp                    |   7 +-
 .../workloads/ConsistencyCheck.actor.cpp      | 380 +++++++++++++-----
 4 files changed, 292 insertions(+), 103 deletions(-)

diff --git a/fdbclient/ClientKnobs.cpp b/fdbclient/ClientKnobs.cpp
index b2535cfdf66..030a6acd5cf 100644
--- a/fdbclient/ClientKnobs.cpp
+++ b/fdbclient/ClientKnobs.cpp
@@ -258,9 +258,11 @@ void ClientKnobs::initialize(Randomize randomize) {
 
 	init( CONSISTENCY_CHECK_RATE_LIMIT_MAX,        50e6 ); // Limit in per sec
 	init( CONSISTENCY_CHECK_ONE_ROUND_TARGET_COMPLETION_TIME,	7 * 24 * 60 * 60 ); // 7 days
-	init( CONSISTENCY_CHECK_CUSTOM_RANGE,          true );
 	init( CONSISTENCY_CHECK_DISTRIBUTED,           true );
+	init( CONSISTENCY_CHECK_AGGRESSIVE_RUN,        true );
+	init( CONSISTENCY_CHECK_COMPLETE_CHECK,        true ); // Take effect only when CONSISTENCY_CHECK_DISTRIBUTED = true
 	init( CONSISTENCY_CHECK_DISTRIBUTED_WIGGLE_ROOM, 50 );
+	init( CONSISTENCY_CHECK_RETRY_DEPTH_MAX,        100 );
 	init( CONSISTENCY_CHECK_RANGE_BEGIN,             "" );
 	init( CONSISTENCY_CHECK_RANGE_END,       "\xff\xff" );
 
diff --git a/fdbclient/ClientKnobs.h b/fdbclient/ClientKnobs.h
index 811d2401349..7aaae7ad867 100644
--- a/fdbclient/ClientKnobs.h
+++ b/fdbclient/ClientKnobs.h
@@ -249,8 +249,10 @@ class ClientKnobs : public KnobsImpl<ClientKnobs> {
 
 	int CONSISTENCY_CHECK_RATE_LIMIT_MAX;
 	int CONSISTENCY_CHECK_ONE_ROUND_TARGET_COMPLETION_TIME;
-	bool CONSISTENCY_CHECK_CUSTOM_RANGE;
 	bool CONSISTENCY_CHECK_DISTRIBUTED;
+	bool CONSISTENCY_CHECK_AGGRESSIVE_RUN;
+	bool CONSISTENCY_CHECK_COMPLETE_CHECK;
+	int CONSISTENCY_CHECK_RETRY_DEPTH_MAX;
 	int CONSISTENCY_CHECK_DISTRIBUTED_WIGGLE_ROOM;
 	std::string CONSISTENCY_CHECK_RANGE_BEGIN;
 	std::string CONSISTENCY_CHECK_RANGE_END;
diff --git a/fdbserver/tester.actor.cpp b/fdbserver/tester.actor.cpp
index 0178c29f0d7..61d7bbf2821 100644
--- a/fdbserver/tester.actor.cpp
+++ b/fdbserver/tester.actor.cpp
@@ -592,6 +592,9 @@ ACTOR Future<Void> runWorkloadAsync(Database cx,
 					TraceEvent("TestChecking", workIface.id()).detail("Workload", workload->description());
 					bool check = wait(timeoutError(workload->check(cx), workload->getCheckTimeout()));
 					checkResult = CheckReply{ (!startResult.present() || !startResult.get().isError()) && check };
+					TraceEvent("TestChecked", workIface.id())
+					    .detail("Workload", workload->description())
+					    .detail("Result", (!startResult.present() || !startResult.get().isError()) && check);
 				} catch (Error& e) {
 					checkResult = operation_failed(); // was: checkResult = false;
 					if (e.code() == error_code_please_reboot || e.code() == error_code_please_reboot_delete)
@@ -960,11 +963,13 @@ ACTOR Future<Void> checkConsistency(Database cx,
 	state double start = now();
 	state bool lastRun = false;
 	loop {
+		TraceEvent("ConsistencyCheckWorkLoadLoopBegin");
 		DistributedTestResults testResults = wait(runWorkload(cx, testers, spec, Optional<TenantName>()));
 		if (testResults.ok() || lastRun) {
 			if (g_network->isSimulated()) {
 				g_simulator.connectionFailuresDisableDuration = connectionFailures;
 			}
+			TraceEvent("ConsistencyCheckWorkLoadEnd");
 			return Void();
 		}
 		if (now() - start > softTimeLimit) {
@@ -972,7 +977,7 @@ ACTOR Future<Void> checkConsistency(Database cx,
 			                               KeyValueRef(LiteralStringRef("failureIsError"), LiteralStringRef("true")));
 			lastRun = true;
 		}
-
+		TraceEvent("ConsistencyCheckWorkLoadRepairDC");
 		wait(repairDeadDatacenter(cx, dbInfo, "ConsistencyCheck"));
 	}
 }
diff --git a/fdbserver/workloads/ConsistencyCheck.actor.cpp b/fdbserver/workloads/ConsistencyCheck.actor.cpp
index 3e7b5a7d849..38ecf91b4ce 100644
--- a/fdbserver/workloads/ConsistencyCheck.actor.cpp
+++ b/fdbserver/workloads/ConsistencyCheck.actor.cpp
@@ -190,17 +190,28 @@ struct ConsistencyCheckWorkload : TestWorkload {
 				TraceEvent("ConsistencyCheck_Suspended")
 				    .detail("Distributed", self->distributed)
 				    .detail("ClientCount", self->clientCount)
-				    .detail("ClientId", self->clientId);
+				    .detail("ClientId", self->clientId)
+				    .detail("Indefinite", self->indefinite)
+				    .detail("Repetitions", self->repetitions);
 				wait(self->suspendConsistencyCheck.onChange());
 			}
 			TraceEvent("ConsistencyCheck_StartingOrResuming")
 			    .detail("Distributed", self->distributed)
 			    .detail("ClientCount", self->clientCount)
-			    .detail("ClientId", self->clientId);
+			    .detail("ClientId", self->clientId)
+			    .detail("Indefinite", self->indefinite)
+			    .detail("Repetitions", self->repetitions);
 			choose {
 				when(wait(self->runCheck(cx, self))) {
-					if (!self->indefinite)
+					if (!self->indefinite) {
+						TraceEvent("ConsistencyCheck_Exit")
+						    .detail("Distributed", self->distributed)
+						    .detail("ClientCount", self->clientCount)
+						    .detail("ClientId", self->clientId)
+						    .detail("Indefinite", self->indefinite)
+						    .detail("Repetitions", self->repetitions);
 						break;
+					}
 					self->repetitions++;
 					wait(delay(5.0));
 				}
@@ -323,12 +334,8 @@ struct ConsistencyCheckWorkload : TestWorkload {
 						self->testFailure("Coordinators incorrect");
 				}
 
-				// Get a list of key servers; verify that the TLogs and master all agree about who the key servers are
-				state Promise<std::vector<std::pair<KeyRange, std::vector<StorageServerInterface>>>> keyServerPromise;
-				// bool keyServerResult = wait(self->getKeyServers(cx, self, keyServerPromise, keyServersKeys));
-
-				KeyRange rangeToCheck = allKeys;
-				if (CLIENT_KNOBS->CONSISTENCY_CHECK_CUSTOM_RANGE) {
+				if (CLIENT_KNOBS->CONSISTENCY_CHECK_DISTRIBUTED) {
+					state KeyRange rangeToCheck = allKeys;
 					Key rangeBegin = StringRef(CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_BEGIN);
 					if (rangeBegin > allKeys.end) {
 						rangeBegin = allKeys.end;
@@ -344,34 +351,61 @@ struct ConsistencyCheckWorkload : TestWorkload {
 					} else {
 						rangeToCheck = allKeys;
 					}
-				}
+					TraceEvent("ConsistencyCheck_Config")
+					    .setMaxEventLength(-1)
+					    .setMaxFieldLength(-1)
+					    .detail("RangeToCheck", rangeToCheck)
+					    .detail("Indefinite", self->indefinite)
+					    .detail("Distributed", self->distributed)
+					    .detail("ClientCount", self->clientCount)
+					    .detail("ClientId", self->clientId)
+					    .detail("PerformQuiescentChecks", self->performQuiescentChecks)
+					    .detail("PerformTSSCheck", self->performTSSCheck)
+					    .detail("PerformCacheCheck", self->performCacheCheck);
+					state std::vector<std::pair<KeyRange, Value>> shardLocationPairListForDistributed =
+					    wait(self->getKeyLocationsForRangeList(cx, { allKeys }, self));
+					// Check that each failed shard has the same data on all storage servers that it resides on
+					wait(::success(self->checkDataConsistency(cx,
+					                                          shardLocationPairListForDistributed,
+					                                          configuration,
+					                                          tssMapping,
+					                                          self,
+					                                          /*consistencyCheckEpoch=*/0)));
 
-				TraceEvent("ConsistencyCheck_Config")
-				    .setMaxEventLength(-1)
-				    .setMaxFieldLength(-1)
-				    .detail("RangeToCheck", rangeToCheck)
-				    .detail("Distributed", self->distributed)
-				    .detail("ClientCount", self->clientCount)
-				    .detail("ClientId", self->clientId);
-
-				bool keyServerResult =
-				    wait(self->getKeyServers(cx, self, keyServerPromise, rangeToCheck.withPrefix(keyServersPrefix)));
-				if (keyServerResult) {
-					state std::vector<std::pair<KeyRange, std::vector<StorageServerInterface>>> keyServers =
-					    keyServerPromise.getFuture().get();
-
-					// Get the locations of all the shards in the database
-					state Promise<Standalone<VectorRef<KeyValueRef>>> keyLocationPromise;
-					bool keyLocationResult = wait(self->getKeyLocations(cx, keyServers, self, keyLocationPromise));
-					if (keyLocationResult) {
-						state Standalone<VectorRef<KeyValueRef>> keyLocations = keyLocationPromise.getFuture().get();
-
-						// Check that each shard has the same data on all storage servers that it resides on
-						wait(::success(self->checkDataConsistency(cx, keyLocations, configuration, tssMapping, self)));
-
-						// Cache consistency check
-						if (self->performCacheCheck)
-							wait(::success(self->checkCacheConsistency(cx, keyLocations, self)));
+				} else {
+					// Get a list of key servers; verify that the TLogs and master all agree about who the key servers
+					// are
+					state Promise<std::vector<std::pair<KeyRange, std::vector<StorageServerInterface>>>>
+					    keyServerPromise;
+					bool keyServerResult = wait(self->getKeyServers(cx, self, keyServerPromise, keyServersKeys));
+					if (keyServerResult) {
+						state std::vector<std::pair<KeyRange, std::vector<StorageServerInterface>>> keyServers =
+						    keyServerPromise.getFuture().get();
+
+						// Get the locations of all the shards in the database
+						state Promise<Standalone<VectorRef<KeyValueRef>>> keyLocationPromise;
+						bool keyLocationResult = wait(self->getKeyLocations(cx, keyServers, self, keyLocationPromise));
+						if (keyLocationResult) {
+							state Standalone<VectorRef<KeyValueRef>> keyLocations =
+							    keyLocationPromise.getFuture().get();
+							state std::vector<std::pair<KeyRange, Value>> shardLocationPairList;
+							// Check that each shard has the same data on all storage servers that it resides on
+							for (int k = 0; k < keyLocations.size() - 1; k++) {
+								shardLocationPairList.push_back(std::make_pair(
+								    Standalone(KeyRangeRef(keyLocations[k].key, keyLocations[k + 1].key)),
+								    Standalone(keyLocations[k].value)));
+							}
+							wait(::success(self->checkDataConsistency(cx,
+							                                          shardLocationPairList,
+							                                          configuration,
+							                                          tssMapping,
+							                                          self,
+							                                          /*consistencyCheckEpoch=*/0)));
+
+							// Cache consistency check
+							if (self->performCacheCheck)
+								wait(::success(self->checkCacheConsistency(cx, keyLocations, self)));
+						}
 					}
 				}
 			} catch (Error& e) {
@@ -379,13 +413,23 @@ struct ConsistencyCheckWorkload : TestWorkload {
 				    e.code() == error_code_wrong_shard_server || e.code() == error_code_all_alternatives_failed ||
 				    e.code() == error_code_process_behind)
 					TraceEvent("ConsistencyCheck_Retry")
-					    .error(e); // FIXME: consistency check does not retry in this case
+					    .error(e) // FIXME: consistency check does not retry in this case
+					    .detail("Distributed", self->distributed)
+					    .detail("ClientCount", self->clientCount)
+					    .detail("ClientId", self->clientId)
+					    .detail("Indefinite", self->indefinite)
+					    .detail("Repetitions", self->repetitions);
 				else
 					self->testFailure(format("Error %d - %s", e.code(), e.name()));
 			}
 		}
 
-		TraceEvent("ConsistencyCheck_FinishedCheck").detail("Repetitions", self->repetitions);
+		TraceEvent("ConsistencyCheck_FinishedCheck")
+		    .detail("Distributed", self->distributed)
+		    .detail("ClientCount", self->clientCount)
+		    .detail("ClientId", self->clientId)
+		    .detail("Indefinite", self->indefinite)
+		    .detail("Repetitions", self->repetitions);
 
 		return Void();
 	}
@@ -908,7 +952,7 @@ struct ConsistencyCheckWorkload : TestWorkload {
 	    Database cx,
 	    ConsistencyCheckWorkload* self,
 	    Promise<std::vector<std::pair<KeyRange, std::vector<StorageServerInterface>>>> keyServersPromise,
-	    KeyRangeRef kr) {
+	    KeyRange kr) {
 		state std::vector<std::pair<KeyRange, std::vector<StorageServerInterface>>> keyServers;
 
 		// Try getting key server locations from the master proxies
@@ -1091,6 +1135,42 @@ struct ConsistencyCheckWorkload : TestWorkload {
 		return true;
 	}
 
+	ACTOR Future<std::vector<std::pair<KeyRange, Value>>> getKeyLocationsForRangeList(Database cx,
+	                                                                                  std::vector<KeyRange> ranges,
+																					  ConsistencyCheckWorkload* self) {
+		state std::vector<std::pair<KeyRange, Value>> res;
+		state Transaction tr(cx);
+		state int idx = 0;
+		for (; idx < ranges.size(); ++idx) {
+			state Key beginKey = ranges[idx].begin;
+			state Key endKey = ranges[idx].end;
+			loop {
+				try {
+					tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
+					tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
+					state KeyRange rangeToRead = Standalone(KeyRangeRef(beginKey, endKey));
+					RangeResult readResult = wait(krmGetRanges(&tr,
+					                                           keyServersPrefix,
+					                                           rangeToRead,
+					                                           CLIENT_KNOBS->TOO_MANY,
+					                                           GetRangeLimits::BYTE_LIMIT_UNLIMITED));
+					for (int i = 0; i < readResult.size() - 1; ++i) {
+						res.push_back(std::make_pair(Standalone(KeyRangeRef(readResult[i].key, readResult[i + 1].key)),
+						                             Standalone(readResult[i].value)));
+						beginKey = readResult[i + 1].key;
+					}
+					if (beginKey >= ranges[idx].end) {
+						break;
+					}
+				} catch (Error& e) {
+					TraceEvent("ConsistencyCheck_RetryGetKeyLocationsForRangeList").error(e);
+					wait(tr.onError(e));
+				}
+			}
+		}
+		return res;
+	}
+
 	// Retrieves a vector of the storage servers' estimates for the size of a particular shard
 	// If a storage server can't be reached, its estimate will be -1
 	// If there is an error, then the returned vector will have 0 size
@@ -1221,11 +1301,16 @@ struct ConsistencyCheckWorkload : TestWorkload {
 
 	// Checks that the data in each shard is the same on each storage server that it resides on.  Also performs some
 	// sanity checks on the sizes of shards and storage servers. Returns false if there is a failure
+	// We create a new checkDataConsistency actor for retrying for failed ranges and we do consistencyCheckEpoch++
 	ACTOR Future<bool> checkDataConsistency(Database cx,
-	                                        VectorRef<KeyValueRef> keyLocations,
+	                                        std::vector<std::pair<KeyRange, Value>> shardLocationPairList,
 	                                        DatabaseConfiguration configuration,
 	                                        std::map<UID, StorageServerInterface> tssMapping,
-	                                        ConsistencyCheckWorkload* self) {
+	                                        ConsistencyCheckWorkload* self,
+	                                        int consistencyCheckEpoch) {
+		if (consistencyCheckEpoch > 0) {
+			ASSERT(CLIENT_KNOBS->CONSISTENCY_CHECK_DISTRIBUTED && CLIENT_KNOBS->CONSISTENCY_CHECK_COMPLETE_CHECK);
+		}
 		// Stores the total number of bytes on each storage server
 		// In a distributed test, this will be an estimated size
 		state std::map<UID, int64_t> storageServerSizes;
@@ -1245,14 +1330,24 @@ struct ConsistencyCheckWorkload : TestWorkload {
 		              self->rateLimitMax,
 		              static_cast<int>(ceil(self->bytesReadInPreviousRound /
 		                                    (float)CLIENT_KNOBS->CONSISTENCY_CHECK_ONE_ROUND_TARGET_COMPLETION_TIME)));
+		if (consistencyCheckEpoch > 0 ||
+		    CLIENT_KNOBS->CONSISTENCY_CHECK_AGGRESSIVE_RUN) { // We set max speed when retrying failed ranges or
+			                                                  // aggressive check is on
+			rateLimitForThisRound = self->rateLimitMax;
+		}
+		TraceEvent("ConsistencyCheck_RateLimitForThisRound")
+		    .detail("RateLimit", rateLimitForThisRound)
+		    .detail("ConsistencyCheckEpoch", consistencyCheckEpoch)
+		    .detail("ShardLocationPairList", shardLocationPairList.size());
 		ASSERT(rateLimitForThisRound >= 0 && rateLimitForThisRound <= self->rateLimitMax);
-		TraceEvent("ConsistencyCheck_RateLimitForThisRound").detail("RateLimit", rateLimitForThisRound);
 		state Reference<IRateControl> rateLimiter = Reference<IRateControl>(new SpeedLimit(rateLimitForThisRound, 1));
 		state double rateLimiterStartTime = now();
 		state int64_t bytesReadInthisRound = 0;
 		state bool testResult = true;
 		state int64_t numCompleteShards = 0;
 		state int64_t numSkippedShards = 0;
+		state KeyRangeMap<bool> failedRanges; // Used to collect failed ranges in the current checkDataConsistency
+		// Which will be used to start the next consistencyCheckEpoch of the checkDataConsistency
 
 		state double dbSize = 100e12;
 		if (g_network->isSimulated()) {
@@ -1261,11 +1356,9 @@ struct ConsistencyCheckWorkload : TestWorkload {
 			dbSize = _dbSize;
 		}
 
-		state std::vector<KeyRangeRef> ranges;
-
-		for (int k = 0; k < keyLocations.size() - 1; k++) {
-			KeyRangeRef range(keyLocations[k].key, keyLocations[k + 1].key);
-			ranges.push_back(range);
+		state std::vector<KeyRange> ranges;
+		for (int k = 0; k < shardLocationPairList.size(); k++) {
+			ranges.push_back(shardLocationPairList[k].first); // Add shard range to ranges
 		}
 
 		state Key beginKeyToCheck = ranges[0].begin;
@@ -1283,32 +1376,40 @@ struct ConsistencyCheckWorkload : TestWorkload {
 
 		state int endPoint;
 		if (CLIENT_KNOBS->CONSISTENCY_CHECK_DISTRIBUTED) {
-			int batchSize = (shardOrder.size() / self->clientCount) + 1;
-			i = self->clientId * batchSize; // overwrite the starting point i
-			i = std::max(0, i - CLIENT_KNOBS->CONSISTENCY_CHECK_DISTRIBUTED_WIGGLE_ROOM);
-			endPoint = (self->clientId + 1) * batchSize;
-			endPoint = std::min(static_cast<int>(shardOrder.size()),
-			                    endPoint + CLIENT_KNOBS->CONSISTENCY_CHECK_DISTRIBUTED_WIGGLE_ROOM);
-			increment = 1;
-			// If the start point i is larger than the list boundary, this client needs not run
-			if (i >= endPoint) {
-				TraceEvent("ConsistencyCheck_NoTaskToRun")
-				    .detail("Distributed", self->distributed)
-				    .detail("ShardCount", ranges.size())
-				    .detail("ClientId", self->clientId)
-				    .detail("ClientCount", self->clientCount)
-				    .detail("Repetitions", self->repetitions);
-				wait(delay(600.0) || self->suspendConsistencyCheck.onChange());
-				return true;
+			if (consistencyCheckEpoch == 0) {
+				int batchSize = (shardOrder.size() / self->clientCount) + 1;
+				i = self->clientId * batchSize; // overwrite the starting point i
+				i = std::max(0, i - CLIENT_KNOBS->CONSISTENCY_CHECK_DISTRIBUTED_WIGGLE_ROOM);
+				endPoint = (self->clientId + 1) * batchSize;
+				endPoint = std::min(static_cast<int>(shardOrder.size()),
+				                    endPoint + CLIENT_KNOBS->CONSISTENCY_CHECK_DISTRIBUTED_WIGGLE_ROOM);
+				increment = 1;
+				// If the start point i is larger than the list boundary, this client needs not run
+				if (i >= endPoint) {
+					TraceEvent("ConsistencyCheck_NoTaskToRun")
+					    .detail("Distributed", self->distributed)
+					    .detail("ShardCount", ranges.size())
+					    .detail("ClientId", self->clientId)
+					    .detail("ClientCount", self->clientCount)
+					    .detail("Repetitions", self->repetitions);
+					wait(delay(600.0) || self->suspendConsistencyCheck.onChange());
+					return true;
+				}
+				// overwrite beginKeyToCheck and endKeyToCheck
+				beginKeyToCheck = ranges[shardOrder[i]].begin;
+				endKeyToCheck = ranges[shardOrder[endPoint - 1]].end;
+			} else { // consistencyCheckEpoch > 0 when handling failure ranges
+				i = 0;
+				increment = 1;
+				endPoint = shardOrder.size();
+				beginKeyToCheck = Key();
+				endKeyToCheck = Key();
 			}
-			// overwrite beginKeyToCheck and endKeyToCheck
-			beginKeyToCheck = ranges[shardOrder[i]].begin;
-			endKeyToCheck = ranges[shardOrder[endPoint - 1]].end;
 		} else {
 			endPoint = shardOrder.size();
 		}
 
-		TraceEvent("ConsistencyCheck_StartTask")
+		TraceEvent(consistencyCheckEpoch == 0 ? "ConsistencyCheck_StartTask" : "ConsistencyCheck_StartFailedTask")
 		    .setMaxEventLength(-1)
 		    .setMaxFieldLength(-1)
 		    .detail("Distributed", self->distributed)
@@ -1320,7 +1421,9 @@ struct ConsistencyCheckWorkload : TestWorkload {
 		    .detail("BeginKey", beginKeyToCheck)
 		    .detail("EndKey", endKeyToCheck)
 		    .detail("Repetitions", self->repetitions)
-		    .trackLatest("ConsistencyCheck_StartTask");
+		    .detail("ConsistencyCheckEpoch", consistencyCheckEpoch)
+		    .trackLatest(consistencyCheckEpoch == 0 ? "ConsistencyCheck_StartTask"
+		                                            : "ConsistencyCheck_StartFailedTask");
 
 		for (; i < endPoint; i += increment) {
 			if (CLIENT_KNOBS->CONSISTENCY_CHECK_DISTRIBUTED && self->suspendConsistencyCheck.get()) {
@@ -1349,8 +1452,9 @@ struct ConsistencyCheckWorkload : TestWorkload {
 
 			RangeResult UIDtoTagMap = wait(tr.getRange(serverTagKeys, CLIENT_KNOBS->TOO_MANY));
 			ASSERT(!UIDtoTagMap.more && UIDtoTagMap.size() < CLIENT_KNOBS->TOO_MANY);
+			// Decode shard location value
 			decodeKeyServersValue(
-			    UIDtoTagMap, keyLocations[shard].value, sourceStorageServers, destStorageServers, false);
+			    UIDtoTagMap, shardLocationPairList[shard].second, sourceStorageServers, destStorageServers, false);
 
 			// If the destStorageServers is non-empty, then this shard is being relocated
 			state bool isRelocating = destStorageServers.size() > 0;
@@ -1497,7 +1601,6 @@ struct ConsistencyCheckWorkload : TestWorkload {
 
 				// Read a limited number of entries at a time, repeating until all keys in the shard have been read
 				loop {
-					completeCheck = true;
 					try {
 						if (CLIENT_KNOBS->CONSISTENCY_CHECK_DISTRIBUTED && self->suspendConsistencyCheck.get()) {
 							TraceEvent("ConsistencyCheck_DataCheckCancelled")
@@ -1699,7 +1802,6 @@ struct ConsistencyCheckWorkload : TestWorkload {
 
 							// If the data is not available and we aren't relocating this shard
 							else if (!isRelocating) {
-								completeCheck = false;
 								Error e =
 								    rangeResult.isError() ? rangeResult.getError() : rangeResult.get().error.get();
 
@@ -1715,15 +1817,33 @@ struct ConsistencyCheckWorkload : TestWorkload {
 								    .detail("UID", storageServerInterfaces[j].id())
 								    .detail("GetKeyValuesToken",
 								            storageServerInterfaces[j].getKeyValues.getEndpoint().token)
-								    .detail("IsTSS", storageServerInterfaces[j].isTss() ? "True" : "False");
+								    .detail("IsTSS", storageServerInterfaces[j].isTss() ? "True" : "False")
+								    .detail("Distributed", self->distributed)
+								    .detail("ClientId", self->clientId)
+								    .detail("ClientCount", self->clientCount)
+								    .detail("Repetitions", self->repetitions)
+								    .detail("ConsistencyCheckEpoch", consistencyCheckEpoch);
+
+								completeCheck = false;
+								if (CLIENT_KNOBS->CONSISTENCY_CHECK_COMPLETE_CHECK) {
+									failedRanges.insert(Standalone(KeyRangeRef(req.begin.getKey(), req.end.getKey())),
+									                    true);
+									break; // Will retry later
+								}
 
 								// All shards should be available in quiscence
 								if (self->performQuiescentChecks && !storageServerInterfaces[j].isTss()) {
 									self->testFailure("Storage server unavailable");
 									return false;
 								}
+
 							} else {
 								completeCheck = false;
+								if (CLIENT_KNOBS->CONSISTENCY_CHECK_COMPLETE_CHECK) {
+									failedRanges.insert(Standalone(KeyRangeRef(req.begin.getKey(), req.end.getKey())),
+									                    true);
+									break; // Will retry later
+								}
 							}
 						}
 
@@ -1804,7 +1924,8 @@ struct ConsistencyCheckWorkload : TestWorkload {
 				}
 
 				if (!completeCheck) {
-					TraceEvent(SevWarnAlways, "ConsistencyCheck_ShardNoCompleteCheck")
+					TraceEvent(CLIENT_KNOBS->CONSISTENCY_CHECK_COMPLETE_CHECK ? SevInfo : SevWarn,
+					           "ConsistencyCheck_ShardFailed")
 					    .setMaxEventLength(-1)
 					    .setMaxFieldLength(-1)
 					    .detail("ClientId", self->clientId)
@@ -1812,7 +1933,26 @@ struct ConsistencyCheckWorkload : TestWorkload {
 					    .detail("ShardBegin", printable(range.begin))
 					    .detail("ShardEnd", printable(range.end))
 					    .detail("Repetitions", self->repetitions)
-					    .trackLatest("ConsistencyCheck_ShardNoCompleteCheck");
+					    .detail("ConsistencyCheckEpoch", consistencyCheckEpoch)
+					    .trackLatest("ConsistencyCheck_ShardCheckFailedRange");
+				} else {
+					numCompleteShards++;
+					TraceEvent(SevInfo, "ConsistencyCheck_ShardComplete")
+					    .suppressFor(1.0)
+					    .setMaxEventLength(-1)
+					    .setMaxFieldLength(-1)
+					    .detail("ClientId", self->clientId)
+					    .detail("ClientCount", self->clientCount)
+					    .detail("Range", range.toString())
+					    .detail("ShardCount", ranges.size())
+					    .detail("NumCompletedShards", numCompleteShards)
+					    .detail("BytesReadInThisRound", bytesReadInthisRound)
+					    .detail("NumSkippedShards", numSkippedShards)
+					    .detail("ConsistencyCheckEpoch", consistencyCheckEpoch);
+				}
+
+				if (CLIENT_KNOBS->CONSISTENCY_CHECK_DISTRIBUTED) {
+					continue; // Bypass following check not-relevant to userdata/metadata consistency
 				}
 
 				canSplit = canSplit && sampledBytes - splitBytes >= shardBounds.min.bytes && sampledBytes > splitBytes;
@@ -1934,32 +2074,57 @@ struct ConsistencyCheckWorkload : TestWorkload {
 				    .detail("Range", range)
 				    .detail("BytesRead", bytesReadInRange);
 			}
-
-			numCompleteShards++;
-			TraceEvent(SevInfo, "ConsistencyCheck_ShardComplete")
-			    .suppressFor(1.0)
-			    .setMaxEventLength(-1)
-			    .setMaxFieldLength(-1)
-			    .detail("ClientId", self->clientId)
-			    .detail("ClientCount", self->clientCount)
-			    .detail("Range", range.toString())
-			    .detail("ShardCount", ranges.size())
-			    .detail("NumCompletedShards", numCompleteShards)
-			    .detail("BytesReadInThisRound", bytesReadInthisRound)
-			    .detail("NumSkippedShards", numSkippedShards);
 		}
 
-		TraceEvent("ConsistencyCheck_EndTask")
-		    .setMaxEventLength(-1)
-		    .setMaxFieldLength(-1)
-		    .detail("Distributed", self->distributed)
-		    .detail("ShardCount", ranges.size())
-		    .detail("ClientId", self->clientId)
-		    .detail("ClientCount", self->clientCount)
-		    .detail("BeginKey", beginKeyToCheck)
-		    .detail("EndKey", endKeyToCheck)
-		    .detail("Repetitions", self->repetitions)
-		    .trackLatest("ConsistencyCheck_EndTask");
+		if (CLIENT_KNOBS->CONSISTENCY_CHECK_DISTRIBUTED && CLIENT_KNOBS->CONSISTENCY_CHECK_COMPLETE_CHECK) {
+			failedRanges.coalesce(allKeys);
+			state std::vector<KeyRange> failedRangesToCheck;
+			state KeyRangeMap<bool>::Ranges failedRangesList = failedRanges.ranges();
+			state KeyRangeMap<bool>::iterator failedRangesIter = failedRangesList.begin();
+			for (; failedRangesIter != failedRangesList.end(); ++failedRangesIter) {
+				if (failedRangesIter->value() == false) {
+					continue; // No failure, so bypass the range
+				}
+				failedRangesToCheck.push_back(failedRangesIter->range());
+			}
+			if (failedRangesToCheck.size() > 0) {
+				wait(delay(60.0)); // Backoff 1 min
+				state std::vector<std::pair<KeyRange, Value>> shardLocationPairListForFailedRanges =
+			    	wait(self->getKeyLocationsForRangeList(cx, failedRangesToCheck, self));
+				TraceEvent(SevInfo, "ConsistencyCheck_StartHandlingFailedRanges")
+					.setMaxEventLength(-1)
+					.setMaxFieldLength(-1)
+					.detail("FailedCollectedRangeCount", failedRangesToCheck.size())
+					.detail("FailedShardCount", shardLocationPairListForFailedRanges.size())
+					.detail("ConsistencyCheckEpoch", consistencyCheckEpoch)
+					.detail("Distributed", self->distributed)
+					.detail("ClientId", self->clientId)
+					.detail("ClientCount", self->clientCount);
+				if (consistencyCheckEpoch < CLIENT_KNOBS->CONSISTENCY_CHECK_RETRY_DEPTH_MAX) {
+					wait(::success(self->checkDataConsistency(cx,
+					                                          shardLocationPairListForFailedRanges,
+					                                          configuration,
+					                                          tssMapping,
+					                                          self,
+					                                          consistencyCheckEpoch + 1)));
+				} else {
+					// We give up retrying when retry too many times
+					for (const auto& missedRange : shardLocationPairListForFailedRanges) {
+						TraceEvent(SevWarnAlways, "ConsistencyCheck_MissingRange")
+						    .setMaxEventLength(-1)
+						    .setMaxFieldLength(-1)
+						    .detail("Reason", "MaxRetryDepthReached")
+						    .detail("ConsistencyCheckEpoch", consistencyCheckEpoch)
+						    .detail("Distributed", self->distributed)
+						    .detail("ShardCount", ranges.size())
+						    .detail("ClientId", self->clientId)
+						    .detail("ClientCount", self->clientCount)
+						    .detail("BeginKey", printable(missedRange.first.begin))
+						    .detail("EndKey", printable(missedRange.first.end));
+					}
+				}
+			} // Otherwise, no failure, we are good to go
+		}
 
 		// SOMEDAY: when background data distribution is implemented, include this test
 		// In a quiescent database, check that the sizes of storage servers are roughly the same
@@ -1983,7 +2148,22 @@ struct ConsistencyCheckWorkload : TestWorkload {
 		    }
 		}*/
 
-		self->bytesReadInPreviousRound = bytesReadInthisRound;
+		if (consistencyCheckEpoch == 0) {
+			TraceEvent("ConsistencyCheck_EndTask")
+			    .setMaxEventLength(-1)
+			    .setMaxFieldLength(-1)
+			    .detail("Distributed", self->distributed)
+			    .detail("ShardCount", ranges.size())
+			    .detail("ClientId", self->clientId)
+			    .detail("ClientCount", self->clientCount)
+			    .detail("BeginKey", beginKeyToCheck)
+			    .detail("EndKey", endKeyToCheck)
+			    .detail("Repetitions", self->repetitions)
+			    .trackLatest("ConsistencyCheck_EndTask");
+			self->bytesReadInPreviousRound =
+			    bytesReadInthisRound; // FIXME: missing bytes read by retrying failed ranges
+		}
+
 		return testResult;
 	}
 

From 33d577ccb68861f9220ba14923f2ebdcbe6d845f Mon Sep 17 00:00:00 2001
From: Zhe Wang <zhe.wang@wustl.edu>
Date: Mon, 11 Dec 2023 00:22:12 -0800
Subject: [PATCH 318/468] repair DC when retry too many times

---
 fdbclient/ClientKnobs.cpp                     |  2 +-
 .../workloads/ConsistencyCheck.actor.cpp      | 24 +++++++++++--------
 2 files changed, 15 insertions(+), 11 deletions(-)

diff --git a/fdbclient/ClientKnobs.cpp b/fdbclient/ClientKnobs.cpp
index 030a6acd5cf..081f9787d11 100644
--- a/fdbclient/ClientKnobs.cpp
+++ b/fdbclient/ClientKnobs.cpp
@@ -262,7 +262,7 @@ void ClientKnobs::initialize(Randomize randomize) {
 	init( CONSISTENCY_CHECK_AGGRESSIVE_RUN,        true );
 	init( CONSISTENCY_CHECK_COMPLETE_CHECK,        true ); // Take effect only when CONSISTENCY_CHECK_DISTRIBUTED = true
 	init( CONSISTENCY_CHECK_DISTRIBUTED_WIGGLE_ROOM, 50 );
-	init( CONSISTENCY_CHECK_RETRY_DEPTH_MAX,        100 );
+	init( CONSISTENCY_CHECK_RETRY_DEPTH_MAX,        100 ); if( randomize && BUGGIFY ) CONSISTENCY_CHECK_RETRY_DEPTH_MAX = 10;
 	init( CONSISTENCY_CHECK_RANGE_BEGIN,             "" );
 	init( CONSISTENCY_CHECK_RANGE_END,       "\xff\xff" );
 
diff --git a/fdbserver/workloads/ConsistencyCheck.actor.cpp b/fdbserver/workloads/ConsistencyCheck.actor.cpp
index 38ecf91b4ce..93a593ff4c2 100644
--- a/fdbserver/workloads/ConsistencyCheck.actor.cpp
+++ b/fdbserver/workloads/ConsistencyCheck.actor.cpp
@@ -1137,7 +1137,7 @@ struct ConsistencyCheckWorkload : TestWorkload {
 
 	ACTOR Future<std::vector<std::pair<KeyRange, Value>>> getKeyLocationsForRangeList(Database cx,
 	                                                                                  std::vector<KeyRange> ranges,
-																					  ConsistencyCheckWorkload* self) {
+	                                                                                  ConsistencyCheckWorkload* self) {
 		state std::vector<std::pair<KeyRange, Value>> res;
 		state Transaction tr(cx);
 		state int idx = 0;
@@ -2090,16 +2090,16 @@ struct ConsistencyCheckWorkload : TestWorkload {
 			if (failedRangesToCheck.size() > 0) {
 				wait(delay(60.0)); // Backoff 1 min
 				state std::vector<std::pair<KeyRange, Value>> shardLocationPairListForFailedRanges =
-			    	wait(self->getKeyLocationsForRangeList(cx, failedRangesToCheck, self));
+				    wait(self->getKeyLocationsForRangeList(cx, failedRangesToCheck, self));
 				TraceEvent(SevInfo, "ConsistencyCheck_StartHandlingFailedRanges")
-					.setMaxEventLength(-1)
-					.setMaxFieldLength(-1)
-					.detail("FailedCollectedRangeCount", failedRangesToCheck.size())
-					.detail("FailedShardCount", shardLocationPairListForFailedRanges.size())
-					.detail("ConsistencyCheckEpoch", consistencyCheckEpoch)
-					.detail("Distributed", self->distributed)
-					.detail("ClientId", self->clientId)
-					.detail("ClientCount", self->clientCount);
+				    .setMaxEventLength(-1)
+				    .setMaxFieldLength(-1)
+				    .detail("FailedCollectedRangeCount", failedRangesToCheck.size())
+				    .detail("FailedShardCount", shardLocationPairListForFailedRanges.size())
+				    .detail("ConsistencyCheckEpoch", consistencyCheckEpoch)
+				    .detail("Distributed", self->distributed)
+				    .detail("ClientId", self->clientId)
+				    .detail("ClientCount", self->clientCount);
 				if (consistencyCheckEpoch < CLIENT_KNOBS->CONSISTENCY_CHECK_RETRY_DEPTH_MAX) {
 					wait(::success(self->checkDataConsistency(cx,
 					                                          shardLocationPairListForFailedRanges,
@@ -2122,6 +2122,10 @@ struct ConsistencyCheckWorkload : TestWorkload {
 						    .detail("BeginKey", printable(missedRange.first.begin))
 						    .detail("EndKey", printable(missedRange.first.end));
 					}
+					if (g_network->isSimulated()) {
+						self->testFailure("Retry too many times");
+						return false;
+					}
 				}
 			} // Otherwise, no failure, we are good to go
 		}

From ed7c80758247ccf45337ec27a7b1dfbdf01a65e3 Mon Sep 17 00:00:00 2001
From: Zhe Wang <zhe.wang@wustl.edu>
Date: Mon, 11 Dec 2023 13:06:36 -0800
Subject: [PATCH 319/468] Range-Customizable Consistency Checker

---
 fdbclient/ClientKnobs.cpp                      | 4 ++--
 fdbserver/tester.actor.cpp                     | 4 +++-
 fdbserver/workloads/ConsistencyCheck.actor.cpp | 5 ++++-
 3 files changed, 9 insertions(+), 4 deletions(-)

diff --git a/fdbclient/ClientKnobs.cpp b/fdbclient/ClientKnobs.cpp
index 081f9787d11..b4c476c8e50 100644
--- a/fdbclient/ClientKnobs.cpp
+++ b/fdbclient/ClientKnobs.cpp
@@ -263,8 +263,8 @@ void ClientKnobs::initialize(Randomize randomize) {
 	init( CONSISTENCY_CHECK_COMPLETE_CHECK,        true ); // Take effect only when CONSISTENCY_CHECK_DISTRIBUTED = true
 	init( CONSISTENCY_CHECK_DISTRIBUTED_WIGGLE_ROOM, 50 );
 	init( CONSISTENCY_CHECK_RETRY_DEPTH_MAX,        100 ); if( randomize && BUGGIFY ) CONSISTENCY_CHECK_RETRY_DEPTH_MAX = 10;
-	init( CONSISTENCY_CHECK_RANGE_BEGIN,             "" );
-	init( CONSISTENCY_CHECK_RANGE_END,       "\xff\xff" );
+	init( CONSISTENCY_CHECK_RANGE_BEGIN,             "" ); if( randomize && BUGGIFY ) CONSISTENCY_CHECK_RANGE_BEGIN = "a";
+	init( CONSISTENCY_CHECK_RANGE_END,       "\xff\xff" ); if( randomize && BUGGIFY ) CONSISTENCY_CHECK_RANGE_END = "z";
 
 	//fdbcli
 	init( CLI_CONNECT_PARALLELISM,                  400 );
diff --git a/fdbserver/tester.actor.cpp b/fdbserver/tester.actor.cpp
index 61d7bbf2821..ad12f5c383f 100644
--- a/fdbserver/tester.actor.cpp
+++ b/fdbserver/tester.actor.cpp
@@ -1014,8 +1014,10 @@ ACTOR Future<bool> runTest(Database cx,
 			fprintf(stderr, "ERROR: Test timed out after %d seconds.\n", spec.timeout);
 			testResults.failures = testers.size();
 			testResults.successes = 0;
-		} else
+		} else {
+			TraceEvent(SevWarnAlways, "TestFailure").error(e).detail("Reason", e.what());
 			throw;
+		}
 	}
 
 	state bool ok = testResults.ok();
diff --git a/fdbserver/workloads/ConsistencyCheck.actor.cpp b/fdbserver/workloads/ConsistencyCheck.actor.cpp
index 93a593ff4c2..5014f482f12 100644
--- a/fdbserver/workloads/ConsistencyCheck.actor.cpp
+++ b/fdbserver/workloads/ConsistencyCheck.actor.cpp
@@ -363,7 +363,7 @@ struct ConsistencyCheckWorkload : TestWorkload {
 					    .detail("PerformTSSCheck", self->performTSSCheck)
 					    .detail("PerformCacheCheck", self->performCacheCheck);
 					state std::vector<std::pair<KeyRange, Value>> shardLocationPairListForDistributed =
-					    wait(self->getKeyLocationsForRangeList(cx, { allKeys }, self));
+					    wait(self->getKeyLocationsForRangeList(cx, { rangeToCheck }, self));
 					// Check that each failed shard has the same data on all storage servers that it resides on
 					wait(::success(self->checkDataConsistency(cx,
 					                                          shardLocationPairListForDistributed,
@@ -1336,6 +1336,9 @@ struct ConsistencyCheckWorkload : TestWorkload {
 			rateLimitForThisRound = self->rateLimitMax;
 		}
 		TraceEvent("ConsistencyCheck_RateLimitForThisRound")
+		    .detail("Distributed", self->distributed)
+		    .detail("ClientId", self->clientId)
+		    .detail("ClientCount", self->clientCount)
 		    .detail("RateLimit", rateLimitForThisRound)
 		    .detail("ConsistencyCheckEpoch", consistencyCheckEpoch)
 		    .detail("ShardLocationPairList", shardLocationPairList.size());

From 51fd3ec216d600192f6c3072d1a18c2b57bf8ea8 Mon Sep 17 00:00:00 2001
From: FoundationDB CI <foundationdb_ci@apple.com>
Date: Tue, 12 Dec 2023 00:55:12 +0000
Subject: [PATCH 320/468] disable AVX for 7.1.46 release

---
 cmake/ConfigureCompiler.cmake | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cmake/ConfigureCompiler.cmake b/cmake/ConfigureCompiler.cmake
index e52d8665ee8..7e50469ba6a 100644
--- a/cmake/ConfigureCompiler.cmake
+++ b/cmake/ConfigureCompiler.cmake
@@ -253,7 +253,7 @@ else()
       set(USE_AVX512F OFF)
     endif()
   endif()
-  set(USE_AVX ON CACHE BOOL "Enable AVX instructions")
+  set(USE_AVX OFF CACHE BOOL "Enable AVX instructions")
   if (USE_AVX)
     if (CMAKE_HOST_SYSTEM_PROCESSOR MATCHES "^x86")
       add_compile_options(-mavx)

From 1a7fe2789c481783c90310fb557f8675d04f822e Mon Sep 17 00:00:00 2001
From: FoundationDB CI <foundationdb_ci@apple.com>
Date: Tue, 12 Dec 2023 00:55:25 +0000
Subject: [PATCH 321/468] enable AVX and update version for 7.1.47 release

---
 CMakeLists.txt                | 2 +-
 cmake/ConfigureCompiler.cmake | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 994e9664fb9..10b81d65b1c 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -23,7 +23,7 @@ else()
 endif()
 
 project(foundationdb
-  VERSION 7.1.46
+  VERSION 7.1.47
   DESCRIPTION "FoundationDB is a scalable, fault-tolerant, ordered key-value store with full ACID transactions."
   HOMEPAGE_URL "http://www.foundationdb.org/"
   LANGUAGES C CXX ASM)
diff --git a/cmake/ConfigureCompiler.cmake b/cmake/ConfigureCompiler.cmake
index 7e50469ba6a..e52d8665ee8 100644
--- a/cmake/ConfigureCompiler.cmake
+++ b/cmake/ConfigureCompiler.cmake
@@ -253,7 +253,7 @@ else()
       set(USE_AVX512F OFF)
     endif()
   endif()
-  set(USE_AVX OFF CACHE BOOL "Enable AVX instructions")
+  set(USE_AVX ON CACHE BOOL "Enable AVX instructions")
   if (USE_AVX)
     if (CMAKE_HOST_SYSTEM_PROCESSOR MATCHES "^x86")
       add_compile_options(-mavx)

From 61e80a4f584eb0052188a7f78189c046c577e690 Mon Sep 17 00:00:00 2001
From: FoundationDB CI <foundationdb_ci@apple.com>
Date: Tue, 12 Dec 2023 15:09:34 +0000
Subject: [PATCH 322/468] update version after 7.1.47 release

---
 CMakeLists.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 10b81d65b1c..d02e78ca2ff 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -23,7 +23,7 @@ else()
 endif()
 
 project(foundationdb
-  VERSION 7.1.47
+  VERSION 7.1.48
   DESCRIPTION "FoundationDB is a scalable, fault-tolerant, ordered key-value store with full ACID transactions."
   HOMEPAGE_URL "http://www.foundationdb.org/"
   LANGUAGES C CXX ASM)

From 3cdf934478a098441a560fdb7b7ae9fee630fe5a Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyu_zhou@apple.com>
Date: Mon, 11 Dec 2023 15:50:05 -0800
Subject: [PATCH 323/468] Merge pull request #11098 from sbodagala/main

Bump up the number of keys that "fdbcli checkall" can fetch per GetKeyValueRequest call
---
 fdbcli/DebugCommands.actor.cpp | 55 +++++++++++++++++++++++++---------
 1 file changed, 41 insertions(+), 14 deletions(-)

diff --git a/fdbcli/DebugCommands.actor.cpp b/fdbcli/DebugCommands.actor.cpp
index c6a8665856d..ecb877e3aee 100644
--- a/fdbcli/DebugCommands.actor.cpp
+++ b/fdbcli/DebugCommands.actor.cpp
@@ -192,18 +192,33 @@ bool checkResults(Version version,
 		}
 
 		allSame = false;
-		printf("#%d  server: %s  key count: %lu\n",
+		printf("#%d  server: %s  key count: %lu, cached: %d, more: %d\n",
 		       firstValidServer,
 		       servers[firstValidServer].address().toString().c_str(),
-		       reference.data.size());
-		printf("#%d  server: %s  key count: %lu\n", j, servers[j].address().toString().c_str(), current.data.size());
+		       reference.data.size(),
+		       reference.cached,
+		       +reference.more);
+		printf("#%d  server: %s  key count: %lu, cached: %d, more: %dn",
+		       j,
+		       servers[j].address().toString().c_str(),
+		       current.data.size(),
+		       current.cached,
+		       current.more);
 		size_t currentI = 0, referenceI = 0;
 		while (currentI < current.data.size() || referenceI < reference.data.size()) {
 			if (currentI >= current.data.size()) {
-				printf(" #%d Unique key: %s\n", firstValidServer, printable(reference.data[referenceI].key).c_str());
+				printf(" #%d CurrentI: %lu ReferenceI: %lu Unique key: %s\n",
+				       firstValidServer,
+				       currentI,
+				       referenceI,
+				       printable(reference.data[referenceI].key).c_str());
 				referenceI++;
 			} else if (referenceI >= reference.data.size()) {
-				printf(" #%d Unique key: %s\n", j, printable(current.data[currentI].key).c_str());
+				printf(" #%d CurrentI: %lu ReferenceI: %lu Unique key: %s\n",
+				       j,
+				       currentI,
+				       referenceI,
+				       printable(current.data[currentI].key).c_str());
 				currentI++;
 			} else {
 				KeyValueRef currentKV = current.data[currentI];
@@ -211,32 +226,44 @@ bool checkResults(Version version,
 
 				if (currentKV.key == referenceKV.key) {
 					if (currentKV.value != referenceKV.value) {
-						printf(" Value mismatch key: %s\n", printable(currentKV.key).c_str());
+						printf(" CurrentI: %lu ReferenceI: %lu Value mismatch key: %s\n",
+						       currentI,
+						       referenceI,
+						       printable(currentKV.key).c_str());
 					}
 					currentI++;
 					referenceI++;
 				} else if (currentKV.key < referenceKV.key) {
-					printf(" #%d Unique key: %s\n", j, printable(currentKV.key).c_str());
+					printf(" #%d CurrentI: %lu ReferenceI: %lu Unique key: %s\n",
+					       j,
+					       currentI,
+					       referenceI,
+					       printable(currentKV.key).c_str());
 					currentI++;
 				} else {
-					printf(" #%d Unique key: %s\n", firstValidServer, printable(referenceKV.key).c_str());
+					printf(" #%d CurrentI: %lu ReferenceI: %lu Unique key: %s\n",
+					       firstValidServer,
+					       currentI,
+					       referenceI,
+					       printable(referenceKV.key).c_str());
 					referenceI++;
 				}
 			}
 		}
 	}
 
-	if (!allSame)
-		return false;
-
 	if (firstValidServer >= 0 && replies[firstValidServer].get().get().more) {
 		const VectorRef<KeyValueRef>& result = replies[firstValidServer].get().get().data;
+		printf("Warning: Consistency check was incomplete, last key of server %d that was checked: %s\n",
+		       firstValidServer,
+		       printable(result[result.size() - 1].key).c_str());
 		begin = firstGreaterThan(result[result.size() - 1].key);
 	} else {
 		printf("Same at version %ld\n", version);
 		begin = end; // signal that we're done
 	}
-	return true;
+
+	return allSame;
 }
 
 // The command is used to check the inconsistency in a keyspace, default is \xff\x02/blog/ keyspace.
@@ -299,8 +326,8 @@ ACTOR Future<bool> checkallCommandActor(Database cx, std::vector<StringRef> toke
 					GetKeyValuesRequest req;
 					req.begin = begin;
 					req.end = end;
-					req.limit = 1e4;
-					req.limitBytes = CLIENT_KNOBS->REPLY_BYTE_LIMIT;
+					req.limit = CLIENT_KNOBS->KRM_GET_RANGE_LIMIT;
+					req.limitBytes = CLIENT_KNOBS->KRM_GET_RANGE_LIMIT_BYTES;
 					req.version = version;
 					req.tags = TagSet();
 

From f3c96168320a4d54102d4f82fab601ac5d4a76e7 Mon Sep 17 00:00:00 2001
From: Zhe Wang <zhe.wang@wustl.edu>
Date: Tue, 12 Dec 2023 21:00:19 -0800
Subject: [PATCH 324/468] improved-getKeyLocationsForRangeList-and-one-time-run

---
 fdbclient/ClientKnobs.cpp                     |  1 +
 fdbclient/ClientKnobs.h                       |  1 +
 .../workloads/ConsistencyCheck.actor.cpp      | 70 +++++++++++--------
 3 files changed, 44 insertions(+), 28 deletions(-)

diff --git a/fdbclient/ClientKnobs.cpp b/fdbclient/ClientKnobs.cpp
index b4c476c8e50..c69905285e5 100644
--- a/fdbclient/ClientKnobs.cpp
+++ b/fdbclient/ClientKnobs.cpp
@@ -261,6 +261,7 @@ void ClientKnobs::initialize(Randomize randomize) {
 	init( CONSISTENCY_CHECK_DISTRIBUTED,           true );
 	init( CONSISTENCY_CHECK_AGGRESSIVE_RUN,        true );
 	init( CONSISTENCY_CHECK_COMPLETE_CHECK,        true ); // Take effect only when CONSISTENCY_CHECK_DISTRIBUTED = true
+	init( CONSISTENCY_CHECK_ONE_TIME_CHECK,        true );
 	init( CONSISTENCY_CHECK_DISTRIBUTED_WIGGLE_ROOM, 50 );
 	init( CONSISTENCY_CHECK_RETRY_DEPTH_MAX,        100 ); if( randomize && BUGGIFY ) CONSISTENCY_CHECK_RETRY_DEPTH_MAX = 10;
 	init( CONSISTENCY_CHECK_RANGE_BEGIN,             "" ); if( randomize && BUGGIFY ) CONSISTENCY_CHECK_RANGE_BEGIN = "a";
diff --git a/fdbclient/ClientKnobs.h b/fdbclient/ClientKnobs.h
index 7aaae7ad867..e9313cc4f2d 100644
--- a/fdbclient/ClientKnobs.h
+++ b/fdbclient/ClientKnobs.h
@@ -252,6 +252,7 @@ class ClientKnobs : public KnobsImpl<ClientKnobs> {
 	bool CONSISTENCY_CHECK_DISTRIBUTED;
 	bool CONSISTENCY_CHECK_AGGRESSIVE_RUN;
 	bool CONSISTENCY_CHECK_COMPLETE_CHECK;
+	bool CONSISTENCY_CHECK_ONE_TIME_CHECK;
 	int CONSISTENCY_CHECK_RETRY_DEPTH_MAX;
 	int CONSISTENCY_CHECK_DISTRIBUTED_WIGGLE_ROOM;
 	std::string CONSISTENCY_CHECK_RANGE_BEGIN;
diff --git a/fdbserver/workloads/ConsistencyCheck.actor.cpp b/fdbserver/workloads/ConsistencyCheck.actor.cpp
index 5014f482f12..628d9f940f7 100644
--- a/fdbserver/workloads/ConsistencyCheck.actor.cpp
+++ b/fdbserver/workloads/ConsistencyCheck.actor.cpp
@@ -203,7 +203,7 @@ struct ConsistencyCheckWorkload : TestWorkload {
 			    .detail("Repetitions", self->repetitions);
 			choose {
 				when(wait(self->runCheck(cx, self))) {
-					if (!self->indefinite) {
+					if (!self->indefinite || CLIENT_KNOBS->CONSISTENCY_CHECK_ONE_TIME_CHECK) {
 						TraceEvent("ConsistencyCheck_Exit")
 						    .detail("Distributed", self->distributed)
 						    .detail("ClientCount", self->clientCount)
@@ -363,7 +363,7 @@ struct ConsistencyCheckWorkload : TestWorkload {
 					    .detail("PerformTSSCheck", self->performTSSCheck)
 					    .detail("PerformCacheCheck", self->performCacheCheck);
 					state std::vector<std::pair<KeyRange, Value>> shardLocationPairListForDistributed =
-					    wait(self->getKeyLocationsForRangeList(cx, { rangeToCheck }, self));
+					    wait(self->getKeyLocationsForRangeList(cx, { rangeToCheck }));
 					// Check that each failed shard has the same data on all storage servers that it resides on
 					wait(::success(self->checkDataConsistency(cx,
 					                                          shardLocationPairListForDistributed,
@@ -1136,36 +1136,50 @@ struct ConsistencyCheckWorkload : TestWorkload {
 	}
 
 	ACTOR Future<std::vector<std::pair<KeyRange, Value>>> getKeyLocationsForRangeList(Database cx,
-	                                                                                  std::vector<KeyRange> ranges,
-	                                                                                  ConsistencyCheckWorkload* self) {
+	                                                                                  std::vector<KeyRange> ranges) {
+		// Get the scope of the input list of ranges
+		state Key beginKeyToReadKeyServer;
+		state Key endKeyToReadKeyServer;
+		for (const auto& range : ranges) {
+			if (beginKeyToReadKeyServer.empty() || range.begin < beginKeyToReadKeyServer) {
+				beginKeyToReadKeyServer = range.begin;
+			}
+			if (endKeyToReadKeyServer.empty() || range.end > endKeyToReadKeyServer) {
+				endKeyToReadKeyServer = range.end;
+			}
+		}
+		// Read KeyServer space within the scope and add shards intersecting with the input ranges
 		state std::vector<std::pair<KeyRange, Value>> res;
 		state Transaction tr(cx);
-		state int idx = 0;
-		for (; idx < ranges.size(); ++idx) {
-			state Key beginKey = ranges[idx].begin;
-			state Key endKey = ranges[idx].end;
-			loop {
-				try {
-					tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
-					tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
-					state KeyRange rangeToRead = Standalone(KeyRangeRef(beginKey, endKey));
-					RangeResult readResult = wait(krmGetRanges(&tr,
-					                                           keyServersPrefix,
-					                                           rangeToRead,
-					                                           CLIENT_KNOBS->TOO_MANY,
-					                                           GetRangeLimits::BYTE_LIMIT_UNLIMITED));
-					for (int i = 0; i < readResult.size() - 1; ++i) {
-						res.push_back(std::make_pair(Standalone(KeyRangeRef(readResult[i].key, readResult[i + 1].key)),
-						                             Standalone(readResult[i].value)));
-						beginKey = readResult[i + 1].key;
+		loop {
+			try {
+				tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
+				tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
+				tr.setOption(FDBTransactionOptions::LOCK_AWARE);
+				KeyRange rangeToRead = Standalone(KeyRangeRef(beginKeyToReadKeyServer, endKeyToReadKeyServer));
+				RangeResult readResult = wait(krmGetRanges(
+				    &tr, keyServersPrefix, rangeToRead, CLIENT_KNOBS->TOO_MANY, GetRangeLimits::BYTE_LIMIT_UNLIMITED));
+				for (int i = 0; i < readResult.size() - 1; ++i) {
+					KeyRange rangeToCheck = Standalone(KeyRangeRef(readResult[i].key, readResult[i + 1].key));
+					Value valueToCheck = Standalone(readResult[i].value);
+					bool toAdd = false;
+					for (const auto& range : ranges) {
+						if (rangeToCheck.intersects(range) == true) {
+							toAdd = true;
+							break;
+						}
 					}
-					if (beginKey >= ranges[idx].end) {
-						break;
+					if (toAdd == true) {
+						res.push_back(std::make_pair(rangeToCheck, valueToCheck));
 					}
-				} catch (Error& e) {
-					TraceEvent("ConsistencyCheck_RetryGetKeyLocationsForRangeList").error(e);
-					wait(tr.onError(e));
+					beginKeyToReadKeyServer = readResult[i + 1].key;
 				}
+				if (beginKeyToReadKeyServer >= endKeyToReadKeyServer) {
+					break;
+				}
+			} catch (Error& e) {
+				TraceEvent("ConsistencyCheck_RetryGetKeyLocationsForRangeList").error(e);
+				wait(tr.onError(e));
 			}
 		}
 		return res;
@@ -2093,7 +2107,7 @@ struct ConsistencyCheckWorkload : TestWorkload {
 			if (failedRangesToCheck.size() > 0) {
 				wait(delay(60.0)); // Backoff 1 min
 				state std::vector<std::pair<KeyRange, Value>> shardLocationPairListForFailedRanges =
-				    wait(self->getKeyLocationsForRangeList(cx, failedRangesToCheck, self));
+				    wait(self->getKeyLocationsForRangeList(cx, failedRangesToCheck));
 				TraceEvent(SevInfo, "ConsistencyCheck_StartHandlingFailedRanges")
 				    .setMaxEventLength(-1)
 				    .setMaxFieldLength(-1)

From f2b1e97d7bbbc449f1a4c8b01f237fdf6b0e80f9 Mon Sep 17 00:00:00 2001
From: Zhe Wang <zhe.wang@wustl.edu>
Date: Tue, 12 Dec 2023 22:10:33 -0800
Subject: [PATCH 325/468] urgent mode

---
 fdbclient/ClientKnobs.cpp                     |  7 ++--
 fdbclient/ClientKnobs.h                       | 13 +++-----
 fdbserver/fdbserver.actor.cpp                 |  2 +-
 fdbserver/tester.actor.cpp                    |  4 +--
 .../workloads/ConsistencyCheck.actor.cpp      | 32 ++++++++++---------
 5 files changed, 27 insertions(+), 31 deletions(-)

diff --git a/fdbclient/ClientKnobs.cpp b/fdbclient/ClientKnobs.cpp
index c69905285e5..2cbf7572b1e 100644
--- a/fdbclient/ClientKnobs.cpp
+++ b/fdbclient/ClientKnobs.cpp
@@ -258,11 +258,8 @@ void ClientKnobs::initialize(Randomize randomize) {
 
 	init( CONSISTENCY_CHECK_RATE_LIMIT_MAX,        50e6 ); // Limit in per sec
 	init( CONSISTENCY_CHECK_ONE_ROUND_TARGET_COMPLETION_TIME,	7 * 24 * 60 * 60 ); // 7 days
-	init( CONSISTENCY_CHECK_DISTRIBUTED,           true );
-	init( CONSISTENCY_CHECK_AGGRESSIVE_RUN,        true );
-	init( CONSISTENCY_CHECK_COMPLETE_CHECK,        true ); // Take effect only when CONSISTENCY_CHECK_DISTRIBUTED = true
-	init( CONSISTENCY_CHECK_ONE_TIME_CHECK,        true );
-	init( CONSISTENCY_CHECK_DISTRIBUTED_WIGGLE_ROOM, 50 );
+	init( CONSISTENCY_CHECK_URGENT_MODE,           true ); if( randomize && BUGGIFY ) CONSISTENCY_CHECK_URGENT_MODE = false;
+	init( CONSISTENCY_CHECK_DISTRIBUTED_WIGGLE_ROOM, 50 ); if( randomize && BUGGIFY ) CONSISTENCY_CHECK_DISTRIBUTED_WIGGLE_ROOM = 1;
 	init( CONSISTENCY_CHECK_RETRY_DEPTH_MAX,        100 ); if( randomize && BUGGIFY ) CONSISTENCY_CHECK_RETRY_DEPTH_MAX = 10;
 	init( CONSISTENCY_CHECK_RANGE_BEGIN,             "" ); if( randomize && BUGGIFY ) CONSISTENCY_CHECK_RANGE_BEGIN = "a";
 	init( CONSISTENCY_CHECK_RANGE_END,       "\xff\xff" ); if( randomize && BUGGIFY ) CONSISTENCY_CHECK_RANGE_END = "z";
diff --git a/fdbclient/ClientKnobs.h b/fdbclient/ClientKnobs.h
index e9313cc4f2d..1d8ac586375 100644
--- a/fdbclient/ClientKnobs.h
+++ b/fdbclient/ClientKnobs.h
@@ -249,14 +249,11 @@ class ClientKnobs : public KnobsImpl<ClientKnobs> {
 
 	int CONSISTENCY_CHECK_RATE_LIMIT_MAX;
 	int CONSISTENCY_CHECK_ONE_ROUND_TARGET_COMPLETION_TIME;
-	bool CONSISTENCY_CHECK_DISTRIBUTED;
-	bool CONSISTENCY_CHECK_AGGRESSIVE_RUN;
-	bool CONSISTENCY_CHECK_COMPLETE_CHECK;
-	bool CONSISTENCY_CHECK_ONE_TIME_CHECK;
-	int CONSISTENCY_CHECK_RETRY_DEPTH_MAX;
-	int CONSISTENCY_CHECK_DISTRIBUTED_WIGGLE_ROOM;
-	std::string CONSISTENCY_CHECK_RANGE_BEGIN;
-	std::string CONSISTENCY_CHECK_RANGE_END;
+	bool CONSISTENCY_CHECK_URGENT_MODE;  // Require Tester Process::Class
+	int CONSISTENCY_CHECK_RETRY_DEPTH_MAX; // Available in urgent mode
+	int CONSISTENCY_CHECK_DISTRIBUTED_WIGGLE_ROOM; // Available in urgent mode
+	std::string CONSISTENCY_CHECK_RANGE_BEGIN; // Available in urgent mode
+	std::string CONSISTENCY_CHECK_RANGE_END; // Available in urgent mode
 
 	// fdbcli
 	int CLI_CONNECT_PARALLELISM;
diff --git a/fdbserver/fdbserver.actor.cpp b/fdbserver/fdbserver.actor.cpp
index d59a1bbd7ac..6f03701c7dd 100644
--- a/fdbserver/fdbserver.actor.cpp
+++ b/fdbserver/fdbserver.actor.cpp
@@ -2181,7 +2181,7 @@ int main(int argc, char* argv[]) {
 			auto m = startSystemMonitor(opts.dataFolder, opts.dcId, opts.zoneId, opts.zoneId);
 			f = stopAfter(runTests(opts.connectionFile,
 			                       TEST_TYPE_CONSISTENCY_CHECK,
-			                       CLIENT_KNOBS->CONSISTENCY_CHECK_DISTRIBUTED ? TEST_ON_TESTERS : TEST_HERE,
+			                       CLIENT_KNOBS->CONSISTENCY_CHECK_URGENT_MODE ? TEST_ON_TESTERS : TEST_HERE,
 			                       1,
 			                       opts.testFile,
 			                       StringRef(),
diff --git a/fdbserver/tester.actor.cpp b/fdbserver/tester.actor.cpp
index ad12f5c383f..bd9f35d3ade 100644
--- a/fdbserver/tester.actor.cpp
+++ b/fdbserver/tester.actor.cpp
@@ -958,7 +958,7 @@ ACTOR Future<Void> checkConsistency(Database cx,
 	// options.push_back_deep(options.arena(), KeyValueRef(LiteralStringRef("distributed"), LiteralStringRef("false")));
 	options.push_back_deep(options.arena(),
 	                       KeyValueRef(LiteralStringRef("distributed"),
-	                                   StringRef(CLIENT_KNOBS->CONSISTENCY_CHECK_DISTRIBUTED ? "true" : "false")));
+	                                   StringRef(CLIENT_KNOBS->CONSISTENCY_CHECK_URGENT_MODE ? "true" : "false")));
 	spec.options.push_back_deep(spec.options.arena(), options);
 	state double start = now();
 	state bool lastRun = false;
@@ -1766,7 +1766,7 @@ ACTOR Future<Void> runTests(Reference<IClusterConnectionRecord> connRecord,
 		//                        KeyValueRef(LiteralStringRef("distributed"), LiteralStringRef("false")));
 		options.push_back_deep(options.arena(),
 		                       KeyValueRef(LiteralStringRef("distributed"),
-		                                   StringRef(CLIENT_KNOBS->CONSISTENCY_CHECK_DISTRIBUTED ? "true" : "false")));
+		                                   StringRef(CLIENT_KNOBS->CONSISTENCY_CHECK_URGENT_MODE ? "true" : "false")));
 		options.push_back_deep(options.arena(),
 		                       KeyValueRef(LiteralStringRef("failureIsError"), LiteralStringRef("true")));
 		options.push_back_deep(options.arena(), KeyValueRef(LiteralStringRef("indefinite"), LiteralStringRef("true")));
diff --git a/fdbserver/workloads/ConsistencyCheck.actor.cpp b/fdbserver/workloads/ConsistencyCheck.actor.cpp
index 628d9f940f7..c699f397adb 100644
--- a/fdbserver/workloads/ConsistencyCheck.actor.cpp
+++ b/fdbserver/workloads/ConsistencyCheck.actor.cpp
@@ -203,7 +203,9 @@ struct ConsistencyCheckWorkload : TestWorkload {
 			    .detail("Repetitions", self->repetitions);
 			choose {
 				when(wait(self->runCheck(cx, self))) {
-					if (!self->indefinite || CLIENT_KNOBS->CONSISTENCY_CHECK_ONE_TIME_CHECK) {
+					if (!self->indefinite || CLIENT_KNOBS->CONSISTENCY_CHECK_URGENT_MODE) {
+						// When consistency checker is distributed, repeated running
+						// can miss ranges, so we exit here
 						TraceEvent("ConsistencyCheck_Exit")
 						    .detail("Distributed", self->distributed)
 						    .detail("ClientCount", self->clientCount)
@@ -334,7 +336,7 @@ struct ConsistencyCheckWorkload : TestWorkload {
 						self->testFailure("Coordinators incorrect");
 				}
 
-				if (CLIENT_KNOBS->CONSISTENCY_CHECK_DISTRIBUTED) {
+				if (CLIENT_KNOBS->CONSISTENCY_CHECK_URGENT_MODE) {
 					state KeyRange rangeToCheck = allKeys;
 					Key rangeBegin = StringRef(CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_BEGIN);
 					if (rangeBegin > allKeys.end) {
@@ -1323,7 +1325,7 @@ struct ConsistencyCheckWorkload : TestWorkload {
 	                                        ConsistencyCheckWorkload* self,
 	                                        int consistencyCheckEpoch) {
 		if (consistencyCheckEpoch > 0) {
-			ASSERT(CLIENT_KNOBS->CONSISTENCY_CHECK_DISTRIBUTED && CLIENT_KNOBS->CONSISTENCY_CHECK_COMPLETE_CHECK);
+			ASSERT(CLIENT_KNOBS->CONSISTENCY_CHECK_URGENT_MODE);
 		}
 		// Stores the total number of bytes on each storage server
 		// In a distributed test, this will be an estimated size
@@ -1345,8 +1347,8 @@ struct ConsistencyCheckWorkload : TestWorkload {
 		              static_cast<int>(ceil(self->bytesReadInPreviousRound /
 		                                    (float)CLIENT_KNOBS->CONSISTENCY_CHECK_ONE_ROUND_TARGET_COMPLETION_TIME)));
 		if (consistencyCheckEpoch > 0 ||
-		    CLIENT_KNOBS->CONSISTENCY_CHECK_AGGRESSIVE_RUN) { // We set max speed when retrying failed ranges or
-			                                                  // aggressive check is on
+		    CLIENT_KNOBS->CONSISTENCY_CHECK_URGENT_MODE) { // We set max speed when retrying failed ranges or
+			                                                  // distributed among testers
 			rateLimitForThisRound = self->rateLimitMax;
 		}
 		TraceEvent("ConsistencyCheck_RateLimitForThisRound")
@@ -1385,14 +1387,14 @@ struct ConsistencyCheckWorkload : TestWorkload {
 		shardOrder.reserve(ranges.size());
 		for (int k = 0; k < ranges.size(); k++)
 			shardOrder.push_back(k);
-		if (self->shuffleShards && !CLIENT_KNOBS->CONSISTENCY_CHECK_DISTRIBUTED) {
+		if (self->shuffleShards && !CLIENT_KNOBS->CONSISTENCY_CHECK_URGENT_MODE) {
 			uint32_t seed = self->sharedRandomNumber + self->repetitions;
 			DeterministicRandom sharedRandom(seed == 0 ? 1 : seed);
 			sharedRandom.randomShuffle(shardOrder);
 		}
 
 		state int endPoint;
-		if (CLIENT_KNOBS->CONSISTENCY_CHECK_DISTRIBUTED) {
+		if (CLIENT_KNOBS->CONSISTENCY_CHECK_URGENT_MODE) {
 			if (consistencyCheckEpoch == 0) {
 				int batchSize = (shardOrder.size() / self->clientCount) + 1;
 				i = self->clientId * batchSize; // overwrite the starting point i
@@ -1443,7 +1445,7 @@ struct ConsistencyCheckWorkload : TestWorkload {
 		                                            : "ConsistencyCheck_StartFailedTask");
 
 		for (; i < endPoint; i += increment) {
-			if (CLIENT_KNOBS->CONSISTENCY_CHECK_DISTRIBUTED && self->suspendConsistencyCheck.get()) {
+			if (CLIENT_KNOBS->CONSISTENCY_CHECK_URGENT_MODE && self->suspendConsistencyCheck.get()) {
 				TraceEvent("ConsistencyCheck_Cancelled")
 				    .setMaxEventLength(-1)
 				    .setMaxFieldLength(-1)
@@ -1596,7 +1598,7 @@ struct ConsistencyCheckWorkload : TestWorkload {
 
 			// The first client may need to skip the rest of the loop contents if it is just processing this shard to
 			// get a size estimate
-			if (CLIENT_KNOBS->CONSISTENCY_CHECK_DISTRIBUTED || !self->firstClient ||
+			if (CLIENT_KNOBS->CONSISTENCY_CHECK_URGENT_MODE || !self->firstClient ||
 			    shard % (effectiveClientCount * self->shardSampleFactor) == 0) {
 				state int shardKeys = 0;
 				state int shardBytes = 0;
@@ -1619,7 +1621,7 @@ struct ConsistencyCheckWorkload : TestWorkload {
 				// Read a limited number of entries at a time, repeating until all keys in the shard have been read
 				loop {
 					try {
-						if (CLIENT_KNOBS->CONSISTENCY_CHECK_DISTRIBUTED && self->suspendConsistencyCheck.get()) {
+						if (CLIENT_KNOBS->CONSISTENCY_CHECK_URGENT_MODE && self->suspendConsistencyCheck.get()) {
 							TraceEvent("ConsistencyCheck_DataCheckCancelled")
 							    .setMaxEventLength(-1)
 							    .setMaxFieldLength(-1)
@@ -1842,7 +1844,7 @@ struct ConsistencyCheckWorkload : TestWorkload {
 								    .detail("ConsistencyCheckEpoch", consistencyCheckEpoch);
 
 								completeCheck = false;
-								if (CLIENT_KNOBS->CONSISTENCY_CHECK_COMPLETE_CHECK) {
+								if (CLIENT_KNOBS->CONSISTENCY_CHECK_URGENT_MODE) {
 									failedRanges.insert(Standalone(KeyRangeRef(req.begin.getKey(), req.end.getKey())),
 									                    true);
 									break; // Will retry later
@@ -1856,7 +1858,7 @@ struct ConsistencyCheckWorkload : TestWorkload {
 
 							} else {
 								completeCheck = false;
-								if (CLIENT_KNOBS->CONSISTENCY_CHECK_COMPLETE_CHECK) {
+								if (CLIENT_KNOBS->CONSISTENCY_CHECK_URGENT_MODE) {
 									failedRanges.insert(Standalone(KeyRangeRef(req.begin.getKey(), req.end.getKey())),
 									                    true);
 									break; // Will retry later
@@ -1941,7 +1943,7 @@ struct ConsistencyCheckWorkload : TestWorkload {
 				}
 
 				if (!completeCheck) {
-					TraceEvent(CLIENT_KNOBS->CONSISTENCY_CHECK_COMPLETE_CHECK ? SevInfo : SevWarn,
+					TraceEvent(CLIENT_KNOBS->CONSISTENCY_CHECK_URGENT_MODE ? SevInfo : SevWarn,
 					           "ConsistencyCheck_ShardFailed")
 					    .setMaxEventLength(-1)
 					    .setMaxFieldLength(-1)
@@ -1968,7 +1970,7 @@ struct ConsistencyCheckWorkload : TestWorkload {
 					    .detail("ConsistencyCheckEpoch", consistencyCheckEpoch);
 				}
 
-				if (CLIENT_KNOBS->CONSISTENCY_CHECK_DISTRIBUTED) {
+				if (CLIENT_KNOBS->CONSISTENCY_CHECK_URGENT_MODE) {
 					continue; // Bypass following check not-relevant to userdata/metadata consistency
 				}
 
@@ -2093,7 +2095,7 @@ struct ConsistencyCheckWorkload : TestWorkload {
 			}
 		}
 
-		if (CLIENT_KNOBS->CONSISTENCY_CHECK_DISTRIBUTED && CLIENT_KNOBS->CONSISTENCY_CHECK_COMPLETE_CHECK) {
+		if (CLIENT_KNOBS->CONSISTENCY_CHECK_URGENT_MODE) {
 			failedRanges.coalesce(allKeys);
 			state std::vector<KeyRange> failedRangesToCheck;
 			state KeyRangeMap<bool>::Ranges failedRangesList = failedRanges.ranges();

From 73d2dfee0c52f7fe6032a494316f4a580c0d0058 Mon Sep 17 00:00:00 2001
From: Zhe Wang <zhe.wang@wustl.edu>
Date: Tue, 12 Dec 2023 22:31:08 -0800
Subject: [PATCH 326/468] fmt

---
 fdbclient/ClientKnobs.h                        | 2 +-
 fdbserver/workloads/ConsistencyCheck.actor.cpp | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/fdbclient/ClientKnobs.h b/fdbclient/ClientKnobs.h
index 1d8ac586375..cc13d9e8e23 100644
--- a/fdbclient/ClientKnobs.h
+++ b/fdbclient/ClientKnobs.h
@@ -249,7 +249,7 @@ class ClientKnobs : public KnobsImpl<ClientKnobs> {
 
 	int CONSISTENCY_CHECK_RATE_LIMIT_MAX;
 	int CONSISTENCY_CHECK_ONE_ROUND_TARGET_COMPLETION_TIME;
-	bool CONSISTENCY_CHECK_URGENT_MODE;  // Require Tester Process::Class
+	bool CONSISTENCY_CHECK_URGENT_MODE; // Require Tester Process::Class
 	int CONSISTENCY_CHECK_RETRY_DEPTH_MAX; // Available in urgent mode
 	int CONSISTENCY_CHECK_DISTRIBUTED_WIGGLE_ROOM; // Available in urgent mode
 	std::string CONSISTENCY_CHECK_RANGE_BEGIN; // Available in urgent mode
diff --git a/fdbserver/workloads/ConsistencyCheck.actor.cpp b/fdbserver/workloads/ConsistencyCheck.actor.cpp
index c699f397adb..1cd7899d403 100644
--- a/fdbserver/workloads/ConsistencyCheck.actor.cpp
+++ b/fdbserver/workloads/ConsistencyCheck.actor.cpp
@@ -1348,7 +1348,7 @@ struct ConsistencyCheckWorkload : TestWorkload {
 		                                    (float)CLIENT_KNOBS->CONSISTENCY_CHECK_ONE_ROUND_TARGET_COMPLETION_TIME)));
 		if (consistencyCheckEpoch > 0 ||
 		    CLIENT_KNOBS->CONSISTENCY_CHECK_URGENT_MODE) { // We set max speed when retrying failed ranges or
-			                                                  // distributed among testers
+			                                               // distributed among testers
 			rateLimitForThisRound = self->rateLimitMax;
 		}
 		TraceEvent("ConsistencyCheck_RateLimitForThisRound")

From 2b9b96f0ed61f156810ab5d5cc242151ff125c68 Mon Sep 17 00:00:00 2001
From: Zhe Wang <zhe.wang@wustl.edu>
Date: Wed, 13 Dec 2023 12:23:41 -0800
Subject: [PATCH 327/468] skip entire shard if any subkeys failed to check

---
 .../workloads/ConsistencyCheck.actor.cpp      | 23 +++++++++++++++----
 1 file changed, 18 insertions(+), 5 deletions(-)

diff --git a/fdbserver/workloads/ConsistencyCheck.actor.cpp b/fdbserver/workloads/ConsistencyCheck.actor.cpp
index 1cd7899d403..acc69182620 100644
--- a/fdbserver/workloads/ConsistencyCheck.actor.cpp
+++ b/fdbserver/workloads/ConsistencyCheck.actor.cpp
@@ -1845,8 +1845,6 @@ struct ConsistencyCheckWorkload : TestWorkload {
 
 								completeCheck = false;
 								if (CLIENT_KNOBS->CONSISTENCY_CHECK_URGENT_MODE) {
-									failedRanges.insert(Standalone(KeyRangeRef(req.begin.getKey(), req.end.getKey())),
-									                    true);
 									break; // Will retry later
 								}
 
@@ -1859,14 +1857,29 @@ struct ConsistencyCheckWorkload : TestWorkload {
 							} else {
 								completeCheck = false;
 								if (CLIENT_KNOBS->CONSISTENCY_CHECK_URGENT_MODE) {
-									failedRanges.insert(Standalone(KeyRangeRef(req.begin.getKey(), req.end.getKey())),
-									                    true);
 									break; // Will retry later
 								}
 							}
 						}
 
-						if (firstValidServer >= 0) {
+						if (CLIENT_KNOBS->CONSISTENCY_CHECK_URGENT_MODE && !completeCheck) {
+							// Add remaining range of the shard to failedRanges,
+							// which will be retried later
+							failedRanges.insert(range, true);
+							TraceEvent(SevInfo, "ConsistencyCheck_ShardAddedToRetry")
+							    .setMaxEventLength(-1)
+							    .setMaxFieldLength(-1)
+							    .detail("ClientId", self->clientId)
+							    .detail("ClientCount", self->clientCount)
+							    .detail("ShardBegin", printable(range.begin))
+							    .detail("ShardEnd", printable(range.end))
+							    .detail("Repetitions", self->repetitions)
+							    .detail("ConsistencyCheckEpoch", consistencyCheckEpoch)
+							    .trackLatest("ConsistencyCheck_ShardAddedToRetry");
+							break; // skip the entire shard, retry later
+						}
+
+						if (firstValidServer >= 0 && !CLIENT_KNOBS->CONSISTENCY_CHECK_URGENT_MODE) {
 							VectorRef<KeyValueRef> data = keyValueFutures[firstValidServer].get().get().data;
 							// Calculate the size of the shard, the variance of the shard size estimate, and the correct
 							// shard size estimate

From 61c2cb2a0c77cceb255abd63eb4e3b806243e9d8 Mon Sep 17 00:00:00 2001
From: Zhe Wang <zhe.wang@wustl.edu>
Date: Wed, 13 Dec 2023 13:41:07 -0800
Subject: [PATCH 328/468] turn on CONSISTENCY_CHECK_URGENT_MODE

---
 fdbclient/ClientKnobs.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/fdbclient/ClientKnobs.cpp b/fdbclient/ClientKnobs.cpp
index 2cbf7572b1e..05bfef5b814 100644
--- a/fdbclient/ClientKnobs.cpp
+++ b/fdbclient/ClientKnobs.cpp
@@ -258,7 +258,7 @@ void ClientKnobs::initialize(Randomize randomize) {
 
 	init( CONSISTENCY_CHECK_RATE_LIMIT_MAX,        50e6 ); // Limit in per sec
 	init( CONSISTENCY_CHECK_ONE_ROUND_TARGET_COMPLETION_TIME,	7 * 24 * 60 * 60 ); // 7 days
-	init( CONSISTENCY_CHECK_URGENT_MODE,           true ); if( randomize && BUGGIFY ) CONSISTENCY_CHECK_URGENT_MODE = false;
+	init( CONSISTENCY_CHECK_URGENT_MODE,           true );
 	init( CONSISTENCY_CHECK_DISTRIBUTED_WIGGLE_ROOM, 50 ); if( randomize && BUGGIFY ) CONSISTENCY_CHECK_DISTRIBUTED_WIGGLE_ROOM = 1;
 	init( CONSISTENCY_CHECK_RETRY_DEPTH_MAX,        100 ); if( randomize && BUGGIFY ) CONSISTENCY_CHECK_RETRY_DEPTH_MAX = 10;
 	init( CONSISTENCY_CHECK_RANGE_BEGIN,             "" ); if( randomize && BUGGIFY ) CONSISTENCY_CHECK_RANGE_BEGIN = "a";

From 392bb217baf3d9ece8663f0dfb3528e0342c16e6 Mon Sep 17 00:00:00 2001
From: Zhe Wang <zhe.wang@wustl.edu>
Date: Wed, 13 Dec 2023 16:00:30 -0800
Subject: [PATCH 329/468] input-multiple-ranges-to-check

---
 fdbclient/ClientKnobs.cpp                     |  32 +++++-
 fdbclient/ClientKnobs.h                       |  28 +++++
 .../workloads/ConsistencyCheck.actor.cpp      | 103 +++++++++++++++---
 3 files changed, 143 insertions(+), 20 deletions(-)

diff --git a/fdbclient/ClientKnobs.cpp b/fdbclient/ClientKnobs.cpp
index 05bfef5b814..b5e3b6ffc55 100644
--- a/fdbclient/ClientKnobs.cpp
+++ b/fdbclient/ClientKnobs.cpp
@@ -261,8 +261,36 @@ void ClientKnobs::initialize(Randomize randomize) {
 	init( CONSISTENCY_CHECK_URGENT_MODE,           true );
 	init( CONSISTENCY_CHECK_DISTRIBUTED_WIGGLE_ROOM, 50 ); if( randomize && BUGGIFY ) CONSISTENCY_CHECK_DISTRIBUTED_WIGGLE_ROOM = 1;
 	init( CONSISTENCY_CHECK_RETRY_DEPTH_MAX,        100 ); if( randomize && BUGGIFY ) CONSISTENCY_CHECK_RETRY_DEPTH_MAX = 10;
-	init( CONSISTENCY_CHECK_RANGE_BEGIN,             "" ); if( randomize && BUGGIFY ) CONSISTENCY_CHECK_RANGE_BEGIN = "a";
-	init( CONSISTENCY_CHECK_RANGE_END,       "\xff\xff" ); if( randomize && BUGGIFY ) CONSISTENCY_CHECK_RANGE_END = "z";
+	init( CONSISTENCY_CHECK_RANGE_BEGIN,             "" ); if( randomize && BUGGIFY ) CONSISTENCY_CHECK_RANGE_BEGIN = "";
+	init( CONSISTENCY_CHECK_RANGE_END,       "\xff\xff" ); if( randomize && BUGGIFY ) CONSISTENCY_CHECK_RANGE_END = "a";
+	init( CONSISTENCY_CHECK_RANGE_BEGIN_MORE_A,      "" ); if( randomize && BUGGIFY ) CONSISTENCY_CHECK_RANGE_BEGIN_MORE_A = "a";
+	init( CONSISTENCY_CHECK_RANGE_END_MORE_A,        "" ); if( randomize && BUGGIFY ) CONSISTENCY_CHECK_RANGE_END_MORE_A = "b";
+	init( CONSISTENCY_CHECK_RANGE_BEGIN_MORE_B,      "" ); if( randomize && BUGGIFY ) CONSISTENCY_CHECK_RANGE_BEGIN_MORE_B = "b";
+	init( CONSISTENCY_CHECK_RANGE_END_MORE_B,        "" ); if( randomize && BUGGIFY ) CONSISTENCY_CHECK_RANGE_END_MORE_B = "c";
+	init( CONSISTENCY_CHECK_RANGE_BEGIN_MORE_C,      "" ); if( randomize && BUGGIFY ) CONSISTENCY_CHECK_RANGE_BEGIN_MORE_C = "c";
+	init( CONSISTENCY_CHECK_RANGE_END_MORE_C,        "" ); if( randomize && BUGGIFY ) CONSISTENCY_CHECK_RANGE_END_MORE_C = "d";
+	init( CONSISTENCY_CHECK_RANGE_BEGIN_MORE_D,      "" ); if( randomize && BUGGIFY ) CONSISTENCY_CHECK_RANGE_BEGIN_MORE_D = "d";
+	init( CONSISTENCY_CHECK_RANGE_END_MORE_D,        "" ); if( randomize && BUGGIFY ) CONSISTENCY_CHECK_RANGE_END_MORE_D = "e";
+	init( CONSISTENCY_CHECK_RANGE_BEGIN_MORE_E,      "" ); if( randomize && BUGGIFY ) CONSISTENCY_CHECK_RANGE_BEGIN_MORE_E = "e";
+	init( CONSISTENCY_CHECK_RANGE_END_MORE_E,        "" ); if( randomize && BUGGIFY ) CONSISTENCY_CHECK_RANGE_END_MORE_E = "f";
+	init( CONSISTENCY_CHECK_RANGE_BEGIN_MORE_F,      "" ); if( randomize && BUGGIFY ) CONSISTENCY_CHECK_RANGE_BEGIN_MORE_F = "f";
+	init( CONSISTENCY_CHECK_RANGE_END_MORE_F,        "" ); if( randomize && BUGGIFY ) CONSISTENCY_CHECK_RANGE_END_MORE_F = "g";
+	init( CONSISTENCY_CHECK_RANGE_BEGIN_MORE_G,      "" ); if( randomize && BUGGIFY ) CONSISTENCY_CHECK_RANGE_BEGIN_MORE_G = "g";
+	init( CONSISTENCY_CHECK_RANGE_END_MORE_G,        "" ); if( randomize && BUGGIFY ) CONSISTENCY_CHECK_RANGE_END_MORE_G = "h";
+	init( CONSISTENCY_CHECK_RANGE_BEGIN_MORE_H,      "" ); if( randomize && BUGGIFY ) CONSISTENCY_CHECK_RANGE_BEGIN_MORE_H = "h";
+	init( CONSISTENCY_CHECK_RANGE_END_MORE_H,        "" ); if( randomize && BUGGIFY ) CONSISTENCY_CHECK_RANGE_END_MORE_H = "i";
+	init( CONSISTENCY_CHECK_RANGE_BEGIN_MORE_I,      "" ); if( randomize && BUGGIFY ) CONSISTENCY_CHECK_RANGE_BEGIN_MORE_I = "i";
+	init( CONSISTENCY_CHECK_RANGE_END_MORE_I,        "" ); if( randomize && BUGGIFY ) CONSISTENCY_CHECK_RANGE_END_MORE_I = "j";
+	init( CONSISTENCY_CHECK_RANGE_BEGIN_MORE_J,      "" ); if( randomize && BUGGIFY ) CONSISTENCY_CHECK_RANGE_BEGIN_MORE_J = "j";
+	init( CONSISTENCY_CHECK_RANGE_END_MORE_J,        "" ); if( randomize && BUGGIFY ) CONSISTENCY_CHECK_RANGE_END_MORE_J = "k";
+	init( CONSISTENCY_CHECK_RANGE_BEGIN_MORE_K,      "" ); if( randomize && BUGGIFY ) CONSISTENCY_CHECK_RANGE_BEGIN_MORE_K = "k";
+	init( CONSISTENCY_CHECK_RANGE_END_MORE_K,        "" ); if( randomize && BUGGIFY ) CONSISTENCY_CHECK_RANGE_END_MORE_K = "l";
+	init( CONSISTENCY_CHECK_RANGE_BEGIN_MORE_L,      "" ); if( randomize && BUGGIFY ) CONSISTENCY_CHECK_RANGE_BEGIN_MORE_L = "l";
+	init( CONSISTENCY_CHECK_RANGE_END_MORE_L,        "" ); if( randomize && BUGGIFY ) CONSISTENCY_CHECK_RANGE_END_MORE_L = "m";
+	init( CONSISTENCY_CHECK_RANGE_BEGIN_MORE_M,      "" ); if( randomize && BUGGIFY ) CONSISTENCY_CHECK_RANGE_BEGIN_MORE_M = "m";
+	init( CONSISTENCY_CHECK_RANGE_END_MORE_M,        "" ); if( randomize && BUGGIFY ) CONSISTENCY_CHECK_RANGE_END_MORE_M = "n";
+	init( CONSISTENCY_CHECK_RANGE_BEGIN_MORE_N,      "" ); if( randomize && BUGGIFY ) CONSISTENCY_CHECK_RANGE_BEGIN_MORE_N = "n";
+	init( CONSISTENCY_CHECK_RANGE_END_MORE_N,        "" ); if( randomize && BUGGIFY ) CONSISTENCY_CHECK_RANGE_END_MORE_N = "o";
 
 	//fdbcli
 	init( CLI_CONNECT_PARALLELISM,                  400 );
diff --git a/fdbclient/ClientKnobs.h b/fdbclient/ClientKnobs.h
index cc13d9e8e23..cbb7166f7b2 100644
--- a/fdbclient/ClientKnobs.h
+++ b/fdbclient/ClientKnobs.h
@@ -254,6 +254,34 @@ class ClientKnobs : public KnobsImpl<ClientKnobs> {
 	int CONSISTENCY_CHECK_DISTRIBUTED_WIGGLE_ROOM; // Available in urgent mode
 	std::string CONSISTENCY_CHECK_RANGE_BEGIN; // Available in urgent mode
 	std::string CONSISTENCY_CHECK_RANGE_END; // Available in urgent mode
+	std::string CONSISTENCY_CHECK_RANGE_BEGIN_MORE_A; // Available in urgent mode
+	std::string CONSISTENCY_CHECK_RANGE_END_MORE_A; // Available in urgent mode
+	std::string CONSISTENCY_CHECK_RANGE_BEGIN_MORE_B;
+	std::string CONSISTENCY_CHECK_RANGE_END_MORE_B;
+	std::string CONSISTENCY_CHECK_RANGE_BEGIN_MORE_C;
+	std::string CONSISTENCY_CHECK_RANGE_END_MORE_C;
+	std::string CONSISTENCY_CHECK_RANGE_BEGIN_MORE_D;
+	std::string CONSISTENCY_CHECK_RANGE_END_MORE_D;
+	std::string CONSISTENCY_CHECK_RANGE_BEGIN_MORE_E;
+	std::string CONSISTENCY_CHECK_RANGE_END_MORE_E;
+	std::string CONSISTENCY_CHECK_RANGE_BEGIN_MORE_F;
+	std::string CONSISTENCY_CHECK_RANGE_END_MORE_F;
+	std::string CONSISTENCY_CHECK_RANGE_BEGIN_MORE_G;
+	std::string CONSISTENCY_CHECK_RANGE_END_MORE_G;
+	std::string CONSISTENCY_CHECK_RANGE_BEGIN_MORE_H;
+	std::string CONSISTENCY_CHECK_RANGE_END_MORE_H;
+	std::string CONSISTENCY_CHECK_RANGE_BEGIN_MORE_I;
+	std::string CONSISTENCY_CHECK_RANGE_END_MORE_I;
+	std::string CONSISTENCY_CHECK_RANGE_BEGIN_MORE_J;
+	std::string CONSISTENCY_CHECK_RANGE_END_MORE_J;
+	std::string CONSISTENCY_CHECK_RANGE_BEGIN_MORE_K;
+	std::string CONSISTENCY_CHECK_RANGE_END_MORE_K;
+	std::string CONSISTENCY_CHECK_RANGE_BEGIN_MORE_L;
+	std::string CONSISTENCY_CHECK_RANGE_END_MORE_L;
+	std::string CONSISTENCY_CHECK_RANGE_BEGIN_MORE_M;
+	std::string CONSISTENCY_CHECK_RANGE_END_MORE_M;
+	std::string CONSISTENCY_CHECK_RANGE_BEGIN_MORE_N;
+	std::string CONSISTENCY_CHECK_RANGE_END_MORE_N;
 
 	// fdbcli
 	int CLI_CONNECT_PARALLELISM;
diff --git a/fdbserver/workloads/ConsistencyCheck.actor.cpp b/fdbserver/workloads/ConsistencyCheck.actor.cpp
index acc69182620..b408b1f80b3 100644
--- a/fdbserver/workloads/ConsistencyCheck.actor.cpp
+++ b/fdbserver/workloads/ConsistencyCheck.actor.cpp
@@ -223,6 +223,88 @@ struct ConsistencyCheckWorkload : TestWorkload {
 		return Void();
 	}
 
+	std::vector<KeyRange> loadRangesToCheckFromKnob() {
+		std::vector<Key> beginKeys = {
+			KeyRef(CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_BEGIN),
+			KeyRef(CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_BEGIN_MORE_A),
+			KeyRef(CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_BEGIN_MORE_B),
+			KeyRef(CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_BEGIN_MORE_C),
+			KeyRef(CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_BEGIN_MORE_D),
+			KeyRef(CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_BEGIN_MORE_E),
+			KeyRef(CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_BEGIN_MORE_F),
+			KeyRef(CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_BEGIN_MORE_G),
+			KeyRef(CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_BEGIN_MORE_H),
+			KeyRef(CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_BEGIN_MORE_I),
+			KeyRef(CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_BEGIN_MORE_J),
+			KeyRef(CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_BEGIN_MORE_K),
+			KeyRef(CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_BEGIN_MORE_L),
+			KeyRef(CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_BEGIN_MORE_M),
+			KeyRef(CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_BEGIN_MORE_N),
+		};
+		std::vector<Key> endKeys = {
+			KeyRef(CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_END),
+			KeyRef(CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_END_MORE_A),
+			KeyRef(CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_END_MORE_B),
+			KeyRef(CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_END_MORE_C),
+			KeyRef(CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_END_MORE_D),
+			KeyRef(CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_END_MORE_E),
+			KeyRef(CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_END_MORE_F),
+			KeyRef(CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_END_MORE_G),
+			KeyRef(CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_END_MORE_H),
+			KeyRef(CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_END_MORE_I),
+			KeyRef(CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_END_MORE_J),
+			KeyRef(CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_END_MORE_K),
+			KeyRef(CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_END_MORE_L),
+			KeyRef(CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_END_MORE_M),
+			KeyRef(CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_END_MORE_N),
+		};
+		ASSERT(beginKeys.size() == endKeys.size());
+
+		KeyRangeMap<bool> rangeToCheckMap;
+		for (int i = 0; i < beginKeys.size(); i++) {
+			Key rangeBegin = beginKeys[i];
+			Key rangeEnd = endKeys[i];
+			if (rangeBegin.empty() && rangeEnd.empty()) {
+				TraceEvent("ConsistencyCheck_EmptyInputRange")
+				    .detail("Index", i)
+				    .detail("RangeBegin", rangeBegin)
+				    .detail("RangeEnd", rangeEnd);
+				continue;
+			}
+			if (rangeBegin > allKeys.end) {
+				rangeBegin = allKeys.end;
+			}
+			if (rangeEnd > allKeys.end) {
+				rangeEnd = allKeys.end;
+			}
+
+			KeyRange rangeToCheck;
+			if (rangeBegin < rangeEnd) {
+				rangeToCheck = Standalone(KeyRangeRef(rangeBegin, rangeEnd));
+			} else if (rangeBegin > rangeEnd) {
+				rangeToCheck = Standalone(KeyRangeRef(rangeEnd, rangeBegin));
+			} else {
+				TraceEvent("ConsistencyCheck_EmptyInputRange")
+				    .detail("Index", i)
+				    .detail("RangeBegin", rangeBegin)
+				    .detail("RangeEnd", rangeEnd);
+				continue;
+			}
+			rangeToCheckMap.insert(rangeToCheck, true);
+		}
+
+		rangeToCheckMap.coalesce(allKeys);
+
+		std::vector<KeyRange> res;
+		for (auto& rangeToCheck : rangeToCheckMap.ranges()) {
+			if (rangeToCheck.value() == false) {
+				continue;
+			}
+			res.push_back(rangeToCheck.range());
+		}
+		return res;
+	}
+
 	ACTOR Future<Void> runCheck(Database cx, ConsistencyCheckWorkload* self) {
 		TEST(self->performQuiescentChecks); // Quiescent consistency check
 		TEST(!self->performQuiescentChecks); // Non-quiescent consistency check
@@ -337,26 +419,11 @@ struct ConsistencyCheckWorkload : TestWorkload {
 				}
 
 				if (CLIENT_KNOBS->CONSISTENCY_CHECK_URGENT_MODE) {
-					state KeyRange rangeToCheck = allKeys;
-					Key rangeBegin = StringRef(CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_BEGIN);
-					if (rangeBegin > allKeys.end) {
-						rangeBegin = allKeys.end;
-					}
-					Key rangeEnd = StringRef(CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_END);
-					if (rangeEnd > allKeys.end) {
-						rangeEnd = allKeys.end;
-					}
-					if (rangeBegin < rangeEnd) {
-						rangeToCheck = KeyRangeRef(rangeBegin, rangeEnd);
-					} else if (rangeBegin > rangeEnd) {
-						rangeToCheck = KeyRangeRef(rangeEnd, rangeBegin);
-					} else {
-						rangeToCheck = allKeys;
-					}
+					std::vector<KeyRange> rangesToCheck = self->loadRangesToCheckFromKnob();
 					TraceEvent("ConsistencyCheck_Config")
 					    .setMaxEventLength(-1)
 					    .setMaxFieldLength(-1)
-					    .detail("RangeToCheck", rangeToCheck)
+					    .detail("RangeToCheck", describe(rangesToCheck))
 					    .detail("Indefinite", self->indefinite)
 					    .detail("Distributed", self->distributed)
 					    .detail("ClientCount", self->clientCount)
@@ -365,7 +432,7 @@ struct ConsistencyCheckWorkload : TestWorkload {
 					    .detail("PerformTSSCheck", self->performTSSCheck)
 					    .detail("PerformCacheCheck", self->performCacheCheck);
 					state std::vector<std::pair<KeyRange, Value>> shardLocationPairListForDistributed =
-					    wait(self->getKeyLocationsForRangeList(cx, { rangeToCheck }));
+					    wait(self->getKeyLocationsForRangeList(cx, rangesToCheck));
 					// Check that each failed shard has the same data on all storage servers that it resides on
 					wait(::success(self->checkDataConsistency(cx,
 					                                          shardLocationPairListForDistributed,

From 33474187b79326f745ee29ccff6d6b76f2f00e22 Mon Sep 17 00:00:00 2001
From: Zhe Wang <zhe.wang@wustl.edu>
Date: Wed, 13 Dec 2023 16:27:40 -0800
Subject: [PATCH 330/468] comment change

---
 fdbserver/workloads/ConsistencyCheck.actor.cpp | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/fdbserver/workloads/ConsistencyCheck.actor.cpp b/fdbserver/workloads/ConsistencyCheck.actor.cpp
index b408b1f80b3..11580eaeff7 100644
--- a/fdbserver/workloads/ConsistencyCheck.actor.cpp
+++ b/fdbserver/workloads/ConsistencyCheck.actor.cpp
@@ -1930,8 +1930,7 @@ struct ConsistencyCheckWorkload : TestWorkload {
 						}
 
 						if (CLIENT_KNOBS->CONSISTENCY_CHECK_URGENT_MODE && !completeCheck) {
-							// Add remaining range of the shard to failedRanges,
-							// which will be retried later
+							// Add the shard to failedRanges, which will be retried later
 							failedRanges.insert(range, true);
 							TraceEvent(SevInfo, "ConsistencyCheck_ShardAddedToRetry")
 							    .setMaxEventLength(-1)

From e876eeabc5da07c560f0437b2e1dfc55f7f25993 Mon Sep 17 00:00:00 2001
From: Zhe Wang <zhe.wang@wustl.edu>
Date: Thu, 14 Dec 2023 10:37:04 -0800
Subject: [PATCH 331/468] fix user input range

---
 fdbclient/ClientKnobs.cpp                     |  86 ++++--
 fdbclient/ClientKnobs.h                       |  84 ++++--
 .../workloads/ConsistencyCheck.actor.cpp      | 284 ++++++++++++------
 3 files changed, 301 insertions(+), 153 deletions(-)

diff --git a/fdbclient/ClientKnobs.cpp b/fdbclient/ClientKnobs.cpp
index b5e3b6ffc55..f1a649af087 100644
--- a/fdbclient/ClientKnobs.cpp
+++ b/fdbclient/ClientKnobs.cpp
@@ -259,38 +259,62 @@ void ClientKnobs::initialize(Randomize randomize) {
 	init( CONSISTENCY_CHECK_RATE_LIMIT_MAX,        50e6 ); // Limit in per sec
 	init( CONSISTENCY_CHECK_ONE_ROUND_TARGET_COMPLETION_TIME,	7 * 24 * 60 * 60 ); // 7 days
 	init( CONSISTENCY_CHECK_URGENT_MODE,           true );
-	init( CONSISTENCY_CHECK_DISTRIBUTED_WIGGLE_ROOM, 50 ); if( randomize && BUGGIFY ) CONSISTENCY_CHECK_DISTRIBUTED_WIGGLE_ROOM = 1;
+	init( CONSISTENCY_CHECK_DISTRIBUTED_WIGGLE_ROOM,  0 ); if( randomize && BUGGIFY ) CONSISTENCY_CHECK_DISTRIBUTED_WIGGLE_ROOM = 1;
 	init( CONSISTENCY_CHECK_RETRY_DEPTH_MAX,        100 ); if( randomize && BUGGIFY ) CONSISTENCY_CHECK_RETRY_DEPTH_MAX = 10;
-	init( CONSISTENCY_CHECK_RANGE_BEGIN,             "" ); if( randomize && BUGGIFY ) CONSISTENCY_CHECK_RANGE_BEGIN = "";
-	init( CONSISTENCY_CHECK_RANGE_END,       "\xff\xff" ); if( randomize && BUGGIFY ) CONSISTENCY_CHECK_RANGE_END = "a";
-	init( CONSISTENCY_CHECK_RANGE_BEGIN_MORE_A,      "" ); if( randomize && BUGGIFY ) CONSISTENCY_CHECK_RANGE_BEGIN_MORE_A = "a";
-	init( CONSISTENCY_CHECK_RANGE_END_MORE_A,        "" ); if( randomize && BUGGIFY ) CONSISTENCY_CHECK_RANGE_END_MORE_A = "b";
-	init( CONSISTENCY_CHECK_RANGE_BEGIN_MORE_B,      "" ); if( randomize && BUGGIFY ) CONSISTENCY_CHECK_RANGE_BEGIN_MORE_B = "b";
-	init( CONSISTENCY_CHECK_RANGE_END_MORE_B,        "" ); if( randomize && BUGGIFY ) CONSISTENCY_CHECK_RANGE_END_MORE_B = "c";
-	init( CONSISTENCY_CHECK_RANGE_BEGIN_MORE_C,      "" ); if( randomize && BUGGIFY ) CONSISTENCY_CHECK_RANGE_BEGIN_MORE_C = "c";
-	init( CONSISTENCY_CHECK_RANGE_END_MORE_C,        "" ); if( randomize && BUGGIFY ) CONSISTENCY_CHECK_RANGE_END_MORE_C = "d";
-	init( CONSISTENCY_CHECK_RANGE_BEGIN_MORE_D,      "" ); if( randomize && BUGGIFY ) CONSISTENCY_CHECK_RANGE_BEGIN_MORE_D = "d";
-	init( CONSISTENCY_CHECK_RANGE_END_MORE_D,        "" ); if( randomize && BUGGIFY ) CONSISTENCY_CHECK_RANGE_END_MORE_D = "e";
-	init( CONSISTENCY_CHECK_RANGE_BEGIN_MORE_E,      "" ); if( randomize && BUGGIFY ) CONSISTENCY_CHECK_RANGE_BEGIN_MORE_E = "e";
-	init( CONSISTENCY_CHECK_RANGE_END_MORE_E,        "" ); if( randomize && BUGGIFY ) CONSISTENCY_CHECK_RANGE_END_MORE_E = "f";
-	init( CONSISTENCY_CHECK_RANGE_BEGIN_MORE_F,      "" ); if( randomize && BUGGIFY ) CONSISTENCY_CHECK_RANGE_BEGIN_MORE_F = "f";
-	init( CONSISTENCY_CHECK_RANGE_END_MORE_F,        "" ); if( randomize && BUGGIFY ) CONSISTENCY_CHECK_RANGE_END_MORE_F = "g";
-	init( CONSISTENCY_CHECK_RANGE_BEGIN_MORE_G,      "" ); if( randomize && BUGGIFY ) CONSISTENCY_CHECK_RANGE_BEGIN_MORE_G = "g";
-	init( CONSISTENCY_CHECK_RANGE_END_MORE_G,        "" ); if( randomize && BUGGIFY ) CONSISTENCY_CHECK_RANGE_END_MORE_G = "h";
-	init( CONSISTENCY_CHECK_RANGE_BEGIN_MORE_H,      "" ); if( randomize && BUGGIFY ) CONSISTENCY_CHECK_RANGE_BEGIN_MORE_H = "h";
-	init( CONSISTENCY_CHECK_RANGE_END_MORE_H,        "" ); if( randomize && BUGGIFY ) CONSISTENCY_CHECK_RANGE_END_MORE_H = "i";
-	init( CONSISTENCY_CHECK_RANGE_BEGIN_MORE_I,      "" ); if( randomize && BUGGIFY ) CONSISTENCY_CHECK_RANGE_BEGIN_MORE_I = "i";
-	init( CONSISTENCY_CHECK_RANGE_END_MORE_I,        "" ); if( randomize && BUGGIFY ) CONSISTENCY_CHECK_RANGE_END_MORE_I = "j";
-	init( CONSISTENCY_CHECK_RANGE_BEGIN_MORE_J,      "" ); if( randomize && BUGGIFY ) CONSISTENCY_CHECK_RANGE_BEGIN_MORE_J = "j";
-	init( CONSISTENCY_CHECK_RANGE_END_MORE_J,        "" ); if( randomize && BUGGIFY ) CONSISTENCY_CHECK_RANGE_END_MORE_J = "k";
-	init( CONSISTENCY_CHECK_RANGE_BEGIN_MORE_K,      "" ); if( randomize && BUGGIFY ) CONSISTENCY_CHECK_RANGE_BEGIN_MORE_K = "k";
-	init( CONSISTENCY_CHECK_RANGE_END_MORE_K,        "" ); if( randomize && BUGGIFY ) CONSISTENCY_CHECK_RANGE_END_MORE_K = "l";
-	init( CONSISTENCY_CHECK_RANGE_BEGIN_MORE_L,      "" ); if( randomize && BUGGIFY ) CONSISTENCY_CHECK_RANGE_BEGIN_MORE_L = "l";
-	init( CONSISTENCY_CHECK_RANGE_END_MORE_L,        "" ); if( randomize && BUGGIFY ) CONSISTENCY_CHECK_RANGE_END_MORE_L = "m";
-	init( CONSISTENCY_CHECK_RANGE_BEGIN_MORE_M,      "" ); if( randomize && BUGGIFY ) CONSISTENCY_CHECK_RANGE_BEGIN_MORE_M = "m";
-	init( CONSISTENCY_CHECK_RANGE_END_MORE_M,        "" ); if( randomize && BUGGIFY ) CONSISTENCY_CHECK_RANGE_END_MORE_M = "n";
-	init( CONSISTENCY_CHECK_RANGE_BEGIN_MORE_N,      "" ); if( randomize && BUGGIFY ) CONSISTENCY_CHECK_RANGE_BEGIN_MORE_N = "n";
-	init( CONSISTENCY_CHECK_RANGE_END_MORE_N,        "" ); if( randomize && BUGGIFY ) CONSISTENCY_CHECK_RANGE_END_MORE_N = "o";
+	init( CONSISTENCY_CHECK_RANGE_BEGIN_0,           "" ); if( randomize && BUGGIFY ) CONSISTENCY_CHECK_RANGE_BEGIN_0 = "";
+	init( CONSISTENCY_CHECK_RANGE_END_0,   "\\xff\\xff" ); if( randomize && BUGGIFY ) CONSISTENCY_CHECK_RANGE_END_0 = "\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x33\\x66\\x63\\x36";
+	init( CONSISTENCY_CHECK_RANGE_BEGIN_1,           "" );
+	init( CONSISTENCY_CHECK_RANGE_END_1,             "" );
+	init( CONSISTENCY_CHECK_RANGE_BEGIN_2,           "" );
+	init( CONSISTENCY_CHECK_RANGE_END_2,             "" );
+	init( CONSISTENCY_CHECK_RANGE_BEGIN_3,           "" );
+	init( CONSISTENCY_CHECK_RANGE_END_3,             "" );
+	init( CONSISTENCY_CHECK_RANGE_BEGIN_4,           "" );
+	init( CONSISTENCY_CHECK_RANGE_END_4,             "" );
+	init( CONSISTENCY_CHECK_RANGE_BEGIN_5,           "" );
+	init( CONSISTENCY_CHECK_RANGE_END_5,             "" );
+	init( CONSISTENCY_CHECK_RANGE_BEGIN_6,           "" ); 
+	init( CONSISTENCY_CHECK_RANGE_END_6,             "" ); 
+	init( CONSISTENCY_CHECK_RANGE_BEGIN_7,           "" );
+	init( CONSISTENCY_CHECK_RANGE_END_7,             "" );
+	init( CONSISTENCY_CHECK_RANGE_BEGIN_8,           "" );
+	init( CONSISTENCY_CHECK_RANGE_END_8,             "" );
+	init( CONSISTENCY_CHECK_RANGE_BEGIN_9,           "" );
+	init( CONSISTENCY_CHECK_RANGE_END_9,             "" );
+	init( CONSISTENCY_CHECK_RANGE_BEGIN_10,          "" );
+	init( CONSISTENCY_CHECK_RANGE_END_10,            "" );
+	init( CONSISTENCY_CHECK_RANGE_BEGIN_11,          "" );
+	init( CONSISTENCY_CHECK_RANGE_END_11,            "" );
+	init( CONSISTENCY_CHECK_RANGE_BEGIN_12,          "" );
+	init( CONSISTENCY_CHECK_RANGE_END_12,            "" );
+	init( CONSISTENCY_CHECK_RANGE_BEGIN_13,          "" );
+	init( CONSISTENCY_CHECK_RANGE_END_13,            "" );
+	init( CONSISTENCY_CHECK_RANGE_BEGIN_14,          "" );
+	init( CONSISTENCY_CHECK_RANGE_END_14,            "" );
+	init( CONSISTENCY_CHECK_RANGE_BEGIN_15,          "" );
+	init( CONSISTENCY_CHECK_RANGE_END_15,            "" );
+	init( CONSISTENCY_CHECK_RANGE_BEGIN_16,          "" );
+	init( CONSISTENCY_CHECK_RANGE_END_16,            "" );
+	init( CONSISTENCY_CHECK_RANGE_BEGIN_17,          "" );
+	init( CONSISTENCY_CHECK_RANGE_END_17,            "" );
+	init( CONSISTENCY_CHECK_RANGE_BEGIN_18,          "" );
+	init( CONSISTENCY_CHECK_RANGE_END_18,            "" );
+	init( CONSISTENCY_CHECK_RANGE_BEGIN_19,          "" );
+	init( CONSISTENCY_CHECK_RANGE_END_19,            "" );
+	init( CONSISTENCY_CHECK_RANGE_BEGIN_20,          "" );
+	init( CONSISTENCY_CHECK_RANGE_END_20,            "" );
+	init( CONSISTENCY_CHECK_RANGE_BEGIN_21,          "" );
+	init( CONSISTENCY_CHECK_RANGE_END_21,            "" );
+	init( CONSISTENCY_CHECK_RANGE_BEGIN_22,          "" );
+	init( CONSISTENCY_CHECK_RANGE_END_22,            "" );
+	init( CONSISTENCY_CHECK_RANGE_BEGIN_23,          "" );
+	init( CONSISTENCY_CHECK_RANGE_END_23,            "" );
+	init( CONSISTENCY_CHECK_RANGE_BEGIN_24,          "" );
+	init( CONSISTENCY_CHECK_RANGE_END_24,            "" );
+	init( CONSISTENCY_CHECK_RANGE_BEGIN_25,          "" );
+	init( CONSISTENCY_CHECK_RANGE_END_25,            "" );
+	init( CONSISTENCY_CHECK_RANGE_BEGIN_26,          "" ); if( randomize && BUGGIFY ) CONSISTENCY_CHECK_RANGE_BEGIN_26 = "\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x33\\x66\\x65\\x34\\x63\\x62";
+	init( CONSISTENCY_CHECK_RANGE_END_26,            "" ); if( randomize && BUGGIFY ) CONSISTENCY_CHECK_RANGE_END_26 = "\\xff\\xff";
 
 	//fdbcli
 	init( CLI_CONNECT_PARALLELISM,                  400 );
diff --git a/fdbclient/ClientKnobs.h b/fdbclient/ClientKnobs.h
index cbb7166f7b2..ccd74e9dd5a 100644
--- a/fdbclient/ClientKnobs.h
+++ b/fdbclient/ClientKnobs.h
@@ -252,36 +252,60 @@ class ClientKnobs : public KnobsImpl<ClientKnobs> {
 	bool CONSISTENCY_CHECK_URGENT_MODE; // Require Tester Process::Class
 	int CONSISTENCY_CHECK_RETRY_DEPTH_MAX; // Available in urgent mode
 	int CONSISTENCY_CHECK_DISTRIBUTED_WIGGLE_ROOM; // Available in urgent mode
-	std::string CONSISTENCY_CHECK_RANGE_BEGIN; // Available in urgent mode
-	std::string CONSISTENCY_CHECK_RANGE_END; // Available in urgent mode
-	std::string CONSISTENCY_CHECK_RANGE_BEGIN_MORE_A; // Available in urgent mode
-	std::string CONSISTENCY_CHECK_RANGE_END_MORE_A; // Available in urgent mode
-	std::string CONSISTENCY_CHECK_RANGE_BEGIN_MORE_B;
-	std::string CONSISTENCY_CHECK_RANGE_END_MORE_B;
-	std::string CONSISTENCY_CHECK_RANGE_BEGIN_MORE_C;
-	std::string CONSISTENCY_CHECK_RANGE_END_MORE_C;
-	std::string CONSISTENCY_CHECK_RANGE_BEGIN_MORE_D;
-	std::string CONSISTENCY_CHECK_RANGE_END_MORE_D;
-	std::string CONSISTENCY_CHECK_RANGE_BEGIN_MORE_E;
-	std::string CONSISTENCY_CHECK_RANGE_END_MORE_E;
-	std::string CONSISTENCY_CHECK_RANGE_BEGIN_MORE_F;
-	std::string CONSISTENCY_CHECK_RANGE_END_MORE_F;
-	std::string CONSISTENCY_CHECK_RANGE_BEGIN_MORE_G;
-	std::string CONSISTENCY_CHECK_RANGE_END_MORE_G;
-	std::string CONSISTENCY_CHECK_RANGE_BEGIN_MORE_H;
-	std::string CONSISTENCY_CHECK_RANGE_END_MORE_H;
-	std::string CONSISTENCY_CHECK_RANGE_BEGIN_MORE_I;
-	std::string CONSISTENCY_CHECK_RANGE_END_MORE_I;
-	std::string CONSISTENCY_CHECK_RANGE_BEGIN_MORE_J;
-	std::string CONSISTENCY_CHECK_RANGE_END_MORE_J;
-	std::string CONSISTENCY_CHECK_RANGE_BEGIN_MORE_K;
-	std::string CONSISTENCY_CHECK_RANGE_END_MORE_K;
-	std::string CONSISTENCY_CHECK_RANGE_BEGIN_MORE_L;
-	std::string CONSISTENCY_CHECK_RANGE_END_MORE_L;
-	std::string CONSISTENCY_CHECK_RANGE_BEGIN_MORE_M;
-	std::string CONSISTENCY_CHECK_RANGE_END_MORE_M;
-	std::string CONSISTENCY_CHECK_RANGE_BEGIN_MORE_N;
-	std::string CONSISTENCY_CHECK_RANGE_END_MORE_N;
+	std::string CONSISTENCY_CHECK_RANGE_BEGIN_0; // Available in urgent mode
+	std::string CONSISTENCY_CHECK_RANGE_END_0; // Available in urgent mode
+	std::string CONSISTENCY_CHECK_RANGE_BEGIN_1; // Available in urgent mode
+	std::string CONSISTENCY_CHECK_RANGE_END_1; // Available in urgent mode
+	std::string CONSISTENCY_CHECK_RANGE_BEGIN_2;
+	std::string CONSISTENCY_CHECK_RANGE_END_2;
+	std::string CONSISTENCY_CHECK_RANGE_BEGIN_3;
+	std::string CONSISTENCY_CHECK_RANGE_END_3;
+	std::string CONSISTENCY_CHECK_RANGE_BEGIN_4;
+	std::string CONSISTENCY_CHECK_RANGE_END_4;
+	std::string CONSISTENCY_CHECK_RANGE_BEGIN_5;
+	std::string CONSISTENCY_CHECK_RANGE_END_5;
+	std::string CONSISTENCY_CHECK_RANGE_BEGIN_6;
+	std::string CONSISTENCY_CHECK_RANGE_END_6;
+	std::string CONSISTENCY_CHECK_RANGE_BEGIN_7;
+	std::string CONSISTENCY_CHECK_RANGE_END_7;
+	std::string CONSISTENCY_CHECK_RANGE_BEGIN_8;
+	std::string CONSISTENCY_CHECK_RANGE_END_8;
+	std::string CONSISTENCY_CHECK_RANGE_BEGIN_9;
+	std::string CONSISTENCY_CHECK_RANGE_END_9;
+	std::string CONSISTENCY_CHECK_RANGE_BEGIN_10;
+	std::string CONSISTENCY_CHECK_RANGE_END_10;
+	std::string CONSISTENCY_CHECK_RANGE_BEGIN_11;
+	std::string CONSISTENCY_CHECK_RANGE_END_11;
+	std::string CONSISTENCY_CHECK_RANGE_BEGIN_12;
+	std::string CONSISTENCY_CHECK_RANGE_END_12;
+	std::string CONSISTENCY_CHECK_RANGE_BEGIN_13;
+	std::string CONSISTENCY_CHECK_RANGE_END_13;
+	std::string CONSISTENCY_CHECK_RANGE_BEGIN_14;
+	std::string CONSISTENCY_CHECK_RANGE_END_14;
+	std::string CONSISTENCY_CHECK_RANGE_BEGIN_15;
+	std::string CONSISTENCY_CHECK_RANGE_END_15;
+	std::string CONSISTENCY_CHECK_RANGE_BEGIN_16;
+	std::string CONSISTENCY_CHECK_RANGE_END_16;
+	std::string CONSISTENCY_CHECK_RANGE_BEGIN_17;
+	std::string CONSISTENCY_CHECK_RANGE_END_17;
+	std::string CONSISTENCY_CHECK_RANGE_BEGIN_18;
+	std::string CONSISTENCY_CHECK_RANGE_END_18;
+	std::string CONSISTENCY_CHECK_RANGE_BEGIN_19;
+	std::string CONSISTENCY_CHECK_RANGE_END_19;
+	std::string CONSISTENCY_CHECK_RANGE_BEGIN_20;
+	std::string CONSISTENCY_CHECK_RANGE_END_20;
+	std::string CONSISTENCY_CHECK_RANGE_BEGIN_21;
+	std::string CONSISTENCY_CHECK_RANGE_END_21;
+	std::string CONSISTENCY_CHECK_RANGE_BEGIN_22;
+	std::string CONSISTENCY_CHECK_RANGE_END_22;
+	std::string CONSISTENCY_CHECK_RANGE_BEGIN_23;
+	std::string CONSISTENCY_CHECK_RANGE_END_23;
+	std::string CONSISTENCY_CHECK_RANGE_BEGIN_24;
+	std::string CONSISTENCY_CHECK_RANGE_END_24;
+	std::string CONSISTENCY_CHECK_RANGE_BEGIN_25;
+	std::string CONSISTENCY_CHECK_RANGE_END_25;
+	std::string CONSISTENCY_CHECK_RANGE_BEGIN_26;
+	std::string CONSISTENCY_CHECK_RANGE_END_26;
 
 	// fdbcli
 	int CLI_CONNECT_PARALLELISM;
diff --git a/fdbserver/workloads/ConsistencyCheck.actor.cpp b/fdbserver/workloads/ConsistencyCheck.actor.cpp
index 11580eaeff7..4fbf91e770d 100644
--- a/fdbserver/workloads/ConsistencyCheck.actor.cpp
+++ b/fdbserver/workloads/ConsistencyCheck.actor.cpp
@@ -43,6 +43,51 @@
 // #define SevCCheckInfo SevVerbose
 #define SevCCheckInfo SevInfo
 
+const std::unordered_map<char, uint8_t> parseCharMap{
+	{ '0', 0 },  { '1', 1 },  { '2', 2 },  { '3', 3 },  { '4', 4 },  { '5', 5 },  { '6', 6 },  { '7', 7 },
+	{ '8', 8 },  { '9', 9 },  { 'a', 10 }, { 'b', 11 }, { 'c', 12 }, { 'd', 13 }, { 'e', 14 }, { 'f', 15 },
+	{ 'A', 10 }, { 'B', 11 }, { 'C', 12 }, { 'D', 13 }, { 'E', 14 }, { 'F', 15 },
+};
+
+Key getKeyFromString(const std::string& str) {
+	Key emptyKey;
+	if (str.size() == 0) {
+		return emptyKey;
+	}
+	if (str.size() % 4 != 0) {
+		TraceEvent(SevWarnAlways, "ConsistencyCheck_GetKeyFromStringError")
+		    .setMaxEventLength(-1)
+		    .setMaxFieldLength(-1)
+		    .detail("Reason", "WrongLength")
+		    .detail("InputStr", str);
+		throw internal_error();
+	}
+	std::vector<uint8_t> byteList;
+	for (int i = 0; i < str.size(); i += 4) {
+		if (str.at(i + 0) != '\\' || str.at(i + 1) != 'x') {
+			TraceEvent(SevWarnAlways, "ConsistencyCheck_GetKeyFromStringError")
+			    .setMaxEventLength(-1)
+			    .setMaxFieldLength(-1)
+			    .detail("Reason", "WrongBytePrefix")
+			    .detail("InputStr", str);
+			throw internal_error();
+		}
+		const char first = str.at(i + 2);
+		const char second = str.at(i + 3);
+		if (parseCharMap.count(first) == 0 || parseCharMap.count(second) == 0) {
+			TraceEvent(SevWarnAlways, "ConsistencyCheck_GetKeyFromStringError")
+			    .setMaxEventLength(-1)
+			    .setMaxFieldLength(-1)
+			    .detail("Reason", "WrongByteContent")
+			    .detail("InputStr", str);
+			throw internal_error();
+		}
+		uint8_t parsedValue = parseCharMap.at(first) * 16 + parseCharMap.at(second);
+		byteList.push_back(parsedValue);
+	}
+	return Standalone(StringRef(byteList.data(), byteList.size()));
+}
+
 struct ConsistencyCheckWorkload : TestWorkload {
 	// Whether or not we should perform checks that will only pass if the database is in a quiescent state
 	bool performQuiescentChecks;
@@ -224,57 +269,71 @@ struct ConsistencyCheckWorkload : TestWorkload {
 	}
 
 	std::vector<KeyRange> loadRangesToCheckFromKnob() {
-		std::vector<Key> beginKeys = {
-			KeyRef(CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_BEGIN),
-			KeyRef(CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_BEGIN_MORE_A),
-			KeyRef(CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_BEGIN_MORE_B),
-			KeyRef(CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_BEGIN_MORE_C),
-			KeyRef(CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_BEGIN_MORE_D),
-			KeyRef(CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_BEGIN_MORE_E),
-			KeyRef(CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_BEGIN_MORE_F),
-			KeyRef(CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_BEGIN_MORE_G),
-			KeyRef(CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_BEGIN_MORE_H),
-			KeyRef(CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_BEGIN_MORE_I),
-			KeyRef(CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_BEGIN_MORE_J),
-			KeyRef(CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_BEGIN_MORE_K),
-			KeyRef(CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_BEGIN_MORE_L),
-			KeyRef(CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_BEGIN_MORE_M),
-			KeyRef(CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_BEGIN_MORE_N),
+		// Load string from knob
+		std::vector<std::string> beginKeyStrs = {
+			CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_BEGIN_0,  CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_BEGIN_1,
+			CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_BEGIN_2,  CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_BEGIN_3,
+			CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_BEGIN_4,  CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_BEGIN_5,
+			CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_BEGIN_6,  CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_BEGIN_7,
+			CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_BEGIN_8,  CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_BEGIN_9,
+			CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_BEGIN_10, CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_BEGIN_11,
+			CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_BEGIN_12, CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_BEGIN_13,
+			CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_BEGIN_14, CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_BEGIN_15,
+			CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_BEGIN_16, CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_BEGIN_17,
+			CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_BEGIN_18, CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_BEGIN_19,
+			CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_BEGIN_20, CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_BEGIN_21,
+			CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_BEGIN_22, CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_BEGIN_23,
+			CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_BEGIN_24, CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_BEGIN_25,
+			CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_BEGIN_26,
 		};
-		std::vector<Key> endKeys = {
-			KeyRef(CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_END),
-			KeyRef(CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_END_MORE_A),
-			KeyRef(CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_END_MORE_B),
-			KeyRef(CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_END_MORE_C),
-			KeyRef(CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_END_MORE_D),
-			KeyRef(CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_END_MORE_E),
-			KeyRef(CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_END_MORE_F),
-			KeyRef(CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_END_MORE_G),
-			KeyRef(CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_END_MORE_H),
-			KeyRef(CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_END_MORE_I),
-			KeyRef(CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_END_MORE_J),
-			KeyRef(CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_END_MORE_K),
-			KeyRef(CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_END_MORE_L),
-			KeyRef(CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_END_MORE_M),
-			KeyRef(CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_END_MORE_N),
+		std::vector<std::string> endKeyStrs = {
+			CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_END_0,  CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_END_1,
+			CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_END_2,  CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_END_3,
+			CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_END_4,  CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_END_5,
+			CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_END_6,  CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_END_7,
+			CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_END_8,  CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_END_9,
+			CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_END_10, CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_END_11,
+			CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_END_12, CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_END_13,
+			CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_END_14, CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_END_15,
+			CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_END_16, CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_END_17,
+			CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_END_18, CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_END_19,
+			CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_END_20, CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_END_21,
+			CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_END_22, CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_END_23,
+			CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_END_24, CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_END_25,
+			CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_END_26,
 		};
+
+		// Get keys from strings
+		std::vector<Key> beginKeys;
+		for (const auto& beginKeyStr : beginKeyStrs) {
+			Key key = getKeyFromString(beginKeyStr);
+			beginKeys.push_back(key);
+		}
+		std::vector<Key> endKeys;
+		for (const auto& endKeyStr : endKeyStrs) {
+			Key key = getKeyFromString(endKeyStr);
+			endKeys.push_back(key);
+		}
 		ASSERT(beginKeys.size() == endKeys.size());
 
+		// Get ranges
 		KeyRangeMap<bool> rangeToCheckMap;
 		for (int i = 0; i < beginKeys.size(); i++) {
 			Key rangeBegin = beginKeys[i];
 			Key rangeEnd = endKeys[i];
 			if (rangeBegin.empty() && rangeEnd.empty()) {
-				TraceEvent("ConsistencyCheck_EmptyInputRange")
-				    .detail("Index", i)
-				    .detail("RangeBegin", rangeBegin)
-				    .detail("RangeEnd", rangeEnd);
 				continue;
 			}
 			if (rangeBegin > allKeys.end) {
 				rangeBegin = allKeys.end;
 			}
 			if (rangeEnd > allKeys.end) {
+				TraceEvent("ConsistencyCheck_ReverseInputRange")
+				    .setMaxEventLength(-1)
+				    .setMaxFieldLength(-1)
+				    .detail("Index", i)
+				    .detail("RangeBegin", rangeBegin)
+				    .detail("RangeEnd", rangeEnd);
 				rangeEnd = allKeys.end;
 			}
 
@@ -285,6 +344,8 @@ struct ConsistencyCheckWorkload : TestWorkload {
 				rangeToCheck = Standalone(KeyRangeRef(rangeEnd, rangeBegin));
 			} else {
 				TraceEvent("ConsistencyCheck_EmptyInputRange")
+				    .setMaxEventLength(-1)
+				    .setMaxFieldLength(-1)
 				    .detail("Index", i)
 				    .detail("RangeBegin", rangeBegin)
 				    .detail("RangeEnd", rangeEnd);
@@ -296,11 +357,17 @@ struct ConsistencyCheckWorkload : TestWorkload {
 		rangeToCheckMap.coalesce(allKeys);
 
 		std::vector<KeyRange> res;
-		for (auto& rangeToCheck : rangeToCheckMap.ranges()) {
-			if (rangeToCheck.value() == false) {
-				continue;
+		for (auto rangeToCheck : rangeToCheckMap.ranges()) {
+			if (rangeToCheck.value() == true) {
+				res.push_back(rangeToCheck.range());
 			}
-			res.push_back(rangeToCheck.range());
+		}
+		TraceEvent e("ConsistencyCheck_LoadedInputRange");
+		e.setMaxEventLength(-1);
+		e.setMaxFieldLength(-1);
+		for (int i = 0; i < res.size(); i++) {
+			e.detail("RangeBegin" + std::to_string(i), res[i].begin);
+			e.detail("RangeEnd" + std::to_string(i), res[i].end);
 		}
 		return res;
 	}
@@ -432,7 +499,7 @@ struct ConsistencyCheckWorkload : TestWorkload {
 					    .detail("PerformTSSCheck", self->performTSSCheck)
 					    .detail("PerformCacheCheck", self->performCacheCheck);
 					state std::vector<std::pair<KeyRange, Value>> shardLocationPairListForDistributed =
-					    wait(self->getKeyLocationsForRangeList(cx, rangesToCheck));
+					    wait(self->getKeyLocationsForRangeList(cx, rangesToCheck, self));
 					// Check that each failed shard has the same data on all storage servers that it resides on
 					wait(::success(self->checkDataConsistency(cx,
 					                                          shardLocationPairListForDistributed,
@@ -1205,18 +1272,25 @@ struct ConsistencyCheckWorkload : TestWorkload {
 	}
 
 	ACTOR Future<std::vector<std::pair<KeyRange, Value>>> getKeyLocationsForRangeList(Database cx,
-	                                                                                  std::vector<KeyRange> ranges) {
+	                                                                                  std::vector<KeyRange> ranges,
+	                                                                                  ConsistencyCheckWorkload* self) {
 		// Get the scope of the input list of ranges
 		state Key beginKeyToReadKeyServer;
 		state Key endKeyToReadKeyServer;
-		for (const auto& range : ranges) {
-			if (beginKeyToReadKeyServer.empty() || range.begin < beginKeyToReadKeyServer) {
-				beginKeyToReadKeyServer = range.begin;
+		for (int i = 0; i < ranges.size(); i++) {
+			if (i == 0 || ranges[i].begin < beginKeyToReadKeyServer) {
+				beginKeyToReadKeyServer = ranges[i].begin;
 			}
-			if (endKeyToReadKeyServer.empty() || range.end > endKeyToReadKeyServer) {
-				endKeyToReadKeyServer = range.end;
+			if (i == 0 || ranges[i].end > endKeyToReadKeyServer) {
+				endKeyToReadKeyServer = ranges[i].end;
 			}
 		}
+		TraceEvent("ConsistencyCheck_GetKeyLocationsForRangeList")
+		    .detail("Ops", "Input")
+		    .detail("RangeBegin", beginKeyToReadKeyServer)
+		    .detail("RangeEnd", endKeyToReadKeyServer)
+		    .detail("ClientCount", self->clientCount)
+		    .detail("ClientId", self->clientId);
 		// Read KeyServer space within the scope and add shards intersecting with the input ranges
 		state std::vector<std::pair<KeyRange, Value>> res;
 		state Transaction tr(cx);
@@ -1247,7 +1321,10 @@ struct ConsistencyCheckWorkload : TestWorkload {
 					break;
 				}
 			} catch (Error& e) {
-				TraceEvent("ConsistencyCheck_RetryGetKeyLocationsForRangeList").error(e);
+				TraceEvent("ConsistencyCheck_RetryGetKeyLocationsForRangeList")
+				    .error(e)
+				    .detail("ClientCount", self->clientCount)
+				    .detail("ClientId", self->clientId);
 				wait(tr.onError(e));
 			}
 		}
@@ -1430,7 +1507,10 @@ struct ConsistencyCheckWorkload : TestWorkload {
 		state double rateLimiterStartTime = now();
 		state int64_t bytesReadInthisRound = 0;
 		state bool testResult = true;
+		state int64_t numShardThisClient = -1;
+		state int64_t numShardToCheck = -1;
 		state int64_t numCompleteShards = 0;
+		state int64_t numFailedShards = 0;
 		state int64_t numSkippedShards = 0;
 		state KeyRangeMap<bool> failedRanges; // Used to collect failed ranges in the current checkDataConsistency
 		// Which will be used to start the next consistencyCheckEpoch of the checkDataConsistency
@@ -1447,9 +1527,6 @@ struct ConsistencyCheckWorkload : TestWorkload {
 			ranges.push_back(shardLocationPairList[k].first); // Add shard range to ranges
 		}
 
-		state Key beginKeyToCheck = ranges[0].begin;
-		state Key endKeyToCheck = ranges[ranges.size() - 1].end;
-
 		state std::vector<int> shardOrder;
 		shardOrder.reserve(ranges.size());
 		for (int k = 0; k < ranges.size(); k++)
@@ -1463,6 +1540,7 @@ struct ConsistencyCheckWorkload : TestWorkload {
 		state int endPoint;
 		if (CLIENT_KNOBS->CONSISTENCY_CHECK_URGENT_MODE) {
 			if (consistencyCheckEpoch == 0) {
+				// Decide which ranges are assigned
 				int batchSize = (shardOrder.size() / self->clientCount) + 1;
 				i = self->clientId * batchSize; // overwrite the starting point i
 				i = std::max(0, i - CLIENT_KNOBS->CONSISTENCY_CHECK_DISTRIBUTED_WIGGLE_ROOM);
@@ -1472,45 +1550,65 @@ struct ConsistencyCheckWorkload : TestWorkload {
 				increment = 1;
 				// If the start point i is larger than the list boundary, this client needs not run
 				if (i >= endPoint) {
-					TraceEvent("ConsistencyCheck_NoTaskToRun")
+					TraceEvent("ConsistencyCheck_ClientExit")
+					    .detail("Reason", "No task assigned")
 					    .detail("Distributed", self->distributed)
 					    .detail("ShardCount", ranges.size())
 					    .detail("ClientId", self->clientId)
 					    .detail("ClientCount", self->clientCount)
 					    .detail("Repetitions", self->repetitions);
-					wait(delay(600.0) || self->suspendConsistencyCheck.onChange());
 					return true;
 				}
-				// overwrite beginKeyToCheck and endKeyToCheck
-				beginKeyToCheck = ranges[shardOrder[i]].begin;
-				endKeyToCheck = ranges[shardOrder[endPoint - 1]].end;
+
+				// Logging range assignment
+				KeyRangeMap<bool> rangeAssignmentMap;
+				std::vector<KeyRange> rangeAssignmentList;
+				// Decide assigned ranges to log
+				for (int idx = i; idx < endPoint; idx += increment) {
+					rangeAssignmentMap.insert(ranges[shardOrder[idx]], true);
+				}
+				rangeAssignmentMap.coalesce(allKeys);
+				for (auto& assignedRange : rangeAssignmentMap.ranges()) {
+					if (assignedRange.value() == true) {
+						rangeAssignmentList.push_back(assignedRange.range());
+					}
+				}
+				numShardThisClient = (endPoint - i) / increment;
+				TraceEvent e("ConsistencyCheck_StartTask");
+				e.setMaxEventLength(-1);
+				e.setMaxFieldLength(-1);
+				e.detail("Distributed", self->distributed);
+				e.detail("ShardCount", ranges.size());
+				e.detail("ClientId", self->clientId);
+				e.detail("ClientCount", self->clientCount);
+				e.detail("StartPoint", i);
+				e.detail("EndPoint", endPoint);
+				e.detail("Repetitions", self->repetitions);
+				e.detail("ConsistencyCheckEpoch", consistencyCheckEpoch);
+				for (int idx = 0; idx < rangeAssignmentList.size(); idx++) {
+					e.detail("RangeToCheckBegin" + std::to_string(idx), rangeAssignmentList[idx].begin);
+					e.detail("RangeToCheckEnd" + std::to_string(idx), rangeAssignmentList[idx].end);
+				}
+				e.trackLatest("ConsistencyCheck_StartTask");
 			} else { // consistencyCheckEpoch > 0 when handling failure ranges
 				i = 0;
 				increment = 1;
 				endPoint = shardOrder.size();
-				beginKeyToCheck = Key();
-				endKeyToCheck = Key();
+				TraceEvent("ConsistencyCheck_RetryFailedTask")
+				    .setMaxEventLength(-1)
+				    .setMaxFieldLength(-1)
+				    .detail("Distributed", self->distributed)
+				    .detail("RetryShardCount", ranges.size())
+				    .detail("ClientId", self->clientId)
+				    .detail("ClientCount", self->clientCount)
+				    .detail("Repetitions", self->repetitions)
+				    .detail("ConsistencyCheckEpoch", consistencyCheckEpoch)
+				    .trackLatest("ConsistencyCheck_RetryFailedTask");
 			}
 		} else {
 			endPoint = shardOrder.size();
 		}
 
-		TraceEvent(consistencyCheckEpoch == 0 ? "ConsistencyCheck_StartTask" : "ConsistencyCheck_StartFailedTask")
-		    .setMaxEventLength(-1)
-		    .setMaxFieldLength(-1)
-		    .detail("Distributed", self->distributed)
-		    .detail("ShardCount", ranges.size())
-		    .detail("ClientId", self->clientId)
-		    .detail("ClientCount", self->clientCount)
-		    .detail("StartPoint", i)
-		    .detail("EndPoint", endPoint)
-		    .detail("BeginKey", beginKeyToCheck)
-		    .detail("EndKey", endKeyToCheck)
-		    .detail("Repetitions", self->repetitions)
-		    .detail("ConsistencyCheckEpoch", consistencyCheckEpoch)
-		    .trackLatest(consistencyCheckEpoch == 0 ? "ConsistencyCheck_StartTask"
-		                                            : "ConsistencyCheck_StartFailedTask");
-
 		for (; i < endPoint; i += increment) {
 			if (CLIENT_KNOBS->CONSISTENCY_CHECK_URGENT_MODE && self->suspendConsistencyCheck.get()) {
 				TraceEvent("ConsistencyCheck_Cancelled")
@@ -1520,13 +1618,13 @@ struct ConsistencyCheckWorkload : TestWorkload {
 				    .detail("ShardCount", ranges.size())
 				    .detail("ClientId", self->clientId)
 				    .detail("ClientCount", self->clientCount)
-				    .detail("CurrentPoint", i)
-				    .detail("EndPoint", endPoint)
-				    .detail("BeginKey", beginKeyToCheck)
-				    .detail("EndKey", endKeyToCheck)
 				    .detail("CompleteEndKey", ranges[shardOrder[i]].begin);
 				break;
 			}
+			if (CLIENT_KNOBS->CONSISTENCY_CHECK_URGENT_MODE) {
+				numShardToCheck = (endPoint - i) / increment;
+			}
+
 			state int shard = shardOrder[i];
 
 			state KeyRangeRef range = ranges[shard];
@@ -1696,10 +1794,6 @@ struct ConsistencyCheckWorkload : TestWorkload {
 							    .detail("ShardCount", ranges.size())
 							    .detail("ClientId", self->clientId)
 							    .detail("ClientCount", self->clientCount)
-							    .detail("CurrentPoint", i)
-							    .detail("EndPoint", endPoint)
-							    .detail("BeginKey", beginKeyToCheck)
-							    .detail("EndKey", endKeyToCheck)
 							    .detail("CompleteEndKey", lastStartSampleKey);
 							break;
 						}
@@ -1937,8 +2031,8 @@ struct ConsistencyCheckWorkload : TestWorkload {
 							    .setMaxFieldLength(-1)
 							    .detail("ClientId", self->clientId)
 							    .detail("ClientCount", self->clientCount)
-							    .detail("ShardBegin", printable(range.begin))
-							    .detail("ShardEnd", printable(range.end))
+							    .detail("ShardBegin", range.begin)
+							    .detail("ShardEnd", range.end)
 							    .detail("Repetitions", self->repetitions)
 							    .detail("ConsistencyCheckEpoch", consistencyCheckEpoch)
 							    .trackLatest("ConsistencyCheck_ShardAddedToRetry");
@@ -2022,17 +2116,22 @@ struct ConsistencyCheckWorkload : TestWorkload {
 				}
 
 				if (!completeCheck) {
+					numFailedShards++;
 					TraceEvent(CLIENT_KNOBS->CONSISTENCY_CHECK_URGENT_MODE ? SevInfo : SevWarn,
 					           "ConsistencyCheck_ShardFailed")
+					    .suppressFor(1.0)
 					    .setMaxEventLength(-1)
 					    .setMaxFieldLength(-1)
 					    .detail("ClientId", self->clientId)
 					    .detail("ClientCount", self->clientCount)
-					    .detail("ShardBegin", printable(range.begin))
-					    .detail("ShardEnd", printable(range.end))
+					    .detail("NumCompletedShards", numCompleteShards)
+					    .detail("NumFailedShards", numFailedShards)
+					    .detail("NumShardThisClient", numShardThisClient)
+					    .detail("NumShardToCheckThisEpoch", numShardToCheck)
+					    .detail("ShardBegin", range.begin)
+					    .detail("ShardEnd", range.end)
 					    .detail("Repetitions", self->repetitions)
-					    .detail("ConsistencyCheckEpoch", consistencyCheckEpoch)
-					    .trackLatest("ConsistencyCheck_ShardCheckFailedRange");
+					    .detail("ConsistencyCheckEpoch", consistencyCheckEpoch);
 				} else {
 					numCompleteShards++;
 					TraceEvent(SevInfo, "ConsistencyCheck_ShardComplete")
@@ -2041,9 +2140,12 @@ struct ConsistencyCheckWorkload : TestWorkload {
 					    .setMaxFieldLength(-1)
 					    .detail("ClientId", self->clientId)
 					    .detail("ClientCount", self->clientCount)
-					    .detail("Range", range.toString())
+					    .detail("Range", range)
 					    .detail("ShardCount", ranges.size())
 					    .detail("NumCompletedShards", numCompleteShards)
+					    .detail("NumFailedShards", numFailedShards)
+					    .detail("NumShardThisClient", numShardThisClient)
+					    .detail("NumShardToCheckThisEpoch", numShardToCheck)
 					    .detail("BytesReadInThisRound", bytesReadInthisRound)
 					    .detail("NumSkippedShards", numSkippedShards)
 					    .detail("ConsistencyCheckEpoch", consistencyCheckEpoch);
@@ -2188,7 +2290,7 @@ struct ConsistencyCheckWorkload : TestWorkload {
 			if (failedRangesToCheck.size() > 0) {
 				wait(delay(60.0)); // Backoff 1 min
 				state std::vector<std::pair<KeyRange, Value>> shardLocationPairListForFailedRanges =
-				    wait(self->getKeyLocationsForRangeList(cx, failedRangesToCheck));
+				    wait(self->getKeyLocationsForRangeList(cx, failedRangesToCheck, self));
 				TraceEvent(SevInfo, "ConsistencyCheck_StartHandlingFailedRanges")
 				    .setMaxEventLength(-1)
 				    .setMaxFieldLength(-1)
@@ -2258,8 +2360,6 @@ struct ConsistencyCheckWorkload : TestWorkload {
 			    .detail("ShardCount", ranges.size())
 			    .detail("ClientId", self->clientId)
 			    .detail("ClientCount", self->clientCount)
-			    .detail("BeginKey", beginKeyToCheck)
-			    .detail("EndKey", endKeyToCheck)
 			    .detail("Repetitions", self->repetitions)
 			    .trackLatest("ConsistencyCheck_EndTask");
 			self->bytesReadInPreviousRound =

From ff79e6f9aee429a5194a7bc3e484550117f85a09 Mon Sep 17 00:00:00 2001
From: Zhe Wang <zhe.wang@wustl.edu>
Date: Thu, 14 Dec 2023 10:45:00 -0800
Subject: [PATCH 332/468] fix knob val

---
 fdbclient/ClientKnobs.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/fdbclient/ClientKnobs.cpp b/fdbclient/ClientKnobs.cpp
index f1a649af087..52bcf149559 100644
--- a/fdbclient/ClientKnobs.cpp
+++ b/fdbclient/ClientKnobs.cpp
@@ -259,7 +259,7 @@ void ClientKnobs::initialize(Randomize randomize) {
 	init( CONSISTENCY_CHECK_RATE_LIMIT_MAX,        50e6 ); // Limit in per sec
 	init( CONSISTENCY_CHECK_ONE_ROUND_TARGET_COMPLETION_TIME,	7 * 24 * 60 * 60 ); // 7 days
 	init( CONSISTENCY_CHECK_URGENT_MODE,           true );
-	init( CONSISTENCY_CHECK_DISTRIBUTED_WIGGLE_ROOM,  0 ); if( randomize && BUGGIFY ) CONSISTENCY_CHECK_DISTRIBUTED_WIGGLE_ROOM = 1;
+	init( CONSISTENCY_CHECK_DISTRIBUTED_WIGGLE_ROOM, 50 ); if( randomize && BUGGIFY ) CONSISTENCY_CHECK_DISTRIBUTED_WIGGLE_ROOM = 1;
 	init( CONSISTENCY_CHECK_RETRY_DEPTH_MAX,        100 ); if( randomize && BUGGIFY ) CONSISTENCY_CHECK_RETRY_DEPTH_MAX = 10;
 	init( CONSISTENCY_CHECK_RANGE_BEGIN_0,           "" ); if( randomize && BUGGIFY ) CONSISTENCY_CHECK_RANGE_BEGIN_0 = "";
 	init( CONSISTENCY_CHECK_RANGE_END_0,   "\\xff\\xff" ); if( randomize && BUGGIFY ) CONSISTENCY_CHECK_RANGE_END_0 = "\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x33\\x66\\x63\\x36";

From 98171499ffc3ee42c2fb42b376c71ebf47c40c3c Mon Sep 17 00:00:00 2001
From: Zhe Wang <zhe.wang@wustl.edu>
Date: Thu, 14 Dec 2023 11:00:59 -0800
Subject: [PATCH 333/468] fix NumShardToCheckThisEpoch counter

---
 fdbserver/workloads/ConsistencyCheck.actor.cpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/fdbserver/workloads/ConsistencyCheck.actor.cpp b/fdbserver/workloads/ConsistencyCheck.actor.cpp
index 4fbf91e770d..47b2e996dda 100644
--- a/fdbserver/workloads/ConsistencyCheck.actor.cpp
+++ b/fdbserver/workloads/ConsistencyCheck.actor.cpp
@@ -2127,7 +2127,7 @@ struct ConsistencyCheckWorkload : TestWorkload {
 					    .detail("NumCompletedShards", numCompleteShards)
 					    .detail("NumFailedShards", numFailedShards)
 					    .detail("NumShardThisClient", numShardThisClient)
-					    .detail("NumShardToCheckThisEpoch", numShardToCheck)
+					    .detail("NumShardToCheckThisEpoch", numShardToCheck - 1)
 					    .detail("ShardBegin", range.begin)
 					    .detail("ShardEnd", range.end)
 					    .detail("Repetitions", self->repetitions)
@@ -2145,7 +2145,7 @@ struct ConsistencyCheckWorkload : TestWorkload {
 					    .detail("NumCompletedShards", numCompleteShards)
 					    .detail("NumFailedShards", numFailedShards)
 					    .detail("NumShardThisClient", numShardThisClient)
-					    .detail("NumShardToCheckThisEpoch", numShardToCheck)
+					    .detail("NumShardToCheckThisEpoch", numShardToCheck - 1)
 					    .detail("BytesReadInThisRound", bytesReadInthisRound)
 					    .detail("NumSkippedShards", numSkippedShards)
 					    .detail("ConsistencyCheckEpoch", consistencyCheckEpoch);

From ca50d27df02bfc13949aa90b5de2cc13288f98cd Mon Sep 17 00:00:00 2001
From: FoundationDB CI <foundationdb_ci@apple.com>
Date: Thu, 14 Dec 2023 23:51:13 +0000
Subject: [PATCH 334/468] disable AVX for 7.1.48 release

---
 cmake/ConfigureCompiler.cmake | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cmake/ConfigureCompiler.cmake b/cmake/ConfigureCompiler.cmake
index e52d8665ee8..7e50469ba6a 100644
--- a/cmake/ConfigureCompiler.cmake
+++ b/cmake/ConfigureCompiler.cmake
@@ -253,7 +253,7 @@ else()
       set(USE_AVX512F OFF)
     endif()
   endif()
-  set(USE_AVX ON CACHE BOOL "Enable AVX instructions")
+  set(USE_AVX OFF CACHE BOOL "Enable AVX instructions")
   if (USE_AVX)
     if (CMAKE_HOST_SYSTEM_PROCESSOR MATCHES "^x86")
       add_compile_options(-mavx)

From 5fe02b5007524978b063687a96474888309aa44e Mon Sep 17 00:00:00 2001
From: FoundationDB CI <foundationdb_ci@apple.com>
Date: Thu, 14 Dec 2023 23:51:23 +0000
Subject: [PATCH 335/468] enable AVX and update version for 7.1.49 release

---
 CMakeLists.txt                | 2 +-
 cmake/ConfigureCompiler.cmake | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index d02e78ca2ff..c0e0df69c0d 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -23,7 +23,7 @@ else()
 endif()
 
 project(foundationdb
-  VERSION 7.1.48
+  VERSION 7.1.49
   DESCRIPTION "FoundationDB is a scalable, fault-tolerant, ordered key-value store with full ACID transactions."
   HOMEPAGE_URL "http://www.foundationdb.org/"
   LANGUAGES C CXX ASM)
diff --git a/cmake/ConfigureCompiler.cmake b/cmake/ConfigureCompiler.cmake
index 7e50469ba6a..e52d8665ee8 100644
--- a/cmake/ConfigureCompiler.cmake
+++ b/cmake/ConfigureCompiler.cmake
@@ -253,7 +253,7 @@ else()
       set(USE_AVX512F OFF)
     endif()
   endif()
-  set(USE_AVX OFF CACHE BOOL "Enable AVX instructions")
+  set(USE_AVX ON CACHE BOOL "Enable AVX instructions")
   if (USE_AVX)
     if (CMAKE_HOST_SYSTEM_PROCESSOR MATCHES "^x86")
       add_compile_options(-mavx)

From 1796b1ffa3f3eb1882e0a29d01183058186febbf Mon Sep 17 00:00:00 2001
From: FoundationDB CI <foundationdb_ci@apple.com>
Date: Fri, 15 Dec 2023 18:51:29 +0000
Subject: [PATCH 336/468] update version after 7.1.49 release

---
 CMakeLists.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index c0e0df69c0d..ca79fc4323b 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -23,7 +23,7 @@ else()
 endif()
 
 project(foundationdb
-  VERSION 7.1.49
+  VERSION 7.1.50
   DESCRIPTION "FoundationDB is a scalable, fault-tolerant, ordered key-value store with full ACID transactions."
   HOMEPAGE_URL "http://www.foundationdb.org/"
   LANGUAGES C CXX ASM)

From 0baa9d9be75218f64c60d76231b2656d78e092cb Mon Sep 17 00:00:00 2001
From: Zhe Wang <zhe.wang@wustl.edu>
Date: Sat, 16 Dec 2023 15:20:35 -0800
Subject: [PATCH 337/468] stateful consistency checker

---
 fdbclient/CMakeLists.txt                      |   3 +
 fdbclient/ClientKnobs.cpp                     |   3 +-
 fdbclient/ClientKnobs.h                       |   3 +-
 fdbclient/ConsistencyCheck.h                  |  67 ++++
 fdbclient/ConsistencyCheckUtil.actor.cpp      | 329 ++++++++++++++++
 fdbclient/ConsistencyCheckUtil.actor.h        |  50 +++
 fdbclient/SystemData.cpp                      |  29 ++
 fdbclient/SystemData.h                        |  10 +
 fdbserver/TesterInterface.actor.h             |   7 +-
 fdbserver/fdbserver.actor.cpp                 |  17 +-
 fdbserver/tester.actor.cpp                    | 240 +++++++++++-
 .../workloads/ConsistencyCheck.actor.cpp      | 359 +++++-------------
 flow/error_definitions.h                      |   1 +
 13 files changed, 847 insertions(+), 271 deletions(-)
 create mode 100644 fdbclient/ConsistencyCheck.h
 create mode 100644 fdbclient/ConsistencyCheckUtil.actor.cpp
 create mode 100644 fdbclient/ConsistencyCheckUtil.actor.h

diff --git a/fdbclient/CMakeLists.txt b/fdbclient/CMakeLists.txt
index c3b626b672e..ba961ea1899 100644
--- a/fdbclient/CMakeLists.txt
+++ b/fdbclient/CMakeLists.txt
@@ -47,6 +47,9 @@ set(FDBCLIENT_SRCS
   ConfigKnobs.h
   ConfigTransactionInterface.cpp
   ConfigTransactionInterface.h
+  ConsistencyCheck.h
+  ConsistencyCheckUtil.actor.cpp
+  ConsistencyCheckUtil.actor.h
   ConvertUTF.h
   CoordinationInterface.h
   DatabaseBackupAgent.actor.cpp
diff --git a/fdbclient/ClientKnobs.cpp b/fdbclient/ClientKnobs.cpp
index 52bcf149559..2325546cbec 100644
--- a/fdbclient/ClientKnobs.cpp
+++ b/fdbclient/ClientKnobs.cpp
@@ -258,8 +258,7 @@ void ClientKnobs::initialize(Randomize randomize) {
 
 	init( CONSISTENCY_CHECK_RATE_LIMIT_MAX,        50e6 ); // Limit in per sec
 	init( CONSISTENCY_CHECK_ONE_ROUND_TARGET_COMPLETION_TIME,	7 * 24 * 60 * 60 ); // 7 days
-	init( CONSISTENCY_CHECK_URGENT_MODE,           true );
-	init( CONSISTENCY_CHECK_DISTRIBUTED_WIGGLE_ROOM, 50 ); if( randomize && BUGGIFY ) CONSISTENCY_CHECK_DISTRIBUTED_WIGGLE_ROOM = 1;
+	init( CONSISTENCY_CHECK_URGENT_MODE_FORCE_INIT, false );
 	init( CONSISTENCY_CHECK_RETRY_DEPTH_MAX,        100 ); if( randomize && BUGGIFY ) CONSISTENCY_CHECK_RETRY_DEPTH_MAX = 10;
 	init( CONSISTENCY_CHECK_RANGE_BEGIN_0,           "" ); if( randomize && BUGGIFY ) CONSISTENCY_CHECK_RANGE_BEGIN_0 = "";
 	init( CONSISTENCY_CHECK_RANGE_END_0,   "\\xff\\xff" ); if( randomize && BUGGIFY ) CONSISTENCY_CHECK_RANGE_END_0 = "\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x33\\x66\\x63\\x36";
diff --git a/fdbclient/ClientKnobs.h b/fdbclient/ClientKnobs.h
index ccd74e9dd5a..eaf75b44b14 100644
--- a/fdbclient/ClientKnobs.h
+++ b/fdbclient/ClientKnobs.h
@@ -249,9 +249,8 @@ class ClientKnobs : public KnobsImpl<ClientKnobs> {
 
 	int CONSISTENCY_CHECK_RATE_LIMIT_MAX;
 	int CONSISTENCY_CHECK_ONE_ROUND_TARGET_COMPLETION_TIME;
-	bool CONSISTENCY_CHECK_URGENT_MODE; // Require Tester Process::Class
+	bool CONSISTENCY_CHECK_URGENT_MODE_FORCE_INIT; // Available in urgent mode
 	int CONSISTENCY_CHECK_RETRY_DEPTH_MAX; // Available in urgent mode
-	int CONSISTENCY_CHECK_DISTRIBUTED_WIGGLE_ROOM; // Available in urgent mode
 	std::string CONSISTENCY_CHECK_RANGE_BEGIN_0; // Available in urgent mode
 	std::string CONSISTENCY_CHECK_RANGE_END_0; // Available in urgent mode
 	std::string CONSISTENCY_CHECK_RANGE_BEGIN_1; // Available in urgent mode
diff --git a/fdbclient/ConsistencyCheck.h b/fdbclient/ConsistencyCheck.h
new file mode 100644
index 00000000000..005472a24ab
--- /dev/null
+++ b/fdbclient/ConsistencyCheck.h
@@ -0,0 +1,67 @@
+/*
+ * ConsistencyCheck.h
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2022 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef FDBCLIENT_CONSISTENCYCHECK_H
+#define FDBCLIENT_CONSISTENCYCHECK_H
+#pragma once
+
+#include "fdbclient/FDBTypes.h"
+#include "fdbrpc/fdbrpc.h"
+
+enum class ConsistencyCheckPhase : uint8_t {
+	Invalid = 0,
+	Complete = 1,
+};
+
+enum class ConsistencyCheckAssignment : uint8_t {
+	Invalid = 0,
+	Assigned = 1,
+};
+
+struct ConsistencyCheckState {
+	constexpr static FileIdentifier file_identifier = 13804349;
+
+	ConsistencyCheckState() : consistencyCheckerId(0), phase(0), assignment(0) {}
+
+	ConsistencyCheckState(uint64_t consistencyCheckerId)
+	  : consistencyCheckerId(consistencyCheckerId), phase(0), assignment(0) {}
+
+	ConsistencyCheckState(ConsistencyCheckPhase phase)
+	  : consistencyCheckerId(0), phase(static_cast<uint8_t>(phase)), assignment(0) {}
+
+	ConsistencyCheckState(ConsistencyCheckAssignment assignment)
+	  : consistencyCheckerId(0), phase(0), assignment(static_cast<uint8_t>(assignment)) {}
+
+	template <class Ar>
+	void serialize(Ar& ar) {
+		serializer(ar, consistencyCheckerId, phase, assignment);
+	}
+
+	inline ConsistencyCheckPhase getPhase() const { return static_cast<ConsistencyCheckPhase>(this->phase); }
+	inline ConsistencyCheckAssignment getAssignment() const {
+		return static_cast<ConsistencyCheckAssignment>(this->assignment);
+	}
+
+	uint64_t consistencyCheckerId;
+	uint8_t phase;
+	uint8_t assignment;
+};
+
+#endif
diff --git a/fdbclient/ConsistencyCheckUtil.actor.cpp b/fdbclient/ConsistencyCheckUtil.actor.cpp
new file mode 100644
index 00000000000..318ae2fc95b
--- /dev/null
+++ b/fdbclient/ConsistencyCheckUtil.actor.cpp
@@ -0,0 +1,329 @@
+/*
+ * ConsistencyCheckUtil.actor.cpp
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2022 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "fdbclient/ConsistencyCheckUtil.actor.h"
+#include "fdbclient/ConsistencyCheck.h"
+#include "flow/actorcompiler.h" // This must be the last #include.
+
+ACTOR Future<std::vector<KeyRange>> loadRangesToCheckFromAssignmentMetadata(Database cx, int clientId) {
+	state std::vector<KeyRange> res;
+	state Transaction tr(cx);
+	state Key rangeToReadBegin = allKeys.begin;
+	state Key rangeToReadEnd = allKeys.end;
+	loop {
+		try {
+			tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
+			tr.setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
+			tr.setOption(FDBTransactionOptions::LOCK_AWARE);
+			KeyRange rangeToRead = Standalone(KeyRangeRef(rangeToReadBegin, rangeToReadEnd));
+			RangeResult res_ = wait(krmGetRanges(&tr,
+			                                     consistencyCheckAssignmentPrefixFor(clientId),
+			                                     rangeToRead,
+			                                     CLIENT_KNOBS->KRM_GET_RANGE_LIMIT,
+			                                     CLIENT_KNOBS->KRM_GET_RANGE_LIMIT_BYTES));
+			for (int i = 0; i < res_.size() - 1; ++i) {
+				KeyRange currentRange = KeyRangeRef(res_[i].key, res_[i + 1].key);
+				if (!res_[i].value.empty()) {
+					ConsistencyCheckState ccState = decodeConsistencyCheckerStateValue(res_[i].value);
+					if (ccState.getAssignment() == ConsistencyCheckAssignment::Assigned) {
+						res.push_back(currentRange);
+					}
+				}
+				rangeToReadBegin = res_[i + 1].key;
+			}
+			break;
+		} catch (Error& e) {
+			wait(tr.onError(e));
+		}
+	}
+	return res;
+}
+
+ACTOR Future<Void> persistConsistencyCheckProgress(Database cx, KeyRange range, uint64_t consistencyCheckerId) {
+	state Transaction tr(cx);
+	loop {
+		try {
+			tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
+			tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
+			tr.setOption(FDBTransactionOptions::LOCK_AWARE);
+			Optional<Value> res_ = wait(tr.get(consistencyCheckerIdKey));
+			if (!res_.present()) {
+				throw key_not_found();
+			} else if (consistencyCheckerId != decodeConsistencyCheckerStateValue(res_.get()).consistencyCheckerId) {
+				throw consistency_check_task_outdated();
+			}
+			wait(krmSetRange(&tr,
+			                 consistencyCheckProgressPrefix,
+			                 range,
+			                 consistencyCheckerStateValue(ConsistencyCheckState(ConsistencyCheckPhase::Complete))));
+			wait(tr.commit());
+			break;
+		} catch (Error& e) {
+			wait(tr.onError(e));
+		}
+	}
+	return Void();
+}
+
+ACTOR Future<Void> initConsistencyCheckMetadata(Database cx,
+                                                std::vector<KeyRange> rangesToCheck,
+                                                uint64_t consistencyCheckerId) {
+	state Transaction tr(cx);
+	loop {
+		try {
+			tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
+			tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
+			tr.setOption(FDBTransactionOptions::LOCK_AWARE);
+			tr.clear(consistencyCheckAssignmentKeys);
+			tr.clear(consistencyCheckProgressKeys);
+			wait(krmSetRange(&tr,
+			                 consistencyCheckProgressPrefix,
+			                 allKeys,
+			                 consistencyCheckerStateValue(
+			                     ConsistencyCheckState(ConsistencyCheckPhase::Complete)))); // Mark all as completes
+			for (const auto& rangeToCheck : rangesToCheck) {
+				wait(krmSetRange(&tr,
+				                 consistencyCheckProgressPrefix,
+				                 rangeToCheck,
+				                 consistencyCheckerStateValue(ConsistencyCheckState(
+				                     ConsistencyCheckPhase::Invalid)))); // Mark rangesToCheck as incomplete
+			}
+			tr.set(consistencyCheckerIdKey, consistencyCheckerStateValue(ConsistencyCheckState(consistencyCheckerId)));
+			wait(tr.commit());
+			break;
+		} catch (Error& e) {
+			wait(tr.onError(e) && delay(10.0));
+		}
+	}
+	return Void();
+}
+
+ACTOR Future<std::vector<KeyRange>> loadRangesToCheckFromProgressMetadata(Database cx) {
+	state std::vector<KeyRange> res;
+	state Transaction tr(cx);
+	state Key rangeToReadBegin = allKeys.begin;
+	state Key rangeToReadEnd = allKeys.end;
+	loop {
+		try {
+			tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
+			tr.setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
+			tr.setOption(FDBTransactionOptions::LOCK_AWARE);
+			KeyRange rangeToRead = Standalone(KeyRangeRef(rangeToReadBegin, rangeToReadEnd));
+			RangeResult res_ = wait(krmGetRanges(&tr,
+			                                     consistencyCheckProgressPrefix,
+			                                     rangeToRead,
+			                                     CLIENT_KNOBS->KRM_GET_RANGE_LIMIT,
+			                                     CLIENT_KNOBS->KRM_GET_RANGE_LIMIT_BYTES));
+			for (int i = 0; i < res_.size() - 1; ++i) {
+				KeyRange currentRange = KeyRangeRef(res_[i].key, res_[i + 1].key);
+				if (!res_[i].value.empty()) {
+					ConsistencyCheckState ccState = decodeConsistencyCheckerStateValue(res_[i].value);
+					if (ccState.getPhase() == ConsistencyCheckPhase::Invalid) {
+						res.push_back(currentRange);
+					}
+				}
+				rangeToReadBegin = res_[i + 1].key;
+			}
+			break;
+		} catch (Error& e) {
+			wait(tr.onError(e));
+		}
+	}
+	return res;
+}
+
+ACTOR Future<Void> persistConsistencyCheckAssignment(Database cx, int clientId, std::vector<KeyRange> assignedRanges) {
+	state Transaction tr(cx);
+	loop {
+		try {
+			tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
+			tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
+			tr.setOption(FDBTransactionOptions::LOCK_AWARE);
+			KeyRangeMap<bool> assignedRangesMap;
+			std::vector<Future<Void>> fs;
+			for (const auto& assignedRange : assignedRanges) {
+				assignedRangesMap.insert(assignedRange, true);
+			}
+			assignedRangesMap.coalesce(allKeys);
+			for (auto& assignedRange : assignedRangesMap.ranges()) {
+				fs.push_back(krmSetRange(
+				    &tr,
+				    consistencyCheckAssignmentPrefixFor(clientId),
+				    assignedRange.range(),
+				    consistencyCheckerStateValue(ConsistencyCheckState(ConsistencyCheckAssignment::Assigned))));
+			}
+			wait(waitForAll(fs));
+			wait(tr.commit());
+			break;
+		} catch (Error& e) {
+			wait(tr.onError(e));
+		}
+	}
+	return Void();
+}
+
+const std::unordered_map<char, uint8_t> parseCharMap{
+	{ '0', 0 },  { '1', 1 },  { '2', 2 },  { '3', 3 },  { '4', 4 },  { '5', 5 },  { '6', 6 },  { '7', 7 },
+	{ '8', 8 },  { '9', 9 },  { 'a', 10 }, { 'b', 11 }, { 'c', 12 }, { 'd', 13 }, { 'e', 14 }, { 'f', 15 },
+	{ 'A', 10 }, { 'B', 11 }, { 'C', 12 }, { 'D', 13 }, { 'E', 14 }, { 'F', 15 },
+};
+
+Key getKeyFromString(const std::string& str) {
+	Key emptyKey;
+	if (str.size() == 0) {
+		return emptyKey;
+	}
+	if (str.size() % 4 != 0) {
+		TraceEvent(SevWarnAlways, "ConsistencyCheck_GetKeyFromStringError")
+		    .setMaxEventLength(-1)
+		    .setMaxFieldLength(-1)
+		    .detail("Reason", "WrongLength")
+		    .detail("InputStr", str);
+		throw internal_error();
+	}
+	std::vector<uint8_t> byteList;
+	for (int i = 0; i < str.size(); i += 4) {
+		if (str.at(i + 0) != '\\' || str.at(i + 1) != 'x') {
+			TraceEvent(SevWarnAlways, "ConsistencyCheck_GetKeyFromStringError")
+			    .setMaxEventLength(-1)
+			    .setMaxFieldLength(-1)
+			    .detail("Reason", "WrongBytePrefix")
+			    .detail("InputStr", str);
+			throw internal_error();
+		}
+		const char first = str.at(i + 2);
+		const char second = str.at(i + 3);
+		if (parseCharMap.count(first) == 0 || parseCharMap.count(second) == 0) {
+			TraceEvent(SevWarnAlways, "ConsistencyCheck_GetKeyFromStringError")
+			    .setMaxEventLength(-1)
+			    .setMaxFieldLength(-1)
+			    .detail("Reason", "WrongByteContent")
+			    .detail("InputStr", str);
+			throw internal_error();
+		}
+		uint8_t parsedValue = parseCharMap.at(first) * 16 + parseCharMap.at(second);
+		byteList.push_back(parsedValue);
+	}
+	return Standalone(StringRef(byteList.data(), byteList.size()));
+}
+
+std::vector<KeyRange> loadRangesToCheckFromKnob() {
+	// Load string from knob
+	std::vector<std::string> beginKeyStrs = {
+		CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_BEGIN_0,  CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_BEGIN_1,
+		CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_BEGIN_2,  CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_BEGIN_3,
+		CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_BEGIN_4,  CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_BEGIN_5,
+		CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_BEGIN_6,  CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_BEGIN_7,
+		CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_BEGIN_8,  CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_BEGIN_9,
+		CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_BEGIN_10, CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_BEGIN_11,
+		CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_BEGIN_12, CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_BEGIN_13,
+		CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_BEGIN_14, CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_BEGIN_15,
+		CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_BEGIN_16, CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_BEGIN_17,
+		CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_BEGIN_18, CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_BEGIN_19,
+		CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_BEGIN_20, CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_BEGIN_21,
+		CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_BEGIN_22, CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_BEGIN_23,
+		CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_BEGIN_24, CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_BEGIN_25,
+		CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_BEGIN_26,
+	};
+	std::vector<std::string> endKeyStrs = {
+		CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_END_0,  CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_END_1,
+		CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_END_2,  CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_END_3,
+		CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_END_4,  CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_END_5,
+		CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_END_6,  CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_END_7,
+		CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_END_8,  CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_END_9,
+		CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_END_10, CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_END_11,
+		CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_END_12, CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_END_13,
+		CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_END_14, CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_END_15,
+		CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_END_16, CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_END_17,
+		CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_END_18, CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_END_19,
+		CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_END_20, CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_END_21,
+		CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_END_22, CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_END_23,
+		CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_END_24, CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_END_25,
+		CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_END_26,
+	};
+
+	// Get keys from strings
+	std::vector<Key> beginKeys;
+	for (const auto& beginKeyStr : beginKeyStrs) {
+		Key key = getKeyFromString(beginKeyStr);
+		beginKeys.push_back(key);
+	}
+	std::vector<Key> endKeys;
+	for (const auto& endKeyStr : endKeyStrs) {
+		Key key = getKeyFromString(endKeyStr);
+		endKeys.push_back(key);
+	}
+	ASSERT(beginKeys.size() == endKeys.size());
+
+	// Get ranges
+	KeyRangeMap<bool> rangeToCheckMap;
+	for (int i = 0; i < beginKeys.size(); i++) {
+		Key rangeBegin = beginKeys[i];
+		Key rangeEnd = endKeys[i];
+		if (rangeBegin.empty() && rangeEnd.empty()) {
+			continue;
+		}
+		if (rangeBegin > allKeys.end) {
+			rangeBegin = allKeys.end;
+		}
+		if (rangeEnd > allKeys.end) {
+			TraceEvent("ConsistencyCheck_ReverseInputRange")
+			    .setMaxEventLength(-1)
+			    .setMaxFieldLength(-1)
+			    .detail("Index", i)
+			    .detail("RangeBegin", rangeBegin)
+			    .detail("RangeEnd", rangeEnd);
+			rangeEnd = allKeys.end;
+		}
+
+		KeyRange rangeToCheck;
+		if (rangeBegin < rangeEnd) {
+			rangeToCheck = Standalone(KeyRangeRef(rangeBegin, rangeEnd));
+		} else if (rangeBegin > rangeEnd) {
+			rangeToCheck = Standalone(KeyRangeRef(rangeEnd, rangeBegin));
+		} else {
+			TraceEvent("ConsistencyCheck_EmptyInputRange")
+			    .setMaxEventLength(-1)
+			    .setMaxFieldLength(-1)
+			    .detail("Index", i)
+			    .detail("RangeBegin", rangeBegin)
+			    .detail("RangeEnd", rangeEnd);
+			continue;
+		}
+		rangeToCheckMap.insert(rangeToCheck, true);
+	}
+
+	rangeToCheckMap.coalesce(allKeys);
+
+	std::vector<KeyRange> res;
+	for (auto rangeToCheck : rangeToCheckMap.ranges()) {
+		if (rangeToCheck.value() == true) {
+			res.push_back(rangeToCheck.range());
+		}
+	}
+	TraceEvent e("ConsistencyCheck_LoadedInputRange");
+	e.setMaxEventLength(-1);
+	e.setMaxFieldLength(-1);
+	for (int i = 0; i < res.size(); i++) {
+		e.detail("RangeBegin" + std::to_string(i), res[i].begin);
+		e.detail("RangeEnd" + std::to_string(i), res[i].end);
+	}
+	return res;
+}
diff --git a/fdbclient/ConsistencyCheckUtil.actor.h b/fdbclient/ConsistencyCheckUtil.actor.h
new file mode 100644
index 00000000000..d2ef0e959c5
--- /dev/null
+++ b/fdbclient/ConsistencyCheckUtil.actor.h
@@ -0,0 +1,50 @@
+/*
+ * ConsistencyCheckUtil.actor.h
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2022 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+// When actually compiled (NO_INTELLISENSE), include the generated version of this file.  In intellisense use the source
+// version.
+#if defined(NO_INTELLISENSE) && !defined(FDBCLIENT_CONSISTENCYCHECKUTIL_ACTOR_G_H)
+#define FDBCLIENT_CONSISTENCYCHECKUTIL_ACTOR_G_H
+#include "fdbclient/ConsistencyCheckUtil.actor.g.h"
+#elif !defined(FDBCLIENT_CONSISTENCYCHECKUTIL_ACTOR_H)
+#define FDBCLIENT_CONSISTENCYCHECKUTIL_ACTOR_H
+
+#include "fdbclient/ConsistencyCheck.h"
+#include "fdbclient/DatabaseContext.h"
+#include "fdbclient/SystemData.h"
+
+ACTOR Future<std::vector<KeyRange>> loadRangesToCheckFromAssignmentMetadata(Database cx, int clientId);
+
+ACTOR Future<Void> persistConsistencyCheckProgress(Database cx, KeyRange range, uint64_t consistencyCheckerId);
+
+ACTOR Future<Void> initConsistencyCheckMetadata(Database cx,
+                                                std::vector<KeyRange> rangesToCheck,
+                                                uint64_t consistencyCheckerId);
+
+ACTOR Future<std::vector<KeyRange>> loadRangesToCheckFromProgressMetadata(Database cx);
+
+ACTOR Future<Void> persistConsistencyCheckAssignment(Database cx, int clientId, std::vector<KeyRange> assignedRanges);
+
+std::vector<KeyRange> loadRangesToCheckFromKnob();
+
+#include "flow/unactorcompiler.h"
+#endif
diff --git a/fdbclient/SystemData.cpp b/fdbclient/SystemData.cpp
index 255fccc6c60..5d311eba3b2 100644
--- a/fdbclient/SystemData.cpp
+++ b/fdbclient/SystemData.cpp
@@ -242,6 +242,35 @@ CheckpointMetaData decodeCheckpointValue(const ValueRef& value) {
 	return checkpoint;
 }
 
+const KeyRangeRef consistencyCheckAssignmentKeys =
+    KeyRangeRef("\xff/consistency_check_assign/"_sr, "\xff/consistency_check_assign0"_sr);
+const KeyRef consistencyCheckAssignmentPrefix = consistencyCheckAssignmentKeys.begin;
+
+const Key consistencyCheckAssignmentPrefixFor(int clientId) {
+	BinaryWriter wr(Unversioned());
+	wr.serializeBytes(consistencyCheckAssignmentPrefix);
+	wr << clientId;
+	wr.serializeBytes("/"_sr);
+	return wr.toValue();
+}
+
+const KeyRangeRef consistencyCheckProgressKeys =
+    KeyRangeRef("\xff/consistency_check_progress/"_sr, "\xff/consistency_check_progress0"_sr);
+const KeyRef consistencyCheckProgressPrefix = consistencyCheckProgressKeys.begin;
+
+const KeyRef consistencyCheckerIdKey = "\xff/consistency_checker_id/"_sr;
+
+const Value consistencyCheckerStateValue(const ConsistencyCheckState& ccState) {
+	return ObjectWriter::toValue(ccState, IncludeVersion());
+}
+
+ConsistencyCheckState decodeConsistencyCheckerStateValue(const ValueRef& value) {
+	ConsistencyCheckState ccState;
+	ObjectReader reader(value.begin(), IncludeVersion());
+	reader.deserialize(ccState);
+	return ccState;
+}
+
 // "\xff/cacheServer/[[UID]] := StorageServerInterface"
 const KeyRangeRef storageCacheServerKeys(LiteralStringRef("\xff/cacheServer/"), LiteralStringRef("\xff/cacheServer0"));
 const KeyRef storageCacheServersPrefix = storageCacheServerKeys.begin;
diff --git a/fdbclient/SystemData.h b/fdbclient/SystemData.h
index 4069c68162a..eef37971dca 100644
--- a/fdbclient/SystemData.h
+++ b/fdbclient/SystemData.h
@@ -24,6 +24,7 @@
 
 // Functions and constants documenting the organization of the reserved keyspace in the database beginning with "\xFF"
 
+#include "fdbclient/ConsistencyCheck.h"
 #include "fdbclient/FDBTypes.h"
 #include "fdbclient/BlobWorkerInterface.h" // TODO move the functions that depend on this out of here and into BlobWorkerInterface.h to remove this depdendency
 #include "fdbclient/StorageServerInterface.h"
@@ -77,6 +78,15 @@ const Value checkpointValue(const CheckpointMetaData& checkpoint);
 UID decodeCheckpointKey(const KeyRef& key);
 CheckpointMetaData decodeCheckpointValue(const ValueRef& value);
 
+extern const KeyRangeRef consistencyCheckAssignmentKeys;
+extern const KeyRef consistencyCheckAssignmentPrefix;
+const Key consistencyCheckAssignmentPrefixFor(int clientId);
+extern const KeyRangeRef consistencyCheckProgressKeys;
+extern const KeyRef consistencyCheckProgressPrefix;
+extern const KeyRef consistencyCheckerIdKey;
+const Value consistencyCheckerStateValue(const ConsistencyCheckState& ccState);
+ConsistencyCheckState decodeConsistencyCheckerStateValue(const ValueRef& value);
+
 // "\xff/storageCacheServer/[[UID]] := StorageServerInterface"
 // This will be added by the cache server on initialization and removed by DD
 // TODO[mpilman]: We will need a way to map uint16_t ids to UIDs in a future
diff --git a/fdbserver/TesterInterface.actor.h b/fdbserver/TesterInterface.actor.h
index a52b11673e5..4c16ffc30a6 100644
--- a/fdbserver/TesterInterface.actor.h
+++ b/fdbserver/TesterInterface.actor.h
@@ -120,7 +120,12 @@ ACTOR Future<Void> testerServerCore(TesterInterface interf,
                                     LocalityData locality);
 
 enum test_location_t { TEST_HERE, TEST_ON_SERVERS, TEST_ON_TESTERS };
-enum test_type_t { TEST_TYPE_FROM_FILE, TEST_TYPE_CONSISTENCY_CHECK, TEST_TYPE_UNIT_TESTS };
+enum test_type_t {
+	TEST_TYPE_FROM_FILE,
+	TEST_TYPE_CONSISTENCY_CHECK,
+	TEST_TYPE_UNIT_TESTS,
+	TEST_TYPE_CONSISTENCY_CHECK_URGENT
+};
 
 ACTOR Future<Void> runTests(Reference<IClusterConnectionRecord> connRecord,
                             test_type_t whatToRun,
diff --git a/fdbserver/fdbserver.actor.cpp b/fdbserver/fdbserver.actor.cpp
index 6f03701c7dd..679e03b0468 100644
--- a/fdbserver/fdbserver.actor.cpp
+++ b/fdbserver/fdbserver.actor.cpp
@@ -978,6 +978,7 @@ void restoreRoleFilesHelper(std::string dirSrc, std::string dirToMove, std::stri
 namespace {
 enum class ServerRole {
 	ConsistencyCheck,
+	ConsistencyCheckUrgent,
 	CreateTemplateDatabase,
 	DSLTest,
 	FDBD,
@@ -1229,6 +1230,8 @@ struct CLIOptions {
 					role = ServerRole::KVFileDump;
 				else if (!strcmp(sRole, "consistencycheck"))
 					role = ServerRole::ConsistencyCheck;
+				else if (!strcmp(sRole, "consistencycheckUrgent"))
+					role = ServerRole::ConsistencyCheckUrgent;
 				else if (!strcmp(sRole, "unittests"))
 					role = ServerRole::UnitTests;
 				else if (!strcmp(sRole, "flowprocess"))
@@ -2177,16 +2180,26 @@ int main(int argc, char* argv[]) {
 			g_network->run();
 		} else if (role == ServerRole::ConsistencyCheck) {
 			setupRunLoopProfiler();
-
 			auto m = startSystemMonitor(opts.dataFolder, opts.dcId, opts.zoneId, opts.zoneId);
 			f = stopAfter(runTests(opts.connectionFile,
 			                       TEST_TYPE_CONSISTENCY_CHECK,
-			                       CLIENT_KNOBS->CONSISTENCY_CHECK_URGENT_MODE ? TEST_ON_TESTERS : TEST_HERE,
+			                       TEST_HERE,
 			                       1,
 			                       opts.testFile,
 			                       StringRef(),
 			                       opts.localities));
 			g_network->run();
+		} else if (role == ServerRole::ConsistencyCheckUrgent) {
+			setupRunLoopProfiler();
+			auto m = startSystemMonitor(opts.dataFolder, opts.dcId, opts.zoneId, opts.zoneId);
+			f = stopAfter(runTests(opts.connectionFile,
+			                       TEST_TYPE_CONSISTENCY_CHECK_URGENT,
+			                       TEST_ON_TESTERS,
+			                       opts.minTesterCount,
+			                       opts.testFile,
+			                       StringRef(),
+			                       opts.localities));
+			g_network->run();
 		} else if (role == ServerRole::UnitTests) {
 			setupRunLoopProfiler();
 			auto m = startSystemMonitor(opts.dataFolder, opts.dcId, opts.zoneId, opts.zoneId);
diff --git a/fdbserver/tester.actor.cpp b/fdbserver/tester.actor.cpp
index bd9f35d3ade..0ff00fa8ab8 100644
--- a/fdbserver/tester.actor.cpp
+++ b/fdbserver/tester.actor.cpp
@@ -28,6 +28,7 @@
 #include "flow/ActorCollection.h"
 #include "fdbrpc/sim_validation.h"
 #include "fdbrpc/simulator.h"
+#include "fdbclient/ConsistencyCheckUtil.actor.h"
 #include "fdbclient/ClusterInterface.h"
 #include "fdbclient/NativeAPI.actor.h"
 #include "fdbclient/SystemData.h"
@@ -557,6 +558,7 @@ ACTOR Future<Void> runWorkloadAsync(Database cx,
 				}
 			}
 			sendResult(setupReq, setupResult);
+			TraceEvent("TestSentResult", workIface.id()).detail("Workload", workload->description());
 		}
 		when(ReplyPromise<Void> req = waitNext(workIface.start.getFuture())) {
 			startReq = req;
@@ -955,10 +957,7 @@ ACTOR Future<Void> checkConsistency(Database cx,
 	options.push_back_deep(options.arena(),
 	                       KeyValueRef(LiteralStringRef("quiescentWaitTimeout"),
 	                                   ValueRef(options.arena(), format("%f", quiescentWaitTimeout))));
-	// options.push_back_deep(options.arena(), KeyValueRef(LiteralStringRef("distributed"), LiteralStringRef("false")));
-	options.push_back_deep(options.arena(),
-	                       KeyValueRef(LiteralStringRef("distributed"),
-	                                   StringRef(CLIENT_KNOBS->CONSISTENCY_CHECK_URGENT_MODE ? "true" : "false")));
+	options.push_back_deep(options.arena(), KeyValueRef(LiteralStringRef("distributed"), LiteralStringRef("false")));
 	spec.options.push_back_deep(spec.options.arena(), options);
 	state double start = now();
 	state bool lastRun = false;
@@ -1704,6 +1703,188 @@ ACTOR Future<Void> runTests(Reference<AsyncVar<Optional<struct ClusterController
 	return Void();
 }
 
+ACTOR Future<std::vector<KeyRange>> getConsistencyCheckShards(Database cx, std::vector<KeyRange> ranges) {
+	// Get the scope of the input list of ranges
+	state Key beginKeyToReadKeyServer;
+	state Key endKeyToReadKeyServer;
+	for (int i = 0; i < ranges.size(); i++) {
+		if (i == 0 || ranges[i].begin < beginKeyToReadKeyServer) {
+			beginKeyToReadKeyServer = ranges[i].begin;
+		}
+		if (i == 0 || ranges[i].end > endKeyToReadKeyServer) {
+			endKeyToReadKeyServer = ranges[i].end;
+		}
+	}
+	TraceEvent("ConsistencyCheck_GetConsistencyCheckShards")
+	    .detail("RangeBegin", beginKeyToReadKeyServer)
+	    .detail("RangeEnd", endKeyToReadKeyServer);
+	// Read KeyServer space within the scope and add shards intersecting with the input ranges
+	state std::vector<KeyRange> res;
+	state Transaction tr(cx);
+	loop {
+		try {
+			tr.setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
+			tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
+			tr.setOption(FDBTransactionOptions::LOCK_AWARE);
+			KeyRange rangeToRead = Standalone(KeyRangeRef(beginKeyToReadKeyServer, endKeyToReadKeyServer));
+			RangeResult readResult = wait(krmGetRanges(
+			    &tr, keyServersPrefix, rangeToRead, CLIENT_KNOBS->TOO_MANY, GetRangeLimits::BYTE_LIMIT_UNLIMITED));
+			for (int i = 0; i < readResult.size() - 1; ++i) {
+				KeyRange rangeToCheck = Standalone(KeyRangeRef(readResult[i].key, readResult[i + 1].key));
+				Value valueToCheck = Standalone(readResult[i].value);
+				bool toAdd = false;
+				for (const auto& range : ranges) {
+					if (rangeToCheck.intersects(range) == true) {
+						toAdd = true;
+						break;
+					}
+				}
+				if (toAdd == true) {
+					res.push_back(rangeToCheck);
+				}
+				beginKeyToReadKeyServer = readResult[i + 1].key;
+			}
+			if (beginKeyToReadKeyServer >= endKeyToReadKeyServer) {
+				break;
+			}
+		} catch (Error& e) {
+			TraceEvent("ConsistencyCheck_GetConsistencyCheckShardsRetry").error(e);
+			wait(tr.onError(e));
+		}
+	}
+	return res;
+}
+
+ACTOR Future<Void> runConsistencyCheckerUrgent(Reference<AsyncVar<Optional<struct ClusterControllerFullInterface>>> cc,
+                                               Reference<AsyncVar<Optional<struct ClusterInterface>>> ci,
+                                               std::vector<TestSpec> tests,
+                                               int minTestersExpected,
+                                               StringRef startingConfiguration,
+                                               LocalityData locality,
+                                               Optional<TenantName> defaultTenant,
+                                               uint64_t consistencyCheckerId) {
+	state Database cx;
+	state Reference<AsyncVar<ServerDBInfo>> dbInfo(new AsyncVar<ServerDBInfo>);
+	state Future<Void> ccMonitor = monitorServerDBInfo(cc, LocalityData(), dbInfo); // FIXME: locality
+	cx = openDBOnServer(dbInfo);
+	cx->defaultTenant = defaultTenant;
+	state int round = 0;
+	TraceEvent("ConsistencyChecker_RunUrgentStart").detail("ConsistencyCheckerId", consistencyCheckerId);
+
+	// Load ranges to check from progress metadata
+	state std::vector<KeyRange> rangesToCheck;
+	wait(store(rangesToCheck, loadRangesToCheckFromProgressMetadata(cx)));
+	if (CLIENT_KNOBS->CONSISTENCY_CHECK_URGENT_MODE_FORCE_INIT || rangesToCheck.size() == 0) {
+		// If force to start with blank or no range to check in progress data
+		// We load the range from knob
+		rangesToCheck = loadRangesToCheckFromKnob();
+		TraceEvent e("ConsistencyChecker_RunUrgentStart");
+		e.setMaxEventLength(-1);
+		e.setMaxFieldLength(-1);
+		e.detail("ConsistencyCheckerId", consistencyCheckerId);
+		for (int i = 0; i < rangesToCheck.size(); i++) {
+			e.detail("RangeToCheckBegin" + std::to_string(i), rangesToCheck[i].begin);
+			e.detail("RangeToCheckEnd" + std::to_string(i), rangesToCheck[i].end);
+		}
+		wait(initConsistencyCheckMetadata(cx, rangesToCheck, consistencyCheckerId));
+	} else {
+		TraceEvent("ConsistencyChecker_RunUrgentResume")
+		    .detail("ConsistencyCheckerId", consistencyCheckerId)
+		    .detail("WorkerCount", workers.size())
+		    .detail("RangesToCheckCount", rangesToCheck.size());
+	}
+
+	TraceEvent("ConsistencyChecker_RunUrgentInitialized").detail("ConsistencyCheckerId", consistencyCheckerId);
+
+	loop {
+		// Recruit workers
+		state int flags = GetWorkersRequest::TESTER_CLASS_ONLY | GetWorkersRequest::NON_EXCLUDED_PROCESSES_ONLY;
+		state Future<Void> testerTimeout = delay(600.0); // wait 600 sec for testers to show up
+		state std::vector<WorkerDetails> workers;
+		loop {
+			choose {
+				when(std::vector<WorkerDetails> w =
+				         wait(cc->get().present()
+				                  ? brokenPromiseToNever(cc->get().get().getWorkers.getReply(GetWorkersRequest(flags)))
+				                  : Never())) {
+					if (w.size() >= minTestersExpected) {
+						workers = w;
+						break;
+					}
+					wait(delay(SERVER_KNOBS->WORKER_POLL_DELAY));
+				}
+				when(wait(cc->onChange())) {}
+				when(wait(testerTimeout)) {
+					TraceEvent(SevError, "TesterRecruitmentTimeout").log();
+					throw timed_out();
+				}
+			}
+		}
+		TraceEvent("ConsistencyChecker_RunUrgentGotWorkers")
+		    .detail("ConsistencyCheckerId", consistencyCheckerId)
+		    .detail("Round", round)
+		    .detail("WorkerCount", workers.size());
+		state std::vector<TesterInterface> ts;
+		ts.reserve(workers.size());
+		for (int i = 0; i < workers.size(); i++)
+			ts.push_back(workers[i].interf.testerInterface);
+
+		// Load shards to check from keyserver space
+		state std::vector<KeyRange> shardsToCheck = wait(getConsistencyCheckShards(cx, rangesToCheck));
+		TraceEvent("ConsistencyChecker_RunUrgentGotShardsToCheck")
+		    .detail("ConsistencyCheckerId", consistencyCheckerId)
+		    .detail("Round", round)
+		    .detail("WorkerCount", workers.size())
+		    .detail("ShardCount", shardsToCheck.size());
+		// Assign shards to workers
+		state std::unordered_map<int, std::vector<KeyRange>> assignment;
+		int batchSize = shardsToCheck.size() / workers.size() + 1;
+		for (int i = 0; i < shardsToCheck.size(); i++) {
+			assignment[i / batchSize].push_back(shardsToCheck[i]);
+		}
+		state std::unordered_map<int, std::vector<KeyRange>>::iterator assignIt;
+		for (assignIt = assignment.begin(); assignIt != assignment.end(); assignIt++) {
+			if (assignIt->second.size() == 0) {
+				TraceEvent("ConsistencyCheck_ClientAssignedNoTask")
+				    .detail("ConsistencyCheckerId", consistencyCheckerId)
+				    .detail("ClientId", assignIt->first);
+				continue;
+			}
+			wait(persistConsistencyCheckAssignment(cx, assignIt->first, assignIt->second));
+		}
+		TraceEvent e("ConsistencyChecker_RunUrgentPersistAssignment");
+		e.detail("ConsistencyCheckerId", consistencyCheckerId);
+		e.detail("Round", round);
+		e.detail("WorkerCount", workers.size());
+		e.detail("ShardCountTotal", shardsToCheck.size());
+		for (const auto& [clientId, assignedShards] : assignment) {
+			e.detail("Client" + std::to_string(clientId), assignedShards.size());
+		}
+		// Run tests and wait for tester complete/fail
+		wait(runTests(cc, ci, ts, tests, startingConfiguration, locality, defaultTenant));
+		TraceEvent("ConsistencyChecker_RunUrgentRoundComplete")
+		    .detail("ConsistencyCheckerId", consistencyCheckerId)
+		    .detail("Round", round);
+		round++;
+
+		// Load progress metadata. If no unfinished range, exit; otherwise, continue
+		rangesToCheck.clear();
+		wait(store(rangesToCheck, loadRangesToCheckFromProgressMetadata(cx)));
+		if (rangesToCheck.size() == 0) {
+			TraceEvent("ConsistencyChecker_RunUrgentComplete")
+			    .detail("ConsistencyCheckerId", consistencyCheckerId)
+			    .detail("Round", round);
+			break;
+		} else {
+			TraceEvent("ConsistencyChecker_RunUrgentContinue")
+			    .detail("ConsistencyCheckerId", consistencyCheckerId)
+			    .detail("Round", round)
+			    .detail("RangesToCheckCount", rangesToCheck.size());
+		}
+	}
+	return Void();
+}
+
 /**
  * \brief Set up testing environment and run the given tests on a cluster.
  *
@@ -1748,8 +1929,37 @@ ACTOR Future<Void> runTests(Reference<IClusterConnectionRecord> connRecord,
 		actors.push_back(reportErrors(monitorLeader(connRecord, cc), "MonitorLeader"));
 		actors.push_back(reportErrors(extractClusterInterface(cc, ci), "ExtractClusterInterface"));
 	}
+	state uint64_t consistencyCheckerId = 0;
 
-	if (whatToRun == TEST_TYPE_CONSISTENCY_CHECK) {
+	if (whatToRun == TEST_TYPE_CONSISTENCY_CHECK_URGENT) {
+		while (consistencyCheckerId == 0)
+			consistencyCheckerId = deterministicRandom()->randomUInt64();
+		// consistencyCheckerId must be not 0, indicating this is in urgent mode of consistency checker
+		TestSpec spec;
+		Standalone<VectorRef<KeyValueRef>> options;
+		spec.title = LiteralStringRef("ConsistencyCheckUrgent");
+		spec.databasePingDelay = 0;
+		spec.timeout = 0;
+		spec.waitForQuiescenceBegin = false;
+		spec.waitForQuiescenceEnd = false;
+		std::string rateLimitMax = format("%d", CLIENT_KNOBS->CONSISTENCY_CHECK_RATE_LIMIT_MAX);
+		options.push_back_deep(options.arena(),
+		                       KeyValueRef(LiteralStringRef("testName"), LiteralStringRef("ConsistencyCheckUrgent")));
+		options.push_back_deep(
+		    options.arena(),
+		    KeyValueRef(LiteralStringRef("consistencyCheckerId"), StringRef(std::to_string(consistencyCheckerId))));
+		options.push_back_deep(options.arena(),
+		                       KeyValueRef(LiteralStringRef("performQuiescentChecks"), LiteralStringRef("false")));
+		options.push_back_deep(options.arena(), KeyValueRef(LiteralStringRef("distributed"), LiteralStringRef("true")));
+		options.push_back_deep(options.arena(),
+		                       KeyValueRef(LiteralStringRef("failureIsError"), LiteralStringRef("true")));
+		options.push_back_deep(options.arena(), KeyValueRef(LiteralStringRef("indefinite"), LiteralStringRef("false")));
+		options.push_back_deep(options.arena(), KeyValueRef(LiteralStringRef("rateLimitMax"), StringRef(rateLimitMax)));
+		options.push_back_deep(options.arena(),
+		                       KeyValueRef(LiteralStringRef("shuffleShards"), LiteralStringRef("false")));
+		spec.options.push_back_deep(spec.options.arena(), options);
+		testSet.testSpecs.push_back(spec);
+	} else if (whatToRun == TEST_TYPE_CONSISTENCY_CHECK) {
 		TestSpec spec;
 		Standalone<VectorRef<KeyValueRef>> options;
 		spec.title = LiteralStringRef("ConsistencyCheck");
@@ -1760,13 +1970,13 @@ ACTOR Future<Void> runTests(Reference<IClusterConnectionRecord> connRecord,
 		std::string rateLimitMax = format("%d", CLIENT_KNOBS->CONSISTENCY_CHECK_RATE_LIMIT_MAX);
 		options.push_back_deep(options.arena(),
 		                       KeyValueRef(LiteralStringRef("testName"), LiteralStringRef("ConsistencyCheck")));
+		options.push_back_deep(
+		    options.arena(),
+		    KeyValueRef(LiteralStringRef("consistencyCheckerId"), StringRef(std::to_string(consistencyCheckerId))));
 		options.push_back_deep(options.arena(),
 		                       KeyValueRef(LiteralStringRef("performQuiescentChecks"), LiteralStringRef("false")));
-		// options.push_back_deep(options.arena(),
-		//                        KeyValueRef(LiteralStringRef("distributed"), LiteralStringRef("false")));
 		options.push_back_deep(options.arena(),
-		                       KeyValueRef(LiteralStringRef("distributed"),
-		                                   StringRef(CLIENT_KNOBS->CONSISTENCY_CHECK_URGENT_MODE ? "true" : "false")));
+		                       KeyValueRef(LiteralStringRef("distributed"), LiteralStringRef("false")));
 		options.push_back_deep(options.arena(),
 		                       KeyValueRef(LiteralStringRef("failureIsError"), LiteralStringRef("true")));
 		options.push_back_deep(options.arena(), KeyValueRef(LiteralStringRef("indefinite"), LiteralStringRef("true")));
@@ -1819,7 +2029,17 @@ ACTOR Future<Void> runTests(Reference<IClusterConnectionRecord> connRecord,
 
 	knobProtectiveGroup = std::make_unique<KnobProtectiveGroup>(testSet.overrideKnobs);
 	Future<Void> tests;
-	if (at == TEST_HERE) {
+	if (whatToRun == TEST_TYPE_CONSISTENCY_CHECK_URGENT) {
+		tests = reportErrors(runConsistencyCheckerUrgent(cc,
+		                                                 ci,
+		                                                 testSet.testSpecs,
+		                                                 minTestersExpected,
+		                                                 startingConfiguration,
+		                                                 locality,
+		                                                 defaultTenant,
+		                                                 consistencyCheckerId),
+		                     "runConsistencyCheckerUrgent");
+	} else if (at == TEST_HERE) {
 		auto db = makeReference<AsyncVar<ServerDBInfo>>();
 		std::vector<TesterInterface> iTesters(1);
 		actors.push_back(
diff --git a/fdbserver/workloads/ConsistencyCheck.actor.cpp b/fdbserver/workloads/ConsistencyCheck.actor.cpp
index 47b2e996dda..140cb498f70 100644
--- a/fdbserver/workloads/ConsistencyCheck.actor.cpp
+++ b/fdbserver/workloads/ConsistencyCheck.actor.cpp
@@ -34,6 +34,7 @@
 #include "fdbserver/QuietDatabase.h"
 #include "fdbserver/TSSMappingUtil.actor.h"
 #include "flow/DeterministicRandom.h"
+#include "fdbclient/ConsistencyCheckUtil.actor.h"
 #include "fdbclient/ManagementAPI.actor.h"
 #include "fdbclient/StorageServerInterface.h"
 #include "flow/network.h"
@@ -43,51 +44,6 @@
 // #define SevCCheckInfo SevVerbose
 #define SevCCheckInfo SevInfo
 
-const std::unordered_map<char, uint8_t> parseCharMap{
-	{ '0', 0 },  { '1', 1 },  { '2', 2 },  { '3', 3 },  { '4', 4 },  { '5', 5 },  { '6', 6 },  { '7', 7 },
-	{ '8', 8 },  { '9', 9 },  { 'a', 10 }, { 'b', 11 }, { 'c', 12 }, { 'd', 13 }, { 'e', 14 }, { 'f', 15 },
-	{ 'A', 10 }, { 'B', 11 }, { 'C', 12 }, { 'D', 13 }, { 'E', 14 }, { 'F', 15 },
-};
-
-Key getKeyFromString(const std::string& str) {
-	Key emptyKey;
-	if (str.size() == 0) {
-		return emptyKey;
-	}
-	if (str.size() % 4 != 0) {
-		TraceEvent(SevWarnAlways, "ConsistencyCheck_GetKeyFromStringError")
-		    .setMaxEventLength(-1)
-		    .setMaxFieldLength(-1)
-		    .detail("Reason", "WrongLength")
-		    .detail("InputStr", str);
-		throw internal_error();
-	}
-	std::vector<uint8_t> byteList;
-	for (int i = 0; i < str.size(); i += 4) {
-		if (str.at(i + 0) != '\\' || str.at(i + 1) != 'x') {
-			TraceEvent(SevWarnAlways, "ConsistencyCheck_GetKeyFromStringError")
-			    .setMaxEventLength(-1)
-			    .setMaxFieldLength(-1)
-			    .detail("Reason", "WrongBytePrefix")
-			    .detail("InputStr", str);
-			throw internal_error();
-		}
-		const char first = str.at(i + 2);
-		const char second = str.at(i + 3);
-		if (parseCharMap.count(first) == 0 || parseCharMap.count(second) == 0) {
-			TraceEvent(SevWarnAlways, "ConsistencyCheck_GetKeyFromStringError")
-			    .setMaxEventLength(-1)
-			    .setMaxFieldLength(-1)
-			    .detail("Reason", "WrongByteContent")
-			    .detail("InputStr", str);
-			throw internal_error();
-		}
-		uint8_t parsedValue = parseCharMap.at(first) * 16 + parseCharMap.at(second);
-		byteList.push_back(parsedValue);
-	}
-	return Standalone(StringRef(byteList.data(), byteList.size()));
-}
-
 struct ConsistencyCheckWorkload : TestWorkload {
 	// Whether or not we should perform checks that will only pass if the database is in a quiescent state
 	bool performQuiescentChecks;
@@ -141,6 +97,8 @@ struct ConsistencyCheckWorkload : TestWorkload {
 
 	Future<Void> monitorConsistencyCheckSettingsActor;
 
+	uint64_t consistencyCheckerId;
+
 	ConsistencyCheckWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
 		performQuiescentChecks = getOption(options, LiteralStringRef("performQuiescentChecks"), false);
 		performCacheCheck = getOption(options, LiteralStringRef("performCacheCheck"), false);
@@ -152,6 +110,7 @@ struct ConsistencyCheckWorkload : TestWorkload {
 		rateLimitMax = getOption(options, LiteralStringRef("rateLimitMax"), 0);
 		shuffleShards = getOption(options, LiteralStringRef("shuffleShards"), false);
 		indefinite = getOption(options, LiteralStringRef("indefinite"), false);
+		consistencyCheckerId = getOption(options, LiteralStringRef("consistencyCheckerId"), (uint64_t)0);
 		suspendConsistencyCheck.set(true);
 
 		success = true;
@@ -233,6 +192,7 @@ struct ConsistencyCheckWorkload : TestWorkload {
 		loop {
 			while (self->suspendConsistencyCheck.get()) {
 				TraceEvent("ConsistencyCheck_Suspended")
+				    .detail("ConsistencyCheckerId", self->consistencyCheckerId)
 				    .detail("Distributed", self->distributed)
 				    .detail("ClientCount", self->clientCount)
 				    .detail("ClientId", self->clientId)
@@ -241,6 +201,7 @@ struct ConsistencyCheckWorkload : TestWorkload {
 				wait(self->suspendConsistencyCheck.onChange());
 			}
 			TraceEvent("ConsistencyCheck_StartingOrResuming")
+			    .detail("ConsistencyCheckerId", self->consistencyCheckerId)
 			    .detail("Distributed", self->distributed)
 			    .detail("ClientCount", self->clientCount)
 			    .detail("ClientId", self->clientId)
@@ -248,10 +209,12 @@ struct ConsistencyCheckWorkload : TestWorkload {
 			    .detail("Repetitions", self->repetitions);
 			choose {
 				when(wait(self->runCheck(cx, self))) {
-					if (!self->indefinite || CLIENT_KNOBS->CONSISTENCY_CHECK_URGENT_MODE) {
+					if (!self->indefinite ||
+					    (self->consistencyCheckerId != 0)) { // self->consistencyCheckerId != 0 when urgent mode
 						// When consistency checker is distributed, repeated running
 						// can miss ranges, so we exit here
 						TraceEvent("ConsistencyCheck_Exit")
+						    .detail("ConsistencyCheckerId", self->consistencyCheckerId)
 						    .detail("Distributed", self->distributed)
 						    .detail("ClientCount", self->clientCount)
 						    .detail("ClientId", self->clientId)
@@ -268,110 +231,16 @@ struct ConsistencyCheckWorkload : TestWorkload {
 		return Void();
 	}
 
-	std::vector<KeyRange> loadRangesToCheckFromKnob() {
-		// Load string from knob
-		std::vector<std::string> beginKeyStrs = {
-			CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_BEGIN_0,  CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_BEGIN_1,
-			CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_BEGIN_2,  CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_BEGIN_3,
-			CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_BEGIN_4,  CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_BEGIN_5,
-			CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_BEGIN_6,  CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_BEGIN_7,
-			CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_BEGIN_8,  CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_BEGIN_9,
-			CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_BEGIN_10, CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_BEGIN_11,
-			CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_BEGIN_12, CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_BEGIN_13,
-			CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_BEGIN_14, CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_BEGIN_15,
-			CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_BEGIN_16, CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_BEGIN_17,
-			CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_BEGIN_18, CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_BEGIN_19,
-			CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_BEGIN_20, CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_BEGIN_21,
-			CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_BEGIN_22, CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_BEGIN_23,
-			CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_BEGIN_24, CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_BEGIN_25,
-			CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_BEGIN_26,
-		};
-		std::vector<std::string> endKeyStrs = {
-			CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_END_0,  CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_END_1,
-			CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_END_2,  CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_END_3,
-			CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_END_4,  CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_END_5,
-			CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_END_6,  CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_END_7,
-			CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_END_8,  CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_END_9,
-			CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_END_10, CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_END_11,
-			CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_END_12, CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_END_13,
-			CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_END_14, CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_END_15,
-			CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_END_16, CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_END_17,
-			CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_END_18, CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_END_19,
-			CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_END_20, CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_END_21,
-			CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_END_22, CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_END_23,
-			CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_END_24, CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_END_25,
-			CLIENT_KNOBS->CONSISTENCY_CHECK_RANGE_END_26,
-		};
-
-		// Get keys from strings
-		std::vector<Key> beginKeys;
-		for (const auto& beginKeyStr : beginKeyStrs) {
-			Key key = getKeyFromString(beginKeyStr);
-			beginKeys.push_back(key);
-		}
-		std::vector<Key> endKeys;
-		for (const auto& endKeyStr : endKeyStrs) {
-			Key key = getKeyFromString(endKeyStr);
-			endKeys.push_back(key);
-		}
-		ASSERT(beginKeys.size() == endKeys.size());
-
-		// Get ranges
-		KeyRangeMap<bool> rangeToCheckMap;
-		for (int i = 0; i < beginKeys.size(); i++) {
-			Key rangeBegin = beginKeys[i];
-			Key rangeEnd = endKeys[i];
-			if (rangeBegin.empty() && rangeEnd.empty()) {
-				continue;
-			}
-			if (rangeBegin > allKeys.end) {
-				rangeBegin = allKeys.end;
-			}
-			if (rangeEnd > allKeys.end) {
-				TraceEvent("ConsistencyCheck_ReverseInputRange")
-				    .setMaxEventLength(-1)
-				    .setMaxFieldLength(-1)
-				    .detail("Index", i)
-				    .detail("RangeBegin", rangeBegin)
-				    .detail("RangeEnd", rangeEnd);
-				rangeEnd = allKeys.end;
-			}
-
-			KeyRange rangeToCheck;
-			if (rangeBegin < rangeEnd) {
-				rangeToCheck = Standalone(KeyRangeRef(rangeBegin, rangeEnd));
-			} else if (rangeBegin > rangeEnd) {
-				rangeToCheck = Standalone(KeyRangeRef(rangeEnd, rangeBegin));
-			} else {
-				TraceEvent("ConsistencyCheck_EmptyInputRange")
-				    .setMaxEventLength(-1)
-				    .setMaxFieldLength(-1)
-				    .detail("Index", i)
-				    .detail("RangeBegin", rangeBegin)
-				    .detail("RangeEnd", rangeEnd);
-				continue;
-			}
-			rangeToCheckMap.insert(rangeToCheck, true);
-		}
-
-		rangeToCheckMap.coalesce(allKeys);
-
-		std::vector<KeyRange> res;
-		for (auto rangeToCheck : rangeToCheckMap.ranges()) {
-			if (rangeToCheck.value() == true) {
-				res.push_back(rangeToCheck.range());
-			}
-		}
-		TraceEvent e("ConsistencyCheck_LoadedInputRange");
-		e.setMaxEventLength(-1);
-		e.setMaxFieldLength(-1);
-		for (int i = 0; i < res.size(); i++) {
-			e.detail("RangeBegin" + std::to_string(i), res[i].begin);
-			e.detail("RangeEnd" + std::to_string(i), res[i].end);
-		}
+	ACTOR Future<std::vector<KeyRange>> loadRangesFromAssignmentMetadata(Database cx, int clientId) {
+		std::vector<KeyRange> res = wait(loadRangesToCheckFromAssignmentMetadata(cx, clientId));
 		return res;
 	}
 
+	ACTOR Future<Void> persistProgress(Database cx, KeyRange range, uint64_t consistencyCheckerId) {
+		wait(persistConsistencyCheckProgress(cx, range, consistencyCheckerId));
+		return Void();
+	}
+
 	ACTOR Future<Void> runCheck(Database cx, ConsistencyCheckWorkload* self) {
 		TEST(self->performQuiescentChecks); // Quiescent consistency check
 		TEST(!self->performQuiescentChecks); // Non-quiescent consistency check
@@ -485,11 +354,27 @@ struct ConsistencyCheckWorkload : TestWorkload {
 						self->testFailure("Coordinators incorrect");
 				}
 
-				if (CLIENT_KNOBS->CONSISTENCY_CHECK_URGENT_MODE) {
-					std::vector<KeyRange> rangesToCheck = self->loadRangesToCheckFromKnob();
+				if (self->consistencyCheckerId != 0) { // Urgent mode
+					std::vector<KeyRange> rangesToCheck =
+					    wait(self->loadRangesFromAssignmentMetadata(cx, self->clientId));
+					if (rangesToCheck.size() == 0) {
+						TraceEvent("ConsistencyCheck_ConfigWithEmptyRangeToCheck")
+						    .setMaxEventLength(-1)
+						    .setMaxFieldLength(-1)
+						    .detail("ConsistencyCheckerId", self->consistencyCheckerId)
+						    .detail("Indefinite", self->indefinite)
+						    .detail("Distributed", self->distributed)
+						    .detail("ClientCount", self->clientCount)
+						    .detail("ClientId", self->clientId)
+						    .detail("PerformQuiescentChecks", self->performQuiescentChecks)
+						    .detail("PerformTSSCheck", self->performTSSCheck)
+						    .detail("PerformCacheCheck", self->performCacheCheck);
+						return Void();
+					}
 					TraceEvent("ConsistencyCheck_Config")
 					    .setMaxEventLength(-1)
 					    .setMaxFieldLength(-1)
+					    .detail("ConsistencyCheckerId", self->consistencyCheckerId)
 					    .detail("RangeToCheck", describe(rangesToCheck))
 					    .detail("Indefinite", self->indefinite)
 					    .detail("Distributed", self->distributed)
@@ -550,6 +435,7 @@ struct ConsistencyCheckWorkload : TestWorkload {
 				    e.code() == error_code_process_behind)
 					TraceEvent("ConsistencyCheck_Retry")
 					    .error(e) // FIXME: consistency check does not retry in this case
+					    .detail("ConsistencyCheckerId", self->consistencyCheckerId)
 					    .detail("Distributed", self->distributed)
 					    .detail("ClientCount", self->clientCount)
 					    .detail("ClientId", self->clientId)
@@ -561,6 +447,7 @@ struct ConsistencyCheckWorkload : TestWorkload {
 		}
 
 		TraceEvent("ConsistencyCheck_FinishedCheck")
+		    .detail("ConsistencyCheckerId", self->consistencyCheckerId)
 		    .detail("Distributed", self->distributed)
 		    .detail("ClientCount", self->clientCount)
 		    .detail("ClientId", self->clientId)
@@ -1296,7 +1183,7 @@ struct ConsistencyCheckWorkload : TestWorkload {
 		state Transaction tr(cx);
 		loop {
 			try {
-				tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
+				tr.setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
 				tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
 				tr.setOption(FDBTransactionOptions::LOCK_AWARE);
 				KeyRange rangeToRead = Standalone(KeyRangeRef(beginKeyToReadKeyServer, endKeyToReadKeyServer));
@@ -1469,7 +1356,7 @@ struct ConsistencyCheckWorkload : TestWorkload {
 	                                        ConsistencyCheckWorkload* self,
 	                                        int consistencyCheckEpoch) {
 		if (consistencyCheckEpoch > 0) {
-			ASSERT(CLIENT_KNOBS->CONSISTENCY_CHECK_URGENT_MODE);
+			ASSERT(self->consistencyCheckerId != 0); // urgent mode
 		}
 		// Stores the total number of bytes on each storage server
 		// In a distributed test, this will be an estimated size
@@ -1490,12 +1377,11 @@ struct ConsistencyCheckWorkload : TestWorkload {
 		              self->rateLimitMax,
 		              static_cast<int>(ceil(self->bytesReadInPreviousRound /
 		                                    (float)CLIENT_KNOBS->CONSISTENCY_CHECK_ONE_ROUND_TARGET_COMPLETION_TIME)));
-		if (consistencyCheckEpoch > 0 ||
-		    CLIENT_KNOBS->CONSISTENCY_CHECK_URGENT_MODE) { // We set max speed when retrying failed ranges or
-			                                               // distributed among testers
+		if (self->consistencyCheckerId != 0) { // We set max speed when urgent mode
 			rateLimitForThisRound = self->rateLimitMax;
 		}
 		TraceEvent("ConsistencyCheck_RateLimitForThisRound")
+		    .detail("ConsistencyCheckerId", self->consistencyCheckerId)
 		    .detail("Distributed", self->distributed)
 		    .detail("ClientId", self->clientId)
 		    .detail("ClientCount", self->clientCount)
@@ -1531,98 +1417,56 @@ struct ConsistencyCheckWorkload : TestWorkload {
 		shardOrder.reserve(ranges.size());
 		for (int k = 0; k < ranges.size(); k++)
 			shardOrder.push_back(k);
-		if (self->shuffleShards && !CLIENT_KNOBS->CONSISTENCY_CHECK_URGENT_MODE) {
+
+		if (self->shuffleShards) {
+			ASSERT(self->consistencyCheckerId != 0); // Urgent mode never does shuffle
 			uint32_t seed = self->sharedRandomNumber + self->repetitions;
 			DeterministicRandom sharedRandom(seed == 0 ? 1 : seed);
 			sharedRandom.randomShuffle(shardOrder);
 		}
 
-		state int endPoint;
-		if (CLIENT_KNOBS->CONSISTENCY_CHECK_URGENT_MODE) {
-			if (consistencyCheckEpoch == 0) {
-				// Decide which ranges are assigned
-				int batchSize = (shardOrder.size() / self->clientCount) + 1;
-				i = self->clientId * batchSize; // overwrite the starting point i
-				i = std::max(0, i - CLIENT_KNOBS->CONSISTENCY_CHECK_DISTRIBUTED_WIGGLE_ROOM);
-				endPoint = (self->clientId + 1) * batchSize;
-				endPoint = std::min(static_cast<int>(shardOrder.size()),
-				                    endPoint + CLIENT_KNOBS->CONSISTENCY_CHECK_DISTRIBUTED_WIGGLE_ROOM);
-				increment = 1;
-				// If the start point i is larger than the list boundary, this client needs not run
-				if (i >= endPoint) {
-					TraceEvent("ConsistencyCheck_ClientExit")
-					    .detail("Reason", "No task assigned")
-					    .detail("Distributed", self->distributed)
-					    .detail("ShardCount", ranges.size())
-					    .detail("ClientId", self->clientId)
-					    .detail("ClientCount", self->clientCount)
-					    .detail("Repetitions", self->repetitions);
-					return true;
+		if (self->consistencyCheckerId != 0) {
+			// In case of the urgent mode, overwrite the i and increment
+			i = 0;
+			increment = 1;
+			int endPoint = shardOrder.size();
+			numShardThisClient = (endPoint - i) / increment;
+			// Logging range assignment
+			KeyRangeMap<bool> rangeAssignmentMap;
+			std::vector<KeyRange> rangeAssignmentList;
+			for (int idx = i; idx < endPoint; idx += increment) {
+				rangeAssignmentMap.insert(ranges[shardOrder[idx]], true);
+			}
+			rangeAssignmentMap.coalesce(allKeys);
+			for (auto& assignedRange : rangeAssignmentMap.ranges()) {
+				if (assignedRange.value() == true) {
+					rangeAssignmentList.push_back(assignedRange.range());
 				}
-
-				// Logging range assignment
-				KeyRangeMap<bool> rangeAssignmentMap;
-				std::vector<KeyRange> rangeAssignmentList;
-				// Decide assigned ranges to log
-				for (int idx = i; idx < endPoint; idx += increment) {
-					rangeAssignmentMap.insert(ranges[shardOrder[idx]], true);
-				}
-				rangeAssignmentMap.coalesce(allKeys);
-				for (auto& assignedRange : rangeAssignmentMap.ranges()) {
-					if (assignedRange.value() == true) {
-						rangeAssignmentList.push_back(assignedRange.range());
-					}
-				}
-				numShardThisClient = (endPoint - i) / increment;
-				TraceEvent e("ConsistencyCheck_StartTask");
-				e.setMaxEventLength(-1);
-				e.setMaxFieldLength(-1);
-				e.detail("Distributed", self->distributed);
-				e.detail("ShardCount", ranges.size());
-				e.detail("ClientId", self->clientId);
-				e.detail("ClientCount", self->clientCount);
-				e.detail("StartPoint", i);
-				e.detail("EndPoint", endPoint);
-				e.detail("Repetitions", self->repetitions);
-				e.detail("ConsistencyCheckEpoch", consistencyCheckEpoch);
+			}
+			TraceEvent e(consistencyCheckEpoch == 0 ? "ConsistencyCheck_StartTask"
+			                                        : "ConsistencyCheck_RetryFailedTask");
+			e.setMaxEventLength(-1);
+			e.setMaxFieldLength(-1);
+			e.detail("ConsistencyCheckerId", self->consistencyCheckerId);
+			e.detail("Distributed", self->distributed);
+			e.detail("ShardCount", ranges.size());
+			e.detail("ClientId", self->clientId);
+			e.detail("ClientCount", self->clientCount);
+			e.detail("Repetitions", self->repetitions);
+			e.detail("ConsistencyCheckEpoch", consistencyCheckEpoch);
+			if (consistencyCheckEpoch == 0) {
 				for (int idx = 0; idx < rangeAssignmentList.size(); idx++) {
 					e.detail("RangeToCheckBegin" + std::to_string(idx), rangeAssignmentList[idx].begin);
 					e.detail("RangeToCheckEnd" + std::to_string(idx), rangeAssignmentList[idx].end);
 				}
-				e.trackLatest("ConsistencyCheck_StartTask");
-			} else { // consistencyCheckEpoch > 0 when handling failure ranges
-				i = 0;
-				increment = 1;
-				endPoint = shardOrder.size();
-				TraceEvent("ConsistencyCheck_RetryFailedTask")
-				    .setMaxEventLength(-1)
-				    .setMaxFieldLength(-1)
-				    .detail("Distributed", self->distributed)
-				    .detail("RetryShardCount", ranges.size())
-				    .detail("ClientId", self->clientId)
-				    .detail("ClientCount", self->clientCount)
-				    .detail("Repetitions", self->repetitions)
-				    .detail("ConsistencyCheckEpoch", consistencyCheckEpoch)
-				    .trackLatest("ConsistencyCheck_RetryFailedTask");
 			}
-		} else {
-			endPoint = shardOrder.size();
+			e.trackLatest(consistencyCheckEpoch == 0 ? "ConsistencyCheck_StartTask"
+			                                         : "ConsistencyCheck_RetryFailedTask");
 		}
 
-		for (; i < endPoint; i += increment) {
-			if (CLIENT_KNOBS->CONSISTENCY_CHECK_URGENT_MODE && self->suspendConsistencyCheck.get()) {
-				TraceEvent("ConsistencyCheck_Cancelled")
-				    .setMaxEventLength(-1)
-				    .setMaxFieldLength(-1)
-				    .detail("Distributed", self->distributed)
-				    .detail("ShardCount", ranges.size())
-				    .detail("ClientId", self->clientId)
-				    .detail("ClientCount", self->clientCount)
-				    .detail("CompleteEndKey", ranges[shardOrder[i]].begin);
-				break;
-			}
-			if (CLIENT_KNOBS->CONSISTENCY_CHECK_URGENT_MODE) {
-				numShardToCheck = (endPoint - i) / increment;
+		for (; i < shardOrder.size(); i += increment) {
+			if (self->consistencyCheckerId != 0) { // Urgent mode
+				numShardToCheck = (shardOrder.size() - i) / increment;
 			}
 
 			state int shard = shardOrder[i];
@@ -1763,7 +1607,7 @@ struct ConsistencyCheckWorkload : TestWorkload {
 
 			// The first client may need to skip the rest of the loop contents if it is just processing this shard to
 			// get a size estimate
-			if (CLIENT_KNOBS->CONSISTENCY_CHECK_URGENT_MODE || !self->firstClient ||
+			if (self->consistencyCheckerId != 0 || !self->firstClient ||
 			    shard % (effectiveClientCount * self->shardSampleFactor) == 0) {
 				state int shardKeys = 0;
 				state int shardBytes = 0;
@@ -1786,18 +1630,6 @@ struct ConsistencyCheckWorkload : TestWorkload {
 				// Read a limited number of entries at a time, repeating until all keys in the shard have been read
 				loop {
 					try {
-						if (CLIENT_KNOBS->CONSISTENCY_CHECK_URGENT_MODE && self->suspendConsistencyCheck.get()) {
-							TraceEvent("ConsistencyCheck_DataCheckCancelled")
-							    .setMaxEventLength(-1)
-							    .setMaxFieldLength(-1)
-							    .detail("Distributed", self->distributed)
-							    .detail("ShardCount", ranges.size())
-							    .detail("ClientId", self->clientId)
-							    .detail("ClientCount", self->clientCount)
-							    .detail("CompleteEndKey", lastStartSampleKey);
-							break;
-						}
-
 						lastSampleKey = lastStartSampleKey;
 
 						// Get the min version of the storage servers
@@ -2005,7 +1837,7 @@ struct ConsistencyCheckWorkload : TestWorkload {
 								    .detail("ConsistencyCheckEpoch", consistencyCheckEpoch);
 
 								completeCheck = false;
-								if (CLIENT_KNOBS->CONSISTENCY_CHECK_URGENT_MODE) {
+								if (self->consistencyCheckerId != 0) {
 									break; // Will retry later
 								}
 
@@ -2017,18 +1849,20 @@ struct ConsistencyCheckWorkload : TestWorkload {
 
 							} else {
 								completeCheck = false;
-								if (CLIENT_KNOBS->CONSISTENCY_CHECK_URGENT_MODE) {
+								if (self->consistencyCheckerId != 0) {
 									break; // Will retry later
 								}
 							}
 						}
 
-						if (CLIENT_KNOBS->CONSISTENCY_CHECK_URGENT_MODE && !completeCheck) {
+						if (self->consistencyCheckerId != 0 && !completeCheck) {
+							// Shard incomplete check in urgent mode
 							// Add the shard to failedRanges, which will be retried later
 							failedRanges.insert(range, true);
 							TraceEvent(SevInfo, "ConsistencyCheck_ShardAddedToRetry")
 							    .setMaxEventLength(-1)
 							    .setMaxFieldLength(-1)
+							    .detail("ConsistencyCheckerId", self->consistencyCheckerId)
 							    .detail("ClientId", self->clientId)
 							    .detail("ClientCount", self->clientCount)
 							    .detail("ShardBegin", range.begin)
@@ -2039,7 +1873,7 @@ struct ConsistencyCheckWorkload : TestWorkload {
 							break; // skip the entire shard, retry later
 						}
 
-						if (firstValidServer >= 0 && !CLIENT_KNOBS->CONSISTENCY_CHECK_URGENT_MODE) {
+						if (firstValidServer >= 0 && self->consistencyCheckerId == 0) {
 							VectorRef<KeyValueRef> data = keyValueFutures[firstValidServer].get().get().data;
 							// Calculate the size of the shard, the variance of the shard size estimate, and the correct
 							// shard size estimate
@@ -2117,11 +1951,11 @@ struct ConsistencyCheckWorkload : TestWorkload {
 
 				if (!completeCheck) {
 					numFailedShards++;
-					TraceEvent(CLIENT_KNOBS->CONSISTENCY_CHECK_URGENT_MODE ? SevInfo : SevWarn,
-					           "ConsistencyCheck_ShardFailed")
+					TraceEvent(self->consistencyCheckerId != 0 ? SevInfo : SevWarn, "ConsistencyCheck_ShardFailed")
 					    .suppressFor(1.0)
 					    .setMaxEventLength(-1)
 					    .setMaxFieldLength(-1)
+					    .detail("ConsistencyCheckerId", self->consistencyCheckerId)
 					    .detail("ClientId", self->clientId)
 					    .detail("ClientCount", self->clientCount)
 					    .detail("NumCompletedShards", numCompleteShards)
@@ -2134,10 +1968,24 @@ struct ConsistencyCheckWorkload : TestWorkload {
 					    .detail("ConsistencyCheckEpoch", consistencyCheckEpoch);
 				} else {
 					numCompleteShards++;
+					if (self->consistencyCheckerId != 0) {
+						// Persist progress in urgent mode
+						try {
+							wait(self->persistProgress(cx, range, self->consistencyCheckerId));
+						} catch (Error& e) {
+							if (e.code() == error_code_key_not_found ||
+							    e.code() == error_code_consistency_check_task_outdated) {
+								return testResult;
+							} else {
+								throw e;
+							}
+						}
+					}
 					TraceEvent(SevInfo, "ConsistencyCheck_ShardComplete")
 					    .suppressFor(1.0)
 					    .setMaxEventLength(-1)
 					    .setMaxFieldLength(-1)
+					    .detail("ConsistencyCheckerId", self->consistencyCheckerId)
 					    .detail("ClientId", self->clientId)
 					    .detail("ClientCount", self->clientCount)
 					    .detail("Range", range)
@@ -2151,7 +1999,7 @@ struct ConsistencyCheckWorkload : TestWorkload {
 					    .detail("ConsistencyCheckEpoch", consistencyCheckEpoch);
 				}
 
-				if (CLIENT_KNOBS->CONSISTENCY_CHECK_URGENT_MODE) {
+				if (self->consistencyCheckerId != 0) { // Urgent mode
 					continue; // Bypass following check not-relevant to userdata/metadata consistency
 				}
 
@@ -2276,7 +2124,7 @@ struct ConsistencyCheckWorkload : TestWorkload {
 			}
 		}
 
-		if (CLIENT_KNOBS->CONSISTENCY_CHECK_URGENT_MODE) {
+		if (self->consistencyCheckerId != 0) {
 			failedRanges.coalesce(allKeys);
 			state std::vector<KeyRange> failedRangesToCheck;
 			state KeyRangeMap<bool>::Ranges failedRangesList = failedRanges.ranges();
@@ -2294,6 +2142,7 @@ struct ConsistencyCheckWorkload : TestWorkload {
 				TraceEvent(SevInfo, "ConsistencyCheck_StartHandlingFailedRanges")
 				    .setMaxEventLength(-1)
 				    .setMaxFieldLength(-1)
+				    .detail("ConsistencyCheckerId", self->consistencyCheckerId)
 				    .detail("FailedCollectedRangeCount", failedRangesToCheck.size())
 				    .detail("FailedShardCount", shardLocationPairListForFailedRanges.size())
 				    .detail("ConsistencyCheckEpoch", consistencyCheckEpoch)
@@ -2313,6 +2162,7 @@ struct ConsistencyCheckWorkload : TestWorkload {
 						TraceEvent(SevWarnAlways, "ConsistencyCheck_MissingRange")
 						    .setMaxEventLength(-1)
 						    .setMaxFieldLength(-1)
+						    .detail("ConsistencyCheckerId", self->consistencyCheckerId)
 						    .detail("Reason", "MaxRetryDepthReached")
 						    .detail("ConsistencyCheckEpoch", consistencyCheckEpoch)
 						    .detail("Distributed", self->distributed)
@@ -2356,6 +2206,7 @@ struct ConsistencyCheckWorkload : TestWorkload {
 			TraceEvent("ConsistencyCheck_EndTask")
 			    .setMaxEventLength(-1)
 			    .setMaxFieldLength(-1)
+			    .detail("ConsistencyCheckerId", self->consistencyCheckerId)
 			    .detail("Distributed", self->distributed)
 			    .detail("ShardCount", ranges.size())
 			    .detail("ClientId", self->clientId)
diff --git a/flow/error_definitions.h b/flow/error_definitions.h
index 6de15721c5b..a6997858332 100755
--- a/flow/error_definitions.h
+++ b/flow/error_definitions.h
@@ -97,6 +97,7 @@ ERROR( unexpected_encoding_type, 1073, "Page content decoding failed" )
 ERROR( encryption_key_not_found, 1074, "Encryption key not found" )
 ERROR( unknown_storage_engine, 1082, "Storage engine type is not recognized." )
 ERROR( manual_shard_split_failed, 1083, "Manual shard split is failed to trigger")
+ERROR( consistency_check_task_outdated, 1084, "Consistency check task is outdated")
 
 ERROR( broken_promise, 1100, "Broken promise" )
 ERROR( operation_cancelled, 1101, "Asynchronous operation cancelled" )

From 650e7291606e99e6b2c3b49bc1352ebc9b31bd48 Mon Sep 17 00:00:00 2001
From: Zhe Wang <zhe.wang@wustl.edu>
Date: Sun, 17 Dec 2023 15:07:15 -0800
Subject: [PATCH 338/468] add to simulation test

---
 fdbclient/ClientKnobs.cpp                     |   1 -
 fdbclient/ClientKnobs.h                       |   5 +-
 fdbclient/ConsistencyCheckUtil.actor.cpp      |  19 +
 fdbclient/ConsistencyCheckUtil.actor.h        |   2 +
 fdbserver/tester.actor.cpp                    | 531 +++++++++++-------
 .../workloads/ConsistencyCheck.actor.cpp      |   5 +-
 6 files changed, 354 insertions(+), 209 deletions(-)

diff --git a/fdbclient/ClientKnobs.cpp b/fdbclient/ClientKnobs.cpp
index 2325546cbec..eb327c4fd4d 100644
--- a/fdbclient/ClientKnobs.cpp
+++ b/fdbclient/ClientKnobs.cpp
@@ -258,7 +258,6 @@ void ClientKnobs::initialize(Randomize randomize) {
 
 	init( CONSISTENCY_CHECK_RATE_LIMIT_MAX,        50e6 ); // Limit in per sec
 	init( CONSISTENCY_CHECK_ONE_ROUND_TARGET_COMPLETION_TIME,	7 * 24 * 60 * 60 ); // 7 days
-	init( CONSISTENCY_CHECK_URGENT_MODE_FORCE_INIT, false );
 	init( CONSISTENCY_CHECK_RETRY_DEPTH_MAX,        100 ); if( randomize && BUGGIFY ) CONSISTENCY_CHECK_RETRY_DEPTH_MAX = 10;
 	init( CONSISTENCY_CHECK_RANGE_BEGIN_0,           "" ); if( randomize && BUGGIFY ) CONSISTENCY_CHECK_RANGE_BEGIN_0 = "";
 	init( CONSISTENCY_CHECK_RANGE_END_0,   "\\xff\\xff" ); if( randomize && BUGGIFY ) CONSISTENCY_CHECK_RANGE_END_0 = "\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x33\\x66\\x63\\x36";
diff --git a/fdbclient/ClientKnobs.h b/fdbclient/ClientKnobs.h
index eaf75b44b14..31d7c15e8d3 100644
--- a/fdbclient/ClientKnobs.h
+++ b/fdbclient/ClientKnobs.h
@@ -247,9 +247,8 @@ class ClientKnobs : public KnobsImpl<ClientKnobs> {
 	int BLOBSTORE_MAX_DELAY_RETRYABLE_ERROR;
 	int BLOBSTORE_MAX_DELAY_CONNECTION_FAILED;
 
-	int CONSISTENCY_CHECK_RATE_LIMIT_MAX;
-	int CONSISTENCY_CHECK_ONE_ROUND_TARGET_COMPLETION_TIME;
-	bool CONSISTENCY_CHECK_URGENT_MODE_FORCE_INIT; // Available in urgent mode
+	int CONSISTENCY_CHECK_RATE_LIMIT_MAX; // Not available in urgent mode
+	int CONSISTENCY_CHECK_ONE_ROUND_TARGET_COMPLETION_TIME; // Not available in urgent mode
 	int CONSISTENCY_CHECK_RETRY_DEPTH_MAX; // Available in urgent mode
 	std::string CONSISTENCY_CHECK_RANGE_BEGIN_0; // Available in urgent mode
 	std::string CONSISTENCY_CHECK_RANGE_END_0; // Available in urgent mode
diff --git a/fdbclient/ConsistencyCheckUtil.actor.cpp b/fdbclient/ConsistencyCheckUtil.actor.cpp
index 318ae2fc95b..c64d002275d 100644
--- a/fdbclient/ConsistencyCheckUtil.actor.cpp
+++ b/fdbclient/ConsistencyCheckUtil.actor.cpp
@@ -115,6 +115,25 @@ ACTOR Future<Void> initConsistencyCheckMetadata(Database cx,
 	return Void();
 }
 
+ACTOR Future<Void> clearConsistencyCheckMetadata(Database cx) {
+	state Transaction tr(cx);
+	loop {
+		try {
+			tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
+			tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
+			tr.setOption(FDBTransactionOptions::LOCK_AWARE);
+			tr.clear(consistencyCheckAssignmentKeys);
+			tr.clear(consistencyCheckProgressKeys);
+			tr.clear(consistencyCheckerIdKey);
+			wait(tr.commit());
+			break;
+		} catch (Error& e) {
+			wait(tr.onError(e) && delay(10.0));
+		}
+	}
+	return Void();
+}
+
 ACTOR Future<std::vector<KeyRange>> loadRangesToCheckFromProgressMetadata(Database cx) {
 	state std::vector<KeyRange> res;
 	state Transaction tr(cx);
diff --git a/fdbclient/ConsistencyCheckUtil.actor.h b/fdbclient/ConsistencyCheckUtil.actor.h
index d2ef0e959c5..80b3bd2e888 100644
--- a/fdbclient/ConsistencyCheckUtil.actor.h
+++ b/fdbclient/ConsistencyCheckUtil.actor.h
@@ -40,6 +40,8 @@ ACTOR Future<Void> initConsistencyCheckMetadata(Database cx,
                                                 std::vector<KeyRange> rangesToCheck,
                                                 uint64_t consistencyCheckerId);
 
+ACTOR Future<Void> clearConsistencyCheckMetadata(Database cx);
+
 ACTOR Future<std::vector<KeyRange>> loadRangesToCheckFromProgressMetadata(Database cx);
 
 ACTOR Future<Void> persistConsistencyCheckAssignment(Database cx, int clientId, std::vector<KeyRange> assignedRanges);
diff --git a/fdbserver/tester.actor.cpp b/fdbserver/tester.actor.cpp
index 0ff00fa8ab8..bde3cb8b00d 100644
--- a/fdbserver/tester.actor.cpp
+++ b/fdbserver/tester.actor.cpp
@@ -981,6 +981,311 @@ ACTOR Future<Void> checkConsistency(Database cx,
 	}
 }
 
+ACTOR Future<Void> dispatchMonitorUrgentConsistencyCheckWorkload(Database cx,
+                                                                 std::vector<TesterInterface> testers,
+                                                                 TestSpec spec,
+                                                                 Optional<TenantName> defaultTenant,
+                                                                 int64_t consistencyCheckerId) {
+	TraceEvent("ConsistencyCheckUrgent_Dispatch")
+	    .detail("WorkloadTitle", spec.title)
+	    .detail("TesterCount", testers.size())
+	    .detail("TestTimeout", spec.timeout)
+	    .detail("ConsistencyCheckerId", consistencyCheckerId);
+
+	state std::vector<Future<WorkloadInterface>> workRequests;
+	for (int i = 0; i < testers.size(); i++) {
+		WorkloadRequest req;
+		req.title = spec.title;
+		req.useDatabase = spec.useDB;
+		req.timeout = spec.timeout;
+		req.databasePingDelay = spec.useDB ? spec.databasePingDelay : 0.0;
+		req.options = spec.options;
+		req.clientId = i;
+		req.clientCount = testers.size();
+		req.sharedRandomNumber = consistencyCheckerId;
+		req.defaultTenant = defaultTenant.castTo<TenantNameRef>();
+		workRequests.push_back(testers[i].recruitments.getReply(req));
+	}
+
+	TraceEvent("ConsistencyCheckUrgent_SetupStart")
+	    .detail("WorkloadTitle", spec.title)
+	    .detail("TesterCount", testers.size())
+	    .detail("TestTimeout", spec.timeout)
+	    .detail("ConsistencyCheckerId", consistencyCheckerId);
+
+	state std::vector<WorkloadInterface> workloads = wait(getAll(workRequests));
+	state double waitForFailureTime = g_network->isSimulated() ? 24 * 60 * 60 : 60;
+	if (spec.phases & TestWorkload::SETUP) {
+		state std::vector<Future<ErrorOr<Void>>> setups;
+		setups.reserve(workloads.size());
+		for (int i = 0; i < workloads.size(); i++)
+			setups.push_back(workloads[i].setup.template getReplyUnlessFailedFor<Void>(waitForFailureTime, 0));
+		wait(waitForAll(setups));
+	}
+
+	TraceEvent("ConsistencyCheckUrgent_SetupComplete")
+	    .detail("WorkloadTitle", spec.title)
+	    .detail("TesterCount", testers.size())
+	    .detail("TestTimeout", spec.timeout)
+	    .detail("ConsistencyCheckerId", consistencyCheckerId);
+
+	if (spec.phases & TestWorkload::EXECUTION) {
+		TraceEvent("TestStarting").detail("WorkloadTitle", spec.title);
+		printf("running test (%s)...\n", printable(spec.title).c_str());
+		state std::vector<Future<ErrorOr<Void>>> starts;
+		starts.reserve(workloads.size());
+		for (int i = 0; i < workloads.size(); i++)
+			starts.push_back(workloads[i].start.template getReplyUnlessFailedFor<Void>(waitForFailureTime, 0));
+		wait(waitForAll(starts));
+	}
+
+	TraceEvent("ConsistencyCheckUrgent_WorkloadComplete")
+	    .detail("WorkloadTitle", spec.title)
+	    .detail("TesterCount", testers.size())
+	    .detail("TestTimeout", spec.timeout)
+	    .detail("ConsistencyCheckerId", consistencyCheckerId);
+
+	return Void();
+}
+
+ACTOR Future<std::vector<KeyRange>> getConsistencyCheckShards(Database cx, std::vector<KeyRange> ranges) {
+	// Get the scope of the input list of ranges
+	state Key beginKeyToReadKeyServer;
+	state Key endKeyToReadKeyServer;
+	for (int i = 0; i < ranges.size(); i++) {
+		if (i == 0 || ranges[i].begin < beginKeyToReadKeyServer) {
+			beginKeyToReadKeyServer = ranges[i].begin;
+		}
+		if (i == 0 || ranges[i].end > endKeyToReadKeyServer) {
+			endKeyToReadKeyServer = ranges[i].end;
+		}
+	}
+	TraceEvent("ConsistencyCheckUrgent_GetConsistencyCheckShards")
+	    .detail("RangeBegin", beginKeyToReadKeyServer)
+	    .detail("RangeEnd", endKeyToReadKeyServer);
+	// Read KeyServer space within the scope and add shards intersecting with the input ranges
+	state std::vector<KeyRange> res;
+	state Transaction tr(cx);
+	loop {
+		try {
+			tr.setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
+			tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
+			tr.setOption(FDBTransactionOptions::LOCK_AWARE);
+			KeyRange rangeToRead = Standalone(KeyRangeRef(beginKeyToReadKeyServer, endKeyToReadKeyServer));
+			RangeResult readResult = wait(krmGetRanges(
+			    &tr, keyServersPrefix, rangeToRead, CLIENT_KNOBS->TOO_MANY, GetRangeLimits::BYTE_LIMIT_UNLIMITED));
+			for (int i = 0; i < readResult.size() - 1; ++i) {
+				KeyRange rangeToCheck = Standalone(KeyRangeRef(readResult[i].key, readResult[i + 1].key));
+				Value valueToCheck = Standalone(readResult[i].value);
+				bool toAdd = false;
+				for (const auto& range : ranges) {
+					if (rangeToCheck.intersects(range) == true) {
+						toAdd = true;
+						break;
+					}
+				}
+				if (toAdd == true) {
+					res.push_back(rangeToCheck);
+				}
+				beginKeyToReadKeyServer = readResult[i + 1].key;
+			}
+			if (beginKeyToReadKeyServer >= endKeyToReadKeyServer) {
+				break;
+			}
+		} catch (Error& e) {
+			TraceEvent("ConsistencyCheckUrgent_GetConsistencyCheckShardsRetry").error(e);
+			wait(tr.onError(e));
+		}
+	}
+	return res;
+}
+
+ACTOR Future<std::vector<TesterInterface>> getTesters(Reference<AsyncVar<Optional<ClusterControllerFullInterface>>> cc,
+                                                      int minTestersExpected) {
+	// Recruit workers
+	state int flags = GetWorkersRequest::TESTER_CLASS_ONLY | GetWorkersRequest::NON_EXCLUDED_PROCESSES_ONLY;
+	state Future<Void> testerTimeout = delay(600.0); // wait 600 sec for testers to show up
+	state std::vector<WorkerDetails> workers;
+	loop {
+		choose {
+			when(std::vector<WorkerDetails> w =
+			         wait(cc->get().present()
+			                  ? brokenPromiseToNever(cc->get().get().getWorkers.getReply(GetWorkersRequest(flags)))
+			                  : Never())) {
+				if (w.size() >= minTestersExpected) {
+					workers = w;
+					break;
+				}
+				wait(delay(SERVER_KNOBS->WORKER_POLL_DELAY));
+			}
+			when(wait(cc->onChange())) {}
+			when(wait(testerTimeout)) {
+				TraceEvent(SevError, "TesterRecruitmentTimeout").log();
+				throw timed_out();
+			}
+		}
+	}
+	state std::vector<TesterInterface> ts;
+	ts.reserve(workers.size());
+	for (int i = 0; i < workers.size(); i++)
+		ts.push_back(workers[i].interf.testerInterface);
+	return ts;
+}
+
+ACTOR Future<Void> runConsistencyCheckerUrgentCore(Reference<AsyncVar<Optional<ClusterControllerFullInterface>>> cc,
+                                                   Database cx,
+                                                   Optional<std::vector<TesterInterface>> testers,
+                                                   int minTestersExpected,
+                                                   TestSpec testSpec,
+                                                   Optional<TenantName> defaultTenant,
+                                                   int64_t consistencyCheckerId) {
+	state int round = 0;
+	// Load ranges to check from progress metadata
+	state std::vector<KeyRange> rangesToCheck;
+	wait(store(rangesToCheck, loadRangesToCheckFromProgressMetadata(cx)));
+	if (rangesToCheck.size() == 0) {
+		// If no range to check in progress data
+		// We load the range from knob
+		rangesToCheck = loadRangesToCheckFromKnob();
+		TraceEvent e("ConsistencyCheckUrgent_Start");
+		e.setMaxEventLength(-1);
+		e.setMaxFieldLength(-1);
+		e.detail("ConsistencyCheckerId", consistencyCheckerId);
+		for (int i = 0; i < rangesToCheck.size(); i++) {
+			e.detail("RangeToCheckBegin" + std::to_string(i), rangesToCheck[i].begin);
+			e.detail("RangeToCheckEnd" + std::to_string(i), rangesToCheck[i].end);
+		}
+		wait(initConsistencyCheckMetadata(cx, rangesToCheck, consistencyCheckerId));
+	} else {
+		TraceEvent("ConsistencyCheckUrgent_Resume")
+		    .detail("ConsistencyCheckerId", consistencyCheckerId)
+		    .detail("RangesToCheckCount", rangesToCheck.size());
+	}
+
+	TraceEvent("ConsistencyCheckUrgent_Initialized").detail("ConsistencyCheckerId", consistencyCheckerId);
+
+	state std::vector<TesterInterface> ts;
+	loop {
+		ts.clear();
+		if (!testers.present()) {
+			wait(store(ts, getTesters(cc, minTestersExpected)));
+		} else {
+			ts = testers.get();
+		}
+		TraceEvent("ConsistencyCheckUrgent_GoTTesters")
+		    .detail("ConsistencyCheckerId", consistencyCheckerId)
+		    .detail("Round", round)
+		    .detail("TesterCount", ts.size());
+
+		// Load shards to check from keyserver space
+		state std::vector<KeyRange> shardsToCheck = wait(getConsistencyCheckShards(cx, rangesToCheck));
+		TraceEvent("ConsistencyCheckUrgent_GotShardsToCheck")
+		    .detail("ConsistencyCheckerId", consistencyCheckerId)
+		    .detail("Round", round)
+		    .detail("TesterCount", ts.size())
+		    .detail("ShardCount", shardsToCheck.size())
+		    .detail("RangesToCheck", describe(rangesToCheck))
+		    .detail("Shards", describe(shardsToCheck));
+		// Assign shards to testers
+		state std::unordered_map<int, std::vector<KeyRange>> assignment;
+		int batchSize = shardsToCheck.size() / ts.size() + 1;
+		for (int i = 0; i < shardsToCheck.size(); i++) {
+			assignment[i / batchSize].push_back(shardsToCheck[i]);
+		}
+		state std::unordered_map<int, std::vector<KeyRange>>::iterator assignIt;
+		for (assignIt = assignment.begin(); assignIt != assignment.end(); assignIt++) {
+			TraceEvent("ConsistencyCheckUrgent_ClientAssignedTask")
+			    .detail("ConsistencyCheckerId", consistencyCheckerId)
+			    .detail("ClientId", assignIt->first)
+			    .detail("Shards", assignIt->second.size());
+			wait(persistConsistencyCheckAssignment(cx, assignIt->first, assignIt->second));
+		}
+		TraceEvent e("ConsistencyCheckUrgent_PersistAssignment");
+		e.detail("ConsistencyCheckerId", consistencyCheckerId);
+		e.detail("Round", round);
+		e.detail("TesterCount", ts.size());
+		e.detail("ShardCountTotal", shardsToCheck.size());
+		for (const auto& [clientId, assignedShards] : assignment) {
+			e.detail("Client" + std::to_string(clientId), assignedShards.size());
+		}
+		// Run test and wait for testers complete/fail
+		try {
+			wait(dispatchMonitorUrgentConsistencyCheckWorkload(cx, ts, testSpec, defaultTenant, consistencyCheckerId));
+		} catch (Error& e) {
+			if (e.code() == error_code_actor_cancelled) {
+				throw;
+			}
+		}
+		TraceEvent("ConsistencyCheckUrgent_RoundComplete")
+		    .detail("ConsistencyCheckerId", consistencyCheckerId)
+		    .detail("Round", round);
+		round++;
+
+		// Load progress metadata. If no unfinished range, exit; otherwise, continue
+		rangesToCheck.clear();
+		wait(store(rangesToCheck, loadRangesToCheckFromProgressMetadata(cx)));
+		if (rangesToCheck.size() == 0) {
+			TraceEvent("ConsistencyCheckUrgent_Complete")
+			    .detail("ConsistencyCheckerId", consistencyCheckerId)
+			    .detail("Round", round);
+			wait(clearConsistencyCheckMetadata(cx));
+			TraceEvent("ConsistencyCheckUrgent_MetadataCleared")
+			    .detail("ConsistencyCheckerId", consistencyCheckerId)
+			    .detail("Round", round);
+			break;
+		} else {
+			TraceEvent("ConsistencyCheckUrgent_Continue")
+			    .detail("ConsistencyCheckerId", consistencyCheckerId)
+			    .detail("Round", round)
+			    .detail("RangesToCheckCount", rangesToCheck.size());
+		}
+	}
+	return Void();
+}
+
+ACTOR Future<Void> checkConsistencyUrgentSim(Database cx,
+                                             std::vector<TesterInterface> testers,
+                                             double softTimeLimit,
+                                             double databasePingDelay,
+                                             Reference<AsyncVar<ServerDBInfo>> dbInfo) {
+	state TestSpec spec;
+	Standalone<VectorRef<KeyValueRef>> options;
+	spec.title = LiteralStringRef("ConsistencyCheck");
+	spec.databasePingDelay = databasePingDelay;
+	spec.timeout = 32000;
+	state int64_t consistencyCheckerId = deterministicRandom()->randomInt64(1, 10000000);
+	options.push_back_deep(options.arena(),
+	                       KeyValueRef(LiteralStringRef("testName"), LiteralStringRef("ConsistencyCheck")));
+	options.push_back_deep(options.arena(),
+	                       KeyValueRef(LiteralStringRef("performQuiescentChecks"), LiteralStringRef("false")));
+	options.push_back_deep(options.arena(),
+	                       KeyValueRef(LiteralStringRef("performCacheCheck"), LiteralStringRef("false")));
+	options.push_back_deep(options.arena(),
+	                       KeyValueRef(LiteralStringRef("performTSSCheck"), LiteralStringRef("false")));
+	options.push_back_deep(options.arena(), KeyValueRef(LiteralStringRef("distributed"), LiteralStringRef("true")));
+	spec.options.push_back_deep(spec.options.arena(), options);
+	loop {
+		TraceEvent("ConsistencyCheckUrgent_LoopBegin").detail("ConsistencyCheckerId", consistencyCheckerId);
+		try {
+			wait(runConsistencyCheckerUrgentCore(Reference<AsyncVar<Optional<ClusterControllerFullInterface>>>(),
+			                                     cx,
+			                                     testers,
+			                                     1,
+			                                     spec,
+			                                     Optional<TenantName>(),
+			                                     consistencyCheckerId));
+			break;
+		} catch (Error& e) {
+			if (e.code() == error_code_actor_cancelled) {
+				throw e;
+			}
+		}
+		TraceEvent("ConsistencyCheckUrgent_RepairDC");
+		wait(repairDeadDatacenter(cx, dbInfo, "ConsistencyCheck"));
+	}
+	return Void();
+}
+
 ACTOR Future<bool> runTest(Database cx,
                            std::vector<TesterInterface> testers,
                            TestSpec spec,
@@ -1036,6 +1341,8 @@ ACTOR Future<bool> runTest(Database cx,
 		// Run the consistency check workload
 		if (spec.runConsistencyCheck) {
 			try {
+				wait(timeoutError(checkConsistencyUrgentSim(cx, testers, 18000, spec.databasePingDelay, dbInfo),
+				                  20000.0));
 				bool quiescent = g_network->isSimulated() ? !BUGGIFY : spec.waitForQuiescenceEnd;
 				wait(timeoutError(checkConsistency(cx,
 				                                   testers,
@@ -1703,188 +2010,6 @@ ACTOR Future<Void> runTests(Reference<AsyncVar<Optional<struct ClusterController
 	return Void();
 }
 
-ACTOR Future<std::vector<KeyRange>> getConsistencyCheckShards(Database cx, std::vector<KeyRange> ranges) {
-	// Get the scope of the input list of ranges
-	state Key beginKeyToReadKeyServer;
-	state Key endKeyToReadKeyServer;
-	for (int i = 0; i < ranges.size(); i++) {
-		if (i == 0 || ranges[i].begin < beginKeyToReadKeyServer) {
-			beginKeyToReadKeyServer = ranges[i].begin;
-		}
-		if (i == 0 || ranges[i].end > endKeyToReadKeyServer) {
-			endKeyToReadKeyServer = ranges[i].end;
-		}
-	}
-	TraceEvent("ConsistencyCheck_GetConsistencyCheckShards")
-	    .detail("RangeBegin", beginKeyToReadKeyServer)
-	    .detail("RangeEnd", endKeyToReadKeyServer);
-	// Read KeyServer space within the scope and add shards intersecting with the input ranges
-	state std::vector<KeyRange> res;
-	state Transaction tr(cx);
-	loop {
-		try {
-			tr.setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
-			tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
-			tr.setOption(FDBTransactionOptions::LOCK_AWARE);
-			KeyRange rangeToRead = Standalone(KeyRangeRef(beginKeyToReadKeyServer, endKeyToReadKeyServer));
-			RangeResult readResult = wait(krmGetRanges(
-			    &tr, keyServersPrefix, rangeToRead, CLIENT_KNOBS->TOO_MANY, GetRangeLimits::BYTE_LIMIT_UNLIMITED));
-			for (int i = 0; i < readResult.size() - 1; ++i) {
-				KeyRange rangeToCheck = Standalone(KeyRangeRef(readResult[i].key, readResult[i + 1].key));
-				Value valueToCheck = Standalone(readResult[i].value);
-				bool toAdd = false;
-				for (const auto& range : ranges) {
-					if (rangeToCheck.intersects(range) == true) {
-						toAdd = true;
-						break;
-					}
-				}
-				if (toAdd == true) {
-					res.push_back(rangeToCheck);
-				}
-				beginKeyToReadKeyServer = readResult[i + 1].key;
-			}
-			if (beginKeyToReadKeyServer >= endKeyToReadKeyServer) {
-				break;
-			}
-		} catch (Error& e) {
-			TraceEvent("ConsistencyCheck_GetConsistencyCheckShardsRetry").error(e);
-			wait(tr.onError(e));
-		}
-	}
-	return res;
-}
-
-ACTOR Future<Void> runConsistencyCheckerUrgent(Reference<AsyncVar<Optional<struct ClusterControllerFullInterface>>> cc,
-                                               Reference<AsyncVar<Optional<struct ClusterInterface>>> ci,
-                                               std::vector<TestSpec> tests,
-                                               int minTestersExpected,
-                                               StringRef startingConfiguration,
-                                               LocalityData locality,
-                                               Optional<TenantName> defaultTenant,
-                                               uint64_t consistencyCheckerId) {
-	state Database cx;
-	state Reference<AsyncVar<ServerDBInfo>> dbInfo(new AsyncVar<ServerDBInfo>);
-	state Future<Void> ccMonitor = monitorServerDBInfo(cc, LocalityData(), dbInfo); // FIXME: locality
-	cx = openDBOnServer(dbInfo);
-	cx->defaultTenant = defaultTenant;
-	state int round = 0;
-	TraceEvent("ConsistencyChecker_RunUrgentStart").detail("ConsistencyCheckerId", consistencyCheckerId);
-
-	// Load ranges to check from progress metadata
-	state std::vector<KeyRange> rangesToCheck;
-	wait(store(rangesToCheck, loadRangesToCheckFromProgressMetadata(cx)));
-	if (CLIENT_KNOBS->CONSISTENCY_CHECK_URGENT_MODE_FORCE_INIT || rangesToCheck.size() == 0) {
-		// If force to start with blank or no range to check in progress data
-		// We load the range from knob
-		rangesToCheck = loadRangesToCheckFromKnob();
-		TraceEvent e("ConsistencyChecker_RunUrgentStart");
-		e.setMaxEventLength(-1);
-		e.setMaxFieldLength(-1);
-		e.detail("ConsistencyCheckerId", consistencyCheckerId);
-		for (int i = 0; i < rangesToCheck.size(); i++) {
-			e.detail("RangeToCheckBegin" + std::to_string(i), rangesToCheck[i].begin);
-			e.detail("RangeToCheckEnd" + std::to_string(i), rangesToCheck[i].end);
-		}
-		wait(initConsistencyCheckMetadata(cx, rangesToCheck, consistencyCheckerId));
-	} else {
-		TraceEvent("ConsistencyChecker_RunUrgentResume")
-		    .detail("ConsistencyCheckerId", consistencyCheckerId)
-		    .detail("WorkerCount", workers.size())
-		    .detail("RangesToCheckCount", rangesToCheck.size());
-	}
-
-	TraceEvent("ConsistencyChecker_RunUrgentInitialized").detail("ConsistencyCheckerId", consistencyCheckerId);
-
-	loop {
-		// Recruit workers
-		state int flags = GetWorkersRequest::TESTER_CLASS_ONLY | GetWorkersRequest::NON_EXCLUDED_PROCESSES_ONLY;
-		state Future<Void> testerTimeout = delay(600.0); // wait 600 sec for testers to show up
-		state std::vector<WorkerDetails> workers;
-		loop {
-			choose {
-				when(std::vector<WorkerDetails> w =
-				         wait(cc->get().present()
-				                  ? brokenPromiseToNever(cc->get().get().getWorkers.getReply(GetWorkersRequest(flags)))
-				                  : Never())) {
-					if (w.size() >= minTestersExpected) {
-						workers = w;
-						break;
-					}
-					wait(delay(SERVER_KNOBS->WORKER_POLL_DELAY));
-				}
-				when(wait(cc->onChange())) {}
-				when(wait(testerTimeout)) {
-					TraceEvent(SevError, "TesterRecruitmentTimeout").log();
-					throw timed_out();
-				}
-			}
-		}
-		TraceEvent("ConsistencyChecker_RunUrgentGotWorkers")
-		    .detail("ConsistencyCheckerId", consistencyCheckerId)
-		    .detail("Round", round)
-		    .detail("WorkerCount", workers.size());
-		state std::vector<TesterInterface> ts;
-		ts.reserve(workers.size());
-		for (int i = 0; i < workers.size(); i++)
-			ts.push_back(workers[i].interf.testerInterface);
-
-		// Load shards to check from keyserver space
-		state std::vector<KeyRange> shardsToCheck = wait(getConsistencyCheckShards(cx, rangesToCheck));
-		TraceEvent("ConsistencyChecker_RunUrgentGotShardsToCheck")
-		    .detail("ConsistencyCheckerId", consistencyCheckerId)
-		    .detail("Round", round)
-		    .detail("WorkerCount", workers.size())
-		    .detail("ShardCount", shardsToCheck.size());
-		// Assign shards to workers
-		state std::unordered_map<int, std::vector<KeyRange>> assignment;
-		int batchSize = shardsToCheck.size() / workers.size() + 1;
-		for (int i = 0; i < shardsToCheck.size(); i++) {
-			assignment[i / batchSize].push_back(shardsToCheck[i]);
-		}
-		state std::unordered_map<int, std::vector<KeyRange>>::iterator assignIt;
-		for (assignIt = assignment.begin(); assignIt != assignment.end(); assignIt++) {
-			if (assignIt->second.size() == 0) {
-				TraceEvent("ConsistencyCheck_ClientAssignedNoTask")
-				    .detail("ConsistencyCheckerId", consistencyCheckerId)
-				    .detail("ClientId", assignIt->first);
-				continue;
-			}
-			wait(persistConsistencyCheckAssignment(cx, assignIt->first, assignIt->second));
-		}
-		TraceEvent e("ConsistencyChecker_RunUrgentPersistAssignment");
-		e.detail("ConsistencyCheckerId", consistencyCheckerId);
-		e.detail("Round", round);
-		e.detail("WorkerCount", workers.size());
-		e.detail("ShardCountTotal", shardsToCheck.size());
-		for (const auto& [clientId, assignedShards] : assignment) {
-			e.detail("Client" + std::to_string(clientId), assignedShards.size());
-		}
-		// Run tests and wait for tester complete/fail
-		wait(runTests(cc, ci, ts, tests, startingConfiguration, locality, defaultTenant));
-		TraceEvent("ConsistencyChecker_RunUrgentRoundComplete")
-		    .detail("ConsistencyCheckerId", consistencyCheckerId)
-		    .detail("Round", round);
-		round++;
-
-		// Load progress metadata. If no unfinished range, exit; otherwise, continue
-		rangesToCheck.clear();
-		wait(store(rangesToCheck, loadRangesToCheckFromProgressMetadata(cx)));
-		if (rangesToCheck.size() == 0) {
-			TraceEvent("ConsistencyChecker_RunUrgentComplete")
-			    .detail("ConsistencyCheckerId", consistencyCheckerId)
-			    .detail("Round", round);
-			break;
-		} else {
-			TraceEvent("ConsistencyChecker_RunUrgentContinue")
-			    .detail("ConsistencyCheckerId", consistencyCheckerId)
-			    .detail("Round", round)
-			    .detail("RangesToCheckCount", rangesToCheck.size());
-		}
-	}
-	return Void();
-}
-
 /**
  * \brief Set up testing environment and run the given tests on a cluster.
  *
@@ -1929,28 +2054,27 @@ ACTOR Future<Void> runTests(Reference<IClusterConnectionRecord> connRecord,
 		actors.push_back(reportErrors(monitorLeader(connRecord, cc), "MonitorLeader"));
 		actors.push_back(reportErrors(extractClusterInterface(cc, ci), "ExtractClusterInterface"));
 	}
-	state uint64_t consistencyCheckerId = 0;
+	state int64_t consistencyCheckerId = 0;
 
 	if (whatToRun == TEST_TYPE_CONSISTENCY_CHECK_URGENT) {
-		while (consistencyCheckerId == 0)
-			consistencyCheckerId = deterministicRandom()->randomUInt64();
+		consistencyCheckerId = deterministicRandom()->randomInt64(1, 10000000);
 		// consistencyCheckerId must be not 0, indicating this is in urgent mode of consistency checker
 		TestSpec spec;
 		Standalone<VectorRef<KeyValueRef>> options;
-		spec.title = LiteralStringRef("ConsistencyCheckUrgent");
+		spec.title = LiteralStringRef("ConsistencyCheck");
 		spec.databasePingDelay = 0;
 		spec.timeout = 0;
 		spec.waitForQuiescenceBegin = false;
 		spec.waitForQuiescenceEnd = false;
 		std::string rateLimitMax = format("%d", CLIENT_KNOBS->CONSISTENCY_CHECK_RATE_LIMIT_MAX);
 		options.push_back_deep(options.arena(),
-		                       KeyValueRef(LiteralStringRef("testName"), LiteralStringRef("ConsistencyCheckUrgent")));
-		options.push_back_deep(
-		    options.arena(),
-		    KeyValueRef(LiteralStringRef("consistencyCheckerId"), StringRef(std::to_string(consistencyCheckerId))));
+		                       KeyValueRef(LiteralStringRef("testName"), LiteralStringRef("ConsistencyCheck")));
 		options.push_back_deep(options.arena(),
 		                       KeyValueRef(LiteralStringRef("performQuiescentChecks"), LiteralStringRef("false")));
-		options.push_back_deep(options.arena(), KeyValueRef(LiteralStringRef("distributed"), LiteralStringRef("true")));
+		options.push_back_deep(
+		    options.arena(),
+		    KeyValueRef(LiteralStringRef("distributed"),
+		                LiteralStringRef("false"))); // The distribution mechanism does not rely on this flag
 		options.push_back_deep(options.arena(),
 		                       KeyValueRef(LiteralStringRef("failureIsError"), LiteralStringRef("true")));
 		options.push_back_deep(options.arena(), KeyValueRef(LiteralStringRef("indefinite"), LiteralStringRef("false")));
@@ -1970,9 +2094,6 @@ ACTOR Future<Void> runTests(Reference<IClusterConnectionRecord> connRecord,
 		std::string rateLimitMax = format("%d", CLIENT_KNOBS->CONSISTENCY_CHECK_RATE_LIMIT_MAX);
 		options.push_back_deep(options.arena(),
 		                       KeyValueRef(LiteralStringRef("testName"), LiteralStringRef("ConsistencyCheck")));
-		options.push_back_deep(
-		    options.arena(),
-		    KeyValueRef(LiteralStringRef("consistencyCheckerId"), StringRef(std::to_string(consistencyCheckerId))));
 		options.push_back_deep(options.arena(),
 		                       KeyValueRef(LiteralStringRef("performQuiescentChecks"), LiteralStringRef("false")));
 		options.push_back_deep(options.arena(),
@@ -2030,15 +2151,19 @@ ACTOR Future<Void> runTests(Reference<IClusterConnectionRecord> connRecord,
 	knobProtectiveGroup = std::make_unique<KnobProtectiveGroup>(testSet.overrideKnobs);
 	Future<Void> tests;
 	if (whatToRun == TEST_TYPE_CONSISTENCY_CHECK_URGENT) {
-		tests = reportErrors(runConsistencyCheckerUrgent(cc,
-		                                                 ci,
-		                                                 testSet.testSpecs,
-		                                                 minTestersExpected,
-		                                                 startingConfiguration,
-		                                                 locality,
-		                                                 defaultTenant,
-		                                                 consistencyCheckerId),
-		                     "runConsistencyCheckerUrgent");
+		state Database cx;
+		state Reference<AsyncVar<ServerDBInfo>> dbInfo(new AsyncVar<ServerDBInfo>);
+		state Future<Void> ccMonitor = monitorServerDBInfo(cc, LocalityData(), dbInfo); // FIXME: locality
+		cx = openDBOnServer(dbInfo);
+		cx->defaultTenant = defaultTenant;
+		tests = reportErrors(runConsistencyCheckerUrgentCore(cc,
+		                                                     cx,
+		                                                     Optional<std::vector<TesterInterface>>(),
+		                                                     minTestersExpected,
+		                                                     testSet.testSpecs[0],
+		                                                     defaultTenant,
+		                                                     consistencyCheckerId),
+		                     "runConsistencyCheckerUrgentCore");
 	} else if (at == TEST_HERE) {
 		auto db = makeReference<AsyncVar<ServerDBInfo>>();
 		std::vector<TesterInterface> iTesters(1);
diff --git a/fdbserver/workloads/ConsistencyCheck.actor.cpp b/fdbserver/workloads/ConsistencyCheck.actor.cpp
index 140cb498f70..96845f0c094 100644
--- a/fdbserver/workloads/ConsistencyCheck.actor.cpp
+++ b/fdbserver/workloads/ConsistencyCheck.actor.cpp
@@ -97,7 +97,7 @@ struct ConsistencyCheckWorkload : TestWorkload {
 
 	Future<Void> monitorConsistencyCheckSettingsActor;
 
-	uint64_t consistencyCheckerId;
+	int64_t consistencyCheckerId;
 
 	ConsistencyCheckWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
 		performQuiescentChecks = getOption(options, LiteralStringRef("performQuiescentChecks"), false);
@@ -110,7 +110,7 @@ struct ConsistencyCheckWorkload : TestWorkload {
 		rateLimitMax = getOption(options, LiteralStringRef("rateLimitMax"), 0);
 		shuffleShards = getOption(options, LiteralStringRef("shuffleShards"), false);
 		indefinite = getOption(options, LiteralStringRef("indefinite"), false);
-		consistencyCheckerId = getOption(options, LiteralStringRef("consistencyCheckerId"), (uint64_t)0);
+		consistencyCheckerId = sharedRandomNumber;
 		suspendConsistencyCheck.set(true);
 
 		success = true;
@@ -1975,6 +1975,7 @@ struct ConsistencyCheckWorkload : TestWorkload {
 						} catch (Error& e) {
 							if (e.code() == error_code_key_not_found ||
 							    e.code() == error_code_consistency_check_task_outdated) {
+								TraceEvent(SevInfo, "Zhe").errorUnsuppressed(e);
 								return testResult;
 							} else {
 								throw e;

From 53f57e73170a85fb8e9739243ddbf5fd95878db8 Mon Sep 17 00:00:00 2001
From: Zhe Wang <zhe.wang@wustl.edu>
Date: Sun, 17 Dec 2023 15:47:22 -0800
Subject: [PATCH 339/468] bypass consistencycheck command

---
 fdbserver/tester.actor.cpp                    |  2 +-
 .../workloads/ConsistencyCheck.actor.cpp      | 47 +++++++++++++++----
 2 files changed, 39 insertions(+), 10 deletions(-)

diff --git a/fdbserver/tester.actor.cpp b/fdbserver/tester.actor.cpp
index bde3cb8b00d..66c698ba78a 100644
--- a/fdbserver/tester.actor.cpp
+++ b/fdbserver/tester.actor.cpp
@@ -1219,7 +1219,6 @@ ACTOR Future<Void> runConsistencyCheckerUrgentCore(Reference<AsyncVar<Optional<C
 		TraceEvent("ConsistencyCheckUrgent_RoundComplete")
 		    .detail("ConsistencyCheckerId", consistencyCheckerId)
 		    .detail("Round", round);
-		round++;
 
 		// Load progress metadata. If no unfinished range, exit; otherwise, continue
 		rangesToCheck.clear();
@@ -1238,6 +1237,7 @@ ACTOR Future<Void> runConsistencyCheckerUrgentCore(Reference<AsyncVar<Optional<C
 			    .detail("ConsistencyCheckerId", consistencyCheckerId)
 			    .detail("Round", round)
 			    .detail("RangesToCheckCount", rangesToCheck.size());
+			round++;
 		}
 	}
 	return Void();
diff --git a/fdbserver/workloads/ConsistencyCheck.actor.cpp b/fdbserver/workloads/ConsistencyCheck.actor.cpp
index 96845f0c094..24c23b580a7 100644
--- a/fdbserver/workloads/ConsistencyCheck.actor.cpp
+++ b/fdbserver/workloads/ConsistencyCheck.actor.cpp
@@ -112,6 +112,7 @@ struct ConsistencyCheckWorkload : TestWorkload {
 		indefinite = getOption(options, LiteralStringRef("indefinite"), false);
 		consistencyCheckerId = sharedRandomNumber;
 		suspendConsistencyCheck.set(true);
+		TraceEvent("ConsistencyCheck_WorkloadReceived").detail("ClientCount", clientCount).detail("ClientId", clientId);
 
 		success = true;
 
@@ -148,7 +149,11 @@ struct ConsistencyCheckWorkload : TestWorkload {
 
 	Future<Void> start(Database const& cx) override {
 		TraceEvent("ConsistencyCheck").log();
-		return _start(cx, this);
+		if (consistencyCheckerId != 0) { // consistencyCheckerId is set when urgent mode
+			return _startUrgent(cx, this);
+		} else {
+			return _start(cx, this);
+		}
 	}
 
 	Future<bool> check(Database const& cx) override { return success; }
@@ -188,38 +193,62 @@ struct ConsistencyCheckWorkload : TestWorkload {
 		}
 	}
 
+	ACTOR Future<Void> _startUrgent(Database cx, ConsistencyCheckWorkload* self) {
+		TraceEvent("ConsistencyCheck_Start")
+		    .detail("Mode", "Urgent")
+		    .detail("ConsistencyCheckerId", self->consistencyCheckerId)
+		    .detail("Distributed", self->distributed)
+		    .detail("ClientCount", self->clientCount)
+		    .detail("ClientId", self->clientId)
+		    .detail("Indefinite", self->indefinite)
+		    .detail("Repetitions", self->repetitions);
+		wait(self->runCheck(cx, self));
+		TraceEvent("ConsistencyCheck_Exit")
+		    .detail("Mode", "Urgent")
+		    .detail("ConsistencyCheckerId", self->consistencyCheckerId)
+		    .detail("Distributed", self->distributed)
+		    .detail("ClientCount", self->clientCount)
+		    .detail("ClientId", self->clientId)
+		    .detail("Indefinite", self->indefinite)
+		    .detail("Repetitions", self->repetitions);
+		return Void();
+	}
+
 	ACTOR Future<Void> _start(Database cx, ConsistencyCheckWorkload* self) {
 		loop {
 			while (self->suspendConsistencyCheck.get()) {
 				TraceEvent("ConsistencyCheck_Suspended")
+				    .detail("Mode", "Normal")
 				    .detail("ConsistencyCheckerId", self->consistencyCheckerId)
 				    .detail("Distributed", self->distributed)
 				    .detail("ClientCount", self->clientCount)
 				    .detail("ClientId", self->clientId)
 				    .detail("Indefinite", self->indefinite)
-				    .detail("Repetitions", self->repetitions);
+				    .detail("Repetitions", self->repetitions)
+				    .detail("SuspendConsistencyCheck", self->suspendConsistencyCheck.get());
 				wait(self->suspendConsistencyCheck.onChange());
 			}
 			TraceEvent("ConsistencyCheck_StartingOrResuming")
+			    .detail("Mode", "Normal")
 			    .detail("ConsistencyCheckerId", self->consistencyCheckerId)
 			    .detail("Distributed", self->distributed)
 			    .detail("ClientCount", self->clientCount)
 			    .detail("ClientId", self->clientId)
 			    .detail("Indefinite", self->indefinite)
-			    .detail("Repetitions", self->repetitions);
+			    .detail("Repetitions", self->repetitions)
+			    .detail("SuspendConsistencyCheck", self->suspendConsistencyCheck.get());
 			choose {
 				when(wait(self->runCheck(cx, self))) {
-					if (!self->indefinite ||
-					    (self->consistencyCheckerId != 0)) { // self->consistencyCheckerId != 0 when urgent mode
-						// When consistency checker is distributed, repeated running
-						// can miss ranges, so we exit here
+					if (!self->indefinite) {
 						TraceEvent("ConsistencyCheck_Exit")
+						    .detail("Mode", "Normal")
 						    .detail("ConsistencyCheckerId", self->consistencyCheckerId)
 						    .detail("Distributed", self->distributed)
 						    .detail("ClientCount", self->clientCount)
 						    .detail("ClientId", self->clientId)
 						    .detail("Indefinite", self->indefinite)
-						    .detail("Repetitions", self->repetitions);
+						    .detail("Repetitions", self->repetitions)
+						    .detail("SuspendConsistencyCheck", self->suspendConsistencyCheck.get());
 						break;
 					}
 					self->repetitions++;
@@ -245,7 +274,7 @@ struct ConsistencyCheckWorkload : TestWorkload {
 		TEST(self->performQuiescentChecks); // Quiescent consistency check
 		TEST(!self->performQuiescentChecks); // Non-quiescent consistency check
 
-		if (self->firstClient || self->distributed) {
+		if (self->firstClient || self->distributed || self->consistencyCheckerId != 0) {
 			try {
 				state DatabaseConfiguration configuration;
 				state std::map<UID, StorageServerInterface> tssMapping;

From 8574938ba85ab7338108545f99b46bc8836e3f0b Mon Sep 17 00:00:00 2001
From: Zhe Wang <zhe.wang@wustl.edu>
Date: Sun, 17 Dec 2023 16:45:27 -0800
Subject: [PATCH 340/468] fix bugs

---
 fdbclient/ConsistencyCheckUtil.actor.cpp | 32 ++++++++++++++++++++++--
 fdbclient/ConsistencyCheckUtil.actor.h   |  2 ++
 fdbserver/tester.actor.cpp               |  8 +++---
 3 files changed, 37 insertions(+), 5 deletions(-)

diff --git a/fdbclient/ConsistencyCheckUtil.actor.cpp b/fdbclient/ConsistencyCheckUtil.actor.cpp
index c64d002275d..ab2e5ad2e60 100644
--- a/fdbclient/ConsistencyCheckUtil.actor.cpp
+++ b/fdbclient/ConsistencyCheckUtil.actor.cpp
@@ -44,11 +44,14 @@ ACTOR Future<std::vector<KeyRange>> loadRangesToCheckFromAssignmentMetadata(Data
 					ConsistencyCheckState ccState = decodeConsistencyCheckerStateValue(res_[i].value);
 					if (ccState.getAssignment() == ConsistencyCheckAssignment::Assigned) {
 						res.push_back(currentRange);
+						TraceEvent("Zhe").detail("CurrentRange", currentRange);
 					}
 				}
 				rangeToReadBegin = res_[i + 1].key;
 			}
-			break;
+			if (rangeToReadBegin >= rangeToReadEnd) {
+				break;
+			}
 		} catch (Error& e) {
 			wait(tr.onError(e));
 		}
@@ -74,6 +77,9 @@ ACTOR Future<Void> persistConsistencyCheckProgress(Database cx, KeyRange range,
 			                 range,
 			                 consistencyCheckerStateValue(ConsistencyCheckState(ConsistencyCheckPhase::Complete))));
 			wait(tr.commit());
+			TraceEvent("ConsistencyCheckUrgent_PersistProgress")
+			    .detail("Range", range)
+			    .detail("ConsistencyCheckerId", consistencyCheckerId);
 			break;
 		} catch (Error& e) {
 			wait(tr.onError(e));
@@ -134,6 +140,23 @@ ACTOR Future<Void> clearConsistencyCheckMetadata(Database cx) {
 	return Void();
 }
 
+ACTOR Future<Void> clearConsistencyCheckAssignment(Database cx) {
+	state Transaction tr(cx);
+	loop {
+		try {
+			tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
+			tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
+			tr.setOption(FDBTransactionOptions::LOCK_AWARE);
+			tr.clear(consistencyCheckAssignmentKeys);
+			wait(tr.commit());
+			break;
+		} catch (Error& e) {
+			wait(tr.onError(e) && delay(10.0));
+		}
+	}
+	return Void();
+}
+
 ACTOR Future<std::vector<KeyRange>> loadRangesToCheckFromProgressMetadata(Database cx) {
 	state std::vector<KeyRange> res;
 	state Transaction tr(cx);
@@ -160,7 +183,9 @@ ACTOR Future<std::vector<KeyRange>> loadRangesToCheckFromProgressMetadata(Databa
 				}
 				rangeToReadBegin = res_[i + 1].key;
 			}
-			break;
+			if (rangeToReadBegin >= rangeToReadEnd) {
+				break;
+			}
 		} catch (Error& e) {
 			wait(tr.onError(e));
 		}
@@ -182,6 +207,9 @@ ACTOR Future<Void> persistConsistencyCheckAssignment(Database cx, int clientId,
 			}
 			assignedRangesMap.coalesce(allKeys);
 			for (auto& assignedRange : assignedRangesMap.ranges()) {
+				if (assignedRange.value() == false) {
+					continue;
+				}
 				fs.push_back(krmSetRange(
 				    &tr,
 				    consistencyCheckAssignmentPrefixFor(clientId),
diff --git a/fdbclient/ConsistencyCheckUtil.actor.h b/fdbclient/ConsistencyCheckUtil.actor.h
index 80b3bd2e888..ec251089b06 100644
--- a/fdbclient/ConsistencyCheckUtil.actor.h
+++ b/fdbclient/ConsistencyCheckUtil.actor.h
@@ -42,6 +42,8 @@ ACTOR Future<Void> initConsistencyCheckMetadata(Database cx,
 
 ACTOR Future<Void> clearConsistencyCheckMetadata(Database cx);
 
+ACTOR Future<Void> clearConsistencyCheckAssignment(Database cx);
+
 ACTOR Future<std::vector<KeyRange>> loadRangesToCheckFromProgressMetadata(Database cx);
 
 ACTOR Future<Void> persistConsistencyCheckAssignment(Database cx, int clientId, std::vector<KeyRange> assignedRanges);
diff --git a/fdbserver/tester.actor.cpp b/fdbserver/tester.actor.cpp
index 66c698ba78a..5ebd167b801 100644
--- a/fdbserver/tester.actor.cpp
+++ b/fdbserver/tester.actor.cpp
@@ -1187,7 +1187,8 @@ ACTOR Future<Void> runConsistencyCheckerUrgentCore(Reference<AsyncVar<Optional<C
 		    .detail("RangesToCheck", describe(rangesToCheck))
 		    .detail("Shards", describe(shardsToCheck));
 		// Assign shards to testers
-		state std::unordered_map<int, std::vector<KeyRange>> assignment;
+		wait(clearConsistencyCheckAssignment(cx)); // Clear existing assignment
+		state std::unordered_map<int, std::vector<KeyRange>> assignment; // Generate assignment
 		int batchSize = shardsToCheck.size() / ts.size() + 1;
 		for (int i = 0; i < shardsToCheck.size(); i++) {
 			assignment[i / batchSize].push_back(shardsToCheck[i]);
@@ -1197,8 +1198,9 @@ ACTOR Future<Void> runConsistencyCheckerUrgentCore(Reference<AsyncVar<Optional<C
 			TraceEvent("ConsistencyCheckUrgent_ClientAssignedTask")
 			    .detail("ConsistencyCheckerId", consistencyCheckerId)
 			    .detail("ClientId", assignIt->first)
-			    .detail("Shards", assignIt->second.size());
-			wait(persistConsistencyCheckAssignment(cx, assignIt->first, assignIt->second));
+			    .detail("ShardsCount", assignIt->second.size())
+			    .detail("Shards", describe(assignIt->second));
+			wait(persistConsistencyCheckAssignment(cx, assignIt->first, assignIt->second)); // Persist assignment
 		}
 		TraceEvent e("ConsistencyCheckUrgent_PersistAssignment");
 		e.detail("ConsistencyCheckerId", consistencyCheckerId);

From 87b61a73f6b7facbd434b6686192fd23384209b7 Mon Sep 17 00:00:00 2001
From: Zhe Wang <zhe.wang@wustl.edu>
Date: Sun, 17 Dec 2023 19:19:40 -0800
Subject: [PATCH 341/468] fix bugs

---
 fdbclient/ClientKnobs.cpp                     |  3 +-
 fdbclient/ClientKnobs.h                       |  1 +
 fdbserver/tester.actor.cpp                    | 24 ++++++++--
 .../workloads/ConsistencyCheck.actor.cpp      | 48 +++++++++----------
 4 files changed, 45 insertions(+), 31 deletions(-)

diff --git a/fdbclient/ClientKnobs.cpp b/fdbclient/ClientKnobs.cpp
index eb327c4fd4d..31c1b4bd29c 100644
--- a/fdbclient/ClientKnobs.cpp
+++ b/fdbclient/ClientKnobs.cpp
@@ -258,7 +258,8 @@ void ClientKnobs::initialize(Randomize randomize) {
 
 	init( CONSISTENCY_CHECK_RATE_LIMIT_MAX,        50e6 ); // Limit in per sec
 	init( CONSISTENCY_CHECK_ONE_ROUND_TARGET_COMPLETION_TIME,	7 * 24 * 60 * 60 ); // 7 days
-	init( CONSISTENCY_CHECK_RETRY_DEPTH_MAX,        100 ); if( randomize && BUGGIFY ) CONSISTENCY_CHECK_RETRY_DEPTH_MAX = 10;
+	init( CONSISTENCY_CHECK_BATCH_SHARD_COUNT,      100 ); if( randomize && BUGGIFY ) CONSISTENCY_CHECK_BATCH_SHARD_COUNT = 2;
+	init( CONSISTENCY_CHECK_RETRY_DEPTH_MAX,         10 ); if( randomize && BUGGIFY ) CONSISTENCY_CHECK_RETRY_DEPTH_MAX = 1;
 	init( CONSISTENCY_CHECK_RANGE_BEGIN_0,           "" ); if( randomize && BUGGIFY ) CONSISTENCY_CHECK_RANGE_BEGIN_0 = "";
 	init( CONSISTENCY_CHECK_RANGE_END_0,   "\\xff\\xff" ); if( randomize && BUGGIFY ) CONSISTENCY_CHECK_RANGE_END_0 = "\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x33\\x66\\x63\\x36";
 	init( CONSISTENCY_CHECK_RANGE_BEGIN_1,           "" );
diff --git a/fdbclient/ClientKnobs.h b/fdbclient/ClientKnobs.h
index 31d7c15e8d3..8eb654ea648 100644
--- a/fdbclient/ClientKnobs.h
+++ b/fdbclient/ClientKnobs.h
@@ -249,6 +249,7 @@ class ClientKnobs : public KnobsImpl<ClientKnobs> {
 
 	int CONSISTENCY_CHECK_RATE_LIMIT_MAX; // Not available in urgent mode
 	int CONSISTENCY_CHECK_ONE_ROUND_TARGET_COMPLETION_TIME; // Not available in urgent mode
+	int CONSISTENCY_CHECK_BATCH_SHARD_COUNT; // Available in urgent mode
 	int CONSISTENCY_CHECK_RETRY_DEPTH_MAX; // Available in urgent mode
 	std::string CONSISTENCY_CHECK_RANGE_BEGIN_0; // Available in urgent mode
 	std::string CONSISTENCY_CHECK_RANGE_END_0; // Available in urgent mode
diff --git a/fdbserver/tester.actor.cpp b/fdbserver/tester.actor.cpp
index 5ebd167b801..5d6ea7f3da6 100644
--- a/fdbserver/tester.actor.cpp
+++ b/fdbserver/tester.actor.cpp
@@ -1138,7 +1138,8 @@ ACTOR Future<Void> runConsistencyCheckerUrgentCore(Reference<AsyncVar<Optional<C
                                                    int minTestersExpected,
                                                    TestSpec testSpec,
                                                    Optional<TenantName> defaultTenant,
-                                                   int64_t consistencyCheckerId) {
+                                                   int64_t consistencyCheckerId,
+                                                   Reference<AsyncVar<ServerDBInfo>> dbInfo) {
 	state int round = 0;
 	// Load ranges to check from progress metadata
 	state std::vector<KeyRange> rangesToCheck;
@@ -1189,8 +1190,15 @@ ACTOR Future<Void> runConsistencyCheckerUrgentCore(Reference<AsyncVar<Optional<C
 		// Assign shards to testers
 		wait(clearConsistencyCheckAssignment(cx)); // Clear existing assignment
 		state std::unordered_map<int, std::vector<KeyRange>> assignment; // Generate assignment
-		int batchSize = shardsToCheck.size() / ts.size() + 1;
-		for (int i = 0; i < shardsToCheck.size(); i++) {
+		int batchSize = CLIENT_KNOBS->CONSISTENCY_CHECK_BATCH_SHARD_COUNT;
+		int64_t startingPoint = 0;
+		if (shardsToCheck.size() > batchSize * ts.size()) {
+			startingPoint = deterministicRandom()->randomInt64(0, shardsToCheck.size() - batchSize * ts.size());
+		}
+		for (int i = startingPoint; i < shardsToCheck.size(); i++) {
+			if (i / batchSize > ts.size() - 1) {
+				break; // Fill up all testers
+			}
 			assignment[i / batchSize].push_back(shardsToCheck[i]);
 		}
 		state std::unordered_map<int, std::vector<KeyRange>>::iterator assignIt;
@@ -1239,6 +1247,10 @@ ACTOR Future<Void> runConsistencyCheckerUrgentCore(Reference<AsyncVar<Optional<C
 			    .detail("ConsistencyCheckerId", consistencyCheckerId)
 			    .detail("Round", round)
 			    .detail("RangesToCheckCount", rangesToCheck.size());
+			if (g_network->isSimulated()) {
+				TraceEvent("ConsistencyCheckUrgent_RepairDC");
+				wait(repairDeadDatacenter(cx, dbInfo, "ConsistencyCheck"));
+			}
 			round++;
 		}
 	}
@@ -1275,7 +1287,8 @@ ACTOR Future<Void> checkConsistencyUrgentSim(Database cx,
 			                                     1,
 			                                     spec,
 			                                     Optional<TenantName>(),
-			                                     consistencyCheckerId));
+			                                     consistencyCheckerId,
+			                                     dbInfo));
 			break;
 		} catch (Error& e) {
 			if (e.code() == error_code_actor_cancelled) {
@@ -2164,7 +2177,8 @@ ACTOR Future<Void> runTests(Reference<IClusterConnectionRecord> connRecord,
 		                                                     minTestersExpected,
 		                                                     testSet.testSpecs[0],
 		                                                     defaultTenant,
-		                                                     consistencyCheckerId),
+		                                                     consistencyCheckerId,
+		                                                     dbInfo),
 		                     "runConsistencyCheckerUrgentCore");
 	} else if (at == TEST_HERE) {
 		auto db = makeReference<AsyncVar<ServerDBInfo>>();
diff --git a/fdbserver/workloads/ConsistencyCheck.actor.cpp b/fdbserver/workloads/ConsistencyCheck.actor.cpp
index 24c23b580a7..e67e47bbf68 100644
--- a/fdbserver/workloads/ConsistencyCheck.actor.cpp
+++ b/fdbserver/workloads/ConsistencyCheck.actor.cpp
@@ -1877,6 +1877,23 @@ struct ConsistencyCheckWorkload : TestWorkload {
 								}
 
 							} else {
+								if (!rangeResult.present()) {
+									TraceEvent("ConsistencyCheck_StorageServerUnavailable")
+									    .suppressFor(1.0)
+									    .detail("ShardBegin", req.begin.getKey())
+									    .detail("ShardEnd", req.end.getKey())
+									    .detail("Reason", "rangeResultNotPresent")
+									    .detail("IsTSS", storageServerInterfaces[j].isTss())
+									    .detail("SSID", storageServerInterfaces[j].id());
+								} else if (rangeResult.get().error.present()) {
+									TraceEvent("ConsistencyCheck_StorageServerUnavailable")
+									    .suppressFor(1.0)
+									    .detail("ShardBegin", req.begin.getKey())
+									    .detail("ShardEnd", req.end.getKey())
+									    .detail("Reason", rangeResult.get().error.get().what())
+									    .detail("IsTSS", storageServerInterfaces[j].isTss())
+									    .detail("SSID", storageServerInterfaces[j].id());
+								}
 								completeCheck = false;
 								if (self->consistencyCheckerId != 0) {
 									break; // Will retry later
@@ -2160,12 +2177,11 @@ struct ConsistencyCheckWorkload : TestWorkload {
 			state KeyRangeMap<bool>::Ranges failedRangesList = failedRanges.ranges();
 			state KeyRangeMap<bool>::iterator failedRangesIter = failedRangesList.begin();
 			for (; failedRangesIter != failedRangesList.end(); ++failedRangesIter) {
-				if (failedRangesIter->value() == false) {
-					continue; // No failure, so bypass the range
+				if (failedRangesIter->value()) {
+					failedRangesToCheck.push_back(failedRangesIter->range());
 				}
-				failedRangesToCheck.push_back(failedRangesIter->range());
 			}
-			if (failedRangesToCheck.size() > 0) {
+			if (failedRangesToCheck.size() > 0) { // Retry for any failed shard
 				wait(delay(60.0)); // Backoff 1 min
 				state std::vector<std::pair<KeyRange, Value>> shardLocationPairListForFailedRanges =
 				    wait(self->getKeyLocationsForRangeList(cx, failedRangesToCheck, self));
@@ -2186,28 +2202,10 @@ struct ConsistencyCheckWorkload : TestWorkload {
 					                                          tssMapping,
 					                                          self,
 					                                          consistencyCheckEpoch + 1)));
-				} else {
-					// We give up retrying when retry too many times
-					for (const auto& missedRange : shardLocationPairListForFailedRanges) {
-						TraceEvent(SevWarnAlways, "ConsistencyCheck_MissingRange")
-						    .setMaxEventLength(-1)
-						    .setMaxFieldLength(-1)
-						    .detail("ConsistencyCheckerId", self->consistencyCheckerId)
-						    .detail("Reason", "MaxRetryDepthReached")
-						    .detail("ConsistencyCheckEpoch", consistencyCheckEpoch)
-						    .detail("Distributed", self->distributed)
-						    .detail("ShardCount", ranges.size())
-						    .detail("ClientId", self->clientId)
-						    .detail("ClientCount", self->clientCount)
-						    .detail("BeginKey", printable(missedRange.first.begin))
-						    .detail("EndKey", printable(missedRange.first.end));
-					}
-					if (g_network->isSimulated()) {
-						self->testFailure("Retry too many times");
-						return false;
-					}
 				}
-			} // Otherwise, no failure, we are good to go
+				// We give up retrying when retry too many times
+				// The failed ranges will be picked up by the next round
+			}
 		}
 
 		// SOMEDAY: when background data distribution is implemented, include this test

From 2c65b84e38cd64bb48e90ec84a50c568922c645f Mon Sep 17 00:00:00 2001
From: Zhe Wang <zhe.wang@wustl.edu>
Date: Sun, 17 Dec 2023 21:36:52 -0800
Subject: [PATCH 342/468] introduce random failure of consistency checker

---
 fdbserver/tester.actor.cpp                    | 28 +++++++++++++++++--
 .../workloads/ConsistencyCheck.actor.cpp      |  4 ++-
 2 files changed, 29 insertions(+), 3 deletions(-)

diff --git a/fdbserver/tester.actor.cpp b/fdbserver/tester.actor.cpp
index 5d6ea7f3da6..557d9ea9d54 100644
--- a/fdbserver/tester.actor.cpp
+++ b/fdbserver/tester.actor.cpp
@@ -1144,6 +1144,9 @@ ACTOR Future<Void> runConsistencyCheckerUrgentCore(Reference<AsyncVar<Optional<C
 	// Load ranges to check from progress metadata
 	state std::vector<KeyRange> rangesToCheck;
 	wait(store(rangesToCheck, loadRangesToCheckFromProgressMetadata(cx)));
+	if (g_network->isSimulated() && deterministicRandom()->random01() < 0.05) {
+		throw operation_failed(); // Introduce random failure
+	}
 	if (rangesToCheck.size() == 0) {
 		// If no range to check in progress data
 		// We load the range from knob
@@ -1157,6 +1160,9 @@ ACTOR Future<Void> runConsistencyCheckerUrgentCore(Reference<AsyncVar<Optional<C
 			e.detail("RangeToCheckEnd" + std::to_string(i), rangesToCheck[i].end);
 		}
 		wait(initConsistencyCheckMetadata(cx, rangesToCheck, consistencyCheckerId));
+		if (g_network->isSimulated() && deterministicRandom()->random01() < 0.05) {
+			throw operation_failed(); // Introduce random failure
+		}
 	} else {
 		TraceEvent("ConsistencyCheckUrgent_Resume")
 		    .detail("ConsistencyCheckerId", consistencyCheckerId)
@@ -1170,6 +1176,9 @@ ACTOR Future<Void> runConsistencyCheckerUrgentCore(Reference<AsyncVar<Optional<C
 		ts.clear();
 		if (!testers.present()) {
 			wait(store(ts, getTesters(cc, minTestersExpected)));
+			if (g_network->isSimulated() && deterministicRandom()->random01() < 0.05) {
+				throw operation_failed(); // Introduce random failure
+			}
 		} else {
 			ts = testers.get();
 		}
@@ -1189,6 +1198,9 @@ ACTOR Future<Void> runConsistencyCheckerUrgentCore(Reference<AsyncVar<Optional<C
 		    .detail("Shards", describe(shardsToCheck));
 		// Assign shards to testers
 		wait(clearConsistencyCheckAssignment(cx)); // Clear existing assignment
+		if (g_network->isSimulated() && deterministicRandom()->random01() < 0.05) {
+			throw operation_failed(); // Introduce random failure
+		}
 		state std::unordered_map<int, std::vector<KeyRange>> assignment; // Generate assignment
 		int batchSize = CLIENT_KNOBS->CONSISTENCY_CHECK_BATCH_SHARD_COUNT;
 		int64_t startingPoint = 0;
@@ -1209,6 +1221,9 @@ ACTOR Future<Void> runConsistencyCheckerUrgentCore(Reference<AsyncVar<Optional<C
 			    .detail("ShardsCount", assignIt->second.size())
 			    .detail("Shards", describe(assignIt->second));
 			wait(persistConsistencyCheckAssignment(cx, assignIt->first, assignIt->second)); // Persist assignment
+			if (g_network->isSimulated() && deterministicRandom()->random01() < 0.05) {
+				throw operation_failed(); // Introduce random failure
+			}
 		}
 		TraceEvent e("ConsistencyCheckUrgent_PersistAssignment");
 		e.detail("ConsistencyCheckerId", consistencyCheckerId);
@@ -1233,11 +1248,17 @@ ACTOR Future<Void> runConsistencyCheckerUrgentCore(Reference<AsyncVar<Optional<C
 		// Load progress metadata. If no unfinished range, exit; otherwise, continue
 		rangesToCheck.clear();
 		wait(store(rangesToCheck, loadRangesToCheckFromProgressMetadata(cx)));
+		if (g_network->isSimulated() && deterministicRandom()->random01() < 0.05) {
+			throw operation_failed(); // Introduce random failure
+		}
 		if (rangesToCheck.size() == 0) {
 			TraceEvent("ConsistencyCheckUrgent_Complete")
 			    .detail("ConsistencyCheckerId", consistencyCheckerId)
 			    .detail("Round", round);
 			wait(clearConsistencyCheckMetadata(cx));
+			if (g_network->isSimulated() && deterministicRandom()->random01() < 0.05) {
+				throw operation_failed(); // Introduce random failure
+			}
 			TraceEvent("ConsistencyCheckUrgent_MetadataCleared")
 			    .detail("ConsistencyCheckerId", consistencyCheckerId)
 			    .detail("Round", round);
@@ -1294,9 +1315,12 @@ ACTOR Future<Void> checkConsistencyUrgentSim(Database cx,
 			if (e.code() == error_code_actor_cancelled) {
 				throw e;
 			}
+			if (e.code() == error_code_operation_failed) {
+				continue;
+			}
+			TraceEvent("ConsistencyCheckUrgent_RepairDC");
+			wait(repairDeadDatacenter(cx, dbInfo, "ConsistencyCheck"));
 		}
-		TraceEvent("ConsistencyCheckUrgent_RepairDC");
-		wait(repairDeadDatacenter(cx, dbInfo, "ConsistencyCheck"));
 	}
 	return Void();
 }
diff --git a/fdbserver/workloads/ConsistencyCheck.actor.cpp b/fdbserver/workloads/ConsistencyCheck.actor.cpp
index e67e47bbf68..a993cff6a7f 100644
--- a/fdbserver/workloads/ConsistencyCheck.actor.cpp
+++ b/fdbserver/workloads/ConsistencyCheck.actor.cpp
@@ -1583,7 +1583,9 @@ struct ConsistencyCheckWorkload : TestWorkload {
 				}
 			}
 
-			if (SERVER_KNOBS->CONSISTENCY_CHECK_ROCKSDB_ENGINE || SERVER_KNOBS->CONSISTENCY_CHECK_SQLITE_ENGINE) {
+			if (self->consistencyCheckerId == 0 &&
+			    (SERVER_KNOBS->CONSISTENCY_CHECK_ROCKSDB_ENGINE || SERVER_KNOBS->CONSISTENCY_CHECK_SQLITE_ENGINE)) {
+				// We can skip for shard for engine only in normal consistency check where consistencyCheckerId is 0
 				std::unordered_map<UID, KeyValueStoreType> storageTypeMapping =
 				    wait(self->getStorageType(storageServerInterfaces));
 

From 9d9afeaa8e5631529dfcd9083ac72e45b8844333 Mon Sep 17 00:00:00 2001
From: Zhe Wang <zhe.wang@wustl.edu>
Date: Mon, 18 Dec 2023 10:50:20 -0800
Subject: [PATCH 343/468] simplifying consistency checker urgent workload

---
 fdbclient/ClientKnobs.cpp                     |   2 +
 fdbclient/ClientKnobs.h                       |   2 +
 fdbclient/ConsistencyCheckUtil.actor.cpp      |   1 -
 fdbserver/tester.actor.cpp                    |  20 +-
 .../workloads/ConsistencyCheck.actor.cpp      | 563 +++++++++++++++---
 5 files changed, 506 insertions(+), 82 deletions(-)

diff --git a/fdbclient/ClientKnobs.cpp b/fdbclient/ClientKnobs.cpp
index 31c1b4bd29c..1091542ade4 100644
--- a/fdbclient/ClientKnobs.cpp
+++ b/fdbclient/ClientKnobs.cpp
@@ -260,6 +260,8 @@ void ClientKnobs::initialize(Randomize randomize) {
 	init( CONSISTENCY_CHECK_ONE_ROUND_TARGET_COMPLETION_TIME,	7 * 24 * 60 * 60 ); // 7 days
 	init( CONSISTENCY_CHECK_BATCH_SHARD_COUNT,      100 ); if( randomize && BUGGIFY ) CONSISTENCY_CHECK_BATCH_SHARD_COUNT = 2;
 	init( CONSISTENCY_CHECK_RETRY_DEPTH_MAX,         10 ); if( randomize && BUGGIFY ) CONSISTENCY_CHECK_RETRY_DEPTH_MAX = 1;
+	init( CONSISTENCY_CHECK_INIT_CLEAR_METADATA,  false );
+	init( CONSISTENCY_CHECK_INIT_CLEAR_METADATA_EXIT, false );
 	init( CONSISTENCY_CHECK_RANGE_BEGIN_0,           "" ); if( randomize && BUGGIFY ) CONSISTENCY_CHECK_RANGE_BEGIN_0 = "";
 	init( CONSISTENCY_CHECK_RANGE_END_0,   "\\xff\\xff" ); if( randomize && BUGGIFY ) CONSISTENCY_CHECK_RANGE_END_0 = "\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x33\\x66\\x63\\x36";
 	init( CONSISTENCY_CHECK_RANGE_BEGIN_1,           "" );
diff --git a/fdbclient/ClientKnobs.h b/fdbclient/ClientKnobs.h
index 8eb654ea648..2b90eb35173 100644
--- a/fdbclient/ClientKnobs.h
+++ b/fdbclient/ClientKnobs.h
@@ -251,6 +251,8 @@ class ClientKnobs : public KnobsImpl<ClientKnobs> {
 	int CONSISTENCY_CHECK_ONE_ROUND_TARGET_COMPLETION_TIME; // Not available in urgent mode
 	int CONSISTENCY_CHECK_BATCH_SHARD_COUNT; // Available in urgent mode
 	int CONSISTENCY_CHECK_RETRY_DEPTH_MAX; // Available in urgent mode
+	bool CONSISTENCY_CHECK_INIT_CLEAR_METADATA; // Available in urgent mode
+	bool CONSISTENCY_CHECK_INIT_CLEAR_METADATA_EXIT; // Available in urgent mode
 	std::string CONSISTENCY_CHECK_RANGE_BEGIN_0; // Available in urgent mode
 	std::string CONSISTENCY_CHECK_RANGE_END_0; // Available in urgent mode
 	std::string CONSISTENCY_CHECK_RANGE_BEGIN_1; // Available in urgent mode
diff --git a/fdbclient/ConsistencyCheckUtil.actor.cpp b/fdbclient/ConsistencyCheckUtil.actor.cpp
index ab2e5ad2e60..ce2aa2efc3b 100644
--- a/fdbclient/ConsistencyCheckUtil.actor.cpp
+++ b/fdbclient/ConsistencyCheckUtil.actor.cpp
@@ -44,7 +44,6 @@ ACTOR Future<std::vector<KeyRange>> loadRangesToCheckFromAssignmentMetadata(Data
 					ConsistencyCheckState ccState = decodeConsistencyCheckerStateValue(res_[i].value);
 					if (ccState.getAssignment() == ConsistencyCheckAssignment::Assigned) {
 						res.push_back(currentRange);
-						TraceEvent("Zhe").detail("CurrentRange", currentRange);
 					}
 				}
 				rangeToReadBegin = res_[i + 1].key;
diff --git a/fdbserver/tester.actor.cpp b/fdbserver/tester.actor.cpp
index 557d9ea9d54..9c0d72c4f2f 100644
--- a/fdbserver/tester.actor.cpp
+++ b/fdbserver/tester.actor.cpp
@@ -1140,6 +1140,15 @@ ACTOR Future<Void> runConsistencyCheckerUrgentCore(Reference<AsyncVar<Optional<C
                                                    Optional<TenantName> defaultTenant,
                                                    int64_t consistencyCheckerId,
                                                    Reference<AsyncVar<ServerDBInfo>> dbInfo) {
+	if (CLIENT_KNOBS->CONSISTENCY_CHECK_INIT_CLEAR_METADATA_EXIT) {
+		wait(clearConsistencyCheckMetadata(cx));
+		TraceEvent("ConsistencyCheckUrgent_InitMetadataClearedExit").log();
+		return Void();
+	}
+	if (CLIENT_KNOBS->CONSISTENCY_CHECK_INIT_CLEAR_METADATA) {
+		wait(clearConsistencyCheckMetadata(cx));
+		TraceEvent("ConsistencyCheckUrgent_InitMetadataCleared").log();
+	}
 	state int round = 0;
 	// Load ranges to check from progress metadata
 	state std::vector<KeyRange> rangesToCheck;
@@ -1203,15 +1212,16 @@ ACTOR Future<Void> runConsistencyCheckerUrgentCore(Reference<AsyncVar<Optional<C
 		}
 		state std::unordered_map<int, std::vector<KeyRange>> assignment; // Generate assignment
 		int batchSize = CLIENT_KNOBS->CONSISTENCY_CHECK_BATCH_SHARD_COUNT;
-		int64_t startingPoint = 0;
+		int startingPoint = 0;
 		if (shardsToCheck.size() > batchSize * ts.size()) {
-			startingPoint = deterministicRandom()->randomInt64(0, shardsToCheck.size() - batchSize * ts.size());
+			startingPoint = deterministicRandom()->randomInt(0, shardsToCheck.size() - batchSize * ts.size());
 		}
 		for (int i = startingPoint; i < shardsToCheck.size(); i++) {
-			if (i / batchSize > ts.size() - 1) {
-				break; // Fill up all testers
+			int testerIdx = (i - startingPoint) / batchSize;
+			if (testerIdx > ts.size() - 1) {
+				break; // Filled up all testers
 			}
-			assignment[i / batchSize].push_back(shardsToCheck[i]);
+			assignment[testerIdx].push_back(shardsToCheck[i]);
 		}
 		state std::unordered_map<int, std::vector<KeyRange>>::iterator assignIt;
 		for (assignIt = assignment.begin(); assignIt != assignment.end(); assignIt++) {
diff --git a/fdbserver/workloads/ConsistencyCheck.actor.cpp b/fdbserver/workloads/ConsistencyCheck.actor.cpp
index a993cff6a7f..4322a76238d 100644
--- a/fdbserver/workloads/ConsistencyCheck.actor.cpp
+++ b/fdbserver/workloads/ConsistencyCheck.actor.cpp
@@ -202,7 +202,7 @@ struct ConsistencyCheckWorkload : TestWorkload {
 		    .detail("ClientId", self->clientId)
 		    .detail("Indefinite", self->indefinite)
 		    .detail("Repetitions", self->repetitions);
-		wait(self->runCheck(cx, self));
+		wait(self->runUrgentCheck(cx, self));
 		TraceEvent("ConsistencyCheck_Exit")
 		    .detail("Mode", "Urgent")
 		    .detail("ConsistencyCheckerId", self->consistencyCheckerId)
@@ -270,11 +270,40 @@ struct ConsistencyCheckWorkload : TestWorkload {
 		return Void();
 	}
 
+	ACTOR Future<Void> runUrgentCheck(Database cx, ConsistencyCheckWorkload* self) {
+		try {
+			std::vector<KeyRange> rangesToCheck = wait(self->loadRangesFromAssignmentMetadata(cx, self->clientId));
+			if (rangesToCheck.size() == 0) {
+				TraceEvent("ConsistencyCheckUrgent_AssignedEmptyRangeToCheck")
+				    .setMaxEventLength(-1)
+				    .setMaxFieldLength(-1)
+				    .detail("ConsistencyCheckerId", self->consistencyCheckerId)
+				    .detail("ClientCount", self->clientCount)
+				    .detail("ClientId", self->clientId);
+				return Void();
+			}
+			wait(self->checkDataConsistencyUrgent(cx,
+			                                      rangesToCheck,
+			                                      self,
+			                                      /*consistencyCheckEpoch=*/0));
+		} catch (Error& e) {
+			if (e.code() == error_code_actor_cancelled) {
+				throw e;
+			}
+			TraceEvent("ConsistencyCheckUrgent_RetryLater")
+			    .error(e)
+			    .detail("ConsistencyCheckerId", self->consistencyCheckerId)
+			    .detail("ClientCount", self->clientCount)
+			    .detail("ClientId", self->clientId);
+		}
+		return Void();
+	}
+
 	ACTOR Future<Void> runCheck(Database cx, ConsistencyCheckWorkload* self) {
 		TEST(self->performQuiescentChecks); // Quiescent consistency check
 		TEST(!self->performQuiescentChecks); // Non-quiescent consistency check
 
-		if (self->firstClient || self->distributed || self->consistencyCheckerId != 0) {
+		if (self->firstClient || self->distributed) {
 			try {
 				state DatabaseConfiguration configuration;
 				state std::map<UID, StorageServerInterface> tssMapping;
@@ -382,80 +411,36 @@ struct ConsistencyCheckWorkload : TestWorkload {
 					if (!coordinatorsCorrect)
 						self->testFailure("Coordinators incorrect");
 				}
-
-				if (self->consistencyCheckerId != 0) { // Urgent mode
-					std::vector<KeyRange> rangesToCheck =
-					    wait(self->loadRangesFromAssignmentMetadata(cx, self->clientId));
-					if (rangesToCheck.size() == 0) {
-						TraceEvent("ConsistencyCheck_ConfigWithEmptyRangeToCheck")
-						    .setMaxEventLength(-1)
-						    .setMaxFieldLength(-1)
-						    .detail("ConsistencyCheckerId", self->consistencyCheckerId)
-						    .detail("Indefinite", self->indefinite)
-						    .detail("Distributed", self->distributed)
-						    .detail("ClientCount", self->clientCount)
-						    .detail("ClientId", self->clientId)
-						    .detail("PerformQuiescentChecks", self->performQuiescentChecks)
-						    .detail("PerformTSSCheck", self->performTSSCheck)
-						    .detail("PerformCacheCheck", self->performCacheCheck);
-						return Void();
-					}
-					TraceEvent("ConsistencyCheck_Config")
-					    .setMaxEventLength(-1)
-					    .setMaxFieldLength(-1)
-					    .detail("ConsistencyCheckerId", self->consistencyCheckerId)
-					    .detail("RangeToCheck", describe(rangesToCheck))
-					    .detail("Indefinite", self->indefinite)
-					    .detail("Distributed", self->distributed)
-					    .detail("ClientCount", self->clientCount)
-					    .detail("ClientId", self->clientId)
-					    .detail("PerformQuiescentChecks", self->performQuiescentChecks)
-					    .detail("PerformTSSCheck", self->performTSSCheck)
-					    .detail("PerformCacheCheck", self->performCacheCheck);
-					state std::vector<std::pair<KeyRange, Value>> shardLocationPairListForDistributed =
-					    wait(self->getKeyLocationsForRangeList(cx, rangesToCheck, self));
-					// Check that each failed shard has the same data on all storage servers that it resides on
-					wait(::success(self->checkDataConsistency(cx,
-					                                          shardLocationPairListForDistributed,
-					                                          configuration,
-					                                          tssMapping,
-					                                          self,
-					                                          /*consistencyCheckEpoch=*/0)));
-
-				} else {
-					// Get a list of key servers; verify that the TLogs and master all agree about who the key servers
-					// are
-					state Promise<std::vector<std::pair<KeyRange, std::vector<StorageServerInterface>>>>
-					    keyServerPromise;
-					bool keyServerResult = wait(self->getKeyServers(cx, self, keyServerPromise, keyServersKeys));
-					if (keyServerResult) {
-						state std::vector<std::pair<KeyRange, std::vector<StorageServerInterface>>> keyServers =
-						    keyServerPromise.getFuture().get();
-
-						// Get the locations of all the shards in the database
-						state Promise<Standalone<VectorRef<KeyValueRef>>> keyLocationPromise;
-						bool keyLocationResult = wait(self->getKeyLocations(cx, keyServers, self, keyLocationPromise));
-						if (keyLocationResult) {
-							state Standalone<VectorRef<KeyValueRef>> keyLocations =
-							    keyLocationPromise.getFuture().get();
-							state std::vector<std::pair<KeyRange, Value>> shardLocationPairList;
-							// Check that each shard has the same data on all storage servers that it resides on
-							for (int k = 0; k < keyLocations.size() - 1; k++) {
-								shardLocationPairList.push_back(std::make_pair(
-								    Standalone(KeyRangeRef(keyLocations[k].key, keyLocations[k + 1].key)),
-								    Standalone(keyLocations[k].value)));
-							}
-							wait(::success(self->checkDataConsistency(cx,
-							                                          shardLocationPairList,
-							                                          configuration,
-							                                          tssMapping,
-							                                          self,
-							                                          /*consistencyCheckEpoch=*/0)));
-
-							// Cache consistency check
-							if (self->performCacheCheck)
-								wait(::success(self->checkCacheConsistency(cx, keyLocations, self)));
+				// Get a list of key servers; verify that the TLogs and master all agree about who the key servers
+				// are
+				state Promise<std::vector<std::pair<KeyRange, std::vector<StorageServerInterface>>>> keyServerPromise;
+				bool keyServerResult = wait(self->getKeyServers(cx, self, keyServerPromise, keyServersKeys));
+				if (keyServerResult) {
+					state std::vector<std::pair<KeyRange, std::vector<StorageServerInterface>>> keyServers =
+					    keyServerPromise.getFuture().get();
+
+					// Get the locations of all the shards in the database
+					state Promise<Standalone<VectorRef<KeyValueRef>>> keyLocationPromise;
+					bool keyLocationResult = wait(self->getKeyLocations(cx, keyServers, self, keyLocationPromise));
+					if (keyLocationResult) {
+						state Standalone<VectorRef<KeyValueRef>> keyLocations = keyLocationPromise.getFuture().get();
+						state std::vector<std::pair<KeyRange, Value>> shardLocationPairList;
+						// Check that each shard has the same data on all storage servers that it resides on
+						for (int k = 0; k < keyLocations.size() - 1; k++) {
+							shardLocationPairList.push_back(
+							    std::make_pair(Standalone(KeyRangeRef(keyLocations[k].key, keyLocations[k + 1].key)),
+							                   Standalone(keyLocations[k].value)));
 						}
+						wait(::success(self->checkDataConsistency(cx,
+						                                          shardLocationPairList,
+						                                          configuration,
+						                                          tssMapping,
+						                                          self,
+						                                          /*consistencyCheckEpoch=*/0)));
+
+						// Cache consistency check
+						if (self->performCacheCheck)
+							wait(::success(self->checkCacheConsistency(cx, keyLocations, self)));
 					}
 				}
 			} catch (Error& e) {
@@ -1375,6 +1360,433 @@ struct ConsistencyCheckWorkload : TestWorkload {
 		}
 	}
 
+	ACTOR Future<Void> checkDataConsistencyUrgent(Database cx,
+	                                              std::vector<KeyRange> rangesToCheck,
+	                                              ConsistencyCheckWorkload* self,
+	                                              int consistencyCheckEpoch) {
+		TraceEvent("ConsistencyCheckUrgent_StartTask")
+		    .setMaxEventLength(-1)
+		    .setMaxFieldLength(-1)
+		    .detail("ConsistencyCheckerId", self->consistencyCheckerId)
+		    .detail("RangeToCheck", rangesToCheck.size())
+		    .detail("ClientCount", self->clientCount)
+		    .detail("ClientId", self->clientId)
+		    .detail("ConsistencyCheckEpoch", consistencyCheckEpoch);
+
+		// Get shard locations of the input rangesToCheck
+		state std::vector<std::pair<KeyRange, Value>> shardLocationPairList;
+		state int retryCount = 0;
+		loop {
+			try {
+				shardLocationPairList.clear();
+				wait(store(shardLocationPairList, self->getKeyLocationsForRangeList(cx, rangesToCheck, self)));
+				break;
+			} catch (Error& e) {
+				if (e.code() == error_code_actor_cancelled) {
+					throw e;
+				}
+				TraceEvent(SevWarn, "ConsistencyCheckUrgent_GetKeyLocationListFailed")
+				    .error(e)
+				    .detail("RetryCount", retryCount)
+				    .detail("ConsistencyCheckerId", self->consistencyCheckerId)
+				    .detail("ClientId", self->clientId)
+				    .detail("ClientCount", self->clientCount)
+				    .detail("ShardBegin", range.begin)
+				    .detail("ShardEnd", range.end)
+				    .detail("Repetitions", self->repetitions)
+				    .detail("ConsistencyCheckEpoch", consistencyCheckEpoch);
+				wait(delay(5.0));
+				retryCount++;
+				if (retryCount > 50) {
+					return Void();
+				}
+			}
+		}
+
+		// Do consistency check shard by shard
+		state Reference<IRateControl> rateLimiter =
+		    Reference<IRateControl>(new SpeedLimit(CLIENT_KNOBS->CONSISTENCY_CHECK_RATE_LIMIT_MAX, 1));
+		state KeyRangeMap<bool> failedRanges; // Used to collect failed ranges in the current checkDataConsistency
+		// Which will be used to start the next consistencyCheckEpoch of the checkDataConsistency
+		state int64_t numShardThisClient = shardLocationPairList.size();
+		state int64_t numShardToCheck = -1;
+		state int64_t numCompleteShards = 0;
+		state int64_t numFailedShards = 0;
+		state int shardIdx = 0;
+		for (; shardIdx < shardLocationPairList.size(); ++shardIdx) {
+			numShardToCheck = numShardThisClient - shardIdx;
+			state KeyRangeRef range = shardLocationPairList[shardIdx].first;
+			// Step 1: Get source server id of the shard
+			state std::vector<UID> sourceStorageServers;
+			state std::vector<UID> destStorageServers;
+			retryCount = 0;
+			loop {
+				try {
+					Transaction tr(cx);
+					RangeResult UIDtoTagMap = wait(tr.getRange(serverTagKeys, CLIENT_KNOBS->TOO_MANY));
+					ASSERT(!UIDtoTagMap.more && UIDtoTagMap.size() < CLIENT_KNOBS->TOO_MANY);
+					decodeKeyServersValue(UIDtoTagMap,
+					                      shardLocationPairList[shardIdx].second,
+					                      sourceStorageServers,
+					                      destStorageServers,
+					                      false);
+					break;
+				} catch (Error& e) {
+					if (e.code() == error_code_actor_cancelled) {
+						throw e;
+					}
+					TraceEvent(SevWarn, "ConsistencyCheckUrgent_GetUIDtoTagMapFailed")
+					    .error(e)
+					    .detail("RetryCount", retryCount)
+					    .detail("ConsistencyCheckerId", self->consistencyCheckerId)
+					    .detail("ClientId", self->clientId)
+					    .detail("ClientCount", self->clientCount)
+					    .detail("ShardBegin", range.begin)
+					    .detail("ShardEnd", range.end)
+					    .detail("Repetitions", self->repetitions)
+					    .detail("ConsistencyCheckEpoch", consistencyCheckEpoch);
+					wait(delay(5.0));
+					retryCount++;
+					if (retryCount > 50) {
+						return Void();
+					}
+				}
+			}
+
+			if (sourceStorageServers.size() == 0) {
+				TraceEvent(SevWarnAlways, "ConsistencyCheckUrgent_EmptySourceServers")
+				    .detail("ConsistencyCheckerId", self->consistencyCheckerId)
+				    .detail("ClientId", self->clientId)
+				    .detail("ClientCount", self->clientCount)
+				    .detail("ShardBegin", range.begin)
+				    .detail("ShardEnd", range.end)
+				    .detail("Repetitions", self->repetitions)
+				    .detail("ConsistencyCheckEpoch", consistencyCheckEpoch);
+				numCompleteShards++;
+				try {
+					wait(self->persistProgress(cx, range, self->consistencyCheckerId));
+				} catch (Error& e) {
+					if (e.code() == error_code_key_not_found ||
+					    e.code() == error_code_consistency_check_task_outdated) {
+						return Void();
+					} else {
+						throw e;
+					}
+				}
+				continue; // Skip to the next shard
+			} else if (sourceStorageServers.size() == 1) {
+				numCompleteShards++;
+				try {
+					wait(self->persistProgress(cx, range, self->consistencyCheckerId));
+				} catch (Error& e) {
+					if (e.code() == error_code_key_not_found ||
+					    e.code() == error_code_consistency_check_task_outdated) {
+						return Void();
+					} else {
+						throw e;
+					}
+				}
+				continue; // Skip to the next shard
+			}
+
+			// Step 2: Get server interfaces
+			state std::vector<UID> storageServers = sourceStorageServers; // We check source server
+			TraceEvent("Zhe").detail("Size", storageServers.size());
+			state std::vector<StorageServerInterface> storageServerInterfaces;
+			retryCount = 0;
+			loop {
+				try {
+					Transaction tr(cx);
+					tr.setOption(FDBTransactionOptions::LOCK_AWARE);
+					std::vector<Future<Optional<Value>>> serverListEntries;
+					for (int s = 0; s < storageServers.size(); s++)
+						serverListEntries.push_back(tr.get(serverListKeyFor(storageServers[s])));
+					state std::vector<Optional<Value>> serverListValues = wait(getAll(serverListEntries));
+					for (int s = 0; s < serverListValues.size(); s++) {
+						if (!serverListValues[s].present()) {
+							TraceEvent(SevWarn, "ConsistencyCheckUrgent_GetServerInterfaceFailed")
+							    .detail("SSID", storageServers[s])
+							    .detail("ConsistencyCheckerId", self->consistencyCheckerId)
+							    .detail("ClientId", self->clientId)
+							    .detail("ClientCount", self->clientCount)
+							    .detail("ShardBegin", range.begin)
+							    .detail("ShardEnd", range.end)
+							    .detail("Repetitions", self->repetitions)
+							    .detail("RetryCount", retryCount)
+							    .detail("ConsistencyCheckEpoch", consistencyCheckEpoch);
+							throw operation_failed();
+						}
+						storageServerInterfaces.push_back(decodeServerListValue(serverListValues[s].get()));
+					}
+					break;
+				} catch (Error& e) {
+					if (e.code() == error_code_actor_cancelled) {
+						throw e;
+					}
+					wait(delay(5.0));
+					retryCount++;
+					if (retryCount > 50) {
+						return Void();
+					}
+				}
+			}
+
+			// Step 3: Read a limited number of entries at a time, repeating until all keys in the shard have been read
+			state int64_t totalReadAmount = 0;
+			state bool valueAvailableToCheck = true;
+			state KeySelector begin = firstGreaterOrEqual(range.begin);
+			loop {
+				try {
+					// Get the min version of the storage servers
+					Version version = wait(self->getVersion(cx, self));
+
+					state GetKeyValuesRequest req;
+					req.begin = begin;
+					req.end = firstGreaterOrEqual(range.end);
+					req.limit = 1e4;
+					req.limitBytes = CLIENT_KNOBS->REPLY_BYTE_LIMIT;
+					req.version = version;
+					req.tags = TagSet();
+					req.debugID = debugRandom()->randomUniqueID();
+
+					// Try getting the entries in the specified range
+					state std::vector<Future<ErrorOr<GetKeyValuesReply>>> keyValueFutures;
+					state int j = 0;
+					for (j = 0; j < storageServerInterfaces.size(); j++) {
+						resetReply(req);
+						keyValueFutures.push_back(
+						    storageServerInterfaces[j].getKeyValues.getReplyUnlessFailedFor(req, 2, 0));
+					}
+					wait(waitForAll(keyValueFutures));
+
+					for (j = 0; j < keyValueFutures.size(); j++) {
+						ErrorOr<GetKeyValuesReply> rangeResult = keyValueFutures[j].get();
+						if (!rangeResult.present() || rangeResult.get().error.present()) {
+							valueAvailableToCheck = false;
+							break;
+						}
+					}
+					if (!valueAvailableToCheck) {
+						failedRanges.insert(range, true);
+						TraceEvent(SevInfo, "ConsistencyCheckUrgent_ShardAddedToRetry")
+						    .setMaxEventLength(-1)
+						    .setMaxFieldLength(-1)
+						    .detail("ConsistencyCheckerId", self->consistencyCheckerId)
+						    .detail("ClientId", self->clientId)
+						    .detail("ClientCount", self->clientCount)
+						    .detail("ShardBegin", range.begin)
+						    .detail("ShardEnd", range.end)
+						    .detail("Repetitions", self->repetitions)
+						    .detail("ConsistencyCheckEpoch", consistencyCheckEpoch);
+						break;
+					}
+
+					state int firstValidServer = -1;
+					totalReadAmount = 0;
+					for (j = 0; j < keyValueFutures.size(); j++) {
+						ErrorOr<GetKeyValuesReply> rangeResult = keyValueFutures[j].get();
+						ASSERT(rangeResult.present() && !rangeResult.get().error.present());
+
+						state GetKeyValuesReply current = rangeResult.get();
+						totalReadAmount += current.data.expectedSize();
+						// If we haven't encountered a valid storage server yet, then mark this as the baseline
+						// to compare against
+						if (firstValidServer == -1) {
+							firstValidServer = j;
+						} else {
+							// Compare this shard against the first
+							GetKeyValuesReply reference = keyValueFutures[firstValidServer].get().get();
+							if (current.data != reference.data || current.more != reference.more) {
+								// Data for trace event
+								// The number of keys unique to the current shard
+								int currentUniques = 0;
+								// The number of keys unique to the reference shard
+								int referenceUniques = 0;
+								// The number of keys in both shards with conflicting values
+								int valueMismatches = 0;
+								// The number of keys in both shards with matching values
+								int matchingKVPairs = 0;
+								// Last unique key on the current shard
+								KeyRef currentUniqueKey;
+								// Last unique key on the reference shard
+								KeyRef referenceUniqueKey;
+								// Last value mismatch
+								KeyRef valueMismatchKey;
+
+								// Loop indeces
+								int currentI = 0;
+								int referenceI = 0;
+								while (currentI < current.data.size() || referenceI < reference.data.size()) {
+									if (currentI >= current.data.size()) {
+										referenceUniqueKey = reference.data[referenceI].key;
+										referenceUniques++;
+										referenceI++;
+									} else if (referenceI >= reference.data.size()) {
+										currentUniqueKey = current.data[currentI].key;
+										currentUniques++;
+										currentI++;
+									} else {
+										KeyValueRef currentKV = current.data[currentI];
+										KeyValueRef referenceKV = reference.data[referenceI];
+										if (currentKV.key == referenceKV.key) {
+											if (currentKV.value == referenceKV.value)
+												matchingKVPairs++;
+											else {
+												valueMismatchKey = currentKV.key;
+												valueMismatches++;
+											}
+											currentI++;
+											referenceI++;
+										} else if (currentKV.key < referenceKV.key) {
+											currentUniqueKey = currentKV.key;
+											currentUniques++;
+											currentI++;
+										} else {
+											referenceUniqueKey = referenceKV.key;
+											referenceUniques++;
+											referenceI++;
+										}
+									}
+								}
+
+								TraceEvent(SevError, "ConsistencyCheck_DataInconsistent")
+								    .setMaxEventLength(-1)
+								    .setMaxFieldLength(-1)
+								    .detail(format("StorageServer%d", j).c_str(), storageServers[j].toString())
+								    .detail(format("StorageServer%d", firstValidServer).c_str(),
+								            storageServers[firstValidServer].toString())
+								    .detail("ShardBegin", req.begin.getKey())
+								    .detail("ShardEnd", req.end.getKey())
+								    .detail("VersionNumber", req.version)
+								    .detail(format("Server%dUniques", j).c_str(), currentUniques)
+								    .detail(format("Server%dUniqueKey", j).c_str(), currentUniqueKey)
+								    .detail(format("Server%dUniques", firstValidServer).c_str(), referenceUniques)
+								    .detail(format("Server%dUniqueKey", firstValidServer).c_str(), referenceUniqueKey)
+								    .detail("ValueMismatches", valueMismatches)
+								    .detail("ValueMismatchKey", valueMismatchKey)
+								    .detail("MatchingKVPairs", matchingKVPairs)
+								    .detail("IsTSS",
+								            storageServerInterfaces[j].isTss() ||
+								                    storageServerInterfaces[firstValidServer].isTss()
+								                ? "True"
+								                : "False");
+							}
+						}
+					}
+
+					// RateKeeping
+					wait(rateLimiter->getAllowance(totalReadAmount));
+
+					// Advance to the next set of entries
+					ASSERT(firstValidServer != -1);
+					if (keyValueFutures[firstValidServer].get().get().more) {
+						VectorRef<KeyValueRef> result = keyValueFutures[firstValidServer].get().get().data;
+						ASSERT(result.size() > 0);
+						begin = firstGreaterThan(result[result.size() - 1].key);
+						ASSERT(begin.getKey() != allKeys.end);
+					} else {
+						break;
+					}
+
+				} catch (Error& e) {
+					if (e.code() == error_code_actor_cancelled) {
+						throw e;
+					}
+					TraceEvent("ConsistencyCheckUrgent_RetryDataConsistency").error(e);
+					wait(delay(5.0));
+				}
+			}
+
+			// Step 4: if the shard failed to check, add it to retry queue
+			// Otherwise, persist the progress
+			if (!valueAvailableToCheck) {
+				numFailedShards++;
+				TraceEvent(SevInfo, "ConsistencyCheckUrgent_ShardFailed")
+				    .suppressFor(1.0)
+				    .setMaxEventLength(-1)
+				    .setMaxFieldLength(-1)
+				    .detail("ConsistencyCheckerId", self->consistencyCheckerId)
+				    .detail("ClientId", self->clientId)
+				    .detail("ClientCount", self->clientCount)
+				    .detail("NumCompletedShards", numCompleteShards)
+				    .detail("NumFailedShards", numFailedShards)
+				    .detail("NumShardThisClient", numShardThisClient)
+				    .detail("NumShardToCheckThisEpoch", numShardToCheck - 1)
+				    .detail("ShardBegin", range.begin)
+				    .detail("ShardEnd", range.end)
+				    .detail("Repetitions", self->repetitions)
+				    .detail("ConsistencyCheckEpoch", consistencyCheckEpoch);
+			} else {
+				numCompleteShards++;
+				try {
+					wait(self->persistProgress(cx, range, self->consistencyCheckerId));
+				} catch (Error& e) {
+					if (e.code() == error_code_key_not_found ||
+					    e.code() == error_code_consistency_check_task_outdated) {
+						return Void();
+					} else {
+						throw e;
+					}
+				}
+				TraceEvent(SevInfo, "ConsistencyCheckUrgent_ShardComplete")
+				    .suppressFor(1.0)
+				    .setMaxEventLength(-1)
+				    .setMaxFieldLength(-1)
+				    .detail("ConsistencyCheckerId", self->consistencyCheckerId)
+				    .detail("ClientId", self->clientId)
+				    .detail("ClientCount", self->clientCount)
+				    .detail("Range", range)
+				    .detail("ShardCount", shardLocationPairList.size())
+				    .detail("NumCompletedShards", numCompleteShards)
+				    .detail("NumFailedShards", numFailedShards)
+				    .detail("NumShardThisClient", numShardThisClient)
+				    .detail("NumShardToCheckThisEpoch", numShardToCheck - 1)
+				    .detail("ConsistencyCheckEpoch", consistencyCheckEpoch);
+			}
+		}
+
+		// For the failed ranges, trigger next epoch of checkDataConsistencyUrgent
+		failedRanges.coalesce(allKeys);
+		state std::vector<KeyRange> failedRangesToCheck;
+		state KeyRangeMap<bool>::Ranges failedRangesList = failedRanges.ranges();
+		state KeyRangeMap<bool>::iterator failedRangesIter = failedRangesList.begin();
+		for (; failedRangesIter != failedRangesList.end(); ++failedRangesIter) {
+			if (failedRangesIter->value()) {
+				failedRangesToCheck.push_back(failedRangesIter->range());
+			}
+		}
+		if (failedRangesToCheck.size() > 0) { // Retry for any failed shard
+			wait(delay(60.0)); // Backoff 1 min
+			TraceEvent(SevInfo, "ConsistencyCheckUrgent_StartHandlingFailedRanges")
+			    .setMaxEventLength(-1)
+			    .setMaxFieldLength(-1)
+			    .detail("ConsistencyCheckerId", self->consistencyCheckerId)
+			    .detail("FailedCollectedRangeCount", failedRangesToCheck.size())
+			    .detail("ConsistencyCheckEpoch", consistencyCheckEpoch)
+			    .detail("Distributed", self->distributed)
+			    .detail("ClientId", self->clientId)
+			    .detail("ClientCount", self->clientCount);
+			if (consistencyCheckEpoch < CLIENT_KNOBS->CONSISTENCY_CHECK_RETRY_DEPTH_MAX) {
+				wait(self->checkDataConsistencyUrgent(cx, failedRangesToCheck, self, consistencyCheckEpoch + 1));
+			}
+			// We give up retrying when retry too many times
+			// The failed ranges will be picked up by the next round
+		}
+		if (consistencyCheckEpoch == 0) {
+			TraceEvent("ConsistencyCheckUrgent_EndTask")
+			    .setMaxEventLength(-1)
+			    .setMaxFieldLength(-1)
+			    .detail("ConsistencyCheckerId", self->consistencyCheckerId)
+			    .detail("Distributed", self->distributed)
+			    .detail("ShardCount", shardLocationPairList.size())
+			    .detail("ClientId", self->clientId)
+			    .detail("ClientCount", self->clientCount)
+			    .detail("Repetitions", self->repetitions)
+			    .trackLatest("ConsistencyCheck_EndTask");
+		}
+		return Void();
+	}
+
 	// Checks that the data in each shard is the same on each storage server that it resides on.  Also performs some
 	// sanity checks on the sizes of shards and storage servers. Returns false if there is a failure
 	// We create a new checkDataConsistency actor for retrying for failed ranges and we do consistencyCheckEpoch++
@@ -2023,7 +2435,6 @@ struct ConsistencyCheckWorkload : TestWorkload {
 						} catch (Error& e) {
 							if (e.code() == error_code_key_not_found ||
 							    e.code() == error_code_consistency_check_task_outdated) {
-								TraceEvent(SevInfo, "Zhe").errorUnsuppressed(e);
 								return testResult;
 							} else {
 								throw e;

From 65df92d1fdf003328ee7c413e9e344b46383be74 Mon Sep 17 00:00:00 2001
From: Zhe Wang <zhe.wang@wustl.edu>
Date: Mon, 18 Dec 2023 11:54:19 -0800
Subject: [PATCH 344/468] more trace event

---
 .../workloads/ConsistencyCheck.actor.cpp      | 33 +++++++++++++++++++
 1 file changed, 33 insertions(+)

diff --git a/fdbserver/workloads/ConsistencyCheck.actor.cpp b/fdbserver/workloads/ConsistencyCheck.actor.cpp
index 4322a76238d..a4531d15a66 100644
--- a/fdbserver/workloads/ConsistencyCheck.actor.cpp
+++ b/fdbserver/workloads/ConsistencyCheck.actor.cpp
@@ -1468,6 +1468,17 @@ struct ConsistencyCheckWorkload : TestWorkload {
 				} catch (Error& e) {
 					if (e.code() == error_code_key_not_found ||
 					    e.code() == error_code_consistency_check_task_outdated) {
+						TraceEvent(SevWarn, "ConsistencyCheckUrgent_ExitChecking")
+						    .error(e)
+						    .detail("ConsistencyCheckerId", self->consistencyCheckerId)
+						    .detail("ClientId", self->clientId)
+						    .detail("ClientCount", self->clientCount)
+						    .detail("ShardCount", shardLocationPairList.size())
+						    .detail("NumCompletedShards", numCompleteShards)
+						    .detail("NumFailedShards", numFailedShards)
+						    .detail("NumShardThisClient", numShardThisClient)
+						    .detail("NumShardToCheckThisEpoch", numShardToCheck - 1)
+						    .detail("ConsistencyCheckEpoch", consistencyCheckEpoch);
 						return Void();
 					} else {
 						throw e;
@@ -1481,6 +1492,17 @@ struct ConsistencyCheckWorkload : TestWorkload {
 				} catch (Error& e) {
 					if (e.code() == error_code_key_not_found ||
 					    e.code() == error_code_consistency_check_task_outdated) {
+						TraceEvent(SevWarn, "ConsistencyCheckUrgent_ExitChecking")
+						    .error(e)
+						    .detail("ConsistencyCheckerId", self->consistencyCheckerId)
+						    .detail("ClientId", self->clientId)
+						    .detail("ClientCount", self->clientCount)
+						    .detail("ShardCount", shardLocationPairList.size())
+						    .detail("NumCompletedShards", numCompleteShards)
+						    .detail("NumFailedShards", numFailedShards)
+						    .detail("NumShardThisClient", numShardThisClient)
+						    .detail("NumShardToCheckThisEpoch", numShardToCheck - 1)
+						    .detail("ConsistencyCheckEpoch", consistencyCheckEpoch);
 						return Void();
 					} else {
 						throw e;
@@ -1723,6 +1745,17 @@ struct ConsistencyCheckWorkload : TestWorkload {
 				} catch (Error& e) {
 					if (e.code() == error_code_key_not_found ||
 					    e.code() == error_code_consistency_check_task_outdated) {
+						TraceEvent(SevWarn, "ConsistencyCheckUrgent_ExitChecking")
+						    .error(e)
+						    .detail("ConsistencyCheckerId", self->consistencyCheckerId)
+						    .detail("ClientId", self->clientId)
+						    .detail("ClientCount", self->clientCount)
+						    .detail("ShardCount", shardLocationPairList.size())
+						    .detail("NumCompletedShards", numCompleteShards)
+						    .detail("NumFailedShards", numFailedShards)
+						    .detail("NumShardThisClient", numShardThisClient)
+						    .detail("NumShardToCheckThisEpoch", numShardToCheck - 1)
+						    .detail("ConsistencyCheckEpoch", consistencyCheckEpoch);
 						return Void();
 					} else {
 						throw e;

From 23089a77413dead8d68518d29da29c61813e36c8 Mon Sep 17 00:00:00 2001
From: Zhe Wang <zhe.wang@wustl.edu>
Date: Mon, 18 Dec 2023 12:50:53 -0800
Subject: [PATCH 345/468] better logic of runConsistencyCheckerUrgentCore

---
 fdbclient/ConsistencyCheckUtil.actor.cpp | 50 +++++++++++++-----------
 fdbclient/ConsistencyCheckUtil.actor.h   | 14 +++----
 fdbserver/tester.actor.cpp               | 34 +++++++---------
 3 files changed, 50 insertions(+), 48 deletions(-)

diff --git a/fdbclient/ConsistencyCheckUtil.actor.cpp b/fdbclient/ConsistencyCheckUtil.actor.cpp
index ce2aa2efc3b..c41f7e88565 100644
--- a/fdbclient/ConsistencyCheckUtil.actor.cpp
+++ b/fdbclient/ConsistencyCheckUtil.actor.cpp
@@ -58,7 +58,7 @@ ACTOR Future<std::vector<KeyRange>> loadRangesToCheckFromAssignmentMetadata(Data
 	return res;
 }
 
-ACTOR Future<Void> persistConsistencyCheckProgress(Database cx, KeyRange range, uint64_t consistencyCheckerId) {
+ACTOR Future<Void> persistConsistencyCheckProgress(Database cx, KeyRange range, int64_t consistencyCheckerId) {
 	state Transaction tr(cx);
 	loop {
 		try {
@@ -87,9 +87,7 @@ ACTOR Future<Void> persistConsistencyCheckProgress(Database cx, KeyRange range,
 	return Void();
 }
 
-ACTOR Future<Void> initConsistencyCheckMetadata(Database cx,
-                                                std::vector<KeyRange> rangesToCheck,
-                                                uint64_t consistencyCheckerId) {
+ACTOR Future<Void> initConsistencyCheckAssignmentMetadata(Database cx, int64_t consistencyCheckerId) {
 	state Transaction tr(cx);
 	loop {
 		try {
@@ -97,19 +95,6 @@ ACTOR Future<Void> initConsistencyCheckMetadata(Database cx,
 			tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 			tr.setOption(FDBTransactionOptions::LOCK_AWARE);
 			tr.clear(consistencyCheckAssignmentKeys);
-			tr.clear(consistencyCheckProgressKeys);
-			wait(krmSetRange(&tr,
-			                 consistencyCheckProgressPrefix,
-			                 allKeys,
-			                 consistencyCheckerStateValue(
-			                     ConsistencyCheckState(ConsistencyCheckPhase::Complete)))); // Mark all as completes
-			for (const auto& rangeToCheck : rangesToCheck) {
-				wait(krmSetRange(&tr,
-				                 consistencyCheckProgressPrefix,
-				                 rangeToCheck,
-				                 consistencyCheckerStateValue(ConsistencyCheckState(
-				                     ConsistencyCheckPhase::Invalid)))); // Mark rangesToCheck as incomplete
-			}
 			tr.set(consistencyCheckerIdKey, consistencyCheckerStateValue(ConsistencyCheckState(consistencyCheckerId)));
 			wait(tr.commit());
 			break;
@@ -120,16 +105,26 @@ ACTOR Future<Void> initConsistencyCheckMetadata(Database cx,
 	return Void();
 }
 
-ACTOR Future<Void> clearConsistencyCheckMetadata(Database cx) {
+ACTOR Future<Void> initConsistencyCheckProgressMetadata(Database cx, std::vector<KeyRange> rangesToCheck) {
 	state Transaction tr(cx);
 	loop {
 		try {
 			tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
 			tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 			tr.setOption(FDBTransactionOptions::LOCK_AWARE);
-			tr.clear(consistencyCheckAssignmentKeys);
 			tr.clear(consistencyCheckProgressKeys);
-			tr.clear(consistencyCheckerIdKey);
+			wait(krmSetRange(&tr,
+			                 consistencyCheckProgressPrefix,
+			                 allKeys,
+			                 consistencyCheckerStateValue(
+			                     ConsistencyCheckState(ConsistencyCheckPhase::Complete)))); // Mark all as completes
+			for (const auto& rangeToCheck : rangesToCheck) {
+				wait(krmSetRange(&tr,
+				                 consistencyCheckProgressPrefix,
+				                 rangeToCheck,
+				                 consistencyCheckerStateValue(ConsistencyCheckState(
+				                     ConsistencyCheckPhase::Invalid)))); // Mark rangesToCheck as incomplete
+			}
 			wait(tr.commit());
 			break;
 		} catch (Error& e) {
@@ -139,7 +134,7 @@ ACTOR Future<Void> clearConsistencyCheckMetadata(Database cx) {
 	return Void();
 }
 
-ACTOR Future<Void> clearConsistencyCheckAssignment(Database cx) {
+ACTOR Future<Void> clearConsistencyCheckMetadata(Database cx) {
 	state Transaction tr(cx);
 	loop {
 		try {
@@ -147,6 +142,8 @@ ACTOR Future<Void> clearConsistencyCheckAssignment(Database cx) {
 			tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 			tr.setOption(FDBTransactionOptions::LOCK_AWARE);
 			tr.clear(consistencyCheckAssignmentKeys);
+			tr.clear(consistencyCheckProgressKeys);
+			tr.clear(consistencyCheckerIdKey);
 			wait(tr.commit());
 			break;
 		} catch (Error& e) {
@@ -192,13 +189,22 @@ ACTOR Future<std::vector<KeyRange>> loadRangesToCheckFromProgressMetadata(Databa
 	return res;
 }
 
-ACTOR Future<Void> persistConsistencyCheckAssignment(Database cx, int clientId, std::vector<KeyRange> assignedRanges) {
+ACTOR Future<Void> persistConsistencyCheckAssignment(Database cx,
+                                                     int clientId,
+                                                     std::vector<KeyRange> assignedRanges,
+                                                     int64_t consistencyCheckerId) {
 	state Transaction tr(cx);
 	loop {
 		try {
 			tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
 			tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 			tr.setOption(FDBTransactionOptions::LOCK_AWARE);
+			Optional<Value> res_ = wait(tr.get(consistencyCheckerIdKey));
+			if (!res_.present()) {
+				throw key_not_found();
+			} else if (consistencyCheckerId != decodeConsistencyCheckerStateValue(res_.get()).consistencyCheckerId) {
+				throw consistency_check_task_outdated();
+			}
 			KeyRangeMap<bool> assignedRangesMap;
 			std::vector<Future<Void>> fs;
 			for (const auto& assignedRange : assignedRanges) {
diff --git a/fdbclient/ConsistencyCheckUtil.actor.h b/fdbclient/ConsistencyCheckUtil.actor.h
index ec251089b06..b84836f98a5 100644
--- a/fdbclient/ConsistencyCheckUtil.actor.h
+++ b/fdbclient/ConsistencyCheckUtil.actor.h
@@ -34,19 +34,19 @@
 
 ACTOR Future<std::vector<KeyRange>> loadRangesToCheckFromAssignmentMetadata(Database cx, int clientId);
 
-ACTOR Future<Void> persistConsistencyCheckProgress(Database cx, KeyRange range, uint64_t consistencyCheckerId);
+ACTOR Future<Void> persistConsistencyCheckProgress(Database cx, KeyRange range, int64_t consistencyCheckerId);
 
-ACTOR Future<Void> initConsistencyCheckMetadata(Database cx,
-                                                std::vector<KeyRange> rangesToCheck,
-                                                uint64_t consistencyCheckerId);
+ACTOR Future<Void> initConsistencyCheckAssignmentMetadata(Database cx, int64_t consistencyCheckerId);
+ACTOR Future<Void> initConsistencyCheckProgressMetadata(Database cx, std::vector<KeyRange> rangesToCheck);
 
 ACTOR Future<Void> clearConsistencyCheckMetadata(Database cx);
 
-ACTOR Future<Void> clearConsistencyCheckAssignment(Database cx);
-
 ACTOR Future<std::vector<KeyRange>> loadRangesToCheckFromProgressMetadata(Database cx);
 
-ACTOR Future<Void> persistConsistencyCheckAssignment(Database cx, int clientId, std::vector<KeyRange> assignedRanges);
+ACTOR Future<Void> persistConsistencyCheckAssignment(Database cx,
+                                                     int clientId,
+                                                     std::vector<KeyRange> assignedRanges,
+                                                     int64_t consistencyCheckerId);
 
 std::vector<KeyRange> loadRangesToCheckFromKnob();
 
diff --git a/fdbserver/tester.actor.cpp b/fdbserver/tester.actor.cpp
index 9c0d72c4f2f..214abb42448 100644
--- a/fdbserver/tester.actor.cpp
+++ b/fdbserver/tester.actor.cpp
@@ -1138,7 +1138,6 @@ ACTOR Future<Void> runConsistencyCheckerUrgentCore(Reference<AsyncVar<Optional<C
                                                    int minTestersExpected,
                                                    TestSpec testSpec,
                                                    Optional<TenantName> defaultTenant,
-                                                   int64_t consistencyCheckerId,
                                                    Reference<AsyncVar<ServerDBInfo>> dbInfo) {
 	if (CLIENT_KNOBS->CONSISTENCY_CHECK_INIT_CLEAR_METADATA_EXIT) {
 		wait(clearConsistencyCheckMetadata(cx));
@@ -1160,28 +1159,27 @@ ACTOR Future<Void> runConsistencyCheckerUrgentCore(Reference<AsyncVar<Optional<C
 		// If no range to check in progress data
 		// We load the range from knob
 		rangesToCheck = loadRangesToCheckFromKnob();
+		wait(initConsistencyCheckProgressMetadata(cx, rangesToCheck));
+		if (g_network->isSimulated() && deterministicRandom()->random01() < 0.05) {
+			throw operation_failed(); // Introduce random failure
+		}
 		TraceEvent e("ConsistencyCheckUrgent_Start");
 		e.setMaxEventLength(-1);
 		e.setMaxFieldLength(-1);
-		e.detail("ConsistencyCheckerId", consistencyCheckerId);
 		for (int i = 0; i < rangesToCheck.size(); i++) {
 			e.detail("RangeToCheckBegin" + std::to_string(i), rangesToCheck[i].begin);
 			e.detail("RangeToCheckEnd" + std::to_string(i), rangesToCheck[i].end);
 		}
-		wait(initConsistencyCheckMetadata(cx, rangesToCheck, consistencyCheckerId));
-		if (g_network->isSimulated() && deterministicRandom()->random01() < 0.05) {
-			throw operation_failed(); // Introduce random failure
-		}
 	} else {
-		TraceEvent("ConsistencyCheckUrgent_Resume")
-		    .detail("ConsistencyCheckerId", consistencyCheckerId)
-		    .detail("RangesToCheckCount", rangesToCheck.size());
+		TraceEvent("ConsistencyCheckUrgent_Resume").detail("RangesToCheckCount", rangesToCheck.size());
 	}
 
-	TraceEvent("ConsistencyCheckUrgent_Initialized").detail("ConsistencyCheckerId", consistencyCheckerId);
-
 	state std::vector<TesterInterface> ts;
 	loop {
+		// Decide consistencyCheckerId -- each assignment corresponds to a unique consistencyCheckerId
+		state int64_t consistencyCheckerId = deterministicRandom()->randomInt64(1, 10000000);
+
+		// Get testers
 		ts.clear();
 		if (!testers.present()) {
 			wait(store(ts, getTesters(cc, minTestersExpected)));
@@ -1205,8 +1203,9 @@ ACTOR Future<Void> runConsistencyCheckerUrgentCore(Reference<AsyncVar<Optional<C
 		    .detail("ShardCount", shardsToCheck.size())
 		    .detail("RangesToCheck", describe(rangesToCheck))
 		    .detail("Shards", describe(shardsToCheck));
+
 		// Assign shards to testers
-		wait(clearConsistencyCheckAssignment(cx)); // Clear existing assignment
+		wait(initConsistencyCheckAssignmentMetadata(cx, consistencyCheckerId));
 		if (g_network->isSimulated() && deterministicRandom()->random01() < 0.05) {
 			throw operation_failed(); // Introduce random failure
 		}
@@ -1230,7 +1229,8 @@ ACTOR Future<Void> runConsistencyCheckerUrgentCore(Reference<AsyncVar<Optional<C
 			    .detail("ClientId", assignIt->first)
 			    .detail("ShardsCount", assignIt->second.size())
 			    .detail("Shards", describe(assignIt->second));
-			wait(persistConsistencyCheckAssignment(cx, assignIt->first, assignIt->second)); // Persist assignment
+			wait(persistConsistencyCheckAssignment(
+			    cx, assignIt->first, assignIt->second, consistencyCheckerId)); // Persist assignment
 			if (g_network->isSimulated() && deterministicRandom()->random01() < 0.05) {
 				throw operation_failed(); // Introduce random failure
 			}
@@ -1243,6 +1243,7 @@ ACTOR Future<Void> runConsistencyCheckerUrgentCore(Reference<AsyncVar<Optional<C
 		for (const auto& [clientId, assignedShards] : assignment) {
 			e.detail("Client" + std::to_string(clientId), assignedShards.size());
 		}
+
 		// Run test and wait for testers complete/fail
 		try {
 			wait(dispatchMonitorUrgentConsistencyCheckWorkload(cx, ts, testSpec, defaultTenant, consistencyCheckerId));
@@ -1298,7 +1299,6 @@ ACTOR Future<Void> checkConsistencyUrgentSim(Database cx,
 	spec.title = LiteralStringRef("ConsistencyCheck");
 	spec.databasePingDelay = databasePingDelay;
 	spec.timeout = 32000;
-	state int64_t consistencyCheckerId = deterministicRandom()->randomInt64(1, 10000000);
 	options.push_back_deep(options.arena(),
 	                       KeyValueRef(LiteralStringRef("testName"), LiteralStringRef("ConsistencyCheck")));
 	options.push_back_deep(options.arena(),
@@ -1310,7 +1310,7 @@ ACTOR Future<Void> checkConsistencyUrgentSim(Database cx,
 	options.push_back_deep(options.arena(), KeyValueRef(LiteralStringRef("distributed"), LiteralStringRef("true")));
 	spec.options.push_back_deep(spec.options.arena(), options);
 	loop {
-		TraceEvent("ConsistencyCheckUrgent_LoopBegin").detail("ConsistencyCheckerId", consistencyCheckerId);
+		TraceEvent("ConsistencyCheckUrgent_LoopBegin");
 		try {
 			wait(runConsistencyCheckerUrgentCore(Reference<AsyncVar<Optional<ClusterControllerFullInterface>>>(),
 			                                     cx,
@@ -1318,7 +1318,6 @@ ACTOR Future<Void> checkConsistencyUrgentSim(Database cx,
 			                                     1,
 			                                     spec,
 			                                     Optional<TenantName>(),
-			                                     consistencyCheckerId,
 			                                     dbInfo));
 			break;
 		} catch (Error& e) {
@@ -2103,10 +2102,8 @@ ACTOR Future<Void> runTests(Reference<IClusterConnectionRecord> connRecord,
 		actors.push_back(reportErrors(monitorLeader(connRecord, cc), "MonitorLeader"));
 		actors.push_back(reportErrors(extractClusterInterface(cc, ci), "ExtractClusterInterface"));
 	}
-	state int64_t consistencyCheckerId = 0;
 
 	if (whatToRun == TEST_TYPE_CONSISTENCY_CHECK_URGENT) {
-		consistencyCheckerId = deterministicRandom()->randomInt64(1, 10000000);
 		// consistencyCheckerId must be not 0, indicating this is in urgent mode of consistency checker
 		TestSpec spec;
 		Standalone<VectorRef<KeyValueRef>> options;
@@ -2211,7 +2208,6 @@ ACTOR Future<Void> runTests(Reference<IClusterConnectionRecord> connRecord,
 		                                                     minTestersExpected,
 		                                                     testSet.testSpecs[0],
 		                                                     defaultTenant,
-		                                                     consistencyCheckerId,
 		                                                     dbInfo),
 		                     "runConsistencyCheckerUrgentCore");
 	} else if (at == TEST_HERE) {

From a0a1a0ee6bdf806612d866285a5172cb90a90f3f Mon Sep 17 00:00:00 2001
From: Zhe Wang <zhe.wang@wustl.edu>
Date: Mon, 18 Dec 2023 13:54:46 -0800
Subject: [PATCH 346/468] better spec

---
 fdbserver/tester.actor.cpp | 215 +++++++++++++++++++++----------------
 1 file changed, 123 insertions(+), 92 deletions(-)

diff --git a/fdbserver/tester.actor.cpp b/fdbserver/tester.actor.cpp
index 214abb42448..fa2a33bb656 100644
--- a/fdbserver/tester.actor.cpp
+++ b/fdbserver/tester.actor.cpp
@@ -1148,7 +1148,8 @@ ACTOR Future<Void> runConsistencyCheckerUrgentCore(Reference<AsyncVar<Optional<C
 		wait(clearConsistencyCheckMetadata(cx));
 		TraceEvent("ConsistencyCheckUrgent_InitMetadataCleared").log();
 	}
-	state int round = 0;
+
+	// Init
 	// Load ranges to check from progress metadata
 	state std::vector<KeyRange> rangesToCheck;
 	wait(store(rangesToCheck, loadRangesToCheckFromProgressMetadata(cx)));
@@ -1174,116 +1175,144 @@ ACTOR Future<Void> runConsistencyCheckerUrgentCore(Reference<AsyncVar<Optional<C
 		TraceEvent("ConsistencyCheckUrgent_Resume").detail("RangesToCheckCount", rangesToCheck.size());
 	}
 
-	state std::vector<TesterInterface> ts;
+	// Main loop
+	state int retryTimes = 0;
+	state int round = 0;
+	state int64_t consistencyCheckerId;
 	loop {
-		// Decide consistencyCheckerId -- each assignment corresponds to a unique consistencyCheckerId
-		state int64_t consistencyCheckerId = deterministicRandom()->randomInt64(1, 10000000);
-
-		// Get testers
-		ts.clear();
-		if (!testers.present()) {
-			wait(store(ts, getTesters(cc, minTestersExpected)));
+		try {
+			// Load ranges to check
+			wait(store(rangesToCheck, loadRangesToCheckFromProgressMetadata(cx)));
 			if (g_network->isSimulated() && deterministicRandom()->random01() < 0.05) {
 				throw operation_failed(); // Introduce random failure
 			}
-		} else {
-			ts = testers.get();
-		}
-		TraceEvent("ConsistencyCheckUrgent_GoTTesters")
-		    .detail("ConsistencyCheckerId", consistencyCheckerId)
-		    .detail("Round", round)
-		    .detail("TesterCount", ts.size());
-
-		// Load shards to check from keyserver space
-		state std::vector<KeyRange> shardsToCheck = wait(getConsistencyCheckShards(cx, rangesToCheck));
-		TraceEvent("ConsistencyCheckUrgent_GotShardsToCheck")
-		    .detail("ConsistencyCheckerId", consistencyCheckerId)
-		    .detail("Round", round)
-		    .detail("TesterCount", ts.size())
-		    .detail("ShardCount", shardsToCheck.size())
-		    .detail("RangesToCheck", describe(rangesToCheck))
-		    .detail("Shards", describe(shardsToCheck));
-
-		// Assign shards to testers
-		wait(initConsistencyCheckAssignmentMetadata(cx, consistencyCheckerId));
-		if (g_network->isSimulated() && deterministicRandom()->random01() < 0.05) {
-			throw operation_failed(); // Introduce random failure
-		}
-		state std::unordered_map<int, std::vector<KeyRange>> assignment; // Generate assignment
-		int batchSize = CLIENT_KNOBS->CONSISTENCY_CHECK_BATCH_SHARD_COUNT;
-		int startingPoint = 0;
-		if (shardsToCheck.size() > batchSize * ts.size()) {
-			startingPoint = deterministicRandom()->randomInt(0, shardsToCheck.size() - batchSize * ts.size());
-		}
-		for (int i = startingPoint; i < shardsToCheck.size(); i++) {
-			int testerIdx = (i - startingPoint) / batchSize;
-			if (testerIdx > ts.size() - 1) {
-				break; // Filled up all testers
+
+			// Decide consistencyCheckerId -- each assignment corresponds to a unique consistencyCheckerId
+			consistencyCheckerId = deterministicRandom()->randomInt64(1, 10000000);
+
+			// Get testers
+			state std::vector<TesterInterface> ts;
+			if (!testers.present()) {
+				wait(store(ts, getTesters(cc, minTestersExpected)));
+				if (g_network->isSimulated() && deterministicRandom()->random01() < 0.05) {
+					throw operation_failed(); // Introduce random failure
+				}
+			} else {
+				ts = testers.get();
 			}
-			assignment[testerIdx].push_back(shardsToCheck[i]);
-		}
-		state std::unordered_map<int, std::vector<KeyRange>>::iterator assignIt;
-		for (assignIt = assignment.begin(); assignIt != assignment.end(); assignIt++) {
-			TraceEvent("ConsistencyCheckUrgent_ClientAssignedTask")
+			TraceEvent("ConsistencyCheckUrgent_GoTTesters")
 			    .detail("ConsistencyCheckerId", consistencyCheckerId)
-			    .detail("ClientId", assignIt->first)
-			    .detail("ShardsCount", assignIt->second.size())
-			    .detail("Shards", describe(assignIt->second));
-			wait(persistConsistencyCheckAssignment(
-			    cx, assignIt->first, assignIt->second, consistencyCheckerId)); // Persist assignment
+			    .detail("Round", round)
+			    .detail("RetryTimes", retryTimes)
+			    .detail("TesterCount", ts.size());
+
+			// Load shards to check from keyserver space
+			state std::vector<KeyRange> shardsToCheck = wait(getConsistencyCheckShards(cx, rangesToCheck));
+			TraceEvent("ConsistencyCheckUrgent_GotShardsToCheck")
+			    .detail("ConsistencyCheckerId", consistencyCheckerId)
+			    .detail("Round", round)
+			    .detail("RetryTimes", retryTimes)
+			    .detail("TesterCount", ts.size())
+			    .detail("ShardCount", shardsToCheck.size())
+			    .detail("RangesToCheck", describe(rangesToCheck))
+			    .detail("Shards", describe(shardsToCheck));
+
+			// Assign shards to testers
+			wait(initConsistencyCheckAssignmentMetadata(cx, consistencyCheckerId));
 			if (g_network->isSimulated() && deterministicRandom()->random01() < 0.05) {
 				throw operation_failed(); // Introduce random failure
 			}
-		}
-		TraceEvent e("ConsistencyCheckUrgent_PersistAssignment");
-		e.detail("ConsistencyCheckerId", consistencyCheckerId);
-		e.detail("Round", round);
-		e.detail("TesterCount", ts.size());
-		e.detail("ShardCountTotal", shardsToCheck.size());
-		for (const auto& [clientId, assignedShards] : assignment) {
-			e.detail("Client" + std::to_string(clientId), assignedShards.size());
-		}
+			state std::unordered_map<int, std::vector<KeyRange>> assignment; // Generate assignment
+			int batchSize = CLIENT_KNOBS->CONSISTENCY_CHECK_BATCH_SHARD_COUNT;
+			int startingPoint = 0;
+			if (shardsToCheck.size() > batchSize * ts.size()) {
+				startingPoint = deterministicRandom()->randomInt(0, shardsToCheck.size() - batchSize * ts.size());
+			}
+			for (int i = startingPoint; i < shardsToCheck.size(); i++) {
+				int testerIdx = (i - startingPoint) / batchSize;
+				if (testerIdx > ts.size() - 1) {
+					break; // Filled up all testers
+				}
+				assignment[testerIdx].push_back(shardsToCheck[i]);
+			}
+			state std::unordered_map<int, std::vector<KeyRange>>::iterator assignIt;
+			for (assignIt = assignment.begin(); assignIt != assignment.end(); assignIt++) {
+				TraceEvent("ConsistencyCheckUrgent_ClientAssignedTask")
+				    .detail("ConsistencyCheckerId", consistencyCheckerId)
+				    .detail("RetryTimes", retryTimes)
+				    .detail("Round", round)
+				    .detail("ClientId", assignIt->first)
+				    .detail("ShardsCount", assignIt->second.size())
+				    .detail("Shards", describe(assignIt->second));
+				wait(persistConsistencyCheckAssignment(
+				    cx, assignIt->first, assignIt->second, consistencyCheckerId)); // Persist assignment
+				if (g_network->isSimulated() && deterministicRandom()->random01() < 0.05) {
+					throw operation_failed(); // Introduce random failure
+				}
+			}
+			TraceEvent e("ConsistencyCheckUrgent_PersistAssignment");
+			e.detail("ConsistencyCheckerId", consistencyCheckerId);
+			e.detail("Round", round);
+			e.detail("RetryTimes", retryTimes);
+			e.detail("TesterCount", ts.size());
+			e.detail("ShardCountTotal", shardsToCheck.size());
+			for (const auto& [clientId, assignedShards] : assignment) {
+				e.detail("Client" + std::to_string(clientId), assignedShards.size());
+			}
 
-		// Run test and wait for testers complete/fail
-		try {
+			// Run test and wait for testers complete/fail
 			wait(dispatchMonitorUrgentConsistencyCheckWorkload(cx, ts, testSpec, defaultTenant, consistencyCheckerId));
-		} catch (Error& e) {
-			if (e.code() == error_code_actor_cancelled) {
-				throw;
+			if (g_network->isSimulated() && deterministicRandom()->random01() < 0.05) {
+				throw operation_failed(); // Introduce random failure
 			}
-		}
-		TraceEvent("ConsistencyCheckUrgent_RoundComplete")
-		    .detail("ConsistencyCheckerId", consistencyCheckerId)
-		    .detail("Round", round);
-
-		// Load progress metadata. If no unfinished range, exit; otherwise, continue
-		rangesToCheck.clear();
-		wait(store(rangesToCheck, loadRangesToCheckFromProgressMetadata(cx)));
-		if (g_network->isSimulated() && deterministicRandom()->random01() < 0.05) {
-			throw operation_failed(); // Introduce random failure
-		}
-		if (rangesToCheck.size() == 0) {
-			TraceEvent("ConsistencyCheckUrgent_Complete")
+			TraceEvent("ConsistencyCheckUrgent_RoundWorkloadComplete")
 			    .detail("ConsistencyCheckerId", consistencyCheckerId)
+			    .detail("RetryTimes", retryTimes)
 			    .detail("Round", round);
-			wait(clearConsistencyCheckMetadata(cx));
+
+			// Load progress metadata. If no unfinished range, exit; otherwise, continue
+			rangesToCheck.clear();
+			wait(store(rangesToCheck, loadRangesToCheckFromProgressMetadata(cx)));
 			if (g_network->isSimulated() && deterministicRandom()->random01() < 0.05) {
 				throw operation_failed(); // Introduce random failure
 			}
-			TraceEvent("ConsistencyCheckUrgent_MetadataCleared")
+			if (rangesToCheck.size() == 0) {
+				TraceEvent("ConsistencyCheckUrgent_Complete")
+				    .detail("ConsistencyCheckerId", consistencyCheckerId)
+				    .detail("RetryTimes", retryTimes)
+				    .detail("Round", round);
+				wait(clearConsistencyCheckMetadata(cx));
+				if (g_network->isSimulated() && deterministicRandom()->random01() < 0.05) {
+					throw operation_failed(); // Introduce random failure
+				}
+				TraceEvent("ConsistencyCheckUrgent_MetadataCleared")
+				    .detail("ConsistencyCheckerId", consistencyCheckerId)
+				    .detail("RetryTimes", retryTimes)
+				    .detail("Round", round);
+				break;
+			} else {
+				TraceEvent("ConsistencyCheckUrgent_Continue")
+				    .detail("ConsistencyCheckerId", consistencyCheckerId)
+				    .detail("RetryTimes", retryTimes)
+				    .detail("Round", round)
+				    .detail("RangesToCheckCount", rangesToCheck.size());
+				if (g_network->isSimulated()) {
+					TraceEvent("ConsistencyCheckUrgent_RepairDC");
+					wait(repairDeadDatacenter(cx, dbInfo, "ConsistencyCheck"));
+				}
+				round++;
+			}
+		} catch (Error& e) {
+			if (e.code() == error_code_actor_cancelled) {
+				throw e;
+			}
+			TraceEvent("ConsistencyCheckUrgent_RetriableFailure")
+			    .errorUnsuppressed(e)
 			    .detail("ConsistencyCheckerId", consistencyCheckerId)
+			    .detail("RetryTimes", retryTimes)
 			    .detail("Round", round);
-			break;
-		} else {
-			TraceEvent("ConsistencyCheckUrgent_Continue")
-			    .detail("ConsistencyCheckerId", consistencyCheckerId)
-			    .detail("Round", round)
-			    .detail("RangesToCheckCount", rangesToCheck.size());
-			if (g_network->isSimulated()) {
-				TraceEvent("ConsistencyCheckUrgent_RepairDC");
-				wait(repairDeadDatacenter(cx, dbInfo, "ConsistencyCheck"));
-			}
-			round++;
+			wait(delay(10.0));
+			retryTimes++;
 		}
 	}
 	return Void();
@@ -1299,6 +1328,7 @@ ACTOR Future<Void> checkConsistencyUrgentSim(Database cx,
 	spec.title = LiteralStringRef("ConsistencyCheck");
 	spec.databasePingDelay = databasePingDelay;
 	spec.timeout = 32000;
+	spec.phases = TestWorkload::SETUP | TestWorkload::EXECUTION;
 	options.push_back_deep(options.arena(),
 	                       KeyValueRef(LiteralStringRef("testName"), LiteralStringRef("ConsistencyCheck")));
 	options.push_back_deep(options.arena(),
@@ -2112,6 +2142,7 @@ ACTOR Future<Void> runTests(Reference<IClusterConnectionRecord> connRecord,
 		spec.timeout = 0;
 		spec.waitForQuiescenceBegin = false;
 		spec.waitForQuiescenceEnd = false;
+		spec.phases = TestWorkload::SETUP | TestWorkload::EXECUTION;
 		std::string rateLimitMax = format("%d", CLIENT_KNOBS->CONSISTENCY_CHECK_RATE_LIMIT_MAX);
 		options.push_back_deep(options.arena(),
 		                       KeyValueRef(LiteralStringRef("testName"), LiteralStringRef("ConsistencyCheck")));

From 30fc0ecaeadc3a61457da0d0f520b1a59a7f0f3e Mon Sep 17 00:00:00 2001
From: Zhe Wang <zhe.wang@wustl.edu>
Date: Mon, 18 Dec 2023 13:56:05 -0800
Subject: [PATCH 347/468] address comments

---
 fdbserver/workloads/ConsistencyCheck.actor.cpp | 1 -
 1 file changed, 1 deletion(-)

diff --git a/fdbserver/workloads/ConsistencyCheck.actor.cpp b/fdbserver/workloads/ConsistencyCheck.actor.cpp
index a4531d15a66..ea15e963012 100644
--- a/fdbserver/workloads/ConsistencyCheck.actor.cpp
+++ b/fdbserver/workloads/ConsistencyCheck.actor.cpp
@@ -1513,7 +1513,6 @@ struct ConsistencyCheckWorkload : TestWorkload {
 
 			// Step 2: Get server interfaces
 			state std::vector<UID> storageServers = sourceStorageServers; // We check source server
-			TraceEvent("Zhe").detail("Size", storageServers.size());
 			state std::vector<StorageServerInterface> storageServerInterfaces;
 			retryCount = 0;
 			loop {

From af3a97eb58fb80fcda3cc0153a90a325b4aa3c19 Mon Sep 17 00:00:00 2001
From: Zhe Wang <zhe.wang@wustl.edu>
Date: Mon, 18 Dec 2023 15:01:21 -0800
Subject: [PATCH 348/468] addressed comments

---
 fdbclient/ConsistencyCheckUtil.actor.cpp | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/fdbclient/ConsistencyCheckUtil.actor.cpp b/fdbclient/ConsistencyCheckUtil.actor.cpp
index c41f7e88565..1beecf6cece 100644
--- a/fdbclient/ConsistencyCheckUtil.actor.cpp
+++ b/fdbclient/ConsistencyCheckUtil.actor.cpp
@@ -94,6 +94,7 @@ ACTOR Future<Void> initConsistencyCheckAssignmentMetadata(Database cx, int64_t c
 			tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
 			tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 			tr.setOption(FDBTransactionOptions::LOCK_AWARE);
+			tr.addReadConflictRange(singleKeyRange(consistencyCheckerIdKey));
 			tr.clear(consistencyCheckAssignmentKeys);
 			tr.set(consistencyCheckerIdKey, consistencyCheckerStateValue(ConsistencyCheckState(consistencyCheckerId)));
 			wait(tr.commit());
@@ -112,6 +113,7 @@ ACTOR Future<Void> initConsistencyCheckProgressMetadata(Database cx, std::vector
 			tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
 			tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 			tr.setOption(FDBTransactionOptions::LOCK_AWARE);
+			tr.addReadConflictRange(singleKeyRange(consistencyCheckerIdKey));
 			tr.clear(consistencyCheckProgressKeys);
 			wait(krmSetRange(&tr,
 			                 consistencyCheckProgressPrefix,
@@ -141,6 +143,7 @@ ACTOR Future<Void> clearConsistencyCheckMetadata(Database cx) {
 			tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
 			tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 			tr.setOption(FDBTransactionOptions::LOCK_AWARE);
+			tr.addReadConflictRange(singleKeyRange(consistencyCheckerIdKey));
 			tr.clear(consistencyCheckAssignmentKeys);
 			tr.clear(consistencyCheckProgressKeys);
 			tr.clear(consistencyCheckerIdKey);

From bfb98405481640257f13cf8e220222fbe54dc5ef Mon Sep 17 00:00:00 2001
From: Zhe Wang <zhe.wang@wustl.edu>
Date: Mon, 18 Dec 2023 18:14:24 -0800
Subject: [PATCH 349/468] make consistency checker operations serializable

---
 fdbclient/ConsistencyCheckUtil.actor.cpp      |  73 +++++-
 fdbclient/ConsistencyCheckUtil.actor.h        |  23 +-
 fdbserver/tester.actor.cpp                    | 219 +++++++++++++-----
 .../workloads/ConsistencyCheck.actor.cpp      | 109 ++++-----
 4 files changed, 289 insertions(+), 135 deletions(-)

diff --git a/fdbclient/ConsistencyCheckUtil.actor.cpp b/fdbclient/ConsistencyCheckUtil.actor.cpp
index 1beecf6cece..030159928c1 100644
--- a/fdbclient/ConsistencyCheckUtil.actor.cpp
+++ b/fdbclient/ConsistencyCheckUtil.actor.cpp
@@ -22,7 +22,9 @@
 #include "fdbclient/ConsistencyCheck.h"
 #include "flow/actorcompiler.h" // This must be the last #include.
 
-ACTOR Future<std::vector<KeyRange>> loadRangesToCheckFromAssignmentMetadata(Database cx, int clientId) {
+ACTOR Future<std::vector<KeyRange>> loadRangesToCheckFromAssignmentMetadata(Database cx,
+                                                                            int clientId,
+                                                                            int64_t consistencyCheckerId) {
 	state std::vector<KeyRange> res;
 	state Transaction tr(cx);
 	state Key rangeToReadBegin = allKeys.begin;
@@ -32,6 +34,12 @@ ACTOR Future<std::vector<KeyRange>> loadRangesToCheckFromAssignmentMetadata(Data
 			tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
 			tr.setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
 			tr.setOption(FDBTransactionOptions::LOCK_AWARE);
+			Optional<Value> res_ = wait(tr.get(consistencyCheckerIdKey));
+			if (!res_.present()) {
+				throw key_not_found();
+			} else if (consistencyCheckerId != decodeConsistencyCheckerStateValue(res_.get()).consistencyCheckerId) {
+				throw consistency_check_task_outdated();
+			}
 			KeyRange rangeToRead = Standalone(KeyRangeRef(rangeToReadBegin, rangeToReadEnd));
 			RangeResult res_ = wait(krmGetRanges(&tr,
 			                                     consistencyCheckAssignmentPrefixFor(clientId),
@@ -94,26 +102,37 @@ ACTOR Future<Void> initConsistencyCheckAssignmentMetadata(Database cx, int64_t c
 			tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
 			tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 			tr.setOption(FDBTransactionOptions::LOCK_AWARE);
-			tr.addReadConflictRange(singleKeyRange(consistencyCheckerIdKey));
+			Optional<Value> res_ = wait(tr.get(consistencyCheckerIdKey));
+			if (!res_.present()) {
+				throw key_not_found();
+			} else if (consistencyCheckerId != decodeConsistencyCheckerStateValue(res_.get()).consistencyCheckerId) {
+				throw consistency_check_task_outdated();
+			}
 			tr.clear(consistencyCheckAssignmentKeys);
-			tr.set(consistencyCheckerIdKey, consistencyCheckerStateValue(ConsistencyCheckState(consistencyCheckerId)));
 			wait(tr.commit());
 			break;
 		} catch (Error& e) {
-			wait(tr.onError(e) && delay(10.0));
+			wait(tr.onError(e) && delay(5.0));
 		}
 	}
 	return Void();
 }
 
-ACTOR Future<Void> initConsistencyCheckProgressMetadata(Database cx, std::vector<KeyRange> rangesToCheck) {
+ACTOR Future<Void> initConsistencyCheckProgressMetadata(Database cx,
+                                                        std::vector<KeyRange> rangesToCheck,
+                                                        int64_t consistencyCheckerId) {
 	state Transaction tr(cx);
 	loop {
 		try {
 			tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
 			tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 			tr.setOption(FDBTransactionOptions::LOCK_AWARE);
-			tr.addReadConflictRange(singleKeyRange(consistencyCheckerIdKey));
+			Optional<Value> res_ = wait(tr.get(consistencyCheckerIdKey));
+			if (!res_.present()) {
+				throw key_not_found();
+			} else if (consistencyCheckerId != decodeConsistencyCheckerStateValue(res_.get()).consistencyCheckerId) {
+				throw consistency_check_task_outdated();
+			}
 			tr.clear(consistencyCheckProgressKeys);
 			wait(krmSetRange(&tr,
 			                 consistencyCheckProgressPrefix,
@@ -136,14 +155,36 @@ ACTOR Future<Void> initConsistencyCheckProgressMetadata(Database cx, std::vector
 	return Void();
 }
 
-ACTOR Future<Void> clearConsistencyCheckMetadata(Database cx) {
+ACTOR Future<Void> persistConsistencyCheckerId(Database cx, int64_t consistencyCheckerId) {
+	state Transaction tr(cx);
+	loop {
+		try {
+			tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
+			tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
+			tr.setOption(FDBTransactionOptions::LOCK_AWARE);
+			tr.set(consistencyCheckerIdKey, consistencyCheckerStateValue(ConsistencyCheckState(consistencyCheckerId)));
+			wait(tr.commit());
+			break;
+		} catch (Error& e) {
+			wait(tr.onError(e) && delay(10.0));
+		}
+	}
+	return Void();
+}
+
+ACTOR Future<Void> clearConsistencyCheckMetadata(Database cx, int64_t consistencyCheckerId) {
 	state Transaction tr(cx);
 	loop {
 		try {
 			tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
 			tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 			tr.setOption(FDBTransactionOptions::LOCK_AWARE);
-			tr.addReadConflictRange(singleKeyRange(consistencyCheckerIdKey));
+			Optional<Value> res_ = wait(tr.get(consistencyCheckerIdKey));
+			if (!res_.present()) {
+				throw key_not_found();
+			} else if (consistencyCheckerId != decodeConsistencyCheckerStateValue(res_.get()).consistencyCheckerId) {
+				throw consistency_check_task_outdated();
+			}
 			tr.clear(consistencyCheckAssignmentKeys);
 			tr.clear(consistencyCheckProgressKeys);
 			tr.clear(consistencyCheckerIdKey);
@@ -156,16 +197,24 @@ ACTOR Future<Void> clearConsistencyCheckMetadata(Database cx) {
 	return Void();
 }
 
-ACTOR Future<std::vector<KeyRange>> loadRangesToCheckFromProgressMetadata(Database cx) {
+ACTOR Future<std::vector<KeyRange>> loadRangesToCheckFromProgressMetadata(Database cx, int64_t consistencyCheckerId) {
 	state std::vector<KeyRange> res;
+	state int64_t shardCompleted = 0;
 	state Transaction tr(cx);
 	state Key rangeToReadBegin = allKeys.begin;
 	state Key rangeToReadEnd = allKeys.end;
 	loop {
 		try {
+			shardCompleted = 0;
 			tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
 			tr.setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
 			tr.setOption(FDBTransactionOptions::LOCK_AWARE);
+			Optional<Value> res_ = wait(tr.get(consistencyCheckerIdKey));
+			if (!res_.present()) {
+				throw key_not_found();
+			} else if (consistencyCheckerId != decodeConsistencyCheckerStateValue(res_.get()).consistencyCheckerId) {
+				throw consistency_check_task_outdated();
+			}
 			KeyRange rangeToRead = Standalone(KeyRangeRef(rangeToReadBegin, rangeToReadEnd));
 			RangeResult res_ = wait(krmGetRanges(&tr,
 			                                     consistencyCheckProgressPrefix,
@@ -178,6 +227,8 @@ ACTOR Future<std::vector<KeyRange>> loadRangesToCheckFromProgressMetadata(Databa
 					ConsistencyCheckState ccState = decodeConsistencyCheckerStateValue(res_[i].value);
 					if (ccState.getPhase() == ConsistencyCheckPhase::Invalid) {
 						res.push_back(currentRange);
+					} else {
+						shardCompleted++;
 					}
 				}
 				rangeToReadBegin = res_[i + 1].key;
@@ -189,6 +240,10 @@ ACTOR Future<std::vector<KeyRange>> loadRangesToCheckFromProgressMetadata(Databa
 			wait(tr.onError(e));
 		}
 	}
+	TraceEvent("ConsistencyCheckUrgent_GlobalProgress")
+	    .suppressFor(10.0)
+	    .detail("ConsistencyCheckerId", consistencyCheckerId)
+	    .detail("ShardCompleted", shardCompleted);
 	return res;
 }
 
diff --git a/fdbclient/ConsistencyCheckUtil.actor.h b/fdbclient/ConsistencyCheckUtil.actor.h
index b84836f98a5..d8771a2d0f1 100644
--- a/fdbclient/ConsistencyCheckUtil.actor.h
+++ b/fdbclient/ConsistencyCheckUtil.actor.h
@@ -32,16 +32,31 @@
 #include "fdbclient/DatabaseContext.h"
 #include "fdbclient/SystemData.h"
 
-ACTOR Future<std::vector<KeyRange>> loadRangesToCheckFromAssignmentMetadata(Database cx, int clientId);
+// Key invariant
+// Only persistConsistencyCheckerId() directly overwrites the consistencyCheckerId
+// Other APIs check the consistencyCheckerId on disk before updating/reading from disk
+// This invariant guarantees that there is at most one consistency checker --- the
+// last checker of overwriting consistencyCheckerIdKey wins.
+// Other checkers will receive consistency_check_task_outdated error and the checker should
+// abort itself.
+
+ACTOR Future<std::vector<KeyRange>> loadRangesToCheckFromAssignmentMetadata(Database cx,
+                                                                            int clientId,
+                                                                            int64_t consistencyCheckerId);
 
 ACTOR Future<Void> persistConsistencyCheckProgress(Database cx, KeyRange range, int64_t consistencyCheckerId);
 
 ACTOR Future<Void> initConsistencyCheckAssignmentMetadata(Database cx, int64_t consistencyCheckerId);
-ACTOR Future<Void> initConsistencyCheckProgressMetadata(Database cx, std::vector<KeyRange> rangesToCheck);
 
-ACTOR Future<Void> clearConsistencyCheckMetadata(Database cx);
+ACTOR Future<Void> initConsistencyCheckProgressMetadata(Database cx,
+                                                        std::vector<KeyRange> rangesToCheck,
+                                                        int64_t consistencyCheckerId);
+
+ACTOR Future<Void> persistConsistencyCheckerId(Database cx, int64_t consistencyCheckerId);
+
+ACTOR Future<Void> clearConsistencyCheckMetadata(Database cx, int64_t consistencyCheckerId);
 
-ACTOR Future<std::vector<KeyRange>> loadRangesToCheckFromProgressMetadata(Database cx);
+ACTOR Future<std::vector<KeyRange>> loadRangesToCheckFromProgressMetadata(Database cx, int64_t consistencyCheckerId);
 
 ACTOR Future<Void> persistConsistencyCheckAssignment(Database cx,
                                                      int clientId,
diff --git a/fdbserver/tester.actor.cpp b/fdbserver/tester.actor.cpp
index fa2a33bb656..6d2f2cae894 100644
--- a/fdbserver/tester.actor.cpp
+++ b/fdbserver/tester.actor.cpp
@@ -1132,6 +1132,80 @@ ACTOR Future<std::vector<TesterInterface>> getTesters(Reference<AsyncVar<Optiona
 	return ts;
 }
 
+ACTOR Future<std::vector<KeyRange>> runConsistencyCheckerUrgentInit(Database cx, int64_t consistencyCheckerId) {
+	state std::vector<KeyRange> rangesToCheck;
+	state int retryTimes = 0;
+	loop {
+		try {
+			// Persist consistencyCheckerId
+			// The system allows one consistency checker at a time
+			// The unique ID is persisted in metadata, indicating which consistency checker takes effect
+			wait(persistConsistencyCheckerId(cx, consistencyCheckerId)); // Always succeed
+			if (g_network->isSimulated() && deterministicRandom()->random01() < 0.05) {
+				throw operation_failed(); // Introduce random failure
+			}
+
+			// If INIT_CLEAR_METADATA_EXIT mode, the metadata is cleared at beginning
+			if (CLIENT_KNOBS->CONSISTENCY_CHECK_INIT_CLEAR_METADATA ||
+			    CLIENT_KNOBS->CONSISTENCY_CHECK_INIT_CLEAR_METADATA_EXIT) {
+				wait(clearConsistencyCheckMetadata(cx, consistencyCheckerId));
+				TraceEvent("ConsistencyCheckUrgent_MetadataClearedWhenInit")
+				    .detail("ConsistencyCheckerId", consistencyCheckerId);
+				return rangesToCheck;
+			}
+
+			// Load ranges to check from progress metadata
+			wait(store(rangesToCheck, loadRangesToCheckFromProgressMetadata(cx, consistencyCheckerId)));
+			if (g_network->isSimulated() && deterministicRandom()->random01() < 0.05) {
+				throw operation_failed(); // Introduce random failure
+			}
+
+			// Prepare for the ranges to check and persist consistency checker id
+			if (rangesToCheck.size() == 0) {
+				// If no range to check in progress data
+				// We load the range from knob
+				rangesToCheck = loadRangesToCheckFromKnob();
+				wait(initConsistencyCheckProgressMetadata(cx, rangesToCheck, consistencyCheckerId));
+				if (g_network->isSimulated() && deterministicRandom()->random01() < 0.05) {
+					throw operation_failed(); // Introduce random failure
+				}
+				TraceEvent e("ConsistencyCheckUrgent_Start");
+				e.setMaxEventLength(-1);
+				e.setMaxFieldLength(-1);
+				e.detail("ConsistencyCheckerId", consistencyCheckerId);
+				for (int i = 0; i < rangesToCheck.size(); i++) {
+					e.detail("RangeToCheckBegin" + std::to_string(i), rangesToCheck[i].begin);
+					e.detail("RangeToCheckEnd" + std::to_string(i), rangesToCheck[i].end);
+				}
+			} else {
+				TraceEvent("ConsistencyCheckUrgent_Resume")
+				    .detail("ConsistencyCheckerId", consistencyCheckerId)
+				    .detail("RangesToCheckCount", rangesToCheck.size());
+			}
+			break;
+
+		} catch (Error& e) {
+			if (e.code() == error_code_actor_cancelled) {
+				throw e;
+			} else if (e.code() == error_code_key_not_found || e.code() == error_code_consistency_check_task_outdated) {
+				throw e;
+			} else {
+				TraceEvent("ConsistencyCheckUrgent_RetriableFailure")
+				    .errorUnsuppressed(e)
+				    .detail("ConsistencyCheckerId", consistencyCheckerId)
+				    .detail("RetryTimes", retryTimes);
+				if (retryTimes > 50) {
+					throw timed_out();
+				}
+				wait(delay(10.0));
+				retryTimes++;
+			}
+		}
+	}
+
+	return rangesToCheck;
+}
+
 ACTOR Future<Void> runConsistencyCheckerUrgentCore(Reference<AsyncVar<Optional<ClusterControllerFullInterface>>> cc,
                                                    Database cx,
                                                    Optional<std::vector<TesterInterface>> testers,
@@ -1139,59 +1213,55 @@ ACTOR Future<Void> runConsistencyCheckerUrgentCore(Reference<AsyncVar<Optional<C
                                                    TestSpec testSpec,
                                                    Optional<TenantName> defaultTenant,
                                                    Reference<AsyncVar<ServerDBInfo>> dbInfo) {
-	if (CLIENT_KNOBS->CONSISTENCY_CHECK_INIT_CLEAR_METADATA_EXIT) {
-		wait(clearConsistencyCheckMetadata(cx));
-		TraceEvent("ConsistencyCheckUrgent_InitMetadataClearedExit").log();
-		return Void();
-	}
-	if (CLIENT_KNOBS->CONSISTENCY_CHECK_INIT_CLEAR_METADATA) {
-		wait(clearConsistencyCheckMetadata(cx));
-		TraceEvent("ConsistencyCheckUrgent_InitMetadataCleared").log();
-	}
-
-	// Init
-	// Load ranges to check from progress metadata
+	state int64_t consistencyCheckerId = deterministicRandom()->randomInt64(1, 10000000);
 	state std::vector<KeyRange> rangesToCheck;
-	wait(store(rangesToCheck, loadRangesToCheckFromProgressMetadata(cx)));
-	if (g_network->isSimulated() && deterministicRandom()->random01() < 0.05) {
-		throw operation_failed(); // Introduce random failure
-	}
-	if (rangesToCheck.size() == 0) {
-		// If no range to check in progress data
-		// We load the range from knob
-		rangesToCheck = loadRangesToCheckFromKnob();
-		wait(initConsistencyCheckProgressMetadata(cx, rangesToCheck));
-		if (g_network->isSimulated() && deterministicRandom()->random01() < 0.05) {
-			throw operation_failed(); // Introduce random failure
-		}
-		TraceEvent e("ConsistencyCheckUrgent_Start");
-		e.setMaxEventLength(-1);
-		e.setMaxFieldLength(-1);
-		for (int i = 0; i < rangesToCheck.size(); i++) {
-			e.detail("RangeToCheckBegin" + std::to_string(i), rangesToCheck[i].begin);
-			e.detail("RangeToCheckEnd" + std::to_string(i), rangesToCheck[i].end);
+	// Init: enforce consistencyCheckerId and prepare for metadata
+	try {
+		wait(store(rangesToCheck, runConsistencyCheckerUrgentInit(cx, consistencyCheckerId)));
+	} catch (Error& e) {
+		if (e.code() == error_code_key_not_found || e.code() == error_code_consistency_check_task_outdated ||
+		    e.code() == error_code_timed_out) {
+			TraceEvent("ConsistencyCheckUrgent_Exit")
+			    .errorUnsuppressed(e)
+			    .detail("Reason", "FailureWhenInit")
+			    .detail("ConsistencyCheckerId", consistencyCheckerId);
+			return Void();
+		} else {
+			throw e;
 		}
-	} else {
-		TraceEvent("ConsistencyCheckUrgent_Resume").detail("RangesToCheckCount", rangesToCheck.size());
 	}
+	// Immediately exit after the clear for INIT_CLEAR_METADATA_EXIT mode
+	if (CLIENT_KNOBS->CONSISTENCY_CHECK_INIT_CLEAR_METADATA_EXIT) {
+		TraceEvent("ConsistencyCheckUrgent_Exit")
+		    .detail("Reason", "SuccessClearMetadataWhenInit")
+		    .detail("ConsistencyCheckerId", consistencyCheckerId);
+		return Void();
+	}
+	// At this point, consistencyCheckerId has the ownership except that another consistency checker overwrites the id
+	// metadata
 
 	// Main loop
 	state int retryTimes = 0;
 	state int round = 0;
-	state int64_t consistencyCheckerId;
+	state std::vector<TesterInterface> ts;
 	loop {
 		try {
 			// Load ranges to check
-			wait(store(rangesToCheck, loadRangesToCheckFromProgressMetadata(cx)));
+			wait(store(rangesToCheck, loadRangesToCheckFromProgressMetadata(cx, consistencyCheckerId)));
 			if (g_network->isSimulated() && deterministicRandom()->random01() < 0.05) {
 				throw operation_failed(); // Introduce random failure
 			}
-
-			// Decide consistencyCheckerId -- each assignment corresponds to a unique consistencyCheckerId
-			consistencyCheckerId = deterministicRandom()->randomInt64(1, 10000000);
+			if (rangesToCheck.size() == 0) {
+				TraceEvent("ConsistencyCheckUrgent_Exit")
+				    .detail("Reason", "Complete")
+				    .detail("ConsistencyCheckerId", consistencyCheckerId)
+				    .detail("RetryTimes", retryTimes)
+				    .detail("Round", round);
+				return Void();
+			}
 
 			// Get testers
-			state std::vector<TesterInterface> ts;
+			ts.clear();
 			if (!testers.present()) {
 				wait(store(ts, getTesters(cc, minTestersExpected)));
 				if (g_network->isSimulated() && deterministicRandom()->random01() < 0.05) {
@@ -1213,9 +1283,7 @@ ACTOR Future<Void> runConsistencyCheckerUrgentCore(Reference<AsyncVar<Optional<C
 			    .detail("Round", round)
 			    .detail("RetryTimes", retryTimes)
 			    .detail("TesterCount", ts.size())
-			    .detail("ShardCount", shardsToCheck.size())
-			    .detail("RangesToCheck", describe(rangesToCheck))
-			    .detail("Shards", describe(shardsToCheck));
+			    .detail("ShardCount", shardsToCheck.size());
 
 			// Assign shards to testers
 			wait(initConsistencyCheckAssignmentMetadata(cx, consistencyCheckerId));
@@ -1227,11 +1295,14 @@ ACTOR Future<Void> runConsistencyCheckerUrgentCore(Reference<AsyncVar<Optional<C
 			int startingPoint = 0;
 			if (shardsToCheck.size() > batchSize * ts.size()) {
 				startingPoint = deterministicRandom()->randomInt(0, shardsToCheck.size() - batchSize * ts.size());
+				// We randomly pick a set of successive shards:
+				// (1) We want to retry for different shards to avoid repeated failure on the same shards
+				// (2) We want to check successive shards to avoid inefficiency incurred by fragments
 			}
 			for (int i = startingPoint; i < shardsToCheck.size(); i++) {
 				int testerIdx = (i - startingPoint) / batchSize;
 				if (testerIdx > ts.size() - 1) {
-					break; // Filled up all testers
+					break; // Have filled up all testers
 				}
 				assignment[testerIdx].push_back(shardsToCheck[i]);
 			}
@@ -1272,7 +1343,8 @@ ACTOR Future<Void> runConsistencyCheckerUrgentCore(Reference<AsyncVar<Optional<C
 
 			// Load progress metadata. If no unfinished range, exit; otherwise, continue
 			rangesToCheck.clear();
-			wait(store(rangesToCheck, loadRangesToCheckFromProgressMetadata(cx)));
+			wait(store(rangesToCheck, loadRangesToCheckFromProgressMetadata(cx, consistencyCheckerId)));
+			// If rangesToCheck has data, rangesToCheck is for the next round
 			if (g_network->isSimulated() && deterministicRandom()->random01() < 0.05) {
 				throw operation_failed(); // Introduce random failure
 			}
@@ -1281,15 +1353,16 @@ ACTOR Future<Void> runConsistencyCheckerUrgentCore(Reference<AsyncVar<Optional<C
 				    .detail("ConsistencyCheckerId", consistencyCheckerId)
 				    .detail("RetryTimes", retryTimes)
 				    .detail("Round", round);
-				wait(clearConsistencyCheckMetadata(cx));
+				wait(clearConsistencyCheckMetadata(cx, consistencyCheckerId));
 				if (g_network->isSimulated() && deterministicRandom()->random01() < 0.05) {
 					throw operation_failed(); // Introduce random failure
 				}
-				TraceEvent("ConsistencyCheckUrgent_MetadataCleared")
+				TraceEvent("ConsistencyCheckUrgent_Exit")
+				    .detail("Reason", "Complete")
 				    .detail("ConsistencyCheckerId", consistencyCheckerId)
 				    .detail("RetryTimes", retryTimes)
 				    .detail("Round", round);
-				break;
+				return Void(); // Exit
 			} else {
 				TraceEvent("ConsistencyCheckUrgent_Continue")
 				    .detail("ConsistencyCheckerId", consistencyCheckerId)
@@ -1305,17 +1378,55 @@ ACTOR Future<Void> runConsistencyCheckerUrgentCore(Reference<AsyncVar<Optional<C
 		} catch (Error& e) {
 			if (e.code() == error_code_actor_cancelled) {
 				throw e;
+			} else if (e.code() == error_code_key_not_found || e.code() == error_code_consistency_check_task_outdated) {
+				TraceEvent("ConsistencyCheckUrgent_Exit")
+				    .errorUnsuppressed(e)
+				    .detail("Reason", "ConsistencyCheckerOutdated")
+				    .detail("ConsistencyCheckerId", consistencyCheckerId)
+				    .detail("RetryTimes", retryTimes)
+				    .detail("Round", round);
+				return Void(); // Exit
+			} else {
+				TraceEvent("ConsistencyCheckUrgent_RetriableFailure")
+				    .errorUnsuppressed(e)
+				    .detail("ConsistencyCheckerId", consistencyCheckerId)
+				    .detail("RetryTimes", retryTimes)
+				    .detail("Round", round);
+				wait(delay(10.0));
+				retryTimes++;
+			}
+		}
+
+		wait(delay(10.0));
+
+		// Decide and enforce the consistencyCheckerId for the next round
+		state int retryTimesForUpdatingCheckerId = 0;
+		loop {
+			try {
+				// Decide and persist consistencyCheckerId for the next round
+				consistencyCheckerId = deterministicRandom()->randomInt64(1, 10000000);
+				wait(persistConsistencyCheckerId(cx, consistencyCheckerId));
+				if (g_network->isSimulated() && deterministicRandom()->random01() < 0.05) {
+					throw operation_failed(); // Introduce random failure
+				}
+				break; // Continue to the next round
+			} catch (Error& e) {
+				if (e.code() == error_code_actor_cancelled) {
+					throw e;
+				}
+				if (retryTimesForUpdatingCheckerId > 50) {
+					TraceEvent("ConsistencyCheckUrgent_Exit")
+					    .errorUnsuppressed(e)
+					    .detail("Reason", "PersistConsistencyCheckerIdFailed")
+					    .detail("ConsistencyCheckerId", consistencyCheckerId)
+					    .detail("Round", round);
+					return Void(); // Exit
+				}
+				wait(delay(1.0));
+				retryTimesForUpdatingCheckerId++;
 			}
-			TraceEvent("ConsistencyCheckUrgent_RetriableFailure")
-			    .errorUnsuppressed(e)
-			    .detail("ConsistencyCheckerId", consistencyCheckerId)
-			    .detail("RetryTimes", retryTimes)
-			    .detail("Round", round);
-			wait(delay(10.0));
-			retryTimes++;
 		}
 	}
-	return Void();
 }
 
 ACTOR Future<Void> checkConsistencyUrgentSim(Database cx,
diff --git a/fdbserver/workloads/ConsistencyCheck.actor.cpp b/fdbserver/workloads/ConsistencyCheck.actor.cpp
index ea15e963012..93eff54ba35 100644
--- a/fdbserver/workloads/ConsistencyCheck.actor.cpp
+++ b/fdbserver/workloads/ConsistencyCheck.actor.cpp
@@ -260,8 +260,10 @@ struct ConsistencyCheckWorkload : TestWorkload {
 		return Void();
 	}
 
-	ACTOR Future<std::vector<KeyRange>> loadRangesFromAssignmentMetadata(Database cx, int clientId) {
-		std::vector<KeyRange> res = wait(loadRangesToCheckFromAssignmentMetadata(cx, clientId));
+	ACTOR Future<std::vector<KeyRange>> loadRangesFromAssignmentMetadata(Database cx,
+	                                                                     int clientId,
+	                                                                     int64_t consistencyCheckerId) {
+		std::vector<KeyRange> res = wait(loadRangesToCheckFromAssignmentMetadata(cx, clientId, consistencyCheckerId));
 		return res;
 	}
 
@@ -272,11 +274,11 @@ struct ConsistencyCheckWorkload : TestWorkload {
 
 	ACTOR Future<Void> runUrgentCheck(Database cx, ConsistencyCheckWorkload* self) {
 		try {
-			std::vector<KeyRange> rangesToCheck = wait(self->loadRangesFromAssignmentMetadata(cx, self->clientId));
+			std::vector<KeyRange> rangesToCheck =
+			    wait(self->loadRangesFromAssignmentMetadata(cx, self->clientId, self->consistencyCheckerId));
 			if (rangesToCheck.size() == 0) {
-				TraceEvent("ConsistencyCheckUrgent_AssignedEmptyRangeToCheck")
-				    .setMaxEventLength(-1)
-				    .setMaxFieldLength(-1)
+				TraceEvent("ConsistencyCheckUrgent_TesterExit")
+				    .detail("Reason", "AssignedEmptyRangeToCheck")
 				    .detail("ConsistencyCheckerId", self->consistencyCheckerId)
 				    .detail("ClientCount", self->clientCount)
 				    .detail("ClientId", self->clientId);
@@ -286,15 +288,36 @@ struct ConsistencyCheckWorkload : TestWorkload {
 			                                      rangesToCheck,
 			                                      self,
 			                                      /*consistencyCheckEpoch=*/0));
+			TraceEvent("ConsistencyCheckUrgent_TesterExit")
+			    .detail("Reason", "CompleteChecker")
+			    .detail("ConsistencyCheckerId", self->consistencyCheckerId)
+			    .detail("ClientCount", self->clientCount)
+			    .detail("ClientId", self->clientId);
 		} catch (Error& e) {
 			if (e.code() == error_code_actor_cancelled) {
 				throw e;
+			} else if (e.code() == error_code_timed_out) {
+				TraceEvent("ConsistencyCheckUrgent_TesterExit")
+				    .error(e)
+				    .detail("Reason", "Operation retried too many times")
+				    .detail("ConsistencyCheckerId", self->consistencyCheckerId)
+				    .detail("ClientCount", self->clientCount)
+				    .detail("ClientId", self->clientId);
+			} else if (e.code() == error_code_key_not_found || e.code() == error_code_consistency_check_task_outdated) {
+				TraceEvent("ConsistencyCheckUrgent_TesterExit")
+				    .error(e)
+				    .detail("Reason", "ConsistencyCheckerIdOutDated")
+				    .detail("ConsistencyCheckerId", self->consistencyCheckerId)
+				    .detail("ClientCount", self->clientCount)
+				    .detail("ClientId", self->clientId);
+			} else {
+				TraceEvent("ConsistencyCheckUrgent_TesterExit")
+				    .error(e)
+				    .detail("Reason", "Unknown failure")
+				    .detail("ConsistencyCheckerId", self->consistencyCheckerId)
+				    .detail("ClientCount", self->clientCount)
+				    .detail("ClientId", self->clientId);
 			}
-			TraceEvent("ConsistencyCheckUrgent_RetryLater")
-			    .error(e)
-			    .detail("ConsistencyCheckerId", self->consistencyCheckerId)
-			    .detail("ClientCount", self->clientCount)
-			    .detail("ClientId", self->clientId);
 		}
 		return Void();
 	}
@@ -1393,12 +1416,11 @@ struct ConsistencyCheckWorkload : TestWorkload {
 				    .detail("ClientCount", self->clientCount)
 				    .detail("ShardBegin", range.begin)
 				    .detail("ShardEnd", range.end)
-				    .detail("Repetitions", self->repetitions)
 				    .detail("ConsistencyCheckEpoch", consistencyCheckEpoch);
 				wait(delay(5.0));
 				retryCount++;
 				if (retryCount > 50) {
-					return Void();
+					throw timed_out();
 				}
 			}
 		}
@@ -1443,12 +1465,11 @@ struct ConsistencyCheckWorkload : TestWorkload {
 					    .detail("ClientCount", self->clientCount)
 					    .detail("ShardBegin", range.begin)
 					    .detail("ShardEnd", range.end)
-					    .detail("Repetitions", self->repetitions)
 					    .detail("ConsistencyCheckEpoch", consistencyCheckEpoch);
 					wait(delay(5.0));
 					retryCount++;
 					if (retryCount > 50) {
-						return Void();
+						throw timed_out();
 					}
 				}
 			}
@@ -1466,23 +1487,7 @@ struct ConsistencyCheckWorkload : TestWorkload {
 				try {
 					wait(self->persistProgress(cx, range, self->consistencyCheckerId));
 				} catch (Error& e) {
-					if (e.code() == error_code_key_not_found ||
-					    e.code() == error_code_consistency_check_task_outdated) {
-						TraceEvent(SevWarn, "ConsistencyCheckUrgent_ExitChecking")
-						    .error(e)
-						    .detail("ConsistencyCheckerId", self->consistencyCheckerId)
-						    .detail("ClientId", self->clientId)
-						    .detail("ClientCount", self->clientCount)
-						    .detail("ShardCount", shardLocationPairList.size())
-						    .detail("NumCompletedShards", numCompleteShards)
-						    .detail("NumFailedShards", numFailedShards)
-						    .detail("NumShardThisClient", numShardThisClient)
-						    .detail("NumShardToCheckThisEpoch", numShardToCheck - 1)
-						    .detail("ConsistencyCheckEpoch", consistencyCheckEpoch);
-						return Void();
-					} else {
-						throw e;
-					}
+					throw e;
 				}
 				continue; // Skip to the next shard
 			} else if (sourceStorageServers.size() == 1) {
@@ -1490,23 +1495,7 @@ struct ConsistencyCheckWorkload : TestWorkload {
 				try {
 					wait(self->persistProgress(cx, range, self->consistencyCheckerId));
 				} catch (Error& e) {
-					if (e.code() == error_code_key_not_found ||
-					    e.code() == error_code_consistency_check_task_outdated) {
-						TraceEvent(SevWarn, "ConsistencyCheckUrgent_ExitChecking")
-						    .error(e)
-						    .detail("ConsistencyCheckerId", self->consistencyCheckerId)
-						    .detail("ClientId", self->clientId)
-						    .detail("ClientCount", self->clientCount)
-						    .detail("ShardCount", shardLocationPairList.size())
-						    .detail("NumCompletedShards", numCompleteShards)
-						    .detail("NumFailedShards", numFailedShards)
-						    .detail("NumShardThisClient", numShardThisClient)
-						    .detail("NumShardToCheckThisEpoch", numShardToCheck - 1)
-						    .detail("ConsistencyCheckEpoch", consistencyCheckEpoch);
-						return Void();
-					} else {
-						throw e;
-					}
+					throw e;
 				}
 				continue; // Skip to the next shard
 			}
@@ -1547,7 +1536,7 @@ struct ConsistencyCheckWorkload : TestWorkload {
 					wait(delay(5.0));
 					retryCount++;
 					if (retryCount > 50) {
-						return Void();
+						throw timed_out();
 					}
 				}
 			}
@@ -1742,23 +1731,7 @@ struct ConsistencyCheckWorkload : TestWorkload {
 				try {
 					wait(self->persistProgress(cx, range, self->consistencyCheckerId));
 				} catch (Error& e) {
-					if (e.code() == error_code_key_not_found ||
-					    e.code() == error_code_consistency_check_task_outdated) {
-						TraceEvent(SevWarn, "ConsistencyCheckUrgent_ExitChecking")
-						    .error(e)
-						    .detail("ConsistencyCheckerId", self->consistencyCheckerId)
-						    .detail("ClientId", self->clientId)
-						    .detail("ClientCount", self->clientCount)
-						    .detail("ShardCount", shardLocationPairList.size())
-						    .detail("NumCompletedShards", numCompleteShards)
-						    .detail("NumFailedShards", numFailedShards)
-						    .detail("NumShardThisClient", numShardThisClient)
-						    .detail("NumShardToCheckThisEpoch", numShardToCheck - 1)
-						    .detail("ConsistencyCheckEpoch", consistencyCheckEpoch);
-						return Void();
-					} else {
-						throw e;
-					}
+					throw e;
 				}
 				TraceEvent(SevInfo, "ConsistencyCheckUrgent_ShardComplete")
 				    .suppressFor(1.0)
@@ -1802,7 +1775,7 @@ struct ConsistencyCheckWorkload : TestWorkload {
 				wait(self->checkDataConsistencyUrgent(cx, failedRangesToCheck, self, consistencyCheckEpoch + 1));
 			}
 			// We give up retrying when retry too many times
-			// The failed ranges will be picked up by the next round
+			// The failed ranges will be picked up by the next round of the consistency checker urgent
 		}
 		if (consistencyCheckEpoch == 0) {
 			TraceEvent("ConsistencyCheckUrgent_EndTask")

From 83dbc5ed6cbc800083376d8e50b02dd0798faa49 Mon Sep 17 00:00:00 2001
From: hao fu <hfu5@apple.com>
Date: Tue, 19 Dec 2023 09:02:18 -0800
Subject: [PATCH 350/468] Fix checkall when shard is large

the begin key has to be updated for checkall command when a shard
is large, this PR makes this change.
---
 fdbcli/DebugCommands.actor.cpp | 65 +++++++++++++++++++++-------------
 1 file changed, 40 insertions(+), 25 deletions(-)

diff --git a/fdbcli/DebugCommands.actor.cpp b/fdbcli/DebugCommands.actor.cpp
index ecb877e3aee..d8565c23cc2 100644
--- a/fdbcli/DebugCommands.actor.cpp
+++ b/fdbcli/DebugCommands.actor.cpp
@@ -164,11 +164,12 @@ ACTOR Future<bool> getallCommandActor(Database cx, std::vector<StringRef> tokens
 CommandFactory getallCommandFactory("getall");
 
 // check that all replies are the same. Update begin to the next key to check
-bool checkResults(Version version,
-                  const std::vector<StorageServerInterface>& servers,
-                  const std::vector<Future<ErrorOr<GetKeyValuesReply>>>& replies,
-                  KeySelectorRef& begin,
-                  const KeySelectorRef& end) {
+std::pair<bool, bool> checkResults(Version version,
+                                   const std::vector<StorageServerInterface>& servers,
+                                   const std::vector<Future<ErrorOr<GetKeyValuesReply>>>& replies,
+                                   KeySelectorRef& begin,
+                                   const KeySelectorRef& end,
+                                   Key& lastEnd) {
 	bool allSame = true;
 	int firstValidServer = -1;
 	for (int j = 0; j < replies.size(); j++) {
@@ -252,18 +253,20 @@ bool checkResults(Version version,
 		}
 	}
 
-	if (firstValidServer >= 0 && replies[firstValidServer].get().get().more) {
+	bool hasMore = firstValidServer >= 0 && replies[firstValidServer].get().get().more;
+	if (hasMore) {
 		const VectorRef<KeyValueRef>& result = replies[firstValidServer].get().get().data;
 		printf("Warning: Consistency check was incomplete, last key of server %d that was checked: %s\n",
 		       firstValidServer,
 		       printable(result[result.size() - 1].key).c_str());
-		begin = firstGreaterThan(result[result.size() - 1].key);
+		lastEnd = result[result.size() - 1].key; // store to a standalone, otherwise memory will be invalid
+		begin = firstGreaterThan(lastEnd);
 	} else {
 		printf("Same at version %ld\n", version);
 		begin = end; // signal that we're done
 	}
 
-	return allSame;
+	return std::make_pair(allSame, hasMore);
 }
 
 // The command is used to check the inconsistency in a keyspace, default is \xff\x02/blog/ keyspace.
@@ -307,6 +310,7 @@ ACTOR Future<bool> checkallCommandActor(Database cx, std::vector<StringRef> toke
 
 			state std::vector<std::pair<KeyRange, std::vector<StorageServerInterface>>> keyServers =
 			    keyServerPromise.getFuture().get();
+			state Key lastEnd; // this is to hold the Key having the value of next begin
 			for (i = 0; i < keyServers.size(); i++) { // for each key range
 				state KeyRange range = keyServers[i].first;
 				range = range & toCheck;
@@ -320,28 +324,39 @@ ACTOR Future<bool> checkallCommandActor(Database cx, std::vector<StringRef> toke
 				for (const auto& server : servers) {
 					printf("  %s\n", server.address().toString().c_str());
 				}
-				wait(store(version, getVersion(cx)));
 				state std::vector<Future<ErrorOr<GetKeyValuesReply>>> replies;
-				for (const auto& s : keyServers[i].second) { // for each storage server
-					GetKeyValuesRequest req;
-					req.begin = begin;
-					req.end = end;
-					req.limit = CLIENT_KNOBS->KRM_GET_RANGE_LIMIT;
-					req.limitBytes = CLIENT_KNOBS->KRM_GET_RANGE_LIMIT_BYTES;
-					req.version = version;
-					req.tags = TagSet();
-
-					replies.push_back(s.getKeyValues.getReplyUnlessFailedFor(req, 2, 0));
-				}
-				// printf("waiting for %lu replies at version: %ld\n", keyServers[i].second.size(), version);
-				wait(waitForAll(replies));
-				if (!checkResults(version, keyServers[i].second, replies, begin, end) && !checkAll) {
-					return false;
+				state bool hasMore = true;
+				state int round = 0;
+				while (hasMore) {
+					wait(store(version, getVersion(cx)));
+					replies.clear();
+					printf("round %d, begin Key: %s\n", round, printable(begin.toString()).c_str());
+					for (const auto& s : keyServers[i].second) { // for each storage server
+						GetKeyValuesRequest req;
+						req.begin = begin;
+						req.end = end;
+						req.limit = CLIENT_KNOBS->KRM_GET_RANGE_LIMIT;
+						req.limitBytes = CLIENT_KNOBS->KRM_GET_RANGE_LIMIT_BYTES;
+						req.version = version;
+						req.tags = TagSet();
+						replies.push_back(s.getKeyValues.getReplyUnlessFailedFor(req, 2, 0));
+					}
+					// printf("waiting for %lu replies at version: %ld\n", keyServers[i].second.size(), version);
+					wait(waitForAll(replies));
+					// if there are more results, continue checking in the same shard
+					auto p = checkResults(version, keyServers[i].second, replies, begin, end, lastEnd);
+					bool allSame = p.first;
+					hasMore = p.second;
+					printf("AllSame %d, hasMore %d, checkAll %d\n", allSame, hasMore, checkAll);
+					if (!allSame && !checkAll) {
+						return false;
+					}
+					round++;
 				}
 				if (begin == end) {
+					// this shard is done, signaled by begin == end
 					toCheck = KeyRangeRef(range.end, toCheck.end);
 				}
-				// TODO: if there are more results, continue checking in the same shard
 			}
 		} catch (Error& e) {
 			printf("Retrying for error: %s\n", e.what());

From a430a68b6d123702497939e665abdaa8493b030e Mon Sep 17 00:00:00 2001
From: Zhe Wang <zhe.wang@wustl.edu>
Date: Tue, 19 Dec 2023 11:07:04 -0800
Subject: [PATCH 351/468] report error in
 dispatchMonitorUrgentConsistencyCheckWorkload

---
 fdbserver/tester.actor.cpp | 22 ++++++++++++++++++++++
 1 file changed, 22 insertions(+)

diff --git a/fdbserver/tester.actor.cpp b/fdbserver/tester.actor.cpp
index 6d2f2cae894..19ac5cb633e 100644
--- a/fdbserver/tester.actor.cpp
+++ b/fdbserver/tester.actor.cpp
@@ -1021,6 +1021,17 @@ ACTOR Future<Void> dispatchMonitorUrgentConsistencyCheckWorkload(Database cx,
 		for (int i = 0; i < workloads.size(); i++)
 			setups.push_back(workloads[i].setup.template getReplyUnlessFailedFor<Void>(waitForFailureTime, 0));
 		wait(waitForAll(setups));
+		for (int i = 0; i < setups.size(); i++) {
+			if (setups[i].isError()) {
+				TraceEvent("ConsistencyCheckUrgent_SetupError")
+				    .errorUnsuppressed(setups[i].getError())
+				    .detail("WorkloadTitle", spec.title)
+				    .detail("ClientId", i)
+				    .detail("ClientCount", testers.size())
+				    .detail("TestTimeout", spec.timeout)
+				    .detail("ConsistencyCheckerId", consistencyCheckerId);
+			}
+		}
 	}
 
 	TraceEvent("ConsistencyCheckUrgent_SetupComplete")
@@ -1037,6 +1048,17 @@ ACTOR Future<Void> dispatchMonitorUrgentConsistencyCheckWorkload(Database cx,
 		for (int i = 0; i < workloads.size(); i++)
 			starts.push_back(workloads[i].start.template getReplyUnlessFailedFor<Void>(waitForFailureTime, 0));
 		wait(waitForAll(starts));
+		for (int i = 0; i < starts.size(); i++) {
+			if (starts[i].isError()) {
+				TraceEvent("ConsistencyCheckUrgent_StartError")
+				    .errorUnsuppressed(starts[i].getError())
+				    .detail("WorkloadTitle", spec.title)
+				    .detail("ClientId", i)
+				    .detail("ClientCount", testers.size())
+				    .detail("TestTimeout", spec.timeout)
+				    .detail("ConsistencyCheckerId", consistencyCheckerId);
+			}
+		}
 	}
 
 	TraceEvent("ConsistencyCheckUrgent_WorkloadComplete")

From 7d593b92bbd04a316561b4dfd843dbbb377c9fb9 Mon Sep 17 00:00:00 2001
From: Zhe Wang <zhe.wang@wustl.edu>
Date: Tue, 19 Dec 2023 11:47:55 -0800
Subject: [PATCH 352/468] nit

---
 fdbserver/fdbserver.actor.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/fdbserver/fdbserver.actor.cpp b/fdbserver/fdbserver.actor.cpp
index 679e03b0468..561f74ea441 100644
--- a/fdbserver/fdbserver.actor.cpp
+++ b/fdbserver/fdbserver.actor.cpp
@@ -1230,7 +1230,7 @@ struct CLIOptions {
 					role = ServerRole::KVFileDump;
 				else if (!strcmp(sRole, "consistencycheck"))
 					role = ServerRole::ConsistencyCheck;
-				else if (!strcmp(sRole, "consistencycheckUrgent"))
+				else if (!strcmp(sRole, "consistencycheckurgent"))
 					role = ServerRole::ConsistencyCheckUrgent;
 				else if (!strcmp(sRole, "unittests"))
 					role = ServerRole::UnitTests;

From 8761ee460e187903611d38f35e58022c578bb184 Mon Sep 17 00:00:00 2001
From: hao fu <hfu5@apple.com>
Date: Tue, 19 Dec 2023 12:31:29 -0800
Subject: [PATCH 353/468] Change comment

---
 fdbcli/DebugCommands.actor.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/fdbcli/DebugCommands.actor.cpp b/fdbcli/DebugCommands.actor.cpp
index d8565c23cc2..30cc3b179ea 100644
--- a/fdbcli/DebugCommands.actor.cpp
+++ b/fdbcli/DebugCommands.actor.cpp
@@ -262,7 +262,7 @@ std::pair<bool, bool> checkResults(Version version,
 		lastEnd = result[result.size() - 1].key; // store to a standalone, otherwise memory will be invalid
 		begin = firstGreaterThan(lastEnd);
 	} else {
-		printf("Same at version %ld\n", version);
+		printf("Consistency check finishes for version %ld\n", version);
 		begin = end; // signal that we're done
 	}
 

From 2a328dce54b27b6a0d55174c3dde6ec8488c7f27 Mon Sep 17 00:00:00 2001
From: Zhe Wang <zhe.wang@wustl.edu>
Date: Tue, 19 Dec 2023 18:18:09 -0800
Subject: [PATCH 354/468] nit

---
 fdbserver/workloads/ConsistencyCheck.actor.cpp | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/fdbserver/workloads/ConsistencyCheck.actor.cpp b/fdbserver/workloads/ConsistencyCheck.actor.cpp
index 93eff54ba35..0a0ea0db742 100644
--- a/fdbserver/workloads/ConsistencyCheck.actor.cpp
+++ b/fdbserver/workloads/ConsistencyCheck.actor.cpp
@@ -1786,8 +1786,7 @@ struct ConsistencyCheckWorkload : TestWorkload {
 			    .detail("ShardCount", shardLocationPairList.size())
 			    .detail("ClientId", self->clientId)
 			    .detail("ClientCount", self->clientCount)
-			    .detail("Repetitions", self->repetitions)
-			    .trackLatest("ConsistencyCheck_EndTask");
+			    .detail("Repetitions", self->repetitions);
 		}
 		return Void();
 	}

From 6c7c834a3bfde8e440a333f19c0b85580f982b04 Mon Sep 17 00:00:00 2001
From: Zhe Wang <zhe.wang@wustl.edu>
Date: Tue, 19 Dec 2023 19:10:42 -0800
Subject: [PATCH 355/468] code clean

---
 fdbserver/tester.actor.cpp                     | 17 ++++++++++-------
 fdbserver/workloads/ConsistencyCheck.actor.cpp |  7 ++++---
 2 files changed, 14 insertions(+), 10 deletions(-)

diff --git a/fdbserver/tester.actor.cpp b/fdbserver/tester.actor.cpp
index 19ac5cb633e..98cb93439f6 100644
--- a/fdbserver/tester.actor.cpp
+++ b/fdbserver/tester.actor.cpp
@@ -1154,7 +1154,7 @@ ACTOR Future<std::vector<TesterInterface>> getTesters(Reference<AsyncVar<Optiona
 	return ts;
 }
 
-ACTOR Future<std::vector<KeyRange>> runConsistencyCheckerUrgentInit(Database cx, int64_t consistencyCheckerId) {
+ACTOR Future<Void> runConsistencyCheckerUrgentInit(Database cx, int64_t consistencyCheckerId) {
 	state std::vector<KeyRange> rangesToCheck;
 	state int retryTimes = 0;
 	loop {
@@ -1173,10 +1173,11 @@ ACTOR Future<std::vector<KeyRange>> runConsistencyCheckerUrgentInit(Database cx,
 				wait(clearConsistencyCheckMetadata(cx, consistencyCheckerId));
 				TraceEvent("ConsistencyCheckUrgent_MetadataClearedWhenInit")
 				    .detail("ConsistencyCheckerId", consistencyCheckerId);
-				return rangesToCheck;
+				return Void();
 			}
 
 			// Load ranges to check from progress metadata
+			rangesToCheck.clear();
 			wait(store(rangesToCheck, loadRangesToCheckFromProgressMetadata(cx, consistencyCheckerId)));
 			if (g_network->isSimulated() && deterministicRandom()->random01() < 0.05) {
 				throw operation_failed(); // Introduce random failure
@@ -1225,7 +1226,7 @@ ACTOR Future<std::vector<KeyRange>> runConsistencyCheckerUrgentInit(Database cx,
 		}
 	}
 
-	return rangesToCheck;
+	return Void();
 }
 
 ACTOR Future<Void> runConsistencyCheckerUrgentCore(Reference<AsyncVar<Optional<ClusterControllerFullInterface>>> cc,
@@ -1239,7 +1240,7 @@ ACTOR Future<Void> runConsistencyCheckerUrgentCore(Reference<AsyncVar<Optional<C
 	state std::vector<KeyRange> rangesToCheck;
 	// Init: enforce consistencyCheckerId and prepare for metadata
 	try {
-		wait(store(rangesToCheck, runConsistencyCheckerUrgentInit(cx, consistencyCheckerId)));
+		wait(runConsistencyCheckerUrgentInit(cx, consistencyCheckerId));
 	} catch (Error& e) {
 		if (e.code() == error_code_key_not_found || e.code() == error_code_consistency_check_task_outdated ||
 		    e.code() == error_code_timed_out) {
@@ -1269,6 +1270,7 @@ ACTOR Future<Void> runConsistencyCheckerUrgentCore(Reference<AsyncVar<Optional<C
 	loop {
 		try {
 			// Load ranges to check
+			rangesToCheck.clear();
 			wait(store(rangesToCheck, loadRangesToCheckFromProgressMetadata(cx, consistencyCheckerId)));
 			if (g_network->isSimulated() && deterministicRandom()->random01() < 0.05) {
 				throw operation_failed(); // Introduce random failure
@@ -1335,8 +1337,7 @@ ACTOR Future<Void> runConsistencyCheckerUrgentCore(Reference<AsyncVar<Optional<C
 				    .detail("RetryTimes", retryTimes)
 				    .detail("Round", round)
 				    .detail("ClientId", assignIt->first)
-				    .detail("ShardsCount", assignIt->second.size())
-				    .detail("Shards", describe(assignIt->second));
+				    .detail("ShardsCount", assignIt->second.size());
 				wait(persistConsistencyCheckAssignment(
 				    cx, assignIt->first, assignIt->second, consistencyCheckerId)); // Persist assignment
 				if (g_network->isSimulated() && deterministicRandom()->random01() < 0.05) {
@@ -1344,6 +1345,8 @@ ACTOR Future<Void> runConsistencyCheckerUrgentCore(Reference<AsyncVar<Optional<C
 				}
 			}
 			TraceEvent e("ConsistencyCheckUrgent_PersistAssignment");
+			e.setMaxEventLength(-1);
+			e.setMaxFieldLength(-1);
 			e.detail("ConsistencyCheckerId", consistencyCheckerId);
 			e.detail("Round", round);
 			e.detail("RetryTimes", retryTimes);
@@ -1419,7 +1422,7 @@ ACTOR Future<Void> runConsistencyCheckerUrgentCore(Reference<AsyncVar<Optional<C
 			}
 		}
 
-		wait(delay(10.0));
+		wait(delay(10.0)); // Backoff 10 seconds for the next round
 
 		// Decide and enforce the consistencyCheckerId for the next round
 		state int retryTimesForUpdatingCheckerId = 0;
diff --git a/fdbserver/workloads/ConsistencyCheck.actor.cpp b/fdbserver/workloads/ConsistencyCheck.actor.cpp
index 0a0ea0db742..96b8f24cce4 100644
--- a/fdbserver/workloads/ConsistencyCheck.actor.cpp
+++ b/fdbserver/workloads/ConsistencyCheck.actor.cpp
@@ -1724,7 +1724,7 @@ struct ConsistencyCheckWorkload : TestWorkload {
 				    .detail("NumShardToCheckThisEpoch", numShardToCheck - 1)
 				    .detail("ShardBegin", range.begin)
 				    .detail("ShardEnd", range.end)
-				    .detail("Repetitions", self->repetitions)
+				    .detail("ReplicaCount", storageServers.size())
 				    .detail("ConsistencyCheckEpoch", consistencyCheckEpoch);
 			} else {
 				numCompleteShards++;
@@ -1740,8 +1740,9 @@ struct ConsistencyCheckWorkload : TestWorkload {
 				    .detail("ConsistencyCheckerId", self->consistencyCheckerId)
 				    .detail("ClientId", self->clientId)
 				    .detail("ClientCount", self->clientCount)
-				    .detail("Range", range)
-				    .detail("ShardCount", shardLocationPairList.size())
+				    .detail("ShardBegin", range.begin)
+				    .detail("ShardEnd", range.end)
+				    .detail("ReplicaCount", storageServers.size())
 				    .detail("NumCompletedShards", numCompleteShards)
 				    .detail("NumFailedShards", numFailedShards)
 				    .detail("NumShardThisClient", numShardThisClient)

From c2ddfca4c4d08dea6b0d9cf33e004cb9918e060a Mon Sep 17 00:00:00 2001
From: Zhe Wang <zhe.wang@wustl.edu>
Date: Mon, 1 Jan 2024 21:41:19 -0800
Subject: [PATCH 356/468] fix bugs

---
 fdbclient/ServerKnobs.cpp                     |  3 ++
 fdbclient/ServerKnobs.h                       |  3 ++
 fdbserver/tester.actor.cpp                    | 37 +++++++++++++++----
 .../workloads/ConsistencyCheck.actor.cpp      | 15 ++++++--
 4 files changed, 47 insertions(+), 11 deletions(-)

diff --git a/fdbclient/ServerKnobs.cpp b/fdbclient/ServerKnobs.cpp
index 4404cbbdda4..486ac9af034 100644
--- a/fdbclient/ServerKnobs.cpp
+++ b/fdbclient/ServerKnobs.cpp
@@ -824,6 +824,9 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi
 	init( STORAGE_SERVER_REBOOT_ON_IO_TIMEOUT,                 false ); if ( randomize && BUGGIFY ) STORAGE_SERVER_REBOOT_ON_IO_TIMEOUT = true;
 	init( CONSISTENCY_CHECK_ROCKSDB_ENGINE,                    false );
 	init( CONSISTENCY_CHECK_SQLITE_ENGINE,                     false );
+	init( TESTER_SHARED_RANDOM_MAX_PLUS_ONE,                10000000 );
+	init( CONSISTENCY_CHECK_ID_MIN, TESTER_SHARED_RANDOM_MAX_PLUS_ONE );
+	init( CONSISTENCY_CHECK_ID_MAX_PLUS_ONE, 10 * TESTER_SHARED_RANDOM_MAX_PLUS_ONE);
 
 	// Test harness
 	init( WORKER_POLL_DELAY,                                     1.0 );
diff --git a/fdbclient/ServerKnobs.h b/fdbclient/ServerKnobs.h
index fd1a453e4c5..9d372c41162 100644
--- a/fdbclient/ServerKnobs.h
+++ b/fdbclient/ServerKnobs.h
@@ -773,6 +773,9 @@ class ServerKnobs : public KnobsImpl<ServerKnobs> {
 	                                      // When both CONSISTENCY_CHECK_ROCKSDB_ENGINE and
 	                                      // CONSISTENCY_CHECK_SQLITE_ENGINE are set, consistency check only checks for
 	                                      // the rocksdb engine.
+	int64_t TESTER_SHARED_RANDOM_MAX_PLUS_ONE;
+	int64_t CONSISTENCY_CHECK_ID_MIN;
+	int64_t CONSISTENCY_CHECK_ID_MAX_PLUS_ONE;
 
 	// Test harness
 	double WORKER_POLL_DELAY;
diff --git a/fdbserver/tester.actor.cpp b/fdbserver/tester.actor.cpp
index 98cb93439f6..406f36fd6f4 100644
--- a/fdbserver/tester.actor.cpp
+++ b/fdbserver/tester.actor.cpp
@@ -805,7 +805,7 @@ ACTOR Future<DistributedTestResults> runWorkload(Database cx,
 	state int i = 0;
 	state int success = 0;
 	state int failure = 0;
-	int64_t sharedRandom = deterministicRandom()->randomInt64(0, 10000000);
+	int64_t sharedRandom = deterministicRandom()->randomInt64(0, SERVER_KNOBS->TESTER_SHARED_RANDOM_MAX_PLUS_ONE);
 	for (; i < testers.size(); i++) {
 		WorkloadRequest req;
 		req.title = spec.title;
@@ -1014,13 +1014,15 @@ ACTOR Future<Void> dispatchMonitorUrgentConsistencyCheckWorkload(Database cx,
 	    .detail("ConsistencyCheckerId", consistencyCheckerId);
 
 	state std::vector<WorkloadInterface> workloads = wait(getAll(workRequests));
+	// workloads follows the same order as workRequests
 	state double waitForFailureTime = g_network->isSimulated() ? 24 * 60 * 60 : 60;
 	if (spec.phases & TestWorkload::SETUP) {
 		state std::vector<Future<ErrorOr<Void>>> setups;
-		setups.reserve(workloads.size());
 		for (int i = 0; i < workloads.size(); i++)
 			setups.push_back(workloads[i].setup.template getReplyUnlessFailedFor<Void>(waitForFailureTime, 0));
 		wait(waitForAll(setups));
+		// setups follows the order of workRequests
+		ASSERT(workRequests.size() == setups.size());
 		for (int i = 0; i < setups.size(); i++) {
 			if (setups[i].isError()) {
 				TraceEvent("ConsistencyCheckUrgent_SetupError")
@@ -1030,6 +1032,14 @@ ACTOR Future<Void> dispatchMonitorUrgentConsistencyCheckWorkload(Database cx,
 				    .detail("ClientCount", testers.size())
 				    .detail("TestTimeout", spec.timeout)
 				    .detail("ConsistencyCheckerId", consistencyCheckerId);
+			} else if (setups[i].get().isError()) {
+				TraceEvent("ConsistencyCheckUrgent_SetupError")
+				    .errorUnsuppressed(setups[i].get().getError())
+				    .detail("WorkloadTitle", spec.title)
+				    .detail("ClientId", i)
+				    .detail("ClientCount", testers.size())
+				    .detail("TestTimeout", spec.timeout)
+				    .detail("ConsistencyCheckerId", consistencyCheckerId);
 			}
 		}
 	}
@@ -1044,10 +1054,11 @@ ACTOR Future<Void> dispatchMonitorUrgentConsistencyCheckWorkload(Database cx,
 		TraceEvent("TestStarting").detail("WorkloadTitle", spec.title);
 		printf("running test (%s)...\n", printable(spec.title).c_str());
 		state std::vector<Future<ErrorOr<Void>>> starts;
-		starts.reserve(workloads.size());
 		for (int i = 0; i < workloads.size(); i++)
 			starts.push_back(workloads[i].start.template getReplyUnlessFailedFor<Void>(waitForFailureTime, 0));
 		wait(waitForAll(starts));
+		// starts follows the order of workRequests
+		ASSERT(workRequests.size() == starts.size());
 		for (int i = 0; i < starts.size(); i++) {
 			if (starts[i].isError()) {
 				TraceEvent("ConsistencyCheckUrgent_StartError")
@@ -1057,6 +1068,14 @@ ACTOR Future<Void> dispatchMonitorUrgentConsistencyCheckWorkload(Database cx,
 				    .detail("ClientCount", testers.size())
 				    .detail("TestTimeout", spec.timeout)
 				    .detail("ConsistencyCheckerId", consistencyCheckerId);
+			} else if (starts[i].get().isError()) {
+				TraceEvent("ConsistencyCheckUrgent_StartError")
+				    .errorUnsuppressed(starts[i].get().getError())
+				    .detail("WorkloadTitle", spec.title)
+				    .detail("ClientId", i)
+				    .detail("ClientCount", testers.size())
+				    .detail("TestTimeout", spec.timeout)
+				    .detail("ConsistencyCheckerId", consistencyCheckerId);
 			}
 		}
 	}
@@ -1236,7 +1255,8 @@ ACTOR Future<Void> runConsistencyCheckerUrgentCore(Reference<AsyncVar<Optional<C
                                                    TestSpec testSpec,
                                                    Optional<TenantName> defaultTenant,
                                                    Reference<AsyncVar<ServerDBInfo>> dbInfo) {
-	state int64_t consistencyCheckerId = deterministicRandom()->randomInt64(1, 10000000);
+	state int64_t consistencyCheckerId = deterministicRandom()->randomInt64(
+	    SERVER_KNOBS->CONSISTENCY_CHECK_ID_MIN, SERVER_KNOBS->CONSISTENCY_CHECK_ID_MAX_PLUS_ONE);
 	state std::vector<KeyRange> rangesToCheck;
 	// Init: enforce consistencyCheckerId and prepare for metadata
 	try {
@@ -1396,7 +1416,7 @@ ACTOR Future<Void> runConsistencyCheckerUrgentCore(Reference<AsyncVar<Optional<C
 				    .detail("RangesToCheckCount", rangesToCheck.size());
 				if (g_network->isSimulated()) {
 					TraceEvent("ConsistencyCheckUrgent_RepairDC");
-					wait(repairDeadDatacenter(cx, dbInfo, "ConsistencyCheck"));
+					wait(repairDeadDatacenter(cx, dbInfo, "ConsistencyCheckUrgent"));
 				}
 				round++;
 			}
@@ -1429,7 +1449,8 @@ ACTOR Future<Void> runConsistencyCheckerUrgentCore(Reference<AsyncVar<Optional<C
 		loop {
 			try {
 				// Decide and persist consistencyCheckerId for the next round
-				consistencyCheckerId = deterministicRandom()->randomInt64(1, 10000000);
+				consistencyCheckerId = deterministicRandom()->randomInt64(
+				    SERVER_KNOBS->CONSISTENCY_CHECK_ID_MIN, SERVER_KNOBS->CONSISTENCY_CHECK_ID_MAX_PLUS_ONE);
 				wait(persistConsistencyCheckerId(cx, consistencyCheckerId));
 				if (g_network->isSimulated() && deterministicRandom()->random01() < 0.05) {
 					throw operation_failed(); // Introduce random failure
@@ -1494,7 +1515,7 @@ ACTOR Future<Void> checkConsistencyUrgentSim(Database cx,
 				continue;
 			}
 			TraceEvent("ConsistencyCheckUrgent_RepairDC");
-			wait(repairDeadDatacenter(cx, dbInfo, "ConsistencyCheck"));
+			wait(repairDeadDatacenter(cx, dbInfo, "ConsistencyCheckUrgent"));
 		}
 	}
 	return Void();
@@ -1555,8 +1576,10 @@ ACTOR Future<bool> runTest(Database cx,
 		// Run the consistency check workload
 		if (spec.runConsistencyCheck) {
 			try {
+				// Urgent consistency check
 				wait(timeoutError(checkConsistencyUrgentSim(cx, testers, 18000, spec.databasePingDelay, dbInfo),
 				                  20000.0));
+				// Normal consistency check
 				bool quiescent = g_network->isSimulated() ? !BUGGIFY : spec.waitForQuiescenceEnd;
 				wait(timeoutError(checkConsistency(cx,
 				                                   testers,
diff --git a/fdbserver/workloads/ConsistencyCheck.actor.cpp b/fdbserver/workloads/ConsistencyCheck.actor.cpp
index 96b8f24cce4..36a782d6fa2 100644
--- a/fdbserver/workloads/ConsistencyCheck.actor.cpp
+++ b/fdbserver/workloads/ConsistencyCheck.actor.cpp
@@ -110,9 +110,13 @@ struct ConsistencyCheckWorkload : TestWorkload {
 		rateLimitMax = getOption(options, LiteralStringRef("rateLimitMax"), 0);
 		shuffleShards = getOption(options, LiteralStringRef("shuffleShards"), false);
 		indefinite = getOption(options, LiteralStringRef("indefinite"), false);
-		consistencyCheckerId = sharedRandomNumber;
+		if (sharedRandomNumber >= SERVER_KNOBS->CONSISTENCY_CHECK_ID_MIN &&
+		    sharedRandomNumber < SERVER_KNOBS->CONSISTENCY_CHECK_ID_MAX_PLUS_ONE) {
+			consistencyCheckerId = sharedRandomNumber; // for urgent mode
+		} else {
+			consistencyCheckerId = 0;
+		}
 		suspendConsistencyCheck.set(true);
-		TraceEvent("ConsistencyCheck_WorkloadReceived").detail("ClientCount", clientCount).detail("ClientId", clientId);
 
 		success = true;
 
@@ -148,8 +152,8 @@ struct ConsistencyCheckWorkload : TestWorkload {
 	}
 
 	Future<Void> start(Database const& cx) override {
-		TraceEvent("ConsistencyCheck").log();
-		if (consistencyCheckerId != 0) { // consistencyCheckerId is set when urgent mode
+		TraceEvent("ConsistencyCheck").detail("ConsistencyCheckerId", consistencyCheckerId);
+		if (consistencyCheckerId != 0) { // urgent mode
 			return _startUrgent(cx, this);
 		} else {
 			return _start(cx, this);
@@ -1445,6 +1449,9 @@ struct ConsistencyCheckWorkload : TestWorkload {
 			loop {
 				try {
 					Transaction tr(cx);
+					tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
+					tr.setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
+					tr.setOption(FDBTransactionOptions::LOCK_AWARE);
 					RangeResult UIDtoTagMap = wait(tr.getRange(serverTagKeys, CLIENT_KNOBS->TOO_MANY));
 					ASSERT(!UIDtoTagMap.more && UIDtoTagMap.size() < CLIENT_KNOBS->TOO_MANY);
 					decodeKeyServersValue(UIDtoTagMap,

From 9a220875ef8b4f741280fdf97b5ed805c8ef1061 Mon Sep 17 00:00:00 2001
From: Zhe Wang <zhe.wang@wustl.edu>
Date: Mon, 1 Jan 2024 22:15:24 -0800
Subject: [PATCH 357/468] add trace of read range failed

---
 fdbserver/workloads/ConsistencyCheck.actor.cpp | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/fdbserver/workloads/ConsistencyCheck.actor.cpp b/fdbserver/workloads/ConsistencyCheck.actor.cpp
index 36a782d6fa2..f5450d5b1bd 100644
--- a/fdbserver/workloads/ConsistencyCheck.actor.cpp
+++ b/fdbserver/workloads/ConsistencyCheck.actor.cpp
@@ -1580,6 +1580,14 @@ struct ConsistencyCheckWorkload : TestWorkload {
 						ErrorOr<GetKeyValuesReply> rangeResult = keyValueFutures[j].get();
 						if (!rangeResult.present() || rangeResult.get().error.present()) {
 							valueAvailableToCheck = false;
+							TraceEvent e(SevInfo, "ConsistencyCheckUrgent_GetRangeError");
+							e.detail("ResultPresent", rangeResult.present());
+							if (rangeResult.present()) {
+								e.detail("ErrorPresent", rangeResult.get().error.present());
+								if (rangeResult.get().error.present()) {
+									e.detail("Error", rangeResult.get().error.get().name());
+								}
+							}
 							break;
 						}
 					}

From 9b3744a29b903be8e9c9b679bc773e2ad5893571 Mon Sep 17 00:00:00 2001
From: Zhe Wang <zhe.wang@wustl.edu>
Date: Tue, 2 Jan 2024 17:00:00 -0800
Subject: [PATCH 358/468] guarantee complete check without persisting metadata

---
 fdbclient/ClientKnobs.cpp                     |   2 +-
 fdbclient/ConsistencyCheckUtil.actor.cpp      |   9 +
 fdbclient/ServerKnobs.cpp                     |   1 +
 fdbclient/ServerKnobs.h                       |   1 +
 fdbserver/TesterInterface.actor.h             |   5 +-
 fdbserver/tester.actor.cpp                    | 343 +++++++++++-------
 .../workloads/ConsistencyCheck.actor.cpp      |  64 ++--
 fdbserver/workloads/workloads.actor.h         |   1 +
 flow/error_definitions.h                      |   1 +
 9 files changed, 270 insertions(+), 157 deletions(-)

diff --git a/fdbclient/ClientKnobs.cpp b/fdbclient/ClientKnobs.cpp
index 1091542ade4..783f1a4d0e0 100644
--- a/fdbclient/ClientKnobs.cpp
+++ b/fdbclient/ClientKnobs.cpp
@@ -258,7 +258,7 @@ void ClientKnobs::initialize(Randomize randomize) {
 
 	init( CONSISTENCY_CHECK_RATE_LIMIT_MAX,        50e6 ); // Limit in per sec
 	init( CONSISTENCY_CHECK_ONE_ROUND_TARGET_COMPLETION_TIME,	7 * 24 * 60 * 60 ); // 7 days
-	init( CONSISTENCY_CHECK_BATCH_SHARD_COUNT,      100 ); if( randomize && BUGGIFY ) CONSISTENCY_CHECK_BATCH_SHARD_COUNT = 2;
+	init( CONSISTENCY_CHECK_BATCH_SHARD_COUNT,       20 ); if( randomize && BUGGIFY ) CONSISTENCY_CHECK_BATCH_SHARD_COUNT = 2;
 	init( CONSISTENCY_CHECK_RETRY_DEPTH_MAX,         10 ); if( randomize && BUGGIFY ) CONSISTENCY_CHECK_RETRY_DEPTH_MAX = 1;
 	init( CONSISTENCY_CHECK_INIT_CLEAR_METADATA,  false );
 	init( CONSISTENCY_CHECK_INIT_CLEAR_METADATA_EXIT, false );
diff --git a/fdbclient/ConsistencyCheckUtil.actor.cpp b/fdbclient/ConsistencyCheckUtil.actor.cpp
index 030159928c1..1b9f9d8a91d 100644
--- a/fdbclient/ConsistencyCheckUtil.actor.cpp
+++ b/fdbclient/ConsistencyCheckUtil.actor.cpp
@@ -20,11 +20,13 @@
 
 #include "fdbclient/ConsistencyCheckUtil.actor.h"
 #include "fdbclient/ConsistencyCheck.h"
+#include "fdbserver/Knobs.h"
 #include "flow/actorcompiler.h" // This must be the last #include.
 
 ACTOR Future<std::vector<KeyRange>> loadRangesToCheckFromAssignmentMetadata(Database cx,
                                                                             int clientId,
                                                                             int64_t consistencyCheckerId) {
+	ASSERT(SERVER_KNOBS->CONSISTENCY_CHECK_USE_PERSIST_DATA);
 	state std::vector<KeyRange> res;
 	state Transaction tr(cx);
 	state Key rangeToReadBegin = allKeys.begin;
@@ -67,6 +69,7 @@ ACTOR Future<std::vector<KeyRange>> loadRangesToCheckFromAssignmentMetadata(Data
 }
 
 ACTOR Future<Void> persistConsistencyCheckProgress(Database cx, KeyRange range, int64_t consistencyCheckerId) {
+	ASSERT(SERVER_KNOBS->CONSISTENCY_CHECK_USE_PERSIST_DATA);
 	state Transaction tr(cx);
 	loop {
 		try {
@@ -96,6 +99,7 @@ ACTOR Future<Void> persistConsistencyCheckProgress(Database cx, KeyRange range,
 }
 
 ACTOR Future<Void> initConsistencyCheckAssignmentMetadata(Database cx, int64_t consistencyCheckerId) {
+	ASSERT(SERVER_KNOBS->CONSISTENCY_CHECK_USE_PERSIST_DATA);
 	state Transaction tr(cx);
 	loop {
 		try {
@@ -121,6 +125,7 @@ ACTOR Future<Void> initConsistencyCheckAssignmentMetadata(Database cx, int64_t c
 ACTOR Future<Void> initConsistencyCheckProgressMetadata(Database cx,
                                                         std::vector<KeyRange> rangesToCheck,
                                                         int64_t consistencyCheckerId) {
+	ASSERT(SERVER_KNOBS->CONSISTENCY_CHECK_USE_PERSIST_DATA);
 	state Transaction tr(cx);
 	loop {
 		try {
@@ -156,6 +161,7 @@ ACTOR Future<Void> initConsistencyCheckProgressMetadata(Database cx,
 }
 
 ACTOR Future<Void> persistConsistencyCheckerId(Database cx, int64_t consistencyCheckerId) {
+	ASSERT(SERVER_KNOBS->CONSISTENCY_CHECK_USE_PERSIST_DATA);
 	state Transaction tr(cx);
 	loop {
 		try {
@@ -173,6 +179,7 @@ ACTOR Future<Void> persistConsistencyCheckerId(Database cx, int64_t consistencyC
 }
 
 ACTOR Future<Void> clearConsistencyCheckMetadata(Database cx, int64_t consistencyCheckerId) {
+	ASSERT(SERVER_KNOBS->CONSISTENCY_CHECK_USE_PERSIST_DATA);
 	state Transaction tr(cx);
 	loop {
 		try {
@@ -198,6 +205,7 @@ ACTOR Future<Void> clearConsistencyCheckMetadata(Database cx, int64_t consistenc
 }
 
 ACTOR Future<std::vector<KeyRange>> loadRangesToCheckFromProgressMetadata(Database cx, int64_t consistencyCheckerId) {
+	ASSERT(SERVER_KNOBS->CONSISTENCY_CHECK_USE_PERSIST_DATA);
 	state std::vector<KeyRange> res;
 	state int64_t shardCompleted = 0;
 	state Transaction tr(cx);
@@ -251,6 +259,7 @@ ACTOR Future<Void> persistConsistencyCheckAssignment(Database cx,
                                                      int clientId,
                                                      std::vector<KeyRange> assignedRanges,
                                                      int64_t consistencyCheckerId) {
+	ASSERT(SERVER_KNOBS->CONSISTENCY_CHECK_USE_PERSIST_DATA);
 	state Transaction tr(cx);
 	loop {
 		try {
diff --git a/fdbclient/ServerKnobs.cpp b/fdbclient/ServerKnobs.cpp
index 486ac9af034..f5a0951bb39 100644
--- a/fdbclient/ServerKnobs.cpp
+++ b/fdbclient/ServerKnobs.cpp
@@ -827,6 +827,7 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi
 	init( TESTER_SHARED_RANDOM_MAX_PLUS_ONE,                10000000 );
 	init( CONSISTENCY_CHECK_ID_MIN, TESTER_SHARED_RANDOM_MAX_PLUS_ONE );
 	init( CONSISTENCY_CHECK_ID_MAX_PLUS_ONE, 10 * TESTER_SHARED_RANDOM_MAX_PLUS_ONE);
+	init( CONSISTENCY_CHECK_USE_PERSIST_DATA,                  false ); if (isSimulated) CONSISTENCY_CHECK_USE_PERSIST_DATA = deterministicRandom()->coinflip();
 
 	// Test harness
 	init( WORKER_POLL_DELAY,                                     1.0 );
diff --git a/fdbclient/ServerKnobs.h b/fdbclient/ServerKnobs.h
index 9d372c41162..d2bce12a70b 100644
--- a/fdbclient/ServerKnobs.h
+++ b/fdbclient/ServerKnobs.h
@@ -776,6 +776,7 @@ class ServerKnobs : public KnobsImpl<ServerKnobs> {
 	int64_t TESTER_SHARED_RANDOM_MAX_PLUS_ONE;
 	int64_t CONSISTENCY_CHECK_ID_MIN;
 	int64_t CONSISTENCY_CHECK_ID_MAX_PLUS_ONE;
+	bool CONSISTENCY_CHECK_USE_PERSIST_DATA;
 
 	// Test harness
 	double WORKER_POLL_DELAY;
diff --git a/fdbserver/TesterInterface.actor.h b/fdbserver/TesterInterface.actor.h
index 4c16ffc30a6..01dee5181d4 100644
--- a/fdbserver/TesterInterface.actor.h
+++ b/fdbserver/TesterInterface.actor.h
@@ -81,6 +81,8 @@ struct WorkloadRequest {
 
 	VectorRef<VectorRef<KeyValueRef>> options;
 
+	Optional<std::vector<KeyRange>> rangesToCheck; // For consistency checker urgent when persisting data is not allowed
+
 	int clientId; // the "id" of the client receiving the request (0 indexed)
 	int clientCount; // the total number of test clients participating in the workload
 	ReplyPromise<struct WorkloadInterface> reply;
@@ -98,7 +100,8 @@ struct WorkloadRequest {
 		           clientCount,
 		           reply,
 		           defaultTenant,
-		           arena);
+		           arena,
+		           rangesToCheck);
 	}
 };
 
diff --git a/fdbserver/tester.actor.cpp b/fdbserver/tester.actor.cpp
index 406f36fd6f4..5d7dd9ed3be 100644
--- a/fdbserver/tester.actor.cpp
+++ b/fdbserver/tester.actor.cpp
@@ -49,7 +49,7 @@ WorkloadContext::WorkloadContext() {}
 
 WorkloadContext::WorkloadContext(const WorkloadContext& r)
   : options(r.options), clientId(r.clientId), clientCount(r.clientCount), sharedRandomNumber(r.sharedRandomNumber),
-    dbInfo(r.dbInfo) {}
+    dbInfo(r.dbInfo), rangesToCheck(r.rangesToCheck) {}
 
 WorkloadContext::~WorkloadContext() {}
 
@@ -353,6 +353,7 @@ Reference<TestWorkload> getWorkloadIface(WorkloadRequest work,
 	wcx.dbInfo = dbInfo;
 	wcx.options = options;
 	wcx.sharedRandomNumber = work.sharedRandomNumber;
+	wcx.rangesToCheck = work.rangesToCheck;
 
 	auto workload = IWorkloadFactory::create(testName.toString(), wcx);
 
@@ -392,6 +393,7 @@ Reference<TestWorkload> getWorkloadIface(WorkloadRequest work, Reference<AsyncVa
 	wcx.clientId = work.clientId;
 	wcx.clientCount = work.clientCount;
 	wcx.sharedRandomNumber = work.sharedRandomNumber;
+	wcx.rangesToCheck = work.rangesToCheck;
 	// FIXME: Other stuff not filled in; why isn't this constructed here and passed down to the other
 	// getWorkloadIface()?
 	auto compound = makeReference<CompoundWorkload>(wcx);
@@ -574,7 +576,9 @@ ACTOR Future<Void> runWorkloadAsync(Database cx,
 					startResult = operation_failed();
 					if (e.code() == error_code_please_reboot || e.code() == error_code_please_reboot_delete)
 						throw;
-					TraceEvent(SevError, "TestFailure", workIface.id())
+					TraceEvent(e.code() == error_code_consistency_check_task_failed ? SevWarn : SevError,
+					           "TestFailure",
+					           workIface.id())
 					    .errorUnsuppressed(e)
 					    .detail("Reason", "Error starting workload")
 					    .detail("Workload", workload->description());
@@ -631,6 +635,11 @@ ACTOR Future<Void> runWorkloadAsync(Database cx,
 			break;
 		}
 	}
+
+	TraceEvent("TestEndAsync", workIface.id())
+	    .detail("Workload", workload->description())
+	    .detail("DatabasePingDelay", databasePingDelay);
+
 	return Void();
 }
 
@@ -817,6 +826,7 @@ ACTOR Future<DistributedTestResults> runWorkload(Database cx,
 		req.clientCount = testers.size();
 		req.sharedRandomNumber = sharedRandom;
 		req.defaultTenant = defaultTenant.castTo<TenantNameRef>();
+		req.rangesToCheck = Optional<std::vector<KeyRange>>();
 		workRequests.push_back(testers[i].recruitments.getReply(req));
 	}
 
@@ -981,17 +991,21 @@ ACTOR Future<Void> checkConsistency(Database cx,
 	}
 }
 
-ACTOR Future<Void> dispatchMonitorUrgentConsistencyCheckWorkload(Database cx,
-                                                                 std::vector<TesterInterface> testers,
-                                                                 TestSpec spec,
-                                                                 Optional<TenantName> defaultTenant,
-                                                                 int64_t consistencyCheckerId) {
+ACTOR Future<std::set<int>> dispatchMonitorUrgentConsistencyCheckWorkload(
+    Database cx,
+    std::vector<TesterInterface> testers,
+    TestSpec spec,
+    Optional<TenantName> defaultTenant,
+    int64_t consistencyCheckerId,
+    std::unordered_map<int, std::vector<KeyRange>> assignment) {
 	TraceEvent("ConsistencyCheckUrgent_Dispatch")
 	    .detail("WorkloadTitle", spec.title)
 	    .detail("TesterCount", testers.size())
 	    .detail("TestTimeout", spec.timeout)
 	    .detail("ConsistencyCheckerId", consistencyCheckerId);
 
+	state std::set<int> completeClientIds;
+
 	state std::vector<Future<WorkloadInterface>> workRequests;
 	for (int i = 0; i < testers.size(); i++) {
 		WorkloadRequest req;
@@ -1004,6 +1018,11 @@ ACTOR Future<Void> dispatchMonitorUrgentConsistencyCheckWorkload(Database cx,
 		req.clientCount = testers.size();
 		req.sharedRandomNumber = consistencyCheckerId;
 		req.defaultTenant = defaultTenant.castTo<TenantNameRef>();
+		if (!SERVER_KNOBS->CONSISTENCY_CHECK_USE_PERSIST_DATA) {
+			req.rangesToCheck = assignment[i];
+		} else {
+			req.rangesToCheck = Optional<std::vector<KeyRange>>();
+		}
 		workRequests.push_back(testers[i].recruitments.getReply(req));
 	}
 
@@ -1025,7 +1044,7 @@ ACTOR Future<Void> dispatchMonitorUrgentConsistencyCheckWorkload(Database cx,
 		ASSERT(workRequests.size() == setups.size());
 		for (int i = 0; i < setups.size(); i++) {
 			if (setups[i].isError()) {
-				TraceEvent("ConsistencyCheckUrgent_SetupError")
+				TraceEvent("ConsistencyCheckUrgent_SetupError1")
 				    .errorUnsuppressed(setups[i].getError())
 				    .detail("WorkloadTitle", spec.title)
 				    .detail("ClientId", i)
@@ -1033,7 +1052,7 @@ ACTOR Future<Void> dispatchMonitorUrgentConsistencyCheckWorkload(Database cx,
 				    .detail("TestTimeout", spec.timeout)
 				    .detail("ConsistencyCheckerId", consistencyCheckerId);
 			} else if (setups[i].get().isError()) {
-				TraceEvent("ConsistencyCheckUrgent_SetupError")
+				TraceEvent("ConsistencyCheckUrgent_SetupError2")
 				    .errorUnsuppressed(setups[i].get().getError())
 				    .detail("WorkloadTitle", spec.title)
 				    .detail("ClientId", i)
@@ -1042,6 +1061,7 @@ ACTOR Future<Void> dispatchMonitorUrgentConsistencyCheckWorkload(Database cx,
 				    .detail("ConsistencyCheckerId", consistencyCheckerId);
 			}
 		}
+		throwIfError(setups, "ConsistencyCheckUrgent_SetupFailedForWorkload");
 	}
 
 	TraceEvent("ConsistencyCheckUrgent_SetupComplete")
@@ -1061,7 +1081,7 @@ ACTOR Future<Void> dispatchMonitorUrgentConsistencyCheckWorkload(Database cx,
 		ASSERT(workRequests.size() == starts.size());
 		for (int i = 0; i < starts.size(); i++) {
 			if (starts[i].isError()) {
-				TraceEvent("ConsistencyCheckUrgent_StartError")
+				TraceEvent("ConsistencyCheckUrgent_StartError1")
 				    .errorUnsuppressed(starts[i].getError())
 				    .detail("WorkloadTitle", spec.title)
 				    .detail("ClientId", i)
@@ -1069,24 +1089,30 @@ ACTOR Future<Void> dispatchMonitorUrgentConsistencyCheckWorkload(Database cx,
 				    .detail("TestTimeout", spec.timeout)
 				    .detail("ConsistencyCheckerId", consistencyCheckerId);
 			} else if (starts[i].get().isError()) {
-				TraceEvent("ConsistencyCheckUrgent_StartError")
+				TraceEvent("ConsistencyCheckUrgent_StartError2")
 				    .errorUnsuppressed(starts[i].get().getError())
 				    .detail("WorkloadTitle", spec.title)
 				    .detail("ClientId", i)
 				    .detail("ClientCount", testers.size())
 				    .detail("TestTimeout", spec.timeout)
 				    .detail("ConsistencyCheckerId", consistencyCheckerId);
+			} else {
+				completeClientIds.insert(i);
 			}
 		}
 	}
 
-	TraceEvent("ConsistencyCheckUrgent_WorkloadComplete")
+	for (int i = 0; i < workloads.size(); i++) {
+		workloads[i].stop.send(ReplyPromise<Void>());
+	}
+
+	TraceEvent("ConsistencyCheckUrgent_WorkloadEnd")
 	    .detail("WorkloadTitle", spec.title)
 	    .detail("TesterCount", testers.size())
 	    .detail("TestTimeout", spec.timeout)
 	    .detail("ConsistencyCheckerId", consistencyCheckerId);
 
-	return Void();
+	return completeClientIds;
 }
 
 ACTOR Future<std::vector<KeyRange>> getConsistencyCheckShards(Database cx, std::vector<KeyRange> ranges) {
@@ -1232,7 +1258,7 @@ ACTOR Future<Void> runConsistencyCheckerUrgentInit(Database cx, int64_t consiste
 			} else if (e.code() == error_code_key_not_found || e.code() == error_code_consistency_check_task_outdated) {
 				throw e;
 			} else {
-				TraceEvent("ConsistencyCheckUrgent_RetriableFailure")
+				TraceEvent("ConsistencyCheckUrgent_InitWithRetriableFailure")
 				    .errorUnsuppressed(e)
 				    .detail("ConsistencyCheckerId", consistencyCheckerId)
 				    .detail("RetryTimes", retryTimes);
@@ -1248,6 +1274,64 @@ ACTOR Future<Void> runConsistencyCheckerUrgentInit(Database cx, int64_t consiste
 	return Void();
 }
 
+ACTOR Future<std::unordered_map<int, std::vector<KeyRange>>> makeTaskAssignment(Database cx,
+                                                                                int64_t consistencyCheckerId,
+                                                                                std::vector<KeyRange> shardsToCheck,
+                                                                                int testersCount,
+                                                                                int round) {
+	state std::unordered_map<int, std::vector<KeyRange>> assignment;
+	if (SERVER_KNOBS->CONSISTENCY_CHECK_USE_PERSIST_DATA) {
+		wait(initConsistencyCheckAssignmentMetadata(cx, consistencyCheckerId));
+		if (g_network->isSimulated() && deterministicRandom()->random01() < 0.05) {
+			throw operation_failed(); // Introduce random failure
+		}
+	}
+	int batchSize = CLIENT_KNOBS->CONSISTENCY_CHECK_BATCH_SHARD_COUNT;
+	int startingPoint = 0;
+	if (shardsToCheck.size() > batchSize * testersCount) {
+		startingPoint = deterministicRandom()->randomInt(0, shardsToCheck.size() - batchSize * testersCount);
+		// We randomly pick a set of successive shards:
+		// (1) We want to retry for different shards to avoid repeated failure on the same shards
+		// (2) We want to check successive shards to avoid inefficiency incurred by fragments
+	}
+	assignment.clear();
+	for (int i = startingPoint; i < shardsToCheck.size(); i++) {
+		int testerIdx = (i - startingPoint) / batchSize;
+		if (testerIdx > testersCount - 1) {
+			break; // Have filled up all testers
+		}
+		assignment[testerIdx].push_back(shardsToCheck[i]);
+	}
+	state std::unordered_map<int, std::vector<KeyRange>>::iterator assignIt;
+	for (assignIt = assignment.begin(); assignIt != assignment.end(); assignIt++) {
+		TraceEvent("ConsistencyCheckUrgent_ClientAssignedTask")
+		    .detail("ConsistencyCheckerId", consistencyCheckerId)
+		    .detail("Round", round)
+		    .detail("ClientId", assignIt->first)
+		    .detail("ShardsCount", assignIt->second.size());
+		if (SERVER_KNOBS->CONSISTENCY_CHECK_USE_PERSIST_DATA) {
+			wait(persistConsistencyCheckAssignment(
+			    cx, assignIt->first, assignIt->second, consistencyCheckerId)); // Persist assignment
+			if (g_network->isSimulated() && deterministicRandom()->random01() < 0.05) {
+				throw operation_failed(); // Introduce random failure
+			}
+		}
+	}
+	if (SERVER_KNOBS->CONSISTENCY_CHECK_USE_PERSIST_DATA) {
+		TraceEvent e("ConsistencyCheckUrgent_PersistAssignment");
+		e.setMaxEventLength(-1);
+		e.setMaxFieldLength(-1);
+		e.detail("ConsistencyCheckerId", consistencyCheckerId);
+		e.detail("Round", round);
+		e.detail("TesterCount", testersCount);
+		e.detail("ShardCountTotal", shardsToCheck.size());
+		for (const auto& [clientId, assignedShards] : assignment) {
+			e.detail("Client" + std::to_string(clientId), assignedShards.size());
+		}
+	}
+	return assignment;
+}
+
 ACTOR Future<Void> runConsistencyCheckerUrgentCore(Reference<AsyncVar<Optional<ClusterControllerFullInterface>>> cc,
                                                    Database cx,
                                                    Optional<std::vector<TesterInterface>> testers,
@@ -1257,43 +1341,69 @@ ACTOR Future<Void> runConsistencyCheckerUrgentCore(Reference<AsyncVar<Optional<C
                                                    Reference<AsyncVar<ServerDBInfo>> dbInfo) {
 	state int64_t consistencyCheckerId = deterministicRandom()->randomInt64(
 	    SERVER_KNOBS->CONSISTENCY_CHECK_ID_MIN, SERVER_KNOBS->CONSISTENCY_CHECK_ID_MAX_PLUS_ONE);
-	state std::vector<KeyRange> rangesToCheck;
-	// Init: enforce consistencyCheckerId and prepare for metadata
-	try {
-		wait(runConsistencyCheckerUrgentInit(cx, consistencyCheckerId));
-	} catch (Error& e) {
-		if (e.code() == error_code_key_not_found || e.code() == error_code_consistency_check_task_outdated ||
-		    e.code() == error_code_timed_out) {
+	state std::vector<KeyRange> rangesToCheck; // get from progress metadata
+	state std::vector<KeyRange> shardsToCheck; // get from keyServer metadata
+	state KeyRangeMap<bool> globalProgressMap; // used to keep track of progress when persisting metadata is not allowed
+	state std::unordered_map<int, std::vector<KeyRange>> assignment; // used to keep track of assignment of tasks
+	state std::vector<TesterInterface> ts; // used to store testers interface
+
+	// Initialization
+	if (SERVER_KNOBS->CONSISTENCY_CHECK_USE_PERSIST_DATA) {
+		// In case when persisting metadata is allowed, enforce consistencyCheckerId and prepare for metadata
+		try {
+			wait(runConsistencyCheckerUrgentInit(cx, consistencyCheckerId));
+		} catch (Error& e) {
+			if (e.code() == error_code_key_not_found || e.code() == error_code_consistency_check_task_outdated ||
+			    e.code() == error_code_timed_out) {
+				TraceEvent("ConsistencyCheckUrgent_Exit")
+				    .errorUnsuppressed(e)
+				    .detail("Reason", "FailureWhenInit")
+				    .detail("ConsistencyCheckerId", consistencyCheckerId);
+				return Void();
+			} else {
+				throw e;
+			}
+		}
+		// Immediately exit after the clear for INIT_CLEAR_METADATA_EXIT mode
+		if (CLIENT_KNOBS->CONSISTENCY_CHECK_INIT_CLEAR_METADATA_EXIT) {
 			TraceEvent("ConsistencyCheckUrgent_Exit")
-			    .errorUnsuppressed(e)
-			    .detail("Reason", "FailureWhenInit")
+			    .detail("Reason", "SuccessClearMetadataWhenInit")
 			    .detail("ConsistencyCheckerId", consistencyCheckerId);
 			return Void();
-		} else {
-			throw e;
 		}
+		// At this point, consistencyCheckerId has the ownership except that another consistency checker overwrites the
+		// id metadata
+	} else {
+		// In case when persisting metadata is not allowed, prepare for globalProgressMap
+		// globalProgressMap is used to keep track of the global progress of checking
+		globalProgressMap.insert(allKeys, true);
+		rangesToCheck = loadRangesToCheckFromKnob();
+		for (const auto& rangeToCheck : rangesToCheck) {
+			// Mark rangesToCheck as incomplete
+			// Those ranges will be checked
+			globalProgressMap.insert(rangeToCheck, false);
+		}
+		globalProgressMap.coalesce(allKeys);
 	}
-	// Immediately exit after the clear for INIT_CLEAR_METADATA_EXIT mode
-	if (CLIENT_KNOBS->CONSISTENCY_CHECK_INIT_CLEAR_METADATA_EXIT) {
-		TraceEvent("ConsistencyCheckUrgent_Exit")
-		    .detail("Reason", "SuccessClearMetadataWhenInit")
-		    .detail("ConsistencyCheckerId", consistencyCheckerId);
-		return Void();
-	}
-	// At this point, consistencyCheckerId has the ownership except that another consistency checker overwrites the id
-	// metadata
 
 	// Main loop
 	state int retryTimes = 0;
 	state int round = 0;
-	state std::vector<TesterInterface> ts;
 	loop {
 		try {
-			// Load ranges to check
+			// Step 1: Load ranges to check
 			rangesToCheck.clear();
-			wait(store(rangesToCheck, loadRangesToCheckFromProgressMetadata(cx, consistencyCheckerId)));
-			if (g_network->isSimulated() && deterministicRandom()->random01() < 0.05) {
-				throw operation_failed(); // Introduce random failure
+			if (SERVER_KNOBS->CONSISTENCY_CHECK_USE_PERSIST_DATA) {
+				wait(store(rangesToCheck, loadRangesToCheckFromProgressMetadata(cx, consistencyCheckerId)));
+				if (g_network->isSimulated() && deterministicRandom()->random01() < 0.05) {
+					throw operation_failed(); // Introduce random failure
+				}
+			} else {
+				for (auto& range : globalProgressMap.ranges()) {
+					if (!range.value()) { // range that is not finished
+						rangesToCheck.push_back(range.range());
+					}
+				}
 			}
 			if (rangesToCheck.size() == 0) {
 				TraceEvent("ConsistencyCheckUrgent_Exit")
@@ -1304,7 +1414,7 @@ ACTOR Future<Void> runConsistencyCheckerUrgentCore(Reference<AsyncVar<Optional<C
 				return Void();
 			}
 
-			// Get testers
+			// Step 2: Get testers
 			ts.clear();
 			if (!testers.present()) {
 				wait(store(ts, getTesters(cc, minTestersExpected)));
@@ -1320,8 +1430,10 @@ ACTOR Future<Void> runConsistencyCheckerUrgentCore(Reference<AsyncVar<Optional<C
 			    .detail("RetryTimes", retryTimes)
 			    .detail("TesterCount", ts.size());
 
-			// Load shards to check from keyserver space
-			state std::vector<KeyRange> shardsToCheck = wait(getConsistencyCheckShards(cx, rangesToCheck));
+			// Step 3: Load shards to check from keyserver space
+			// Shard is the unit for the task assignment
+			shardsToCheck.clear();
+			wait(store(shardsToCheck, getConsistencyCheckShards(cx, rangesToCheck)));
 			TraceEvent("ConsistencyCheckUrgent_GotShardsToCheck")
 			    .detail("ConsistencyCheckerId", consistencyCheckerId)
 			    .detail("Round", round)
@@ -1329,78 +1441,58 @@ ACTOR Future<Void> runConsistencyCheckerUrgentCore(Reference<AsyncVar<Optional<C
 			    .detail("TesterCount", ts.size())
 			    .detail("ShardCount", shardsToCheck.size());
 
-			// Assign shards to testers
-			wait(initConsistencyCheckAssignmentMetadata(cx, consistencyCheckerId));
+			// Step 4: Decide assignment of shards to testers
+			assignment.clear();
+			wait(store(assignment, makeTaskAssignment(cx, consistencyCheckerId, shardsToCheck, ts.size(), round)));
+
+			// Step 5: Run checking on testers
+			std::set<int> completeClients = wait(dispatchMonitorUrgentConsistencyCheckWorkload(
+			    cx, ts, testSpec, defaultTenant, consistencyCheckerId, assignment));
 			if (g_network->isSimulated() && deterministicRandom()->random01() < 0.05) {
 				throw operation_failed(); // Introduce random failure
 			}
-			state std::unordered_map<int, std::vector<KeyRange>> assignment; // Generate assignment
-			int batchSize = CLIENT_KNOBS->CONSISTENCY_CHECK_BATCH_SHARD_COUNT;
-			int startingPoint = 0;
-			if (shardsToCheck.size() > batchSize * ts.size()) {
-				startingPoint = deterministicRandom()->randomInt(0, shardsToCheck.size() - batchSize * ts.size());
-				// We randomly pick a set of successive shards:
-				// (1) We want to retry for different shards to avoid repeated failure on the same shards
-				// (2) We want to check successive shards to avoid inefficiency incurred by fragments
-			}
-			for (int i = startingPoint; i < shardsToCheck.size(); i++) {
-				int testerIdx = (i - startingPoint) / batchSize;
-				if (testerIdx > ts.size() - 1) {
-					break; // Have filled up all testers
-				}
-				assignment[testerIdx].push_back(shardsToCheck[i]);
-			}
-			state std::unordered_map<int, std::vector<KeyRange>>::iterator assignIt;
-			for (assignIt = assignment.begin(); assignIt != assignment.end(); assignIt++) {
-				TraceEvent("ConsistencyCheckUrgent_ClientAssignedTask")
-				    .detail("ConsistencyCheckerId", consistencyCheckerId)
-				    .detail("RetryTimes", retryTimes)
-				    .detail("Round", round)
-				    .detail("ClientId", assignIt->first)
-				    .detail("ShardsCount", assignIt->second.size());
-				wait(persistConsistencyCheckAssignment(
-				    cx, assignIt->first, assignIt->second, consistencyCheckerId)); // Persist assignment
-				if (g_network->isSimulated() && deterministicRandom()->random01() < 0.05) {
-					throw operation_failed(); // Introduce random failure
+			if (!SERVER_KNOBS->CONSISTENCY_CHECK_USE_PERSIST_DATA) {
+				// In case when persisting metadata is not allowed, we use
+				// the complete client to decide which ranges are completed
+				for (const auto& clientId : completeClients) {
+					for (const auto& range : assignment[clientId]) {
+						globalProgressMap.insert(range, true); // Mark the ranges as complete
+					}
 				}
 			}
-			TraceEvent e("ConsistencyCheckUrgent_PersistAssignment");
-			e.setMaxEventLength(-1);
-			e.setMaxFieldLength(-1);
-			e.detail("ConsistencyCheckerId", consistencyCheckerId);
-			e.detail("Round", round);
-			e.detail("RetryTimes", retryTimes);
-			e.detail("TesterCount", ts.size());
-			e.detail("ShardCountTotal", shardsToCheck.size());
-			for (const auto& [clientId, assignedShards] : assignment) {
-				e.detail("Client" + std::to_string(clientId), assignedShards.size());
-			}
-
-			// Run test and wait for testers complete/fail
-			wait(dispatchMonitorUrgentConsistencyCheckWorkload(cx, ts, testSpec, defaultTenant, consistencyCheckerId));
-			if (g_network->isSimulated() && deterministicRandom()->random01() < 0.05) {
-				throw operation_failed(); // Introduce random failure
-			}
-			TraceEvent("ConsistencyCheckUrgent_RoundWorkloadComplete")
+			TraceEvent("ConsistencyCheckUrgent_RoundComplete")
 			    .detail("ConsistencyCheckerId", consistencyCheckerId)
 			    .detail("RetryTimes", retryTimes)
+			    .detail("SucceedTesterCount", completeClients.size())
+			    .detail("SucceedTesters", describe(completeClients))
+			    .detail("TesterCount", ts.size())
 			    .detail("Round", round);
 
-			// Load progress metadata. If no unfinished range, exit; otherwise, continue
+			// Step 6: Check progress: If no unfinished range, exit; otherwise, continue
 			rangesToCheck.clear();
-			wait(store(rangesToCheck, loadRangesToCheckFromProgressMetadata(cx, consistencyCheckerId)));
-			// If rangesToCheck has data, rangesToCheck is for the next round
-			if (g_network->isSimulated() && deterministicRandom()->random01() < 0.05) {
-				throw operation_failed(); // Introduce random failure
+			if (SERVER_KNOBS->CONSISTENCY_CHECK_USE_PERSIST_DATA) {
+				wait(store(rangesToCheck, loadRangesToCheckFromProgressMetadata(cx, consistencyCheckerId)));
+				// If rangesToCheck has data, rangesToCheck is for the next round
+				if (g_network->isSimulated() && deterministicRandom()->random01() < 0.05) {
+					throw operation_failed(); // Introduce random failure
+				}
+			} else {
+				for (auto& range : globalProgressMap.ranges()) {
+					if (!range.value()) { // range that is not finished
+						rangesToCheck.push_back(range.range());
+					}
+				}
 			}
 			if (rangesToCheck.size() == 0) {
 				TraceEvent("ConsistencyCheckUrgent_Complete")
 				    .detail("ConsistencyCheckerId", consistencyCheckerId)
 				    .detail("RetryTimes", retryTimes)
 				    .detail("Round", round);
-				wait(clearConsistencyCheckMetadata(cx, consistencyCheckerId));
-				if (g_network->isSimulated() && deterministicRandom()->random01() < 0.05) {
-					throw operation_failed(); // Introduce random failure
+				if (SERVER_KNOBS->CONSISTENCY_CHECK_USE_PERSIST_DATA) {
+					wait(clearConsistencyCheckMetadata(cx, consistencyCheckerId));
+					if (g_network->isSimulated() && deterministicRandom()->random01() < 0.05) {
+						throw operation_failed(); // Introduce random failure
+					}
 				}
 				TraceEvent("ConsistencyCheckUrgent_Exit")
 				    .detail("Reason", "Complete")
@@ -1424,7 +1516,7 @@ ACTOR Future<Void> runConsistencyCheckerUrgentCore(Reference<AsyncVar<Optional<C
 			if (e.code() == error_code_actor_cancelled) {
 				throw e;
 			} else if (e.code() == error_code_key_not_found || e.code() == error_code_consistency_check_task_outdated) {
-				TraceEvent("ConsistencyCheckUrgent_Exit")
+				TraceEvent("ConsistencyCheckUrgent_Exit") // Happens only when persisting data is allowed
 				    .errorUnsuppressed(e)
 				    .detail("Reason", "ConsistencyCheckerOutdated")
 				    .detail("ConsistencyCheckerId", consistencyCheckerId)
@@ -1432,7 +1524,7 @@ ACTOR Future<Void> runConsistencyCheckerUrgentCore(Reference<AsyncVar<Optional<C
 				    .detail("Round", round);
 				return Void(); // Exit
 			} else {
-				TraceEvent("ConsistencyCheckUrgent_RetriableFailure")
+				TraceEvent("ConsistencyCheckUrgent_CoreWithRetriableFailure")
 				    .errorUnsuppressed(e)
 				    .detail("ConsistencyCheckerId", consistencyCheckerId)
 				    .detail("RetryTimes", retryTimes)
@@ -1445,31 +1537,32 @@ ACTOR Future<Void> runConsistencyCheckerUrgentCore(Reference<AsyncVar<Optional<C
 		wait(delay(10.0)); // Backoff 10 seconds for the next round
 
 		// Decide and enforce the consistencyCheckerId for the next round
-		state int retryTimesForUpdatingCheckerId = 0;
-		loop {
-			try {
-				// Decide and persist consistencyCheckerId for the next round
-				consistencyCheckerId = deterministicRandom()->randomInt64(
-				    SERVER_KNOBS->CONSISTENCY_CHECK_ID_MIN, SERVER_KNOBS->CONSISTENCY_CHECK_ID_MAX_PLUS_ONE);
-				wait(persistConsistencyCheckerId(cx, consistencyCheckerId));
-				if (g_network->isSimulated() && deterministicRandom()->random01() < 0.05) {
-					throw operation_failed(); // Introduce random failure
-				}
-				break; // Continue to the next round
-			} catch (Error& e) {
-				if (e.code() == error_code_actor_cancelled) {
-					throw e;
-				}
-				if (retryTimesForUpdatingCheckerId > 50) {
-					TraceEvent("ConsistencyCheckUrgent_Exit")
-					    .errorUnsuppressed(e)
-					    .detail("Reason", "PersistConsistencyCheckerIdFailed")
-					    .detail("ConsistencyCheckerId", consistencyCheckerId)
-					    .detail("Round", round);
-					return Void(); // Exit
+		consistencyCheckerId = deterministicRandom()->randomInt64(SERVER_KNOBS->CONSISTENCY_CHECK_ID_MIN,
+		                                                          SERVER_KNOBS->CONSISTENCY_CHECK_ID_MAX_PLUS_ONE);
+		if (SERVER_KNOBS->CONSISTENCY_CHECK_USE_PERSIST_DATA) {
+			state int retryTimesForUpdatingCheckerId = 0;
+			loop {
+				try {
+					wait(persistConsistencyCheckerId(cx, consistencyCheckerId));
+					if (g_network->isSimulated() && deterministicRandom()->random01() < 0.05) {
+						throw operation_failed(); // Introduce random failure
+					}
+					break; // Continue to the next round
+				} catch (Error& e) {
+					if (e.code() == error_code_actor_cancelled) {
+						throw e;
+					}
+					if (retryTimesForUpdatingCheckerId > 50) {
+						TraceEvent("ConsistencyCheckUrgent_Exit")
+						    .errorUnsuppressed(e)
+						    .detail("Reason", "PersistConsistencyCheckerIdFailed")
+						    .detail("ConsistencyCheckerId", consistencyCheckerId)
+						    .detail("Round", round);
+						return Void(); // Exit
+					}
+					wait(delay(1.0));
+					retryTimesForUpdatingCheckerId++;
 				}
-				wait(delay(1.0));
-				retryTimesForUpdatingCheckerId++;
 			}
 		}
 	}
@@ -1497,7 +1590,7 @@ ACTOR Future<Void> checkConsistencyUrgentSim(Database cx,
 	options.push_back_deep(options.arena(), KeyValueRef(LiteralStringRef("distributed"), LiteralStringRef("true")));
 	spec.options.push_back_deep(spec.options.arena(), options);
 	loop {
-		TraceEvent("ConsistencyCheckUrgent_LoopBegin");
+		TraceEvent("ConsistencyCheckUrgent_SimBegin");
 		try {
 			wait(runConsistencyCheckerUrgentCore(Reference<AsyncVar<Optional<ClusterControllerFullInterface>>>(),
 			                                     cx,
diff --git a/fdbserver/workloads/ConsistencyCheck.actor.cpp b/fdbserver/workloads/ConsistencyCheck.actor.cpp
index f5450d5b1bd..999c4d887b2 100644
--- a/fdbserver/workloads/ConsistencyCheck.actor.cpp
+++ b/fdbserver/workloads/ConsistencyCheck.actor.cpp
@@ -278,8 +278,13 @@ struct ConsistencyCheckWorkload : TestWorkload {
 
 	ACTOR Future<Void> runUrgentCheck(Database cx, ConsistencyCheckWorkload* self) {
 		try {
-			std::vector<KeyRange> rangesToCheck =
-			    wait(self->loadRangesFromAssignmentMetadata(cx, self->clientId, self->consistencyCheckerId));
+			state std::vector<KeyRange> rangesToCheck;
+			if (!self->rangesToCheck.present()) {
+				wait(store(rangesToCheck,
+				           self->loadRangesFromAssignmentMetadata(cx, self->clientId, self->consistencyCheckerId)));
+			} else {
+				rangesToCheck = self->rangesToCheck.get(); // In case persisting data is not allowed
+			}
 			if (rangesToCheck.size() == 0) {
 				TraceEvent("ConsistencyCheckUrgent_TesterExit")
 				    .detail("Reason", "AssignedEmptyRangeToCheck")
@@ -288,6 +293,12 @@ struct ConsistencyCheckWorkload : TestWorkload {
 				    .detail("ClientId", self->clientId);
 				return Void();
 			}
+			if (g_network->isSimulated() && deterministicRandom()->coinflip()) {
+				TraceEvent("ConsistencyCheckUrgent_SimTesterFailure")
+				    .detail("ClientCount", self->clientCount)
+				    .detail("ClientId", self->clientId);
+				throw operation_failed(); // mimic tester failure
+			}
 			wait(self->checkDataConsistencyUrgent(cx,
 			                                      rangesToCheck,
 			                                      self,
@@ -307,8 +318,9 @@ struct ConsistencyCheckWorkload : TestWorkload {
 				    .detail("ConsistencyCheckerId", self->consistencyCheckerId)
 				    .detail("ClientCount", self->clientCount)
 				    .detail("ClientId", self->clientId);
+				throw consistency_check_task_failed();
 			} else if (e.code() == error_code_key_not_found || e.code() == error_code_consistency_check_task_outdated) {
-				TraceEvent("ConsistencyCheckUrgent_TesterExit")
+				TraceEvent("ConsistencyCheckUrgent_TesterExit") // Happens only when persisting data is allowed
 				    .error(e)
 				    .detail("Reason", "ConsistencyCheckerIdOutDated")
 				    .detail("ConsistencyCheckerId", self->consistencyCheckerId)
@@ -321,6 +333,7 @@ struct ConsistencyCheckWorkload : TestWorkload {
 				    .detail("ConsistencyCheckerId", self->consistencyCheckerId)
 				    .detail("ClientCount", self->clientCount)
 				    .detail("ClientId", self->clientId);
+				throw consistency_check_task_failed();
 			}
 		}
 		return Void();
@@ -1491,18 +1504,22 @@ struct ConsistencyCheckWorkload : TestWorkload {
 				    .detail("Repetitions", self->repetitions)
 				    .detail("ConsistencyCheckEpoch", consistencyCheckEpoch);
 				numCompleteShards++;
-				try {
-					wait(self->persistProgress(cx, range, self->consistencyCheckerId));
-				} catch (Error& e) {
-					throw e;
+				if (!self->rangesToCheck.present()) { // In case we are able to persist progress
+					try {
+						wait(self->persistProgress(cx, range, self->consistencyCheckerId));
+					} catch (Error& e) {
+						throw e;
+					}
 				}
 				continue; // Skip to the next shard
 			} else if (sourceStorageServers.size() == 1) {
 				numCompleteShards++;
-				try {
-					wait(self->persistProgress(cx, range, self->consistencyCheckerId));
-				} catch (Error& e) {
-					throw e;
+				if (!self->rangesToCheck.present()) { // In case we are able to persist progress
+					try {
+						wait(self->persistProgress(cx, range, self->consistencyCheckerId));
+					} catch (Error& e) {
+						throw e;
+					}
 				}
 				continue; // Skip to the next shard
 			}
@@ -1727,7 +1744,6 @@ struct ConsistencyCheckWorkload : TestWorkload {
 			if (!valueAvailableToCheck) {
 				numFailedShards++;
 				TraceEvent(SevInfo, "ConsistencyCheckUrgent_ShardFailed")
-				    .suppressFor(1.0)
 				    .setMaxEventLength(-1)
 				    .setMaxFieldLength(-1)
 				    .detail("ConsistencyCheckerId", self->consistencyCheckerId)
@@ -1743,13 +1759,14 @@ struct ConsistencyCheckWorkload : TestWorkload {
 				    .detail("ConsistencyCheckEpoch", consistencyCheckEpoch);
 			} else {
 				numCompleteShards++;
-				try {
-					wait(self->persistProgress(cx, range, self->consistencyCheckerId));
-				} catch (Error& e) {
-					throw e;
+				if (!self->rangesToCheck.present()) { // In case we are able to persist progress
+					try {
+						wait(self->persistProgress(cx, range, self->consistencyCheckerId));
+					} catch (Error& e) {
+						throw e;
+					}
 				}
 				TraceEvent(SevInfo, "ConsistencyCheckUrgent_ShardComplete")
-				    .suppressFor(1.0)
 				    .setMaxEventLength(-1)
 				    .setMaxFieldLength(-1)
 				    .detail("ConsistencyCheckerId", self->consistencyCheckerId)
@@ -2448,19 +2465,6 @@ struct ConsistencyCheckWorkload : TestWorkload {
 					    .detail("ConsistencyCheckEpoch", consistencyCheckEpoch);
 				} else {
 					numCompleteShards++;
-					if (self->consistencyCheckerId != 0) {
-						// Persist progress in urgent mode
-						try {
-							wait(self->persistProgress(cx, range, self->consistencyCheckerId));
-						} catch (Error& e) {
-							if (e.code() == error_code_key_not_found ||
-							    e.code() == error_code_consistency_check_task_outdated) {
-								return testResult;
-							} else {
-								throw e;
-							}
-						}
-					}
 					TraceEvent(SevInfo, "ConsistencyCheck_ShardComplete")
 					    .suppressFor(1.0)
 					    .setMaxEventLength(-1)
diff --git a/fdbserver/workloads/workloads.actor.h b/fdbserver/workloads/workloads.actor.h
index c7c0557b682..7efdd6753fe 100644
--- a/fdbserver/workloads/workloads.actor.h
+++ b/fdbserver/workloads/workloads.actor.h
@@ -51,6 +51,7 @@ struct WorkloadContext {
 	int clientId, clientCount;
 	int64_t sharedRandomNumber;
 	Reference<AsyncVar<struct ServerDBInfo> const> dbInfo;
+	Optional<std::vector<KeyRange>> rangesToCheck; // for urgent consistency checker when persisting data is not allowed
 
 	WorkloadContext();
 	WorkloadContext(const WorkloadContext&);
diff --git a/flow/error_definitions.h b/flow/error_definitions.h
index a6997858332..f46cf0c3503 100755
--- a/flow/error_definitions.h
+++ b/flow/error_definitions.h
@@ -98,6 +98,7 @@ ERROR( encryption_key_not_found, 1074, "Encryption key not found" )
 ERROR( unknown_storage_engine, 1082, "Storage engine type is not recognized." )
 ERROR( manual_shard_split_failed, 1083, "Manual shard split is failed to trigger")
 ERROR( consistency_check_task_outdated, 1084, "Consistency check task is outdated")
+ERROR( consistency_check_task_failed, 1085, "Consistency check task is failed")
 
 ERROR( broken_promise, 1100, "Broken promise" )
 ERROR( operation_cancelled, 1101, "Asynchronous operation cancelled" )

From 60ae45bed8dca82a89c1d280fe52cc43495e562d Mon Sep 17 00:00:00 2001
From: Zhe Wang <zhe.wang@wustl.edu>
Date: Wed, 3 Jan 2024 12:39:18 -0800
Subject: [PATCH 359/468] code cleanup and nits

---
 fdbserver/tester.actor.cpp                    | 65 ++++++-------------
 .../workloads/ConsistencyCheck.actor.cpp      | 35 ++++++----
 2 files changed, 41 insertions(+), 59 deletions(-)

diff --git a/fdbserver/tester.actor.cpp b/fdbserver/tester.actor.cpp
index 5d7dd9ed3be..f74b33347f2 100644
--- a/fdbserver/tester.actor.cpp
+++ b/fdbserver/tester.actor.cpp
@@ -1391,7 +1391,13 @@ ACTOR Future<Void> runConsistencyCheckerUrgentCore(Reference<AsyncVar<Optional<C
 	state int round = 0;
 	loop {
 		try {
-			// Step 1: Load ranges to check
+			// Step 1: Load ranges to check, if nothing to run, exit
+			TraceEvent("ConsistencyCheckUrgent_RoundBegin")
+			    .detail("ConsistencyCheckerId", consistencyCheckerId)
+			    .detail("RetryTimes", retryTimes)
+			    .detail("TesterCount", ts.size())
+			    .detail("Round", round);
+
 			rangesToCheck.clear();
 			if (SERVER_KNOBS->CONSISTENCY_CHECK_USE_PERSIST_DATA) {
 				wait(store(rangesToCheck, loadRangesToCheckFromProgressMetadata(cx, consistencyCheckerId)));
@@ -1406,6 +1412,16 @@ ACTOR Future<Void> runConsistencyCheckerUrgentCore(Reference<AsyncVar<Optional<C
 				}
 			}
 			if (rangesToCheck.size() == 0) {
+				TraceEvent("ConsistencyCheckUrgent_Complete")
+				    .detail("ConsistencyCheckerId", consistencyCheckerId)
+				    .detail("RetryTimes", retryTimes)
+				    .detail("Round", round);
+				if (SERVER_KNOBS->CONSISTENCY_CHECK_USE_PERSIST_DATA) {
+					wait(clearConsistencyCheckMetadata(cx, consistencyCheckerId));
+					if (g_network->isSimulated() && deterministicRandom()->random01() < 0.05) {
+						throw operation_failed(); // Introduce random failure
+					}
+				}
 				TraceEvent("ConsistencyCheckUrgent_Exit")
 				    .detail("Reason", "Complete")
 				    .detail("ConsistencyCheckerId", consistencyCheckerId)
@@ -1460,58 +1476,15 @@ ACTOR Future<Void> runConsistencyCheckerUrgentCore(Reference<AsyncVar<Optional<C
 					}
 				}
 			}
-			TraceEvent("ConsistencyCheckUrgent_RoundComplete")
+			TraceEvent("ConsistencyCheckUrgent_RoundEnd")
 			    .detail("ConsistencyCheckerId", consistencyCheckerId)
 			    .detail("RetryTimes", retryTimes)
 			    .detail("SucceedTesterCount", completeClients.size())
 			    .detail("SucceedTesters", describe(completeClients))
 			    .detail("TesterCount", ts.size())
 			    .detail("Round", round);
+			round++;
 
-			// Step 6: Check progress: If no unfinished range, exit; otherwise, continue
-			rangesToCheck.clear();
-			if (SERVER_KNOBS->CONSISTENCY_CHECK_USE_PERSIST_DATA) {
-				wait(store(rangesToCheck, loadRangesToCheckFromProgressMetadata(cx, consistencyCheckerId)));
-				// If rangesToCheck has data, rangesToCheck is for the next round
-				if (g_network->isSimulated() && deterministicRandom()->random01() < 0.05) {
-					throw operation_failed(); // Introduce random failure
-				}
-			} else {
-				for (auto& range : globalProgressMap.ranges()) {
-					if (!range.value()) { // range that is not finished
-						rangesToCheck.push_back(range.range());
-					}
-				}
-			}
-			if (rangesToCheck.size() == 0) {
-				TraceEvent("ConsistencyCheckUrgent_Complete")
-				    .detail("ConsistencyCheckerId", consistencyCheckerId)
-				    .detail("RetryTimes", retryTimes)
-				    .detail("Round", round);
-				if (SERVER_KNOBS->CONSISTENCY_CHECK_USE_PERSIST_DATA) {
-					wait(clearConsistencyCheckMetadata(cx, consistencyCheckerId));
-					if (g_network->isSimulated() && deterministicRandom()->random01() < 0.05) {
-						throw operation_failed(); // Introduce random failure
-					}
-				}
-				TraceEvent("ConsistencyCheckUrgent_Exit")
-				    .detail("Reason", "Complete")
-				    .detail("ConsistencyCheckerId", consistencyCheckerId)
-				    .detail("RetryTimes", retryTimes)
-				    .detail("Round", round);
-				return Void(); // Exit
-			} else {
-				TraceEvent("ConsistencyCheckUrgent_Continue")
-				    .detail("ConsistencyCheckerId", consistencyCheckerId)
-				    .detail("RetryTimes", retryTimes)
-				    .detail("Round", round)
-				    .detail("RangesToCheckCount", rangesToCheck.size());
-				if (g_network->isSimulated()) {
-					TraceEvent("ConsistencyCheckUrgent_RepairDC");
-					wait(repairDeadDatacenter(cx, dbInfo, "ConsistencyCheckUrgent"));
-				}
-				round++;
-			}
 		} catch (Error& e) {
 			if (e.code() == error_code_actor_cancelled) {
 				throw e;
diff --git a/fdbserver/workloads/ConsistencyCheck.actor.cpp b/fdbserver/workloads/ConsistencyCheck.actor.cpp
index 999c4d887b2..910bf06bfa4 100644
--- a/fdbserver/workloads/ConsistencyCheck.actor.cpp
+++ b/fdbserver/workloads/ConsistencyCheck.actor.cpp
@@ -304,7 +304,7 @@ struct ConsistencyCheckWorkload : TestWorkload {
 			                                      self,
 			                                      /*consistencyCheckEpoch=*/0));
 			TraceEvent("ConsistencyCheckUrgent_TesterExit")
-			    .detail("Reason", "CompleteChecker")
+			    .detail("Reason", "CompleteCheck")
 			    .detail("ConsistencyCheckerId", self->consistencyCheckerId)
 			    .detail("ClientCount", self->clientCount)
 			    .detail("ClientId", self->clientId);
@@ -1404,15 +1404,6 @@ struct ConsistencyCheckWorkload : TestWorkload {
 	                                              std::vector<KeyRange> rangesToCheck,
 	                                              ConsistencyCheckWorkload* self,
 	                                              int consistencyCheckEpoch) {
-		TraceEvent("ConsistencyCheckUrgent_StartTask")
-		    .setMaxEventLength(-1)
-		    .setMaxFieldLength(-1)
-		    .detail("ConsistencyCheckerId", self->consistencyCheckerId)
-		    .detail("RangeToCheck", rangesToCheck.size())
-		    .detail("ClientCount", self->clientCount)
-		    .detail("ClientId", self->clientId)
-		    .detail("ConsistencyCheckEpoch", consistencyCheckEpoch);
-
 		// Get shard locations of the input rangesToCheck
 		state std::vector<std::pair<KeyRange, Value>> shardLocationPairList;
 		state int retryCount = 0;
@@ -1442,6 +1433,14 @@ struct ConsistencyCheckWorkload : TestWorkload {
 			}
 		}
 
+		TraceEvent("ConsistencyCheckUrgent_StartTask")
+		    .detail("ConsistencyCheckerId", self->consistencyCheckerId)
+		    .detail("RangeToCheck", rangesToCheck.size())
+		    .detail("ShardToCheck", shardLocationPairList.size())
+		    .detail("ClientCount", self->clientCount)
+		    .detail("ClientId", self->clientId)
+		    .detail("ConsistencyCheckEpoch", consistencyCheckEpoch);
+
 		// Do consistency check shard by shard
 		state Reference<IRateControl> rateLimiter =
 		    Reference<IRateControl>(new SpeedLimit(CLIENT_KNOBS->CONSISTENCY_CHECK_RATE_LIMIT_MAX, 1));
@@ -1513,6 +1512,14 @@ struct ConsistencyCheckWorkload : TestWorkload {
 				}
 				continue; // Skip to the next shard
 			} else if (sourceStorageServers.size() == 1) {
+				TraceEvent(SevWarn, "ConsistencyCheckUrgent_SingleReplica")
+				    .detail("ConsistencyCheckerId", self->consistencyCheckerId)
+				    .detail("ClientId", self->clientId)
+				    .detail("ClientCount", self->clientCount)
+				    .detail("ShardBegin", range.begin)
+				    .detail("ShardEnd", range.end)
+				    .detail("Repetitions", self->repetitions)
+				    .detail("ConsistencyCheckEpoch", consistencyCheckEpoch);
 				numCompleteShards++;
 				if (!self->rangesToCheck.present()) { // In case we are able to persist progress
 					try {
@@ -1697,8 +1704,8 @@ struct ConsistencyCheckWorkload : TestWorkload {
 								    .detail(format("StorageServer%d", j).c_str(), storageServers[j].toString())
 								    .detail(format("StorageServer%d", firstValidServer).c_str(),
 								            storageServers[firstValidServer].toString())
-								    .detail("ShardBegin", req.begin.getKey())
-								    .detail("ShardEnd", req.end.getKey())
+								    .detail("RangeBegin", req.begin.getKey())
+								    .detail("RangeEnd", req.end.getKey())
 								    .detail("VersionNumber", req.version)
 								    .detail(format("Server%dUniques", j).c_str(), currentUniques)
 								    .detail(format("Server%dUniqueKey", j).c_str(), currentUniqueKey)
@@ -1711,7 +1718,9 @@ struct ConsistencyCheckWorkload : TestWorkload {
 								            storageServerInterfaces[j].isTss() ||
 								                    storageServerInterfaces[firstValidServer].isTss()
 								                ? "True"
-								                : "False");
+								                : "False")
+								    .detail("ShardBegin", range.begin)
+								    .detail("ShardEnd", range.end);
 							}
 						}
 					}

From ec996050508016d23f423459942bb223f8dd6371 Mon Sep 17 00:00:00 2001
From: Zhe Wang <zhe.wang@wustl.edu>
Date: Wed, 3 Jan 2024 13:41:48 -0800
Subject: [PATCH 360/468] tester should notify checker if it gives up retrying

---
 .../workloads/ConsistencyCheck.actor.cpp      | 39 ++++++++++++-------
 1 file changed, 26 insertions(+), 13 deletions(-)

diff --git a/fdbserver/workloads/ConsistencyCheck.actor.cpp b/fdbserver/workloads/ConsistencyCheck.actor.cpp
index 910bf06bfa4..54417d2e9a3 100644
--- a/fdbserver/workloads/ConsistencyCheck.actor.cpp
+++ b/fdbserver/workloads/ConsistencyCheck.actor.cpp
@@ -326,6 +326,14 @@ struct ConsistencyCheckWorkload : TestWorkload {
 				    .detail("ConsistencyCheckerId", self->consistencyCheckerId)
 				    .detail("ClientCount", self->clientCount)
 				    .detail("ClientId", self->clientId);
+			} else if (e.code() == error_code_consistency_check_task_failed) {
+				TraceEvent("ConsistencyCheckUrgent_TesterExit")
+				    .error(e)
+				    .detail("Reason", "Retry failed ranges for too many times")
+				    .detail("ConsistencyCheckerId", self->consistencyCheckerId)
+				    .detail("ClientCount", self->clientCount)
+				    .detail("ClientId", self->clientId);
+				throw consistency_check_task_failed();
 			} else {
 				TraceEvent("ConsistencyCheckUrgent_TesterExit")
 				    .error(e)
@@ -1803,26 +1811,31 @@ struct ConsistencyCheckWorkload : TestWorkload {
 			}
 		}
 		if (failedRangesToCheck.size() > 0) { // Retry for any failed shard
-			wait(delay(60.0)); // Backoff 1 min
-			TraceEvent(SevInfo, "ConsistencyCheckUrgent_StartHandlingFailedRanges")
-			    .setMaxEventLength(-1)
-			    .setMaxFieldLength(-1)
-			    .detail("ConsistencyCheckerId", self->consistencyCheckerId)
-			    .detail("FailedCollectedRangeCount", failedRangesToCheck.size())
-			    .detail("ConsistencyCheckEpoch", consistencyCheckEpoch)
-			    .detail("Distributed", self->distributed)
-			    .detail("ClientId", self->clientId)
-			    .detail("ClientCount", self->clientCount);
 			if (consistencyCheckEpoch < CLIENT_KNOBS->CONSISTENCY_CHECK_RETRY_DEPTH_MAX) {
+				wait(delay(60.0)); // Backoff 1 min
+				TraceEvent(SevInfo, "ConsistencyCheckUrgent_RetryFailedRanges")
+				    .detail("ConsistencyCheckerId", self->consistencyCheckerId)
+				    .detail("FailedCollectedRangeCount", failedRangesToCheck.size())
+				    .detail("ConsistencyCheckEpoch", consistencyCheckEpoch)
+				    .detail("Distributed", self->distributed)
+				    .detail("ClientId", self->clientId)
+				    .detail("ClientCount", self->clientCount);
 				wait(self->checkDataConsistencyUrgent(cx, failedRangesToCheck, self, consistencyCheckEpoch + 1));
+			} else if (self->rangesToCheck.present()) { // In case we are not able to persist progress
+				TraceEvent(SevWarn, "ConsistencyCheckUrgent_RetryDepthMax")
+				    .detail("ConsistencyCheckerId", self->consistencyCheckerId)
+				    .detail("FailedCollectedRangeCount", failedRangesToCheck.size())
+				    .detail("ConsistencyCheckEpoch", consistencyCheckEpoch)
+				    .detail("Distributed", self->distributed)
+				    .detail("ClientId", self->clientId)
+				    .detail("ClientCount", self->clientCount);
+				throw consistency_check_task_failed(); // Notify the checker that this tester is failed
 			}
-			// We give up retrying when retry too many times
+			// When we are able to persist progress, we simply give up retrying when retry too many times
 			// The failed ranges will be picked up by the next round of the consistency checker urgent
 		}
 		if (consistencyCheckEpoch == 0) {
 			TraceEvent("ConsistencyCheckUrgent_EndTask")
-			    .setMaxEventLength(-1)
-			    .setMaxFieldLength(-1)
 			    .detail("ConsistencyCheckerId", self->consistencyCheckerId)
 			    .detail("Distributed", self->distributed)
 			    .detail("ShardCount", shardLocationPairList.size())

From 581f0aab12e30fd279851fa66da32b8a67890bbf Mon Sep 17 00:00:00 2001
From: Zhe Wang <zhe.wang@wustl.edu>
Date: Wed, 3 Jan 2024 19:00:35 -0800
Subject: [PATCH 361/468] tolerate when CC returns stale testers interface

---
 fdbclient/ClientKnobs.cpp  |   2 +-
 fdbserver/tester.actor.cpp | 154 +++++++++++++++++++------------------
 2 files changed, 80 insertions(+), 76 deletions(-)

diff --git a/fdbclient/ClientKnobs.cpp b/fdbclient/ClientKnobs.cpp
index 783f1a4d0e0..67e4df59800 100644
--- a/fdbclient/ClientKnobs.cpp
+++ b/fdbclient/ClientKnobs.cpp
@@ -258,7 +258,7 @@ void ClientKnobs::initialize(Randomize randomize) {
 
 	init( CONSISTENCY_CHECK_RATE_LIMIT_MAX,        50e6 ); // Limit in per sec
 	init( CONSISTENCY_CHECK_ONE_ROUND_TARGET_COMPLETION_TIME,	7 * 24 * 60 * 60 ); // 7 days
-	init( CONSISTENCY_CHECK_BATCH_SHARD_COUNT,       20 ); if( randomize && BUGGIFY ) CONSISTENCY_CHECK_BATCH_SHARD_COUNT = 2;
+	init( CONSISTENCY_CHECK_BATCH_SHARD_COUNT,       10 ); if( randomize && BUGGIFY ) CONSISTENCY_CHECK_BATCH_SHARD_COUNT = 2;
 	init( CONSISTENCY_CHECK_RETRY_DEPTH_MAX,         10 ); if( randomize && BUGGIFY ) CONSISTENCY_CHECK_RETRY_DEPTH_MAX = 1;
 	init( CONSISTENCY_CHECK_INIT_CLEAR_METADATA,  false );
 	init( CONSISTENCY_CHECK_INIT_CLEAR_METADATA_EXIT, false );
diff --git a/fdbserver/tester.actor.cpp b/fdbserver/tester.actor.cpp
index f74b33347f2..c35b2f06a49 100644
--- a/fdbserver/tester.actor.cpp
+++ b/fdbserver/tester.actor.cpp
@@ -991,7 +991,7 @@ ACTOR Future<Void> checkConsistency(Database cx,
 	}
 }
 
-ACTOR Future<std::set<int>> dispatchMonitorUrgentConsistencyCheckWorkload(
+ACTOR Future<std::unordered_set<int>> dispatchMonitorUrgentConsistencyCheckWorkload(
     Database cx,
     std::vector<TesterInterface> testers,
     TestSpec spec,
@@ -999,14 +999,12 @@ ACTOR Future<std::set<int>> dispatchMonitorUrgentConsistencyCheckWorkload(
     int64_t consistencyCheckerId,
     std::unordered_map<int, std::vector<KeyRange>> assignment) {
 	TraceEvent("ConsistencyCheckUrgent_Dispatch")
-	    .detail("WorkloadTitle", spec.title)
 	    .detail("TesterCount", testers.size())
-	    .detail("TestTimeout", spec.timeout)
 	    .detail("ConsistencyCheckerId", consistencyCheckerId);
 
-	state std::set<int> completeClientIds;
+	state double waitForFailureTime = g_network->isSimulated() ? 24 * 60 * 60 : 60;
 
-	state std::vector<Future<WorkloadInterface>> workRequests;
+	state std::vector<Future<ErrorOr<WorkloadInterface>>> workRequests;
 	for (int i = 0; i < testers.size(); i++) {
 		WorkloadRequest req;
 		req.title = spec.title;
@@ -1023,93 +1021,98 @@ ACTOR Future<std::set<int>> dispatchMonitorUrgentConsistencyCheckWorkload(
 		} else {
 			req.rangesToCheck = Optional<std::vector<KeyRange>>();
 		}
-		workRequests.push_back(testers[i].recruitments.getReply(req));
+		workRequests.push_back(errorOr(timeoutError(testers[i].recruitments.getReply(req), waitForFailureTime)));
+		// workRequests follows the order of clientId of assignment
 	}
+	wait(waitForAll(workRequests));
 
 	TraceEvent("ConsistencyCheckUrgent_SetupStart")
-	    .detail("WorkloadTitle", spec.title)
 	    .detail("TesterCount", testers.size())
-	    .detail("TestTimeout", spec.timeout)
 	    .detail("ConsistencyCheckerId", consistencyCheckerId);
-
-	state std::vector<WorkloadInterface> workloads = wait(getAll(workRequests));
-	// workloads follows the same order as workRequests
-	state double waitForFailureTime = g_network->isSimulated() ? 24 * 60 * 60 : 60;
-	if (spec.phases & TestWorkload::SETUP) {
-		state std::vector<Future<ErrorOr<Void>>> setups;
-		for (int i = 0; i < workloads.size(); i++)
-			setups.push_back(workloads[i].setup.template getReplyUnlessFailedFor<Void>(waitForFailureTime, 0));
-		wait(waitForAll(setups));
-		// setups follows the order of workRequests
-		ASSERT(workRequests.size() == setups.size());
-		for (int i = 0; i < setups.size(); i++) {
-			if (setups[i].isError()) {
-				TraceEvent("ConsistencyCheckUrgent_SetupError1")
-				    .errorUnsuppressed(setups[i].getError())
-				    .detail("WorkloadTitle", spec.title)
-				    .detail("ClientId", i)
-				    .detail("ClientCount", testers.size())
-				    .detail("TestTimeout", spec.timeout)
-				    .detail("ConsistencyCheckerId", consistencyCheckerId);
-			} else if (setups[i].get().isError()) {
-				TraceEvent("ConsistencyCheckUrgent_SetupError2")
-				    .errorUnsuppressed(setups[i].get().getError())
-				    .detail("WorkloadTitle", spec.title)
-				    .detail("ClientId", i)
-				    .detail("ClientCount", testers.size())
-				    .detail("TestTimeout", spec.timeout)
-				    .detail("ConsistencyCheckerId", consistencyCheckerId);
-			}
+	state std::vector<int> clientIds; // record the clientId for setups/starts
+	// clientIds follows the same order as setups/starts
+	state std::vector<Future<ErrorOr<Void>>> setups;
+	for (int i = 0; i < workRequests.size(); i++) {
+		ASSERT(workRequests[i].isReady());
+		if (workRequests[i].get().isError()) {
+			TraceEvent("ConsistencyCheckUrgent_FailedToContactToClient")
+			    .error(workRequests[i].get().getError())
+			    .detail("TesterCount", testers.size())
+			    .detail("TesterId", i)
+			    .detail("ConsistencyCheckerId", consistencyCheckerId);
+			continue; // ignore any failed tester
+		} else {
+			setups.push_back(
+			    workRequests[i].get().get().setup.template getReplyUnlessFailedFor<Void>(waitForFailureTime, 0));
+			clientIds.push_back(i); // same order as setups
+		}
+	}
+	wait(waitForAll(setups));
+	for (int i = 0; i < setups.size(); i++) {
+		if (setups[i].isError()) {
+			TraceEvent("ConsistencyCheckUrgent_SetupError1")
+			    .errorUnsuppressed(setups[i].getError())
+			    .detail("ClientId", clientIds[i])
+			    .detail("ClientCount", testers.size())
+			    .detail("ConsistencyCheckerId", consistencyCheckerId);
+		} else if (setups[i].get().isError()) {
+			TraceEvent("ConsistencyCheckUrgent_SetupError2")
+			    .errorUnsuppressed(setups[i].get().getError())
+			    .detail("ClientId", clientIds[i])
+			    .detail("ClientCount", testers.size())
+			    .detail("ConsistencyCheckerId", consistencyCheckerId);
 		}
-		throwIfError(setups, "ConsistencyCheckUrgent_SetupFailedForWorkload");
 	}
+	throwIfError(setups, "ConsistencyCheckUrgent_SetupFailedForWorkload");
+	// Error out if the setup fails
 
 	TraceEvent("ConsistencyCheckUrgent_SetupComplete")
-	    .detail("WorkloadTitle", spec.title)
 	    .detail("TesterCount", testers.size())
-	    .detail("TestTimeout", spec.timeout)
 	    .detail("ConsistencyCheckerId", consistencyCheckerId);
 
-	if (spec.phases & TestWorkload::EXECUTION) {
-		TraceEvent("TestStarting").detail("WorkloadTitle", spec.title);
-		printf("running test (%s)...\n", printable(spec.title).c_str());
-		state std::vector<Future<ErrorOr<Void>>> starts;
-		for (int i = 0; i < workloads.size(); i++)
-			starts.push_back(workloads[i].start.template getReplyUnlessFailedFor<Void>(waitForFailureTime, 0));
-		wait(waitForAll(starts));
-		// starts follows the order of workRequests
-		ASSERT(workRequests.size() == starts.size());
-		for (int i = 0; i < starts.size(); i++) {
-			if (starts[i].isError()) {
-				TraceEvent("ConsistencyCheckUrgent_StartError1")
-				    .errorUnsuppressed(starts[i].getError())
-				    .detail("WorkloadTitle", spec.title)
-				    .detail("ClientId", i)
-				    .detail("ClientCount", testers.size())
-				    .detail("TestTimeout", spec.timeout)
-				    .detail("ConsistencyCheckerId", consistencyCheckerId);
-			} else if (starts[i].get().isError()) {
-				TraceEvent("ConsistencyCheckUrgent_StartError2")
-				    .errorUnsuppressed(starts[i].get().getError())
-				    .detail("WorkloadTitle", spec.title)
-				    .detail("ClientId", i)
-				    .detail("ClientCount", testers.size())
-				    .detail("TestTimeout", spec.timeout)
-				    .detail("ConsistencyCheckerId", consistencyCheckerId);
-			} else {
-				completeClientIds.insert(i);
-			}
+	state std::unordered_set<int> completeClientIds;
+	clientIds.clear();
+	state std::vector<Future<ErrorOr<Void>>> starts;
+	for (int i = 0; i < workRequests.size(); i++) {
+		ASSERT(workRequests[i].isReady());
+		if (!workRequests[i].get().isError()) {
+			starts.push_back(
+			    workRequests[i].get().get().start.template getReplyUnlessFailedFor<Void>(waitForFailureTime, 0));
+			clientIds.push_back(i); // same order as starts
+		}
+	}
+	wait(waitForAll(starts));
+	for (int i = 0; i < starts.size(); i++) {
+		if (starts[i].isError()) {
+			TraceEvent("ConsistencyCheckUrgent_StartError1")
+			    .errorUnsuppressed(starts[i].getError())
+			    .detail("ClientId", clientIds[i])
+			    .detail("ClientCount", testers.size())
+			    .detail("ConsistencyCheckerId", consistencyCheckerId);
+		} else if (starts[i].get().isError()) {
+			TraceEvent("ConsistencyCheckUrgent_StartError2")
+			    .errorUnsuppressed(starts[i].get().getError())
+			    .detail("ClientId", clientIds[i])
+			    .detail("ClientCount", testers.size())
+			    .detail("ConsistencyCheckerId", consistencyCheckerId);
+		} else {
+			TraceEvent("ConsistencyCheckUrgent_ClientComplete")
+			    .detail("ClientId", clientIds[i])
+			    .detail("ClientCount", testers.size())
+			    .detail("ConsistencyCheckerId", consistencyCheckerId);
+			completeClientIds.insert(clientIds[i]);
 		}
 	}
 
-	for (int i = 0; i < workloads.size(); i++) {
-		workloads[i].stop.send(ReplyPromise<Void>());
+	for (int i = 0; i < workRequests.size(); i++) {
+		ASSERT(workRequests[i].isReady());
+		if (!workRequests[i].get().isError()) {
+			workRequests[i].get().get().stop.send(ReplyPromise<Void>());
+		}
 	}
 
 	TraceEvent("ConsistencyCheckUrgent_WorkloadEnd")
-	    .detail("WorkloadTitle", spec.title)
 	    .detail("TesterCount", testers.size())
-	    .detail("TestTimeout", spec.timeout)
 	    .detail("ConsistencyCheckerId", consistencyCheckerId);
 
 	return completeClientIds;
@@ -1196,6 +1199,7 @@ ACTOR Future<std::vector<TesterInterface>> getTesters(Reference<AsyncVar<Optiona
 	ts.reserve(workers.size());
 	for (int i = 0; i < workers.size(); i++)
 		ts.push_back(workers[i].interf.testerInterface);
+	deterministicRandom()->randomShuffle(ts);
 	return ts;
 }
 
@@ -1457,12 +1461,12 @@ ACTOR Future<Void> runConsistencyCheckerUrgentCore(Reference<AsyncVar<Optional<C
 			    .detail("TesterCount", ts.size())
 			    .detail("ShardCount", shardsToCheck.size());
 
-			// Step 4: Decide assignment of shards to testers
+			// Step 4: Assign tasks to clientId
 			assignment.clear();
 			wait(store(assignment, makeTaskAssignment(cx, consistencyCheckerId, shardsToCheck, ts.size(), round)));
 
 			// Step 5: Run checking on testers
-			std::set<int> completeClients = wait(dispatchMonitorUrgentConsistencyCheckWorkload(
+			std::unordered_set<int> completeClients = wait(dispatchMonitorUrgentConsistencyCheckWorkload(
 			    cx, ts, testSpec, defaultTenant, consistencyCheckerId, assignment));
 			if (g_network->isSimulated() && deterministicRandom()->random01() < 0.05) {
 				throw operation_failed(); // Introduce random failure

From f2cd454eb83921a4eecd053446ad05742a49b799 Mon Sep 17 00:00:00 2001
From: Zhe Wang <zhe.wang@wustl.edu>
Date: Wed, 3 Jan 2024 19:30:54 -0800
Subject: [PATCH 362/468] nits

---
 fdbserver/tester.actor.cpp | 44 ++++++++++++++++++++++++--------------
 1 file changed, 28 insertions(+), 16 deletions(-)

diff --git a/fdbserver/tester.actor.cpp b/fdbserver/tester.actor.cpp
index c35b2f06a49..ce4b33bf72f 100644
--- a/fdbserver/tester.actor.cpp
+++ b/fdbserver/tester.actor.cpp
@@ -991,7 +991,7 @@ ACTOR Future<Void> checkConsistency(Database cx,
 	}
 }
 
-ACTOR Future<std::unordered_set<int>> dispatchMonitorUrgentConsistencyCheckWorkload(
+ACTOR Future<std::unordered_set<int>> runUrgentConsistencyCheckWorkload(
     Database cx,
     std::vector<TesterInterface> testers,
     TestSpec spec,
@@ -1001,9 +1001,9 @@ ACTOR Future<std::unordered_set<int>> dispatchMonitorUrgentConsistencyCheckWorkl
 	TraceEvent("ConsistencyCheckUrgent_Dispatch")
 	    .detail("TesterCount", testers.size())
 	    .detail("ConsistencyCheckerId", consistencyCheckerId);
-
 	state double waitForFailureTime = g_network->isSimulated() ? 24 * 60 * 60 : 60;
 
+	// Step 1: Get interfaces for running workloads
 	state std::vector<Future<ErrorOr<WorkloadInterface>>> workRequests;
 	for (int i = 0; i < testers.size(); i++) {
 		WorkloadRequest req;
@@ -1021,12 +1021,13 @@ ACTOR Future<std::unordered_set<int>> dispatchMonitorUrgentConsistencyCheckWorkl
 		} else {
 			req.rangesToCheck = Optional<std::vector<KeyRange>>();
 		}
-		workRequests.push_back(errorOr(timeoutError(testers[i].recruitments.getReply(req), waitForFailureTime)));
+		workRequests.push_back(testers[i].recruitments.getReplyUnlessFailedFor(req, waitForFailureTime, 0));
 		// workRequests follows the order of clientId of assignment
 	}
 	wait(waitForAll(workRequests));
 
-	TraceEvent("ConsistencyCheckUrgent_SetupStart")
+	// Step 2: Setup workloads via the interfaces
+	TraceEvent("ConsistencyCheckUrgent_SetupWorkloads")
 	    .detail("TesterCount", testers.size())
 	    .detail("ConsistencyCheckerId", consistencyCheckerId);
 	state std::vector<int> clientIds; // record the clientId for setups/starts
@@ -1050,26 +1051,36 @@ ACTOR Future<std::unordered_set<int>> dispatchMonitorUrgentConsistencyCheckWorkl
 	wait(waitForAll(setups));
 	for (int i = 0; i < setups.size(); i++) {
 		if (setups[i].isError()) {
-			TraceEvent("ConsistencyCheckUrgent_SetupError1")
+			TraceEvent("ConsistencyCheckUrgent_SetupWorkloadError1")
 			    .errorUnsuppressed(setups[i].getError())
 			    .detail("ClientId", clientIds[i])
 			    .detail("ClientCount", testers.size())
 			    .detail("ConsistencyCheckerId", consistencyCheckerId);
 		} else if (setups[i].get().isError()) {
-			TraceEvent("ConsistencyCheckUrgent_SetupError2")
+			TraceEvent("ConsistencyCheckUrgent_SetupWorkloadError2")
 			    .errorUnsuppressed(setups[i].get().getError())
 			    .detail("ClientId", clientIds[i])
 			    .detail("ClientCount", testers.size())
 			    .detail("ConsistencyCheckerId", consistencyCheckerId);
 		}
 	}
-	throwIfError(setups, "ConsistencyCheckUrgent_SetupFailedForWorkload");
-	// Error out if the setup fails
+	try {
+		throwIfError(setups, "ConsistencyCheckUrgent_SetupWorkloadFailed");
+	} catch (Error& e) {
+		// Give up this round if any setup failed
+		for (int i = 0; i < workRequests.size(); i++) {
+			ASSERT(workRequests[i].isReady());
+			if (!workRequests[i].get().isError()) {
+				workRequests[i].get().get().stop.send(ReplyPromise<Void>());
+			}
+		}
+		throw e;
+	}
 
-	TraceEvent("ConsistencyCheckUrgent_SetupComplete")
+	// Step 3: Run workloads via the interfaces
+	TraceEvent("ConsistencyCheckUrgent_RunWorkloads")
 	    .detail("TesterCount", testers.size())
 	    .detail("ConsistencyCheckerId", consistencyCheckerId);
-
 	state std::unordered_set<int> completeClientIds;
 	clientIds.clear();
 	state std::vector<Future<ErrorOr<Void>>> starts;
@@ -1084,26 +1095,27 @@ ACTOR Future<std::unordered_set<int>> dispatchMonitorUrgentConsistencyCheckWorkl
 	wait(waitForAll(starts));
 	for (int i = 0; i < starts.size(); i++) {
 		if (starts[i].isError()) {
-			TraceEvent("ConsistencyCheckUrgent_StartError1")
+			TraceEvent("ConsistencyCheckUrgent_RunWorkloadError1")
 			    .errorUnsuppressed(starts[i].getError())
 			    .detail("ClientId", clientIds[i])
 			    .detail("ClientCount", testers.size())
 			    .detail("ConsistencyCheckerId", consistencyCheckerId);
 		} else if (starts[i].get().isError()) {
-			TraceEvent("ConsistencyCheckUrgent_StartError2")
+			TraceEvent("ConsistencyCheckUrgent_RunWorkloadError2")
 			    .errorUnsuppressed(starts[i].get().getError())
 			    .detail("ClientId", clientIds[i])
 			    .detail("ClientCount", testers.size())
 			    .detail("ConsistencyCheckerId", consistencyCheckerId);
 		} else {
-			TraceEvent("ConsistencyCheckUrgent_ClientComplete")
+			TraceEvent("ConsistencyCheckUrgent_RunWorkloadComplete")
 			    .detail("ClientId", clientIds[i])
 			    .detail("ClientCount", testers.size())
 			    .detail("ConsistencyCheckerId", consistencyCheckerId);
-			completeClientIds.insert(clientIds[i]);
+			completeClientIds.insert(clientIds[i]); // Add complete clients
 		}
 	}
 
+	// Step 4: Stop workloads via the interfaces
 	for (int i = 0; i < workRequests.size(); i++) {
 		ASSERT(workRequests[i].isReady());
 		if (!workRequests[i].get().isError()) {
@@ -1111,7 +1123,7 @@ ACTOR Future<std::unordered_set<int>> dispatchMonitorUrgentConsistencyCheckWorkl
 		}
 	}
 
-	TraceEvent("ConsistencyCheckUrgent_WorkloadEnd")
+	TraceEvent("ConsistencyCheckUrgent_DispatchEnd")
 	    .detail("TesterCount", testers.size())
 	    .detail("ConsistencyCheckerId", consistencyCheckerId);
 
@@ -1466,7 +1478,7 @@ ACTOR Future<Void> runConsistencyCheckerUrgentCore(Reference<AsyncVar<Optional<C
 			wait(store(assignment, makeTaskAssignment(cx, consistencyCheckerId, shardsToCheck, ts.size(), round)));
 
 			// Step 5: Run checking on testers
-			std::unordered_set<int> completeClients = wait(dispatchMonitorUrgentConsistencyCheckWorkload(
+			std::unordered_set<int> completeClients = wait(runUrgentConsistencyCheckWorkload(
 			    cx, ts, testSpec, defaultTenant, consistencyCheckerId, assignment));
 			if (g_network->isSimulated() && deterministicRandom()->random01() < 0.05) {
 				throw operation_failed(); // Introduce random failure

From 9311ad04415e8c7a2bebd7cd5615b9d0ad84c1ce Mon Sep 17 00:00:00 2001
From: Zhe Wang <zhe.wang@wustl.edu>
Date: Wed, 3 Jan 2024 22:18:35 -0800
Subject: [PATCH 363/468] fmt

---
 fdbserver/tester.actor.cpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/fdbserver/tester.actor.cpp b/fdbserver/tester.actor.cpp
index ce4b33bf72f..8f60c4102ec 100644
--- a/fdbserver/tester.actor.cpp
+++ b/fdbserver/tester.actor.cpp
@@ -1478,8 +1478,8 @@ ACTOR Future<Void> runConsistencyCheckerUrgentCore(Reference<AsyncVar<Optional<C
 			wait(store(assignment, makeTaskAssignment(cx, consistencyCheckerId, shardsToCheck, ts.size(), round)));
 
 			// Step 5: Run checking on testers
-			std::unordered_set<int> completeClients = wait(runUrgentConsistencyCheckWorkload(
-			    cx, ts, testSpec, defaultTenant, consistencyCheckerId, assignment));
+			std::unordered_set<int> completeClients = wait(
+			    runUrgentConsistencyCheckWorkload(cx, ts, testSpec, defaultTenant, consistencyCheckerId, assignment));
 			if (g_network->isSimulated() && deterministicRandom()->random01() < 0.05) {
 				throw operation_failed(); // Introduce random failure
 			}

From 82c59601233704cae886f1336f2c32ee666575c5 Mon Sep 17 00:00:00 2001
From: FoundationDB CI <foundationdb_ci@apple.com>
Date: Fri, 5 Jan 2024 00:29:14 +0000
Subject: [PATCH 364/468] disable AVX for 7.1.50 release

---
 cmake/ConfigureCompiler.cmake | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cmake/ConfigureCompiler.cmake b/cmake/ConfigureCompiler.cmake
index e52d8665ee8..7e50469ba6a 100644
--- a/cmake/ConfigureCompiler.cmake
+++ b/cmake/ConfigureCompiler.cmake
@@ -253,7 +253,7 @@ else()
       set(USE_AVX512F OFF)
     endif()
   endif()
-  set(USE_AVX ON CACHE BOOL "Enable AVX instructions")
+  set(USE_AVX OFF CACHE BOOL "Enable AVX instructions")
   if (USE_AVX)
     if (CMAKE_HOST_SYSTEM_PROCESSOR MATCHES "^x86")
       add_compile_options(-mavx)

From 778e31b6a27ebd7f418c5405682737d59f1a4496 Mon Sep 17 00:00:00 2001
From: FoundationDB CI <foundationdb_ci@apple.com>
Date: Fri, 5 Jan 2024 00:29:27 +0000
Subject: [PATCH 365/468] enable AVX and update version for 7.1.51 release

---
 CMakeLists.txt                | 2 +-
 cmake/ConfigureCompiler.cmake | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index ca79fc4323b..21d3b11d5b7 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -23,7 +23,7 @@ else()
 endif()
 
 project(foundationdb
-  VERSION 7.1.50
+  VERSION 7.1.51
   DESCRIPTION "FoundationDB is a scalable, fault-tolerant, ordered key-value store with full ACID transactions."
   HOMEPAGE_URL "http://www.foundationdb.org/"
   LANGUAGES C CXX ASM)
diff --git a/cmake/ConfigureCompiler.cmake b/cmake/ConfigureCompiler.cmake
index 7e50469ba6a..e52d8665ee8 100644
--- a/cmake/ConfigureCompiler.cmake
+++ b/cmake/ConfigureCompiler.cmake
@@ -253,7 +253,7 @@ else()
       set(USE_AVX512F OFF)
     endif()
   endif()
-  set(USE_AVX OFF CACHE BOOL "Enable AVX instructions")
+  set(USE_AVX ON CACHE BOOL "Enable AVX instructions")
   if (USE_AVX)
     if (CMAKE_HOST_SYSTEM_PROCESSOR MATCHES "^x86")
       add_compile_options(-mavx)

From ddaa2fd3d10c3a946e44e1571d195d874f3ded4d Mon Sep 17 00:00:00 2001
From: FoundationDB CI <foundationdb_ci@apple.com>
Date: Fri, 5 Jan 2024 08:57:14 +0000
Subject: [PATCH 366/468] update version after 7.1.51 release

---
 CMakeLists.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 21d3b11d5b7..57bfe402545 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -23,7 +23,7 @@ else()
 endif()
 
 project(foundationdb
-  VERSION 7.1.51
+  VERSION 7.1.52
   DESCRIPTION "FoundationDB is a scalable, fault-tolerant, ordered key-value store with full ACID transactions."
   HOMEPAGE_URL "http://www.foundationdb.org/"
   LANGUAGES C CXX ASM)

From 584e4ad06497b03378632b5963c9aaff6ee4565c Mon Sep 17 00:00:00 2001
From: Zhe Wang <zhe.wang@wustl.edu>
Date: Sat, 13 Jan 2024 20:40:54 -0800
Subject: [PATCH 367/468] consistency checker urgent should tolerate busy
 restart loop of the checker

---
 fdbclient/ConsistencyCheckUtil.actor.cpp      | 12 ++---
 fdbserver/tester.actor.cpp                    | 53 +++++++++++++++++--
 .../workloads/ConsistencyCheck.actor.cpp      | 26 +++++----
 3 files changed, 73 insertions(+), 18 deletions(-)

diff --git a/fdbclient/ConsistencyCheckUtil.actor.cpp b/fdbclient/ConsistencyCheckUtil.actor.cpp
index 1b9f9d8a91d..3a5ecacea00 100644
--- a/fdbclient/ConsistencyCheckUtil.actor.cpp
+++ b/fdbclient/ConsistencyCheckUtil.actor.cpp
@@ -310,7 +310,7 @@ Key getKeyFromString(const std::string& str) {
 		return emptyKey;
 	}
 	if (str.size() % 4 != 0) {
-		TraceEvent(SevWarnAlways, "ConsistencyCheck_GetKeyFromStringError")
+		TraceEvent(SevWarnAlways, "ConsistencyCheckUrgent_GetKeyFromStringError")
 		    .setMaxEventLength(-1)
 		    .setMaxFieldLength(-1)
 		    .detail("Reason", "WrongLength")
@@ -320,7 +320,7 @@ Key getKeyFromString(const std::string& str) {
 	std::vector<uint8_t> byteList;
 	for (int i = 0; i < str.size(); i += 4) {
 		if (str.at(i + 0) != '\\' || str.at(i + 1) != 'x') {
-			TraceEvent(SevWarnAlways, "ConsistencyCheck_GetKeyFromStringError")
+			TraceEvent(SevWarnAlways, "ConsistencyCheckUrgent_GetKeyFromStringError")
 			    .setMaxEventLength(-1)
 			    .setMaxFieldLength(-1)
 			    .detail("Reason", "WrongBytePrefix")
@@ -330,7 +330,7 @@ Key getKeyFromString(const std::string& str) {
 		const char first = str.at(i + 2);
 		const char second = str.at(i + 3);
 		if (parseCharMap.count(first) == 0 || parseCharMap.count(second) == 0) {
-			TraceEvent(SevWarnAlways, "ConsistencyCheck_GetKeyFromStringError")
+			TraceEvent(SevWarnAlways, "ConsistencyCheckUrgent_GetKeyFromStringError")
 			    .setMaxEventLength(-1)
 			    .setMaxFieldLength(-1)
 			    .detail("Reason", "WrongByteContent")
@@ -403,7 +403,7 @@ std::vector<KeyRange> loadRangesToCheckFromKnob() {
 			rangeBegin = allKeys.end;
 		}
 		if (rangeEnd > allKeys.end) {
-			TraceEvent("ConsistencyCheck_ReverseInputRange")
+			TraceEvent("ConsistencyCheckUrgent_ReverseInputRange")
 			    .setMaxEventLength(-1)
 			    .setMaxFieldLength(-1)
 			    .detail("Index", i)
@@ -418,7 +418,7 @@ std::vector<KeyRange> loadRangesToCheckFromKnob() {
 		} else if (rangeBegin > rangeEnd) {
 			rangeToCheck = Standalone(KeyRangeRef(rangeEnd, rangeBegin));
 		} else {
-			TraceEvent("ConsistencyCheck_EmptyInputRange")
+			TraceEvent("ConsistencyCheckUrgent_EmptyInputRange")
 			    .setMaxEventLength(-1)
 			    .setMaxFieldLength(-1)
 			    .detail("Index", i)
@@ -437,7 +437,7 @@ std::vector<KeyRange> loadRangesToCheckFromKnob() {
 			res.push_back(rangeToCheck.range());
 		}
 	}
-	TraceEvent e("ConsistencyCheck_LoadedInputRange");
+	TraceEvent e("ConsistencyCheckUrgent_LoadedInputRange");
 	e.setMaxEventLength(-1);
 	e.setMaxFieldLength(-1);
 	for (int i = 0; i < res.size(); i++) {
diff --git a/fdbserver/tester.actor.cpp b/fdbserver/tester.actor.cpp
index 8f60c4102ec..7aa7da02857 100644
--- a/fdbserver/tester.actor.cpp
+++ b/fdbserver/tester.actor.cpp
@@ -710,11 +710,45 @@ ACTOR Future<Void> testerServerCore(TesterInterface interf,
 	state PromiseStream<Future<Void>> addWorkload;
 	state Future<Void> workerFatalError = actorCollection(addWorkload.getFuture());
 
-	TraceEvent("StartingTesterServerCore", interf.id());
+	// Dedicated to consistencyCheckerUrgent
+	// At any time, we only allow at most 1 consistency checker workload on a server
+	state std::pair<int64_t, Future<Void>> consistencyCheckerUrgentTester = std::make_pair(0, Future<Void>());
+
+	TraceEvent(SevInfo, "StartingTesterServerCore", interf.id());
 	loop choose {
 		when(wait(workerFatalError)) {}
+		when(wait(consistencyCheckerUrgentTester.second.isValid() ? consistencyCheckerUrgentTester.second : Never())) {
+			ASSERT(consistencyCheckerUrgentTester.first != 0);
+			TraceEvent(SevInfo, "ConsistencyCheckUrgent_ServerWorkloadEnd", interf.id())
+			    .detail("ConsistencyCheckerId", consistencyCheckerUrgentTester.first);
+			consistencyCheckerUrgentTester = std::make_pair(0, Future<Void>()); // reset
+		}
 		when(WorkloadRequest work = waitNext(interf.recruitments.getFuture())) {
-			addWorkload.send(testerServerWorkload(work, ccr, dbInfo, locality));
+			if (work.sharedRandomNumber > SERVER_KNOBS->CONSISTENCY_CHECK_ID_MIN &&
+			    work.sharedRandomNumber < SERVER_KNOBS->CONSISTENCY_CHECK_ID_MAX_PLUS_ONE) {
+				// The workload is a consistency checker urgent workload
+				if (work.sharedRandomNumber == consistencyCheckerUrgentTester.first) {
+					TraceEvent(SevInfo, "ConsistencyCheckUrgent_ServerDuplicatedRequest", interf.id())
+					    .detail("ConsistencyCheckerId", work.sharedRandomNumber)
+					    .detail("ClientId", work.clientId)
+					    .detail("ClientCount", work.clientCount);
+				} else if (consistencyCheckerUrgentTester.second.isValid() &&
+				           !consistencyCheckerUrgentTester.second.isReady()) {
+					TraceEvent(SevWarnAlways, "ConsistencyCheckUrgent_ServerConflict", interf.id())
+					    .detail("ExistingConsistencyCheckerId", consistencyCheckerUrgentTester.first)
+					    .detail("ArrivingConsistencyCheckerId", work.sharedRandomNumber)
+					    .detail("ClientId", work.clientId)
+					    .detail("ClientCount", work.clientCount);
+				}
+				consistencyCheckerUrgentTester =
+				    std::make_pair(work.sharedRandomNumber, testerServerWorkload(work, ccr, dbInfo, locality));
+				TraceEvent(SevInfo, "ConsistencyCheckUrgent_ServerWorkloadStart", interf.id())
+				    .detail("ConsistencyCheckerId", consistencyCheckerUrgentTester.first)
+				    .detail("ClientId", work.clientId)
+				    .detail("ClientCount", work.clientCount);
+			} else {
+				addWorkload.send(testerServerWorkload(work, ccr, dbInfo, locality));
+			}
 		}
 	}
 }
@@ -1036,8 +1070,9 @@ ACTOR Future<std::unordered_set<int>> runUrgentConsistencyCheckWorkload(
 	for (int i = 0; i < workRequests.size(); i++) {
 		ASSERT(workRequests[i].isReady());
 		if (workRequests[i].get().isError()) {
+			Error e = workRequests[i].get().getError();
 			TraceEvent("ConsistencyCheckUrgent_FailedToContactToClient")
-			    .error(workRequests[i].get().getError())
+			    .error(e)
 			    .detail("TesterCount", testers.size())
 			    .detail("TesterId", i)
 			    .detail("ConsistencyCheckerId", consistencyCheckerId);
@@ -1119,7 +1154,19 @@ ACTOR Future<std::unordered_set<int>> runUrgentConsistencyCheckWorkload(
 	for (int i = 0; i < workRequests.size(); i++) {
 		ASSERT(workRequests[i].isReady());
 		if (!workRequests[i].get().isError()) {
+			TraceEvent("ConsistencyCheckUrgent_RunWorkloadStopSignal")
+			    .detail("State", "Succeed")
+			    .detail("ClientId", i)
+			    .detail("ClientCount", testers.size())
+			    .detail("ConsistencyCheckerId", consistencyCheckerId);
 			workRequests[i].get().get().stop.send(ReplyPromise<Void>());
+			// This signal is not reliable but acceptable
+		} else {
+			TraceEvent("ConsistencyCheckUrgent_RunWorkloadStopSignal")
+			    .detail("State", "Not issue since the interface is failed to fetch")
+			    .detail("ClientId", i)
+			    .detail("ClientCount", testers.size())
+			    .detail("ConsistencyCheckerId", consistencyCheckerId);
 		}
 	}
 
diff --git a/fdbserver/workloads/ConsistencyCheck.actor.cpp b/fdbserver/workloads/ConsistencyCheck.actor.cpp
index 54417d2e9a3..2ccb2a92867 100644
--- a/fdbserver/workloads/ConsistencyCheck.actor.cpp
+++ b/fdbserver/workloads/ConsistencyCheck.actor.cpp
@@ -198,8 +198,7 @@ struct ConsistencyCheckWorkload : TestWorkload {
 	}
 
 	ACTOR Future<Void> _startUrgent(Database cx, ConsistencyCheckWorkload* self) {
-		TraceEvent("ConsistencyCheck_Start")
-		    .detail("Mode", "Urgent")
+		TraceEvent("ConsistencyCheckUrgent_Start")
 		    .detail("ConsistencyCheckerId", self->consistencyCheckerId)
 		    .detail("Distributed", self->distributed)
 		    .detail("ClientCount", self->clientCount)
@@ -207,8 +206,7 @@ struct ConsistencyCheckWorkload : TestWorkload {
 		    .detail("Indefinite", self->indefinite)
 		    .detail("Repetitions", self->repetitions);
 		wait(self->runUrgentCheck(cx, self));
-		TraceEvent("ConsistencyCheck_Exit")
-		    .detail("Mode", "Urgent")
+		TraceEvent("ConsistencyCheckUrgent_Exit")
 		    .detail("ConsistencyCheckerId", self->consistencyCheckerId)
 		    .detail("Distributed", self->distributed)
 		    .detail("ClientCount", self->clientCount)
@@ -222,7 +220,6 @@ struct ConsistencyCheckWorkload : TestWorkload {
 		loop {
 			while (self->suspendConsistencyCheck.get()) {
 				TraceEvent("ConsistencyCheck_Suspended")
-				    .detail("Mode", "Normal")
 				    .detail("ConsistencyCheckerId", self->consistencyCheckerId)
 				    .detail("Distributed", self->distributed)
 				    .detail("ClientCount", self->clientCount)
@@ -233,7 +230,6 @@ struct ConsistencyCheckWorkload : TestWorkload {
 				wait(self->suspendConsistencyCheck.onChange());
 			}
 			TraceEvent("ConsistencyCheck_StartingOrResuming")
-			    .detail("Mode", "Normal")
 			    .detail("ConsistencyCheckerId", self->consistencyCheckerId)
 			    .detail("Distributed", self->distributed)
 			    .detail("ClientCount", self->clientCount)
@@ -1582,6 +1578,8 @@ struct ConsistencyCheckWorkload : TestWorkload {
 
 			// Step 3: Read a limited number of entries at a time, repeating until all keys in the shard have been read
 			state int64_t totalReadAmount = 0;
+			state int64_t shardReadAmount = 0;
+			state int64_t shardKeyCompared = 0;
 			state bool valueAvailableToCheck = true;
 			state KeySelector begin = firstGreaterOrEqual(range.begin);
 			loop {
@@ -1617,8 +1615,10 @@ struct ConsistencyCheckWorkload : TestWorkload {
 							if (rangeResult.present()) {
 								e.detail("ErrorPresent", rangeResult.get().error.present());
 								if (rangeResult.get().error.present()) {
-									e.detail("Error", rangeResult.get().error.get().name());
+									e.detail("Error", rangeResult.get().error.get().what());
 								}
+							} else {
+								e.detail("ResultNotPresentWithError", rangeResult.getError().what());
 							}
 							break;
 						}
@@ -1650,6 +1650,8 @@ struct ConsistencyCheckWorkload : TestWorkload {
 						// to compare against
 						if (firstValidServer == -1) {
 							firstValidServer = j;
+							GetKeyValuesReply reference = keyValueFutures[firstValidServer].get().get();
+							shardKeyCompared += current.data.size();
 						} else {
 							// Compare this shard against the first
 							GetKeyValuesReply reference = keyValueFutures[firstValidServer].get().get();
@@ -1736,6 +1738,8 @@ struct ConsistencyCheckWorkload : TestWorkload {
 					// RateKeeping
 					wait(rateLimiter->getAllowance(totalReadAmount));
 
+					shardReadAmount += totalReadAmount;
+
 					// Advance to the next set of entries
 					ASSERT(firstValidServer != -1);
 					if (keyValueFutures[firstValidServer].get().get().more) {
@@ -1773,7 +1777,9 @@ struct ConsistencyCheckWorkload : TestWorkload {
 				    .detail("ShardBegin", range.begin)
 				    .detail("ShardEnd", range.end)
 				    .detail("ReplicaCount", storageServers.size())
-				    .detail("ConsistencyCheckEpoch", consistencyCheckEpoch);
+				    .detail("ConsistencyCheckEpoch", consistencyCheckEpoch)
+				    .detail("ShardBytesRead", shardReadAmount)
+				    .detail("ShardKeysCompared", shardKeyCompared);
 			} else {
 				numCompleteShards++;
 				if (!self->rangesToCheck.present()) { // In case we are able to persist progress
@@ -1796,7 +1802,9 @@ struct ConsistencyCheckWorkload : TestWorkload {
 				    .detail("NumFailedShards", numFailedShards)
 				    .detail("NumShardThisClient", numShardThisClient)
 				    .detail("NumShardToCheckThisEpoch", numShardToCheck - 1)
-				    .detail("ConsistencyCheckEpoch", consistencyCheckEpoch);
+				    .detail("ConsistencyCheckEpoch", consistencyCheckEpoch)
+				    .detail("ShardBytesRead", shardReadAmount)
+				    .detail("ShardKeysCompared", shardKeyCompared);
 			}
 		}
 

From 7e9594f0ea3dfd4d58750e3159c41f48e6f7762f Mon Sep 17 00:00:00 2001
From: Zhe Wang <zhe.wang@wustl.edu>
Date: Tue, 16 Jan 2024 13:47:38 -0800
Subject: [PATCH 368/468] fix a simulation failure due to setup error

---
 fdbserver/tester.actor.cpp | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/fdbserver/tester.actor.cpp b/fdbserver/tester.actor.cpp
index 7aa7da02857..9d1f8ade3fc 100644
--- a/fdbserver/tester.actor.cpp
+++ b/fdbserver/tester.actor.cpp
@@ -1100,8 +1100,15 @@ ACTOR Future<std::unordered_set<int>> runUrgentConsistencyCheckWorkload(
 		}
 	}
 	try {
-		throwIfError(setups, "ConsistencyCheckUrgent_SetupWorkloadFailed");
+		for (auto& setup : setups) {
+			if (setup.isError()) {
+				throw setup.getError();
+			} else if (setup.get().isError()) {
+				throw setup.get().getError();
+			}
+		}
 	} catch (Error& e) {
+		TraceEvent(SevWarn, "ConsistencyCheckUrgent_SetupWorkloadFailed").error(e);
 		// Give up this round if any setup failed
 		for (int i = 0; i < workRequests.size(); i++) {
 			ASSERT(workRequests[i].isReady());

From f5b190ef318e615cc8112816636dfff4ca7cd078 Mon Sep 17 00:00:00 2001
From: FoundationDB CI <foundationdb_ci@apple.com>
Date: Wed, 17 Jan 2024 13:57:29 +0000
Subject: [PATCH 369/468] disable AVX for 7.1.52 release

---
 cmake/ConfigureCompiler.cmake | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cmake/ConfigureCompiler.cmake b/cmake/ConfigureCompiler.cmake
index e52d8665ee8..7e50469ba6a 100644
--- a/cmake/ConfigureCompiler.cmake
+++ b/cmake/ConfigureCompiler.cmake
@@ -253,7 +253,7 @@ else()
       set(USE_AVX512F OFF)
     endif()
   endif()
-  set(USE_AVX ON CACHE BOOL "Enable AVX instructions")
+  set(USE_AVX OFF CACHE BOOL "Enable AVX instructions")
   if (USE_AVX)
     if (CMAKE_HOST_SYSTEM_PROCESSOR MATCHES "^x86")
       add_compile_options(-mavx)

From d69857d5bd033fd4262608db4b3f0d5f9d88dd5f Mon Sep 17 00:00:00 2001
From: FoundationDB CI <foundationdb_ci@apple.com>
Date: Wed, 17 Jan 2024 13:58:04 +0000
Subject: [PATCH 370/468] enable AVX and update version for 7.1.53 release

---
 CMakeLists.txt                | 2 +-
 cmake/ConfigureCompiler.cmake | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 57bfe402545..3cf8e36126e 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -23,7 +23,7 @@ else()
 endif()
 
 project(foundationdb
-  VERSION 7.1.52
+  VERSION 7.1.53
   DESCRIPTION "FoundationDB is a scalable, fault-tolerant, ordered key-value store with full ACID transactions."
   HOMEPAGE_URL "http://www.foundationdb.org/"
   LANGUAGES C CXX ASM)
diff --git a/cmake/ConfigureCompiler.cmake b/cmake/ConfigureCompiler.cmake
index 7e50469ba6a..e52d8665ee8 100644
--- a/cmake/ConfigureCompiler.cmake
+++ b/cmake/ConfigureCompiler.cmake
@@ -253,7 +253,7 @@ else()
       set(USE_AVX512F OFF)
     endif()
   endif()
-  set(USE_AVX OFF CACHE BOOL "Enable AVX instructions")
+  set(USE_AVX ON CACHE BOOL "Enable AVX instructions")
   if (USE_AVX)
     if (CMAKE_HOST_SYSTEM_PROCESSOR MATCHES "^x86")
       add_compile_options(-mavx)

From 7c63eedb2f34ffcb048ae4c39dee607e5dd13b45 Mon Sep 17 00:00:00 2001
From: FoundationDB CI <foundationdb_ci@apple.com>
Date: Wed, 17 Jan 2024 20:05:06 +0000
Subject: [PATCH 371/468] update version after 7.1.53 release

---
 CMakeLists.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 3cf8e36126e..5eda1f4ff9a 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -23,7 +23,7 @@ else()
 endif()
 
 project(foundationdb
-  VERSION 7.1.53
+  VERSION 7.1.54
   DESCRIPTION "FoundationDB is a scalable, fault-tolerant, ordered key-value store with full ACID transactions."
   HOMEPAGE_URL "http://www.foundationdb.org/"
   LANGUAGES C CXX ASM)

From 19b0f923b77a50299b070e90aae4e307401f4145 Mon Sep 17 00:00:00 2001
From: Zhe Wang <zhe.wang@wustl.edu>
Date: Thu, 18 Jan 2024 13:26:11 -0800
Subject: [PATCH 372/468] fix check all command

---
 fdbcli/DebugCommands.actor.cpp | 25 ++++++++++++-------------
 1 file changed, 12 insertions(+), 13 deletions(-)

diff --git a/fdbcli/DebugCommands.actor.cpp b/fdbcli/DebugCommands.actor.cpp
index 30cc3b179ea..93090b143cc 100644
--- a/fdbcli/DebugCommands.actor.cpp
+++ b/fdbcli/DebugCommands.actor.cpp
@@ -279,21 +279,20 @@ ACTOR Future<bool> checkallCommandActor(Database cx, std::vector<StringRef> toke
 	state bool checkAll = false; // do not return on first error, continue checking all keys
 	state KeyRange toCheck = backupLogKeys;
 
-	if (tokens.size() == 2 && tokens[1] == "ALL"_sr) {
-		printf("Checking all keys for corruption...\n");
-		checkAll = true;
-	}
-	if (tokens.size() == 2 && tokens[1] == "help"_sr) {
-		printf("checkall [ALL]|[<KEY> <KEY2>]\n"
-		       "Check inconsistency in a keyspace by comparing all replicas and print any corruptions.\n"
-		       "The default behavior is to stop on the first shard where corruption is found and the\n"
-		       "default keyspace is \\xff\\x02/blog/.\n"
-		       "The default keyspace can be changed by specifying a range. Note this is intended to check\n"
-		       "a small range of keys, not the entire database (consider consistencycheck for that purpose).\n");
-		return false;
-	}
 	if (tokens.size() == 3) {
 		toCheck = KeyRangeRef(tokens[1], tokens[2]);
+	} else if (tokens.size() == 4 && tokens[3] == "all"_sr) {
+		toCheck = KeyRangeRef(tokens[1], tokens[2]);
+		checkAll = true;
+	} else {
+		printf(
+		    "checkall [<KEY> <KEY2>] (all)\n"
+		    "Check inconsistency of the input range by comparing all replicas and print any corruptions.\n"
+		    "The default behavior is to stop on the first subrange where corruption is found\n"
+		    "`all` is optional. When `all` is appended, the checker does not stop until all subranges have checked.\n"
+		    "Note this is intended to check a small range of keys, not the entire database (consider consistencycheck "
+		    "for that purpose).\n");
+		return false;
 	}
 
 	loop {

From a2a56d1edf334e41c6487a1e3dc30c3feb02fa52 Mon Sep 17 00:00:00 2001
From: Zhe Wang <zhe.wang@wustl.edu>
Date: Fri, 19 Jan 2024 17:51:04 -0800
Subject: [PATCH 373/468] fix-checkall-false-alarm-bug

---
 fdbcli/DebugCommands.actor.cpp | 215 +++++++++++++++++++--------------
 1 file changed, 121 insertions(+), 94 deletions(-)

diff --git a/fdbcli/DebugCommands.actor.cpp b/fdbcli/DebugCommands.actor.cpp
index 93090b143cc..8228bd31dfc 100644
--- a/fdbcli/DebugCommands.actor.cpp
+++ b/fdbcli/DebugCommands.actor.cpp
@@ -164,125 +164,133 @@ ACTOR Future<bool> getallCommandActor(Database cx, std::vector<StringRef> tokens
 CommandFactory getallCommandFactory("getall");
 
 // check that all replies are the same. Update begin to the next key to check
-std::pair<bool, bool> checkResults(Version version,
-                                   const std::vector<StorageServerInterface>& servers,
-                                   const std::vector<Future<ErrorOr<GetKeyValuesReply>>>& replies,
-                                   KeySelectorRef& begin,
-                                   const KeySelectorRef& end,
-                                   Key& lastEnd) {
-	bool allSame = true;
-	int firstValidServer = -1;
+// checkResults keeps invariants:
+// (1) hasMore = true if any server has more data not read yet
+// (2) nextBeginKey is the minimal key returned from all servers
+// (3) checkResults reports inconsistency of keys only before the nextBeginKey if hasMore=true
+// Therefore, whether to proceed to the next round depends on hasMore
+// If there is a next round, it starts from the minimal key returned from all servers
+std::tuple<bool, bool, Key> checkResults(Version version,
+                                         const std::vector<StorageServerInterface>& servers,
+                                         const std::vector<Future<ErrorOr<GetKeyValuesReply>>>& replies) {
+	// Decide comparison scope
+	Key claimEndKey; // used for the next round exists
+	bool hasMore = false;
 	for (int j = 0; j < replies.size(); j++) {
 		auto reply = replies[j].get();
-		if (!reply.present() || reply.get().error.present()) {
-			printf("Error from %s: %s\n",
-			       servers[j].address().toString().c_str(),
-			       reply.present() ? reply.get().error.get().what() : "no reply");
-			continue;
+		if (reply.isError()) {
+			printf("checkResults error: %s\n", reply.getError().what());
+			throw reply.getError();
+		} else if (reply.get().error.present()) {
+			printf("checkResults error: %s\n", reply.get().error.get().what());
+			throw reply.get().error.get();
 		}
 		GetKeyValuesReply current = reply.get();
+		if (current.data.size() == 0) {
+			continue; // Ignore if no data has replied
+		}
+		if (claimEndKey.empty() || current.data[current.data.size() - 1].key < claimEndKey) {
+			claimEndKey = current.data[current.data.size() - 1].key;
+		}
+		hasMore = hasMore || current.more;
+	}
+	ASSERT(!claimEndKey.empty());
+
+	// Compare servers
+	bool allSame = true;
+	int firstValidServer = -1;
+	for (int j = 0; j < replies.size(); j++) {
+		auto reply = replies[j].get();
+		ASSERT(reply.present() && !reply.get().error.present()); // has thrown eariler of error
 		if (firstValidServer == -1) {
 			firstValidServer = j;
-			continue;
+			continue; // always select the first server as reference
 		}
+		// compare reference and current
+		GetKeyValuesReply current = reply.get();
 		GetKeyValuesReply reference = replies[firstValidServer].get().get();
-
-		// compare the two replicas
 		if (current.data == reference.data && current.more == reference.more) {
 			continue;
 		}
 
+		// Detecting corrupted keys for any mismatching replies between current and reference servers
 		allSame = false;
-		printf("#%d  server: %s  key count: %lu, cached: %d, more: %d\n",
-		       firstValidServer,
-		       servers[firstValidServer].address().toString().c_str(),
-		       reference.data.size(),
-		       reference.cached,
-		       +reference.more);
-		printf("#%d  server: %s  key count: %lu, cached: %d, more: %dn",
-		       j,
-		       servers[j].address().toString().c_str(),
-		       current.data.size(),
-		       current.cached,
-		       current.more);
 		size_t currentI = 0, referenceI = 0;
 		while (currentI < current.data.size() || referenceI < reference.data.size()) {
+			if (hasMore &&
+			    (reference.data[referenceI].key >= claimEndKey || current.data[currentI].key >= claimEndKey)) {
+				// If there will be a next round and the key is out of claimEndKey
+				// We will delay the detection to the next round
+				break;
+			}
 			if (currentI >= current.data.size()) {
-				printf(" #%d CurrentI: %lu ReferenceI: %lu Unique key: %s\n",
-				       firstValidServer,
+				printf("UniqueKey, %s(1), %s(0), CurrentIndex %lu, ReferenceIndex %lu, Version %ld, UniqueKey %s\n",
+				       servers[firstValidServer].address().toString().c_str(),
+				       servers[j].address().toString().c_str(),
 				       currentI,
 				       referenceI,
+				       version,
 				       printable(reference.data[referenceI].key).c_str());
 				referenceI++;
 			} else if (referenceI >= reference.data.size()) {
-				printf(" #%d CurrentI: %lu ReferenceI: %lu Unique key: %s\n",
-				       j,
+				printf("UniqueKey %s(1), %s(0), CurrentIndex %lu, ReferenceIndex %lu, Version %ld, UniqueKey %s\n",
+				       servers[j].address().toString().c_str(),
+				       servers[firstValidServer].address().toString().c_str(),
 				       currentI,
 				       referenceI,
+				       version,
 				       printable(current.data[currentI].key).c_str());
 				currentI++;
 			} else {
 				KeyValueRef currentKV = current.data[currentI];
 				KeyValueRef referenceKV = reference.data[referenceI];
-
 				if (currentKV.key == referenceKV.key) {
 					if (currentKV.value != referenceKV.value) {
-						printf(" CurrentI: %lu ReferenceI: %lu Value mismatch key: %s\n",
+						printf("MismatchValue %s(1), %s(1), CurrentIndex %lu, ReferenceIndex %lu, Version %ld, "
+						       "UniqueKey %s\n",
+						       servers[firstValidServer].address().toString().c_str(),
+						       servers[j].address().toString().c_str(),
 						       currentI,
 						       referenceI,
+						       version,
 						       printable(currentKV.key).c_str());
 					}
 					currentI++;
 					referenceI++;
 				} else if (currentKV.key < referenceKV.key) {
-					printf(" #%d CurrentI: %lu ReferenceI: %lu Unique key: %s\n",
-					       j,
+					printf("UniqueKey %s(1), %s(0), CurrentIndex %lu, ReferenceIndex %lu, Version %ld, UniqueKey %s\n",
+					       servers[j].address().toString().c_str(),
+					       servers[firstValidServer].address().toString().c_str(),
 					       currentI,
 					       referenceI,
+					       version,
 					       printable(currentKV.key).c_str());
 					currentI++;
 				} else {
-					printf(" #%d CurrentI: %lu ReferenceI: %lu Unique key: %s\n",
-					       firstValidServer,
+					printf("UniqueKey %s(1), %s(0), CurrentIndex %lu, ReferenceIndex %lu, Version %ld, UniqueKey %s\n",
+					       servers[firstValidServer].address().toString().c_str(),
+					       servers[j].address().toString().c_str(),
 					       currentI,
 					       referenceI,
-					       printable(referenceKV.key).c_str());
+					       version,
+					       printable(currentKV.key).c_str());
 					referenceI++;
 				}
 			}
 		}
 	}
-
-	bool hasMore = firstValidServer >= 0 && replies[firstValidServer].get().get().more;
-	if (hasMore) {
-		const VectorRef<KeyValueRef>& result = replies[firstValidServer].get().get().data;
-		printf("Warning: Consistency check was incomplete, last key of server %d that was checked: %s\n",
-		       firstValidServer,
-		       printable(result[result.size() - 1].key).c_str());
-		lastEnd = result[result.size() - 1].key; // store to a standalone, otherwise memory will be invalid
-		begin = firstGreaterThan(lastEnd);
-	} else {
-		printf("Consistency check finishes for version %ld\n", version);
-		begin = end; // signal that we're done
-	}
-
-	return std::make_pair(allSame, hasMore);
+	return std::make_tuple(allSame, hasMore, claimEndKey);
 }
 
 // The command is used to check the inconsistency in a keyspace, default is \xff\x02/blog/ keyspace.
 ACTOR Future<bool> checkallCommandActor(Database cx, std::vector<StringRef> tokens) {
-	// ignore tokens for now
 	state Transaction onErrorTr(cx); // This transaction exists only to access onError and its backoff behavior
-	state int i = 0;
-	state Version version;
-	state KeySelectorRef begin, end;
-	state bool checkAll = false; // do not return on first error, continue checking all keys
-	state KeyRange toCheck = backupLogKeys;
-
+	state bool checkAll = false; // If set, do not return on first error, continue checking all keys
+	state KeyRange inputRange;
 	if (tokens.size() == 3) {
-		toCheck = KeyRangeRef(tokens[1], tokens[2]);
+		inputRange = KeyRangeRef(tokens[1], tokens[2]);
 	} else if (tokens.size() == 4 && tokens[3] == "all"_sr) {
-		toCheck = KeyRangeRef(tokens[1], tokens[2]);
+		inputRange = KeyRangeRef(tokens[1], tokens[2]);
 		checkAll = true;
 	} else {
 		printf(
@@ -294,78 +302,97 @@ ACTOR Future<bool> checkallCommandActor(Database cx, std::vector<StringRef> toke
 		    "for that purpose).\n");
 		return false;
 	}
+	if (inputRange.empty()) {
+		return true;
+	}
 
+	// At this point, we have a non-empty inputRange to check
 	loop {
-		printf("Start checking range: %s\n", printable(toCheck).c_str());
 		try {
+			printf("Start checking for range: %s\n", printable(inputRange).c_str());
+			// Get SS interface for each shard of the inputRange
 			state Promise<std::vector<std::pair<KeyRange, std::vector<StorageServerInterface>>>> keyServerPromise;
-			bool foundKeyServers = wait(getKeyServers(cx, keyServerPromise, toCheck));
-
+			bool foundKeyServers = wait(getKeyServers(cx, keyServerPromise, inputRange));
 			if (!foundKeyServers) {
-				printf("key server locations for %s not found, retrying in 1s...\n", printable(toCheck).c_str());
+				printf("key server locations for %s not found, retrying in 1s...\n", printable(inputRange).c_str());
 				wait(delay(1.0));
 				continue;
 			}
-
 			state std::vector<std::pair<KeyRange, std::vector<StorageServerInterface>>> keyServers =
 			    keyServerPromise.getFuture().get();
-			state Key lastEnd; // this is to hold the Key having the value of next begin
-			for (i = 0; i < keyServers.size(); i++) { // for each key range
-				state KeyRange range = keyServers[i].first;
-				range = range & toCheck;
-				if (range.empty()) {
+
+			// Checking each shard
+			state int i = 0;
+			for (; i < keyServers.size(); i++) { // for each shard
+				state KeyRange rangeToCheck = keyServers[i].first;
+				rangeToCheck = rangeToCheck & inputRange; // Only check the shard part within the inputRange
+				if (rangeToCheck.empty()) {
 					continue;
 				}
 				const auto& servers = keyServers[i].second;
-				begin = firstGreaterOrEqual(range.begin);
-				end = firstGreaterOrEqual(range.end);
-				printf("Key range: %s\n", printable(range).c_str());
+				state Key beginKeyToCheck = rangeToCheck.begin;
+				printf("Key range to check: %s\n", printable(rangeToCheck).c_str());
 				for (const auto& server : servers) {
 					printf("  %s\n", server.address().toString().c_str());
 				}
 				state std::vector<Future<ErrorOr<GetKeyValuesReply>>> replies;
 				state bool hasMore = true;
 				state int round = 0;
+				state Version version;
 				while (hasMore) {
 					wait(store(version, getVersion(cx)));
 					replies.clear();
-					printf("round %d, begin Key: %s\n", round, printable(begin.toString()).c_str());
+					printf("Round %d: %s - %s\n",
+					       round,
+					       printable(beginKeyToCheck.toString()).c_str(),
+					       printable(rangeToCheck.end.toString()).c_str());
 					for (const auto& s : keyServers[i].second) { // for each storage server
 						GetKeyValuesRequest req;
-						req.begin = begin;
-						req.end = end;
+						req.begin = firstGreaterOrEqual(beginKeyToCheck);
+						req.end = firstGreaterOrEqual(rangeToCheck.end);
 						req.limit = CLIENT_KNOBS->KRM_GET_RANGE_LIMIT;
 						req.limitBytes = CLIENT_KNOBS->KRM_GET_RANGE_LIMIT_BYTES;
-						req.version = version;
+						req.version = version; // all replica should read at the same version
 						req.tags = TagSet();
 						replies.push_back(s.getKeyValues.getReplyUnlessFailedFor(req, 2, 0));
 					}
-					// printf("waiting for %lu replies at version: %ld\n", keyServers[i].second.size(), version);
 					wait(waitForAll(replies));
-					// if there are more results, continue checking in the same shard
-					auto p = checkResults(version, keyServers[i].second, replies, begin, end, lastEnd);
-					bool allSame = p.first;
-					hasMore = p.second;
-					printf("AllSame %d, hasMore %d, checkAll %d\n", allSame, hasMore, checkAll);
+					// checkResults keeps invariants:
+					// (1) hasMore = true if any server has more data not read yet
+					// (2) nextBeginKey is the minimal key returned from all servers
+					// (3) checkResults reports inconsistency of keys only before the nextBeginKey if hasMore=true
+					// Therefore, whether to proceed to the next round depends on hasMore
+					// If there is a next round, it starts from the minimal key returned from all servers
+					auto res = checkResults(version, keyServers[i].second, replies);
+					bool allSame = std::get<0>(res);
+					hasMore = std::get<1>(res);
+					Key nextBeginKey = std::get<2>(res);
+					// Using claimEndKey for the current round as the nextBeginKey for the next round
+					// Note that claimEndKey is not compared in the current round
+					// This key will be compared in the next round
+					printf("Result: compared %s - %s\n",
+					       printable(beginKeyToCheck.toString()).c_str(),
+					       printable(nextBeginKey.toString()).c_str());
+					beginKeyToCheck = nextBeginKey;
+					printf("allSame %d, hasMore %d, checkAll %d\n", allSame, hasMore, checkAll);
 					if (!allSame && !checkAll) {
 						return false;
 					}
 					round++;
 				}
-				if (begin == end) {
-					// this shard is done, signaled by begin == end
-					toCheck = KeyRangeRef(range.end, toCheck.end);
-				}
 			}
+			break;
+
 		} catch (Error& e) {
-			printf("Retrying for error: %s\n", e.what());
+			printf("Error: %s", e.what());
 			wait(onErrorTr.onError(e));
-		}
-		if (toCheck.begin == toCheck.end) {
-			return true;
+			printf(", retrying in 1s...\n");
 		}
 		wait(delay(1.0));
 	}
+
+	printf("Checking complete.\n");
+	return true;
 }
 
 CommandFactory checkallCommandFactory("checkall");

From ab79237db8cf089e92f2f8805bb80cb059209e89 Mon Sep 17 00:00:00 2001
From: Zhe Wang <zhe.wang@wustl.edu>
Date: Fri, 19 Jan 2024 21:13:48 -0800
Subject: [PATCH 374/468] address comments

---
 fdbcli/DebugCommands.actor.cpp | 22 +++++++++-------------
 1 file changed, 9 insertions(+), 13 deletions(-)

diff --git a/fdbcli/DebugCommands.actor.cpp b/fdbcli/DebugCommands.actor.cpp
index 8228bd31dfc..62594042f3c 100644
--- a/fdbcli/DebugCommands.actor.cpp
+++ b/fdbcli/DebugCommands.actor.cpp
@@ -224,7 +224,9 @@ std::tuple<bool, bool, Key> checkResults(Version version,
 				break;
 			}
 			if (currentI >= current.data.size()) {
-				printf("UniqueKey, %s(1), %s(0), CurrentIndex %lu, ReferenceIndex %lu, Version %ld, UniqueKey %s\n",
+				// ServerA(1), ServerB(0): 1 indicates that ServerA has the key while 0 indicates that ServerB does not
+				// have the key
+				printf("UniqueKey, %s(1), %s(0), CurrentIndex %lu, ReferenceIndex %lu, Version %ld, Key %s\n",
 				       servers[firstValidServer].address().toString().c_str(),
 				       servers[j].address().toString().c_str(),
 				       currentI,
@@ -233,7 +235,7 @@ std::tuple<bool, bool, Key> checkResults(Version version,
 				       printable(reference.data[referenceI].key).c_str());
 				referenceI++;
 			} else if (referenceI >= reference.data.size()) {
-				printf("UniqueKey %s(1), %s(0), CurrentIndex %lu, ReferenceIndex %lu, Version %ld, UniqueKey %s\n",
+				printf("UniqueKey %s(1), %s(0), CurrentIndex %lu, ReferenceIndex %lu, Version %ld, Key %s\n",
 				       servers[j].address().toString().c_str(),
 				       servers[firstValidServer].address().toString().c_str(),
 				       currentI,
@@ -247,7 +249,7 @@ std::tuple<bool, bool, Key> checkResults(Version version,
 				if (currentKV.key == referenceKV.key) {
 					if (currentKV.value != referenceKV.value) {
 						printf("MismatchValue %s(1), %s(1), CurrentIndex %lu, ReferenceIndex %lu, Version %ld, "
-						       "UniqueKey %s\n",
+						       "Key %s\n",
 						       servers[firstValidServer].address().toString().c_str(),
 						       servers[j].address().toString().c_str(),
 						       currentI,
@@ -258,7 +260,7 @@ std::tuple<bool, bool, Key> checkResults(Version version,
 					currentI++;
 					referenceI++;
 				} else if (currentKV.key < referenceKV.key) {
-					printf("UniqueKey %s(1), %s(0), CurrentIndex %lu, ReferenceIndex %lu, Version %ld, UniqueKey %s\n",
+					printf("UniqueKey %s(1), %s(0), CurrentIndex %lu, ReferenceIndex %lu, Version %ld, Key %s\n",
 					       servers[j].address().toString().c_str(),
 					       servers[firstValidServer].address().toString().c_str(),
 					       currentI,
@@ -267,13 +269,13 @@ std::tuple<bool, bool, Key> checkResults(Version version,
 					       printable(currentKV.key).c_str());
 					currentI++;
 				} else {
-					printf("UniqueKey %s(1), %s(0), CurrentIndex %lu, ReferenceIndex %lu, Version %ld, UniqueKey %s\n",
+					printf("UniqueKey %s(1), %s(0), CurrentIndex %lu, ReferenceIndex %lu, Version %ld, Key %s\n",
 					       servers[firstValidServer].address().toString().c_str(),
 					       servers[j].address().toString().c_str(),
 					       currentI,
 					       referenceI,
 					       version,
-					       printable(currentKV.key).c_str());
+					       printable(referenceKV.key).c_str());
 					referenceI++;
 				}
 			}
@@ -357,17 +359,11 @@ ACTOR Future<bool> checkallCommandActor(Database cx, std::vector<StringRef> toke
 						replies.push_back(s.getKeyValues.getReplyUnlessFailedFor(req, 2, 0));
 					}
 					wait(waitForAll(replies));
-					// checkResults keeps invariants:
-					// (1) hasMore = true if any server has more data not read yet
-					// (2) nextBeginKey is the minimal key returned from all servers
-					// (3) checkResults reports inconsistency of keys only before the nextBeginKey if hasMore=true
-					// Therefore, whether to proceed to the next round depends on hasMore
-					// If there is a next round, it starts from the minimal key returned from all servers
 					auto res = checkResults(version, keyServers[i].second, replies);
 					bool allSame = std::get<0>(res);
 					hasMore = std::get<1>(res);
 					Key nextBeginKey = std::get<2>(res);
-					// Using claimEndKey for the current round as the nextBeginKey for the next round
+					// Using claimEndKey of the current round as the nextBeginKey for the next round
 					// Note that claimEndKey is not compared in the current round
 					// This key will be compared in the next round
 					printf("Result: compared %s - %s\n",

From 4d1dbb2ebee9c6532fba7dcc5d31c74115d07676 Mon Sep 17 00:00:00 2001
From: Zhe Wang <zhe.wang@wustl.edu>
Date: Mon, 22 Jan 2024 16:08:07 -0800
Subject: [PATCH 375/468] address comments and nits

---
 fdbcli/DebugCommands.actor.cpp | 21 +++++++++++++++------
 1 file changed, 15 insertions(+), 6 deletions(-)

diff --git a/fdbcli/DebugCommands.actor.cpp b/fdbcli/DebugCommands.actor.cpp
index 62594042f3c..c8025a96f5e 100644
--- a/fdbcli/DebugCommands.actor.cpp
+++ b/fdbcli/DebugCommands.actor.cpp
@@ -194,7 +194,10 @@ std::tuple<bool, bool, Key> checkResults(Version version,
 		}
 		hasMore = hasMore || current.more;
 	}
-	ASSERT(!claimEndKey.empty());
+	if (claimEndKey.empty()) {
+		printf("checkResults error: claimEndKey is empty\nPlease re-run the checkall.\n");
+		throw operation_failed();
+	}
 
 	// Compare servers
 	bool allSame = true;
@@ -226,7 +229,8 @@ std::tuple<bool, bool, Key> checkResults(Version version,
 			if (currentI >= current.data.size()) {
 				// ServerA(1), ServerB(0): 1 indicates that ServerA has the key while 0 indicates that ServerB does not
 				// have the key
-				printf("UniqueKey, %s(1), %s(0), CurrentIndex %lu, ReferenceIndex %lu, Version %ld, Key %s\n",
+				printf("Inconsistency: UniqueKey, %s(1), %s(0), CurrentIndex %lu, ReferenceIndex %lu, Version %ld, Key "
+				       "%s\n",
 				       servers[firstValidServer].address().toString().c_str(),
 				       servers[j].address().toString().c_str(),
 				       currentI,
@@ -235,7 +239,8 @@ std::tuple<bool, bool, Key> checkResults(Version version,
 				       printable(reference.data[referenceI].key).c_str());
 				referenceI++;
 			} else if (referenceI >= reference.data.size()) {
-				printf("UniqueKey %s(1), %s(0), CurrentIndex %lu, ReferenceIndex %lu, Version %ld, Key %s\n",
+				printf("Inconsistency: UniqueKey, %s(1), %s(0), CurrentIndex %lu, ReferenceIndex %lu, Version %ld, Key "
+				       "%s\n",
 				       servers[j].address().toString().c_str(),
 				       servers[firstValidServer].address().toString().c_str(),
 				       currentI,
@@ -248,7 +253,8 @@ std::tuple<bool, bool, Key> checkResults(Version version,
 				KeyValueRef referenceKV = reference.data[referenceI];
 				if (currentKV.key == referenceKV.key) {
 					if (currentKV.value != referenceKV.value) {
-						printf("MismatchValue %s(1), %s(1), CurrentIndex %lu, ReferenceIndex %lu, Version %ld, "
+						printf("Inconsistency: MismatchValue, %s(1), %s(1), CurrentIndex %lu, ReferenceIndex %lu, "
+						       "Version %ld, "
 						       "Key %s\n",
 						       servers[firstValidServer].address().toString().c_str(),
 						       servers[j].address().toString().c_str(),
@@ -260,7 +266,8 @@ std::tuple<bool, bool, Key> checkResults(Version version,
 					currentI++;
 					referenceI++;
 				} else if (currentKV.key < referenceKV.key) {
-					printf("UniqueKey %s(1), %s(0), CurrentIndex %lu, ReferenceIndex %lu, Version %ld, Key %s\n",
+					printf("Inconsistency: UniqueKey, %s(1), %s(0), CurrentIndex %lu, ReferenceIndex %lu, Version %ld, "
+					       "Key %s\n",
 					       servers[j].address().toString().c_str(),
 					       servers[firstValidServer].address().toString().c_str(),
 					       currentI,
@@ -269,7 +276,8 @@ std::tuple<bool, bool, Key> checkResults(Version version,
 					       printable(currentKV.key).c_str());
 					currentI++;
 				} else {
-					printf("UniqueKey %s(1), %s(0), CurrentIndex %lu, ReferenceIndex %lu, Version %ld, Key %s\n",
+					printf("Inconsistency: UniqueKey, %s(1), %s(0), CurrentIndex %lu, ReferenceIndex %lu, Version %ld, "
+					       "Key %s\n",
 					       servers[firstValidServer].address().toString().c_str(),
 					       servers[j].address().toString().c_str(),
 					       currentI,
@@ -305,6 +313,7 @@ ACTOR Future<bool> checkallCommandActor(Database cx, std::vector<StringRef> toke
 		return false;
 	}
 	if (inputRange.empty()) {
+		printf("Input empty range: %s.\nImmediately exit.\n", printable(inputRange).c_str());
 		return true;
 	}
 

From 9aec70316e7f959a6d3cf292c980658fd9459307 Mon Sep 17 00:00:00 2001
From: Zhe Wang <zhe.wang@wustl.edu>
Date: Wed, 24 Jan 2024 14:50:14 -0800
Subject: [PATCH 376/468] fix-check-all-block-issue

---
 fdbcli/DebugCommands.actor.cpp | 183 +++++++++++++++++++--------------
 1 file changed, 106 insertions(+), 77 deletions(-)

diff --git a/fdbcli/DebugCommands.actor.cpp b/fdbcli/DebugCommands.actor.cpp
index c8025a96f5e..e662c72d4f8 100644
--- a/fdbcli/DebugCommands.actor.cpp
+++ b/fdbcli/DebugCommands.actor.cpp
@@ -170,58 +170,31 @@ CommandFactory getallCommandFactory("getall");
 // (3) checkResults reports inconsistency of keys only before the nextBeginKey if hasMore=true
 // Therefore, whether to proceed to the next round depends on hasMore
 // If there is a next round, it starts from the minimal key returned from all servers
-std::tuple<bool, bool, Key> checkResults(Version version,
-                                         const std::vector<StorageServerInterface>& servers,
-                                         const std::vector<Future<ErrorOr<GetKeyValuesReply>>>& replies) {
-	// Decide comparison scope
-	Key claimEndKey; // used for the next round exists
-	bool hasMore = false;
-	for (int j = 0; j < replies.size(); j++) {
-		auto reply = replies[j].get();
-		if (reply.isError()) {
-			printf("checkResults error: %s\n", reply.getError().what());
-			throw reply.getError();
-		} else if (reply.get().error.present()) {
-			printf("checkResults error: %s\n", reply.get().error.get().what());
-			throw reply.get().error.get();
-		}
-		GetKeyValuesReply current = reply.get();
-		if (current.data.size() == 0) {
-			continue; // Ignore if no data has replied
-		}
-		if (claimEndKey.empty() || current.data[current.data.size() - 1].key < claimEndKey) {
-			claimEndKey = current.data[current.data.size() - 1].key;
-		}
-		hasMore = hasMore || current.more;
-	}
-	if (claimEndKey.empty()) {
-		printf("checkResults error: claimEndKey is empty\nPlease re-run the checkall.\n");
-		throw operation_failed();
-	}
-
+bool checkResults(Version version,
+                  bool hasMore,
+                  Key claimEndKey,
+                  const std::vector<StorageServerInterface>& servers,
+                  const std::vector<GetKeyValuesReply>& replies) {
 	// Compare servers
 	bool allSame = true;
 	int firstValidServer = -1;
 	for (int j = 0; j < replies.size(); j++) {
-		auto reply = replies[j].get();
-		ASSERT(reply.present() && !reply.get().error.present()); // has thrown eariler of error
 		if (firstValidServer == -1) {
 			firstValidServer = j;
 			continue; // always select the first server as reference
 		}
 		// compare reference and current
-		GetKeyValuesReply current = reply.get();
-		GetKeyValuesReply reference = replies[firstValidServer].get().get();
+		GetKeyValuesReply current = replies[j];
+		GetKeyValuesReply reference = replies[firstValidServer];
 		if (current.data == reference.data && current.more == reference.more) {
 			continue;
 		}
-
 		// Detecting corrupted keys for any mismatching replies between current and reference servers
 		allSame = false;
 		size_t currentI = 0, referenceI = 0;
 		while (currentI < current.data.size() || referenceI < reference.data.size()) {
-			if (hasMore &&
-			    (reference.data[referenceI].key >= claimEndKey || current.data[currentI].key >= claimEndKey)) {
+			if (hasMore && ((referenceI < reference.data.size() && reference.data[referenceI].key >= claimEndKey) ||
+			                (currentI < current.data.size() && current.data[currentI].key >= claimEndKey))) {
 				// If there will be a next round and the key is out of claimEndKey
 				// We will delay the detection to the next round
 				break;
@@ -289,35 +262,15 @@ std::tuple<bool, bool, Key> checkResults(Version version,
 			}
 		}
 	}
-	return std::make_tuple(allSame, hasMore, claimEndKey);
+	return allSame;
 }
 
-// The command is used to check the inconsistency in a keyspace, default is \xff\x02/blog/ keyspace.
-ACTOR Future<bool> checkallCommandActor(Database cx, std::vector<StringRef> tokens) {
-	state Transaction onErrorTr(cx); // This transaction exists only to access onError and its backoff behavior
-	state bool checkAll = false; // If set, do not return on first error, continue checking all keys
-	state KeyRange inputRange;
-	if (tokens.size() == 3) {
-		inputRange = KeyRangeRef(tokens[1], tokens[2]);
-	} else if (tokens.size() == 4 && tokens[3] == "all"_sr) {
-		inputRange = KeyRangeRef(tokens[1], tokens[2]);
-		checkAll = true;
-	} else {
-		printf(
-		    "checkall [<KEY> <KEY2>] (all)\n"
-		    "Check inconsistency of the input range by comparing all replicas and print any corruptions.\n"
-		    "The default behavior is to stop on the first subrange where corruption is found\n"
-		    "`all` is optional. When `all` is appended, the checker does not stop until all subranges have checked.\n"
-		    "Note this is intended to check a small range of keys, not the entire database (consider consistencycheck "
-		    "for that purpose).\n");
-		return false;
-	}
-	if (inputRange.empty()) {
-		printf("Input empty range: %s.\nImmediately exit.\n", printable(inputRange).c_str());
-		return true;
-	}
+ACTOR Future<bool> doCheckAll(Database cx, KeyRange inputRange, bool checkAll);
 
-	// At this point, we have a non-empty inputRange to check
+// Return whether inconsistency is detected in the inputRange
+ACTOR Future<bool> doCheckAll(Database cx, KeyRange inputRange, bool checkAll) {
+	state Transaction onErrorTr(cx); // This transaction exists only to access onError and its backoff behavior
+	state bool consistent = true;
 	loop {
 		try {
 			printf("Start checking for range: %s\n", printable(inputRange).c_str());
@@ -344,7 +297,7 @@ ACTOR Future<bool> checkallCommandActor(Database cx, std::vector<StringRef> toke
 				state Key beginKeyToCheck = rangeToCheck.begin;
 				printf("Key range to check: %s\n", printable(rangeToCheck).c_str());
 				for (const auto& server : servers) {
-					printf("  %s\n", server.address().toString().c_str());
+					printf("\t%s\n", server.address().toString().c_str());
 				}
 				state std::vector<Future<ErrorOr<GetKeyValuesReply>>> replies;
 				state bool hasMore = true;
@@ -368,19 +321,68 @@ ACTOR Future<bool> checkallCommandActor(Database cx, std::vector<StringRef> toke
 						replies.push_back(s.getKeyValues.getReplyUnlessFailedFor(req, 2, 0));
 					}
 					wait(waitForAll(replies));
-					auto res = checkResults(version, keyServers[i].second, replies);
-					bool allSame = std::get<0>(res);
-					hasMore = std::get<1>(res);
-					Key nextBeginKey = std::get<2>(res);
-					// Using claimEndKey of the current round as the nextBeginKey for the next round
-					// Note that claimEndKey is not compared in the current round
-					// This key will be compared in the next round
-					printf("Result: compared %s - %s\n",
-					       printable(beginKeyToCheck.toString()).c_str(),
-					       printable(nextBeginKey.toString()).c_str());
-					beginKeyToCheck = nextBeginKey;
-					printf("allSame %d, hasMore %d, checkAll %d\n", allSame, hasMore, checkAll);
-					if (!allSame && !checkAll) {
+
+					// Decide comparison scope
+					Key claimEndKey; // used for the next round if hasMore == true
+					Key maxEndKey;
+					hasMore = false; // re-calculate hasMore according to replies
+					for (int j = 0; j < replies.size(); j++) {
+						auto reply = replies[j].get();
+						if (reply.isError()) {
+							printf("checkResults error: %s\n", reply.getError().what());
+							throw reply.getError();
+						} else if (reply.get().error.present()) {
+							printf("checkResults error: %s\n", reply.get().error.get().what());
+							throw reply.get().error.get();
+						}
+						GetKeyValuesReply current = reply.get();
+						if (current.data.size() == 0) {
+							continue; // Ignore if no data has replied
+						}
+						if (claimEndKey.empty() || current.data[current.data.size() - 1].key < claimEndKey) {
+							claimEndKey = current.data[current.data.size() - 1].key;
+						}
+						if (maxEndKey.empty() || current.data[current.data.size() - 1].key > maxEndKey) {
+							maxEndKey = current.data[current.data.size() - 1].key;
+						}
+						hasMore = hasMore || current.more;
+					}
+					printf("Compare scope has been decided\n\tBeginKey: %s\n\tEndKey: %s\n\tHasMore: %d\n",
+					       printable(beginKeyToCheck).c_str(),
+					       printable(claimEndKey).c_str(),
+					       hasMore);
+					if (claimEndKey.empty()) {
+						printf("checkResults error: claimEndKey is empty\nPlease re-run the checkall.\n");
+						throw operation_failed();
+					} else if ((beginKeyToCheck == claimEndKey) && hasMore) {
+						// In case rangeBegin == claimEndKey == next beginKeyToCheck
+						// we spawn a new checkall for a smaller range in this case
+						state KeyRange spawnedRangeToCheck = Standalone(KeyRangeRef(beginKeyToCheck, maxEndKey));
+						printf("Spawn new checkall for range %s\n", printable(spawnedRangeToCheck).c_str());
+						bool allSame = wait(doCheckAll(cx, spawnedRangeToCheck, checkAll));
+						beginKeyToCheck = spawnedRangeToCheck.end;
+						consistent = consistent && allSame; // !allSame of any subrange results in !consistent
+					} else {
+						std::vector<GetKeyValuesReply> keyValueReplies;
+						for (int j = 0; j < replies.size(); j++) {
+							auto reply = replies[j].get();
+							ASSERT(reply.present() && !reply.get().error.present()); // has thrown eariler of error
+							keyValueReplies.push_back(reply.get());
+						}
+						// keyServers and keyValueReplies must follow the same order
+						bool allSame =
+						    checkResults(version, hasMore, claimEndKey, keyServers[i].second, keyValueReplies);
+						// Using claimEndKey of the current round as the nextBeginKey for the next round
+						// Note that claimEndKey is not compared in the current round
+						// This key will be compared in the next round
+						printf("Result: compared %s - %s\n",
+						       printable(beginKeyToCheck.toString()).c_str(),
+						       printable(claimEndKey.toString()).c_str());
+						beginKeyToCheck = claimEndKey;
+						printf("allSame %d, hasMore %d, checkAll %d\n", allSame, hasMore, checkAll);
+						consistent = consistent && allSame; // !allSame of any subrange results in !consistent
+					}
+					if (!consistent && !checkAll) {
 						return false;
 					}
 					round++;
@@ -395,8 +397,35 @@ ACTOR Future<bool> checkallCommandActor(Database cx, std::vector<StringRef> toke
 		}
 		wait(delay(1.0));
 	}
+	return consistent;
+}
 
-	printf("Checking complete.\n");
+// The command is used to check the inconsistency in a keyspace, default is \xff\x02/blog/ keyspace.
+ACTOR Future<bool> checkallCommandActor(Database cx, std::vector<StringRef> tokens) {
+	state bool checkAll = false; // If set, do not return on first error, continue checking all keys
+	state KeyRange inputRange;
+	if (tokens.size() == 3) {
+		inputRange = KeyRangeRef(tokens[1], tokens[2]);
+	} else if (tokens.size() == 4 && tokens[3] == "all"_sr) {
+		inputRange = KeyRangeRef(tokens[1], tokens[2]);
+		checkAll = true;
+	} else {
+		printf(
+		    "checkall [<KEY> <KEY2>] (all)\n"
+		    "Check inconsistency of the input range by comparing all replicas and print any corruptions.\n"
+		    "The default behavior is to stop on the first subrange where corruption is found\n"
+		    "`all` is optional. When `all` is appended, the checker does not stop until all subranges have checked.\n"
+		    "Note this is intended to check a small range of keys, not the entire database (consider consistencycheck "
+		    "for that purpose).\n");
+		return false;
+	}
+	if (inputRange.empty()) {
+		printf("Input empty range: %s.\nImmediately exit.\n", printable(inputRange).c_str());
+		return false;
+	}
+	// At this point, we have a non-empty inputRange to check
+	bool res = wait(doCheckAll(cx, inputRange, checkAll));
+	printf("Checking complete. AllSame: %d\n", res);
 	return true;
 }
 

From 2a2adf3b9564a4e5349dd41f7e749c6affc140f9 Mon Sep 17 00:00:00 2001
From: Zhe Wang <zhe.wang@wustl.edu>
Date: Thu, 25 Jan 2024 14:02:37 -0800
Subject: [PATCH 377/468] address comments

---
 fdbcli/DebugCommands.actor.cpp | 27 ++++++++++++++++++---------
 1 file changed, 18 insertions(+), 9 deletions(-)

diff --git a/fdbcli/DebugCommands.actor.cpp b/fdbcli/DebugCommands.actor.cpp
index e662c72d4f8..cc08a7ce4e0 100644
--- a/fdbcli/DebugCommands.actor.cpp
+++ b/fdbcli/DebugCommands.actor.cpp
@@ -266,7 +266,6 @@ bool checkResults(Version version,
 }
 
 ACTOR Future<bool> doCheckAll(Database cx, KeyRange inputRange, bool checkAll);
-
 // Return whether inconsistency is detected in the inputRange
 ACTOR Future<bool> doCheckAll(Database cx, KeyRange inputRange, bool checkAll) {
 	state Transaction onErrorTr(cx); // This transaction exists only to access onError and its backoff behavior
@@ -284,14 +283,14 @@ ACTOR Future<bool> doCheckAll(Database cx, KeyRange inputRange, bool checkAll) {
 			}
 			state std::vector<std::pair<KeyRange, std::vector<StorageServerInterface>>> keyServers =
 			    keyServerPromise.getFuture().get();
-
-			// Checking each shard
+			// We partition the entire input range into shards
+			// and we conduct comparison shard by shard
 			state int i = 0;
 			for (; i < keyServers.size(); i++) { // for each shard
 				state KeyRange rangeToCheck = keyServers[i].first;
 				rangeToCheck = rangeToCheck & inputRange; // Only check the shard part within the inputRange
 				if (rangeToCheck.empty()) {
-					continue;
+					continue; // Skip the shard if it is outside of the inputRange
 				}
 				const auto& servers = keyServers[i].second;
 				state Key beginKeyToCheck = rangeToCheck.begin;
@@ -352,11 +351,21 @@ ACTOR Future<bool> doCheckAll(Database cx, KeyRange inputRange, bool checkAll) {
 					       printable(claimEndKey).c_str(),
 					       hasMore);
 					if (claimEndKey.empty()) {
-						printf("checkResults error: claimEndKey is empty\nPlease re-run the checkall.\n");
-						throw operation_failed();
+						// It is possible that there is clear operation between the prev round and the current round
+						// which result in empty claimEndKey --- nothing to compare
+						// In this case, we simply skip the current shard
+						ASSERT(hasMore == false);
+						continue;
 					} else if ((beginKeyToCheck == claimEndKey) && hasMore) {
-						// In case rangeBegin == claimEndKey == next beginKeyToCheck
-						// we spawn a new checkall for a smaller range in this case
+						// This is a special case: rangeBegin == claimEndKey == next beginKeyToCheck
+						// We separate this case and the third case to solve a corner issue led by the
+						// third code path: the progress will get stuck on repeatedly checking beginKeyToCheck.
+						// In the third code path, if hasMore == true and beginKeyToCheck == claimEndKey,
+						// The next round of beginKeyToCheck (aka claimEndKey) will always be beginKeyToCheck of the
+						// current round. To avoid this issue, we spawn a child checkall on a smaller range
+						// (beginKeyToCheck ~ maxEndKey). This smaller range guarantees that the hasMore is always false
+						// and the child checkall will complete and the global progress will move forward. Once the
+						// child checkall is done, we move to the range: maxEndKey ~ rangeToCheck.end
 						state KeyRange spawnedRangeToCheck = Standalone(KeyRangeRef(beginKeyToCheck, maxEndKey));
 						printf("Spawn new checkall for range %s\n", printable(spawnedRangeToCheck).c_str());
 						bool allSame = wait(doCheckAll(cx, spawnedRangeToCheck, checkAll));
@@ -400,7 +409,7 @@ ACTOR Future<bool> doCheckAll(Database cx, KeyRange inputRange, bool checkAll) {
 	return consistent;
 }
 
-// The command is used to check the inconsistency in a keyspace, default is \xff\x02/blog/ keyspace.
+// The command is used to check the data inconsistency of the user input range
 ACTOR Future<bool> checkallCommandActor(Database cx, std::vector<StringRef> tokens) {
 	state bool checkAll = false; // If set, do not return on first error, continue checking all keys
 	state KeyRange inputRange;

From d2e26d93080b742093b771922ef124b7980b93bd Mon Sep 17 00:00:00 2001
From: Zhe Wang <zhe.wang@wustl.edu>
Date: Fri, 26 Jan 2024 15:32:54 -0800
Subject: [PATCH 378/468] fix tester stale interface issue at CC

---
 fdbserver/ClusterController.actor.cpp | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/fdbserver/ClusterController.actor.cpp b/fdbserver/ClusterController.actor.cpp
index 9d53bc63346..a0040addcf0 100644
--- a/fdbserver/ClusterController.actor.cpp
+++ b/fdbserver/ClusterController.actor.cpp
@@ -917,10 +917,9 @@ ACTOR Future<Void> rebootAndCheck(ClusterControllerData* cluster, Optional<Stand
 ACTOR Future<Void> workerAvailabilityWatch(WorkerInterface worker,
                                            ProcessClass startingClass,
                                            ClusterControllerData* cluster) {
-	state Future<Void> failed =
-	    (worker.address() == g_network->getLocalAddress() || startingClass.classType() == ProcessClass::TesterClass)
-	        ? Never()
-	        : waitFailureClient(worker.waitFailure, SERVER_KNOBS->WORKER_FAILURE_TIME);
+	state Future<Void> failed = (worker.address() == g_network->getLocalAddress())
+	                                ? Never()
+	                                : waitFailureClient(worker.waitFailure, SERVER_KNOBS->WORKER_FAILURE_TIME);
 	cluster->updateWorkerList.set(worker.locality.processId(),
 	                              ProcessData(worker.locality, startingClass, worker.stableAddress()));
 	// This switching avoids a race where the worker can be added to id_worker map after the workerAvailabilityWatch

From 462304aacc2aafce431be27ee46609396cd1bc2d Mon Sep 17 00:00:00 2001
From: hao fu <hfu5@apple.com>
Date: Mon, 29 Jan 2024 09:33:28 -0800
Subject: [PATCH 379/468] Allow AsyncFileWriteChecker history.timestamp to have
 better precision

---
 fdbrpc/AsyncFileWriteChecker.h | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/fdbrpc/AsyncFileWriteChecker.h b/fdbrpc/AsyncFileWriteChecker.h
index fa082bcf796..16cb099b5e6 100644
--- a/fdbrpc/AsyncFileWriteChecker.h
+++ b/fdbrpc/AsyncFileWriteChecker.h
@@ -143,8 +143,9 @@ class AsyncFileWriteChecker : public IAsyncFile, public ReferenceCounted<AsyncFi
 #endif
 
 			// For writes, just update the stored sum
+			double millisecondsPerSecond = 1000;
 			if (write) {
-				history.timestamp = (uint32_t)now();
+				history.timestamp = (uint32_t)(now() * millisecondsPerSecond);
 				history.checksum = checksum;
 			} else {
 				if (history.checksum != 0 && history.checksum != checksum) {
@@ -155,7 +156,7 @@ class AsyncFileWriteChecker : public IAsyncFile, public ReferenceCounted<AsyncFi
 					    .detail("PageNumber", page)
 					    .detail("ChecksumOfPage", checksum)
 					    .detail("ChecksumHistory", history.checksum)
-					    .detail("LastWriteTime", history.timestamp);
+					    .detail("LastWriteTime", history.timestamp / millisecondsPerSecond);
 					history.checksum = 0;
 				}
 			}

From a75ec4da15fdd5f0bf6d319f2358de3fa470c773 Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Mon, 29 Jan 2024 11:19:39 -0800
Subject: [PATCH 380/468] Update value to be hex output

So we can use the output easily for fdbcli and analyzing scripts.
---
 fdbcli/DebugCommands.actor.cpp | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/fdbcli/DebugCommands.actor.cpp b/fdbcli/DebugCommands.actor.cpp
index cc08a7ce4e0..2cf3ae6392d 100644
--- a/fdbcli/DebugCommands.actor.cpp
+++ b/fdbcli/DebugCommands.actor.cpp
@@ -124,6 +124,15 @@ ACTOR Future<bool> getLocationCommandActor(Database cx, std::vector<StringRef> t
 // hidden commands, no help text for now
 CommandFactory getLocationCommandFactory("getlocation");
 
+std::string toHex(Value v) {
+	std::string result;
+	result.reserve(v.size() * 4);
+	for (int i = 0; i < v.size(); i++) {
+		result.append(format("\\x%02x", v[i]));
+	}
+	return result;
+}
+
 // The command is used to get values from all storage servers that have the given key.
 ACTOR Future<bool> getallCommandActor(Database cx, std::vector<StringRef> tokens, Version version) {
 	if (tokens.size() != 2) {
@@ -152,7 +161,7 @@ ACTOR Future<bool> getallCommandActor(Database cx, std::vector<StringRef> tokens
 				fprintf(stderr, "ERROR: %s %s\n", ssi.c_str(), replies[i].getError().what());
 			} else {
 				Optional<Value> v = replies[i].get().value;
-				printf(" %s %s\n", ssi.c_str(), v.present() ? printable(v.get()).c_str() : "(not found)");
+				printf(" %s %s\n", ssi.c_str(), v.present() ? toHex(v.get()).c_str() : "(not found)");
 			}
 		}
 	} else {

From 3cf894c30549531a8b3d189048e21ece07f2486f Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Mon, 29 Jan 2024 13:51:09 -0800
Subject: [PATCH 381/468] Use hex for checkall command output

---
 fdbcli/DebugCommands.actor.cpp | 49 ++++++++++++++++------------------
 1 file changed, 23 insertions(+), 26 deletions(-)

diff --git a/fdbcli/DebugCommands.actor.cpp b/fdbcli/DebugCommands.actor.cpp
index 2cf3ae6392d..e30a49b507a 100644
--- a/fdbcli/DebugCommands.actor.cpp
+++ b/fdbcli/DebugCommands.actor.cpp
@@ -27,6 +27,15 @@
 
 namespace fdb_cli {
 
+std::string toHex(StringRef v) {
+	std::string result;
+	result.reserve(v.size() * 4);
+	for (int i = 0; i < v.size(); i++) {
+		result.append(format("\\x%02x", v[i]));
+	}
+	return result;
+}
+
 // Gets a version at which to read from the storage servers
 ACTOR Future<Version> getVersion(Database cx) {
 	loop {
@@ -108,7 +117,7 @@ ACTOR Future<bool> getLocationCommandActor(Database cx, std::vector<StringRef> t
 	state Promise<std::vector<std::pair<KeyRange, std::vector<StorageServerInterface>>>> keyServersPromise;
 	bool found = wait(getKeyServers(cx, keyServersPromise, kr));
 	if (!found) {
-		printf("[%s, %s]locations not found\n", printable(tokens[1]).c_str(), printable(tokens[2]).c_str());
+		printf("[%s, %s]locations not found\n", toHex(tokens[1]).c_str(), toHex(tokens[2]).c_str());
 		return false;
 	}
 	std::vector<std::pair<KeyRange, std::vector<StorageServerInterface>>> keyServers =
@@ -124,15 +133,6 @@ ACTOR Future<bool> getLocationCommandActor(Database cx, std::vector<StringRef> t
 // hidden commands, no help text for now
 CommandFactory getLocationCommandFactory("getlocation");
 
-std::string toHex(Value v) {
-	std::string result;
-	result.reserve(v.size() * 4);
-	for (int i = 0; i < v.size(); i++) {
-		result.append(format("\\x%02x", v[i]));
-	}
-	return result;
-}
-
 // The command is used to get values from all storage servers that have the given key.
 ACTOR Future<bool> getallCommandActor(Database cx, std::vector<StringRef> tokens, Version version) {
 	if (tokens.size() != 2) {
@@ -147,7 +147,7 @@ ACTOR Future<bool> getallCommandActor(Database cx, std::vector<StringRef> tokens
 
 	if (loc.locations) {
 		printf("version is %ld\n", version);
-		printf("`%s' is at:\n", printable(tokens[1]).c_str());
+		printf("`%s' is at:\n", toHex(tokens[1]).c_str());
 		state Reference<LocationInfo::Locations> locations = loc.locations->locations();
 		state std::vector<Future<GetValueReply>> replies;
 		for (int i = 0; locations && i < locations->size(); i++) {
@@ -165,7 +165,7 @@ ACTOR Future<bool> getallCommandActor(Database cx, std::vector<StringRef> tokens
 			}
 		}
 	} else {
-		printf("`%s': location not found\n", printable(tokens[1]).c_str());
+		printf("`%s': location not found\n", toHex(tokens[1]).c_str());
 	}
 	return true;
 }
@@ -218,7 +218,7 @@ bool checkResults(Version version,
 				       currentI,
 				       referenceI,
 				       version,
-				       printable(reference.data[referenceI].key).c_str());
+				       toHex(reference.data[referenceI].key).c_str());
 				referenceI++;
 			} else if (referenceI >= reference.data.size()) {
 				printf("Inconsistency: UniqueKey, %s(1), %s(0), CurrentIndex %lu, ReferenceIndex %lu, Version %ld, Key "
@@ -228,7 +228,7 @@ bool checkResults(Version version,
 				       currentI,
 				       referenceI,
 				       version,
-				       printable(current.data[currentI].key).c_str());
+				       toHex(current.data[currentI].key).c_str());
 				currentI++;
 			} else {
 				KeyValueRef currentKV = current.data[currentI];
@@ -243,7 +243,7 @@ bool checkResults(Version version,
 						       currentI,
 						       referenceI,
 						       version,
-						       printable(currentKV.key).c_str());
+						       toHex(currentKV.key).c_str());
 					}
 					currentI++;
 					referenceI++;
@@ -255,7 +255,7 @@ bool checkResults(Version version,
 					       currentI,
 					       referenceI,
 					       version,
-					       printable(currentKV.key).c_str());
+					       toHex(currentKV.key).c_str());
 					currentI++;
 				} else {
 					printf("Inconsistency: UniqueKey, %s(1), %s(0), CurrentIndex %lu, ReferenceIndex %lu, Version %ld, "
@@ -265,7 +265,7 @@ bool checkResults(Version version,
 					       currentI,
 					       referenceI,
 					       version,
-					       printable(referenceKV.key).c_str());
+					       toHex(referenceKV.key).c_str());
 					referenceI++;
 				}
 			}
@@ -314,10 +314,8 @@ ACTOR Future<bool> doCheckAll(Database cx, KeyRange inputRange, bool checkAll) {
 				while (hasMore) {
 					wait(store(version, getVersion(cx)));
 					replies.clear();
-					printf("Round %d: %s - %s\n",
-					       round,
-					       printable(beginKeyToCheck.toString()).c_str(),
-					       printable(rangeToCheck.end.toString()).c_str());
+					printf(
+					    "Round %d: %s - %s\n", round, toHex(beginKeyToCheck).c_str(), toHex(rangeToCheck.end).c_str());
 					for (const auto& s : keyServers[i].second) { // for each storage server
 						GetKeyValuesRequest req;
 						req.begin = firstGreaterOrEqual(beginKeyToCheck);
@@ -356,8 +354,8 @@ ACTOR Future<bool> doCheckAll(Database cx, KeyRange inputRange, bool checkAll) {
 						hasMore = hasMore || current.more;
 					}
 					printf("Compare scope has been decided\n\tBeginKey: %s\n\tEndKey: %s\n\tHasMore: %d\n",
-					       printable(beginKeyToCheck).c_str(),
-					       printable(claimEndKey).c_str(),
+					       toHex(beginKeyToCheck).c_str(),
+					       toHex(claimEndKey).c_str(),
 					       hasMore);
 					if (claimEndKey.empty()) {
 						// It is possible that there is clear operation between the prev round and the current round
@@ -393,9 +391,8 @@ ACTOR Future<bool> doCheckAll(Database cx, KeyRange inputRange, bool checkAll) {
 						// Using claimEndKey of the current round as the nextBeginKey for the next round
 						// Note that claimEndKey is not compared in the current round
 						// This key will be compared in the next round
-						printf("Result: compared %s - %s\n",
-						       printable(beginKeyToCheck.toString()).c_str(),
-						       printable(claimEndKey.toString()).c_str());
+						printf(
+						    "Result: compared %s - %s\n", toHex(beginKeyToCheck).c_str(), toHex(claimEndKey).c_str());
 						beginKeyToCheck = claimEndKey;
 						printf("allSame %d, hasMore %d, checkAll %d\n", allSame, hasMore, checkAll);
 						consistent = consistent && allSame; // !allSame of any subrange results in !consistent

From e831f2fa093cb0117571e1b685fcbf35ff748945 Mon Sep 17 00:00:00 2001
From: He Liu <heliu05023@gmail.com>
Date: Tue, 30 Jan 2024 09:27:14 -0800
Subject: [PATCH 382/468] Enable DD verbose tracing by default.

---
 fdbclient/ServerKnobs.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/fdbclient/ServerKnobs.cpp b/fdbclient/ServerKnobs.cpp
index f5a0951bb39..560b70177dc 100644
--- a/fdbclient/ServerKnobs.cpp
+++ b/fdbclient/ServerKnobs.cpp
@@ -283,7 +283,7 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi
 	init( DD_EXCLUDE_MIN_REPLICAS,                                 1 );
 	init( DD_VALIDATE_LOCALITY,                                 true ); if( randomize && BUGGIFY ) DD_VALIDATE_LOCALITY = false;
 	init( DD_CHECK_INVALID_LOCALITY_DELAY,                       60  ); if( randomize && BUGGIFY ) DD_CHECK_INVALID_LOCALITY_DELAY = 1 + deterministicRandom()->random01() * 600;
-	init( DD_ENABLE_VERBOSE_TRACING,                           false ); if( randomize && BUGGIFY ) DD_ENABLE_VERBOSE_TRACING = true;
+	init( DD_ENABLE_VERBOSE_TRACING,                            true ); if( randomize && BUGGIFY ) DD_ENABLE_VERBOSE_TRACING = false;
 	init( DD_SS_FAILURE_VERSIONLAG,                        250000000 );
 	init( DD_SS_ALLOWED_VERSIONLAG,                        200000000 ); if( randomize && BUGGIFY ) { DD_SS_FAILURE_VERSIONLAG = deterministicRandom()->randomInt(15000000, 500000000); DD_SS_ALLOWED_VERSIONLAG = 0.75 * DD_SS_FAILURE_VERSIONLAG; }
 	init( DD_SS_STUCK_TIME_LIMIT,                              300.0 ); if( randomize && BUGGIFY ) { DD_SS_STUCK_TIME_LIMIT = 200.0 + deterministicRandom()->random01() * 100.0; }

From 97aca0ff35b6b25e68b63fad78c03a608de6b600 Mon Sep 17 00:00:00 2001
From: Johannes Scheuermann <johscheuer@users.noreply.github.com>
Date: Thu, 1 Feb 2024 18:36:42 +0100
Subject: [PATCH 383/468] Fix complie issue for ALLOC_INSTRUMENTATION (#11164)

---
 flow/Platform.actor.cpp | 2 +-
 flow/SystemMonitor.cpp  | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/flow/Platform.actor.cpp b/flow/Platform.actor.cpp
index 486025e5877..aaa3915d513 100644
--- a/flow/Platform.actor.cpp
+++ b/flow/Platform.actor.cpp
@@ -3109,10 +3109,10 @@ void outOfMemory() {
 	    .detail("BackTraces", traceCounts.size());
 
 	for (auto i = traceCounts.begin(); i != traceCounts.end(); ++i) {
-		char buf[1024];
 		std::vector<void*>* frames = i->second.backTrace;
 		std::string backTraceStr;
 #if defined(_WIN32)
+		char buf[1024];
 		for (int j = 1; j < frames->size(); j++) {
 			_snprintf(buf, 1024, "%p ", frames->at(j));
 			backTraceStr += buf;
diff --git a/flow/SystemMonitor.cpp b/flow/SystemMonitor.cpp
index c4572c0b9dc..0729ed38456 100644
--- a/flow/SystemMonitor.cpp
+++ b/flow/SystemMonitor.cpp
@@ -355,10 +355,10 @@ SystemStatistics customSystemMonitor(std::string const& eventName, StatisticsSta
 			uint64_t totalSize = 0;
 			uint64_t totalCount = 0;
 			for (auto i = traceCounts.begin(); i != traceCounts.end(); ++i) {
-				char buf[1024];
 				std::vector<void*>* frames = i->second.backTrace;
 				std::string backTraceStr;
 #if defined(_WIN32)
+				char buf[1024];
 				for (int j = 1; j < frames->size(); j++) {
 					_snprintf(buf, 1024, "%p ", frames->at(j));
 					backTraceStr += buf;
@@ -427,4 +427,4 @@ Future<Void> startMemoryUsageMonitor(uint64_t memLimit) {
 		}
 	};
 	return recurring(checkMemoryUsage, FLOW_KNOBS->MEMORY_USAGE_CHECK_INTERVAL);
-}
\ No newline at end of file
+}

From c44522469b5ceb5809f73d15c6ff61336cf29582 Mon Sep 17 00:00:00 2001
From: Aaron Molitor <ammolitor@users.noreply.github.com>
Date: Tue, 31 Oct 2023 18:10:12 -0500
Subject: [PATCH 384/468] Upgrade rocksdb to 8.6.7

---------

Co-authored-by: Yao Xiao <yxiao6@apple.com>

Upgrade rocksdb to 8.6.7

Upgrade rocksdb to 8.6.7
---
 cmake/CompileRocksDB.cmake    |  9 +++------
 cmake/FDBComponents.cmake     | 12 +++++-------
 fdbserver/FDBRocksDBVersion.h |  4 ++--
 3 files changed, 10 insertions(+), 15 deletions(-)

diff --git a/cmake/CompileRocksDB.cmake b/cmake/CompileRocksDB.cmake
index 436ac97630f..621b3c14491 100644
--- a/cmake/CompileRocksDB.cmake
+++ b/cmake/CompileRocksDB.cmake
@@ -1,15 +1,12 @@
 # FindRocksDB
 
-find_package(RocksDB 8.1.1)
+find_package(RocksDB 8.6.7)
 
 include(ExternalProject)
 
 set(RocksDB_CMAKE_ARGS
   -DUSE_RTTI=1
   -DPORTABLE=${PORTABLE_ROCKSDB}
-  -DFORCE_SSE42=${ROCKSDB_SSE42}
-  -DFORCE_AVX=${ROCKSDB_AVX}
-  -DFORCE_AVX2=${ROCKSDB_AVX2}
   -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER}
   -DCMAKE_C_FLAGS=${CMAKE_C_FLAGS}
   -DCMAKE_CXX_STANDARD=${CMAKE_CXX_STANDARD}
@@ -54,8 +51,8 @@ if(ROCKSDB_FOUND)
     ${BINARY_DIR}/librocksdb.a)
 else()
   ExternalProject_Add(rocksdb
-    URL https://github.com/facebook/rocksdb/archive/refs/tags/v8.1.1.tar.gz
-    URL_HASH SHA256=9102704e169cfb53e7724a30750eeeb3e71307663852f01fa08d5a320e6155a8
+    URL https://github.com/facebook/rocksdb/archive/refs/tags/v8.6.7.tar.gz
+    URL_HASH SHA256=cdb2fc3c6a556f20591f564cb8e023e56828469aa3f76e1d9535c443ba1f0c1a
     CMAKE_ARGS ${RocksDB_CMAKE_ARGS}
     BUILD_BYPRODUCTS <BINARY_DIR>/librocksdb.a
     INSTALL_COMMAND ""
diff --git a/cmake/FDBComponents.cmake b/cmake/FDBComponents.cmake
index 8c110c75cbe..b5c747bdc14 100644
--- a/cmake/FDBComponents.cmake
+++ b/cmake/FDBComponents.cmake
@@ -162,15 +162,13 @@ endif()
 ################################################################################
 
 set(SSD_ROCKSDB_EXPERIMENTAL ON CACHE BOOL "Build with experimental RocksDB support")
-set(PORTABLE_ROCKSDB ON CACHE BOOL "Compile RocksDB in portable mode") # Set this to OFF to compile RocksDB with `-march=native`
-set(ROCKSDB_SSE42 OFF CACHE BOOL "Compile RocksDB with SSE42 enabled")
-set(ROCKSDB_AVX ${USE_AVX} CACHE BOOL "Compile RocksDB with AVX enabled")
-set(ROCKSDB_AVX2 OFF CACHE BOOL "Compile RocksDB with AVX2 enabled")
+# PORTABLE flag for RockdDB changed as of this PR (with v8.3.2 and after): https://github.com/facebook/rocksdb/pull/11419
+# https://github.com/facebook/rocksdb/blob/v8.6.7/CMakeLists.txt#L256
+set(PORTABLE_ROCKSDB 1 CACHE STRING "Minimum CPU arch to support (i.e. skylake, haswell, etc., or 0 = current CPU, 1 = baseline CPU")
 set(ROCKSDB_TOOLS OFF CACHE BOOL "Compile RocksDB tools")
 set(WITH_LIBURING OFF CACHE BOOL "Build with liburing enabled") # Set this to ON to include liburing
-# RocksDB is currently enabled by default for GCC but does not build with the latest
-# Clang.
-if (SSD_ROCKSDB_EXPERIMENTAL AND GCC)
+
+if (SSD_ROCKSDB_EXPERIMENTAL AND NOT WIN32)
   set(WITH_ROCKSDB_EXPERIMENTAL ON)
 else()
   set(WITH_ROCKSDB_EXPERIMENTAL OFF)
diff --git a/fdbserver/FDBRocksDBVersion.h b/fdbserver/FDBRocksDBVersion.h
index d8c1272a168..ec3de38666b 100644
--- a/fdbserver/FDBRocksDBVersion.h
+++ b/fdbserver/FDBRocksDBVersion.h
@@ -22,7 +22,7 @@
 #define FDBSERVER_FDBROCKSDBVERSION_H
 
 #define FDB_ROCKSDB_MAJOR 8
-#define FDB_ROCKSDB_MINOR 1
-#define FDB_ROCKSDB_PATCH 1
+#define FDB_ROCKSDB_MINOR 6
+#define FDB_ROCKSDB_PATCH 7
 
 #endif
\ No newline at end of file

From 16f5875705a0b3c0b05960be4ebc875f9e271911 Mon Sep 17 00:00:00 2001
From: Aaron Molitor <amolitor@apple.com>
Date: Fri, 3 Nov 2023 10:33:18 -0500
Subject: [PATCH 385/468] add sha to opentest4j download

---
 bindings/java/CMakeLists.txt | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/bindings/java/CMakeLists.txt b/bindings/java/CMakeLists.txt
index 22564dccc86..c502ddbe26d 100644
--- a/bindings/java/CMakeLists.txt
+++ b/bindings/java/CMakeLists.txt
@@ -319,7 +319,8 @@ if(NOT OPEN_FOR_IDE)
       ${CMAKE_BINARY_DIR}/packages/apiguardian-api-1.1.1.jar
       EXPECTED_HASH SHA256=fc68f0d28633caccf3980fdf1e99628fba9c49424ee56dc685cd8b4d2a9fefde)
     file(DOWNLOAD "https://search.maven.org/remotecontent?filepath=org/opentest4j/opentest4j/1.2.0/opentest4j-1.2.0.jar"
-      ${CMAKE_BINARY_DIR}/packages/opentest4j-1.2.0.jar)
+      ${CMAKE_BINARY_DIR}/packages/opentest4j-1.2.0.jar
+      EXPECTED_HASH SHA256=58812de60898d976fb81ef3b62da05c6604c18fd4a249f5044282479fc286af2)
 
     set(JUNIT_CLASSPATH "${CMAKE_BINARY_DIR}/packages/junit-jupiter-api-5.7.1.jar:${CMAKE_BINARY_DIR}/packages/junit-jupiter-engine-5.7.1.jar:${CMAKE_BINARY_DIR}/packages/junit-jupiter-params-5.7.1.jar:${CMAKE_BINARY_DIR}/packages/opentest4j-1.2.0.jar:${CMAKE_BINARY_DIR}/packages/apiguardian-api-1.1.1.jar")
     set(JUNIT_CLASSPATH "${JUNIT_CLASSPATH}:${CMAKE_BINARY_DIR}/packages/junit-platform-console-1.7.1.jar:${CMAKE_BINARY_DIR}/packages/junit-platform-commons-1.7.1.jar")

From 546c440e3aa82dbe4c9543dac0bf0c8f029a7fba Mon Sep 17 00:00:00 2001
From: Xiaoge Su <magichp@gmail.com>
Date: Sat, 27 Jan 2024 17:11:43 -0800
Subject: [PATCH 386/468] Fix the compile warning in
 KeyValueStoreRocksDB.actor.cpp

---
 fdbserver/KeyValueStoreRocksDB.actor.cpp | 16 +++++++---------
 1 file changed, 7 insertions(+), 9 deletions(-)

diff --git a/fdbserver/KeyValueStoreRocksDB.actor.cpp b/fdbserver/KeyValueStoreRocksDB.actor.cpp
index 78b13b05e89..41e6113cfb9 100644
--- a/fdbserver/KeyValueStoreRocksDB.actor.cpp
+++ b/fdbserver/KeyValueStoreRocksDB.actor.cpp
@@ -106,24 +106,22 @@ std::shared_ptr<rocksdb::Cache> rocksdb_block_cache = nullptr;
 
 class SharedRocksDBState {
 public:
-	SharedRocksDBState(UID id);
+	SharedRocksDBState(const UID& id_);
 
+	const UID id;
 	std::vector<std::shared_ptr<LatencySample>> readLatency;
 	std::vector<std::shared_ptr<LatencySample>> scanLatency;
 	std::vector<std::shared_ptr<LatencySample>> readQueueLatency;
 	LatencySample commitLatency;
 	LatencySample commitQueueLatency;
 	LatencySample dbWriteLatency;
-
-private:
-	const UID id;
 };
 
-SharedRocksDBState::SharedRocksDBState(UID id)
-  : id(id), commitLatency(LatencySample("RocksDBCommitLatency",
-                                        id,
-                                        SERVER_KNOBS->LATENCY_METRICS_LOGGING_INTERVAL,
-                                        SERVER_KNOBS->LATENCY_SAMPLE_SIZE)),
+SharedRocksDBState::SharedRocksDBState(const UID& id_)
+  : id(id_), commitLatency(LatencySample("RocksDBCommitLatency",
+                                         id,
+                                         SERVER_KNOBS->LATENCY_METRICS_LOGGING_INTERVAL,
+                                         SERVER_KNOBS->LATENCY_SAMPLE_SIZE)),
     commitQueueLatency(LatencySample("RocksDBCommitQueueLatency",
                                      id,
                                      SERVER_KNOBS->LATENCY_METRICS_LOGGING_INTERVAL,

From 13934fb6fc454ab762b93940c37c5060dbbdfaa7 Mon Sep 17 00:00:00 2001
From: Xiaoge Su <magichp@gmail.com>
Date: Wed, 31 Jan 2024 15:17:09 -0800
Subject: [PATCH 387/468] Make FoundationDB Compilable Again

---
 fdbserver/KeyValueStoreRocksDB.actor.cpp | 72 ++++++++++++++----------
 flow/IThreadPool.h                       |  1 +
 2 files changed, 43 insertions(+), 30 deletions(-)

diff --git a/fdbserver/KeyValueStoreRocksDB.actor.cpp b/fdbserver/KeyValueStoreRocksDB.actor.cpp
index 41e6113cfb9..db3e691a1be 100644
--- a/fdbserver/KeyValueStoreRocksDB.actor.cpp
+++ b/fdbserver/KeyValueStoreRocksDB.actor.cpp
@@ -459,8 +459,8 @@ struct Counters {
 struct ReadIterator {
 	uint64_t index; // incrementing counter to uniquely identify read iterator.
 	bool inUse;
-	std::shared_ptr<rocksdb::Iterator> iter;
 	double creationTime;
+	std::shared_ptr<rocksdb::Iterator> iter;
 	KeyRange keyRange;
 	std::shared_ptr<rocksdb::Slice> beginSlice, endSlice;
 	ReadIterator(CF& cf, uint64_t index, DB& db)
@@ -1121,22 +1121,26 @@ struct RocksDBKeyValueStore : IKeyValueStore {
 		CF& cf;
 
 		UID id;
-		std::shared_ptr<rocksdb::RateLimiter> rateLimiter;
+
+		std::shared_ptr<SharedRocksDBState> sharedState;
 		std::shared_ptr<ReadIteratorPool> readIterPool;
 		std::shared_ptr<PerfContextMetrics> perfContextMetrics;
+
 		int threadIndex;
+
+		std::shared_ptr<rocksdb::RateLimiter> rateLimiter;
+
 		ThreadReturnPromiseStream<std::tuple<int, std::string, double>> metricPromiseStream;
-		std::shared_ptr<SharedRocksDBState> sharedState;
 
-		explicit Writer(DB& db,
-		                CF& cf,
-		                UID id,
-		                std::shared_ptr<SharedRocksDBState> sharedState,
-		                std::shared_ptr<ReadIteratorPool> readIterPool,
-		                std::shared_ptr<PerfContextMetrics> perfContextMetrics,
-		                int threadIndex)
-		  : db(db), cf(cf), id(id), sharedState(sharedState), readIterPool(readIterPool),
-		    perfContextMetrics(perfContextMetrics), threadIndex(threadIndex),
+		Writer(DB& db_,
+		       CF& cf_,
+		       const UID id_,
+		       const std::shared_ptr<SharedRocksDBState> sharedState_,
+		       const std::shared_ptr<ReadIteratorPool> readIterPool_,
+		       const std::shared_ptr<PerfContextMetrics> perfContextMetrics_,
+		       const int threadIndex_)
+		  : db(db_), cf(cf_), id(id_), sharedState(sharedState_), readIterPool(readIterPool_),
+		    perfContextMetrics(perfContextMetrics_), threadIndex(threadIndex_),
 		    rateLimiter(SERVER_KNOBS->ROCKSDB_WRITE_RATE_LIMITER_BYTES_PER_SEC > 0
 		                    ? rocksdb::NewGenericRateLimiter(
 		                          SERVER_KNOBS->ROCKSDB_WRITE_RATE_LIMITER_BYTES_PER_SEC, // rate_bytes_per_sec
@@ -1152,7 +1156,7 @@ struct RocksDBKeyValueStore : IKeyValueStore {
 			}
 		}
 
-		~Writer() override {
+		virtual ~Writer() override {
 			if (db) {
 				delete db;
 			}
@@ -1395,7 +1399,7 @@ struct RocksDBKeyValueStore : IKeyValueStore {
 				std::set<std::string> columnFamilies{ "default" };
 				columnFamilies.insert(SERVER_KNOBS->DEFAULT_FDB_ROCKSDB_COLUMN_FAMILY);
 				std::vector<rocksdb::ColumnFamilyDescriptor> descriptors;
-				for (const std::string name : columnFamilies) {
+				for (const std::string& name : columnFamilies) {
 					descriptors.push_back(rocksdb::ColumnFamilyDescriptor{ name, getCFOptions() });
 				}
 				s = rocksdb::DestroyDB(a.path, getOptions(), descriptors);
@@ -1756,19 +1760,21 @@ struct RocksDBKeyValueStore : IKeyValueStore {
 
 		struct ReadRangeAction : TypedAction<Reader, ReadRangeAction>, FastAllocated<ReadRangeAction> {
 			KeyRange keys;
-			int rowLimit, byteLimit;
+			int rowLimit;
+			int byteLimit;
 			ReadType type;
+			Counters& counters;
 			double startTime;
 			bool getHistograms;
 			ThreadReturnPromise<RangeResult> result;
-			Counters& counters;
-			ReadRangeAction(KeyRange keys, int rowLimit, int byteLimit, ReadType type, Counters& counters)
-			  : keys(keys), rowLimit(rowLimit), byteLimit(byteLimit), type(type), startTime(timer_monotonic()),
-			    counters(counters),
+			ReadRangeAction(KeyRange keys_, int rowLimit_, int byteLimit_, ReadType type_, Counters& counters_)
+			  : keys(keys_), rowLimit(rowLimit_), byteLimit(byteLimit_), type(type_), counters(counters_),
+			    startTime(timer_monotonic()),
 			    getHistograms(deterministicRandom()->random01() < SERVER_KNOBS->ROCKSDB_HISTOGRAMS_SAMPLE_RATE),
 			    result(static_cast<TaskPriority>(SERVER_KNOBS->ROCKSDB_THREAD_PROMISE_PRIORITY)) {}
 			double getTimeEstimate() const override { return SERVER_KNOBS->READ_RANGE_TIME_ESTIMATE; }
 		};
+
 		void action(ReadRangeAction& a) {
 			++a.counters.rocksdbReadRangeQueries;
 			bool doPerfContextMetrics =
@@ -1900,11 +1906,21 @@ struct RocksDBKeyValueStore : IKeyValueStore {
 	UID id;
 	std::shared_ptr<SharedRocksDBState> sharedState;
 	std::shared_ptr<PerfContextMetrics> perfContextMetrics;
+	std::shared_ptr<ReadIteratorPool> readIterPool;
+	FlowLock readSemaphore;
+	FlowLock fetchSemaphore;
+
+	int numReadWaiters;
+	int numFetchWaiters;
+
+	std::shared_ptr<RocksDBErrorListener> errorListener;
+	Future<Void> errorFuture;
+
 	rocksdb::ColumnFamilyHandle* defaultFdbCF = nullptr;
+
 	Reference<IThreadPool> writeThread;
 	Reference<IThreadPool> readThreads;
-	std::shared_ptr<RocksDBErrorListener> errorListener;
-	Future<Void> errorFuture;
+
 	Promise<Void> closePromise;
 	Future<Void> openFuture;
 	std::unique_ptr<rocksdb::WriteBatch> writeBatch;
@@ -1913,17 +1929,13 @@ struct RocksDBKeyValueStore : IKeyValueStore {
 	// maximum number of single key deletes in a commit, if ROCKSDB_SINGLEKEY_DELETES_ON_CLEARRANGE is enabled.
 	int maxDeletes;
 	Optional<Future<Void>> metrics;
-	FlowLock readSemaphore;
-	int numReadWaiters;
-	FlowLock fetchSemaphore;
-	int numFetchWaiters;
-	std::shared_ptr<ReadIteratorPool> readIterPool;
+
 	std::vector<Future<Void>> actors;
 	Counters counters;
 
-	explicit RocksDBKeyValueStore(const std::string& path, UID id)
-	  : path(path), id(id), sharedState(std::make_shared<SharedRocksDBState>(id)),
-	    perfContextMetrics(new PerfContextMetrics()), readIterPool(new ReadIteratorPool(id, db, defaultFdbCF)),
+	RocksDBKeyValueStore(const std::string& path_, const UID id_)
+	  : path(path_), id(id_), sharedState(std::make_shared<SharedRocksDBState>(id_)),
+	    perfContextMetrics(new PerfContextMetrics()), readIterPool(new ReadIteratorPool(id_, db, defaultFdbCF)),
 	    readSemaphore(SERVER_KNOBS->ROCKSDB_READ_QUEUE_SOFT_MAX),
 	    fetchSemaphore(SERVER_KNOBS->ROCKSDB_FETCH_QUEUE_SOFT_MAX),
 	    numReadWaiters(SERVER_KNOBS->ROCKSDB_READ_QUEUE_HARD_MAX - SERVER_KNOBS->ROCKSDB_READ_QUEUE_SOFT_MAX),
@@ -2347,7 +2359,7 @@ struct RocksDBKeyValueStore : IKeyValueStore {
 			for (const LiveFileMetaData& file : rocksCF.sstFiles) {
 				dirs.insert(file.db_path);
 			}
-			for (const std::string dir : dirs) {
+			for (const std::string& dir : dirs) {
 				platform::eraseDirectoryRecursive(dir);
 				TraceEvent("DeleteCheckpointRemovedDir", id)
 				    .detail("CheckpointID", checkpoint.checkpointID)
diff --git a/flow/IThreadPool.h b/flow/IThreadPool.h
index 8aea2723265..d7ebc79f51d 100644
--- a/flow/IThreadPool.h
+++ b/flow/IThreadPool.h
@@ -77,6 +77,7 @@ class TypedAction : public ThreadAction {
 		delete (ActionType*)this;
 	}
 	void cancel() override { delete (ActionType*)this; }
+	virtual ~TypedAction() {}
 };
 
 template <class T>

From 94e0a14604a193f2d3e669b55931494c3c0b2a8c Mon Sep 17 00:00:00 2001
From: neethuhaneesha <nbingi@apple.com>
Date: Thu, 1 Feb 2024 13:47:03 -0800
Subject: [PATCH 388/468] Removing the type check unit test (#11168)

---
 fdbserver/KeyValueStoreRocksDB.actor.cpp | 8 --------
 1 file changed, 8 deletions(-)

diff --git a/fdbserver/KeyValueStoreRocksDB.actor.cpp b/fdbserver/KeyValueStoreRocksDB.actor.cpp
index db3e691a1be..9315b35acfb 100644
--- a/fdbserver/KeyValueStoreRocksDB.actor.cpp
+++ b/fdbserver/KeyValueStoreRocksDB.actor.cpp
@@ -2533,14 +2533,6 @@ TEST_CASE("noSim/fdbserver/KeyValueStoreRocksDB/CheckpointRestore") {
 	return Void();
 }
 
-TEST_CASE("noSim/fdbserver/KeyValueStoreRocksDB/RocksDBTypes") {
-	// If the following assertion fails, update SstFileMetaData and LiveFileMetaData in RocksDBCheckpointUtils.actor.h
-	// to be the same as rocksdb::SstFileMetaData and rocksdb::LiveFileMetaData.
-	ASSERT_EQ(sizeof(rocksdb::LiveFileMetaData), 208);
-	ASSERT_EQ(sizeof(rocksdb::ExportImportFilesMetaData), 32);
-	return Void();
-}
-
 } // namespace
 
 #endif // SSD_ROCKSDB_EXPERIMENTAL

From a92485b3f8794aa7f41a41de12f0a8f8247a4b56 Mon Sep 17 00:00:00 2001
From: FoundationDB CI <foundationdb_ci@apple.com>
Date: Tue, 6 Feb 2024 21:49:39 +0000
Subject: [PATCH 389/468] disable AVX for 7.1.54 release

---
 cmake/ConfigureCompiler.cmake | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cmake/ConfigureCompiler.cmake b/cmake/ConfigureCompiler.cmake
index e52d8665ee8..7e50469ba6a 100644
--- a/cmake/ConfigureCompiler.cmake
+++ b/cmake/ConfigureCompiler.cmake
@@ -253,7 +253,7 @@ else()
       set(USE_AVX512F OFF)
     endif()
   endif()
-  set(USE_AVX ON CACHE BOOL "Enable AVX instructions")
+  set(USE_AVX OFF CACHE BOOL "Enable AVX instructions")
   if (USE_AVX)
     if (CMAKE_HOST_SYSTEM_PROCESSOR MATCHES "^x86")
       add_compile_options(-mavx)

From 0b6e96bf4ccdb43922e1e698264df4ce59bbcd08 Mon Sep 17 00:00:00 2001
From: FoundationDB CI <foundationdb_ci@apple.com>
Date: Tue, 6 Feb 2024 21:49:50 +0000
Subject: [PATCH 390/468] enable AVX and update version for 7.1.55 release

---
 CMakeLists.txt                | 2 +-
 cmake/ConfigureCompiler.cmake | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 5eda1f4ff9a..ac8416cdb4b 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -23,7 +23,7 @@ else()
 endif()
 
 project(foundationdb
-  VERSION 7.1.54
+  VERSION 7.1.55
   DESCRIPTION "FoundationDB is a scalable, fault-tolerant, ordered key-value store with full ACID transactions."
   HOMEPAGE_URL "http://www.foundationdb.org/"
   LANGUAGES C CXX ASM)
diff --git a/cmake/ConfigureCompiler.cmake b/cmake/ConfigureCompiler.cmake
index 7e50469ba6a..e52d8665ee8 100644
--- a/cmake/ConfigureCompiler.cmake
+++ b/cmake/ConfigureCompiler.cmake
@@ -253,7 +253,7 @@ else()
       set(USE_AVX512F OFF)
     endif()
   endif()
-  set(USE_AVX OFF CACHE BOOL "Enable AVX instructions")
+  set(USE_AVX ON CACHE BOOL "Enable AVX instructions")
   if (USE_AVX)
     if (CMAKE_HOST_SYSTEM_PROCESSOR MATCHES "^x86")
       add_compile_options(-mavx)

From 4a67f6940986370ae34014889f8d6ecbcf432941 Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyu_zhou@apple.com>
Date: Tue, 6 Feb 2024 21:00:49 -0800
Subject: [PATCH 391/468] Add 7.1.44 to 7.1.55 release notes (#11183)

---
 .../release-notes/release-notes-710.rst       | 63 +++++++++++++++++++
 1 file changed, 63 insertions(+)

diff --git a/documentation/sphinx/source/release-notes/release-notes-710.rst b/documentation/sphinx/source/release-notes/release-notes-710.rst
index 6bdbf6b6392..e76f233c8ee 100644
--- a/documentation/sphinx/source/release-notes/release-notes-710.rst
+++ b/documentation/sphinx/source/release-notes/release-notes-710.rst
@@ -4,6 +4,69 @@
 Release Notes
 #############
 
+7.1.55
+======
+* Same as 7.1.54 release with AVX enabled.
+
+7.1.54
+======
+* Released with AVX disabled.
+* Upgraded RocksDB to 8.6.7. `(PR #11160) <https://github.com/apple/foundationdb/pull/11160>`_
+* Enabled data distribution verbose logging by default. `(PR #11158) <https://github.com/apple/foundationdb/pull/11158>`_
+* Added logging for invalid mutations on storage servers. `(PR #11097) <https://github.com/apple/foundationdb/pull/11097>`_
+* Enhanced AsyncFileWriteChecker history.timestamp to have better millisecond precision. `(PR #11155) <https://github.com/apple/foundationdb/pull/11155>`_
+* Fixed checkall debug command to output all inconsistent keys. `(PR #11153) <https://github.com/apple/foundationdb/pull/11153>`_, `(PR #11146) <https://github.com/apple/foundationdb/pull/11146>`_, `(PR #11141) <https://github.com/apple/foundationdb/pull/11141>`_, and `(PR #11138) <https://github.com/apple/foundationdb/pull/11138>`_
+* Fixed unreachable processes in status json due to stale tester interfaces at cluster controller. `(PR #11149) <https://github.com/apple/foundationdb/pull/11149>`_
+
+7.1.53
+======
+* Same as 7.1.52 release with AVX enabled.
+
+7.1.52
+======
+* Released with AVX disabled.
+* Fixed testers from being overloaded by consistency checkers. `(PR #11127) <https://github.com/apple/foundationdb/pull/11127>`_
+
+7.1.51
+======
+* Same as 7.1.50 release with AVX enabled.
+
+7.1.50
+======
+* Released with AVX disabled.
+* Added urgent consistency checker mode that uses multiple tester processes. `(PR #11103) <https://github.com/apple/foundationdb/pull/11103>`_
+* Fixed checkall debug command when shard to check is large. `(PR #11105) <https://github.com/apple/foundationdb/pull/11105>`_
+
+7.1.49
+======
+* Same as 7.1.48 release with AVX enabled.
+
+7.1.48
+======
+* Released with AVX disabled.
+* Added consistency checker urgent mode. `(PR #11102) <https://github.com/apple/foundationdb/pull/11102>`_
+* Increased the number of keys that "fdbcli checkall" command can fetch per GetKeyValueRequest call. `(PR #11098) <https://github.com/apple/foundationdb/pull/11098>`_
+
+7.1.47
+======
+* Same as 7.1.46 release with AVX enabled.
+
+7.1.46
+======
+* Released with AVX disabled.
+* Added range-customizable consistency checker and fixed retriable errors during the check. `(PR #11100) <https://github.com/apple/foundationdb/pull/11100>`_ and `(PR #11096) <https://github.com/apple/foundationdb/pull/11096>`_
+* Added key ranges as filters for fdbdecode. `(PR #11099) <https://github.com/apple/foundationdb/pull/11099>`_
+
+7.1.45
+======
+* Same as 7.1.44 release with AVX enabled.
+
+7.1.44
+======
+* Released with AVX disabled.
+* Added distributed consistency checker. `(PR #11088) <https://github.com/apple/foundationdb/pull/11088>`_
+* Fixed the exclusions of localities that are not matching any process. `(PR #11034) <https://github.com/apple/foundationdb/pull/11034>`_
+
 7.1.43
 ======
 * Same as 7.1.42 release with AVX enabled.

From 0c062ef815c950651a5a4922bcedbdecca443312 Mon Sep 17 00:00:00 2001
From: FoundationDB CI <foundationdb_ci@apple.com>
Date: Wed, 7 Feb 2024 20:04:00 +0000
Subject: [PATCH 392/468] update version after 7.1.55 release

---
 CMakeLists.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index ac8416cdb4b..4c23e845278 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -23,7 +23,7 @@ else()
 endif()
 
 project(foundationdb
-  VERSION 7.1.55
+  VERSION 7.1.56
   DESCRIPTION "FoundationDB is a scalable, fault-tolerant, ordered key-value store with full ACID transactions."
   HOMEPAGE_URL "http://www.foundationdb.org/"
   LANGUAGES C CXX ASM)

From 009af240b7a2ae58c6e2f818c8de4f415fb02249 Mon Sep 17 00:00:00 2001
From: Aaron Molitor <amolitor@apple.com>
Date: Thu, 8 Feb 2024 11:44:27 -0600
Subject: [PATCH 393/468] fix find toml in release-7.1 to avoid macos pr
 failures

---
 cmake/FDBComponents.cmake | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/cmake/FDBComponents.cmake b/cmake/FDBComponents.cmake
index b5c747bdc14..8f8e6379dc6 100644
--- a/cmake/FDBComponents.cmake
+++ b/cmake/FDBComponents.cmake
@@ -183,14 +183,16 @@ endif()
 find_package(toml11 QUIET)
 if(toml11_FOUND)
   add_library(toml11_target INTERFACE)
-  add_dependencies(toml11_target INTERFACE toml11::toml11)
+  target_link_libraries(toml11_target INTERFACE toml11::toml11)
 else()
   include(ExternalProject)
-
   ExternalProject_add(toml11Project
     URL "https://github.com/ToruNiina/toml11/archive/v3.4.0.tar.gz"
     URL_HASH SHA256=bc6d733efd9216af8c119d8ac64a805578c79cc82b813e4d1d880ca128bd154d
     CMAKE_CACHE_ARGS
+      -DCMAKE_BUILD_TYPE:STRING=${CMAKE_BUILD_TYPE}
+      -DCMAKE_C_COMPILER:FILEPATH=${CMAKE_C_COMPILER}
+      -DCMAKE_CXX_COMPILER:FILEPATH=${CMAKE_CXX_COMPILER}
       -DCMAKE_INSTALL_PREFIX:PATH=${CMAKE_CURRENT_BINARY_DIR}/toml11
       -Dtoml11_BUILD_TEST:BOOL=OFF
     BUILD_ALWAYS ON)

From 3d6161e3b292cd4ed9e22ae8a21de153c4674a17 Mon Sep 17 00:00:00 2001
From: Yao Xiao <yxiao6@apple.com>
Date: Mon, 5 Feb 2024 12:05:16 -0800
Subject: [PATCH 394/468] Upgrade to RocksDB 8.10.0.

---
 cmake/CompileRocksDB.cmake    | 6 +++---
 fdbserver/FDBRocksDBVersion.h | 4 ++--
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/cmake/CompileRocksDB.cmake b/cmake/CompileRocksDB.cmake
index 621b3c14491..f3cac793c48 100644
--- a/cmake/CompileRocksDB.cmake
+++ b/cmake/CompileRocksDB.cmake
@@ -1,6 +1,6 @@
 # FindRocksDB
 
-find_package(RocksDB 8.6.7)
+find_package(RocksDB 8.10.0)
 
 include(ExternalProject)
 
@@ -51,8 +51,8 @@ if(ROCKSDB_FOUND)
     ${BINARY_DIR}/librocksdb.a)
 else()
   ExternalProject_Add(rocksdb
-    URL https://github.com/facebook/rocksdb/archive/refs/tags/v8.6.7.tar.gz
-    URL_HASH SHA256=cdb2fc3c6a556f20591f564cb8e023e56828469aa3f76e1d9535c443ba1f0c1a
+    URL https://github.com/facebook/rocksdb/archive/refs/tags/v8.10.0.tar.gz
+    URL_HASH SHA256=2dc107551cc864dbcf7908fdee96f2318cbb680df2b3fe1f85b0d545c2b5673b
     CMAKE_ARGS ${RocksDB_CMAKE_ARGS}
     BUILD_BYPRODUCTS <BINARY_DIR>/librocksdb.a
     INSTALL_COMMAND ""
diff --git a/fdbserver/FDBRocksDBVersion.h b/fdbserver/FDBRocksDBVersion.h
index ec3de38666b..d66531c200a 100644
--- a/fdbserver/FDBRocksDBVersion.h
+++ b/fdbserver/FDBRocksDBVersion.h
@@ -22,7 +22,7 @@
 #define FDBSERVER_FDBROCKSDBVERSION_H
 
 #define FDB_ROCKSDB_MAJOR 8
-#define FDB_ROCKSDB_MINOR 6
-#define FDB_ROCKSDB_PATCH 7
+#define FDB_ROCKSDB_MINOR 10
+#define FDB_ROCKSDB_PATCH 0
 
 #endif
\ No newline at end of file

From a5da13c9d040fb6294d8df11e9ab22950c97492d Mon Sep 17 00:00:00 2001
From: Johannes Scheuermann <johscheuer@users.noreply.github.com>
Date: Tue, 13 Feb 2024 20:41:53 +0100
Subject: [PATCH 395/468] [RELEASE-7.1] Abort process when abnormal shutdown is
 initiated to allow coredumps to be generated (#11191)

* Abort process when abnormal shutdown is initiated to allow coredumps to be generated

* Update link to abort function

* Add knob to allow fdbserver to abort under abnormal behaviour
---
 flow/Knobs.cpp          | 1 +
 flow/Knobs.h            | 3 +++
 flow/Platform.actor.cpp | 7 +++++++
 3 files changed, 11 insertions(+)

diff --git a/flow/Knobs.cpp b/flow/Knobs.cpp
index 1adbbe95874..5468d0f9189 100644
--- a/flow/Knobs.cpp
+++ b/flow/Knobs.cpp
@@ -71,6 +71,7 @@ void FlowKnobs::initialize(Randomize randomize, IsSimulated isSimulated) {
 	init( FAST_ALLOC_LOGGING_BYTES,                           10e6 );
 	init( HUGE_ARENA_LOGGING_BYTES,                          100e6 );
 	init( HUGE_ARENA_LOGGING_INTERVAL,                         5.0 );
+	init( ABORT_ON_FAILURE,                                  false );
 
 	init( MEMORY_USAGE_CHECK_INTERVAL,                         1.0 );
 
diff --git a/flow/Knobs.h b/flow/Knobs.h
index 848e953b624..8b45431d5b7 100644
--- a/flow/Knobs.h
+++ b/flow/Knobs.h
@@ -133,6 +133,9 @@ class FlowKnobs : public KnobsImpl<FlowKnobs> {
 	double FAST_ALLOC_LOGGING_BYTES;
 	double HUGE_ARENA_LOGGING_BYTES;
 	double HUGE_ARENA_LOGGING_INTERVAL;
+	// This setting allows to let the fdbserver abort instead of exit to generate coredumps
+	// in case of a failure.
+	bool ABORT_ON_FAILURE;
 
 	double MEMORY_USAGE_CHECK_INTERVAL;
 
diff --git a/flow/Platform.actor.cpp b/flow/Platform.actor.cpp
index aaa3915d513..748050b3767 100644
--- a/flow/Platform.actor.cpp
+++ b/flow/Platform.actor.cpp
@@ -3247,6 +3247,13 @@ extern "C" void flushAndExit(int exitCode) {
 	// to the crashAndDie call below.
 	TerminateProcess(GetCurrentProcess(), exitCode);
 #else
+	// Send a signal to allow the Kernel to generate a coredump for this process.
+	// See: https://man7.org/linux/man-pages/man5/core.5.html
+	// The abort method will send a SIGABRT, which causes the kernel to collect a coredump.
+	// See: https://man7.org/linux/man-pages/man3/abort.3.html.
+	if (exitCode != FDB_EXIT_SUCCESS && FLOW_KNOBS->ABORT_ON_FAILURE)
+		abort();
+	// In the success case exit the process gracefully.
 	_exit(exitCode);
 #endif
 	// should never reach here, but you never know

From ac8bdfceaa017aff461eb0e6c02b0d7ea9f140f6 Mon Sep 17 00:00:00 2001
From: Zhe Wang <zhe.wang@wustl.edu>
Date: Thu, 15 Feb 2024 18:44:45 -0800
Subject: [PATCH 396/468] Fix single consistency checker (#11205)

* reproduce the issue

* revert single consistency checker back
---
 fdbserver/tester.actor.cpp                    |   1 +
 .../workloads/ConsistencyCheck.actor.cpp      | 316 ++----------------
 2 files changed, 33 insertions(+), 284 deletions(-)

diff --git a/fdbserver/tester.actor.cpp b/fdbserver/tester.actor.cpp
index 9d1f8ade3fc..239c9163aad 100644
--- a/fdbserver/tester.actor.cpp
+++ b/fdbserver/tester.actor.cpp
@@ -1002,6 +1002,7 @@ ACTOR Future<Void> checkConsistency(Database cx,
 	                       KeyValueRef(LiteralStringRef("quiescentWaitTimeout"),
 	                                   ValueRef(options.arena(), format("%f", quiescentWaitTimeout))));
 	options.push_back_deep(options.arena(), KeyValueRef(LiteralStringRef("distributed"), LiteralStringRef("false")));
+	options.push_back_deep(options.arena(), KeyValueRef(LiteralStringRef("shuffleShards"), LiteralStringRef("true")));
 	spec.options.push_back_deep(spec.options.arena(), options);
 	state double start = now();
 	state bool lastRun = false;
diff --git a/fdbserver/workloads/ConsistencyCheck.actor.cpp b/fdbserver/workloads/ConsistencyCheck.actor.cpp
index 2ccb2a92867..070542934ff 100644
--- a/fdbserver/workloads/ConsistencyCheck.actor.cpp
+++ b/fdbserver/workloads/ConsistencyCheck.actor.cpp
@@ -219,38 +219,14 @@ struct ConsistencyCheckWorkload : TestWorkload {
 	ACTOR Future<Void> _start(Database cx, ConsistencyCheckWorkload* self) {
 		loop {
 			while (self->suspendConsistencyCheck.get()) {
-				TraceEvent("ConsistencyCheck_Suspended")
-				    .detail("ConsistencyCheckerId", self->consistencyCheckerId)
-				    .detail("Distributed", self->distributed)
-				    .detail("ClientCount", self->clientCount)
-				    .detail("ClientId", self->clientId)
-				    .detail("Indefinite", self->indefinite)
-				    .detail("Repetitions", self->repetitions)
-				    .detail("SuspendConsistencyCheck", self->suspendConsistencyCheck.get());
+				TraceEvent("ConsistencyCheck_Suspended").log();
 				wait(self->suspendConsistencyCheck.onChange());
 			}
-			TraceEvent("ConsistencyCheck_StartingOrResuming")
-			    .detail("ConsistencyCheckerId", self->consistencyCheckerId)
-			    .detail("Distributed", self->distributed)
-			    .detail("ClientCount", self->clientCount)
-			    .detail("ClientId", self->clientId)
-			    .detail("Indefinite", self->indefinite)
-			    .detail("Repetitions", self->repetitions)
-			    .detail("SuspendConsistencyCheck", self->suspendConsistencyCheck.get());
+			TraceEvent("ConsistencyCheck_StartingOrResuming").log();
 			choose {
 				when(wait(self->runCheck(cx, self))) {
-					if (!self->indefinite) {
-						TraceEvent("ConsistencyCheck_Exit")
-						    .detail("Mode", "Normal")
-						    .detail("ConsistencyCheckerId", self->consistencyCheckerId)
-						    .detail("Distributed", self->distributed)
-						    .detail("ClientCount", self->clientCount)
-						    .detail("ClientId", self->clientId)
-						    .detail("Indefinite", self->indefinite)
-						    .detail("Repetitions", self->repetitions)
-						    .detail("SuspendConsistencyCheck", self->suspendConsistencyCheck.get());
+					if (!self->indefinite)
 						break;
-					}
 					self->repetitions++;
 					wait(delay(5.0));
 				}
@@ -455,8 +431,8 @@ struct ConsistencyCheckWorkload : TestWorkload {
 					if (!coordinatorsCorrect)
 						self->testFailure("Coordinators incorrect");
 				}
-				// Get a list of key servers; verify that the TLogs and master all agree about who the key servers
-				// are
+
+				// Get a list of key servers; verify that the TLogs and master all agree about who the key servers are
 				state Promise<std::vector<std::pair<KeyRange, std::vector<StorageServerInterface>>>> keyServerPromise;
 				bool keyServerResult = wait(self->getKeyServers(cx, self, keyServerPromise, keyServersKeys));
 				if (keyServerResult) {
@@ -468,19 +444,9 @@ struct ConsistencyCheckWorkload : TestWorkload {
 					bool keyLocationResult = wait(self->getKeyLocations(cx, keyServers, self, keyLocationPromise));
 					if (keyLocationResult) {
 						state Standalone<VectorRef<KeyValueRef>> keyLocations = keyLocationPromise.getFuture().get();
-						state std::vector<std::pair<KeyRange, Value>> shardLocationPairList;
+
 						// Check that each shard has the same data on all storage servers that it resides on
-						for (int k = 0; k < keyLocations.size() - 1; k++) {
-							shardLocationPairList.push_back(
-							    std::make_pair(Standalone(KeyRangeRef(keyLocations[k].key, keyLocations[k + 1].key)),
-							                   Standalone(keyLocations[k].value)));
-						}
-						wait(::success(self->checkDataConsistency(cx,
-						                                          shardLocationPairList,
-						                                          configuration,
-						                                          tssMapping,
-						                                          self,
-						                                          /*consistencyCheckEpoch=*/0)));
+						wait(::success(self->checkDataConsistency(cx, keyLocations, configuration, tssMapping, self)));
 
 						// Cache consistency check
 						if (self->performCacheCheck)
@@ -492,25 +458,13 @@ struct ConsistencyCheckWorkload : TestWorkload {
 				    e.code() == error_code_wrong_shard_server || e.code() == error_code_all_alternatives_failed ||
 				    e.code() == error_code_process_behind)
 					TraceEvent("ConsistencyCheck_Retry")
-					    .error(e) // FIXME: consistency check does not retry in this case
-					    .detail("ConsistencyCheckerId", self->consistencyCheckerId)
-					    .detail("Distributed", self->distributed)
-					    .detail("ClientCount", self->clientCount)
-					    .detail("ClientId", self->clientId)
-					    .detail("Indefinite", self->indefinite)
-					    .detail("Repetitions", self->repetitions);
+					    .error(e); // FIXME: consistency check does not retry in this case
 				else
 					self->testFailure(format("Error %d - %s", e.code(), e.name()));
 			}
 		}
 
-		TraceEvent("ConsistencyCheck_FinishedCheck")
-		    .detail("ConsistencyCheckerId", self->consistencyCheckerId)
-		    .detail("Distributed", self->distributed)
-		    .detail("ClientCount", self->clientCount)
-		    .detail("ClientId", self->clientId)
-		    .detail("Indefinite", self->indefinite)
-		    .detail("Repetitions", self->repetitions);
+		TraceEvent("ConsistencyCheck_FinishedCheck").detail("Repetitions", self->repetitions);
 
 		return Void();
 	}
@@ -1856,16 +1810,11 @@ struct ConsistencyCheckWorkload : TestWorkload {
 
 	// Checks that the data in each shard is the same on each storage server that it resides on.  Also performs some
 	// sanity checks on the sizes of shards and storage servers. Returns false if there is a failure
-	// We create a new checkDataConsistency actor for retrying for failed ranges and we do consistencyCheckEpoch++
 	ACTOR Future<bool> checkDataConsistency(Database cx,
-	                                        std::vector<std::pair<KeyRange, Value>> shardLocationPairList,
+	                                        VectorRef<KeyValueRef> keyLocations,
 	                                        DatabaseConfiguration configuration,
 	                                        std::map<UID, StorageServerInterface> tssMapping,
-	                                        ConsistencyCheckWorkload* self,
-	                                        int consistencyCheckEpoch) {
-		if (consistencyCheckEpoch > 0) {
-			ASSERT(self->consistencyCheckerId != 0); // urgent mode
-		}
+	                                        ConsistencyCheckWorkload* self) {
 		// Stores the total number of bytes on each storage server
 		// In a distributed test, this will be an estimated size
 		state std::map<UID, int64_t> storageServerSizes;
@@ -1885,29 +1834,14 @@ struct ConsistencyCheckWorkload : TestWorkload {
 		              self->rateLimitMax,
 		              static_cast<int>(ceil(self->bytesReadInPreviousRound /
 		                                    (float)CLIENT_KNOBS->CONSISTENCY_CHECK_ONE_ROUND_TARGET_COMPLETION_TIME)));
-		if (self->consistencyCheckerId != 0) { // We set max speed when urgent mode
-			rateLimitForThisRound = self->rateLimitMax;
-		}
-		TraceEvent("ConsistencyCheck_RateLimitForThisRound")
-		    .detail("ConsistencyCheckerId", self->consistencyCheckerId)
-		    .detail("Distributed", self->distributed)
-		    .detail("ClientId", self->clientId)
-		    .detail("ClientCount", self->clientCount)
-		    .detail("RateLimit", rateLimitForThisRound)
-		    .detail("ConsistencyCheckEpoch", consistencyCheckEpoch)
-		    .detail("ShardLocationPairList", shardLocationPairList.size());
+		TraceEvent("ConsistencyCheck_RateLimitForThisRound").detail("RateLimit", rateLimitForThisRound);
 		ASSERT(rateLimitForThisRound >= 0 && rateLimitForThisRound <= self->rateLimitMax);
 		state Reference<IRateControl> rateLimiter = Reference<IRateControl>(new SpeedLimit(rateLimitForThisRound, 1));
 		state double rateLimiterStartTime = now();
 		state int64_t bytesReadInthisRound = 0;
 		state bool testResult = true;
-		state int64_t numShardThisClient = -1;
-		state int64_t numShardToCheck = -1;
 		state int64_t numCompleteShards = 0;
-		state int64_t numFailedShards = 0;
 		state int64_t numSkippedShards = 0;
-		state KeyRangeMap<bool> failedRanges; // Used to collect failed ranges in the current checkDataConsistency
-		// Which will be used to start the next consistencyCheckEpoch of the checkDataConsistency
 
 		state double dbSize = 100e12;
 		if (g_network->isSimulated()) {
@@ -1916,67 +1850,24 @@ struct ConsistencyCheckWorkload : TestWorkload {
 			dbSize = _dbSize;
 		}
 
-		state std::vector<KeyRange> ranges;
-		for (int k = 0; k < shardLocationPairList.size(); k++) {
-			ranges.push_back(shardLocationPairList[k].first); // Add shard range to ranges
+		state std::vector<KeyRangeRef> ranges;
+
+		for (int k = 0; k < keyLocations.size() - 1; k++) {
+			KeyRangeRef range(keyLocations[k].key, keyLocations[k + 1].key);
+			ranges.push_back(range);
 		}
 
 		state std::vector<int> shardOrder;
 		shardOrder.reserve(ranges.size());
 		for (int k = 0; k < ranges.size(); k++)
 			shardOrder.push_back(k);
-
 		if (self->shuffleShards) {
-			ASSERT(self->consistencyCheckerId != 0); // Urgent mode never does shuffle
 			uint32_t seed = self->sharedRandomNumber + self->repetitions;
 			DeterministicRandom sharedRandom(seed == 0 ? 1 : seed);
 			sharedRandom.randomShuffle(shardOrder);
 		}
 
-		if (self->consistencyCheckerId != 0) {
-			// In case of the urgent mode, overwrite the i and increment
-			i = 0;
-			increment = 1;
-			int endPoint = shardOrder.size();
-			numShardThisClient = (endPoint - i) / increment;
-			// Logging range assignment
-			KeyRangeMap<bool> rangeAssignmentMap;
-			std::vector<KeyRange> rangeAssignmentList;
-			for (int idx = i; idx < endPoint; idx += increment) {
-				rangeAssignmentMap.insert(ranges[shardOrder[idx]], true);
-			}
-			rangeAssignmentMap.coalesce(allKeys);
-			for (auto& assignedRange : rangeAssignmentMap.ranges()) {
-				if (assignedRange.value() == true) {
-					rangeAssignmentList.push_back(assignedRange.range());
-				}
-			}
-			TraceEvent e(consistencyCheckEpoch == 0 ? "ConsistencyCheck_StartTask"
-			                                        : "ConsistencyCheck_RetryFailedTask");
-			e.setMaxEventLength(-1);
-			e.setMaxFieldLength(-1);
-			e.detail("ConsistencyCheckerId", self->consistencyCheckerId);
-			e.detail("Distributed", self->distributed);
-			e.detail("ShardCount", ranges.size());
-			e.detail("ClientId", self->clientId);
-			e.detail("ClientCount", self->clientCount);
-			e.detail("Repetitions", self->repetitions);
-			e.detail("ConsistencyCheckEpoch", consistencyCheckEpoch);
-			if (consistencyCheckEpoch == 0) {
-				for (int idx = 0; idx < rangeAssignmentList.size(); idx++) {
-					e.detail("RangeToCheckBegin" + std::to_string(idx), rangeAssignmentList[idx].begin);
-					e.detail("RangeToCheckEnd" + std::to_string(idx), rangeAssignmentList[idx].end);
-				}
-			}
-			e.trackLatest(consistencyCheckEpoch == 0 ? "ConsistencyCheck_StartTask"
-			                                         : "ConsistencyCheck_RetryFailedTask");
-		}
-
-		for (; i < shardOrder.size(); i += increment) {
-			if (self->consistencyCheckerId != 0) { // Urgent mode
-				numShardToCheck = (shardOrder.size() - i) / increment;
-			}
-
+		for (; i < ranges.size(); i += increment) {
 			state int shard = shardOrder[i];
 
 			state KeyRangeRef range = ranges[shard];
@@ -1988,9 +1879,8 @@ struct ConsistencyCheckWorkload : TestWorkload {
 
 			RangeResult UIDtoTagMap = wait(tr.getRange(serverTagKeys, CLIENT_KNOBS->TOO_MANY));
 			ASSERT(!UIDtoTagMap.more && UIDtoTagMap.size() < CLIENT_KNOBS->TOO_MANY);
-			// Decode shard location value
 			decodeKeyServersValue(
-			    UIDtoTagMap, shardLocationPairList[shard].second, sourceStorageServers, destStorageServers, false);
+			    UIDtoTagMap, keyLocations[shard].value, sourceStorageServers, destStorageServers, false);
 
 			// If the destStorageServers is non-empty, then this shard is being relocated
 			state bool isRelocating = destStorageServers.size() > 0;
@@ -2062,9 +1952,7 @@ struct ConsistencyCheckWorkload : TestWorkload {
 				}
 			}
 
-			if (self->consistencyCheckerId == 0 &&
-			    (SERVER_KNOBS->CONSISTENCY_CHECK_ROCKSDB_ENGINE || SERVER_KNOBS->CONSISTENCY_CHECK_SQLITE_ENGINE)) {
-				// We can skip for shard for engine only in normal consistency check where consistencyCheckerId is 0
+			if (SERVER_KNOBS->CONSISTENCY_CHECK_ROCKSDB_ENGINE || SERVER_KNOBS->CONSISTENCY_CHECK_SQLITE_ENGINE) {
 				std::unordered_map<UID, KeyValueStoreType> storageTypeMapping =
 				    wait(self->getStorageType(storageServerInterfaces));
 
@@ -2117,8 +2005,7 @@ struct ConsistencyCheckWorkload : TestWorkload {
 
 			// The first client may need to skip the rest of the loop contents if it is just processing this shard to
 			// get a size estimate
-			if (self->consistencyCheckerId != 0 || !self->firstClient ||
-			    shard % (effectiveClientCount * self->shardSampleFactor) == 0) {
+			if (!self->firstClient || shard % (effectiveClientCount * self->shardSampleFactor) == 0) {
 				state int shardKeys = 0;
 				state int shardBytes = 0;
 				state int sampledBytes = 0;
@@ -2131,7 +2018,6 @@ struct ConsistencyCheckWorkload : TestWorkload {
 				state Key lastSampleKey;
 				state Key lastStartSampleKey;
 				state int64_t totalReadAmount = 0;
-				state bool completeCheck = true;
 
 				state KeySelector begin = firstGreaterOrEqual(range.begin);
 				state Transaction onErrorTr(
@@ -2288,8 +2174,6 @@ struct ConsistencyCheckWorkload : TestWorkload {
 										}
 
 										TraceEvent("ConsistencyCheck_DataInconsistent")
-										    .setMaxEventLength(-1)
-										    .setMaxFieldLength(-1)
 										    .detail(format("StorageServer%d", j).c_str(), storageServers[j].toString())
 										    .detail(format("StorageServer%d", firstValidServer).c_str(),
 										            storageServers[firstValidServer].toString())
@@ -2330,8 +2214,6 @@ struct ConsistencyCheckWorkload : TestWorkload {
 								TraceEvent("ConsistencyCheck_StorageServerUnavailable")
 								    .errorUnsuppressed(e)
 								    .suppressFor(1.0)
-								    .setMaxEventLength(-1)
-								    .setMaxFieldLength(-1)
 								    .detail("StorageServer", storageServers[j])
 								    .detail("ShardBegin", printable(range.begin))
 								    .detail("ShardEnd", printable(range.end))
@@ -2339,68 +2221,17 @@ struct ConsistencyCheckWorkload : TestWorkload {
 								    .detail("UID", storageServerInterfaces[j].id())
 								    .detail("GetKeyValuesToken",
 								            storageServerInterfaces[j].getKeyValues.getEndpoint().token)
-								    .detail("IsTSS", storageServerInterfaces[j].isTss() ? "True" : "False")
-								    .detail("Distributed", self->distributed)
-								    .detail("ClientId", self->clientId)
-								    .detail("ClientCount", self->clientCount)
-								    .detail("Repetitions", self->repetitions)
-								    .detail("ConsistencyCheckEpoch", consistencyCheckEpoch);
-
-								completeCheck = false;
-								if (self->consistencyCheckerId != 0) {
-									break; // Will retry later
-								}
+								    .detail("IsTSS", storageServerInterfaces[j].isTss() ? "True" : "False");
 
 								// All shards should be available in quiscence
 								if (self->performQuiescentChecks && !storageServerInterfaces[j].isTss()) {
 									self->testFailure("Storage server unavailable");
 									return false;
 								}
-
-							} else {
-								if (!rangeResult.present()) {
-									TraceEvent("ConsistencyCheck_StorageServerUnavailable")
-									    .suppressFor(1.0)
-									    .detail("ShardBegin", req.begin.getKey())
-									    .detail("ShardEnd", req.end.getKey())
-									    .detail("Reason", "rangeResultNotPresent")
-									    .detail("IsTSS", storageServerInterfaces[j].isTss())
-									    .detail("SSID", storageServerInterfaces[j].id());
-								} else if (rangeResult.get().error.present()) {
-									TraceEvent("ConsistencyCheck_StorageServerUnavailable")
-									    .suppressFor(1.0)
-									    .detail("ShardBegin", req.begin.getKey())
-									    .detail("ShardEnd", req.end.getKey())
-									    .detail("Reason", rangeResult.get().error.get().what())
-									    .detail("IsTSS", storageServerInterfaces[j].isTss())
-									    .detail("SSID", storageServerInterfaces[j].id());
-								}
-								completeCheck = false;
-								if (self->consistencyCheckerId != 0) {
-									break; // Will retry later
-								}
 							}
 						}
 
-						if (self->consistencyCheckerId != 0 && !completeCheck) {
-							// Shard incomplete check in urgent mode
-							// Add the shard to failedRanges, which will be retried later
-							failedRanges.insert(range, true);
-							TraceEvent(SevInfo, "ConsistencyCheck_ShardAddedToRetry")
-							    .setMaxEventLength(-1)
-							    .setMaxFieldLength(-1)
-							    .detail("ConsistencyCheckerId", self->consistencyCheckerId)
-							    .detail("ClientId", self->clientId)
-							    .detail("ClientCount", self->clientCount)
-							    .detail("ShardBegin", range.begin)
-							    .detail("ShardEnd", range.end)
-							    .detail("Repetitions", self->repetitions)
-							    .detail("ConsistencyCheckEpoch", consistencyCheckEpoch)
-							    .trackLatest("ConsistencyCheck_ShardAddedToRetry");
-							break; // skip the entire shard, retry later
-						}
-
-						if (firstValidServer >= 0 && self->consistencyCheckerId == 0) {
+						if (firstValidServer >= 0) {
 							VectorRef<KeyValueRef> data = keyValueFutures[firstValidServer].get().get().data;
 							// Calculate the size of the shard, the variance of the shard size estimate, and the correct
 							// shard size estimate
@@ -2476,47 +2307,6 @@ struct ConsistencyCheckWorkload : TestWorkload {
 					}
 				}
 
-				if (!completeCheck) {
-					numFailedShards++;
-					TraceEvent(self->consistencyCheckerId != 0 ? SevInfo : SevWarn, "ConsistencyCheck_ShardFailed")
-					    .suppressFor(1.0)
-					    .setMaxEventLength(-1)
-					    .setMaxFieldLength(-1)
-					    .detail("ConsistencyCheckerId", self->consistencyCheckerId)
-					    .detail("ClientId", self->clientId)
-					    .detail("ClientCount", self->clientCount)
-					    .detail("NumCompletedShards", numCompleteShards)
-					    .detail("NumFailedShards", numFailedShards)
-					    .detail("NumShardThisClient", numShardThisClient)
-					    .detail("NumShardToCheckThisEpoch", numShardToCheck - 1)
-					    .detail("ShardBegin", range.begin)
-					    .detail("ShardEnd", range.end)
-					    .detail("Repetitions", self->repetitions)
-					    .detail("ConsistencyCheckEpoch", consistencyCheckEpoch);
-				} else {
-					numCompleteShards++;
-					TraceEvent(SevInfo, "ConsistencyCheck_ShardComplete")
-					    .suppressFor(1.0)
-					    .setMaxEventLength(-1)
-					    .setMaxFieldLength(-1)
-					    .detail("ConsistencyCheckerId", self->consistencyCheckerId)
-					    .detail("ClientId", self->clientId)
-					    .detail("ClientCount", self->clientCount)
-					    .detail("Range", range)
-					    .detail("ShardCount", ranges.size())
-					    .detail("NumCompletedShards", numCompleteShards)
-					    .detail("NumFailedShards", numFailedShards)
-					    .detail("NumShardThisClient", numShardThisClient)
-					    .detail("NumShardToCheckThisEpoch", numShardToCheck - 1)
-					    .detail("BytesReadInThisRound", bytesReadInthisRound)
-					    .detail("NumSkippedShards", numSkippedShards)
-					    .detail("ConsistencyCheckEpoch", consistencyCheckEpoch);
-				}
-
-				if (self->consistencyCheckerId != 0) { // Urgent mode
-					continue; // Bypass following check not-relevant to userdata/metadata consistency
-				}
-
 				canSplit = canSplit && sampledBytes - splitBytes >= shardBounds.min.bytes && sampledBytes > splitBytes;
 
 				// Update the size of all storage servers containing this shard
@@ -2636,43 +2426,15 @@ struct ConsistencyCheckWorkload : TestWorkload {
 				    .detail("Range", range)
 				    .detail("BytesRead", bytesReadInRange);
 			}
-		}
 
-		if (self->consistencyCheckerId != 0) {
-			failedRanges.coalesce(allKeys);
-			state std::vector<KeyRange> failedRangesToCheck;
-			state KeyRangeMap<bool>::Ranges failedRangesList = failedRanges.ranges();
-			state KeyRangeMap<bool>::iterator failedRangesIter = failedRangesList.begin();
-			for (; failedRangesIter != failedRangesList.end(); ++failedRangesIter) {
-				if (failedRangesIter->value()) {
-					failedRangesToCheck.push_back(failedRangesIter->range());
-				}
-			}
-			if (failedRangesToCheck.size() > 0) { // Retry for any failed shard
-				wait(delay(60.0)); // Backoff 1 min
-				state std::vector<std::pair<KeyRange, Value>> shardLocationPairListForFailedRanges =
-				    wait(self->getKeyLocationsForRangeList(cx, failedRangesToCheck, self));
-				TraceEvent(SevInfo, "ConsistencyCheck_StartHandlingFailedRanges")
-				    .setMaxEventLength(-1)
-				    .setMaxFieldLength(-1)
-				    .detail("ConsistencyCheckerId", self->consistencyCheckerId)
-				    .detail("FailedCollectedRangeCount", failedRangesToCheck.size())
-				    .detail("FailedShardCount", shardLocationPairListForFailedRanges.size())
-				    .detail("ConsistencyCheckEpoch", consistencyCheckEpoch)
-				    .detail("Distributed", self->distributed)
-				    .detail("ClientId", self->clientId)
-				    .detail("ClientCount", self->clientCount);
-				if (consistencyCheckEpoch < CLIENT_KNOBS->CONSISTENCY_CHECK_RETRY_DEPTH_MAX) {
-					wait(::success(self->checkDataConsistency(cx,
-					                                          shardLocationPairListForFailedRanges,
-					                                          configuration,
-					                                          tssMapping,
-					                                          self,
-					                                          consistencyCheckEpoch + 1)));
-				}
-				// We give up retrying when retry too many times
-				// The failed ranges will be picked up by the next round
-			}
+			numCompleteShards++;
+			TraceEvent(SevInfo, "ConsistencyCheck_ShardComplete")
+			    .suppressFor(1.0)
+			    .detail("Range", range.toString())
+			    .detail("ShardCount", ranges.size())
+			    .detail("NumCompletedShards", numCompleteShards)
+			    .detail("BytesReadInThisRound", bytesReadInthisRound)
+			    .detail("NumSkippedShards", numSkippedShards);
 		}
 
 		// SOMEDAY: when background data distribution is implemented, include this test
@@ -2697,21 +2459,7 @@ struct ConsistencyCheckWorkload : TestWorkload {
 		    }
 		}*/
 
-		if (consistencyCheckEpoch == 0) {
-			TraceEvent("ConsistencyCheck_EndTask")
-			    .setMaxEventLength(-1)
-			    .setMaxFieldLength(-1)
-			    .detail("ConsistencyCheckerId", self->consistencyCheckerId)
-			    .detail("Distributed", self->distributed)
-			    .detail("ShardCount", ranges.size())
-			    .detail("ClientId", self->clientId)
-			    .detail("ClientCount", self->clientCount)
-			    .detail("Repetitions", self->repetitions)
-			    .trackLatest("ConsistencyCheck_EndTask");
-			self->bytesReadInPreviousRound =
-			    bytesReadInthisRound; // FIXME: missing bytes read by retrying failed ranges
-		}
-
+		self->bytesReadInPreviousRound = bytesReadInthisRound;
 		return testResult;
 	}
 

From e8856e9e2af6e084baf5f7c2feb56cf170b5c217 Mon Sep 17 00:00:00 2001
From: Zhe Wang <zhe.wang@wustl.edu>
Date: Thu, 15 Feb 2024 21:16:54 -0800
Subject: [PATCH 397/468] fix sev error in distributed consistency checker
 (#11203)

---
 fdbserver/tester.actor.cpp | 24 +++++++++++++-----------
 1 file changed, 13 insertions(+), 11 deletions(-)

diff --git a/fdbserver/tester.actor.cpp b/fdbserver/tester.actor.cpp
index 239c9163aad..8e08c7020c8 100644
--- a/fdbserver/tester.actor.cpp
+++ b/fdbserver/tester.actor.cpp
@@ -525,7 +525,8 @@ void sendResult(ReplyPromise<T>& reply, Optional<ErrorOr<T>> const& result) {
 ACTOR Future<Void> runWorkloadAsync(Database cx,
                                     WorkloadInterface workIface,
                                     Reference<TestWorkload> workload,
-                                    double databasePingDelay) {
+                                    double databasePingDelay,
+                                    bool isConsistencyCheckUrgent) {
 	state Optional<ErrorOr<Void>> setupResult;
 	state Optional<ErrorOr<Void>> startResult;
 	state Optional<ErrorOr<CheckReply>> checkResult;
@@ -552,7 +553,7 @@ ACTOR Future<Void> runWorkloadAsync(Database cx,
 					setupResult = Void();
 				} catch (Error& e) {
 					setupResult = operation_failed();
-					TraceEvent(SevError, "TestSetupError", workIface.id())
+					TraceEvent(isConsistencyCheckUrgent ? SevWarn : SevError, "TestSetupError", workIface.id())
 					    .error(e)
 					    .detail("Workload", workload->description());
 					if (e.code() == error_code_please_reboot || e.code() == error_code_please_reboot_delete)
@@ -576,9 +577,7 @@ ACTOR Future<Void> runWorkloadAsync(Database cx,
 					startResult = operation_failed();
 					if (e.code() == error_code_please_reboot || e.code() == error_code_please_reboot_delete)
 						throw;
-					TraceEvent(e.code() == error_code_consistency_check_task_failed ? SevWarn : SevError,
-					           "TestFailure",
-					           workIface.id())
+					TraceEvent(isConsistencyCheckUrgent ? SevWarn : SevError, "TestFailure", workIface.id())
 					    .errorUnsuppressed(e)
 					    .detail("Reason", "Error starting workload")
 					    .detail("Workload", workload->description());
@@ -646,7 +645,8 @@ ACTOR Future<Void> runWorkloadAsync(Database cx,
 ACTOR Future<Void> testerServerWorkload(WorkloadRequest work,
                                         Reference<IClusterConnectionRecord> ccr,
                                         Reference<AsyncVar<struct ServerDBInfo> const> dbInfo,
-                                        LocalityData locality) {
+                                        LocalityData locality,
+                                        bool isConsistencyCheckUrgent) {
 	state WorkloadInterface workIface;
 	state bool replied = false;
 	state Database cx;
@@ -672,8 +672,9 @@ ACTOR Future<Void> testerServerWorkload(WorkloadRequest work,
 			fprintf(stderr, "ERROR: The workload could not be created.\n");
 			throw test_specification_invalid();
 		}
-		Future<Void> test = runWorkloadAsync(cx, workIface, workload, work.databasePingDelay) ||
-		                    traceRole(Role::TESTER, workIface.id());
+		Future<Void> test =
+		    runWorkloadAsync(cx, workIface, workload, work.databasePingDelay, isConsistencyCheckUrgent) ||
+		    traceRole(Role::TESTER, workIface.id());
 		work.reply.send(workIface);
 		replied = true;
 
@@ -740,14 +741,15 @@ ACTOR Future<Void> testerServerCore(TesterInterface interf,
 					    .detail("ClientId", work.clientId)
 					    .detail("ClientCount", work.clientCount);
 				}
-				consistencyCheckerUrgentTester =
-				    std::make_pair(work.sharedRandomNumber, testerServerWorkload(work, ccr, dbInfo, locality));
+				consistencyCheckerUrgentTester = std::make_pair(
+				    work.sharedRandomNumber,
+				    testerServerWorkload(work, ccr, dbInfo, locality, /*isConsistencyCheckUrgent=*/true));
 				TraceEvent(SevInfo, "ConsistencyCheckUrgent_ServerWorkloadStart", interf.id())
 				    .detail("ConsistencyCheckerId", consistencyCheckerUrgentTester.first)
 				    .detail("ClientId", work.clientId)
 				    .detail("ClientCount", work.clientCount);
 			} else {
-				addWorkload.send(testerServerWorkload(work, ccr, dbInfo, locality));
+				addWorkload.send(testerServerWorkload(work, ccr, dbInfo, locality, /*isConsistencyCheckUrgent=*/false));
 			}
 		}
 	}

From 8e434dfeaf00b47abafb74e647b8c8a9ec1d23ae Mon Sep 17 00:00:00 2001
From: Johannes Scheuermann <johscheuer@users.noreply.github.com>
Date: Fri, 16 Feb 2024 23:23:54 +0100
Subject: [PATCH 398/468] Allow to disable the removal of maintenance mode when
 a SS outside of the maintenance zone fails (#11207)

---
 fdbclient/ServerKnobs.cpp            | 1 +
 fdbclient/ServerKnobs.h              | 2 ++
 fdbserver/DDTeamCollection.actor.cpp | 3 ++-
 3 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/fdbclient/ServerKnobs.cpp b/fdbclient/ServerKnobs.cpp
index 560b70177dc..930d6076d8d 100644
--- a/fdbclient/ServerKnobs.cpp
+++ b/fdbclient/ServerKnobs.cpp
@@ -293,6 +293,7 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi
 	init( DD_STORAGE_WIGGLE_PAUSE_THRESHOLD,                      10 ); if( randomize && BUGGIFY ) DD_STORAGE_WIGGLE_PAUSE_THRESHOLD = 1000;
 	init( DD_STORAGE_WIGGLE_STUCK_THRESHOLD,                      20 );
 	init( DD_BUILD_EXTRA_TEAMS_OVERRIDE,                          10 ); if( randomize && BUGGIFY ) DD_BUILD_EXTRA_TEAMS_OVERRIDE = 2;
+	init( DD_REMOVE_MAINTENANCE_ON_FAILURE,                     true ); if( randomize && BUGGIFY ) DD_REMOVE_MAINTENANCE_ON_FAILURE = false;
 	init( ENABLE_STORAGE_QUEUE_AWARE_TEAM_SELECTION,           false ); if( randomize && BUGGIFY ) ENABLE_STORAGE_QUEUE_AWARE_TEAM_SELECTION = true;
 	init( TRACE_STORAGE_QUEUE_AWARE_GET_TEAM_FOR_MANUAL_SPLIT_ONLY, true ); if (isSimulated) TRACE_STORAGE_QUEUE_AWARE_GET_TEAM_FOR_MANUAL_SPLIT_ONLY = false;
 	init( DD_TARGET_STORAGE_QUEUE_SIZE, TARGET_BYTES_PER_STORAGE_SERVER*0.35 ); if( randomize && BUGGIFY ) DD_TARGET_STORAGE_QUEUE_SIZE = TARGET_BYTES_PER_STORAGE_SERVER*0.035;
diff --git a/fdbclient/ServerKnobs.h b/fdbclient/ServerKnobs.h
index d2bce12a70b..4389bdbebbb 100644
--- a/fdbclient/ServerKnobs.h
+++ b/fdbclient/ServerKnobs.h
@@ -265,6 +265,8 @@ class ServerKnobs : public KnobsImpl<ServerKnobs> {
 	double DD_FAILURE_TIME;
 	double DD_ZERO_HEALTHY_TEAM_DELAY;
 	int DD_BUILD_EXTRA_TEAMS_OVERRIDE; // build extra teams to allow data movement to progress. must be larger than 0
+	bool DD_REMOVE_MAINTENANCE_ON_FAILURE; // If set to true DD will remove the maintenance mode if another SS fails
+	                                       // outside of the maintenance zone.
 
 	// Run storage enginee on a child process on the same machine with storage process
 	bool REMOTE_KV_STORE;
diff --git a/fdbserver/DDTeamCollection.actor.cpp b/fdbserver/DDTeamCollection.actor.cpp
index 0f91367a325..d8a7e08e009 100644
--- a/fdbserver/DDTeamCollection.actor.cpp
+++ b/fdbserver/DDTeamCollection.actor.cpp
@@ -1684,7 +1684,8 @@ class DDTeamCollectionImpl {
 							    .detail("ServerID", interf.id())
 							    .detail("Status", status->toString());
 							status->isFailed = false;
-						} else if (self->clearHealthyZoneFuture.isReady()) {
+						} else if (SERVER_KNOBS->DD_REMOVE_MAINTENANCE_ON_FAILURE &&
+						           self->clearHealthyZoneFuture.isReady()) {
 							self->clearHealthyZoneFuture = clearHealthyZone(self->cx);
 							TraceEvent("MaintenanceZoneCleared", self->distributorId).log();
 							self->healthyZone.set(Optional<Key>());

From 56cce59607451747e19f74c9ba70dc8df213b0a0 Mon Sep 17 00:00:00 2001
From: FoundationDB CI <foundationdb_ci@apple.com>
Date: Sun, 18 Feb 2024 20:31:22 +0000
Subject: [PATCH 399/468] disable AVX for 7.1.56 release

---
 cmake/ConfigureCompiler.cmake | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cmake/ConfigureCompiler.cmake b/cmake/ConfigureCompiler.cmake
index e52d8665ee8..7e50469ba6a 100644
--- a/cmake/ConfigureCompiler.cmake
+++ b/cmake/ConfigureCompiler.cmake
@@ -253,7 +253,7 @@ else()
       set(USE_AVX512F OFF)
     endif()
   endif()
-  set(USE_AVX ON CACHE BOOL "Enable AVX instructions")
+  set(USE_AVX OFF CACHE BOOL "Enable AVX instructions")
   if (USE_AVX)
     if (CMAKE_HOST_SYSTEM_PROCESSOR MATCHES "^x86")
       add_compile_options(-mavx)

From 05afc09840a77e93f012746ef5ad8ebd1c0bea43 Mon Sep 17 00:00:00 2001
From: FoundationDB CI <foundationdb_ci@apple.com>
Date: Sun, 18 Feb 2024 20:31:34 +0000
Subject: [PATCH 400/468] enable AVX and update version for 7.1.57 release

---
 CMakeLists.txt                | 2 +-
 cmake/ConfigureCompiler.cmake | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 4c23e845278..8a4dd18d834 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -23,7 +23,7 @@ else()
 endif()
 
 project(foundationdb
-  VERSION 7.1.56
+  VERSION 7.1.57
   DESCRIPTION "FoundationDB is a scalable, fault-tolerant, ordered key-value store with full ACID transactions."
   HOMEPAGE_URL "http://www.foundationdb.org/"
   LANGUAGES C CXX ASM)
diff --git a/cmake/ConfigureCompiler.cmake b/cmake/ConfigureCompiler.cmake
index 7e50469ba6a..e52d8665ee8 100644
--- a/cmake/ConfigureCompiler.cmake
+++ b/cmake/ConfigureCompiler.cmake
@@ -253,7 +253,7 @@ else()
       set(USE_AVX512F OFF)
     endif()
   endif()
-  set(USE_AVX OFF CACHE BOOL "Enable AVX instructions")
+  set(USE_AVX ON CACHE BOOL "Enable AVX instructions")
   if (USE_AVX)
     if (CMAKE_HOST_SYSTEM_PROCESSOR MATCHES "^x86")
       add_compile_options(-mavx)

From d5707f72064db58cda503670a45a6c8fbbd01c3f Mon Sep 17 00:00:00 2001
From: FoundationDB CI <foundationdb_ci@apple.com>
Date: Mon, 19 Feb 2024 15:08:07 +0000
Subject: [PATCH 401/468] update version after 7.1.57 release

---
 CMakeLists.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 8a4dd18d834..c3cddb7749c 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -23,7 +23,7 @@ else()
 endif()
 
 project(foundationdb
-  VERSION 7.1.57
+  VERSION 7.1.58
   DESCRIPTION "FoundationDB is a scalable, fault-tolerant, ordered key-value store with full ACID transactions."
   HOMEPAGE_URL "http://www.foundationdb.org/"
   LANGUAGES C CXX ASM)

From 8cbc54471543730e426f860162e7b76aba19236b Mon Sep 17 00:00:00 2001
From: Zhe Wang <zhe.wang@wustl.edu>
Date: Mon, 19 Feb 2024 10:43:39 -0800
Subject: [PATCH 402/468] Add 7.1.56 to 7.1.57 release notes [release-7.1]
 (#11209)

---
 .../source/release-notes/release-notes-710.rst     | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/documentation/sphinx/source/release-notes/release-notes-710.rst b/documentation/sphinx/source/release-notes/release-notes-710.rst
index e76f233c8ee..4fbf393de79 100644
--- a/documentation/sphinx/source/release-notes/release-notes-710.rst
+++ b/documentation/sphinx/source/release-notes/release-notes-710.rst
@@ -4,6 +4,20 @@
 Release Notes
 #############
 
+7.1.57
+======
+* Same as 7.1.56 release with AVX enabled.
+
+7.1.56
+======
+* Released with AVX disabled.
+* Added a knob DD_REMOVE_MAINTENANCE_ON_FAILURE to remove the maintenance mode if a failure outside of the maintenance zone happens. `(PR #11207) <https://github.com/apple/foundationdb/pull/11207>`_
+* Fixed start failure of single-threaded consistency checker. `(PR #11205) <https://github.com/apple/foundationdb/pull/11205>`_
+* Fixed false alarms of Sev40 TestFailure in consistency checker urgent mode. `(PR #11203) <https://github.com/apple/foundationdb/pull/11203>`_
+* Added a knob ABORT_ON_FAILURE to generate core dumps for failures. `(PR #11191) <https://github.com/apple/foundationdb/pull/11191>`_
+* Upgraded to RocksDB 8.10.0. `(PR #11176) <https://github.com/apple/foundationdb/pull/11176>`_
+* Fixed find toml in release-7.1 to avoid macos pr failures. `(PR #11190) <https://github.com/apple/foundationdb/pull/11190>`_
+
 7.1.55
 ======
 * Same as 7.1.54 release with AVX enabled.

From 8cfba79a1b04f322372739e327e4ab3991111018 Mon Sep 17 00:00:00 2001
From: Zhe Wang <zhe.wang@wustl.edu>
Date: Fri, 1 Mar 2024 10:52:28 -0800
Subject: [PATCH 403/468] cherrypick pr 11229 (#11230)

---
 fdbserver/tester.actor.cpp                     | 7 +++++--
 fdbserver/workloads/ConsistencyCheck.actor.cpp | 7 +++++--
 2 files changed, 10 insertions(+), 4 deletions(-)

diff --git a/fdbserver/tester.actor.cpp b/fdbserver/tester.actor.cpp
index 8e08c7020c8..178c6c7f77e 100644
--- a/fdbserver/tester.actor.cpp
+++ b/fdbserver/tester.actor.cpp
@@ -1211,8 +1211,11 @@ ACTOR Future<std::vector<KeyRange>> getConsistencyCheckShards(Database cx, std::
 			tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
 			tr.setOption(FDBTransactionOptions::LOCK_AWARE);
 			KeyRange rangeToRead = Standalone(KeyRangeRef(beginKeyToReadKeyServer, endKeyToReadKeyServer));
-			RangeResult readResult = wait(krmGetRanges(
-			    &tr, keyServersPrefix, rangeToRead, CLIENT_KNOBS->TOO_MANY, GetRangeLimits::BYTE_LIMIT_UNLIMITED));
+			RangeResult readResult = wait(krmGetRanges(&tr,
+			                                           keyServersPrefix,
+			                                           rangeToRead,
+			                                           SERVER_KNOBS->MOVE_KEYS_KRM_LIMIT,
+			                                           SERVER_KNOBS->MOVE_KEYS_KRM_LIMIT_BYTES));
 			for (int i = 0; i < readResult.size() - 1; ++i) {
 				KeyRange rangeToCheck = Standalone(KeyRangeRef(readResult[i].key, readResult[i + 1].key));
 				Value valueToCheck = Standalone(readResult[i].value);
diff --git a/fdbserver/workloads/ConsistencyCheck.actor.cpp b/fdbserver/workloads/ConsistencyCheck.actor.cpp
index 070542934ff..9132f0df826 100644
--- a/fdbserver/workloads/ConsistencyCheck.actor.cpp
+++ b/fdbserver/workloads/ConsistencyCheck.actor.cpp
@@ -1199,8 +1199,11 @@ struct ConsistencyCheckWorkload : TestWorkload {
 				tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
 				tr.setOption(FDBTransactionOptions::LOCK_AWARE);
 				KeyRange rangeToRead = Standalone(KeyRangeRef(beginKeyToReadKeyServer, endKeyToReadKeyServer));
-				RangeResult readResult = wait(krmGetRanges(
-				    &tr, keyServersPrefix, rangeToRead, CLIENT_KNOBS->TOO_MANY, GetRangeLimits::BYTE_LIMIT_UNLIMITED));
+				RangeResult readResult = wait(krmGetRanges(&tr,
+				                                           keyServersPrefix,
+				                                           rangeToRead,
+				                                           SERVER_KNOBS->MOVE_KEYS_KRM_LIMIT,
+				                                           SERVER_KNOBS->MOVE_KEYS_KRM_LIMIT_BYTES));
 				for (int i = 0; i < readResult.size() - 1; ++i) {
 					KeyRange rangeToCheck = Standalone(KeyRangeRef(readResult[i].key, readResult[i + 1].key));
 					Value valueToCheck = Standalone(readResult[i].value);

From 11b41cdb994c9a896f5abe0c343e0dd34cc25e9b Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Thu, 29 Feb 2024 15:38:09 -0800
Subject: [PATCH 404/468] Update doctest to version 2.4.8

---
 bindings/c/test/unit/third_party/CMakeLists.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bindings/c/test/unit/third_party/CMakeLists.txt b/bindings/c/test/unit/third_party/CMakeLists.txt
index 014157553c8..5f26630744f 100644
--- a/bindings/c/test/unit/third_party/CMakeLists.txt
+++ b/bindings/c/test/unit/third_party/CMakeLists.txt
@@ -6,7 +6,7 @@ ExternalProject_Add(
     doctest
     PREFIX ${CMAKE_BINARY_DIR}/doctest
     GIT_REPOSITORY https://github.com/onqtam/doctest.git
-    GIT_TAG 8424be522357e68d8c6178375546bb0cf9d5f6b3 # v2.4.1
+    GIT_TAG 7b9885133108ae301ddd16e2651320f54cafeba7 # v2.4.8
     TIMEOUT 10
     CONFIGURE_COMMAND ""
     BUILD_COMMAND ""

From a286cfddd84f8d07e58abdffb2372c10ab851886 Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Fri, 8 Mar 2024 18:01:51 -0800
Subject: [PATCH 405/468] Fix a DR corruption bug

The clear in status call can cause corruptions.

To reproduce with clang build:
seed: -f ./tests/slow/ApiCorrectnessSwitchover.toml -s 223736449 -b on
commit: f8eca6a80
Apply the ./tests/slow/ApiCorrectnessSwitchover.toml file in this commit.
---
 fdbserver/Status.actor.cpp                | 7 -------
 tests/slow/ApiCorrectnessSwitchover.toml  | 4 ++++
 tests/slow/WriteDuringReadSwitchover.toml | 4 ++++
 3 files changed, 8 insertions(+), 7 deletions(-)

diff --git a/fdbserver/Status.actor.cpp b/fdbserver/Status.actor.cpp
index c657f265c18..07e2eef1cda 100644
--- a/fdbserver/Status.actor.cpp
+++ b/fdbserver/Status.actor.cpp
@@ -1531,13 +1531,6 @@ ACTOR static Future<Void> logRangeWarningFetcher(Database cx,
 							break;
 						}
 						existingRanges.insert(rangePair);
-					} else {
-						// This cleanup is done during status, because it should only be required once after upgrading
-						// to 6.2.7 or later. There is no other good location to detect that the metadata is mismatched.
-						TraceEvent(SevWarnAlways, "CleaningDestUidLookup")
-						    .detail("K", it.key.printable())
-						    .detail("V", it.value.printable());
-						tr.clear(it.key);
 					}
 				}
 				wait(tr.commit() || timeoutFuture);
diff --git a/tests/slow/ApiCorrectnessSwitchover.toml b/tests/slow/ApiCorrectnessSwitchover.toml
index 3b4bfd56809..2b560cf633f 100644
--- a/tests/slow/ApiCorrectnessSwitchover.toml
+++ b/tests/slow/ApiCorrectnessSwitchover.toml
@@ -30,3 +30,7 @@ runSetup = true
 
     [[test.workload]]
     testName = 'AtomicSwitchover'
+
+    [[test.workload]]
+    testName = 'Status'
+    testDuration = 30.0
diff --git a/tests/slow/WriteDuringReadSwitchover.toml b/tests/slow/WriteDuringReadSwitchover.toml
index b5232c91191..bd546ce0a79 100644
--- a/tests/slow/WriteDuringReadSwitchover.toml
+++ b/tests/slow/WriteDuringReadSwitchover.toml
@@ -39,3 +39,7 @@ simBackupAgents = 'BackupToDB'
     machinesToLeave = 3
     reboot = true
     testDuration = 60.0
+
+    [[test.workload]]
+    testName = 'Status'
+    testDuration = 60.0

From 3316566c50927a166278ec91da4c5aa15f8509b4 Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Fri, 8 Mar 2024 18:32:13 -0800
Subject: [PATCH 406/468] Fix clang compiling errors

---
 bindings/c/test/unit/third_party/CMakeLists.txt | 2 +-
 bindings/c/test/unit/unit_tests.cpp             | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/bindings/c/test/unit/third_party/CMakeLists.txt b/bindings/c/test/unit/third_party/CMakeLists.txt
index 5f26630744f..04b9097a644 100644
--- a/bindings/c/test/unit/third_party/CMakeLists.txt
+++ b/bindings/c/test/unit/third_party/CMakeLists.txt
@@ -6,7 +6,7 @@ ExternalProject_Add(
     doctest
     PREFIX ${CMAKE_BINARY_DIR}/doctest
     GIT_REPOSITORY https://github.com/onqtam/doctest.git
-    GIT_TAG 7b9885133108ae301ddd16e2651320f54cafeba7 # v2.4.8
+    GIT_TAG ae7a13539fb71f270b87eb2e874fbac80bc8dda2 # v2.4.11
     TIMEOUT 10
     CONFIGURE_COMMAND ""
     BUILD_COMMAND ""
diff --git a/bindings/c/test/unit/unit_tests.cpp b/bindings/c/test/unit/unit_tests.cpp
index e9c029fc413..179e8b35436 100644
--- a/bindings/c/test/unit/unit_tests.cpp
+++ b/bindings/c/test/unit/unit_tests.cpp
@@ -1062,7 +1062,7 @@ TEST_CASE("fdb_transaction_get_mapped_range") {
 
 		int id = beginId;
 		for (int i = 0; i < expectSize; i++, id++) {
-			const auto& [key, value, begin, end, range_results] = result.mkvs[i];
+			const auto [key, value, begin, end, range_results] = result.mkvs[i];
 			CHECK(indexEntryKey(id).compare(key) == 0);
 			CHECK(EMPTY.compare(value) == 0);
 			CHECK(range_results.size() == SPLIT_SIZE);

From c9b9cc1621a69fade2697fdd066a7dbd16ba41f9 Mon Sep 17 00:00:00 2001
From: FoundationDB CI <foundationdb_ci@apple.com>
Date: Tue, 12 Mar 2024 21:58:48 +0000
Subject: [PATCH 407/468] disable AVX for 7.1.58 release

---
 cmake/ConfigureCompiler.cmake | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cmake/ConfigureCompiler.cmake b/cmake/ConfigureCompiler.cmake
index e52d8665ee8..7e50469ba6a 100644
--- a/cmake/ConfigureCompiler.cmake
+++ b/cmake/ConfigureCompiler.cmake
@@ -253,7 +253,7 @@ else()
       set(USE_AVX512F OFF)
     endif()
   endif()
-  set(USE_AVX ON CACHE BOOL "Enable AVX instructions")
+  set(USE_AVX OFF CACHE BOOL "Enable AVX instructions")
   if (USE_AVX)
     if (CMAKE_HOST_SYSTEM_PROCESSOR MATCHES "^x86")
       add_compile_options(-mavx)

From 38852379c1972ad759d1e1e8109ba01dc8e80d7a Mon Sep 17 00:00:00 2001
From: FoundationDB CI <foundationdb_ci@apple.com>
Date: Tue, 12 Mar 2024 21:59:50 +0000
Subject: [PATCH 408/468] enable AVX and update version for 7.1.59 release

---
 CMakeLists.txt                | 2 +-
 cmake/ConfigureCompiler.cmake | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index c3cddb7749c..8aabac3f63a 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -23,7 +23,7 @@ else()
 endif()
 
 project(foundationdb
-  VERSION 7.1.58
+  VERSION 7.1.59
   DESCRIPTION "FoundationDB is a scalable, fault-tolerant, ordered key-value store with full ACID transactions."
   HOMEPAGE_URL "http://www.foundationdb.org/"
   LANGUAGES C CXX ASM)
diff --git a/cmake/ConfigureCompiler.cmake b/cmake/ConfigureCompiler.cmake
index 7e50469ba6a..e52d8665ee8 100644
--- a/cmake/ConfigureCompiler.cmake
+++ b/cmake/ConfigureCompiler.cmake
@@ -253,7 +253,7 @@ else()
       set(USE_AVX512F OFF)
     endif()
   endif()
-  set(USE_AVX OFF CACHE BOOL "Enable AVX instructions")
+  set(USE_AVX ON CACHE BOOL "Enable AVX instructions")
   if (USE_AVX)
     if (CMAKE_HOST_SYSTEM_PROCESSOR MATCHES "^x86")
       add_compile_options(-mavx)

From 476b6125c1210f7a2299555ecfa7bdd4b54c92b2 Mon Sep 17 00:00:00 2001
From: FoundationDB CI <foundationdb_ci@apple.com>
Date: Thu, 14 Mar 2024 16:58:56 +0000
Subject: [PATCH 409/468] update version after 7.1.59 release

---
 CMakeLists.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 8aabac3f63a..bee878481f2 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -23,7 +23,7 @@ else()
 endif()
 
 project(foundationdb
-  VERSION 7.1.59
+  VERSION 7.1.60
   DESCRIPTION "FoundationDB is a scalable, fault-tolerant, ordered key-value store with full ACID transactions."
   HOMEPAGE_URL "http://www.foundationdb.org/"
   LANGUAGES C CXX ASM)

From 96bffe11d6286e7f252bff82e31e129c7880edd2 Mon Sep 17 00:00:00 2001
From: Aaron Molitor <amolitor@apple.com>
Date: Mon, 1 Apr 2024 16:15:26 -0400
Subject: [PATCH 410/468] align script with main branch

---
 packaging/docker/build-images.sh | 48 ++++++++++++++++----------------
 1 file changed, 24 insertions(+), 24 deletions(-)

diff --git a/packaging/docker/build-images.sh b/packaging/docker/build-images.sh
index 4b306a1535b..206c152baae 100755
--- a/packaging/docker/build-images.sh
+++ b/packaging/docker/build-images.sh
@@ -9,9 +9,9 @@ function logg () {
     printf "${blue}##### $(date +"%H:%M:%S") #  %-56.55s #####${reset}\n" "${1}"
 }
 
- function loge () {
-     printf "${red}##### $(date +"%H:%M:%S") #  %-56.55s #####${reset}\n" "${1}"
- }
+function loge () {
+    printf "${red}##### $(date +"%H:%M:%S") #  %-56.55s #####${reset}\n" "${1}"
+}
 
 function pushd () {
     command pushd "$@" > /dev/null
@@ -22,18 +22,18 @@ function popd () {
 }
 
 function error_exit () {
-     echo "${red}################################################################################${reset}"
-     loge "${0} FAILED"
-     echo "${red}################################################################################${reset}"
- }
+    echo "${red}################################################################################${reset}"
+    loge "${0} FAILED"
+    echo "${red}################################################################################${reset}"
+}
 
- trap error_exit ERR
+trap error_exit ERR
 
- function create_fake_website_directory () {
-     if [ ${#} -ne 1 ]; then
-         loge "INCORRECT NUMBER OF ARGS FOR ${FUNCNAME[0]}"
-     fi
-     local stripped_binaries_and_from_where="${1}"
+function create_fake_website_directory () {
+    if [ ${#} -ne 1 ]; then
+        loge "INCORRECT NUMBER OF ARGS FOR ${FUNCNAME[0]}"
+    fi
+    local stripped_binaries_and_from_where="${1}"
     fdb_binaries=( 'fdbbackup' 'fdbcli' 'fdbserver' 'fdbmonitor' )
     logg "PREPARING WEBSITE"
     website_directory="${script_dir}/website"
@@ -168,12 +168,12 @@ function compile_ycsb () {
 }
 
 function build_and_push_images () {
-     if [ ${#} -ne 3 ]; then
-         loge "INCORRECT NUMBER OF ARGS FOR ${FUNCNAME[0]}"
-     fi
-     local dockerfile_name="${1}"
-     local use_development_java_bindings="${2}"
-     local push_docker_images="${3}"
+    if [ ${#} -ne 3 ]; then
+        loge "INCORRECT NUMBER OF ARGS FOR ${FUNCNAME[0]}"
+    fi
+    local dockerfile_name="${1}"
+    local use_development_java_bindings="${2}"
+    local push_docker_images="${3}"
     declare -a tags_to_push=()
     for image in "${image_list[@]}"; do
         logg "BUILDING ${image}"
@@ -278,12 +278,12 @@ if [ -n "${OKTETO_NAMESPACE+x}" ]; then
     fi
 
     # build regular images
-     create_fake_website_directory stripped_local
-     build_and_push_images Dockerfile true true
+    create_fake_website_directory stripped_local
+    build_and_push_images Dockerfile true true
 
-     # build debug images
-     create_fake_website_directory unstripped_local
-     build_and_push_images Dockerfile.eks true true
+    # build debug images
+    create_fake_website_directory unstripped_local
+    build_and_push_images Dockerfile.eks true true
 else
     echo "Dear ${USER}, you probably need to edit this file before running it. "
     echo "${0} has a very narrow set of situations where it will be successful,"

From 1bd34657ecbb59c6500be90ad9a2a3bde2f08d93 Mon Sep 17 00:00:00 2001
From: Griffin Dunn <griffin.dunn@datadoghq.com>
Date: Thu, 25 May 2023 17:25:21 -0400
Subject: [PATCH 411/468] cherry picked
 https://github.com/DataDog/foundationdb/commits/griffin/go-tenant-impl/

---
 bindings/go/CMakeLists.txt      |   2 +
 bindings/go/src/fdb/database.go |  35 +++--
 bindings/go/src/fdb/errors.go   |   4 +
 bindings/go/src/fdb/fdb_test.go | 234 +++++++++++++++++++++++++++++
 bindings/go/src/fdb/tenant.go   | 256 ++++++++++++++++++++++++++++++++
 5 files changed, 518 insertions(+), 13 deletions(-)
 create mode 100644 bindings/go/src/fdb/tenant.go

diff --git a/bindings/go/CMakeLists.txt b/bindings/go/CMakeLists.txt
index cf4aeb1aab3..7d3d8ea6cff 100644
--- a/bindings/go/CMakeLists.txt
+++ b/bindings/go/CMakeLists.txt
@@ -1,3 +1,4 @@
+
 set(SRCS
   src/_stacktester/directory.go
   src/fdb/directory/allocator.go
@@ -19,6 +20,7 @@ set(SRCS
   src/fdb/tuple/tuple_test.go
   src/fdb/database.go
   src/fdb/directory/directorySubspace.go
+  src/fdb/tenant.go
   src/fdb/fdb_test.go
   src/fdb/snapshot.go)
 
diff --git a/bindings/go/src/fdb/database.go b/bindings/go/src/fdb/database.go
index 7d3be8fabe8..006378465c2 100644
--- a/bindings/go/src/fdb/database.go
+++ b/bindings/go/src/fdb/database.go
@@ -131,6 +131,22 @@ func retryable(wrapped func() (interface{}, error), onError func(Error) FutureNi
 	}
 }
 
+func transact(tr Transaction, f func(Transaction) (interface{}, error)) (interface{}, error) {
+	wrapped := func() (ret interface{}, e error) {
+		defer panicToError(&e)
+
+		ret, e = f(tr)
+
+		if e == nil {
+			e = tr.Commit().Get()
+		}
+
+		return
+	}
+
+	return retryable(wrapped, tr.OnError)
+}
+
 // Transact runs a caller-provided function inside a retry loop, providing it
 // with a newly created Transaction. After the function returns, the Transaction
 // will be committed automatically. Any error during execution of the function
@@ -160,6 +176,11 @@ func (d Database) Transact(f func(Transaction) (interface{}, error)) (interface{
 		return nil, e
 	}
 
+	return transact(tr, f)
+}
+
+
+func readTransact(tr Transaction, f func(ReadTransaction) (interface{}, error)) (interface{}, error) {
 	wrapped := func() (ret interface{}, e error) {
 		defer panicToError(&e)
 
@@ -203,19 +224,7 @@ func (d Database) ReadTransact(f func(ReadTransaction) (interface{}, error)) (in
 		return nil, e
 	}
 
-	wrapped := func() (ret interface{}, e error) {
-		defer panicToError(&e)
-
-		ret, e = f(tr)
-
-		if e == nil {
-			e = tr.Commit().Get()
-		}
-
-		return
-	}
-
-	return retryable(wrapped, tr.OnError)
+	return readTransact(tr, f)
 }
 
 // Options returns a DatabaseOptions instance suitable for setting options
diff --git a/bindings/go/src/fdb/errors.go b/bindings/go/src/fdb/errors.go
index 9ce11ca150f..2c955026908 100644
--- a/bindings/go/src/fdb/errors.go
+++ b/bindings/go/src/fdb/errors.go
@@ -54,4 +54,8 @@ var (
 	errAPIVersionUnset        = Error{2200}
 	errAPIVersionAlreadySet   = Error{2201}
 	errAPIVersionNotSupported = Error{2203}
+
+	errTenantNotFound    = Error{2131}
+	errTenantExists      = Error{2132}
+	errTenantNameInvalid = Error{2134}
 )
diff --git a/bindings/go/src/fdb/fdb_test.go b/bindings/go/src/fdb/fdb_test.go
index 690b49d8d54..ab1cd5a20dc 100644
--- a/bindings/go/src/fdb/fdb_test.go
+++ b/bindings/go/src/fdb/fdb_test.go
@@ -23,6 +23,7 @@
 package fdb_test
 
 import (
+	"bytes"
 	"fmt"
 	"testing"
 
@@ -311,3 +312,236 @@ func ExamplePrintable() {
 	fmt.Println(fdb.Printable([]byte{0, 1, 2, 'a', 'b', 'c', '1', '2', '3', '!', '?', 255}))
 	// Output: \x00\x01\x02abc123!?\xff
 }
+
+func TestDatabaseCloseRemovesResources(t *testing.T) {
+	err := fdb.APIVersion(API_VERSION)
+	if err != nil {
+		t.Fatalf("Unable to set API version: %v\n", err)
+	}
+
+	// OpenDefault opens the database described by the platform-specific default
+	// cluster file
+	db, err := fdb.OpenDefault()
+	if err != nil {
+		t.Fatalf("Unable to set API version: %v\n", err)
+	}
+
+	// Close the database after usage
+	db.Close()
+
+	// Open the same database again, if the database is still in the cache we would return the same object, if not we create a new object with a new pointer
+	newDB, err := fdb.OpenDefault()
+	if err != nil {
+		t.Fatalf("Unable to set API version: %v\n", err)
+	}
+
+	if db == newDB {
+		t.Fatalf("Expected a different database object, got: %v and %v\n", db, newDB)
+	}
+}
+
+func ExampleOpenWithConnectionString() {
+	fdb.MustAPIVersion(API_VERSION)
+
+	clusterFileContent, err := os.ReadFile(os.Getenv("FDB_CLUSTER_FILE"))
+	if err != nil {
+		fmt.Printf("Unable to read cluster file: %v\n", err)
+		return
+	}
+
+	// OpenWithConnectionString opens the database described by the connection string
+	db, err := fdb.OpenWithConnectionString(string(clusterFileContent))
+	if err != nil {
+		fmt.Printf("Unable to open database: %v\n", err)
+		return
+	}
+
+	// Close the database after usage
+	defer db.Close()
+
+	// Do work here
+
+	// Output:
+}
+
+// Copied from errors.go so that these types aren't public
+var (
+	errTenantNotFound    = fdb.Error{2131}
+	errTenantExists      = fdb.Error{2132}
+	errTenantNameInvalid = fdb.Error{2134}
+)
+
+func TestCreateTenant(t *testing.T) {
+	err := fdb.APIVersion(API_VERSION)
+	if err != nil {
+		t.Fatalf("Unable to set API version: %v\n", err)
+	}
+
+	// OpenDefault opens the database described by the platform-specific default
+	// cluster file
+	db, err := fdb.OpenDefault()
+	if err != nil {
+		t.Fatalf("Unable to set API version: %v\n", err)
+	}
+
+	var testTenantName = fdb.Key("test-create-tenant")
+
+	err = db.CreateTenant(testTenantName)
+	if err != nil {
+		t.Fatalf("Unable to create tenant: %v\n", err)
+	}
+
+	_, err = db.OpenTenant(testTenantName)
+	if err != nil {
+		t.Fatalf("Unable to open tenant: %v\n", err)
+	}
+}
+
+func TestCreateExistTenant(t *testing.T) {
+	err := fdb.APIVersion(API_VERSION)
+	if err != nil {
+		t.Fatalf("Unable to set API version: %v\n", err)
+	}
+
+	// OpenDefault opens the database described by the platform-specific default
+	// cluster file
+	db, err := fdb.OpenDefault()
+	if err != nil {
+		t.Fatalf("Unable to set API version: %v\n", err)
+	}
+
+	var testTenantName = fdb.Key("test-exist-tenant")
+
+	err = db.CreateTenant(testTenantName)
+	if err != nil {
+		t.Fatalf("Unable to create tenant: %v\n", err)
+	}
+
+	// This should fail
+	err = db.CreateTenant(testTenantName)
+	assertErrorCodeEqual(t, err, errTenantExists)
+}
+
+func TestOpenNotExistTenant(t *testing.T) {
+	err := fdb.APIVersion(API_VERSION)
+	if err != nil {
+		t.Fatalf("Unable to set API version: %v\n", err)
+	}
+
+	// OpenDefault opens the database described by the platform-specific default
+	// cluster file
+	db, err := fdb.OpenDefault()
+	if err != nil {
+		t.Fatalf("Unable to set API version: %v\n", err)
+	}
+
+	var testTenantName = fdb.Key("test-not-exist-tenant")
+
+	// this should fail
+	_, err = db.OpenTenant(testTenantName)
+	assertErrorCodeEqual(t, err, errTenantNotFound)
+}
+
+func TestDeleteNotExistTenant(t *testing.T) {
+	err := fdb.APIVersion(API_VERSION)
+	if err != nil {
+		t.Fatalf("Unable to set API version: %v\n", err)
+	}
+
+	// OpenDefault opens the database described by the platform-specific default
+	// cluster file
+	db, err := fdb.OpenDefault()
+	if err != nil {
+		t.Fatalf("Unable to set API version: %v\n", err)
+	}
+
+	var testTenantName = fdb.Key("test-not-exist-tenant")
+
+	// this should fail
+	err = db.DeleteTenant(testTenantName)
+	assertErrorCodeEqual(t, err, errTenantNotFound)
+}
+
+func inSlice(sl []fdb.Key, t fdb.Key) bool {
+	for _, s := range sl {
+		if bytes.Equal(s, t) {
+			return true
+		}
+	}
+	return false
+}
+
+func assertErrorCodeEqual(t *testing.T, actual error, expected fdb.Error) {
+	if actual == nil {
+		t.Fatalf("Error is nil when it should be: %v\n", expected.Code)
+	}
+
+	castErr, ok := actual.(fdb.Error)
+	if !ok {
+		t.Fatalf("Error is wrong type %v, expected %v\n", actual, expected)
+	}
+
+	if castErr.Code != expected.Code {
+		t.Fatalf("Error is wrong code, expected %v, actual %v\n", expected.Code, castErr.Code)
+	}
+}
+
+func TestListTenant(t *testing.T) {
+	err := fdb.APIVersion(API_VERSION)
+	if err != nil {
+		t.Fatalf("Unable to set API version: %v\n", err)
+	}
+
+	// OpenDefault opens the database described by the platform-specific default
+	// cluster file
+	db, err := fdb.OpenDefault()
+	if err != nil {
+		t.Fatalf("Unable to set API version: %v\n", err)
+	}
+
+	var testTenantName1 = fdb.Key("1-test-list-1-tenant-1")
+	var testTenantName2 = fdb.Key("2-test-list-2-tenant-2")
+
+	err = db.CreateTenant(testTenantName1)
+	if err != nil {
+		t.Fatalf("Unable to create tenant 1: %v\n", err)
+	}
+
+	err = db.CreateTenant(testTenantName2)
+	if err != nil {
+		t.Fatalf("Unable to create tenant 2: %v\n", err)
+	}
+
+	ls, err := db.ListTenants()
+	if err != nil {
+		t.Fatalf("Unable to list tenants: %v\n", err)
+	}
+
+	if !inSlice(ls, testTenantName1) {
+		t.Fatalf("tenant 1 not in slice %#v", ls)
+	}
+
+	if !inSlice(ls, testTenantName2) {
+		t.Fatalf("tenant 2 not in slice, %#v", ls)
+	}
+}
+
+func TestInvalidPrefixTenant(t *testing.T) {
+	err := fdb.APIVersion(API_VERSION)
+	if err != nil {
+		t.Fatalf("Unable to set API version: %v\n", err)
+	}
+
+	// OpenDefault opens the database described by the platform-specific default
+	// cluster file
+	db, err := fdb.OpenDefault()
+	if err != nil {
+		t.Fatalf("Unable to set API version: %v\n", err)
+	}
+
+	var testTenantName = fdb.Key("\xFFtest-invalid-prefix-tenant")
+
+	// this should fail
+	err = db.CreateTenant(testTenantName)
+	assertErrorCodeEqual(t, err, errTenantNameInvalid)
+}
diff --git a/bindings/go/src/fdb/tenant.go b/bindings/go/src/fdb/tenant.go
new file mode 100644
index 00000000000..33da5e5d5f3
--- /dev/null
+++ b/bindings/go/src/fdb/tenant.go
@@ -0,0 +1,256 @@
+/*
+ * tenant.go
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2018 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// FoundationDB Go API
+
+package fdb
+
+// #define FDB_API_VERSION 720
+// #include <foundationdb/fdb_c.h>
+// #include <stdlib.h>
+import "C"
+
+import (
+	"runtime"
+)
+
+// CreateTenant creates a new tenant in the cluster. The tenant name cannot
+// start with the \xff byte.
+func (t Transaction) CreateTenant(name KeyConvertible) error {
+	tenantName := name.FDBKey()
+	if len(tenantName) > 0 && tenantName[0] == '\xFF' {
+		return errTenantNameInvalid
+	}
+
+	if err := t.Options().SetSpecialKeySpaceEnableWrites(); err != nil {
+		return err
+	}
+
+	key := append(Key("\xFF\xFF/management/tenant/map/"), tenantName...)
+
+	exist, err := t.checkTenantExist(key)
+	if err != nil {
+		return err
+	}
+	if exist {
+		return errTenantExists
+	}
+
+	t.Set(key, nil)
+
+	return nil
+}
+
+// CreateTenant creates a new tenant in the cluster. The tenant name cannot
+// start with the \xff byte.
+func (d Database) CreateTenant(name KeyConvertible) error {
+	_, err := d.Transact(func(t Transaction) (interface{}, error) {
+		return nil, t.CreateTenant(name)
+	})
+	return err
+}
+
+// DeleteTenant deletes an existing tenant in the cluster.
+func (t Transaction) DeleteTenant(name KeyConvertible) error {
+	if err := t.Options().SetSpecialKeySpaceEnableWrites(); err != nil {
+		return err
+	}
+
+	key := append(Key("\xFF\xFF/management/tenant/map/"), name.FDBKey()...)
+
+	exist, err := t.checkTenantExist(key)
+	if err != nil {
+		return err
+	}
+	if !exist {
+		return errTenantNotFound
+	}
+
+	t.Clear(key)
+
+	return nil
+}
+
+// DeleteTenant creates a new tenant in the cluster. The tenant name cannot
+// start with the \xff byte.
+func (d Database) DeleteTenant(name KeyConvertible) error {
+	_, err := d.Transact(func(t Transaction) (interface{}, error) {
+		return nil, t.DeleteTenant(name)
+	})
+	return err
+}
+
+// ListTenants lists all existings tenants in the cluster.
+func (t Transaction) ListTenants() ([]Key, error) {
+	if err := t.Options().SetSpecialKeySpaceEnableWrites(); err != nil {
+		return nil, err
+	}
+
+	kr, err := PrefixRange(Key("\xFF\xFF/management/tenant/map/"))
+	if err != nil {
+		return nil, err
+	}
+
+	rawTenants, err := t.GetRange(kr, RangeOptions{}).GetSliceWithError()
+	if err != nil {
+		return nil, err
+	}
+
+	// trim tenant prefix
+	tenants := make([]Key, len(rawTenants))
+	for i, rt := range rawTenants {
+		tenants[i] = rt.Key[25:] // len of tenant prefix
+	}
+
+	return tenants, nil
+}
+
+// CreateTenant creates a new tenant in the cluster. The tenant name cannot
+// start with the \xff byte.
+func (d Database) ListTenants() ([]Key, error) {
+	tenants, err := d.Transact(func(t Transaction) (interface{}, error) {
+		return t.ListTenants()
+	})
+	return tenants.([]Key), err
+}
+
+func (t Transaction) checkTenantExist(tenantPath Key) (bool, error) {
+	buf, err := t.Get(tenantPath).Get()
+	if err != nil {
+		return true, err
+	}
+
+	return buf != nil, nil
+}
+
+// Tenant is a handle to a FoundationDB tenant. Tenant is a lightweight
+// object that may be efficiently copied, and is safe for concurrent use by
+// multiple goroutines.
+type Tenant struct {
+	*tenant
+
+	db Database
+}
+
+type tenant struct {
+	ptr *C.FDBTenant
+}
+
+// OpenTenant returns a tenant handle identified by the given name. All transactions
+// created by this tenant will operate on the tenant’s key-space.
+func (d Database) OpenTenant(name KeyConvertible) (Tenant, error) {
+	var outt *C.FDBTenant
+	if err := C.fdb_database_open_tenant(d.database.ptr, byteSliceToPtr(name.FDBKey()), C.int(len(name.FDBKey())), &outt); err != 0 {
+		return Tenant{}, Error{int(err)}
+	}
+
+	tnt := &tenant{outt}
+	runtime.SetFinalizer(tnt, (*tenant).destroy)
+
+	return Tenant{tnt, d}, nil
+}
+
+func (t *tenant) destroy() {
+	if t.ptr == nil {
+		return
+	}
+
+	C.fdb_tenant_destroy(t.ptr)
+}
+
+// CreateTransaction returns a new FoundationDB transaction. It is generally
+// preferable to use the (Tenant).Transact method, which handles
+// automatically creating and committing a transaction with appropriate retry
+// behavior.
+func (t Tenant) CreateTransaction() (Transaction, error) {
+	var outt *C.FDBTransaction
+
+	if err := C.fdb_tenant_create_transaction(t.ptr, &outt); err != 0 {
+		return Transaction{}, Error{int(err)}
+	}
+
+	trx := &transaction{outt, t.db}
+	runtime.SetFinalizer(trx, (*transaction).destroy)
+
+	return Transaction{trx}, nil
+}
+
+// Transact runs a caller-provided function inside a retry loop, providing it
+// with a newly created Transaction. After the function returns, the Transaction
+// will be committed automatically. Any error during execution of the function
+// (by panic or return) or the commit will cause the function and commit to be
+// retried or, if fatal, return the error to the caller.
+//
+// When working with Future objects in a transactional function, you may either
+// explicitly check and return error values using Get, or call MustGet. Transact
+// will recover a panicked Error and either retry the transaction or return the
+// error.
+//
+// The transaction is retried if the error is or wraps a retryable Error.
+// The error is unwrapped.
+//
+// Do not return Future objects from the function provided to Transact. The
+// Transaction created by Transact may be finalized at any point after Transact
+// returns, resulting in the cancellation of any outstanding
+// reads. Additionally, any errors returned or panicked by the Future will no
+// longer be able to trigger a retry of the caller-provided function.
+//
+// See the Transactor interface for an example of using Transact with
+// Transaction and Database objects.
+func (t Tenant) Transact(f func(Transaction) (interface{}, error)) (interface{}, error) {
+	tr, e := t.CreateTransaction()
+	// Any error here is non-retryable
+	if e != nil {
+		return nil, e
+	}
+
+	return transact(tr, f)
+}
+
+// ReadTransact runs a caller-provided function inside a retry loop, providing
+// it with a newly created Transaction (as a ReadTransaction). Any error during
+// execution of the function (by panic or return) will cause the function to be
+// retried or, if fatal, return the error to the caller.
+//
+// When working with Future objects in a read-only transactional function, you
+// may either explicitly check and return error values using Get, or call
+// MustGet. ReadTransact will recover a panicked Error and either retry the
+// transaction or return the error.
+//
+// The transaction is retried if the error is or wraps a retryable Error.
+// The error is unwrapped.
+//
+// Do not return Future objects from the function provided to ReadTransact. The
+// Transaction created by ReadTransact may be finalized at any point after
+// ReadTransact returns, resulting in the cancellation of any outstanding
+// reads. Additionally, any errors returned or panicked by the Future will no
+// longer be able to trigger a retry of the caller-provided function.
+//
+// See the ReadTransactor interface for an example of using ReadTransact with
+// Transaction, Snapshot and Database objects.
+func (t Tenant) ReadTransact(f func(ReadTransaction) (interface{}, error)) (interface{}, error) {
+	tr, e := t.CreateTransaction()
+	// Any error here is non-retryable
+	if e != nil {
+		return nil, e
+	}
+
+	return readTransact(tr, f)
+}

From 89122c8095f6329fc8db782e9472f26ee8d009b3 Mon Sep 17 00:00:00 2001
From: Matthew Newhook <matthew@customer.io>
Date: Tue, 2 Apr 2024 13:22:23 -0400
Subject: [PATCH 412/468] 720 -> 710

---
 bindings/go/src/fdb/tenant.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bindings/go/src/fdb/tenant.go b/bindings/go/src/fdb/tenant.go
index 33da5e5d5f3..6d448a43098 100644
--- a/bindings/go/src/fdb/tenant.go
+++ b/bindings/go/src/fdb/tenant.go
@@ -22,7 +22,7 @@
 
 package fdb
 
-// #define FDB_API_VERSION 720
+// #define FDB_API_VERSION 710
 // #include <foundationdb/fdb_c.h>
 // #include <stdlib.h>
 import "C"

From 7b685b925fca3ff7bf8cd87b4a60fe6211086010 Mon Sep 17 00:00:00 2001
From: Matthew Newhook <matthew@customer.io>
Date: Tue, 2 Apr 2024 15:27:34 -0400
Subject: [PATCH 413/468] tenant/map -> tenant_map

---
 bindings/go/src/fdb/tenant.go | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/bindings/go/src/fdb/tenant.go b/bindings/go/src/fdb/tenant.go
index 6d448a43098..bc4a9c46385 100644
--- a/bindings/go/src/fdb/tenant.go
+++ b/bindings/go/src/fdb/tenant.go
@@ -43,7 +43,7 @@ func (t Transaction) CreateTenant(name KeyConvertible) error {
 		return err
 	}
 
-	key := append(Key("\xFF\xFF/management/tenant/map/"), tenantName...)
+	key := append(Key("\xFF\xFF/management/tenant_map/"), tenantName...)
 
 	exist, err := t.checkTenantExist(key)
 	if err != nil {
@@ -73,7 +73,7 @@ func (t Transaction) DeleteTenant(name KeyConvertible) error {
 		return err
 	}
 
-	key := append(Key("\xFF\xFF/management/tenant/map/"), name.FDBKey()...)
+	key := append(Key("\xFF\xFF/management/tenant_map/"), name.FDBKey()...)
 
 	exist, err := t.checkTenantExist(key)
 	if err != nil {
@@ -103,7 +103,7 @@ func (t Transaction) ListTenants() ([]Key, error) {
 		return nil, err
 	}
 
-	kr, err := PrefixRange(Key("\xFF\xFF/management/tenant/map/"))
+	kr, err := PrefixRange(Key("\xFF\xFF/management/tenant_map/"))
 	if err != nil {
 		return nil, err
 	}

From 3235657d1ed78c9b46341d6fb82f06a6941109da Mon Sep 17 00:00:00 2001
From: Matthew Newhook <matthew@customer.io>
Date: Tue, 9 Apr 2024 10:12:33 -0400
Subject: [PATCH 414/468] Address some PR review comments.

---
 bindings/go/src/fdb/fdb_test.go | 155 ++++++--------------------------
 bindings/go/src/fdb/tenant.go   |  29 +++---
 2 files changed, 43 insertions(+), 141 deletions(-)

diff --git a/bindings/go/src/fdb/fdb_test.go b/bindings/go/src/fdb/fdb_test.go
index ab1cd5a20dc..3e98c4c9e3d 100644
--- a/bindings/go/src/fdb/fdb_test.go
+++ b/bindings/go/src/fdb/fdb_test.go
@@ -313,57 +313,6 @@ func ExamplePrintable() {
 	// Output: \x00\x01\x02abc123!?\xff
 }
 
-func TestDatabaseCloseRemovesResources(t *testing.T) {
-	err := fdb.APIVersion(API_VERSION)
-	if err != nil {
-		t.Fatalf("Unable to set API version: %v\n", err)
-	}
-
-	// OpenDefault opens the database described by the platform-specific default
-	// cluster file
-	db, err := fdb.OpenDefault()
-	if err != nil {
-		t.Fatalf("Unable to set API version: %v\n", err)
-	}
-
-	// Close the database after usage
-	db.Close()
-
-	// Open the same database again, if the database is still in the cache we would return the same object, if not we create a new object with a new pointer
-	newDB, err := fdb.OpenDefault()
-	if err != nil {
-		t.Fatalf("Unable to set API version: %v\n", err)
-	}
-
-	if db == newDB {
-		t.Fatalf("Expected a different database object, got: %v and %v\n", db, newDB)
-	}
-}
-
-func ExampleOpenWithConnectionString() {
-	fdb.MustAPIVersion(API_VERSION)
-
-	clusterFileContent, err := os.ReadFile(os.Getenv("FDB_CLUSTER_FILE"))
-	if err != nil {
-		fmt.Printf("Unable to read cluster file: %v\n", err)
-		return
-	}
-
-	// OpenWithConnectionString opens the database described by the connection string
-	db, err := fdb.OpenWithConnectionString(string(clusterFileContent))
-	if err != nil {
-		fmt.Printf("Unable to open database: %v\n", err)
-		return
-	}
-
-	// Close the database after usage
-	defer db.Close()
-
-	// Do work here
-
-	// Output:
-}
-
 // Copied from errors.go so that these types aren't public
 var (
 	errTenantNotFound    = fdb.Error{2131}
@@ -372,21 +321,12 @@ var (
 )
 
 func TestCreateTenant(t *testing.T) {
-	err := fdb.APIVersion(API_VERSION)
-	if err != nil {
-		t.Fatalf("Unable to set API version: %v\n", err)
-	}
-
-	// OpenDefault opens the database described by the platform-specific default
-	// cluster file
-	db, err := fdb.OpenDefault()
-	if err != nil {
-		t.Fatalf("Unable to set API version: %v\n", err)
-	}
+	fdb.MustAPIVersion(710)
+	db := fdb.MustOpenDefault()
 
-	var testTenantName = fdb.Key("test-create-tenant")
+	testTenantName := fdb.Key("test-create-tenant")
 
-	err = db.CreateTenant(testTenantName)
+	err := db.CreateTenant(testTenantName)
 	if err != nil {
 		t.Fatalf("Unable to create tenant: %v\n", err)
 	}
@@ -398,21 +338,12 @@ func TestCreateTenant(t *testing.T) {
 }
 
 func TestCreateExistTenant(t *testing.T) {
-	err := fdb.APIVersion(API_VERSION)
-	if err != nil {
-		t.Fatalf("Unable to set API version: %v\n", err)
-	}
-
-	// OpenDefault opens the database described by the platform-specific default
-	// cluster file
-	db, err := fdb.OpenDefault()
-	if err != nil {
-		t.Fatalf("Unable to set API version: %v\n", err)
-	}
+	fdb.MustAPIVersion(710)
+	db := fdb.MustOpenDefault()
 
-	var testTenantName = fdb.Key("test-exist-tenant")
+	testTenantName := fdb.Key("test-exist-tenant")
 
-	err = db.CreateTenant(testTenantName)
+	err := db.CreateTenant(testTenantName)
 	if err != nil {
 		t.Fatalf("Unable to create tenant: %v\n", err)
 	}
@@ -423,42 +354,24 @@ func TestCreateExistTenant(t *testing.T) {
 }
 
 func TestOpenNotExistTenant(t *testing.T) {
-	err := fdb.APIVersion(API_VERSION)
-	if err != nil {
-		t.Fatalf("Unable to set API version: %v\n", err)
-	}
-
-	// OpenDefault opens the database described by the platform-specific default
-	// cluster file
-	db, err := fdb.OpenDefault()
-	if err != nil {
-		t.Fatalf("Unable to set API version: %v\n", err)
-	}
+	fdb.MustAPIVersion(710)
+	db := fdb.MustOpenDefault()
 
-	var testTenantName = fdb.Key("test-not-exist-tenant")
+	testTenantName := fdb.Key("test-not-exist-tenant")
 
 	// this should fail
-	_, err = db.OpenTenant(testTenantName)
+	_, err := db.OpenTenant(testTenantName)
 	assertErrorCodeEqual(t, err, errTenantNotFound)
 }
 
 func TestDeleteNotExistTenant(t *testing.T) {
-	err := fdb.APIVersion(API_VERSION)
-	if err != nil {
-		t.Fatalf("Unable to set API version: %v\n", err)
-	}
-
-	// OpenDefault opens the database described by the platform-specific default
-	// cluster file
-	db, err := fdb.OpenDefault()
-	if err != nil {
-		t.Fatalf("Unable to set API version: %v\n", err)
-	}
+	fdb.MustAPIVersion(710)
+	db := fdb.MustOpenDefault()
 
-	var testTenantName = fdb.Key("test-not-exist-tenant")
+	testTenantName := fdb.Key("test-not-exist-tenant")
 
 	// this should fail
-	err = db.DeleteTenant(testTenantName)
+	err := db.DeleteTenant(testTenantName)
 	assertErrorCodeEqual(t, err, errTenantNotFound)
 }
 
@@ -487,22 +400,13 @@ func assertErrorCodeEqual(t *testing.T, actual error, expected fdb.Error) {
 }
 
 func TestListTenant(t *testing.T) {
-	err := fdb.APIVersion(API_VERSION)
-	if err != nil {
-		t.Fatalf("Unable to set API version: %v\n", err)
-	}
-
-	// OpenDefault opens the database described by the platform-specific default
-	// cluster file
-	db, err := fdb.OpenDefault()
-	if err != nil {
-		t.Fatalf("Unable to set API version: %v\n", err)
-	}
+	fdb.MustAPIVersion(710)
+	db := fdb.MustOpenDefault()
 
-	var testTenantName1 = fdb.Key("1-test-list-1-tenant-1")
-	var testTenantName2 = fdb.Key("2-test-list-2-tenant-2")
+	testTenantName1 := fdb.Key("1-test-list-1-tenant-1")
+	testTenantName2 := fdb.Key("2-test-list-2-tenant-2")
 
-	err = db.CreateTenant(testTenantName1)
+	err := db.CreateTenant(testTenantName1)
 	if err != nil {
 		t.Fatalf("Unable to create tenant 1: %v\n", err)
 	}
@@ -527,21 +431,12 @@ func TestListTenant(t *testing.T) {
 }
 
 func TestInvalidPrefixTenant(t *testing.T) {
-	err := fdb.APIVersion(API_VERSION)
-	if err != nil {
-		t.Fatalf("Unable to set API version: %v\n", err)
-	}
-
-	// OpenDefault opens the database described by the platform-specific default
-	// cluster file
-	db, err := fdb.OpenDefault()
-	if err != nil {
-		t.Fatalf("Unable to set API version: %v\n", err)
-	}
+	fdb.MustAPIVersion(710)
+	db := fdb.MustOpenDefault()
 
-	var testTenantName = fdb.Key("\xFFtest-invalid-prefix-tenant")
+	testTenantName := fdb.Key("\xFFtest-invalid-prefix-tenant")
 
 	// this should fail
-	err = db.CreateTenant(testTenantName)
+	err := db.CreateTenant(testTenantName)
 	assertErrorCodeEqual(t, err, errTenantNameInvalid)
 }
diff --git a/bindings/go/src/fdb/tenant.go b/bindings/go/src/fdb/tenant.go
index bc4a9c46385..25d6aa3c72c 100644
--- a/bindings/go/src/fdb/tenant.go
+++ b/bindings/go/src/fdb/tenant.go
@@ -31,11 +31,13 @@ import (
 	"runtime"
 )
 
+const tenantMapKey = "\xff\xff/management/tenant_map/"
+
 // CreateTenant creates a new tenant in the cluster. The tenant name cannot
 // start with the \xff byte.
 func (t Transaction) CreateTenant(name KeyConvertible) error {
 	tenantName := name.FDBKey()
-	if len(tenantName) > 0 && tenantName[0] == '\xFF' {
+	if len(tenantName) == 0 || tenantName[0] == '\xff' {
 		return errTenantNameInvalid
 	}
 
@@ -43,7 +45,7 @@ func (t Transaction) CreateTenant(name KeyConvertible) error {
 		return err
 	}
 
-	key := append(Key("\xFF\xFF/management/tenant_map/"), tenantName...)
+	key := append(Key(tenantMapKey), tenantName...)
 
 	exist, err := t.checkTenantExist(key)
 	if err != nil {
@@ -67,13 +69,17 @@ func (d Database) CreateTenant(name KeyConvertible) error {
 	return err
 }
 
-// DeleteTenant deletes an existing tenant in the cluster.
+// DeleteTenant deletes an existing tenant in the cluster. If the provided tenant name doesn't exist an error will be thrown.
 func (t Transaction) DeleteTenant(name KeyConvertible) error {
+	tenantName := name.FDBKey()
+	if len(tenantName) == 0 || tenantName[0] == '\xff' {
+		return errTenantNameInvalid
+	}
 	if err := t.Options().SetSpecialKeySpaceEnableWrites(); err != nil {
 		return err
 	}
 
-	key := append(Key("\xFF\xFF/management/tenant_map/"), name.FDBKey()...)
+	key := append(Key(tenantMapKey), name.FDBKey()...)
 
 	exist, err := t.checkTenantExist(key)
 	if err != nil {
@@ -88,8 +94,7 @@ func (t Transaction) DeleteTenant(name KeyConvertible) error {
 	return nil
 }
 
-// DeleteTenant creates a new tenant in the cluster. The tenant name cannot
-// start with the \xff byte.
+// DeleteTenant deletes an existing tenant in the cluster. If the provided tenant name doesn't exist an error will be thrown.
 func (d Database) DeleteTenant(name KeyConvertible) error {
 	_, err := d.Transact(func(t Transaction) (interface{}, error) {
 		return nil, t.DeleteTenant(name)
@@ -103,7 +108,7 @@ func (t Transaction) ListTenants() ([]Key, error) {
 		return nil, err
 	}
 
-	kr, err := PrefixRange(Key("\xFF\xFF/management/tenant_map/"))
+	kr, err := PrefixRange(Key(tenantMapKey))
 	if err != nil {
 		return nil, err
 	}
@@ -116,25 +121,27 @@ func (t Transaction) ListTenants() ([]Key, error) {
 	// trim tenant prefix
 	tenants := make([]Key, len(rawTenants))
 	for i, rt := range rawTenants {
-		tenants[i] = rt.Key[25:] // len of tenant prefix
+		tenants[i] = rt.Key[len(tenantMapKey):]
 	}
 
 	return tenants, nil
 }
 
-// CreateTenant creates a new tenant in the cluster. The tenant name cannot
-// start with the \xff byte.
+// ListTenants lists all existings tenants in the cluster.
 func (d Database) ListTenants() ([]Key, error) {
 	tenants, err := d.Transact(func(t Transaction) (interface{}, error) {
 		return t.ListTenants()
 	})
+	if err != nil {
+		return nil, err
+	}
 	return tenants.([]Key), err
 }
 
 func (t Transaction) checkTenantExist(tenantPath Key) (bool, error) {
 	buf, err := t.Get(tenantPath).Get()
 	if err != nil {
-		return true, err
+		return false, err
 	}
 
 	return buf != nil, nil

From d8ae0b4e57d41b5cdd261cfb4ee343ba5264e292 Mon Sep 17 00:00:00 2001
From: Matthew Newhook <matthew@customer.io>
Date: Fri, 12 Apr 2024 08:44:34 -0400
Subject: [PATCH 415/468] update copyright notice.

---
 bindings/go/src/fdb/tenant.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bindings/go/src/fdb/tenant.go b/bindings/go/src/fdb/tenant.go
index 25d6aa3c72c..87cf654705d 100644
--- a/bindings/go/src/fdb/tenant.go
+++ b/bindings/go/src/fdb/tenant.go
@@ -3,7 +3,7 @@
  *
  * This source file is part of the FoundationDB open source project
  *
- * Copyright 2013-2018 Apple Inc. and the FoundationDB project authors
+ * Copyright 2013-2024 Apple Inc. and the FoundationDB project authors
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.

From b6e94335eaed69cba6cdc53295dec82114cf2d0b Mon Sep 17 00:00:00 2001
From: Aaron Molitor <amolitor@apple.com>
Date: Fri, 12 Apr 2024 12:44:03 -0500
Subject: [PATCH 416/468] update fdbkubernetesmonitor to match main, and  add
 fdb-kubernetes-monitor to the standard build flow

---
 cmake/InstallLayout.cmake                  |   1 +
 fdbkubernetesmonitor/.gitignore            |   2 +
 fdbkubernetesmonitor/.golangci.yml         |  54 +++
 fdbkubernetesmonitor/README.md             |  48 ++-
 fdbkubernetesmonitor/api/config.go         |  71 ++-
 fdbkubernetesmonitor/api/config_test.go    | 208 ++++++++-
 fdbkubernetesmonitor/copy.go               | 104 ++++-
 fdbkubernetesmonitor/copy_test.go          | 427 ++++++++++++++++++
 fdbkubernetesmonitor/errcheck_excludes.txt |   2 +
 fdbkubernetesmonitor/go.mod                |  70 ++-
 fdbkubernetesmonitor/go.sum                | 475 +++++++--------------
 fdbkubernetesmonitor/kubernetes.go         | 230 +++++++---
 fdbkubernetesmonitor/main.go               | 101 +++--
 fdbkubernetesmonitor/monitor.go            |  86 +++-
 fdbkubernetesmonitor/suite_test.go         |  37 ++
 packaging/docker/Dockerfile                |   6 +-
 packaging/docker/Dockerfile.eks            |   6 +-
 packaging/docker/build-images.sh           |   6 +-
 packaging/docker/fdb_single.bash           |  90 ++++
 19 files changed, 1521 insertions(+), 503 deletions(-)
 create mode 100644 fdbkubernetesmonitor/.gitignore
 create mode 100644 fdbkubernetesmonitor/.golangci.yml
 create mode 100644 fdbkubernetesmonitor/copy_test.go
 create mode 100644 fdbkubernetesmonitor/errcheck_excludes.txt
 create mode 100644 fdbkubernetesmonitor/suite_test.go
 create mode 100644 packaging/docker/fdb_single.bash

diff --git a/cmake/InstallLayout.cmake b/cmake/InstallLayout.cmake
index 568b4c51fe5..a2197183a50 100644
--- a/cmake/InstallLayout.cmake
+++ b/cmake/InstallLayout.cmake
@@ -181,6 +181,7 @@ install(DIRECTORY "${script_dir}/clients/usr/lib/cmake"
 ################################################################################
 
 file(COPY "${PROJECT_SOURCE_DIR}/packaging/docker" DESTINATION "${PROJECT_BINARY_DIR}/packages/")
+file(COPY "${PROJECT_SOURCE_DIR}/fdbkubernetesmonitor" DESTINATION "${PROJECT_BINARY_DIR}/packages/docker/")
 
 ################################################################################
 # General CPack configuration
diff --git a/fdbkubernetesmonitor/.gitignore b/fdbkubernetesmonitor/.gitignore
new file mode 100644
index 00000000000..1c2fdffed51
--- /dev/null
+++ b/fdbkubernetesmonitor/.gitignore
@@ -0,0 +1,2 @@
+fdbkubernetesmonitor
+cover.out
diff --git a/fdbkubernetesmonitor/.golangci.yml b/fdbkubernetesmonitor/.golangci.yml
new file mode 100644
index 00000000000..b2b452d7dc9
--- /dev/null
+++ b/fdbkubernetesmonitor/.golangci.yml
@@ -0,0 +1,54 @@
+issues:
+  # don't skip warning about doc comments
+  # don't exclude the default set of lint
+  exclude-use-default: false
+  exclude-rules:
+  # exclude deprecation warning otherwise we get an error for our own deprecations
+  - linters: [staticcheck]
+    text: "SA1019:"
+  - linters: [ govet ]
+    text: 'declaration of "(err|ctx)" shadows declaration at'
+
+linters:
+  disable-all: true
+  # TODO(johscheuer): activate linters again and fix issues
+  enable:
+    - deadcode
+    #- dupl
+    - errcheck
+    #- goconst
+    #- gocyclo
+    - gofmt
+    - goimports
+    - revive
+    #- gosec
+    - gosimple
+    - govet
+    - ineffassign
+    #- interfacer
+    #- lll
+    #- maligned
+    - misspell
+    - nakedret
+    - prealloc
+    - exportloopref
+    - staticcheck
+    - structcheck
+    - typecheck
+    - unconvert
+    - unparam
+    - unused
+    - varcheck
+
+linters-settings:
+  errcheck:
+    # path to a file containing a list of functions to exclude from checking
+    # see https://github.com/kisielk/errcheck#excluding-functions for details
+    exclude: errcheck_excludes.txt
+  govet:
+    enable:
+      - shadow
+run:
+  deadline: 5m
+  skip-files:
+    - ".*_generated.*.go$"
diff --git a/fdbkubernetesmonitor/README.md b/fdbkubernetesmonitor/README.md
index b9539c01cd2..4e2af35f468 100644
--- a/fdbkubernetesmonitor/README.md
+++ b/fdbkubernetesmonitor/README.md
@@ -1,30 +1,42 @@
+# FoundationDB Kubernetes monitor
 
 This package provides a launcher program for running FoundationDB in Kubernetes.
 
+## Testing
+
+### Unit tests
+
+You can run the unit tests by running `go test ./...` from this directory.
+
+### Manual testing
+
 To test this, run the following commands from the root of the FoundationDB
 repository:
 
 ```bash
-docker build -t foundationdb/foundationdb-kubernetes:6.3.13-local --target fdb-kubernetes-monitor --build-arg FDB_VERSION=6.3.13 --build-arg FDB_LIBRARY_VERSIONS="6.3.13 6.2.30 6.1.13" -f packaging/docker/Dockerfile .
-docker build -t foundationdb/foundationdb-kubernetes:6.3.15-local --target fdb-kubernetes-monitor --build-arg FDB_VERSION=6.3.15 --build-arg FDB_LIBRARY_VERSIONS="6.3.15 6.2.30 6.1.13" -f packaging/docker/Dockerfile .
+mkdir -p website
+docker build -t foundationdb/foundationdb-kubernetes:7.1.5-local --target fdb-kubernetes-monitor --build-arg FDB_VERSION=7.1.5 --build-arg FDB_LIBRARY_VERSIONS="7.1.5 6.3.24 6.2.30" -f packaging/docker/Dockerfile .
+docker build -t foundationdb/foundationdb-kubernetes:7.1.6-local --target fdb-kubernetes-monitor --build-arg FDB_VERSION=7.1.6 --build-arg FDB_LIBRARY_VERSIONS="7.1.6 6.3.24 6.2.30" -f packaging/docker/Dockerfile .
 kubectl apply -f packaging/docker/kubernetes/test_config.yaml
 # Wait for the pods to become ready
+kubectl rollout status sts/fdb-kubernetes-example
 ips=$(kubectl get pod -l app=fdb-kubernetes-example -o json | jq -j '[[.items|.[]|select(.status.podIP!="")]|limit(3;.[])|.status.podIP+":4501"]|join(",")')
 sed -e "s/fdb.cluster: \"\"/fdb.cluster: \"test:test@$ips\"/" -e "s/\"runProcesses\": false/\"runProcesses\": true/" packaging/docker/kubernetes/test_config.yaml | kubectl apply -f -
-kubectl get pod -l app=fdb-kubernetes-example -o name | xargs -I {} kubectl annotate {} foundationdb.org/outdated-config-map-seen=$(date +%s) --overwrite
+kubectl annotate pod -l app=fdb-kubernetes-example foundationdb.org/outdated-config-map-seen=$(date +%s) --overwrite
 # Watch the logs for the fdb-kubernetes-example pods to confirm that they have launched the fdbserver processes.
 kubectl exec -it sts/fdb-kubernetes-example -- fdbcli --exec "configure new double ssd"
 ```
 
-This will set up a cluster in your Kubernetes environment using a statefulset, to provide a simple subset of what the Kubernetes operator does to set up the cluster. Note: This assumes that you are running Docker Desktop on your local machine, with Kubernetes configured through Docker Desktop.
+This will set up a cluster in your Kubernetes environment using a StatefulSet, to provide a simple subset of what the Kubernetes operator does to set up the cluster.
+Note: This assumes that you are running Docker Desktop on your local machine, with Kubernetes configured through Docker Desktop.
 
-You can then make changes to the data in the config map and update the fdbserver processes:
+You can then make changes to the data in the ConfigMap and update the `fdbserver` processes:
 
 ```bash
 sed -e "s/fdb.cluster: \"\"/fdb.cluster: \"test:test@$ips\"/" -e "s/\"runProcesses\": false/\"runProcesses\": true/" packaging/docker/kubernetes/test_config.yaml | kubectl apply -f -
 
 # You can apply an annotation to speed up the propagation of config
-kubectl get pod -l app=fdb-kubernetes-example -o name | xargs -I {} kubectl annotate {} foundationdb.org/outdated-config-map-seen=$(date +%s) --overwrite
+kubectl annotate pod -l app=fdb-kubernetes-example foundationdb.org/outdated-config-map-seen=$(date +%s) --overwrite
 
 # Watch the logs for the fdb-kubernetes-example pods to confirm that they have reloaded their configuration, and then do a bounce.
 kubectl exec -it sts/fdb-kubernetes-example -- fdbcli --exec "kill; kill all; status"
@@ -33,5 +45,27 @@ kubectl exec -it sts/fdb-kubernetes-example -- fdbcli --exec "kill; kill all; st
 Once you are done, you can tear down the example with the following command:
 
 ```bash
-kubectl delete -f packaging/docker/kubernetes/test_config.yaml; kubectl delete pvc -l app=fdb-kubernetes-example
+kubectl delete -f packaging/docker/kubernetes/test_config.yaml
+kubectl delete pvc -l app=fdb-kubernetes-example
+```
+
+### FDB Kubernetes operator
+
+The following steps assume that you already have a [local development](https://github.com/FoundationDB/fdb-kubernetes-operator#local-development) setup for the fdb-kubernetes-operator.
+
+```bash
+mkdir -p website
+# Change this version if you want to create a cluster with a different version
+docker build -t foundationdb/foundationdb-kubernetes:7.1.11-local --target fdb-kubernetes-monitor --build-arg FDB_VERSION=7.1.11 --build-arg FDB_LIBRARY_VERSIONS="7.1.11 6.3.24 6.2.30" -f packaging/docker/Dockerfile .
+```
+
+Depending on the local Kubernetes setup you use you might have to push the newly build image to a local registry.
+Now you should change to the directory that contains the [fdb-kubernetes-operator](https://github.com/FoundationDB/fdb-kubernetes-operator) repository.
+In the top directory run:
+
+```bash
+# Adjust the version for the cluster if it differs from the build image
+vim ./config/tests/base/cluster.yaml
+# Now you can create the cluster
+kubectl apply -k ./config/tests/unified_image
 ```
diff --git a/fdbkubernetesmonitor/api/config.go b/fdbkubernetesmonitor/api/config.go
index c44834987b8..07839ccf374 100644
--- a/fdbkubernetesmonitor/api/config.go
+++ b/fdbkubernetesmonitor/api/config.go
@@ -21,8 +21,10 @@ package api
 
 import (
 	"fmt"
+	"net"
 	"os"
 	"strconv"
+	"strings"
 )
 
 // ProcessConfiguration models the configuration for starting a FoundationDB
@@ -39,7 +41,7 @@ type ProcessConfiguration struct {
 	// BinaryPath provides the path to the binary to launch.
 	BinaryPath string `json:"-"`
 
-	// Arguments provides the arugments to the process.
+	// Arguments provides the arguments to the process.
 	Arguments []Argument `json:"arguments,omitempty"`
 }
 
@@ -63,8 +65,11 @@ type Argument struct {
 	Multiplier int `json:"multiplier,omitempty"`
 
 	// Offset provides an offset to add to the process number for ProcessNumber
-	// type argujments.
+	// type arguments.
 	Offset int `json:"offset,omitempty"`
+
+	// IPFamily provides the family to use for IPList type arguments.
+	IPFamily int `json:"ipFamily,omitempty"`
 }
 
 // ArgumentType defines the types for arguments.
@@ -84,6 +89,10 @@ const (
 	// ProcessNumberArgumentType defines an argument that is calculated using
 	// the number of the process in the process list.
 	ProcessNumberArgumentType = "ProcessNumber"
+
+	// IPListArgumentType defines an argument that is a comma-separated list of
+	// IP addresses, provided through an environment variable.
+	IPListArgumentType = "IPList"
 )
 
 // GenerateArgument processes an argument and generates its string
@@ -111,25 +120,53 @@ func (argument Argument) GenerateArgument(processNumber int, env map[string]stri
 		}
 		number = number + argument.Offset
 		return strconv.Itoa(number), nil
-	case EnvironmentArgumentType:
-		var value string
-		var present bool
-		if env != nil {
-			value, present = env[argument.Source]
-		}
-		if !present {
-			value, present = os.LookupEnv(argument.Source)
-		}
-		if !present {
-			return "", fmt.Errorf("Missing environment variable %s", argument.Source)
-		}
-		return value, nil
+	case EnvironmentArgumentType, IPListArgumentType:
+		return argument.LookupEnv(env)
 	default:
-		return "", fmt.Errorf("Unsupported argument type %s", argument.ArgumentType)
+		return "", fmt.Errorf("unsupported argument type %s", argument.ArgumentType)
+	}
+}
+
+// LookupEnv looks up the value for an argument from the environment.
+func (argument Argument) LookupEnv(env map[string]string) (string, error) {
+	var value string
+	var present bool
+	if env != nil {
+		value, present = env[argument.Source]
+	}
+	if !present {
+		value, present = os.LookupEnv(argument.Source)
+	}
+	if !present {
+		return "", fmt.Errorf("missing environment variable %s", argument.Source)
+	}
+
+	if argument.ArgumentType == IPListArgumentType {
+		ips := strings.Split(value, ",")
+		for _, ipString := range ips {
+			ip := net.ParseIP(ipString)
+			if ip == nil {
+				continue
+			}
+			switch argument.IPFamily {
+			case 4:
+				if ip.To4() != nil {
+					return ipString, nil
+				}
+			case 6:
+				if ip.To16() != nil && ip.To4() == nil {
+					return ipString, nil
+				}
+			default:
+				return "", fmt.Errorf("unsupported IP family %d", argument.IPFamily)
+			}
+		}
+		return "", fmt.Errorf("could not find IP with family %d", argument.IPFamily)
 	}
+	return value, nil
 }
 
-// GenerateArguments intreprets the arguments in the process configuration and
+// GenerateArguments interprets the arguments in the process configuration and
 // generates a command invocation.
 func (configuration *ProcessConfiguration) GenerateArguments(processNumber int, env map[string]string) ([]string, error) {
 	results := make([]string, 0, len(configuration.Arguments)+1)
diff --git a/fdbkubernetesmonitor/api/config_test.go b/fdbkubernetesmonitor/api/config_test.go
index adaf49457dc..5f80e75f352 100644
--- a/fdbkubernetesmonitor/api/config_test.go
+++ b/fdbkubernetesmonitor/api/config_test.go
@@ -120,10 +120,216 @@ func TestGeneratingArgumentForEnvironmentVariable(t *testing.T) {
 		t.Fail()
 		return
 	}
-	expectedError := "Missing environment variable FDB_ZONE_ID"
+	expectedError := "missing environment variable FDB_ZONE_ID"
 	if err.Error() != expectedError {
 		t.Logf("Expected error %s, but got error %s", expectedError, err)
 		t.Fail()
 		return
 	}
 }
+
+func TestGeneratingArgumentForIPList(t *testing.T) {
+	argument := Argument{ArgumentType: IPListArgumentType, Source: "FDB_PUBLIC_IP", IPFamily: 4}
+
+	result, err := argument.GenerateArgument(1, map[string]string{"FDB_PUBLIC_IP": "127.0.0.1,::1"})
+	if err != nil {
+		t.Error(err)
+		return
+	}
+	if result != "127.0.0.1" {
+		t.Logf("Expected result 127.0.0.1, but got result %v", result)
+		t.Fail()
+		return
+	}
+
+	result, err = argument.GenerateArgument(1, map[string]string{"FDB_PUBLIC_IP": "::1,127.0.0.1"})
+	if err != nil {
+		t.Error(err)
+		return
+	}
+	if result != "127.0.0.1" {
+		t.Logf("Expected result 127.0.0.1, but got result %v", result)
+		t.Fail()
+		return
+	}
+
+	argument.IPFamily = 6
+
+	result, err = argument.GenerateArgument(1, map[string]string{"FDB_PUBLIC_IP": "127.0.0.1,::1"})
+	if err != nil {
+		t.Error(err)
+		return
+	}
+	if result != "::1" {
+		t.Logf("Expected result ::1, but got result %v", result)
+		t.Fail()
+		return
+	}
+
+	result, err = argument.GenerateArgument(1, map[string]string{"FDB_PUBLIC_IP": "::1,127.0.0.1"})
+	if err != nil {
+		t.Error(err)
+		return
+	}
+	if result != "::1" {
+		t.Logf("Expected result ::1, but got result %v", result)
+		t.Fail()
+		return
+	}
+
+	result, err = argument.GenerateArgument(1, map[string]string{"FDB_PUBLIC_IP": "bad,::1"})
+	if err != nil {
+		t.Error(err)
+		return
+	}
+	if result != "::1" {
+		t.Logf("Expected result ::1, but got result %v", result)
+		t.Fail()
+		return
+	}
+
+	_, err = argument.GenerateArgument(1, map[string]string{"FDB_PUBLIC_IP": "127.0.0.1"})
+	if err == nil {
+		t.Logf("Expected error, but did not get an error")
+		t.Fail()
+		return
+	}
+	expectedError := "could not find IP with family 6"
+	if err.Error() != expectedError {
+		t.Logf("Expected error %s, but got error %s", expectedError, err.Error())
+		t.Fail()
+		return
+	}
+
+	argument.IPFamily = 5
+
+	_, err = argument.GenerateArgument(1, map[string]string{"FDB_PUBLIC_IP": "127.0.0.1"})
+	if err == nil {
+		t.Logf("Expected error, but did not get an error")
+		t.Fail()
+		return
+	}
+	expectedError = "unsupported IP family 5"
+	if err.Error() != expectedError {
+		t.Logf("Expected error %s, but got error %s", expectedError, err.Error())
+		t.Fail()
+		return
+	}
+}
+
+func TestLookupEnvForEnvironmentVariable(t *testing.T) {
+	argument := Argument{ArgumentType: EnvironmentArgumentType, Source: "FDB_ZONE_ID"}
+
+	result, err := argument.LookupEnv(map[string]string{"FDB_ZONE_ID": "zone1", "FDB_MACHINE_ID": "machine1"})
+	if err != nil {
+		t.Error(err)
+		return
+	}
+	if result != "zone1" {
+		t.Logf("Expected result zone1, but got result %v", result)
+		t.Fail()
+		return
+	}
+
+	_, err = argument.LookupEnv(map[string]string{"FDB_MACHINE_ID": "machine1"})
+	if err == nil {
+		t.Logf("Expected error result, but did not get an error")
+		t.Fail()
+		return
+	}
+	expectedError := "missing environment variable FDB_ZONE_ID"
+	if err.Error() != expectedError {
+		t.Logf("Expected error %s, but got error %s", expectedError, err)
+		t.Fail()
+		return
+	}
+}
+
+func TestLookupEnvForIPList(t *testing.T) {
+	argument := Argument{ArgumentType: IPListArgumentType, Source: "FDB_PUBLIC_IP", IPFamily: 4}
+
+	result, err := argument.LookupEnv(map[string]string{"FDB_PUBLIC_IP": "127.0.0.1,::1"})
+	if err != nil {
+		t.Error(err)
+		return
+	}
+	if result != "127.0.0.1" {
+		t.Logf("Expected result 127.0.0.1, but got result %v", result)
+		t.Fail()
+		return
+	}
+
+	result, err = argument.LookupEnv(map[string]string{"FDB_PUBLIC_IP": "::1,127.0.0.1"})
+	if err != nil {
+		t.Error(err)
+		return
+	}
+	if result != "127.0.0.1" {
+		t.Logf("Expected result 127.0.0.1, but got result %v", result)
+		t.Fail()
+		return
+	}
+
+	argument.IPFamily = 6
+
+	result, err = argument.LookupEnv(map[string]string{"FDB_PUBLIC_IP": "127.0.0.1,::1"})
+	if err != nil {
+		t.Error(err)
+		return
+	}
+	if result != "::1" {
+		t.Logf("Expected result ::1, but got result %v", result)
+		t.Fail()
+		return
+	}
+
+	result, err = argument.LookupEnv(map[string]string{"FDB_PUBLIC_IP": "::1,127.0.0.1"})
+	if err != nil {
+		t.Error(err)
+		return
+	}
+	if result != "::1" {
+		t.Logf("Expected result ::1, but got result %v", result)
+		t.Fail()
+		return
+	}
+
+	result, err = argument.LookupEnv(map[string]string{"FDB_PUBLIC_IP": "bad,::1"})
+	if err != nil {
+		t.Error(err)
+		return
+	}
+	if result != "::1" {
+		t.Logf("Expected result ::1, but got result %v", result)
+		t.Fail()
+		return
+	}
+
+	_, err = argument.LookupEnv(map[string]string{"FDB_PUBLIC_IP": "127.0.0.1"})
+	if err == nil {
+		t.Logf("Expected error, but did not get an error")
+		t.Fail()
+		return
+	}
+	expectedError := "could not find IP with family 6"
+	if err.Error() != expectedError {
+		t.Logf("Expected error %s, but got error %s", expectedError, err.Error())
+		t.Fail()
+		return
+	}
+
+	argument.IPFamily = 5
+
+	_, err = argument.LookupEnv(map[string]string{"FDB_PUBLIC_IP": "127.0.0.1"})
+	if err == nil {
+		t.Logf("Expected error, but did not get an error")
+		t.Fail()
+		return
+	}
+	expectedError = "unsupported IP family 5"
+	if err.Error() != expectedError {
+		t.Logf("Expected error %s, but got error %s", expectedError, err.Error())
+		t.Fail()
+		return
+	}
+}
diff --git a/fdbkubernetesmonitor/copy.go b/fdbkubernetesmonitor/copy.go
index 2414a8cf7d8..0a311da095d 100644
--- a/fdbkubernetesmonitor/copy.go
+++ b/fdbkubernetesmonitor/copy.go
@@ -23,10 +23,19 @@ import (
 	"fmt"
 	"os"
 	"path"
+	"regexp"
 
 	"github.com/go-logr/logr"
 )
 
+const (
+	libraryTestDirectoryEnv = "TEST_LIBRARY_DIRECTORY"
+	binaryTestDirectoryEnv  = "TEST_BINARY_DIRECTORY"
+)
+
+// versionRegex represents the regex to parse the compact version string.
+var versionRegex = regexp.MustCompile(`^(\d+)\.(\d+)`)
+
 // copyFile copies a file into the output directory.
 func copyFile(logger logr.Logger, inputPath string, outputPath string, required bool) error {
 	logger.Info("Copying file", "inputPath", inputPath, "outputPath", outputPath)
@@ -44,12 +53,10 @@ func copyFile(logger logr.Logger, inputPath string, outputPath string, required
 	}
 
 	if required && inputInfo.Size() == 0 {
-		return fmt.Errorf("File %s is empty", inputPath)
+		return fmt.Errorf("file %s is empty", inputPath)
 	}
 
-	outputDir := path.Dir(outputPath)
-
-	tempFile, err := os.CreateTemp(outputDir, "")
+	tempFile, err := os.CreateTemp(path.Dir(outputPath), "")
 	if err != nil {
 		return err
 	}
@@ -60,11 +67,6 @@ func copyFile(logger logr.Logger, inputPath string, outputPath string, required
 		return err
 	}
 
-	err = tempFile.Close()
-	if err != nil {
-		return err
-	}
-
 	err = os.Chmod(tempFile.Name(), inputInfo.Mode())
 	if err != nil {
 		return err
@@ -90,11 +92,91 @@ func CopyFiles(logger logr.Logger, outputDir string, copyDetails map[string]stri
 			return err
 		}
 
-		required := requiredCopies[inputPath]
-		err = copyFile(logger, inputPath, outputPath, required)
+		err = copyFile(logger, inputPath, outputPath, requiredCopies[inputPath])
 		if err != nil {
 			return err
 		}
 	}
+
 	return nil
 }
+
+// getCompactVersion will return the compact version representation for the input version. The compact version consists
+// of the major and minor version.
+func getCompactVersion(version string) (string, error) {
+	matches := versionRegex.FindStringSubmatch(version)
+	if matches == nil {
+		return "", fmt.Errorf("could not parse FDB compact version from %s", version)
+	}
+
+	return matches[0], nil
+}
+
+// getBinaryDirectory returns the directory where the binaries are located. This will return `/usr/bin` if the env
+// variable TEST_BINARY_DIRECTORY is unset.
+func getBinaryDirectory() string {
+	binaryDirectory, ok := os.LookupEnv(binaryTestDirectoryEnv)
+	if ok {
+		return binaryDirectory
+	}
+
+	return "/usr/bin"
+}
+
+// getLibraryPath returns the library where the binaries are located. This will return `/usr/lib/fdb/multiversion` if the env
+// variable TEST_LIBRARY_DIRECTORY is unset.
+func getLibraryPath() string {
+	libraryDirectory, ok := os.LookupEnv(libraryTestDirectoryEnv)
+	if ok {
+		return libraryDirectory
+	}
+
+	return "/usr/lib/fdb/multiversion"
+}
+
+// getCopyDetails will generate the details for all the files that should be copied.
+func getCopyDetails(inputDir string, copyPrimaryLibrary string, binaryOutputDirectory string, copyFiles []string, copyBinaries []string, copyLibraries []string, requiredCopyFiles []string, currentContainerVersion string) (map[string]string, map[string]bool, error) {
+	copyDetails := make(map[string]string, len(copyFiles)+len(copyBinaries))
+
+	for _, filePath := range copyFiles {
+		copyDetails[path.Join(inputDir, filePath)] = ""
+	}
+
+	if len(copyBinaries) > 0 {
+		if binaryOutputDirectory == "" {
+			compactVersion, err := getCompactVersion(currentContainerVersion)
+			if err != nil {
+				return nil, nil, err
+			}
+
+			binaryOutputDirectory = compactVersion
+		}
+
+		binaryDirectory := getBinaryDirectory()
+		for _, copyBinary := range copyBinaries {
+			copyDetails[path.Join(binaryDirectory, copyBinary)] = path.Join(binaryOutputDirectory, copyBinary)
+		}
+	}
+
+	libraryPath := getLibraryPath()
+	for _, library := range copyLibraries {
+		libraryName := fmt.Sprintf("libfdb_c_%s.so", library)
+		copyDetails[path.Join(libraryPath, libraryName)] = libraryName
+	}
+
+	if copyPrimaryLibrary != "" {
+		copyDetails[path.Join(libraryPath, fmt.Sprintf("libfdb_c_%s.so", copyPrimaryLibrary))] = "libfdb_c.so"
+	}
+
+	requiredCopyMap := make(map[string]bool, len(requiredCopyFiles))
+	for _, filePath := range requiredCopyFiles {
+		fullFilePath := path.Join(inputDir, filePath)
+		_, present := copyDetails[fullFilePath]
+		if !present {
+			return nil, nil, fmt.Errorf("file %s is required, but is not in the --copy-file list", filePath)
+		}
+		requiredCopyMap[fullFilePath] = true
+	}
+
+	return copyDetails, requiredCopyMap, nil
+}
diff --git a/fdbkubernetesmonitor/copy_test.go b/fdbkubernetesmonitor/copy_test.go
new file mode 100644
index 00000000000..1f208ea0156
--- /dev/null
+++ b/fdbkubernetesmonitor/copy_test.go
@@ -0,0 +1,427 @@
+// copy_test.go
+//
+// This source file is part of the FoundationDB open source project
+//
+// Copyright 2023 Apple Inc. and the FoundationDB project authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package main
+
+import (
+	"fmt"
+	"os"
+	"path"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("Testing the copy methods", func() {
+	When("getting the copy details", func() {
+		var copyFiles, copyBinaries, copyLibraries, requiredCopyFiles []string
+		var currentContainerVersion, inputDir, copyPrimaryLibrary, binaryOutputDirectory string
+
+		AfterEach(func() {
+			copyFiles = []string{}
+			copyBinaries = []string{}
+			copyLibraries = []string{}
+			requiredCopyFiles = []string{}
+		})
+
+		BeforeEach(func() {
+			currentContainerVersion = "7.1.43"
+		})
+
+		When("no files should be copied", func() {
+			It("no error should be thrown", func() {
+				copyDetails, requiredCopyMap, err := getCopyDetails(inputDir, copyPrimaryLibrary, binaryOutputDirectory, copyFiles, copyBinaries, copyLibraries, requiredCopyFiles, currentContainerVersion)
+				Expect(err).NotTo(HaveOccurred())
+				Expect(copyDetails).To(BeEmpty())
+				Expect(requiredCopyMap).To(BeEmpty())
+			})
+		})
+
+		When("the fdbserver binary should be copied", func() {
+			When("the binary output directory is not set", func() {
+				BeforeEach(func() {
+					copyBinaries = []string{"fdbserver"}
+				})
+
+				It("no error should be thrown", func() {
+					copyDetails, requiredCopyMap, err := getCopyDetails(inputDir, copyPrimaryLibrary, binaryOutputDirectory, copyFiles, copyBinaries, copyLibraries, requiredCopyFiles, currentContainerVersion)
+					Expect(err).NotTo(HaveOccurred())
+					Expect(copyDetails).To(HaveKeyWithValue("/usr/bin/fdbserver", "7.1/fdbserver"))
+					Expect(copyDetails).To(HaveLen(1))
+					Expect(requiredCopyMap).To(BeEmpty())
+				})
+			})
+
+			When("the binary output directory is set", func() {
+				BeforeEach(func() {
+					copyBinaries = []string{"fdbserver"}
+					binaryOutputDirectory = "testing"
+				})
+
+				AfterEach(func() {
+					binaryOutputDirectory = ""
+				})
+
+				It("no error should be thrown", func() {
+					copyDetails, requiredCopyMap, err := getCopyDetails(inputDir, copyPrimaryLibrary, binaryOutputDirectory, copyFiles, copyBinaries, copyLibraries, requiredCopyFiles, currentContainerVersion)
+					Expect(err).NotTo(HaveOccurred())
+					Expect(copyDetails).To(HaveKeyWithValue("/usr/bin/fdbserver", "testing/fdbserver"))
+					Expect(copyDetails).To(HaveLen(1))
+					Expect(requiredCopyMap).To(BeEmpty())
+				})
+			})
+		})
+
+		When("the fdbserver and fdbbackup binary should be copied", func() {
+			BeforeEach(func() {
+				copyBinaries = []string{"fdbserver", "fdbbackup"}
+			})
+
+			It("no error should be thrown", func() {
+				copyDetails, requiredCopyMap, err := getCopyDetails(inputDir, copyPrimaryLibrary, binaryOutputDirectory, copyFiles, copyBinaries, copyLibraries, requiredCopyFiles, currentContainerVersion)
+				Expect(err).NotTo(HaveOccurred())
+				Expect(copyDetails).To(
+					And(
+						HaveKeyWithValue("/usr/bin/fdbserver", "7.1/fdbserver"),
+						HaveKeyWithValue("/usr/bin/fdbbackup", "7.1/fdbbackup"),
+					),
+				)
+				Expect(copyDetails).To(HaveLen(2))
+				Expect(requiredCopyMap).To(BeEmpty())
+			})
+		})
+
+		When("libraries should be copied", func() {
+			BeforeEach(func() {
+				copyLibraries = []string{"7.1.43", "7.3.27"}
+			})
+
+			It("no error should be thrown", func() {
+				copyDetails, requiredCopyMap, err := getCopyDetails(inputDir, copyPrimaryLibrary, binaryOutputDirectory, copyFiles, copyBinaries, copyLibraries, requiredCopyFiles, currentContainerVersion)
+				Expect(err).NotTo(HaveOccurred())
+				Expect(copyDetails).To(
+					And(
+						HaveKeyWithValue("/usr/lib/fdb/multiversion/libfdb_c_7.1.43.so", "libfdb_c_7.1.43.so"),
+						HaveKeyWithValue("/usr/lib/fdb/multiversion/libfdb_c_7.3.27.so", "libfdb_c_7.3.27.so"),
+					),
+				)
+				Expect(copyDetails).To(HaveLen(2))
+				Expect(requiredCopyMap).To(BeEmpty())
+			})
+		})
+
+		When("the fdbserver and fdbbackup binary should be copied", func() {
+			BeforeEach(func() {
+				copyBinaries = []string{"fdbserver", "fdbbackup"}
+			})
+
+			It("no error should be thrown", func() {
+				copyDetails, requiredCopyMap, err := getCopyDetails(inputDir, copyPrimaryLibrary, binaryOutputDirectory, copyFiles, copyBinaries, copyLibraries, requiredCopyFiles, currentContainerVersion)
+				Expect(err).NotTo(HaveOccurred())
+				Expect(copyDetails).To(
+					And(
+						HaveKeyWithValue("/usr/bin/fdbserver", "7.1/fdbserver"),
+						HaveKeyWithValue("/usr/bin/fdbbackup", "7.1/fdbbackup"),
+					),
+				)
+				Expect(copyDetails).To(HaveLen(2))
+				Expect(requiredCopyMap).To(BeEmpty())
+			})
+		})
+
+		When("libraries and binaries should be copied", func() {
+			BeforeEach(func() {
+				copyLibraries = []string{"7.1.43", "7.3.27"}
+				copyBinaries = []string{"fdbserver", "fdbbackup"}
+			})
+
+			It("no error should be thrown", func() {
+				copyDetails, requiredCopyMap, err := getCopyDetails(inputDir, copyPrimaryLibrary, binaryOutputDirectory, copyFiles, copyBinaries, copyLibraries, requiredCopyFiles, currentContainerVersion)
+				Expect(err).NotTo(HaveOccurred())
+				Expect(copyDetails).To(
+					And(
+						HaveKeyWithValue("/usr/bin/fdbserver", "7.1/fdbserver"),
+						HaveKeyWithValue("/usr/bin/fdbbackup", "7.1/fdbbackup"),
+						HaveKeyWithValue("/usr/lib/fdb/multiversion/libfdb_c_7.1.43.so", "libfdb_c_7.1.43.so"),
+						HaveKeyWithValue("/usr/lib/fdb/multiversion/libfdb_c_7.3.27.so", "libfdb_c_7.3.27.so"),
+					),
+				)
+				Expect(copyDetails).To(HaveLen(4))
+				Expect(requiredCopyMap).To(BeEmpty())
+			})
+		})
+
+		When("files should be copied", func() {
+			BeforeEach(func() {
+				copyFiles = []string{"testFile"}
+			})
+
+			When("no input directory is set", func() {
+				It("no error should be thrown", func() {
+					copyDetails, requiredCopyMap, err := getCopyDetails(inputDir, copyPrimaryLibrary, binaryOutputDirectory, copyFiles, copyBinaries, copyLibraries, requiredCopyFiles, currentContainerVersion)
+					Expect(err).NotTo(HaveOccurred())
+					Expect(copyDetails).To(
+						And(
+							HaveKeyWithValue("testFile", ""),
+						),
+					)
+					Expect(copyDetails).To(HaveLen(1))
+					Expect(requiredCopyMap).To(BeEmpty())
+				})
+			})
+
+			When("the input directory is set", func() {
+				BeforeEach(func() {
+					inputDir = "/testing"
+				})
+
+				AfterEach(func() {
+					inputDir = ""
+				})
+
+				It("no error should be thrown", func() {
+					copyDetails, requiredCopyMap, err := getCopyDetails(inputDir, copyPrimaryLibrary, binaryOutputDirectory, copyFiles, copyBinaries, copyLibraries, requiredCopyFiles, currentContainerVersion)
+					Expect(err).NotTo(HaveOccurred())
+					Expect(copyDetails).To(
+						And(
+							HaveKeyWithValue("/testing/testFile", ""),
+						),
+					)
+					Expect(copyDetails).To(HaveLen(1))
+					Expect(requiredCopyMap).To(BeEmpty())
+				})
+			})
+		})
+
+		When("the primary fdb library should be copied", func() {
+			BeforeEach(func() {
+				copyPrimaryLibrary = "7.1.43"
+			})
+
+			AfterEach(func() {
+				copyPrimaryLibrary = ""
+			})
+
+			It("no error should be thrown", func() {
+				copyDetails, requiredCopyMap, err := getCopyDetails(inputDir, copyPrimaryLibrary, binaryOutputDirectory, copyFiles, copyBinaries, copyLibraries, requiredCopyFiles, currentContainerVersion)
+				Expect(err).NotTo(HaveOccurred())
+				Expect(copyDetails).To(
+					And(
+						HaveKeyWithValue("/usr/lib/fdb/multiversion/libfdb_c_7.1.43.so", "libfdb_c.so"),
+					),
+				)
+				Expect(copyDetails).To(HaveLen(1))
+				Expect(requiredCopyMap).To(BeEmpty())
+			})
+		})
+
+		When("some files are required not to be empty", func() {
+			BeforeEach(func() {
+				requiredCopyFiles = []string{"testFile"}
+				copyBinaries = []string{"fdbserver"}
+				inputDir = "/testing"
+			})
+
+			AfterEach(func() {
+				inputDir = ""
+			})
+
+			When("a file is present in the require not empty list but should not be copied", func() {
+				BeforeEach(func() {
+					copyFiles = nil
+				})
+
+				It("should throw an error", func() {
+					copyDetails, requiredCopyMap, err := getCopyDetails(inputDir, copyPrimaryLibrary, binaryOutputDirectory, copyFiles, copyBinaries, copyLibraries, requiredCopyFiles, currentContainerVersion)
+					Expect(err).To(HaveOccurred())
+					Expect(copyDetails).To(BeEmpty())
+					Expect(requiredCopyMap).To(BeEmpty())
+				})
+			})
+
+			When("all files are present", func() {
+				BeforeEach(func() {
+					copyFiles = requiredCopyFiles
+				})
+
+				It("should not throw an error and include all files", func() {
+					copyDetails, requiredCopyMap, err := getCopyDetails(inputDir, copyPrimaryLibrary, binaryOutputDirectory, copyFiles, copyBinaries, copyLibraries, requiredCopyFiles, currentContainerVersion)
+					Expect(err).NotTo(HaveOccurred())
+					Expect(copyDetails).To(
+						And(
+							HaveKeyWithValue("/usr/bin/fdbserver", "7.1/fdbserver"),
+							HaveKeyWithValue("/testing/testFile", ""),
+						),
+					)
+					Expect(copyDetails).To(HaveLen(2))
+					Expect(requiredCopyMap).To(HaveLen(1))
+					Expect(requiredCopyMap).To(HaveKeyWithValue("/testing/testFile", true))
+				})
+			})
+		})
+	})
+
+	When("copying the binaries from the input directory to the output directory", func() {
+		var inputBinaryDir, outputBinaryDir string
+		binaries := []string{"fdbserver", "fdbbackup", "fdbrestore"}
+
+		BeforeEach(func() {
+			inputBinaryDir = GinkgoT().TempDir()
+			outputBinaryDir = GinkgoT().TempDir()
+
+			for _, binary := range binaries {
+				_, err := os.Create(path.Join(inputBinaryDir, binary))
+				Expect(err).NotTo(HaveOccurred())
+			}
+		})
+
+		When("copying the binaries", func() {
+			var copyDetails map[string]string
+
+			BeforeEach(func() {
+				// Simulate the binary directory
+				GinkgoT().Setenv(binaryTestDirectoryEnv, inputBinaryDir)
+
+				var err error
+				copyDetails, _, err = getCopyDetails("", "", "", nil, binaries, nil, nil, "7.1.43")
+				Expect(err).NotTo(HaveOccurred())
+				Expect(copyDetails).To(HaveLen(3))
+				Expect(CopyFiles(GinkgoLogr, outputBinaryDir, copyDetails, map[string]bool{})).NotTo(HaveOccurred())
+			})
+
+			It("should copy all the files", func() {
+				Expect(path.Join(outputBinaryDir, "7.1", "fdbserver")).Should(BeAnExistingFile())
+				Expect(path.Join(outputBinaryDir, "7.1", "fdbrestore")).Should(BeAnExistingFile())
+				Expect(path.Join(outputBinaryDir, "7.1", "fdbbackup")).Should(BeAnExistingFile())
+			})
+		})
+	})
+
+	When("copying the libraries from the input directory to the output directory", func() {
+		var inputLibraryDir, outputLibraryDir string
+		libraries := []string{"7.1", "7.3", "6.3"}
+
+		BeforeEach(func() {
+			inputLibraryDir = GinkgoT().TempDir()
+			outputLibraryDir = GinkgoT().TempDir()
+
+			for _, library := range libraries {
+				_, err := os.Create(path.Join(inputLibraryDir, fmt.Sprintf("libfdb_c_%s.so", library)))
+				Expect(err).NotTo(HaveOccurred())
+			}
+
+			// Simulate the binary directory
+			GinkgoT().Setenv(libraryTestDirectoryEnv, inputLibraryDir)
+		})
+
+		When("copying the libraries without a primary library", func() {
+			var copyDetails map[string]string
+
+			BeforeEach(func() {
+				var err error
+				copyDetails, _, err = getCopyDetails("", "", "", nil, nil, libraries, nil, "7.1.43")
+				Expect(err).NotTo(HaveOccurred())
+				Expect(copyDetails).To(HaveLen(3))
+				Expect(CopyFiles(GinkgoLogr, outputLibraryDir, copyDetails, map[string]bool{})).NotTo(HaveOccurred())
+			})
+
+			It("should copy all the files", func() {
+				Expect(path.Join(outputLibraryDir, "libfdb_c_7.1.so")).Should(BeAnExistingFile())
+				Expect(path.Join(outputLibraryDir, "libfdb_c_6.3.so")).Should(BeAnExistingFile())
+				Expect(path.Join(outputLibraryDir, "libfdb_c_7.3.so")).Should(BeAnExistingFile())
+			})
+		})
+
+		When("copying the libraries with a primary library", func() {
+			var copyDetails map[string]string
+
+			BeforeEach(func() {
+				var err error
+				copyDetails, _, err = getCopyDetails("", "7.1", "", nil, nil, libraries, nil, "7.1.43")
+				Expect(err).NotTo(HaveOccurred())
+				Expect(copyDetails).To(HaveLen(3))
+				Expect(CopyFiles(GinkgoLogr, outputLibraryDir, copyDetails, map[string]bool{})).NotTo(HaveOccurred())
+			})
+
+			It("should copy all the files", func() {
+				Expect(path.Join(outputLibraryDir, "libfdb_c.so")).Should(BeAnExistingFile())
+				Expect(path.Join(outputLibraryDir, "libfdb_c_6.3.so")).Should(BeAnExistingFile())
+				Expect(path.Join(outputLibraryDir, "libfdb_c_7.3.so")).Should(BeAnExistingFile())
+			})
+		})
+
+		When("copying the binaries from the input directory to the output directory", func() {
+			var testInputDir, testOutputDir string
+
+			BeforeEach(func() {
+				testInputDir = GinkgoT().TempDir()
+				testOutputDir = GinkgoT().TempDir()
+
+				_, err := os.Create(path.Join(testInputDir, "testfile"))
+				Expect(err).NotTo(HaveOccurred())
+			})
+
+			When("copying the file", func() {
+				var copyDetails map[string]string
+
+				BeforeEach(func() {
+					var err error
+					copyDetails, _, err = getCopyDetails(testInputDir, "", "", []string{"testfile"}, nil, nil, nil, "7.1.43")
+					Expect(err).NotTo(HaveOccurred())
+					Expect(copyDetails).To(HaveLen(1))
+					Expect(CopyFiles(GinkgoLogr, testOutputDir, copyDetails, map[string]bool{})).NotTo(HaveOccurred())
+				})
+
+				It("should copy the file", func() {
+					Expect(path.Join(testOutputDir, "testfile")).Should(BeAnExistingFile())
+				})
+			})
+
+			When("the file is required to not be empty", func() {
+				var copyDetails map[string]string
+				var requiredFiles map[string]bool
+
+				BeforeEach(func() {
+					var err error
+					copyDetails, requiredFiles, err = getCopyDetails(testInputDir, "", "", []string{"testfile"}, nil, nil, []string{"testfile"}, "7.1.43")
+					Expect(err).NotTo(HaveOccurred())
+					Expect(copyDetails).To(HaveLen(1))
+				})
+
+				When("the file is empty", func() {
+					It("should not copy the file", func() {
+						Expect(CopyFiles(GinkgoLogr, testOutputDir, copyDetails, requiredFiles)).To(HaveOccurred())
+						Expect(path.Join(testOutputDir, "testfile")).NotTo(BeAnExistingFile())
+					})
+				})
+
+				When("the file is not empty", func() {
+					BeforeEach(func() {
+						Expect(os.WriteFile(path.Join(testInputDir, "testfile"), []byte("Hello World"), 0644)).NotTo(HaveOccurred())
+					})
+
+					It("should copy the file", func() {
+						Expect(CopyFiles(GinkgoLogr, testOutputDir, copyDetails, requiredFiles)).NotTo(HaveOccurred())
+						Expect(path.Join(testOutputDir, "testfile")).To(BeAnExistingFile())
+					})
+				})
+			})
+		})
+	})
+})
diff --git a/fdbkubernetesmonitor/errcheck_excludes.txt b/fdbkubernetesmonitor/errcheck_excludes.txt
new file mode 100644
index 00000000000..5b46ca2672d
--- /dev/null
+++ b/fdbkubernetesmonitor/errcheck_excludes.txt
@@ -0,0 +1,2 @@
+(io.Closer).Close
+(*os.File).Close
diff --git a/fdbkubernetesmonitor/go.mod b/fdbkubernetesmonitor/go.mod
index fec774f3277..4e33ab244a5 100644
--- a/fdbkubernetesmonitor/go.mod
+++ b/fdbkubernetesmonitor/go.mod
@@ -19,15 +19,69 @@
 
 module github.com/apple/foundationdb/fdbkubernetesmonitor
 
-go 1.16
+go 1.20
 
 require (
-	github.com/fsnotify/fsnotify v1.5.0
-	github.com/go-logr/logr v0.4.0
-	github.com/go-logr/zapr v0.4.0
+	github.com/fsnotify/fsnotify v1.6.0
+	github.com/go-logr/logr v1.2.4
+	github.com/go-logr/zapr v1.2.4
+	github.com/onsi/ginkgo/v2 v2.9.7
+	github.com/onsi/gomega v1.27.8
+	github.com/prometheus/client_golang v1.16.0
 	github.com/spf13/pflag v1.0.5
-	go.uber.org/zap v1.19.0
-	k8s.io/api v0.20.2
-	k8s.io/apimachinery v0.20.2
-	k8s.io/client-go v0.20.2
+	go.uber.org/zap v1.25.0
+	gopkg.in/natefinch/lumberjack.v2 v2.2.1
+	k8s.io/api v0.26.1
+	k8s.io/apimachinery v0.26.1
+	k8s.io/client-go v0.26.1
+	k8s.io/utils v0.0.0-20230115233650-391b47cb4029
+	sigs.k8s.io/controller-runtime v0.14.6
+)
+
+require (
+	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/cespare/xxhash/v2 v2.2.0 // indirect
+	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/emicklei/go-restful/v3 v3.9.0 // indirect
+	github.com/evanphx/json-patch/v5 v5.6.0 // indirect
+	github.com/go-openapi/jsonpointer v0.19.5 // indirect
+	github.com/go-openapi/jsonreference v0.20.0 // indirect
+	github.com/go-openapi/swag v0.19.14 // indirect
+	github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect
+	github.com/gogo/protobuf v1.3.2 // indirect
+	github.com/golang/protobuf v1.5.3 // indirect
+	github.com/google/gnostic v0.5.7-v3refs // indirect
+	github.com/google/go-cmp v0.5.9 // indirect
+	github.com/google/gofuzz v1.1.0 // indirect
+	github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38 // indirect
+	github.com/josharian/intern v1.0.0 // indirect
+	github.com/json-iterator/go v1.1.12 // indirect
+	github.com/mailru/easyjson v0.7.6 // indirect
+	github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
+	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
+	github.com/modern-go/reflect2 v1.0.2 // indirect
+	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
+	github.com/pkg/errors v0.9.1 // indirect
+	github.com/prometheus/client_model v0.3.0 // indirect
+	github.com/prometheus/common v0.42.0 // indirect
+	github.com/prometheus/procfs v0.10.1 // indirect
+	github.com/rogpeppe/go-internal v1.12.0 // indirect
+	go.uber.org/multierr v1.10.0 // indirect
+	golang.org/x/net v0.17.0 // indirect
+	golang.org/x/oauth2 v0.5.0 // indirect
+	golang.org/x/sys v0.13.0 // indirect
+	golang.org/x/term v0.13.0 // indirect
+	golang.org/x/text v0.13.0 // indirect
+	golang.org/x/time v0.3.0 // indirect
+	golang.org/x/tools v0.9.1 // indirect
+	google.golang.org/appengine v1.6.7 // indirect
+	google.golang.org/protobuf v1.33.0 // indirect
+	gopkg.in/inf.v0 v0.9.1 // indirect
+	gopkg.in/yaml.v2 v2.4.0 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
+	k8s.io/klog/v2 v2.80.1 // indirect
+	k8s.io/kube-openapi v0.0.0-20221012153701-172d655c2280 // indirect
+	sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2 // indirect
+	sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect
+	sigs.k8s.io/yaml v1.3.0 // indirect
 )
diff --git a/fdbkubernetesmonitor/go.sum b/fdbkubernetesmonitor/go.sum
index 1c021ce5283..0b3cc3df237 100644
--- a/fdbkubernetesmonitor/go.sum
+++ b/fdbkubernetesmonitor/go.sum
@@ -1,439 +1,284 @@
 cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
-cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
-cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU=
-cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU=
-cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY=
-cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc=
-cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0=
-cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To=
-cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4=
-cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M=
-cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc=
-cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o=
-cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE=
-cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc=
-cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE=
-cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk=
-cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I=
-cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw=
-cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA=
-cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw=
-cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos=
-cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk=
-dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
-github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24=
-github.com/Azure/go-autorest/autorest v0.11.1/go.mod h1:JFgpikqFJ/MleTTxwepExTKnFUKKszPS8UavbQYUMuw=
-github.com/Azure/go-autorest/autorest/adal v0.9.0/go.mod h1:/c022QCutn2P7uY+/oQWWNcK9YU+MH96NgK+jErpbcg=
-github.com/Azure/go-autorest/autorest/adal v0.9.5/go.mod h1:B7KF7jKIeC9Mct5spmyCB/A8CG/sEz1vwIRGv/bbw7A=
-github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSYnokU+TrmwEsOqdt8Y6sso74=
-github.com/Azure/go-autorest/autorest/mocks v0.4.0/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k=
-github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k=
-github.com/Azure/go-autorest/logger v0.2.0/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8=
-github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
-github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
-github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ=
-github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0=
-github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE=
-github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
-github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8=
 github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
+github.com/benbjohnson/clock v1.3.0 h1:ip6w0uFQkncKQ979AypyG0ER7mqUSBdKLOgAle/AT8A=
+github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
+github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
+github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44=
+github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
 github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
 github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
+github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
-github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM=
 github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE=
-github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc=
-github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
+github.com/emicklei/go-restful/v3 v3.9.0 h1:XwGDlfxEnQZzuopoqxwSEllNcCOM9DhhFyhFIIGKwxE=
+github.com/emicklei/go-restful/v3 v3.9.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc=
 github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
-github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
-github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k=
-github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
-github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
-github.com/fsnotify/fsnotify v1.5.0 h1:NO5hkcB+srp1x6QmwvNZLeaOgbM8cmBTN32THzjvu2k=
-github.com/fsnotify/fsnotify v1.5.0/go.mod h1:BX0DCEr5pT4jm2CnQdVP1lFV521fcCNcyEeNp4DQQDk=
-github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
-github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
-github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
-github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
-github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas=
-github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU=
-github.com/go-logr/logr v0.4.0 h1:K7/B1jt6fIBQVd4Owv2MqGQClcgf0R266+7C/QjRcLc=
-github.com/go-logr/logr v0.4.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU=
-github.com/go-logr/zapr v0.4.0 h1:uc1uML3hRYL9/ZZPdgHS/n8Nzo+eaYL/Efxkkamf7OM=
-github.com/go-logr/zapr v0.4.0/go.mod h1:tabnROwaDl0UNxkVeFRbY8bwB37GwRv0P8lg6aAiEnk=
-github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg=
+github.com/evanphx/json-patch/v5 v5.6.0 h1:b91NhWfaz02IuVxO9faSllyAtNXHMPkC5J8sJCLunww=
+github.com/evanphx/json-patch/v5 v5.6.0/go.mod h1:G79N1coSVB93tBe7j6PhzjmR3/2VvlbKOFpnXhI9Bw4=
+github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY=
+github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw=
+github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
+github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ=
+github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
+github.com/go-logr/zapr v1.2.4 h1:QHVo+6stLbfJmYGkQ7uGHUCu5hnAFAj6mDe6Ea0SeOo=
+github.com/go-logr/zapr v1.2.4/go.mod h1:FyHWQIzQORZ0QVE1BtVHv3cKtNLuXsbNLtpuhNapBOA=
 github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg=
-github.com/go-openapi/jsonreference v0.19.2/go.mod h1:jMjeRr2HHw6nAVajTXJ4eiUwohSTlpa0o73RUL1owJc=
-github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8=
-github.com/go-openapi/spec v0.19.3/go.mod h1:FpwSN1ksY1eteniUU7X0N/BgJ7a4WvBFVA8Lj9mJglo=
-github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk=
+github.com/go-openapi/jsonpointer v0.19.5 h1:gZr+CIYByUqjcgeLXnQu2gHYQC9o73G2XUeOFYEICuY=
+github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg=
+github.com/go-openapi/jsonreference v0.20.0 h1:MYlu0sBgChmCfJxxUKZ8g1cPWFOB37YSZqewK7OKeyA=
+github.com/go-openapi/jsonreference v0.20.0/go.mod h1:Ag74Ico3lPc+zR+qjn4XBUmXymS4zJbYVCZmcgkasdo=
 github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk=
-github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls=
-github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o=
+github.com/go-openapi/swag v0.19.14 h1:gm3vOOXfiuw5i9p5N9xJvfjvuofpyvLA9Wr6QfK5Fng=
+github.com/go-openapi/swag v0.19.14/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ=
+github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI=
+github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls=
+github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
+github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
-github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
-github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
-github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE=
 github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
-github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
-github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y=
-github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
-github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
-github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
-github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
+github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk=
 github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
 github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
 github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
 github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
 github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
 github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
-github.com/golang/protobuf v1.4.3 h1:JjCZWpVbqXDqFVmTfYWEVTMIYrL/NPdPSCHPJ0T/raM=
-github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
-github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
-github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
+github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
+github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
+github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg=
+github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
+github.com/google/gnostic v0.5.7-v3refs h1:FhTMOKj2VhjpouxvWJAV1TL304uMlb9zcDqkl6cEI54=
+github.com/google/gnostic v0.5.7-v3refs/go.mod h1:73MKFl6jIHelAJNaBGFzt3SPtZULs9dYrGFt8OiIsHQ=
 github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
 github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.2 h1:X2ev0eStA3AbceY54o37/0PQ/UWqKEiiO2dKL5OPaFM=
-github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
+github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g=
 github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
-github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
-github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
-github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
-github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
-github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
-github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
-github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
-github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
-github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
-github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
-github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
-github.com/googleapis/gnostic v0.4.1 h1:DLJCy1n/vrD4HPjOvYcT8aYQXpPIzoRZONaYwyycI+I=
-github.com/googleapis/gnostic v0.4.1/go.mod h1:LRhVm6pbyptWbWbuZ38d1eyptfvIytN3ir6b65WBswg=
-github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA=
-github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
-github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
-github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
-github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
-github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
-github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
-github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr68=
-github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
-github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
-github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
-github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00=
+github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38 h1:yAJXTCF9TqKcTiHJAE8dj7HMvPfh66eeA2JYW7eFpSE=
+github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
+github.com/google/uuid v1.1.2 h1:EVhdT+1Kseyi1/pUmXKaFxYsDNy9RQYkMWRH68J/W7Y=
+github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
+github.com/imdario/mergo v0.3.6 h1:xTNEAn+kxVO7dTZGu0CegyqKZmoWFI0rF8UxjlB2d28=
+github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
+github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
+github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
+github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
+github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
+github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
-github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs=
 github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
+github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
-github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA=
-github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
+github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
 github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
 github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
-github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
+github.com/mailru/easyjson v0.7.6 h1:8yTIVnZgCoiM1TgqoeTl+LfU5Jg6/xL3QhGQnimLYnA=
+github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
+github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo=
+github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4=
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
-github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
-github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI=
-github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
-github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
-github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw=
-github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
-github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
-github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
-github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA=
-github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
-github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU=
+github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
+github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
+github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
+github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
+github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
+github.com/onsi/ginkgo/v2 v2.9.7 h1:06xGQy5www2oN160RtEZoTvnP2sPhEfePYmCDc2szss=
+github.com/onsi/ginkgo/v2 v2.9.7/go.mod h1:cxrmXWykAwTwhQsJOPfdIDiJ+l2RYq7U8hFU+M/1uw0=
+github.com/onsi/gomega v1.27.8 h1:gegWiwZjBsf2DgiSbf5hpokZ98JVDMcWkUiigk6/KXc=
+github.com/onsi/gomega v1.27.8/go.mod h1:2J8vzI/s+2shY9XHRApDkdgPo1TKT7P2u6fXeJKFnNQ=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/prometheus/client_golang v1.16.0 h1:yk/hx9hDbrGHovbci4BY+pRMfSuuat626eFsHb7tmT8=
+github.com/prometheus/client_golang v1.16.0/go.mod h1:Zsulrv/L9oM40tJ7T815tM89lFEugiJ9HzIqaAx4LKc=
 github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
-github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
-github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk=
-github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
+github.com/prometheus/client_model v0.3.0 h1:UBgGFHqYdG/TPFD1B1ogZywDqEkwp3fBMvqdiQ7Xew4=
+github.com/prometheus/client_model v0.3.0/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w=
+github.com/prometheus/common v0.42.0 h1:EKsfXEYo4JpWMHH5cg+KOUWeuJSov1Id8zGR8eeI1YM=
+github.com/prometheus/common v0.42.0/go.mod h1:xBwqVerjNdUDjgODMpudtOMwlOwf2SaTr1yjz4b7Zbc=
+github.com/prometheus/procfs v0.10.1 h1:kYK1Va/YMlutzCGazswoHKo//tZVlFpKYh+PymziUAg=
+github.com/prometheus/procfs v0.10.1/go.mod h1:nwNm2aOCAYw8uTR/9bWRREkZFxAUcWzPHWJq+XBB/FM=
+github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8=
+github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4=
 github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
 github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
+github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
-github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE=
+github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
-github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
+github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
-github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
-go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
-go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
-go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
-go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
-go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw=
+github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
+github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk=
+github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
 go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
-go.uber.org/goleak v1.1.10 h1:z+mqJhf6ss6BSfSM671tgKyZBFPTTJM+HLxnhPC3wu0=
-go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A=
-go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4=
+go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ=
+go.uber.org/goleak v1.2.0 h1:xqgm/S+aQvhWFTtR0XK3Jvg7z8kGV8P4X14IzwN3Eqk=
 go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU=
-go.uber.org/zap v1.19.0 h1:mZQZefskPPCMIBCSEH0v2/iUqqLrYtaeqwD6FUGUnFE=
-go.uber.org/zap v1.19.0/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI=
+go.uber.org/multierr v1.10.0 h1:S0h4aNzvfcFsC3dRF1jLoaov7oRaKqRGC/pUEJ2yvPQ=
+go.uber.org/multierr v1.10.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
+go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg=
+go.uber.org/zap v1.25.0 h1:4Hvk6GtkucQ790dqmj7l1eEnRdKm3k3ZUrUMS2d5+5c=
+go.uber.org/zap v1.25.0/go.mod h1:JIAUzQIH94IC4fOJQm7gMmBJP5k7wQfdcnYdPoEXJYk=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
-golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
-golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
-golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
-golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0 h1:hb9wdF1z5waM+dSIICn1l0DkLVDT3hqhhQsDNUmHPRE=
-golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
-golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
-golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
-golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek=
-golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY=
-golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
-golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
-golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
-golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM=
-golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU=
-golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
-golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
 golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
-golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
 golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
-golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
-golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
 golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
-golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs=
-golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
-golang.org/x/lint v0.0.0-20200302205851-738671d3881b h1:Wh+f8QHJXR411sJR8/vRBTZ7YapZaRvUcLFFJhusH0k=
-golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
-golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE=
-golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o=
-golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc=
-golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY=
-golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
-golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
 golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.10.0 h1:lFO9qtOdlre5W1jxS3r/4szv2/6iXxScdzjoBMXNhYk=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
-golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
-golang.org/x/net v0.0.0-20201110031124-69a78807bb2b h1:uwuIcX0g4Yl1NC5XAz37xsr2lTtcqevgzYNVt49waME=
-golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
+golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
+golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
+golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM=
+golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
-golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
-golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
-golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
-golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d h1:TzXSXBo42m9gQenoE3b9BGiEpg5IG2JkU5FkPIawgtw=
-golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/oauth2 v0.5.0 h1:HuArIo48skDwlrvM3sEdHXElYslAMsf3KwRkkW4MC4s=
+golang.org/x/oauth2 v0.5.0/go.mod h1:9/XBHVqLaWO3/BRHs5jbpYCnOZVjj5V0ndyaAM7KB4I=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20201112073958-5cba982894dd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c h1:F1jZWGFhYfh0Ci55sIpILtKKK8p3i2/krTr0H1rg74I=
-golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE=
+golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/term v0.13.0 h1:bb+I9cTfFazGW51MZqBVmZy7+JEJMouUHTUSKVQLBek=
+golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
-golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/text v0.3.4 h1:0YWbFKbhXG/wIiuHDSKpS0Iy7FSA+u45VtBMfQcFTTc=
-golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
-golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
-golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
-golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e h1:EHBhcS0mlXEAVwNyO2dLfjToGsyY4j24pTs2ScHnX7s=
-golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k=
+golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
+golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4=
+golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
-golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
 golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
-golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
-golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
-golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
-golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
 golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
-golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
-golang.org/x/tools v0.0.0-20190614205625-5aca471b1d59/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
-golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
-golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
-golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
-golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
-golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
-golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
-golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
-golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
-golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
-golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
-golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
-golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb h1:iKlO7ROJc6SttHKlxzwGytRtBUqX4VARrNTgP2YLX5M=
-golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw=
+golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
+golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
+golang.org/x/tools v0.9.1 h1:8WMNJAz3zrtPmnYC7ISf5dEn3MT0gY7jBJfw27yrrLo=
+golang.org/x/tools v0.9.1/go.mod h1:owI94Op576fPu3cIGQeHs3joujW/2Oc6MtlxbF5dfNc=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
-google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M=
-google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
-google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
-google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
-google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
-google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
-google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
-google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
-google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
+gomodules.xyz/jsonpatch/v2 v2.2.0 h1:4pT439QV83L+G9FkcCriY6EkpcK6r6bK+A5FBUMI7qY=
 google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
 google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
-google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
-google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0=
-google.golang.org/appengine v1.6.5 h1:tycE03LOZYQNhDpS27tcQdAzLCVMaj7QT2SXxebnpCM=
-google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
+google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c=
+google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
 google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
-google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
-google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
-google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
-google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
-google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
 google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
-google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8=
-google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
-google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
-google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
-google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
-google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
-google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
-google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA=
-google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
-google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
-google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
 google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
+google.golang.org/genproto v0.0.0-20201019141844-1ed22bb0c154/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
 google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
-google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
-google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
 google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
-google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
 google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
-google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
 google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
 google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
 google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
 google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
 google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
 google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
-google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
 google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
-google.golang.org/protobuf v1.25.0 h1:Ejskq+SyPohKW+1uil0JJMtmHCgJPJ/qWTxr8qp+R4c=
-google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
+google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4=
+google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
+google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
+google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI=
+google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
-gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
+gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
 gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc=
 gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
-gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
-gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc=
+gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10=
 gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
+gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo=
-gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
-honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
-honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
-honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
-honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
-k8s.io/api v0.20.2 h1:y/HR22XDZY3pniu9hIFDLpUCPq2w5eQ6aV/VFQ7uJMw=
-k8s.io/api v0.20.2/go.mod h1:d7n6Ehyzx+S+cE3VhTGfVNNqtGc/oL9DCdYYahlurV8=
-k8s.io/apimachinery v0.20.2 h1:hFx6Sbt1oG0n6DZ+g4bFt5f6BoMkOjKWsQFu077M3Vg=
-k8s.io/apimachinery v0.20.2/go.mod h1:WlLqWAHZGg07AeltaI0MV5uk1Omp8xaN0JGLY6gkRpU=
-k8s.io/client-go v0.20.2 h1:uuf+iIAbfnCSw8IGAv/Rg0giM+2bOzHLOsbbrwrdhNQ=
-k8s.io/client-go v0.20.2/go.mod h1:kH5brqWqp7HDxUFKoEgiI4v8G1xzbe9giaCenUWJzgE=
-k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0=
-k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE=
-k8s.io/klog/v2 v2.4.0 h1:7+X0fUguPyrKEC4WjH8iGDg3laWgMo5tMnRTIGTTxGQ=
-k8s.io/klog/v2 v2.4.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y=
-k8s.io/kube-openapi v0.0.0-20201113171705-d219536bb9fd/go.mod h1:WOJ3KddDSol4tAGcJo0Tvi+dK12EcqSLqcWsryKMpfM=
-k8s.io/utils v0.0.0-20201110183641-67b214c5f920 h1:CbnUZsM497iRC5QMVkHwyl8s2tB3g7yaSHkYPkpgelw=
-k8s.io/utils v0.0.0-20201110183641-67b214c5f920/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
-rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
-rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
-rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=
-sigs.k8s.io/structured-merge-diff/v4 v4.0.2 h1:YHQV7Dajm86OuqnIR6zAelnDWBRjo+YhYV9PmGrh1s8=
-sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw=
-sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o=
-sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q=
-sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc=
+k8s.io/api v0.26.1 h1:f+SWYiPd/GsiWwVRz+NbFyCgvv75Pk9NK6dlkZgpCRQ=
+k8s.io/api v0.26.1/go.mod h1:xd/GBNgR0f707+ATNyPmQ1oyKSgndzXij81FzWGsejg=
+k8s.io/apiextensions-apiserver v0.26.1 h1:cB8h1SRk6e/+i3NOrQgSFij1B2S0Y0wDoNl66bn8RMI=
+k8s.io/apimachinery v0.26.1 h1:8EZ/eGJL+hY/MYCNwhmDzVqq2lPl3N3Bo8rvweJwXUQ=
+k8s.io/apimachinery v0.26.1/go.mod h1:tnPmbONNJ7ByJNz9+n9kMjNP8ON+1qoAIIC70lztu74=
+k8s.io/client-go v0.26.1 h1:87CXzYJnAMGaa/IDDfRdhTzxk/wzGZ+/HUQpqgVSZXU=
+k8s.io/client-go v0.26.1/go.mod h1:IWNSglg+rQ3OcvDkhY6+QLeasV4OYHDjdqeWkDQZwGE=
+k8s.io/component-base v0.26.1 h1:4ahudpeQXHZL5kko+iDHqLj/FSGAEUnSVO0EBbgDd+4=
+k8s.io/klog/v2 v2.80.1 h1:atnLQ121W371wYYFawwYx1aEY2eUfs4l3J72wtgAwV4=
+k8s.io/klog/v2 v2.80.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0=
+k8s.io/kube-openapi v0.0.0-20221012153701-172d655c2280 h1:+70TFaan3hfJzs+7VK2o+OGxg8HsuBr/5f6tVAjDu6E=
+k8s.io/kube-openapi v0.0.0-20221012153701-172d655c2280/go.mod h1:+Axhij7bCpeqhklhUTe3xmOn6bWxolyZEeyaFpjGtl4=
+k8s.io/utils v0.0.0-20230115233650-391b47cb4029 h1:L8zDtT4jrxj+TaQYD0k8KNlr556WaVQylDXswKmX+dE=
+k8s.io/utils v0.0.0-20230115233650-391b47cb4029/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0=
+sigs.k8s.io/controller-runtime v0.14.6 h1:oxstGVvXGNnMvY7TAESYk+lzr6S3V5VFxQ6d92KcwQA=
+sigs.k8s.io/controller-runtime v0.14.6/go.mod h1:WqIdsAY6JBsjfc/CqO0CORmNtoCtE4S6qbPc9s68h+0=
+sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2 h1:iXTIw73aPyC+oRdyqqvVJuloN1p0AC/kzH07hu3NE+k=
+sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0=
+sigs.k8s.io/structured-merge-diff/v4 v4.2.3 h1:PRbqxJClWWYMNV1dhaG4NsibJbArud9kFxnAMREiWFE=
+sigs.k8s.io/structured-merge-diff/v4 v4.2.3/go.mod h1:qjx8mGObPmV2aSZepjQjbmb2ihdVs8cGKBraizNC69E=
+sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo=
+sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8=
diff --git a/fdbkubernetesmonitor/kubernetes.go b/fdbkubernetesmonitor/kubernetes.go
index da4072aac1e..497ec3af918 100644
--- a/fdbkubernetesmonitor/kubernetes.go
+++ b/fdbkubernetesmonitor/kubernetes.go
@@ -2,7 +2,7 @@
 //
 // This source file is part of the FoundationDB open source project
 //
-// Copyright 2021 Apple Inc. and the FoundationDB project authors
+// Copyright 2023 Apple Inc. and the FoundationDB project authors
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -22,7 +22,6 @@ package main
 import (
 	"context"
 	"encoding/json"
-	"fmt"
 	"os"
 	"path"
 	"strconv"
@@ -31,11 +30,12 @@ import (
 	"github.com/go-logr/logr"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/apimachinery/pkg/types"
-	"k8s.io/apimachinery/pkg/watch"
-	"k8s.io/client-go/kubernetes"
-	typedv1 "k8s.io/client-go/kubernetes/typed/core/v1"
+	"k8s.io/apimachinery/pkg/fields"
+	"k8s.io/apimachinery/pkg/runtime"
+	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
 	"k8s.io/client-go/rest"
+	"sigs.k8s.io/controller-runtime/pkg/cache"
+	"sigs.k8s.io/controller-runtime/pkg/client"
 )
 
 const (
@@ -50,15 +50,17 @@ const (
 	// OutdatedConfigMapAnnotation is the annotation we read to get notified of
 	// outdated configuration.
 	OutdatedConfigMapAnnotation = "foundationdb.org/outdated-config-map-seen"
+
+	// DelayShutdownAnnotation defines how long the FDB Kubernetes monitor process should sleep before shutting itself down.
+	// The FDB Kubernetes monitor will always shutdown all fdbserver processes, independent of this setting.
+	// The value of this annotation must be a duration like "60s".
+	DelayShutdownAnnotation = "foundationdb.org/delay-shutdown"
 )
 
 // PodClient is a wrapper around the pod API.
 type PodClient struct {
-	// podApi is the raw API
-	podApi typedv1.PodInterface
-
-	// pod is the latest pod configuration
-	pod *corev1.Pod
+	// metadata is the latest metadata that was seen by the fdb-kubernetes-monitor for the according Pod.
+	metadata *metav1.PartialObjectMetadata
 
 	// TimestampFeed is a channel where the pod client will send updates with
 	// the values from OutdatedConfigMapAnnotation.
@@ -66,53 +68,122 @@ type PodClient struct {
 
 	// Logger is the logger we use for this client.
 	Logger logr.Logger
+
+	// Adds the controller runtime client to the PodClient.
+	client.Client
 }
 
 // CreatePodClient creates a new client for working with the pod object.
-func CreatePodClient(logger logr.Logger) (*PodClient, error) {
+func CreatePodClient(ctx context.Context, logger logr.Logger) (*PodClient, error) {
 	config, err := rest.InClusterConfig()
 	if err != nil {
 		return nil, err
 	}
-	client, err := kubernetes.NewForConfig(config)
+
+	scheme := runtime.NewScheme()
+	err = clientgoscheme.AddToScheme(scheme)
+	if err != nil {
+		return nil, err
+	}
+
+	// Create the new client for writes. This client will also be used to setup the cache.
+	internalClient, err := client.NewWithWatch(config, client.Options{
+		Scheme: scheme,
+	})
 	if err != nil {
 		return nil, err
 	}
 
-	podApi := client.CoreV1().Pods(os.Getenv("FDB_POD_NAMESPACE"))
-	pod, err := podApi.Get(context.Background(), os.Getenv("FDB_POD_NAME"), metav1.GetOptions{ResourceVersion: "0"})
+	namespace := os.Getenv("FDB_POD_NAMESPACE")
+	podName := os.Getenv("FDB_POD_NAME")
+
+	internalCache, err := cache.New(config, cache.Options{
+		Scheme:    scheme,
+		Mapper:    internalClient.RESTMapper(),
+		Namespace: namespace,
+		SelectorsByObject: map[client.Object]cache.ObjectSelector{
+			&corev1.Pod{}: {
+				Field: fields.OneTermEqualSelector(metav1.ObjectNameField, podName),
+			},
+		},
+	})
 	if err != nil {
 		return nil, err
 	}
 
-	podClient := &PodClient{podApi: podApi, pod: pod, TimestampFeed: make(chan int64, 10), Logger: logger}
-	err = podClient.watchPod()
+	// Fetch the informer for the FoundationDBCluster resource
+	informer, err := internalCache.GetInformer(ctx, &corev1.Pod{})
 	if err != nil {
 		return nil, err
 	}
 
+	podClient := &PodClient{
+		metadata:      nil,
+		TimestampFeed: make(chan int64, 10),
+		Logger:        logger,
+	}
+
+	// Setup an event handler to make sue we get events for new clusters and directly reload.
+	_, err = informer.AddEventHandler(podClient)
+	if err != nil {
+		return nil, err
+	}
+
+	// Make sure the internal cache is started.
+	go func() {
+		_ = internalCache.Start(ctx)
+	}()
+
+	// This should be fairly quick as no informers are provided by default.
+	internalCache.WaitForCacheSync(ctx)
+	controllerClient, err := client.NewDelegatingClient(client.NewDelegatingClientInput{
+		CacheReader:       internalCache,
+		Client:            internalClient,
+		UncachedObjects:   nil,
+		CacheUnstructured: false,
+	})
+
+	if err != nil {
+		return nil, err
+	}
+
+	podClient.Client = controllerClient
+
+	// Fetch the current metadata before returning the PodClient
+	currentMetadata := &metav1.PartialObjectMetadata{}
+	currentMetadata.SetGroupVersionKind(corev1.SchemeGroupVersion.WithKind("Pod"))
+	err = podClient.Client.Get(ctx, client.ObjectKey{Namespace: namespace, Name: podName}, currentMetadata)
+	if err != nil {
+		return nil, err
+	}
+
+	podClient.metadata = currentMetadata
+
 	return podClient, nil
 }
 
 // retrieveEnvironmentVariables extracts the environment variables we have for
 // an argument into a map.
-func retrieveEnvironmentVariables(argument api.Argument, target map[string]string) {
+func retrieveEnvironmentVariables(monitor *Monitor, argument api.Argument, target map[string]string) {
 	if argument.Source != "" {
-		target[argument.Source] = os.Getenv(argument.Source)
+		value, err := argument.LookupEnv(monitor.CustomEnvironment)
+		if err == nil {
+			target[argument.Source] = value
+		}
 	}
 	if argument.Values != nil {
 		for _, childArgument := range argument.Values {
-			retrieveEnvironmentVariables(childArgument, target)
+			retrieveEnvironmentVariables(monitor, childArgument, target)
 		}
 	}
 }
 
 // UpdateAnnotations updates annotations on the pod after loading new
 // configuration.
-func (client *PodClient) UpdateAnnotations(monitor *Monitor) error {
+func (podClient *PodClient) UpdateAnnotations(monitor *Monitor) error {
 	environment := make(map[string]string)
 	for _, argument := range monitor.ActiveConfiguration.Arguments {
-		retrieveEnvironmentVariables(argument, environment)
+		retrieveEnvironmentVariables(monitor, argument, environment)
 	}
 	environment["BINARY_DIR"] = path.Dir(monitor.ActiveConfiguration.BinaryPath)
 	jsonEnvironment, err := json.Marshal(environment)
@@ -120,72 +191,91 @@ func (client *PodClient) UpdateAnnotations(monitor *Monitor) error {
 		return err
 	}
 
-	patch := map[string]interface{}{
-		"metadata": map[string]interface{}{
-			"annotations": map[string]string{
-				CurrentConfigurationAnnotation: string(monitor.ActiveConfigurationBytes),
-				EnvironmentAnnotation:          string(jsonEnvironment),
-			},
-		},
+	annotations := podClient.metadata.Annotations
+	if len(annotations) > 0 {
+		annotations = map[string]string{}
 	}
+	annotations[CurrentConfigurationAnnotation] = string(monitor.ActiveConfigurationBytes)
+	annotations[EnvironmentAnnotation] = string(jsonEnvironment)
 
-	patchJson, err := json.Marshal(patch)
-	if err != nil {
-		return err
+	return podClient.Patch(context.Background(), &corev1.Pod{
+		TypeMeta: metav1.TypeMeta{
+			Kind:       "Pod",
+			APIVersion: "v1",
+		},
+		ObjectMeta: metav1.ObjectMeta{
+			Namespace:   podClient.metadata.Namespace,
+			Name:        podClient.metadata.Name,
+			Annotations: annotations,
+		},
+	}, client.Apply, client.FieldOwner("fdb-kubernetes-monitor"), client.ForceOwnership)
+}
+
+// OnAdd is called when an object is added.
+func (podClient *PodClient) OnAdd(obj interface{}) {
+	pod, ok := obj.(*corev1.Pod)
+	if !ok {
+		return
 	}
 
-	pod, err := client.podApi.Patch(context.Background(), client.pod.Name, types.MergePatchType, patchJson, metav1.PatchOptions{})
-	if err != nil {
-		return err
+	podClient.Logger.Info("Got event for OnAdd", "name", pod.Name, "namespace", pod.Namespace)
+	podClient.metadata = &metav1.PartialObjectMetadata{
+		TypeMeta:   pod.TypeMeta,
+		ObjectMeta: pod.ObjectMeta,
 	}
-	client.pod = pod
-	return nil
 }
 
-// watchPod starts a watch on the pod.
-func (client *PodClient) watchPod() error {
-	podWatch, err := client.podApi.Watch(
-		context.Background(),
-		metav1.ListOptions{
-			Watch:           true,
-			ResourceVersion: "0",
-			FieldSelector:   fmt.Sprintf("metadata.name=%s", os.Getenv("FDB_POD_NAME")),
-		},
-	)
-	if err != nil {
-		return err
+// OnUpdate is called when an object is modified. Note that oldObj is the
+// last known state of the object-- it is possible that several changes
+// were combined together, so you can't use this to see every single
+// change. OnUpdate is also called when a re-list happens, and it will
+// get called even if nothing changed. This is useful for periodically
+// evaluating or syncing something.
+func (podClient *PodClient) OnUpdate(_, newObj interface{}) {
+	pod, ok := newObj.(*corev1.Pod)
+	if !ok {
+		return
 	}
-	results := podWatch.ResultChan()
-	go func() {
-		for event := range results {
-			if event.Type == watch.Modified {
-				pod, valid := event.Object.(*corev1.Pod)
-				if !valid {
-					client.Logger.Error(nil, "Error getting pod information from watch", "event", event)
-				}
-				client.processPodUpdate(pod)
-			}
-		}
-	}()
 
-	return nil
-}
+	podClient.Logger.Info("Got event for OnAdd", "name", pod.Name, "namespace", pod.Namespace, "generation", pod.Generation)
 
-// processPodUpdate handles an update for a pod.
-func (client *PodClient) processPodUpdate(pod *corev1.Pod) {
-	client.pod = pod
-	if pod.Annotations == nil {
+	podClient.metadata = &metav1.PartialObjectMetadata{
+		TypeMeta:   pod.TypeMeta,
+		ObjectMeta: pod.ObjectMeta,
+	}
+
+	if podClient.metadata == nil {
 		return
 	}
-	annotation := client.pod.Annotations[OutdatedConfigMapAnnotation]
+
+	annotation := podClient.metadata.Annotations[OutdatedConfigMapAnnotation]
 	if annotation == "" {
 		return
 	}
+
 	timestamp, err := strconv.ParseInt(annotation, 10, 64)
 	if err != nil {
-		client.Logger.Error(err, "Error parsing annotation", "key", OutdatedConfigMapAnnotation, "rawAnnotation", annotation)
+		podClient.Logger.Error(err, "Error parsing annotation", "key", OutdatedConfigMapAnnotation, "rawAnnotation", annotation)
 		return
 	}
 
-	client.TimestampFeed <- timestamp
+	podClient.TimestampFeed <- timestamp
+}
+
+// OnDelete will get the final state of the item if it is known, otherwise
+// it will get an object of type DeletedFinalStateUnknown. This can
+// happen if the watch is closed and misses the delete event and we don't
+// notice the deletion until the subsequent re-list.
+func (podClient *PodClient) OnDelete(obj interface{}) {
+	pod, ok := obj.(*corev1.Pod)
+	if !ok {
+		return
+	}
+
+	podClient.Logger.Info("Got event for OnDelete", "name", pod.Name, "namespace", pod.Namespace)
+
+	podClient.metadata = &metav1.PartialObjectMetadata{
+		TypeMeta:   pod.TypeMeta,
+		ObjectMeta: pod.ObjectMeta,
+	}
 }
diff --git a/fdbkubernetesmonitor/main.go b/fdbkubernetesmonitor/main.go
index d91493f3dd7..ddf70b203eb 100644
--- a/fdbkubernetesmonitor/main.go
+++ b/fdbkubernetesmonitor/main.go
@@ -21,9 +21,10 @@ package main
 
 import (
 	"bufio"
+	"context"
 	"fmt"
+	"io"
 	"os"
-	"path"
 	"regexp"
 	"strings"
 
@@ -31,6 +32,8 @@ import (
 	"github.com/go-logr/zapr"
 	"github.com/spf13/pflag"
 	"go.uber.org/zap"
+	"go.uber.org/zap/zapcore"
+	"gopkg.in/natefinch/lumberjack.v2"
 )
 
 var (
@@ -42,16 +45,18 @@ var (
 	logPath                 string
 	executionModeString     string
 	outputDir               string
-	copyFiles               []string
-	copyBinaries            []string
-	binaryOutputDirectory   string
-	copyLibraries           []string
-	copyPrimaryLibrary      string
-	requiredCopyFiles       []string
 	mainContainerVersion    string
 	currentContainerVersion string
 	additionalEnvFile       string
+	binaryOutputDirectory   string
+	listenAddress           string
+	copyPrimaryLibrary      string
+	requiredCopyFiles       []string
+	copyFiles               []string
+	copyBinaries            []string
+	copyLibraries           []string
 	processCount            int
+	enablePprof             bool
 )
 
 type executionMode string
@@ -62,7 +67,29 @@ const (
 	executionModeSidecar  executionMode = "sidecar"
 )
 
+func initLogger(logPath string) *zap.Logger {
+	var logWriter io.Writer
+
+	if logPath != "" {
+		lumberjackLogger := &lumberjack.Logger{
+			Filename:   logPath,
+			MaxSize:    100,
+			MaxAge:     7,
+			MaxBackups: 2,
+			Compress:   false,
+		}
+		logWriter = io.MultiWriter(os.Stdout, lumberjackLogger)
+	} else {
+		logWriter = os.Stdout
+	}
+
+	return zap.New(zapcore.NewCore(zapcore.NewJSONEncoder(zap.NewProductionEncoderConfig()), zapcore.AddSync(logWriter), zapcore.InfoLevel))
+}
+
 func main() {
+	var copyFiles, copyBinaries, copyLibraries, requiredCopyFiles []string
+	var inputDir, copyPrimaryLibrary, binaryOutputDirectory string
+
 	pflag.StringVar(&executionModeString, "mode", "launcher", "Execution mode. Valid options are launcher, sidecar, and init")
 	pflag.StringVar(&fdbserverPath, "fdbserver-path", "/usr/bin/fdbserver", "Path to the fdbserver binary")
 	pflag.StringVar(&inputDir, "input-dir", ".", "Directory containing input files")
@@ -80,25 +107,19 @@ func main() {
 	pflag.StringVar(&mainContainerVersion, "main-container-version", "", "For sidecar mode, this specifies the version of the main container. If this is equal to the current container version, no files will be copied")
 	pflag.StringVar(&additionalEnvFile, "additional-env-file", "", "A file with additional environment variables to use when interpreting the monitor configuration")
 	pflag.IntVar(&processCount, "process-count", 1, "The number of processes to start")
+	pflag.BoolVar(&enablePprof, "enable-pprof", false, "Enables /debug/pprof endpoints on the listen address")
+	pflag.StringVar(&listenAddress, "listen-address", ":8081", "An address and port to listen on")
 	pflag.Parse()
 
-	zapConfig := zap.NewProductionConfig()
-	if logPath != "" {
-		zapConfig.OutputPaths = append(zapConfig.OutputPaths, logPath)
-	}
-	zapLogger, err := zapConfig.Build()
-	if err != nil {
-		panic(err)
-	}
+	logger := zapr.NewLogger(initLogger(logPath))
 
 	versionBytes, err := os.ReadFile(versionFilePath)
 	if err != nil {
 		panic(err)
 	}
-	currentContainerVersion = strings.TrimSpace(string(versionBytes))
+	currentContainerVersion := strings.TrimSpace(string(versionBytes))
 
-	logger := zapr.NewLogger(zapLogger)
-	copyDetails, requiredCopies, err := getCopyDetails()
+	copyDetails, requiredCopies, err := getCopyDetails(inputDir, copyPrimaryLibrary, binaryOutputDirectory, copyFiles, copyBinaries, copyLibraries, requiredCopyFiles, currentContainerVersion)
 	if err != nil {
 		logger.Error(err, "Error getting list of files to copy")
 		os.Exit(1)
@@ -112,7 +133,7 @@ func main() {
 			logger.Error(err, "Error loading additional environment")
 			os.Exit(1)
 		}
-		StartMonitor(logger, fmt.Sprintf("%s/%s", inputDir, monitorConfFile), customEnvironment, processCount)
+		StartMonitor(context.Background(), logger, fmt.Sprintf("%s/%s", inputDir, monitorConfFile), customEnvironment, processCount, listenAddress, enablePprof, currentContainerVersion)
 	case executionModeInit:
 		err = CopyFiles(logger, outputDir, copyDetails, requiredCopies)
 		if err != nil {
@@ -136,46 +157,21 @@ func main() {
 	}
 }
 
-func getCopyDetails() (map[string]string, map[string]bool, error) {
-	copyDetails := make(map[string]string, len(copyFiles)+len(copyBinaries))
-
-	for _, filePath := range copyFiles {
-		copyDetails[path.Join(inputDir, filePath)] = ""
-	}
-	if copyBinaries != nil {
-		if binaryOutputDirectory == "" {
-			binaryOutputDirectory = currentContainerVersion
-		}
-		for _, copyBinary := range copyBinaries {
-			copyDetails[fmt.Sprintf("/usr/bin/%s", copyBinary)] = path.Join("bin", binaryOutputDirectory, copyBinary)
-		}
-	}
-	for _, library := range copyLibraries {
-		copyDetails[fmt.Sprintf("/usr/lib/fdb/multiversion/libfdb_c_%s.so", library)] = path.Join("lib", "multiversion", fmt.Sprintf("libfdb_c_%s.so", library))
-	}
-	if copyPrimaryLibrary != "" {
-		copyDetails[fmt.Sprintf("/usr/lib/fdb/multiversion/libfdb_c_%s.so", copyPrimaryLibrary)] = path.Join("lib", "libfdb_c.so")
-	}
-	requiredCopyMap := make(map[string]bool, len(requiredCopyFiles))
-	for _, filePath := range requiredCopyFiles {
-		fullFilePath := path.Join(inputDir, filePath)
-		_, present := copyDetails[fullFilePath]
-		if !present {
-			return nil, nil, fmt.Errorf("File %s is required, but is not in the --copy-file list", filePath)
-		}
-		requiredCopyMap[fullFilePath] = true
-	}
-	return copyDetails, requiredCopyMap, nil
-}
-
 func loadAdditionalEnvironment(logger logr.Logger) (map[string]string, error) {
 	var customEnvironment = make(map[string]string)
-	environmentPattern := regexp.MustCompile(`export ([A-Za-z0-9_]+)=([^\n]*)`)
 	if additionalEnvFile != "" {
+		environmentPattern := regexp.MustCompile(`export ([A-Za-z0-9_]+)=([^\n]*)`)
+
 		file, err := os.Open(additionalEnvFile)
 		if err != nil {
 			return nil, err
 		}
+		defer func() {
+			err := file.Close()
+			if err != nil {
+				logger.Error(err, "error when closing file")
+			}
+		}()
 
 		envScanner := bufio.NewScanner(file)
 		for envScanner.Scan() {
@@ -188,5 +184,6 @@ func loadAdditionalEnvironment(logger logr.Logger) (map[string]string, error) {
 			customEnvironment[matches[1]] = matches[2]
 		}
 	}
+
 	return customEnvironment, nil
 }
diff --git a/fdbkubernetesmonitor/monitor.go b/fdbkubernetesmonitor/monitor.go
index c23f20e8326..2ca667b824c 100644
--- a/fdbkubernetesmonitor/monitor.go
+++ b/fdbkubernetesmonitor/monitor.go
@@ -21,9 +21,12 @@ package main
 
 import (
 	"bufio"
+	"context"
 	"encoding/json"
 	"fmt"
 	"io"
+	"net/http"
+	"net/http/pprof"
 	"os"
 	"os/exec"
 	"os/signal"
@@ -32,6 +35,9 @@ import (
 	"syscall"
 	"time"
 
+	"github.com/prometheus/client_golang/prometheus/promhttp"
+	"k8s.io/utils/pointer"
+
 	"github.com/apple/foundationdb/fdbkubernetesmonitor/api"
 	"github.com/fsnotify/fsnotify"
 	"github.com/go-logr/logr"
@@ -48,6 +54,9 @@ type Monitor struct {
 	// ConfigFile defines the path to the config file to load.
 	ConfigFile string
 
+	// CurrentContainerVersion defines the version of the container. This will be the same as the fdbserver version.
+	CurrentContainerVersion string
+
 	// CustomEnvironment defines the custom environment variables to use when
 	// interpreting the monitor configuration.
 	CustomEnvironment map[string]string
@@ -86,21 +95,50 @@ type Monitor struct {
 }
 
 // StartMonitor starts the monitor loop.
-func StartMonitor(logger logr.Logger, configFile string, customEnvironment map[string]string, processCount int) {
-	podClient, err := CreatePodClient(logger)
+func StartMonitor(ctx context.Context, logger logr.Logger, configFile string, customEnvironment map[string]string, processCount int, listenAddr string, enableDebug bool, currentContainerVersion string) {
+	podClient, err := CreatePodClient(ctx, logger)
 	if err != nil {
-		panic(err)
+		logger.Error(err, "could not create Pod client")
+		os.Exit(1)
 	}
 
 	monitor := &Monitor{
-		ConfigFile:        configFile,
-		PodClient:         podClient,
-		Logger:            logger,
-		CustomEnvironment: customEnvironment,
-		ProcessCount:      processCount,
+		ConfigFile:              configFile,
+		PodClient:               podClient,
+		Logger:                  logger,
+		CustomEnvironment:       customEnvironment,
+		ProcessCount:            processCount,
+		CurrentContainerVersion: currentContainerVersion,
 	}
 
 	go func() { monitor.WatchPodTimestamps() }()
+
+	mux := http.NewServeMux()
+	// Enable pprof endpoints for debugging purposes.
+	if enableDebug {
+		mux.Handle("/debug/pprof/heap", pprof.Handler("heap"))
+		mux.Handle("/debug/pprof/goroutine", pprof.Handler("goroutine"))
+		mux.Handle("/debug/pprof/threadcreate", pprof.Handler("threadcreate"))
+		mux.Handle("/debug/pprof/allocs", pprof.Handler("allocs"))
+		mux.Handle("/debug/pprof/block", pprof.Handler("block"))
+		mux.Handle("/debug/pprof/mutex", pprof.Handler("mutex"))
+		mux.HandleFunc("/debug/pprof/", pprof.Index)
+		mux.HandleFunc("/debug/pprof/profile", pprof.Profile)
+		mux.HandleFunc("/debug/pprof/symbol", pprof.Symbol)
+		mux.HandleFunc("/debug/pprof/cmdline", pprof.Cmdline)
+		mux.HandleFunc("/debug/pprof/trace", pprof.Trace)
+	}
+
+	// Add Prometheus support
+	mux.Handle("/metrics", promhttp.Handler())
+	go func() {
+		err := http.ListenAndServe(listenAddr, mux)
+		if err != nil {
+			logger.Error(err, "could not start HTTP server")
+			os.Exit(1)
+		}
+	}()
+
 	monitor.Run()
 }
 
@@ -123,7 +161,7 @@ func (monitor *Monitor) LoadConfiguration() {
 		return
 	}
 
-	if currentContainerVersion == configuration.Version {
+	if monitor.CurrentContainerVersion == configuration.Version {
 		configuration.BinaryPath = fdbserverPath
 	} else {
 		configuration.BinaryPath = path.Join(sharedBinaryDir, configuration.Version, "fdbserver")
@@ -152,7 +190,7 @@ func checkOwnerExecutable(path string) error {
 		return err
 	}
 	if binaryStat.Mode()&0o100 == 0 {
-		return fmt.Errorf("Binary is not executable")
+		return fmt.Errorf("binary is not executable")
 	}
 	return nil
 }
@@ -289,12 +327,13 @@ func (monitor *Monitor) checkProcessRequired(processNumber int) bool {
 	monitor.Mutex.Lock()
 	defer monitor.Mutex.Unlock()
 	logger := monitor.Logger.WithValues("processNumber", processNumber, "area", "checkProcessRequired")
-	runProcesses := monitor.ActiveConfiguration.RunServers
-	if monitor.ProcessCount < processNumber || (runProcesses != nil && !*runProcesses) {
+	runProcesses := pointer.BoolDeref(monitor.ActiveConfiguration.RunServers, true)
+	if monitor.ProcessCount < processNumber || !runProcesses {
 		logger.Info("Terminating run loop")
 		monitor.ProcessIDs[processNumber] = 0
 		return false
 	}
+
 	return true
 }
 
@@ -341,6 +380,9 @@ func (monitor *Monitor) Run() {
 	go func() {
 		latestSignal := <-signals
 		monitor.Logger.Info("Received system signal", "signal", latestSignal)
+
+		// Reset the ProcessCount to 0 to make sure the monitor doesn't try to restart the processes.
+		monitor.ProcessCount = 0
 		for processNumber, processID := range monitor.ProcessIDs {
 			if processID > 0 {
 				subprocessLogger := monitor.Logger.WithValues("processNumber", processNumber, "PID", processID)
@@ -357,6 +399,18 @@ func (monitor *Monitor) Run() {
 				}
 			}
 		}
+
+		annotations := monitor.PodClient.metadata.Annotations
+		if len(annotations) > 0 {
+			delayValue, ok := annotations[DelayShutdownAnnotation]
+			if ok {
+				delay, err := time.ParseDuration(delayValue)
+				if err == nil {
+					time.Sleep(delay)
+				}
+			}
+		}
+
 		done <- true
 	}()
 
@@ -370,12 +424,18 @@ func (monitor *Monitor) Run() {
 		panic(err)
 	}
 
-	defer watcher.Close()
+	defer func(watcher *fsnotify.Watcher) {
+		err := watcher.Close()
+		if err != nil {
+			monitor.Logger.Error(err, "could not close watcher")
+		}
+	}(watcher)
 	go func() { monitor.WatchConfiguration(watcher) }()
 
 	<-done
 }
 
+// WatchPodTimestamps watches the timestamp feed to reload the configuration.
 func (monitor *Monitor) WatchPodTimestamps() {
 	for timestamp := range monitor.PodClient.TimestampFeed {
 		if timestamp > monitor.LastConfigurationTime.Unix() {
diff --git a/fdbkubernetesmonitor/suite_test.go b/fdbkubernetesmonitor/suite_test.go
new file mode 100644
index 00000000000..3883bc7a8f5
--- /dev/null
+++ b/fdbkubernetesmonitor/suite_test.go
@@ -0,0 +1,37 @@
+/*
+ * suite_test.go
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2023 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main
+
+import (
+	"testing"
+	"time"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+// These tests use Ginkgo (BDD-style Go testing framework). Refer to
+// http://onsi.github.io/ginkgo/ to learn more about Ginkgo.
+func TestAPIs(t *testing.T) {
+	RegisterFailHandler(Fail)
+	SetDefaultEventuallyTimeout(10 * time.Second)
+	RunSpecs(t, "FDB Kubernetes Monitor")
+}
diff --git a/packaging/docker/Dockerfile b/packaging/docker/Dockerfile
index 75dbda3e9da..ae9b9336719 100644
--- a/packaging/docker/Dockerfile
+++ b/packaging/docker/Dockerfile
@@ -59,7 +59,7 @@ RUN curl -Ls https://github.com/krallin/tini/releases/download/v0.19.0/tini-amd6
 
 WORKDIR /
 
-FROM golang:1.16.7-bullseye AS go-build
+FROM golang:1.22.2-bullseye AS go-build
 
 COPY fdbkubernetesmonitor/ /fdbkubernetesmonitor
 WORKDIR /fdbkubernetesmonitor
@@ -158,8 +158,8 @@ RUN curl -LsO https://raw.githubusercontent.com/brendangregg/FlameGraph/90533539
     rm -rf /tmp/*
 WORKDIR /
 # Set Up Runtime Scripts and Directories
-ADD fdb.bash /var/fdb/scripts/
-RUN chmod a+x /var/fdb/scripts/fdb.bash
+ADD fdb.bash fdb_single.bash /var/fdb/scripts/
+RUN chmod a+x /var/fdb/scripts/fdb.bash /var/fdb/scripts/fdb_single.bash
 VOLUME /var/fdb/data
 ENV FDB_PORT 4500
 ENV FDB_CLUSTER_FILE /var/fdb/fdb.cluster
diff --git a/packaging/docker/Dockerfile.eks b/packaging/docker/Dockerfile.eks
index f8dec2421a2..75962136530 100644
--- a/packaging/docker/Dockerfile.eks
+++ b/packaging/docker/Dockerfile.eks
@@ -67,7 +67,7 @@ RUN NO_PROXY="" no_proxy="" curl -Ls https://s3.us-west-2.amazonaws.com/amazon-e
 
 WORKDIR /
 
-FROM golang:1.16.7-bullseye AS go-build
+FROM golang:1.22.2-bullseye AS go-build
 
 COPY fdbkubernetesmonitor/ /fdbkubernetesmonitor
 WORKDIR /fdbkubernetesmonitor
@@ -166,8 +166,8 @@ RUN curl -LsO https://raw.githubusercontent.com/brendangregg/FlameGraph/90533539
     rm -rf /tmp/*
 WORKDIR /
 # Set Up Runtime Scripts and Directories
-ADD fdb.bash /var/fdb/scripts/
-RUN chmod a+x /var/fdb/scripts/fdb.bash
+ADD fdb.bash fdb_single.bash /var/fdb/scripts/
+RUN chmod a+x /var/fdb/scripts/fdb.bash /var/fdb/scripts/fdb_single.bash
 VOLUME /var/fdb/data
 ENV FDB_PORT 4500
 ENV FDB_CLUSTER_FILE /var/fdb/fdb.cluster
diff --git a/packaging/docker/build-images.sh b/packaging/docker/build-images.sh
index 206c152baae..63669a4616d 100755
--- a/packaging/docker/build-images.sh
+++ b/packaging/docker/build-images.sh
@@ -247,14 +247,14 @@ build_output_directory="${script_dir}/../../"
 source_code_diretory=$(awk -F= '/foundationdb_SOURCE_DIR:STATIC/{print $2}' "${build_output_directory}/CMakeCache.txt")
 commit_sha=$(cd "${source_code_diretory}" && git rev-parse --verify HEAD --short=10)
 fdb_version=$(cat "${build_output_directory}/version.txt")
-fdb_library_versions=( '5.1.7' '6.1.13' '6.2.30' '6.3.18' "${fdb_version}" )
+fdb_library_versions=( '6.3.25' "${fdb_version}" )
 fdb_website="https://github.com/apple/foundationdb/releases/download"
 image_list=(
     'base'
-    # 'go-build'
+    'go-build'
     'foundationdb-base'
     'foundationdb'
-    # 'foundationdb-kubernetes-monitor'
+    'fdb-kubernetes-monitor'
     'foundationdb-kubernetes-sidecar'
     'ycsb'
 )
diff --git a/packaging/docker/fdb_single.bash b/packaging/docker/fdb_single.bash
new file mode 100644
index 00000000000..a44bc5716db
--- /dev/null
+++ b/packaging/docker/fdb_single.bash
@@ -0,0 +1,90 @@
+#!/usr/bin/env bash
+set -Eeuo pipefail
+set -m
+#
+# fdb_single.bash
+#
+# This source file is part of the FoundationDB open source project
+#
+# Copyright 2013-2023 Apple Inc. and the FoundationDB project authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+function create_cluster_file() {
+    FDB_CLUSTER_FILE=${FDB_CLUSTER_FILE:-/etc/foundationdb/fdb.cluster}
+    mkdir -p "$(dirname "$FDB_CLUSTER_FILE")"
+
+    if [[ -n "$FDB_CLUSTER_FILE_CONTENTS" ]]; then
+        echo "$FDB_CLUSTER_FILE_CONTENTS" > "$FDB_CLUSTER_FILE"
+    elif [[ -n $FDB_COORDINATOR ]]; then
+        coordinator_ip=$(dig +short "$FDB_COORDINATOR")
+        if [[ -z "$coordinator_ip" ]]; then
+            echo "Failed to look up coordinator address for $FDB_COORDINATOR" 1>&2
+            exit 1
+        fi
+        coordinator_port=${FDB_COORDINATOR_PORT:-4500}
+        echo "docker:docker@$coordinator_ip:$coordinator_port" > "$FDB_CLUSTER_FILE"
+    else
+        echo "FDB_COORDINATOR environment variable not defined" 1>&2
+        exit 1
+    fi
+}
+
+function create_server_environment() {
+    env_file=/var/fdb/.fdbenv
+
+    if [[ "$FDB_NETWORKING_MODE" == "host" ]]; then
+        public_ip=127.0.0.1
+    elif [[ "$FDB_NETWORKING_MODE" == "container" ]]; then
+        public_ip=$(hostname -i | awk '{print $1}')
+    else
+        echo "Unknown FDB Networking mode \"$FDB_NETWORKING_MODE\"" 1>&2
+        exit 1
+    fi
+
+    echo "export PUBLIC_IP=$public_ip" > $env_file
+    if [[ -z $FDB_COORDINATOR && -z "$FDB_CLUSTER_FILE_CONTENTS" ]]; then
+        FDB_CLUSTER_FILE_CONTENTS="docker:docker@$public_ip:$FDB_PORT"
+    fi
+
+    create_cluster_file
+}
+
+function start_fdb () {
+    create_server_environment
+    source /var/fdb/.fdbenv
+    echo "Starting FDB server on $PUBLIC_IP:$FDB_PORT"
+    fdbserver --listen-address 0.0.0.0:"$FDB_PORT" \
+              --public-address "$PUBLIC_IP:$FDB_PORT" \
+              --datadir /var/fdb/data \
+              --logdir /var/fdb/logs \
+              --locality-zoneid="$(hostname)" \
+              --locality-machineid="$(hostname)" \
+              --knob_disable_posix_kernel_aio=1 \
+              --class "$FDB_PROCESS_CLASS" &
+    fdb_pid=$(jobs -p)
+    echo "fdbserver pid is: ${fdb_pid}"
+}
+
+function configure_fdb_single () {
+    echo "Configuring new single memory FDB database"
+    fdbcli --exec 'configure new single memory'
+    sleep 3
+    fdbcli --exec 'status'
+}
+
+start_fdb
+sleep 5
+configure_fdb_single
+fg %1

From 684422ed142d0313d8d5a1804f3e7009c774c095 Mon Sep 17 00:00:00 2001
From: Dan Lambright <dlambrig@gmail.com>
Date: Fri, 19 Apr 2024 14:53:39 -0400
Subject: [PATCH 417/468] Cherry pick #11309 Raise visibility of gray failure
 actions (#11314)

* Cherry pick #11308 Raise visibility of gray failure actions

* format change

---------

Co-authored-by: Dan Lambright <hlambright@apple.com>
---
 fdbserver/ClusterController.actor.cpp | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/fdbserver/ClusterController.actor.cpp b/fdbserver/ClusterController.actor.cpp
index a0040addcf0..e2ff221267b 100644
--- a/fdbserver/ClusterController.actor.cpp
+++ b/fdbserver/ClusterController.actor.cpp
@@ -2613,19 +2613,19 @@ ACTOR Future<Void> workerHealthMonitor(ClusterControllerData* self) {
 							self->excludedDegradedServers = self->degradationInfo.degradedServers;
 							self->excludedDegradedServers.insert(self->degradationInfo.disconnectedServers.begin(),
 							                                     self->degradationInfo.disconnectedServers.end());
-							TraceEvent("DegradedServerDetectedAndTriggerRecovery")
+							TraceEvent(SevWarnAlways, "DegradedServerDetectedAndTriggerRecovery")
 							    .detail("RecentRecoveryCountDueToHealth", self->recentRecoveryCountDueToHealth());
 							self->db.forceMasterFailure.trigger();
 						}
 					} else {
 						self->excludedDegradedServers.clear();
-						TraceEvent("DegradedServerDetectedAndSuggestRecovery").log();
+						TraceEvent(SevWarnAlways, "DegradedServerDetectedAndSuggestRecovery").log();
 					}
 				} else if (self->shouldTriggerFailoverDueToDegradedServers()) {
 					double ccUpTime = now() - machineStartTime();
 					if (SERVER_KNOBS->CC_HEALTH_TRIGGER_FAILOVER &&
 					    ccUpTime > SERVER_KNOBS->INITIAL_UPDATE_CROSS_DC_INFO_DELAY) {
-						TraceEvent("DegradedServerDetectedAndTriggerFailover").log();
+						TraceEvent(SevWarnAlways, "DegradedServerDetectedAndTriggerFailover").log();
 						std::vector<Optional<Key>> dcPriority;
 						auto remoteDcId = self->db.config.regions[0].dcId == self->clusterControllerDcId.get()
 						                      ? self->db.config.regions[1].dcId
@@ -2637,7 +2637,8 @@ ACTOR Future<Void> workerHealthMonitor(ClusterControllerData* self) {
 						dcPriority.push_back(self->clusterControllerDcId);
 						self->desiredDcIds.set(dcPriority);
 					} else {
-						TraceEvent("DegradedServerDetectedAndSuggestFailover").detail("CCUpTime", ccUpTime);
+						TraceEvent(SevWarnAlways, "DegradedServerDetectedAndSuggestFailover")
+						    .detail("CCUpTime", ccUpTime);
 					}
 				}
 			}

From 98a18d5c0a1c7420fa876b52773ec698a56fbf1d Mon Sep 17 00:00:00 2001
From: Aaron Molitor <amolitor@apple.com>
Date: Thu, 25 Apr 2024 13:50:52 -0500
Subject: [PATCH 418/468] update go bindings build to play nice with golang
 1.22

---
 bindings/go/CMakeLists.txt | 2 --
 1 file changed, 2 deletions(-)

diff --git a/bindings/go/CMakeLists.txt b/bindings/go/CMakeLists.txt
index 7d3d8ea6cff..24798a1dab6 100644
--- a/bindings/go/CMakeLists.txt
+++ b/bindings/go/CMakeLists.txt
@@ -102,8 +102,6 @@ function(build_go_package)
   endif()
   add_custom_command(OUTPUT ${outfile}
     COMMAND ${CMAKE_COMMAND} -E env ${go_env}
-            ${GO_EXECUTABLE} get -d ${GO_IMPORT_PATH}/${BGP_PATH} &&
-            ${CMAKE_COMMAND} -E env ${go_env}
             ${GO_EXECUTABLE} install ${GO_IMPORT_PATH}/${BGP_PATH}
     DEPENDS ${fdb_options_file}
     COMMENT "Building ${BGP_NAME}")

From 30948b4b958e281ff17c05bc40f92882eb190c5c Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyu_zhou@apple.com>
Date: Fri, 26 Apr 2024 11:49:07 -0700
Subject: [PATCH 419/468] Increase CommitProxyTerminated severity for
 failed_to_progress errors. (#11316)

For better visibility.
---
 fdbserver/CommitProxyServer.actor.cpp | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/fdbserver/CommitProxyServer.actor.cpp b/fdbserver/CommitProxyServer.actor.cpp
index a29a29bd38e..8723eab2e02 100644
--- a/fdbserver/CommitProxyServer.actor.cpp
+++ b/fdbserver/CommitProxyServer.actor.cpp
@@ -2493,7 +2493,8 @@ ACTOR Future<Void> commitProxyServer(CommitProxyInterface proxy,
 		                                                whitelistBinPaths);
 		wait(core || checkRemoved(db, req.recoveryCount, proxy));
 	} catch (Error& e) {
-		TraceEvent("CommitProxyTerminated", proxy.id()).errorUnsuppressed(e);
+		Severity sev = e.code() == error_code_failed_to_progress ? SevWarnAlways : SevInfo;
+		TraceEvent(sev, "CommitProxyTerminated", proxy.id()).errorUnsuppressed(e);
 
 		if (e.code() != error_code_worker_removed && e.code() != error_code_tlog_stopped &&
 		    e.code() != error_code_tlog_failed && e.code() != error_code_coordinators_changed &&

From 6cfd21fd30a69e5c7404707fffb57156bb433398 Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyu_zhou@apple.com>
Date: Thu, 2 May 2024 11:55:03 -0700
Subject: [PATCH 420/468] Throw errors in getConsistentReadVersion (#11352)

In the current code, errors are retried in getConsistentReadVersion, so it's
possible that the client has cancelled the GRV request, but readVersionBatcher
continue retrying, which can lead to many clients DDoS GRV proxies, especially
when the database has become unavailable for a while and clients are issuing
many GRV requests.
---
 fdbclient/NativeAPI.actor.cpp | 8 +-------
 1 file changed, 1 insertion(+), 7 deletions(-)

diff --git a/fdbclient/NativeAPI.actor.cpp b/fdbclient/NativeAPI.actor.cpp
index bdd46fc03fa..73e34996f5b 100644
--- a/fdbclient/NativeAPI.actor.cpp
+++ b/fdbclient/NativeAPI.actor.cpp
@@ -6761,13 +6761,7 @@ ACTOR Future<GetReadVersionReply> getConsistentReadVersion(SpanID parentSpan,
 			if (e.code() != error_code_broken_promise && e.code() != error_code_batch_transaction_throttled &&
 			    e.code() != error_code_proxy_memory_limit_exceeded)
 				TraceEvent(SevError, "GetConsistentReadVersionError").error(e);
-			if ((e.code() == error_code_batch_transaction_throttled ||
-			     e.code() == error_code_proxy_memory_limit_exceeded) &&
-			    !cx->apiVersionAtLeast(630)) {
-				wait(delayJittered(5.0));
-			} else {
-				throw;
-			}
+			throw;
 		}
 	}
 }

From 5321186750fed4f478cfd2ae0dc22334ffa4a3bf Mon Sep 17 00:00:00 2001
From: Zhe Wang <zhe.wang@wustl.edu>
Date: Fri, 3 May 2024 12:39:14 -0700
Subject: [PATCH 421/468] cherry-pick-improve-dcc (#11351)

---
 fdbclient/ClientKnobs.cpp  |  1 +
 fdbclient/ClientKnobs.h    |  1 +
 fdbserver/tester.actor.cpp | 44 ++++++++++++++++++++++++++++++--------
 3 files changed, 37 insertions(+), 9 deletions(-)

diff --git a/fdbclient/ClientKnobs.cpp b/fdbclient/ClientKnobs.cpp
index 67e4df59800..e2036582094 100644
--- a/fdbclient/ClientKnobs.cpp
+++ b/fdbclient/ClientKnobs.cpp
@@ -258,6 +258,7 @@ void ClientKnobs::initialize(Randomize randomize) {
 
 	init( CONSISTENCY_CHECK_RATE_LIMIT_MAX,        50e6 ); // Limit in per sec
 	init( CONSISTENCY_CHECK_ONE_ROUND_TARGET_COMPLETION_TIME,	7 * 24 * 60 * 60 ); // 7 days
+	init( CONSISTENCY_CHECK_URGENT_NEXT_WAIT_TIME,  600 );
 	init( CONSISTENCY_CHECK_BATCH_SHARD_COUNT,       10 ); if( randomize && BUGGIFY ) CONSISTENCY_CHECK_BATCH_SHARD_COUNT = 2;
 	init( CONSISTENCY_CHECK_RETRY_DEPTH_MAX,         10 ); if( randomize && BUGGIFY ) CONSISTENCY_CHECK_RETRY_DEPTH_MAX = 1;
 	init( CONSISTENCY_CHECK_INIT_CLEAR_METADATA,  false );
diff --git a/fdbclient/ClientKnobs.h b/fdbclient/ClientKnobs.h
index 2b90eb35173..cf77077b8fb 100644
--- a/fdbclient/ClientKnobs.h
+++ b/fdbclient/ClientKnobs.h
@@ -249,6 +249,7 @@ class ClientKnobs : public KnobsImpl<ClientKnobs> {
 
 	int CONSISTENCY_CHECK_RATE_LIMIT_MAX; // Not available in urgent mode
 	int CONSISTENCY_CHECK_ONE_ROUND_TARGET_COMPLETION_TIME; // Not available in urgent mode
+	int CONSISTENCY_CHECK_URGENT_NEXT_WAIT_TIME; // Available in urgent mode
 	int CONSISTENCY_CHECK_BATCH_SHARD_COUNT; // Available in urgent mode
 	int CONSISTENCY_CHECK_RETRY_DEPTH_MAX; // Available in urgent mode
 	bool CONSISTENCY_CHECK_INIT_CLEAR_METADATA; // Available in urgent mode
diff --git a/fdbserver/tester.actor.cpp b/fdbserver/tester.actor.cpp
index 178c6c7f77e..52fd3c4db76 100644
--- a/fdbserver/tester.actor.cpp
+++ b/fdbserver/tester.actor.cpp
@@ -1262,7 +1262,7 @@ ACTOR Future<std::vector<TesterInterface>> getTesters(Reference<AsyncVar<Optiona
 			}
 			when(wait(cc->onChange())) {}
 			when(wait(testerTimeout)) {
-				TraceEvent(SevError, "TesterRecruitmentTimeout").log();
+				TraceEvent(SevWarnAlways, "TesterRecruitmentTimeout").log();
 				throw timed_out();
 			}
 		}
@@ -1419,6 +1419,7 @@ ACTOR Future<Void> runConsistencyCheckerUrgentCore(Reference<AsyncVar<Optional<C
 	    SERVER_KNOBS->CONSISTENCY_CHECK_ID_MIN, SERVER_KNOBS->CONSISTENCY_CHECK_ID_MAX_PLUS_ONE);
 	state std::vector<KeyRange> rangesToCheck; // get from progress metadata
 	state std::vector<KeyRange> shardsToCheck; // get from keyServer metadata
+	state Optional<double> whenFailedToGetTesterStart;
 	state KeyRangeMap<bool> globalProgressMap; // used to keep track of progress when persisting metadata is not allowed
 	state std::unordered_map<int, std::vector<KeyRange>> assignment; // used to keep track of assignment of tasks
 	state std::vector<TesterInterface> ts; // used to store testers interface
@@ -1509,7 +1510,19 @@ ACTOR Future<Void> runConsistencyCheckerUrgentCore(Reference<AsyncVar<Optional<C
 			// Step 2: Get testers
 			ts.clear();
 			if (!testers.present()) {
-				wait(store(ts, getTesters(cc, minTestersExpected)));
+				try {
+					wait(store(ts, getTesters(cc, minTestersExpected)));
+					whenFailedToGetTesterStart.reset();
+				} catch (Error& e) {
+					if (e.code() == error_code_timed_out) {
+						if (!whenFailedToGetTesterStart.present()) {
+							whenFailedToGetTesterStart = now();
+						} else if (now() - whenFailedToGetTesterStart.get() > 3600 * 24) { // 1 day
+							TraceEvent(SevError, "TesterRecruitmentTimeout").log();
+						}
+					}
+					throw e;
+				}
 				if (g_network->isSimulated() && deterministicRandom()->random01() < 0.05) {
 					throw operation_failed(); // Introduce random failure
 				}
@@ -2389,6 +2402,19 @@ ACTOR Future<Void> runTests(Reference<AsyncVar<Optional<struct ClusterController
 	return Void();
 }
 
+ACTOR Future<Void> runConsistencyCheckerUrgentHolder(Reference<AsyncVar<Optional<ClusterControllerFullInterface>>> cc,
+                                                     Database cx,
+                                                     Optional<std::vector<TesterInterface>> testers,
+                                                     int minTestersExpected,
+                                                     TestSpec testSpec,
+                                                     Optional<TenantName> defaultTenant,
+                                                     Reference<AsyncVar<ServerDBInfo>> dbInf) {
+	loop {
+		wait(runConsistencyCheckerUrgentCore(cc, cx, testers, minTestersExpected, testSpec, defaultTenant, dbInf));
+		wait(delay(CLIENT_KNOBS->CONSISTENCY_CHECK_URGENT_NEXT_WAIT_TIME));
+	}
+}
+
 /**
  * \brief Set up testing environment and run the given tests on a cluster.
  *
@@ -2534,13 +2560,13 @@ ACTOR Future<Void> runTests(Reference<IClusterConnectionRecord> connRecord,
 		state Future<Void> ccMonitor = monitorServerDBInfo(cc, LocalityData(), dbInfo); // FIXME: locality
 		cx = openDBOnServer(dbInfo);
 		cx->defaultTenant = defaultTenant;
-		tests = reportErrors(runConsistencyCheckerUrgentCore(cc,
-		                                                     cx,
-		                                                     Optional<std::vector<TesterInterface>>(),
-		                                                     minTestersExpected,
-		                                                     testSet.testSpecs[0],
-		                                                     defaultTenant,
-		                                                     dbInfo),
+		tests = reportErrors(runConsistencyCheckerUrgentHolder(cc,
+		                                                       cx,
+		                                                       Optional<std::vector<TesterInterface>>(),
+		                                                       minTestersExpected,
+		                                                       testSet.testSpecs[0],
+		                                                       defaultTenant,
+		                                                       dbInfo),
 		                     "runConsistencyCheckerUrgentCore");
 	} else if (at == TEST_HERE) {
 		auto db = makeReference<AsyncVar<ServerDBInfo>>();

From 4ef0d02dee4622a1ef74557346e21e4b15a0e33b Mon Sep 17 00:00:00 2001
From: Dan Lambright <dlambrig@gmail.com>
Date: Wed, 8 May 2024 13:01:01 -0400
Subject: [PATCH 422/468] Add logging to LogRouter's waitForVersion function
 (#11359) (#11370)

* Add logging to waitForVersion

* Respond to review comments.

---------

Co-authored-by: Dan Lambright <hlambright@apple.com>
---
 fdbserver/LogRouter.actor.cpp | 25 +++++++++++++++++++++++--
 1 file changed, 23 insertions(+), 2 deletions(-)

diff --git a/fdbserver/LogRouter.actor.cpp b/fdbserver/LogRouter.actor.cpp
index e0a8e3c36f4..5f71baadca6 100644
--- a/fdbserver/LogRouter.actor.cpp
+++ b/fdbserver/LogRouter.actor.cpp
@@ -264,6 +264,7 @@ ACTOR Future<Void> waitForVersion(LogRouterData* self, Version ver) {
 	// The only time the log router should allow a gap in versions larger than MAX_READ_TRANSACTION_LIFE_VERSIONS is
 	// when processing epoch end. Since one set of log routers is created per generation of transaction logs, the gap
 	// caused by epoch end will be within MAX_VERSIONS_IN_FLIGHT of the log routers start version.
+
 	state double startTime = now();
 	if (self->version.get() < self->startVersion) {
 		// Log router needs to wait for remote tLogs to process data, whose version is less than self->startVersion,
@@ -305,6 +306,26 @@ ACTOR Future<Void> waitForVersion(LogRouterData* self, Version ver) {
 	return Void();
 }
 
+ACTOR Future<Void> waitForVersionAndLog(LogRouterData* self, Version ver) {
+	state Future<Void> f = waitForVersion(self, ver);
+	state double emitInterval = 60.0;
+	loop {
+		choose {
+			when(wait(f)) {
+				return Void();
+			}
+			when(wait(delay(emitInterval))) {
+				TraceEvent("LogRouterWaitForVersionLongDelay", self->dbgid)
+				    .detail("WaitForVersion", ver)
+				    .detail("StartVersion", self->startVersion)
+				    .detail("Version", self->version.get())
+				    .detail("MinPopped", self->minPopped.get())
+				    .detail("FoundEpochEnd", self->foundEpochEnd);
+			}
+		}
+	}
+}
+
 ACTOR Future<Reference<ILogSystem::IPeekCursor>> getPeekCursorData(LogRouterData* self,
                                                                    Reference<ILogSystem::IPeekCursor> r,
                                                                    Version startVersion) {
@@ -378,7 +399,7 @@ ACTOR Future<Void> pullAsyncData(LogRouterData* self) {
 			if (!foundMessage || r->version().version != ver) {
 				ASSERT(r->version().version > lastVer);
 				if (ver) {
-					wait(waitForVersion(self, ver));
+					wait(waitForVersionAndLog(self, ver));
 
 					commitMessages(self, ver, messages);
 					self->version.set(ver);
@@ -393,7 +414,7 @@ ACTOR Future<Void> pullAsyncData(LogRouterData* self) {
 				if (!foundMessage) {
 					ver--; // ver is the next possible version we will get data for
 					if (ver > self->version.get() && ver >= r->popped()) {
-						wait(waitForVersion(self, ver));
+						wait(waitForVersionAndLog(self, ver));
 
 						self->version.set(ver);
 						wait(yield(TaskPriority::TLogCommit));

From 725a08a3ffd1d3be29428c90eb31bfcd8c7015e2 Mon Sep 17 00:00:00 2001
From: Xiaoge Su <magichp@gmail.com>
Date: Fri, 10 May 2024 11:02:14 -0700
Subject: [PATCH 423/468] [Release 7.1] Disable TestOpenNotExistTenant test
 (#11380)

* Disable fdb_test.go:TestOpenNotExistTenant

This is not supported yet.

* fixup! Fix the comments
---
 bindings/go/src/fdb/fdb_test.go | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/bindings/go/src/fdb/fdb_test.go b/bindings/go/src/fdb/fdb_test.go
index 3e98c4c9e3d..1594cc9e95c 100644
--- a/bindings/go/src/fdb/fdb_test.go
+++ b/bindings/go/src/fdb/fdb_test.go
@@ -353,6 +353,7 @@ func TestCreateExistTenant(t *testing.T) {
 	assertErrorCodeEqual(t, err, errTenantExists)
 }
 
+/* FIXME: This test will not pass as FDB does not properly return error code TenantNotFound
 func TestOpenNotExistTenant(t *testing.T) {
 	fdb.MustAPIVersion(710)
 	db := fdb.MustOpenDefault()
@@ -363,6 +364,7 @@ func TestOpenNotExistTenant(t *testing.T) {
 	_, err := db.OpenTenant(testTenantName)
 	assertErrorCodeEqual(t, err, errTenantNotFound)
 }
+*/
 
 func TestDeleteNotExistTenant(t *testing.T) {
 	fdb.MustAPIVersion(710)

From 923ca8e678e3a52fe600ee968f2cafbbd0378729 Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Tue, 14 May 2024 12:26:59 -0700
Subject: [PATCH 424/468] Fix an assertion failure when waiting for recovery

CC's checkBetterSingletons() calls getUsedIds() that asserts proxy interfaces
are present. However, when a GRV/commit proxy failed, before CC starts a new
recovery, the proxy's processId becomes empty, thus triggering the failure.

The fix is to cancel the caller while waiting for recovery.

To reproduce 7.1 commit 725a08a3ff clang build:

./fdbserver.6.0.15 -r simulation -f ./tests/restarting/from_5.2.0_until_6.3.0/ClientTransactionProfilingCorrectness-1.txt -s 900000399 -b on
-f ./tests/restarting/from_5.2.0_until_6.3.0/ClientTransactionProfilingCorrectness-2.txt --restarting -s 900000400 -b on
---
 fdbserver/ClusterController.actor.cpp | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/fdbserver/ClusterController.actor.cpp b/fdbserver/ClusterController.actor.cpp
index e2ff221267b..cbb8347b94e 100644
--- a/fdbserver/ClusterController.actor.cpp
+++ b/fdbserver/ClusterController.actor.cpp
@@ -421,6 +421,13 @@ ACTOR Future<Void> clusterWatchDatabase(ClusterControllerData* cluster,
 				wait(delay(0.0));
 
 			recoveryCore.cancel();
+			if (cluster->outstandingRemoteRequestChecker.isValid()) {
+				cluster->outstandingRemoteRequestChecker.cancel();
+			}
+
+			if (cluster->outstandingRequestChecker.isValid()) {
+				cluster->outstandingRequestChecker.cancel();
+			}
 			wait(cleanupRecoveryActorCollection(db->recoveryData, true /* exThrown */));
 			ASSERT(addActor.isEmpty());
 

From ae0da6a850b84d8d68c841e4c7af5bc3f9a3d98e Mon Sep 17 00:00:00 2001
From: "Johannes M. Scheuermann" <jscheuermann@apple.com>
Date: Tue, 14 May 2024 13:48:32 +0200
Subject: [PATCH 425/468] Add default timeout for fdbbackup

---
 fdbbackup/backup.actor.cpp | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/fdbbackup/backup.actor.cpp b/fdbbackup/backup.actor.cpp
index a8dd6a8f4e4..4a601e76da6 100644
--- a/fdbbackup/backup.actor.cpp
+++ b/fdbbackup/backup.actor.cpp
@@ -4098,6 +4098,12 @@ int main(int argc, char* argv[]) {
 			Optional<Database> result = connectToCluster(clusterFile, localities, quiet);
 			if (result.present()) {
 				db = result.get();
+				// Make sure we are setting a transaction timeout and retry limit to prevent cases
+				// where the fdbbackup command hangs infinitely. 60 seconds should be more than
+				// enough for all cases to finish and 5 retries should also be good enough for
+				// most cases.
+				db->setOption(FDBDatabaseOptions::TRANSACTION_TIMEOUT, Optional<StringRef>(StringRef((const uint8_t*)60000, 8)));
+				db->setOption(FDBDatabaseOptions::TRANSACTION_RETRY_LIMIT, Optional<StringRef>(StringRef((const uint8_t*)5, 8)));
 			}
 
 			return result.present();
@@ -4114,6 +4120,12 @@ int main(int argc, char* argv[]) {
 			Optional<Database> result = connectToCluster(sourceClusterFile, localities, quiet);
 			if (result.present()) {
 				sourceDb = result.get();
+				// Make sure we are setting a transaction timeout and retry limit to prevent cases
+				// where the fdbbackup command hangs infinitely. 60 seconds should be more than
+				// enough for all cases to finish and 5 retries should also be good enough for
+				// most cases.
+				sourceDb->setOption(FDBDatabaseOptions::TRANSACTION_TIMEOUT, Optional<StringRef>(StringRef((const uint8_t*)60000, 8)));
+				sourceDb->setOption(FDBDatabaseOptions::TRANSACTION_RETRY_LIMIT, Optional<StringRef>(StringRef((const uint8_t*)5, 8)));
 			}
 
 			return result.present();

From 797ade27a49a4fc476f3bcd27ead89704dcfc544 Mon Sep 17 00:00:00 2001
From: "Johannes M. Scheuermann" <jscheuermann@apple.com>
Date: Tue, 14 May 2024 14:57:41 +0200
Subject: [PATCH 426/468] Correct format

---
 fdbbackup/backup.actor.cpp | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/fdbbackup/backup.actor.cpp b/fdbbackup/backup.actor.cpp
index 4a601e76da6..d228752f0b6 100644
--- a/fdbbackup/backup.actor.cpp
+++ b/fdbbackup/backup.actor.cpp
@@ -4102,8 +4102,10 @@ int main(int argc, char* argv[]) {
 				// where the fdbbackup command hangs infinitely. 60 seconds should be more than
 				// enough for all cases to finish and 5 retries should also be good enough for
 				// most cases.
-				db->setOption(FDBDatabaseOptions::TRANSACTION_TIMEOUT, Optional<StringRef>(StringRef((const uint8_t*)60000, 8)));
-				db->setOption(FDBDatabaseOptions::TRANSACTION_RETRY_LIMIT, Optional<StringRef>(StringRef((const uint8_t*)5, 8)));
+				db->setOption(FDBDatabaseOptions::TRANSACTION_TIMEOUT,
+				              Optional<StringRef>(StringRef((const uint8_t*)60000, 8)));
+				db->setOption(FDBDatabaseOptions::TRANSACTION_RETRY_LIMIT,
+				              Optional<StringRef>(StringRef((const uint8_t*)5, 8)));
 			}
 
 			return result.present();
@@ -4124,8 +4126,10 @@ int main(int argc, char* argv[]) {
 				// where the fdbbackup command hangs infinitely. 60 seconds should be more than
 				// enough for all cases to finish and 5 retries should also be good enough for
 				// most cases.
-				sourceDb->setOption(FDBDatabaseOptions::TRANSACTION_TIMEOUT, Optional<StringRef>(StringRef((const uint8_t*)60000, 8)));
-				sourceDb->setOption(FDBDatabaseOptions::TRANSACTION_RETRY_LIMIT, Optional<StringRef>(StringRef((const uint8_t*)5, 8)));
+				sourceDb->setOption(FDBDatabaseOptions::TRANSACTION_TIMEOUT,
+				                    Optional<StringRef>(StringRef((const uint8_t*)60000, 8)));
+				sourceDb->setOption(FDBDatabaseOptions::TRANSACTION_RETRY_LIMIT,
+				                    Optional<StringRef>(StringRef((const uint8_t*)5, 8)));
 			}
 
 			return result.present();

From 834bb30381b2206fff511c8923e8d216bc702c31 Mon Sep 17 00:00:00 2001
From: "Johannes M. Scheuermann" <jscheuermann@apple.com>
Date: Wed, 15 May 2024 11:46:29 +0200
Subject: [PATCH 427/468] Correct imports for release-7.1

---
 fdbbackup/backup.actor.cpp | 1 +
 1 file changed, 1 insertion(+)

diff --git a/fdbbackup/backup.actor.cpp b/fdbbackup/backup.actor.cpp
index d228752f0b6..89d393cfc51 100644
--- a/fdbbackup/backup.actor.cpp
+++ b/fdbbackup/backup.actor.cpp
@@ -36,6 +36,7 @@
 #include "flow/genericactors.actor.h"
 #include "flow/TLSConfig.actor.h"
 
+#include "fdbclient/DatabaseContext.h"
 #include "fdbclient/FDBTypes.h"
 #include "fdbclient/BackupAgent.actor.h"
 #include "fdbclient/Status.h"

From 51779a5dca44f142363519c6ec34c6dd66d1bf77 Mon Sep 17 00:00:00 2001
From: stack <stack@apache.org>
Date: Wed, 8 May 2024 09:11:15 -0700
Subject: [PATCH 428/468] Make it so can run the ycsb container standalone
 without need of a k8s context (useful for running ycsb against bare metal
 cluster).

* packaging/docker/Dockerfile
* packaging/docker/Dockerfile.eks
 Make the ycsb target inherit from foundationdb-base so we pick up
 libfdb_c.so. Add in a version of run_ycsb.sh that doesn't presume
 k8s. Use 'entrypoint' rather than 'cmd' so can override on
 'docker run'.

* packaging/docker/run_ycsb_standalone.sh
 Version of run_ycsb.sh w/o the presumption of k8s.
---
 packaging/docker/Dockerfile             | 10 +++--
 packaging/docker/Dockerfile.eks         |  9 +++--
 packaging/docker/run_ycsb_standalone.sh | 54 +++++++++++++++++++++++++
 3 files changed, 65 insertions(+), 8 deletions(-)
 create mode 100644 packaging/docker/run_ycsb_standalone.sh

diff --git a/packaging/docker/Dockerfile b/packaging/docker/Dockerfile
index ae9b9336719..6cf1e067859 100644
--- a/packaging/docker/Dockerfile
+++ b/packaging/docker/Dockerfile
@@ -170,7 +170,7 @@ ENV FDB_CLUSTER_FILE_CONTENTS ""
 ENV FDB_PROCESS_CLASS unset
 ENTRYPOINT ["/usr/bin/tini", "-g", "--", "/var/fdb/scripts/fdb.bash"]
 
-FROM base as ycsb
+FROM foundationdb-base as ycsb
 
 RUN yum -y install \
     java-11-openjdk-11.0.13.0.8-1.el7_9 && \
@@ -192,9 +192,11 @@ RUN NO_PROXY="" no_proxy="" curl -Ls https://s3.us-west-2.amazonaws.com/amazon-e
 
 # TODO: Log4J complains that it's eating the HTracer logs.  Even without it, we get per-operation
 # time series graphs of throughput, median, 90, 99, 99.9 and 99.99 (in usec).
-ADD run_ycsb.sh /usr/local/bin/run_ycsb.sh
+ADD run_ycsb.sh run_ycsb_standalone.sh /usr/local/bin
 RUN mkdir -p /var/log/fdb-trace-logs && \
-    chmod +x /usr/local/bin/run_ycsb.sh
+    chmod +x /usr/local/bin/run_ycsb.sh && \
+    chmod +x /usr/local/bin/run_ycsb_standalone.sh
+
 
 ADD YCSB /YCSB
 WORKDIR /YCSB
@@ -202,4 +204,4 @@ ENV FDB_NETWORK_OPTION_EXTERNAL_CLIENT_DIRECTORY=/var/dynamic-conf/lib/multivers
 ENV FDB_NETWORK_OPTION_TRACE_ENABLE=/var/log/fdb-trace-logs
 ENV LD_LIBRARY_PATH=/var/dynamic-conf/lib/
 ENV BUCKET=""
-CMD ["run_ycsb.sh"]
+ENTRYPOINT ["run_ycsb.sh"]
diff --git a/packaging/docker/Dockerfile.eks b/packaging/docker/Dockerfile.eks
index 75962136530..1edf3768035 100644
--- a/packaging/docker/Dockerfile.eks
+++ b/packaging/docker/Dockerfile.eks
@@ -178,7 +178,7 @@ ENV FDB_CLUSTER_FILE_CONTENTS ""
 ENV FDB_PROCESS_CLASS unset
 ENTRYPOINT ["/usr/bin/tini", "-g", "--", "/var/fdb/scripts/fdb.bash"]
 
-FROM base as ycsb
+FROM foundationdb-base as ycsb
 
 RUN yum -y install \
     java-11-amazon-corretto-11.0.13+8-1.amzn2 && \
@@ -187,9 +187,10 @@ RUN yum -y install \
 
 # TODO: Log4J complains that it's eating the HTracer logs.  Even without it, we get per-operation
 # time series graphs of throughput, median, 90, 99, 99.9 and 99.99 (in usec).
-ADD run_ycsb.sh /usr/local/bin/run_ycsb.sh
+ADD run_ycsb.sh run_ycsb_standalone.sh /usr/local/bin
 RUN mkdir -p /var/log/fdb-trace-logs && \
-    chmod +x /usr/local/bin/run_ycsb.sh
+    chmod +x /usr/local/bin/run_ycsb.sh && \
+    chmod +x /usr/local/bin/run_ycsb_standalone.sh
 
 ADD YCSB /YCSB
 WORKDIR /YCSB
@@ -197,4 +198,4 @@ ENV FDB_NETWORK_OPTION_EXTERNAL_CLIENT_DIRECTORY=/var/dynamic-conf/lib/multivers
 ENV FDB_NETWORK_OPTION_TRACE_ENABLE=/var/log/fdb-trace-logs
 ENV LD_LIBRARY_PATH=/var/dynamic-conf/lib/
 ENV BUCKET=""
-CMD ["run_ycsb.sh"]
+ENTRYPOINT ["run_ycsb.sh"]
diff --git a/packaging/docker/run_ycsb_standalone.sh b/packaging/docker/run_ycsb_standalone.sh
new file mode 100644
index 00000000000..41459534b43
--- /dev/null
+++ b/packaging/docker/run_ycsb_standalone.sh
@@ -0,0 +1,54 @@
+#!/usr/bin/env bash
+# This script is like the run_ycsb.sh script only there is no presumption of
+# a k8s context. It also takes a method from the adjacent fdb.bash script for
+# writing out an fdb.cluster file based off environment variables.
+set -Eeuo pipefail
+
+function logg () {
+    printf "##### $(date +'%Y-%m-%dT%H:%M:%SZ') #  %-56.55s #####\n" "${1}"
+}
+
+function error_exit () {
+    echo "################################################################################"
+    logg "${0} FAILED"
+    logg "RUN_ID: ${RUN_ID}"
+    logg "WORKLOAD: ${WORKLOAD}"
+    logg "ENVIRONMENT IS:"
+    env
+    echo "################################################################################"
+}
+
+function create_cluster_file() {
+    FDB_CLUSTER_FILE=${FDB_CLUSTER_FILE:-/etc/foundationdb/fdb.cluster}
+    mkdir -p "$(dirname $FDB_CLUSTER_FILE)"
+
+    if [[ -n "$FDB_CLUSTER_FILE_CONTENTS" ]]; then
+        echo "$FDB_CLUSTER_FILE_CONTENTS" > "$FDB_CLUSTER_FILE"
+    elif [[ -n $FDB_COORDINATOR ]]; then
+        coordinator_ip=$(dig +short "$FDB_COORDINATOR")
+        if [[ -z "$coordinator_ip" ]]; then
+            echo "Failed to look up coordinator address for $FDB_COORDINATOR" 1>&2
+            exit 1
+        fi
+        coordinator_port=${FDB_COORDINATOR_PORT:-4500}
+        echo "docker:docker@$coordinator_ip:$coordinator_port" > "$FDB_CLUSTER_FILE"
+    else
+        echo "FDB_COORDINATOR environment variable not defined" 1>&2
+        exit 1
+    fi
+}
+
+trap error_exit ERR
+
+logg "RUNNING YCSB ${WORKLOAD}"
+set -x
+create_cluster_file
+./bin/ycsb.sh "${MODE}" foundationdb -s -P "workloads/${WORKLOAD}" "${YCSB_ARGS}"
+set +x
+logg "YCSB ${WORKLOAD} FINISHED"
+
+echo "################################################################################"
+logg "COMPLETED ${0}"
+logg "RUN_ID: ${RUN_ID}"
+logg "WORKLOAD: ${WORKLOAD}"
+echo "################################################################################"

From 4645713e41385db67b53d688df8ab9b0e4821585 Mon Sep 17 00:00:00 2001
From: FoundationDB CI <foundationdb_ci@apple.com>
Date: Wed, 15 May 2024 14:17:12 +0000
Subject: [PATCH 429/468] disable AVX for 7.1.60 release

---
 cmake/ConfigureCompiler.cmake | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cmake/ConfigureCompiler.cmake b/cmake/ConfigureCompiler.cmake
index e52d8665ee8..7e50469ba6a 100644
--- a/cmake/ConfigureCompiler.cmake
+++ b/cmake/ConfigureCompiler.cmake
@@ -253,7 +253,7 @@ else()
       set(USE_AVX512F OFF)
     endif()
   endif()
-  set(USE_AVX ON CACHE BOOL "Enable AVX instructions")
+  set(USE_AVX OFF CACHE BOOL "Enable AVX instructions")
   if (USE_AVX)
     if (CMAKE_HOST_SYSTEM_PROCESSOR MATCHES "^x86")
       add_compile_options(-mavx)

From 262c6fe778ad229cbb58bfcd6d337bb0b4227622 Mon Sep 17 00:00:00 2001
From: FoundationDB CI <foundationdb_ci@apple.com>
Date: Wed, 15 May 2024 14:18:16 +0000
Subject: [PATCH 430/468] enable AVX and update version for 7.1.61 release

---
 CMakeLists.txt                | 2 +-
 cmake/ConfigureCompiler.cmake | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index bee878481f2..a1b2dff93b6 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -23,7 +23,7 @@ else()
 endif()
 
 project(foundationdb
-  VERSION 7.1.60
+  VERSION 7.1.61
   DESCRIPTION "FoundationDB is a scalable, fault-tolerant, ordered key-value store with full ACID transactions."
   HOMEPAGE_URL "http://www.foundationdb.org/"
   LANGUAGES C CXX ASM)
diff --git a/cmake/ConfigureCompiler.cmake b/cmake/ConfigureCompiler.cmake
index 7e50469ba6a..e52d8665ee8 100644
--- a/cmake/ConfigureCompiler.cmake
+++ b/cmake/ConfigureCompiler.cmake
@@ -253,7 +253,7 @@ else()
       set(USE_AVX512F OFF)
     endif()
   endif()
-  set(USE_AVX OFF CACHE BOOL "Enable AVX instructions")
+  set(USE_AVX ON CACHE BOOL "Enable AVX instructions")
   if (USE_AVX)
     if (CMAKE_HOST_SYSTEM_PROCESSOR MATCHES "^x86")
       add_compile_options(-mavx)

From d0ce1f80df81464c403328ef2d70b20e877de4ae Mon Sep 17 00:00:00 2001
From: FoundationDB CI <foundationdb_ci@apple.com>
Date: Wed, 15 May 2024 22:16:02 +0000
Subject: [PATCH 431/468] update version after 7.1.61 release

---
 CMakeLists.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index a1b2dff93b6..9296bea0564 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -23,7 +23,7 @@ else()
 endif()
 
 project(foundationdb
-  VERSION 7.1.61
+  VERSION 7.1.62
   DESCRIPTION "FoundationDB is a scalable, fault-tolerant, ordered key-value store with full ACID transactions."
   HOMEPAGE_URL "http://www.foundationdb.org/"
   LANGUAGES C CXX ASM)

From fe767feaa89286b222257bfbdd5ff9c85f77386e Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyu_zhou@apple.com>
Date: Fri, 17 May 2024 14:15:28 -0700
Subject: [PATCH 432/468] Add 7.1.58 to 7.1.61 release notes (#11405)

---
 .../release-notes/release-notes-710.rst       | 24 +++++++++++++++++++
 1 file changed, 24 insertions(+)

diff --git a/documentation/sphinx/source/release-notes/release-notes-710.rst b/documentation/sphinx/source/release-notes/release-notes-710.rst
index 4fbf393de79..63671679509 100644
--- a/documentation/sphinx/source/release-notes/release-notes-710.rst
+++ b/documentation/sphinx/source/release-notes/release-notes-710.rst
@@ -4,6 +4,30 @@
 Release Notes
 #############
 
+7.1.61
+======
+* Same as 7.1.60 release with AVX enabled.
+
+7.1.60
+======
+* Released with AVX disabled.
+* Fixed a DR corruption bug due to a race condition. `(PR #11245) <https://github.com/apple/foundationdb/pull/11245>`_
+* Added Go tenanting support for 7.1. `(PR #11278) <https://github.com/apple/foundationdb/pull/11278>`_
+* Increased visibility of gray failure actions. `(PR #11314) <https://github.com/apple/foundationdb/pull/11314>`_
+* Increase visibility of CommitProxyTerminated events for failed_to_progress errors. `(PR #11316) <https://github.com/apple/foundationdb/pull/11316>`_
+* Fixed an infinite retry of GRV request bug. `(PR #11352) <https://github.com/apple/foundationdb/pull/11352>`_
+* Improved distributed consistency checker to continuously run by default and visibility of recruitment errors. `(PR #11351) <https://github.com/apple/foundationdb/pull/11351>`_
+* Fix an assertion failure of cluster controller when waiting for recovery. `(PR #11398) <https://github.com/apple/foundationdb/pull/11398>`_
+
+7.1.59
+======
+* Same as 7.1.59 release with AVX enabled.
+
+7.1.58
+======
+* Released with AVX disabled.
+* Fixed a bug in consistency checker urgent mode. `(PR #11230) <https://github.com/apple/foundationdb/pull/11230>`_
+
 7.1.57
 ======
 * Same as 7.1.56 release with AVX enabled.

From db7269f1aeebd0caee8effbea0cb571056e3115f Mon Sep 17 00:00:00 2001
From: "Johannes M. Scheuermann" <jscheuermann@apple.com>
Date: Thu, 23 May 2024 14:17:32 +0200
Subject: [PATCH 433/468] Correct the path where the fdb-kubernetes-monitor
 copies the binary into when running in sidecar mode

---
 fdbkubernetesmonitor/copy.go      | 25 ++++----
 fdbkubernetesmonitor/copy_test.go | 98 ++++++++++++++++++++-----------
 fdbkubernetesmonitor/main.go      |  5 +-
 3 files changed, 80 insertions(+), 48 deletions(-)

diff --git a/fdbkubernetesmonitor/copy.go b/fdbkubernetesmonitor/copy.go
index 0a311da095d..abb43eaf9ec 100644
--- a/fdbkubernetesmonitor/copy.go
+++ b/fdbkubernetesmonitor/copy.go
@@ -82,11 +82,11 @@ func copyFile(logger logr.Logger, inputPath string, outputPath string, required
 
 // CopyFiles copies a list of files into the output directory.
 func CopyFiles(logger logr.Logger, outputDir string, copyDetails map[string]string, requiredCopies map[string]bool) error {
-	for inputPath, outputSubpath := range copyDetails {
-		if outputSubpath == "" {
-			outputSubpath = path.Base(inputPath)
+	for inputPath, outputSubPath := range copyDetails {
+		if outputSubPath == "" {
+			outputSubPath = path.Base(inputPath)
 		}
-		outputPath := path.Join(outputDir, outputSubpath)
+		outputPath := path.Join(outputDir, outputSubPath)
 		err := os.MkdirAll(path.Dir(outputPath), os.ModeDir|os.ModePerm)
 		if err != nil {
 			return err
@@ -135,7 +135,7 @@ func getLibraryPath() string {
 }
 
 // getCopyDetails will generate the details for all the files that should be copied.
-func getCopyDetails(inputDir string, copyPrimaryLibrary string, binaryOutputDirectory string, copyFiles []string, copyBinaries []string, copyLibraries []string, requiredCopyFiles []string, currentContainerVersion string) (map[string]string, map[string]bool, error) {
+func getCopyDetails(inputDir string, copyPrimaryLibrary string, binaryOutputDirectory string, copyFiles []string, copyBinaries []string, copyLibraries []string, requiredCopyFiles []string, currentContainerVersion string, mode executionMode) (map[string]string, map[string]bool, error) {
 	copyDetails := make(map[string]string, len(copyFiles)+len(copyBinaries))
 
 	for _, filePath := range copyFiles {
@@ -144,12 +144,17 @@ func getCopyDetails(inputDir string, copyPrimaryLibrary string, binaryOutputDire
 
 	if len(copyBinaries) > 0 {
 		if binaryOutputDirectory == "" {
-			compactVersion, err := getCompactVersion(currentContainerVersion)
-			if err != nil {
-				return nil, nil, err
+			// The operator expects that the sidecar will copy the binaries into /var/fdb/shared-binaries/bin/<new version>/fdbserver
+			if mode == executionModeSidecar {
+				binaryOutputDirectory = path.Join("bin", currentContainerVersion)
+			} else {
+				compactVersion, err := getCompactVersion(currentContainerVersion)
+				if err != nil {
+					return nil, nil, err
+				}
+
+				binaryOutputDirectory = compactVersion
 			}
-
-			binaryOutputDirectory = compactVersion
 		}
 
 		binaryDirectory := getBinaryDirectory()
diff --git a/fdbkubernetesmonitor/copy_test.go b/fdbkubernetesmonitor/copy_test.go
index 1f208ea0156..95d18e1f425 100644
--- a/fdbkubernetesmonitor/copy_test.go
+++ b/fdbkubernetesmonitor/copy_test.go
@@ -46,7 +46,7 @@ var _ = Describe("Testing the copy methods", func() {
 
 		When("no files should be copied", func() {
 			It("no error should be thrown", func() {
-				copyDetails, requiredCopyMap, err := getCopyDetails(inputDir, copyPrimaryLibrary, binaryOutputDirectory, copyFiles, copyBinaries, copyLibraries, requiredCopyFiles, currentContainerVersion)
+				copyDetails, requiredCopyMap, err := getCopyDetails(inputDir, copyPrimaryLibrary, binaryOutputDirectory, copyFiles, copyBinaries, copyLibraries, requiredCopyFiles, currentContainerVersion, executionModeInit)
 				Expect(err).NotTo(HaveOccurred())
 				Expect(copyDetails).To(BeEmpty())
 				Expect(requiredCopyMap).To(BeEmpty())
@@ -59,12 +59,24 @@ var _ = Describe("Testing the copy methods", func() {
 					copyBinaries = []string{"fdbserver"}
 				})
 
-				It("no error should be thrown", func() {
-					copyDetails, requiredCopyMap, err := getCopyDetails(inputDir, copyPrimaryLibrary, binaryOutputDirectory, copyFiles, copyBinaries, copyLibraries, requiredCopyFiles, currentContainerVersion)
-					Expect(err).NotTo(HaveOccurred())
-					Expect(copyDetails).To(HaveKeyWithValue("/usr/bin/fdbserver", "7.1/fdbserver"))
-					Expect(copyDetails).To(HaveLen(1))
-					Expect(requiredCopyMap).To(BeEmpty())
+				When("the execution mode is init", func() {
+					It("no error should be thrown", func() {
+						copyDetails, requiredCopyMap, err := getCopyDetails(inputDir, copyPrimaryLibrary, binaryOutputDirectory, copyFiles, copyBinaries, copyLibraries, requiredCopyFiles, currentContainerVersion, executionModeInit)
+						Expect(err).NotTo(HaveOccurred())
+						Expect(copyDetails).To(HaveKeyWithValue("/usr/bin/fdbserver", "7.1/fdbserver"))
+						Expect(copyDetails).To(HaveLen(1))
+						Expect(requiredCopyMap).To(BeEmpty())
+					})
+				})
+
+				When("the execution mode is sidecar", func() {
+					It("no error should be thrown", func() {
+						copyDetails, requiredCopyMap, err := getCopyDetails(inputDir, copyPrimaryLibrary, binaryOutputDirectory, copyFiles, copyBinaries, copyLibraries, requiredCopyFiles, currentContainerVersion, executionModeSidecar)
+						Expect(err).NotTo(HaveOccurred())
+						Expect(copyDetails).To(HaveKeyWithValue("/usr/bin/fdbserver", "bin/7.1.43/fdbserver"))
+						Expect(copyDetails).To(HaveLen(1))
+						Expect(requiredCopyMap).To(BeEmpty())
+					})
 				})
 			})
 
@@ -79,7 +91,7 @@ var _ = Describe("Testing the copy methods", func() {
 				})
 
 				It("no error should be thrown", func() {
-					copyDetails, requiredCopyMap, err := getCopyDetails(inputDir, copyPrimaryLibrary, binaryOutputDirectory, copyFiles, copyBinaries, copyLibraries, requiredCopyFiles, currentContainerVersion)
+					copyDetails, requiredCopyMap, err := getCopyDetails(inputDir, copyPrimaryLibrary, binaryOutputDirectory, copyFiles, copyBinaries, copyLibraries, requiredCopyFiles, currentContainerVersion, executionModeInit)
 					Expect(err).NotTo(HaveOccurred())
 					Expect(copyDetails).To(HaveKeyWithValue("/usr/bin/fdbserver", "testing/fdbserver"))
 					Expect(copyDetails).To(HaveLen(1))
@@ -94,7 +106,7 @@ var _ = Describe("Testing the copy methods", func() {
 			})
 
 			It("no error should be thrown", func() {
-				copyDetails, requiredCopyMap, err := getCopyDetails(inputDir, copyPrimaryLibrary, binaryOutputDirectory, copyFiles, copyBinaries, copyLibraries, requiredCopyFiles, currentContainerVersion)
+				copyDetails, requiredCopyMap, err := getCopyDetails(inputDir, copyPrimaryLibrary, binaryOutputDirectory, copyFiles, copyBinaries, copyLibraries, requiredCopyFiles, currentContainerVersion, executionModeInit)
 				Expect(err).NotTo(HaveOccurred())
 				Expect(copyDetails).To(
 					And(
@@ -113,7 +125,7 @@ var _ = Describe("Testing the copy methods", func() {
 			})
 
 			It("no error should be thrown", func() {
-				copyDetails, requiredCopyMap, err := getCopyDetails(inputDir, copyPrimaryLibrary, binaryOutputDirectory, copyFiles, copyBinaries, copyLibraries, requiredCopyFiles, currentContainerVersion)
+				copyDetails, requiredCopyMap, err := getCopyDetails(inputDir, copyPrimaryLibrary, binaryOutputDirectory, copyFiles, copyBinaries, copyLibraries, requiredCopyFiles, currentContainerVersion, executionModeInit)
 				Expect(err).NotTo(HaveOccurred())
 				Expect(copyDetails).To(
 					And(
@@ -132,7 +144,7 @@ var _ = Describe("Testing the copy methods", func() {
 			})
 
 			It("no error should be thrown", func() {
-				copyDetails, requiredCopyMap, err := getCopyDetails(inputDir, copyPrimaryLibrary, binaryOutputDirectory, copyFiles, copyBinaries, copyLibraries, requiredCopyFiles, currentContainerVersion)
+				copyDetails, requiredCopyMap, err := getCopyDetails(inputDir, copyPrimaryLibrary, binaryOutputDirectory, copyFiles, copyBinaries, copyLibraries, requiredCopyFiles, currentContainerVersion, executionModeInit)
 				Expect(err).NotTo(HaveOccurred())
 				Expect(copyDetails).To(
 					And(
@@ -152,7 +164,7 @@ var _ = Describe("Testing the copy methods", func() {
 			})
 
 			It("no error should be thrown", func() {
-				copyDetails, requiredCopyMap, err := getCopyDetails(inputDir, copyPrimaryLibrary, binaryOutputDirectory, copyFiles, copyBinaries, copyLibraries, requiredCopyFiles, currentContainerVersion)
+				copyDetails, requiredCopyMap, err := getCopyDetails(inputDir, copyPrimaryLibrary, binaryOutputDirectory, copyFiles, copyBinaries, copyLibraries, requiredCopyFiles, currentContainerVersion, executionModeInit)
 				Expect(err).NotTo(HaveOccurred())
 				Expect(copyDetails).To(
 					And(
@@ -174,7 +186,7 @@ var _ = Describe("Testing the copy methods", func() {
 
 			When("no input directory is set", func() {
 				It("no error should be thrown", func() {
-					copyDetails, requiredCopyMap, err := getCopyDetails(inputDir, copyPrimaryLibrary, binaryOutputDirectory, copyFiles, copyBinaries, copyLibraries, requiredCopyFiles, currentContainerVersion)
+					copyDetails, requiredCopyMap, err := getCopyDetails(inputDir, copyPrimaryLibrary, binaryOutputDirectory, copyFiles, copyBinaries, copyLibraries, requiredCopyFiles, currentContainerVersion, executionModeInit)
 					Expect(err).NotTo(HaveOccurred())
 					Expect(copyDetails).To(
 						And(
@@ -196,7 +208,7 @@ var _ = Describe("Testing the copy methods", func() {
 				})
 
 				It("no error should be thrown", func() {
-					copyDetails, requiredCopyMap, err := getCopyDetails(inputDir, copyPrimaryLibrary, binaryOutputDirectory, copyFiles, copyBinaries, copyLibraries, requiredCopyFiles, currentContainerVersion)
+					copyDetails, requiredCopyMap, err := getCopyDetails(inputDir, copyPrimaryLibrary, binaryOutputDirectory, copyFiles, copyBinaries, copyLibraries, requiredCopyFiles, currentContainerVersion, executionModeInit)
 					Expect(err).NotTo(HaveOccurred())
 					Expect(copyDetails).To(
 						And(
@@ -219,7 +231,7 @@ var _ = Describe("Testing the copy methods", func() {
 			})
 
 			It("no error should be thrown", func() {
-				copyDetails, requiredCopyMap, err := getCopyDetails(inputDir, copyPrimaryLibrary, binaryOutputDirectory, copyFiles, copyBinaries, copyLibraries, requiredCopyFiles, currentContainerVersion)
+				copyDetails, requiredCopyMap, err := getCopyDetails(inputDir, copyPrimaryLibrary, binaryOutputDirectory, copyFiles, copyBinaries, copyLibraries, requiredCopyFiles, currentContainerVersion, executionModeInit)
 				Expect(err).NotTo(HaveOccurred())
 				Expect(copyDetails).To(
 					And(
@@ -248,7 +260,7 @@ var _ = Describe("Testing the copy methods", func() {
 				})
 
 				It("should throw an error", func() {
-					copyDetails, requiredCopyMap, err := getCopyDetails(inputDir, copyPrimaryLibrary, binaryOutputDirectory, copyFiles, copyBinaries, copyLibraries, requiredCopyFiles, currentContainerVersion)
+					copyDetails, requiredCopyMap, err := getCopyDetails(inputDir, copyPrimaryLibrary, binaryOutputDirectory, copyFiles, copyBinaries, copyLibraries, requiredCopyFiles, currentContainerVersion, executionModeInit)
 					Expect(err).To(HaveOccurred())
 					Expect(copyDetails).To(BeEmpty())
 					Expect(requiredCopyMap).To(BeEmpty())
@@ -261,7 +273,7 @@ var _ = Describe("Testing the copy methods", func() {
 				})
 
 				It("should not throw an error and include all files", func() {
-					copyDetails, requiredCopyMap, err := getCopyDetails(inputDir, copyPrimaryLibrary, binaryOutputDirectory, copyFiles, copyBinaries, copyLibraries, requiredCopyFiles, currentContainerVersion)
+					copyDetails, requiredCopyMap, err := getCopyDetails(inputDir, copyPrimaryLibrary, binaryOutputDirectory, copyFiles, copyBinaries, copyLibraries, requiredCopyFiles, currentContainerVersion, executionModeInit)
 					Expect(err).NotTo(HaveOccurred())
 					Expect(copyDetails).To(
 						And(
@@ -292,24 +304,40 @@ var _ = Describe("Testing the copy methods", func() {
 		})
 
 		When("copying the binaries", func() {
-			var copyDetails map[string]string
-
-			BeforeEach(func() {
-				// Simulate the binary directory
-				GinkgoT().Setenv(binaryTestDirectoryEnv, inputBinaryDir)
+			When("the execution mode is init", func() {
+				BeforeEach(func() {
+					// Simulate the binary directory
+					GinkgoT().Setenv(binaryTestDirectoryEnv, inputBinaryDir)
+					copyDetails, _, err := getCopyDetails("", "", "", nil, binaries, nil, nil, "7.1.43", executionModeInit)
+					Expect(err).NotTo(HaveOccurred())
+					Expect(copyDetails).To(HaveLen(3))
+					Expect(CopyFiles(GinkgoLogr, outputBinaryDir, copyDetails, map[string]bool{})).NotTo(HaveOccurred())
+				})
 
-				var err error
-				copyDetails, _, err = getCopyDetails("", "", "", nil, binaries, nil, nil, "7.1.43")
-				Expect(err).NotTo(HaveOccurred())
-				Expect(copyDetails).To(HaveLen(3))
-				Expect(CopyFiles(GinkgoLogr, outputBinaryDir, copyDetails, map[string]bool{})).NotTo(HaveOccurred())
+				It("should copy all the files", func() {
+					Expect(path.Join(outputBinaryDir, "7.1", "fdbserver")).Should(BeAnExistingFile())
+					Expect(path.Join(outputBinaryDir, "7.1", "fdbrestore")).Should(BeAnExistingFile())
+					Expect(path.Join(outputBinaryDir, "7.1", "fdbbackup")).Should(BeAnExistingFile())
+				})
 			})
 
-			It("should copy all the files", func() {
-				Expect(path.Join(outputBinaryDir, "7.1", "fdbserver")).Should(BeAnExistingFile())
-				Expect(path.Join(outputBinaryDir, "7.1", "fdbrestore")).Should(BeAnExistingFile())
-				Expect(path.Join(outputBinaryDir, "7.1", "fdbbackup")).Should(BeAnExistingFile())
+			When("the execution mode is sidecar", func() {
+				BeforeEach(func() {
+					// Simulate the binary directory
+					GinkgoT().Setenv(binaryTestDirectoryEnv, inputBinaryDir)
+					copyDetails, _, err := getCopyDetails("", "", "", nil, binaries, nil, nil, "7.1.43", executionModeSidecar)
+					Expect(err).NotTo(HaveOccurred())
+					Expect(copyDetails).To(HaveLen(3))
+					Expect(CopyFiles(GinkgoLogr, outputBinaryDir, copyDetails, map[string]bool{})).NotTo(HaveOccurred())
+				})
+
+				It("should copy all the files", func() {
+					Expect(path.Join(outputBinaryDir, "bin", "7.1.43", "fdbserver")).Should(BeAnExistingFile())
+					Expect(path.Join(outputBinaryDir, "bin", "7.1.43", "fdbrestore")).Should(BeAnExistingFile())
+					Expect(path.Join(outputBinaryDir, "bin", "7.1.43", "fdbbackup")).Should(BeAnExistingFile())
+				})
 			})
+
 		})
 	})
 
@@ -335,7 +363,7 @@ var _ = Describe("Testing the copy methods", func() {
 
 			BeforeEach(func() {
 				var err error
-				copyDetails, _, err = getCopyDetails("", "", "", nil, nil, libraries, nil, "7.1.43")
+				copyDetails, _, err = getCopyDetails("", "", "", nil, nil, libraries, nil, "7.1.43", executionModeInit)
 				Expect(err).NotTo(HaveOccurred())
 				Expect(copyDetails).To(HaveLen(3))
 				Expect(CopyFiles(GinkgoLogr, outputLibraryDir, copyDetails, map[string]bool{})).NotTo(HaveOccurred())
@@ -353,7 +381,7 @@ var _ = Describe("Testing the copy methods", func() {
 
 			BeforeEach(func() {
 				var err error
-				copyDetails, _, err = getCopyDetails("", "7.1", "", nil, nil, libraries, nil, "7.1.43")
+				copyDetails, _, err = getCopyDetails("", "7.1", "", nil, nil, libraries, nil, "7.1.43", executionModeInit)
 				Expect(err).NotTo(HaveOccurred())
 				Expect(copyDetails).To(HaveLen(3))
 				Expect(CopyFiles(GinkgoLogr, outputLibraryDir, copyDetails, map[string]bool{})).NotTo(HaveOccurred())
@@ -382,7 +410,7 @@ var _ = Describe("Testing the copy methods", func() {
 
 				BeforeEach(func() {
 					var err error
-					copyDetails, _, err = getCopyDetails(testInputDir, "", "", []string{"testfile"}, nil, nil, nil, "7.1.43")
+					copyDetails, _, err = getCopyDetails(testInputDir, "", "", []string{"testfile"}, nil, nil, nil, "7.1.43", executionModeInit)
 					Expect(err).NotTo(HaveOccurred())
 					Expect(copyDetails).To(HaveLen(1))
 					Expect(CopyFiles(GinkgoLogr, testOutputDir, copyDetails, map[string]bool{})).NotTo(HaveOccurred())
@@ -399,7 +427,7 @@ var _ = Describe("Testing the copy methods", func() {
 
 				BeforeEach(func() {
 					var err error
-					copyDetails, requiredFiles, err = getCopyDetails(testInputDir, "", "", []string{"testfile"}, nil, nil, []string{"testfile"}, "7.1.43")
+					copyDetails, requiredFiles, err = getCopyDetails(testInputDir, "", "", []string{"testfile"}, nil, nil, []string{"testfile"}, "7.1.43", executionModeInit)
 					Expect(err).NotTo(HaveOccurred())
 					Expect(copyDetails).To(HaveLen(1))
 				})
diff --git a/fdbkubernetesmonitor/main.go b/fdbkubernetesmonitor/main.go
index ddf70b203eb..404209c934a 100644
--- a/fdbkubernetesmonitor/main.go
+++ b/fdbkubernetesmonitor/main.go
@@ -118,14 +118,13 @@ func main() {
 		panic(err)
 	}
 	currentContainerVersion := strings.TrimSpace(string(versionBytes))
-
-	copyDetails, requiredCopies, err := getCopyDetails(inputDir, copyPrimaryLibrary, binaryOutputDirectory, copyFiles, copyBinaries, copyLibraries, requiredCopyFiles, currentContainerVersion)
+	mode := executionMode(executionModeString)
+	copyDetails, requiredCopies, err := getCopyDetails(inputDir, copyPrimaryLibrary, binaryOutputDirectory, copyFiles, copyBinaries, copyLibraries, requiredCopyFiles, currentContainerVersion, mode)
 	if err != nil {
 		logger.Error(err, "Error getting list of files to copy")
 		os.Exit(1)
 	}
 
-	mode := executionMode(executionModeString)
 	switch mode {
 	case executionModeLauncher:
 		customEnvironment, err := loadAdditionalEnvironment(logger)

From bf5abf411f4070e3c4156eeec3f66e82721d32ff Mon Sep 17 00:00:00 2001
From: "Johannes M. Scheuermann" <jscheuermann@apple.com>
Date: Fri, 24 May 2024 14:55:24 +0200
Subject: [PATCH 434/468] Revert variable renaming

---
 fdbkubernetesmonitor/copy.go | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/fdbkubernetesmonitor/copy.go b/fdbkubernetesmonitor/copy.go
index abb43eaf9ec..c24d6abce6b 100644
--- a/fdbkubernetesmonitor/copy.go
+++ b/fdbkubernetesmonitor/copy.go
@@ -82,11 +82,11 @@ func copyFile(logger logr.Logger, inputPath string, outputPath string, required
 
 // CopyFiles copies a list of files into the output directory.
 func CopyFiles(logger logr.Logger, outputDir string, copyDetails map[string]string, requiredCopies map[string]bool) error {
-	for inputPath, outputSubPath := range copyDetails {
-		if outputSubPath == "" {
-			outputSubPath = path.Base(inputPath)
+	for inputPath, outputSubpath := range copyDetails {
+		if outputSubpath == "" {
+			outputSubpath = path.Base(inputPath)
 		}
-		outputPath := path.Join(outputDir, outputSubPath)
+		outputPath := path.Join(outputDir, outputSubpath)
 		err := os.MkdirAll(path.Dir(outputPath), os.ModeDir|os.ModePerm)
 		if err != nil {
 			return err

From 1e3f0130860b5e736c3f118c0ef9cfd9a08a250d Mon Sep 17 00:00:00 2001
From: Zhe Wang <zhe.wang@wustl.edu>
Date: Mon, 10 Jun 2024 19:58:52 -0700
Subject: [PATCH 435/468] Improve checkall tool (#11440)

* improve checkall

* fmt

* simplify

* nit

* simplify

* nit
---
 fdbcli/DebugCommands.actor.cpp | 44 ++++++++++++++++++++++++++--------
 1 file changed, 34 insertions(+), 10 deletions(-)

diff --git a/fdbcli/DebugCommands.actor.cpp b/fdbcli/DebugCommands.actor.cpp
index e30a49b507a..6bd307f842d 100644
--- a/fdbcli/DebugCommands.actor.cpp
+++ b/fdbcli/DebugCommands.actor.cpp
@@ -172,6 +172,24 @@ ACTOR Future<bool> getallCommandActor(Database cx, std::vector<StringRef> tokens
 // hidden commands, no help text for now
 CommandFactory getallCommandFactory("getall");
 
+std::string printStorageServerMachineInfo(const StorageServerInterface& server) {
+	std::string serverIp = server.address().toString();
+	std::string serverLocality = server.locality.toString();
+	return serverLocality + " " + serverIp;
+}
+
+std::string printAllStorageServerMachineInfo(const std::vector<StorageServerInterface>& servers) {
+	std::string res;
+	for (int i = 0; i < servers.size(); i++) {
+		if (i == 0) {
+			res = printStorageServerMachineInfo(servers[i]);
+		} else {
+			res = res + "; " + printStorageServerMachineInfo(servers[i]);
+		}
+	}
+	return res;
+}
+
 // check that all replies are the same. Update begin to the next key to check
 // checkResults keeps invariants:
 // (1) hasMore = true if any server has more data not read yet
@@ -190,6 +208,11 @@ bool checkResults(Version version,
 	for (int j = 0; j < replies.size(); j++) {
 		if (firstValidServer == -1) {
 			firstValidServer = j;
+			// Print full list of comparing servers and the reference server
+			// Used to check server info which does not produce an inconsistency log
+			printf("CheckResult: servers: %s, reference server: %s\n",
+			       printAllStorageServerMachineInfo(servers).c_str(),
+			       printStorageServerMachineInfo(servers[firstValidServer]).c_str());
 			continue; // always select the first server as reference
 		}
 		// compare reference and current
@@ -213,8 +236,8 @@ bool checkResults(Version version,
 				// have the key
 				printf("Inconsistency: UniqueKey, %s(1), %s(0), CurrentIndex %lu, ReferenceIndex %lu, Version %ld, Key "
 				       "%s\n",
-				       servers[firstValidServer].address().toString().c_str(),
-				       servers[j].address().toString().c_str(),
+				       printStorageServerMachineInfo(servers[firstValidServer]).c_str(),
+				       printStorageServerMachineInfo(servers[j]).c_str(),
 				       currentI,
 				       referenceI,
 				       version,
@@ -223,8 +246,8 @@ bool checkResults(Version version,
 			} else if (referenceI >= reference.data.size()) {
 				printf("Inconsistency: UniqueKey, %s(1), %s(0), CurrentIndex %lu, ReferenceIndex %lu, Version %ld, Key "
 				       "%s\n",
-				       servers[j].address().toString().c_str(),
-				       servers[firstValidServer].address().toString().c_str(),
+				       printStorageServerMachineInfo(servers[j]).c_str(),
+				       printStorageServerMachineInfo(servers[firstValidServer]).c_str(),
 				       currentI,
 				       referenceI,
 				       version,
@@ -238,8 +261,8 @@ bool checkResults(Version version,
 						printf("Inconsistency: MismatchValue, %s(1), %s(1), CurrentIndex %lu, ReferenceIndex %lu, "
 						       "Version %ld, "
 						       "Key %s\n",
-						       servers[firstValidServer].address().toString().c_str(),
-						       servers[j].address().toString().c_str(),
+						       printStorageServerMachineInfo(servers[firstValidServer]).c_str(),
+						       printStorageServerMachineInfo(servers[j]).c_str(),
 						       currentI,
 						       referenceI,
 						       version,
@@ -250,8 +273,8 @@ bool checkResults(Version version,
 				} else if (currentKV.key < referenceKV.key) {
 					printf("Inconsistency: UniqueKey, %s(1), %s(0), CurrentIndex %lu, ReferenceIndex %lu, Version %ld, "
 					       "Key %s\n",
-					       servers[j].address().toString().c_str(),
-					       servers[firstValidServer].address().toString().c_str(),
+					       printStorageServerMachineInfo(servers[j]).c_str(),
+					       printStorageServerMachineInfo(servers[firstValidServer]).c_str(),
 					       currentI,
 					       referenceI,
 					       version,
@@ -260,8 +283,8 @@ bool checkResults(Version version,
 				} else {
 					printf("Inconsistency: UniqueKey, %s(1), %s(0), CurrentIndex %lu, ReferenceIndex %lu, Version %ld, "
 					       "Key %s\n",
-					       servers[firstValidServer].address().toString().c_str(),
-					       servers[j].address().toString().c_str(),
+					       printStorageServerMachineInfo(servers[firstValidServer]).c_str(),
+					       printStorageServerMachineInfo(servers[j]).c_str(),
 					       currentI,
 					       referenceI,
 					       version,
@@ -271,6 +294,7 @@ bool checkResults(Version version,
 			}
 		}
 	}
+
 	return allSame;
 }
 

From 4e89a102ea2e05f8ed5a2a83b865ba919ad113b2 Mon Sep 17 00:00:00 2001
From: Johannes Scheuermann <johscheuer@users.noreply.github.com>
Date: Wed, 12 Jun 2024 15:09:17 +0200
Subject: [PATCH 436/468] Add exponential backoff mechanism for restarting
 fdbserver processes in the monitor (#11452)

---
 fdbkubernetesmonitor/monitor.go      | 59 ++++++++++++++++++++--------
 fdbkubernetesmonitor/monitor_test.go | 54 +++++++++++++++++++++++++
 2 files changed, 96 insertions(+), 17 deletions(-)
 create mode 100644 fdbkubernetesmonitor/monitor_test.go

diff --git a/fdbkubernetesmonitor/monitor.go b/fdbkubernetesmonitor/monitor.go
index 2ca667b824c..835842b7617 100644
--- a/fdbkubernetesmonitor/monitor.go
+++ b/fdbkubernetesmonitor/monitor.go
@@ -43,11 +43,9 @@ import (
 	"github.com/go-logr/logr"
 )
 
-// errorBackoffSeconds is the time to wait after a process fails before starting
-// another process.
-// This delay will only be applied when there has been more than one failure
-// within this time window.
-const errorBackoffSeconds = 60
+// maxErrorBackoffSeconds is the maximum time to wait after a process fails before starting another process.
+// The actual delay will be based on the observed errors and will increase until maxErrorBackoffSeconds is hit.
+const maxErrorBackoffSeconds = 60 * time.Second
 
 // Monitor provides the main monitor loop
 type Monitor struct {
@@ -228,20 +226,44 @@ func (monitor *Monitor) acceptConfiguration(configuration *api.ProcessConfigurat
 	}
 }
 
+// getBackoffDuration returns the backoff duration. The backoff time will increase exponential with a maximum of 60 seconds.
+func getBackoffDuration(errorCounter int) time.Duration {
+	timeToBackoff := time.Duration(errorCounter*errorCounter) * time.Second
+	if timeToBackoff > maxErrorBackoffSeconds {
+		return maxErrorBackoffSeconds
+	}
+
+	return timeToBackoff
+}
+
 // RunProcess runs a loop to continually start and watch a process.
 func (monitor *Monitor) RunProcess(processNumber int) {
 	pid := 0
 	logger := monitor.Logger.WithValues("processNumber", processNumber, "area", "RunProcess")
 	logger.Info("Starting run loop")
+	startTime := time.Now()
+	// Counts the successive errors that occurred during process start up. Based on the error count the backoff time
+	// will be calculated.
+	var errorCounter int
+
 	for {
 		if !monitor.checkProcessRequired(processNumber) {
 			return
 		}
 
+		durationSinceLastStart := time.Since(startTime)
+		// If for more than 5 minutes no error have occurred we reset the error counter to reset the backoff time.
+		if durationSinceLastStart > 5*time.Minute {
+			errorCounter = 0
+		}
+
 		arguments, err := monitor.ActiveConfiguration.GenerateArguments(processNumber, monitor.CustomEnvironment)
 		if err != nil {
-			logger.Error(err, "Error generating arguments for subprocess", "configuration", monitor.ActiveConfiguration)
-			time.Sleep(errorBackoffSeconds * time.Second)
+			backoffDuration := getBackoffDuration(errorCounter)
+			logger.Error(err, "Error generating arguments for subprocess", "configuration", monitor.ActiveConfiguration, "errorCounter", errorCounter, "backoffDuration", backoffDuration.String())
+			time.Sleep(backoffDuration)
+			errorCounter++
+			continue
 		}
 		cmd := exec.Cmd{
 			Path: arguments[0],
@@ -262,8 +284,10 @@ func (monitor *Monitor) RunProcess(processNumber int) {
 
 		err = cmd.Start()
 		if err != nil {
-			logger.Error(err, "Error starting subprocess")
-			time.Sleep(errorBackoffSeconds * time.Second)
+			backoffDuration := getBackoffDuration(errorCounter)
+			logger.Error(err, "Error starting subprocess", "backoffDuration", backoffDuration.String())
+			time.Sleep(backoffDuration)
+			errorCounter++
 			continue
 		}
 
@@ -273,7 +297,7 @@ func (monitor *Monitor) RunProcess(processNumber int) {
 			logger.Error(nil, "No Process information available for subprocess")
 		}
 
-		startTime := time.Now()
+		startTime = time.Now()
 		logger.Info("Subprocess started", "PID", pid)
 
 		monitor.updateProcessID(processNumber, pid)
@@ -305,15 +329,16 @@ func (monitor *Monitor) RunProcess(processNumber int) {
 			exitCode = cmd.ProcessState.ExitCode()
 		}
 
-		logger.Info("Subprocess terminated", "exitCode", exitCode, "PID", pid)
-
-		endTime := time.Now()
+		processDuration := time.Since(startTime)
+		logger.Info("Subprocess terminated", "exitCode", exitCode, "PID", pid, "lastExecutionDurationSeconds", processDuration.String())
 		monitor.updateProcessID(processNumber, -1)
 
-		processDuration := endTime.Sub(startTime)
-		if processDuration.Seconds() < errorBackoffSeconds {
-			logger.Info("Backing off from restarting subprocess", "backOffTimeSeconds", errorBackoffSeconds, "lastExecutionDurationSeconds", processDuration)
-			time.Sleep(errorBackoffSeconds * time.Second)
+		// Only backoff if the exit code is non-zero.
+		if exitCode != 0 {
+			backoffDuration := getBackoffDuration(errorCounter)
+			logger.Info("Backing off from restarting subprocess", "backoffDuration", backoffDuration.String(), "lastExecutionDurationSeconds", processDuration.String(), "errorCounter", errorCounter, "exitCode", exitCode)
+			time.Sleep(backoffDuration)
+			errorCounter++
 		}
 	}
 }
diff --git a/fdbkubernetesmonitor/monitor_test.go b/fdbkubernetesmonitor/monitor_test.go
new file mode 100644
index 00000000000..a520c89781f
--- /dev/null
+++ b/fdbkubernetesmonitor/monitor_test.go
@@ -0,0 +1,54 @@
+// monitor_test.go
+//
+// This source file is part of the FoundationDB open source project
+//
+// Copyright 2021-2024 Apple Inc. and the FoundationDB project authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package main
+
+import (
+	"time"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("Testing FDB Kubernetes Monitor", func() {
+	DescribeTable("when getting the backoff time", func(errorCount int, expected time.Duration) {
+		Expect(getBackoffDuration(errorCount)).To(Equal(expected))
+	},
+		Entry("no errors have occurred",
+			0,
+			time.Duration(0),
+		),
+		Entry("one error have occurred",
+			1,
+			1*time.Second,
+		),
+		Entry("two errors have occurred",
+			2,
+			4*time.Second,
+		),
+		Entry("three errors have occurred",
+			3,
+			9*time.Second,
+		),
+		Entry("ten errors have occurred, should return the max backoff seconds",
+			100,
+			60*time.Second,
+		),
+	)
+})

From a22fd429b0a0839c9d9e095b619a698fea827c0d Mon Sep 17 00:00:00 2001
From: "Johannes M. Scheuermann" <jscheuermann@apple.com>
Date: Mon, 10 Jun 2024 18:18:51 +0200
Subject: [PATCH 437/468] Remove some steps are only needed once

---
 fdbkubernetesmonitor/kubernetes.go |  2 +-
 fdbkubernetesmonitor/monitor.go    | 61 ++++++++++++++++--------------
 2 files changed, 33 insertions(+), 30 deletions(-)

diff --git a/fdbkubernetesmonitor/kubernetes.go b/fdbkubernetesmonitor/kubernetes.go
index 497ec3af918..80e4057f8f4 100644
--- a/fdbkubernetesmonitor/kubernetes.go
+++ b/fdbkubernetesmonitor/kubernetes.go
@@ -237,7 +237,7 @@ func (podClient *PodClient) OnUpdate(_, newObj interface{}) {
 		return
 	}
 
-	podClient.Logger.Info("Got event for OnAdd", "name", pod.Name, "namespace", pod.Namespace, "generation", pod.Generation)
+	podClient.Logger.Info("Got event for OnUpdate", "name", pod.Name, "namespace", pod.Namespace, "generation", pod.Generation)
 
 	podClient.metadata = &metav1.PartialObjectMetadata{
 		TypeMeta:   pod.TypeMeta,
diff --git a/fdbkubernetesmonitor/monitor.go b/fdbkubernetesmonitor/monitor.go
index 835842b7617..1662a2fc48e 100644
--- a/fdbkubernetesmonitor/monitor.go
+++ b/fdbkubernetesmonitor/monitor.go
@@ -35,12 +35,12 @@ import (
 	"syscall"
 	"time"
 
-	"github.com/prometheus/client_golang/prometheus/promhttp"
-	"k8s.io/utils/pointer"
-
 	"github.com/apple/foundationdb/fdbkubernetesmonitor/api"
 	"github.com/fsnotify/fsnotify"
 	"github.com/go-logr/logr"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
+	"k8s.io/apimachinery/pkg/api/equality"
+	"k8s.io/utils/pointer"
 )
 
 // maxErrorBackoffSeconds is the maximum time to wait after a process fails before starting another process.
@@ -106,6 +106,7 @@ func StartMonitor(ctx context.Context, logger logr.Logger, configFile string, cu
 		Logger:                  logger,
 		CustomEnvironment:       customEnvironment,
 		ProcessCount:            processCount,
+		ProcessIDs:              make([]int, processCount+1),
 		CurrentContainerVersion: currentContainerVersion,
 	}
 
@@ -198,16 +199,13 @@ func checkOwnerExecutable(path string) error {
 func (monitor *Monitor) acceptConfiguration(configuration *api.ProcessConfiguration, configurationBytes []byte) {
 	monitor.Mutex.Lock()
 	defer monitor.Mutex.Unlock()
-	monitor.Logger.Info("Received new configuration file", "configuration", configuration)
 
-	if monitor.ProcessIDs == nil {
-		monitor.ProcessIDs = make([]int, monitor.ProcessCount+1)
-	} else {
-		for len(monitor.ProcessIDs) <= monitor.ProcessCount {
-			monitor.ProcessIDs = append(monitor.ProcessIDs, 0)
-		}
+	// If the configuration hasn't changed ignore those events to prevent noisy logging.
+	if equality.Semantic.DeepEqual(monitor.ActiveConfiguration, configuration) {
+		return
 	}
 
+	monitor.Logger.Info("Received new configuration file", "configuration", configuration)
 	monitor.ActiveConfiguration = configuration
 	monitor.ActiveConfigurationBytes = configurationBytes
 	monitor.LastConfigurationTime = time.Now()
@@ -247,7 +245,7 @@ func (monitor *Monitor) RunProcess(processNumber int) {
 	var errorCounter int
 
 	for {
-		if !monitor.checkProcessRequired(processNumber) {
+		if !monitor.processRequired(processNumber) {
 			return
 		}
 
@@ -343,19 +341,22 @@ func (monitor *Monitor) RunProcess(processNumber int) {
 	}
 }
 
-// checkProcessRequired determines if the latest configuration requires that a
+// processRequired determines if the latest configuration requires that a
 // process stay running.
 // If the process is no longer desired, this will remove it from the process ID
 // list and return false. If the process is still desired, this will return
 // true.
-func (monitor *Monitor) checkProcessRequired(processNumber int) bool {
+func (monitor *Monitor) processRequired(processNumber int) bool {
 	monitor.Mutex.Lock()
 	defer monitor.Mutex.Unlock()
-	logger := monitor.Logger.WithValues("processNumber", processNumber, "area", "checkProcessRequired")
+	logger := monitor.Logger.WithValues("processNumber", processNumber, "area", "processRequired")
 	runProcesses := pointer.BoolDeref(monitor.ActiveConfiguration.RunServers, true)
 	if monitor.ProcessCount < processNumber || !runProcesses {
-		logger.Info("Terminating run loop")
-		monitor.ProcessIDs[processNumber] = 0
+		if monitor.ProcessIDs[processNumber] != 0 {
+			logger.Info("Terminating run loop")
+			monitor.ProcessIDs[processNumber] = 0
+		}
+
 		return false
 	}
 
@@ -409,19 +410,21 @@ func (monitor *Monitor) Run() {
 		// Reset the ProcessCount to 0 to make sure the monitor doesn't try to restart the processes.
 		monitor.ProcessCount = 0
 		for processNumber, processID := range monitor.ProcessIDs {
-			if processID > 0 {
-				subprocessLogger := monitor.Logger.WithValues("processNumber", processNumber, "PID", processID)
-				process, err := os.FindProcess(processID)
-				if err != nil {
-					subprocessLogger.Error(err, "Error finding subprocess")
-					continue
-				}
-				subprocessLogger.Info("Sending signal to subprocess", "signal", latestSignal)
-				err = process.Signal(latestSignal)
-				if err != nil {
-					subprocessLogger.Error(err, "Error signaling subprocess")
-					continue
-				}
+			if processID <= 0 {
+				continue
+			}
+
+			subprocessLogger := monitor.Logger.WithValues("processNumber", processNumber, "PID", processID)
+			process, err := os.FindProcess(processID)
+			if err != nil {
+				subprocessLogger.Error(err, "Error finding subprocess")
+				continue
+			}
+			subprocessLogger.Info("Sending signal to subprocess", "signal", latestSignal)
+			err = process.Signal(latestSignal)
+			if err != nil {
+				subprocessLogger.Error(err, "Error signaling subprocess")
+				continue
 			}
 		}
 

From 3af2e0ad17631acdeef952495a8356088b654469 Mon Sep 17 00:00:00 2001
From: "Johannes M. Scheuermann" <jscheuermann@apple.com>
Date: Wed, 12 Jun 2024 15:02:45 +0200
Subject: [PATCH 438/468] fdbkubernetesmonitor: Update the pod annotation if
 the cluster file changes

---
 fdbkubernetesmonitor/kubernetes.go | 29 ++++++++++++++--
 fdbkubernetesmonitor/main.go       |  4 +--
 fdbkubernetesmonitor/monitor.go    | 56 ++++++++++++++++++++++++++----
 3 files changed, 77 insertions(+), 12 deletions(-)

diff --git a/fdbkubernetesmonitor/kubernetes.go b/fdbkubernetesmonitor/kubernetes.go
index 80e4057f8f4..915f38bd717 100644
--- a/fdbkubernetesmonitor/kubernetes.go
+++ b/fdbkubernetesmonitor/kubernetes.go
@@ -25,6 +25,7 @@ import (
 	"os"
 	"path"
 	"strconv"
+	"time"
 
 	"github.com/apple/foundationdb/fdbkubernetesmonitor/api"
 	"github.com/go-logr/logr"
@@ -55,6 +56,9 @@ const (
 	// The FDB Kubernetes monitor will always shutdown all fdbserver processes, independent of this setting.
 	// The value of this annotation must be a duration like "60s".
 	DelayShutdownAnnotation = "foundationdb.org/delay-shutdown"
+
+	// ClusterFileChangeDetectedAnnotation is the annotation that will be updated if the fdb.cluster file is updated.
+	ClusterFileChangeDetectedAnnotation = "foundationdb.org/cluster-file-change"
 )
 
 // PodClient is a wrapper around the pod API.
@@ -191,12 +195,31 @@ func (podClient *PodClient) UpdateAnnotations(monitor *Monitor) error {
 		return err
 	}
 
+	return podClient.updateAnnotationsOnPod(map[string]string{
+		CurrentConfigurationAnnotation: string(monitor.ActiveConfigurationBytes),
+		EnvironmentAnnotation:          string(jsonEnvironment),
+	})
+}
+
+// updateFdbClusterTimestampAnnotation updates the ClusterFileChangeDetectedAnnotation annotation on the pod
+// after a change to the fdb.cluster file was detected, e.g. because the coordinators were changed.
+func (podClient *PodClient) updateFdbClusterTimestampAnnotation() error {
+	return podClient.updateAnnotationsOnPod(map[string]string{
+		ClusterFileChangeDetectedAnnotation: strconv.FormatInt(time.Now().Unix(), 10),
+	})
+}
+
+// updateAnnotationsOnPod will update the annotations with the provided annotationChanges. If an annotation exists, it
+// will be updated if the annotation is absent it will be added.
+func (podClient *PodClient) updateAnnotationsOnPod(annotationChanges map[string]string) error {
 	annotations := podClient.metadata.Annotations
-	if len(annotations) > 0 {
+	if len(annotations) == 0 {
 		annotations = map[string]string{}
 	}
-	annotations[CurrentConfigurationAnnotation] = string(monitor.ActiveConfigurationBytes)
-	annotations[EnvironmentAnnotation] = string(jsonEnvironment)
+
+	for key, val := range annotationChanges {
+		annotations[key] = val
+	}
 
 	return podClient.Patch(context.Background(), &corev1.Pod{
 		TypeMeta: metav1.TypeMeta{
diff --git a/fdbkubernetesmonitor/main.go b/fdbkubernetesmonitor/main.go
index 404209c934a..e732cbc4409 100644
--- a/fdbkubernetesmonitor/main.go
+++ b/fdbkubernetesmonitor/main.go
@@ -22,9 +22,9 @@ package main
 import (
 	"bufio"
 	"context"
-	"fmt"
 	"io"
 	"os"
+	"path"
 	"regexp"
 	"strings"
 
@@ -132,7 +132,7 @@ func main() {
 			logger.Error(err, "Error loading additional environment")
 			os.Exit(1)
 		}
-		StartMonitor(context.Background(), logger, fmt.Sprintf("%s/%s", inputDir, monitorConfFile), customEnvironment, processCount, listenAddress, enablePprof, currentContainerVersion)
+		StartMonitor(context.Background(), logger, path.Join(inputDir, monitorConfFile), customEnvironment, processCount, listenAddress, enablePprof, currentContainerVersion)
 	case executionModeInit:
 		err = CopyFiles(logger, outputDir, copyDetails, requiredCopies)
 		if err != nil {
diff --git a/fdbkubernetesmonitor/monitor.go b/fdbkubernetesmonitor/monitor.go
index 1662a2fc48e..d3d1286c2dd 100644
--- a/fdbkubernetesmonitor/monitor.go
+++ b/fdbkubernetesmonitor/monitor.go
@@ -23,6 +23,7 @@ import (
 	"bufio"
 	"context"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"io"
 	"net/http"
@@ -43,9 +44,16 @@ import (
 	"k8s.io/utils/pointer"
 )
 
-// maxErrorBackoffSeconds is the maximum time to wait after a process fails before starting another process.
-// The actual delay will be based on the observed errors and will increase until maxErrorBackoffSeconds is hit.
-const maxErrorBackoffSeconds = 60 * time.Second
+const (
+	// maxErrorBackoffSeconds is the maximum time to wait after a process fails before starting another process.
+	// The actual delay will be based on the observed errors and will increase until maxErrorBackoffSeconds is hit.
+	maxErrorBackoffSeconds = 60 * time.Second
+
+	// fdbClusterFilePath defines the default path to the fdb cluster file that contains the current connection string.
+	// This file is managed by the fdbserver processes itself and they will automatically update the file if the
+	// coordinators have changed.
+	fdbClusterFilePath = "/var/fdb/data/fdb.cluster"
+)
 
 // Monitor provides the main monitor loop
 type Monitor struct {
@@ -378,15 +386,16 @@ func (monitor *Monitor) WatchConfiguration(watcher *fsnotify.Watcher) {
 			if !ok {
 				return
 			}
-			monitor.Logger.Info("Detected event on monitor conf file", "event", event)
+
+			monitor.Logger.Info("Detected event on monitor conf file or cluster file", "event", event)
 			if event.Op&fsnotify.Write == fsnotify.Write || event.Op&fsnotify.Create == fsnotify.Create {
-				monitor.LoadConfiguration()
+				monitor.handleFileChange(event.Name)
 			} else if event.Op&fsnotify.Remove == fsnotify.Remove {
-				err := watcher.Add(monitor.ConfigFile)
+				err := watcher.Add(event.Name)
 				if err != nil {
 					panic(err)
 				}
-				monitor.LoadConfiguration()
+				monitor.handleFileChange(event.Name)
 			}
 		case err, ok := <-watcher.Errors:
 			if !ok {
@@ -397,6 +406,19 @@ func (monitor *Monitor) WatchConfiguration(watcher *fsnotify.Watcher) {
 	}
 }
 
+// handleFileChange will perform the required action based on the changed/modified file.
+func (monitor *Monitor) handleFileChange(changedFile string) {
+	if changedFile == fdbClusterFilePath {
+		err := monitor.PodClient.updateFdbClusterTimestampAnnotation()
+		if err != nil {
+			monitor.Logger.Error(err, fmt.Sprintf("could not update %s annotation", ClusterFileChangeDetectedAnnotation))
+		}
+		return
+	}
+
+	monitor.LoadConfiguration()
+}
+
 // Run runs the monitor loop.
 func (monitor *Monitor) Run() {
 	done := make(chan bool, 1)
@@ -447,6 +469,7 @@ func (monitor *Monitor) Run() {
 	if err != nil {
 		panic(err)
 	}
+	monitor.Logger.Info("adding watch for file", "path", path.Base(monitor.ConfigFile))
 	err = watcher.Add(monitor.ConfigFile)
 	if err != nil {
 		panic(err)
@@ -460,6 +483,25 @@ func (monitor *Monitor) Run() {
 	}(watcher)
 	go func() { monitor.WatchConfiguration(watcher) }()
 
+	// The cluster file will be created and managed by the fdbserver processes, so we have to wait until the fdbserver
+	// processes have been started. Except for the initial cluster creation his file should be present as soon as the
+	// monitor starts the processes.
+	for {
+		_, err = os.Stat(fdbClusterFilePath)
+		if errors.Is(err, os.ErrNotExist) {
+			monitor.Logger.Info("waiting for file to be created", "path", fdbClusterFilePath)
+			time.Sleep(5 * time.Second)
+			continue
+		}
+
+		monitor.Logger.Info("adding watch for file", "path", fdbClusterFilePath)
+		err = watcher.Add(fdbClusterFilePath)
+		if err != nil {
+			panic(err)
+		}
+		break
+	}
+
 	<-done
 }
 

From cc63a3d07bc1f5ef70e2c6443fe0385a6b291c19 Mon Sep 17 00:00:00 2001
From: "Johannes M. Scheuermann" <jscheuermann@apple.com>
Date: Tue, 11 Jun 2024 11:59:26 +0200
Subject: [PATCH 439/468] Initial add of custom prometheus metrics

---
 fdbkubernetesmonitor/metrics.go      | 140 ++++++++++++++
 fdbkubernetesmonitor/metrics_test.go | 271 +++++++++++++++++++++++++++
 fdbkubernetesmonitor/monitor.go      |  19 +-
 3 files changed, 429 insertions(+), 1 deletion(-)
 create mode 100644 fdbkubernetesmonitor/metrics.go
 create mode 100644 fdbkubernetesmonitor/metrics_test.go

diff --git a/fdbkubernetesmonitor/metrics.go b/fdbkubernetesmonitor/metrics.go
new file mode 100644
index 00000000000..d652c23f958
--- /dev/null
+++ b/fdbkubernetesmonitor/metrics.go
@@ -0,0 +1,140 @@
+// metrics.go
+//
+// This source file is part of the FoundationDB open source project
+//
+// Copyright 2021-2024 Apple Inc. and the FoundationDB project authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"strconv"
+
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+const (
+	// versionLabel represents the version label for the prometheus metrics.
+	versionLabel = "version"
+	// processLabel represents the process label for the prometheus metrics.
+	processLabel = "process"
+	// namespace is the prometheus namespace for the metrics
+	namespace = "fdbkubernetesmonitor"
+	// restartCountMetricName represents the name of the restart metric.
+	restartCountMetricName = "restart_count"
+	// configurationChangeCountMetricName represents the configuration_change_count metric.
+	configurationChangeCountMetricName = "configuration_change_count"
+	// lastAppliedConfigurationTimestampMetricName represents the last_applied_configuration_timestamp metric.
+	lastAppliedConfigurationTimestampMetricName = "last_applied_configuration_timestamp"
+	// startTimestampMetricName represents the start_timestamp metric.
+	startTimestampMetricName = "start_timestamp"
+	// runningVersionMetricName represents the running_version metric.
+	runningVersionMetricName = "running_version"
+	// desiredVersionMetricName represents the desired_version metric.
+	desiredVersionMetricName = "desired_version"
+)
+
+// metrics represents the custom prometheus metrics for the Monitor.
+type metrics struct {
+	// restartCount represents the total number of fdbserver process restarts.
+	restartCount *prometheus.CounterVec
+	// configurationChangeCount represents the total number of observed configuration changes.
+	configurationChangeCount prometheus.Counter
+	// lastAppliedConfigurationTimestamp provides a unix timestamp when the last configuration was applied.
+	lastAppliedConfigurationTimestamp prometheus.Gauge
+	// startTimestamp represents the timestamp when a specific process was started.
+	startTimestamp *prometheus.GaugeVec
+	// runningVersion represents the current running version of the binaries.
+	runningVersion *prometheus.GaugeVec
+	// desiredVersion represents the desired running version of the binaries.
+	desiredVersion *prometheus.GaugeVec
+	// previousDesiredVersion keeps the previous seen desired version.
+	previousDesiredVersion string
+	// previousRunningVersion keeps the prvious seen running version.
+	previousRunningVersion string
+}
+
+// registerConfigurationChange will update the current prometheus metrics related to configuration changes.
+// This method doesn't use latch as it will only be called from acceptConfiguration which already uses a mutex.
+func (metrics *metrics) registerConfigurationChange(version string) {
+	// Update the prometheus metrics.
+	metrics.lastAppliedConfigurationTimestamp.SetToCurrentTime()
+	metrics.configurationChangeCount.Inc()
+	// Update the desired version for the fdbserver process.
+	metrics.desiredVersion.With(prometheus.Labels{versionLabel: version}).Set(1.0)
+	// If we had a different desired version before we want to make sure to reset this value.
+	if metrics.previousDesiredVersion != "" && metrics.previousDesiredVersion != version {
+		metrics.desiredVersion.With(prometheus.Labels{versionLabel: metrics.previousDesiredVersion}).Set(0.0)
+	}
+	metrics.previousDesiredVersion = version
+}
+
+// registerProcessStartup will update the current prometheus metrics related to process startup and running versions.
+func (metrics *metrics) registerProcessStartup(processNumber int, version string) {
+	castedProcessNumber := strconv.Itoa(processNumber)
+	metrics.restartCount.With(prometheus.Labels{processLabel: castedProcessNumber}).Inc()
+	metrics.startTimestamp.With(prometheus.Labels{processLabel: castedProcessNumber}).SetToCurrentTime()
+	metrics.runningVersion.With(prometheus.Labels{versionLabel: version}).Set(1.0)
+	// If we had a different desired version before we want to make sure to reset this value.
+	if metrics.previousRunningVersion != "" && metrics.previousRunningVersion != version {
+		metrics.runningVersion.With(prometheus.Labels{versionLabel: metrics.previousRunningVersion}).Set(0.0)
+	}
+	metrics.previousRunningVersion = version
+}
+
+// registerMetrics will register the monitor metrics and returns a metrics struct to update the current metrics.
+func registerMetrics(reg prometheus.Registerer) *metrics {
+	monitorMetrics := &metrics{
+		restartCount: prometheus.NewCounterVec(
+			prometheus.CounterOpts{
+				Namespace: namespace,
+				Name:      restartCountMetricName,
+				Help:      "Number of fdbserver process restarts in total.",
+			}, []string{processLabel}),
+		configurationChangeCount: prometheus.NewCounter(prometheus.CounterOpts{
+			Namespace: namespace,
+			Name:      configurationChangeCountMetricName,
+			Help:      "Number of observed configuration changes.",
+		}),
+		lastAppliedConfigurationTimestamp: prometheus.NewGauge(prometheus.GaugeOpts{
+			Namespace: namespace,
+			Name:      lastAppliedConfigurationTimestampMetricName,
+			Help:      "Timestamp when the last time the configuration was applied.",
+		}),
+		startTimestamp: prometheus.NewGaugeVec(prometheus.GaugeOpts{
+			Namespace: namespace,
+			Name:      startTimestampMetricName,
+			Help:      "Timestamp when the last time the configuration was applied.",
+		}, []string{processLabel}),
+		runningVersion: prometheus.NewGaugeVec(prometheus.GaugeOpts{
+			Namespace: namespace,
+			Name:      runningVersionMetricName,
+			Help:      "The current running version of the fdbserver processes started by this monitor.",
+		}, []string{versionLabel}),
+		desiredVersion: prometheus.NewGaugeVec(prometheus.GaugeOpts{
+			Namespace: namespace,
+			Name:      desiredVersionMetricName,
+			Help:      "The desired running version of the fdbserver processes started by this monitor.",
+		}, []string{versionLabel}),
+	}
+
+	reg.MustRegister(monitorMetrics.restartCount)
+	reg.MustRegister(monitorMetrics.configurationChangeCount)
+	reg.MustRegister(monitorMetrics.lastAppliedConfigurationTimestamp)
+	reg.MustRegister(monitorMetrics.startTimestamp)
+	reg.MustRegister(monitorMetrics.runningVersion)
+	reg.MustRegister(monitorMetrics.desiredVersion)
+
+	return monitorMetrics
+}
diff --git a/fdbkubernetesmonitor/metrics_test.go b/fdbkubernetesmonitor/metrics_test.go
new file mode 100644
index 00000000000..8ccb58d2d88
--- /dev/null
+++ b/fdbkubernetesmonitor/metrics_test.go
@@ -0,0 +1,271 @@
+// metrics_test.go
+//
+// This source file is part of the FoundationDB open source project
+//
+// Copyright 2021-2024 Apple Inc. and the FoundationDB project authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"strings"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+	"github.com/prometheus/client_golang/prometheus"
+	"k8s.io/utils/pointer"
+)
+
+var _ = Describe("Testing Monitor metrics", func() {
+	When("getting the copy details", func() {
+		sevenOneVersion := "7.1.57"
+		sevenThreeVersion := "7.3.37"
+		var registry *prometheus.Registry
+		var monitorMetrics *metrics
+
+		BeforeEach(func() {
+			registry = prometheus.NewRegistry()
+			monitorMetrics = registerMetrics(registry)
+		})
+
+		It("shouldn't throw any error", func() {
+			Expect(monitorMetrics).NotTo(BeNil())
+		})
+
+		When("no metrics are added", func() {
+			It("only the counters should be setup", func() {
+				metrics, err := registry.Gather()
+				Expect(err).NotTo(HaveOccurred())
+				Expect(metrics).To(HaveLen(2))
+			})
+		})
+
+		When("a configuration change is registered", func() {
+			BeforeEach(func() {
+				monitorMetrics.registerConfigurationChange(sevenOneVersion)
+			})
+
+			It("should update the metrics", func() {
+				metrics, err := registry.Gather()
+				Expect(err).NotTo(HaveOccurred())
+				Expect(metrics).To(HaveLen(3))
+
+				for _, metric := range metrics {
+					Expect(metric.Metric).To(HaveLen(1))
+					name := pointer.StringDeref(metric.Name, "")
+					if strings.HasSuffix(name, configurationChangeCountMetricName) {
+						Expect(*metric.Metric[0].Counter.Value).To(BeNumerically("==", 1))
+						continue
+					}
+					if strings.HasSuffix(name, desiredVersionMetricName) {
+						Expect(*metric.Metric[0].Gauge.Value).To(BeNumerically("==", 1))
+						Expect(metric.Metric[0].Label).To(HaveLen(1))
+						Expect(*metric.Metric[0].Label[0].Name).To(Equal(versionLabel))
+						Expect(*metric.Metric[0].Label[0].Value).To(Equal(sevenOneVersion))
+						continue
+					}
+				}
+			})
+
+			When("a new configuration change is registered for the same version", func() {
+				BeforeEach(func() {
+					monitorMetrics.registerConfigurationChange(sevenOneVersion)
+				})
+
+				It("should update the metrics", func() {
+					metrics, err := registry.Gather()
+					Expect(err).NotTo(HaveOccurred())
+					Expect(metrics).To(HaveLen(3))
+
+					for _, metric := range metrics {
+						Expect(metric.Metric).To(HaveLen(1))
+						name := pointer.StringDeref(metric.Name, "")
+						if strings.HasSuffix(name, configurationChangeCountMetricName) {
+							Expect(*metric.Metric[0].Counter.Value).To(BeNumerically("==", 2))
+							continue
+						}
+						if strings.HasSuffix(name, desiredVersionMetricName) {
+							Expect(*metric.Metric[0].Gauge.Value).To(BeNumerically("==", 1))
+							Expect(metric.Metric[0].Label).To(HaveLen(1))
+							Expect(*metric.Metric[0].Label[0].Name).To(Equal(versionLabel))
+							Expect(*metric.Metric[0].Label[0].Value).To(Equal(sevenOneVersion))
+							continue
+						}
+					}
+				})
+			})
+
+			When("a new configuration change is registered for a different version", func() {
+				BeforeEach(func() {
+					monitorMetrics.registerConfigurationChange(sevenThreeVersion)
+				})
+
+				It("should update the metrics", func() {
+					metrics, err := registry.Gather()
+					Expect(err).NotTo(HaveOccurred())
+					Expect(metrics).To(HaveLen(3))
+
+					for _, metric := range metrics {
+						name := pointer.StringDeref(metric.Name, "")
+						if strings.HasSuffix(name, desiredVersionMetricName) {
+							Expect(metric.Metric).To(HaveLen(2))
+							for _, versionMetric := range metric.Metric {
+								Expect(versionMetric.Label).To(HaveLen(1))
+								Expect(*versionMetric.Label[0].Name).To(Equal(versionLabel))
+								expected := 0
+								if *versionMetric.Label[0].Value == sevenThreeVersion {
+									expected = 1
+								}
+								Expect(*versionMetric.Gauge.Value).To(BeNumerically("==", expected))
+							}
+							continue
+						}
+
+						Expect(metric.Metric).To(HaveLen(1))
+						if strings.HasSuffix(name, configurationChangeCountMetricName) {
+							Expect(*metric.Metric[0].Counter.Value).To(BeNumerically("==", 2))
+							continue
+						}
+					}
+				})
+			})
+		})
+
+		When("a process startup registered", func() {
+			BeforeEach(func() {
+				monitorMetrics.registerProcessStartup(1, sevenOneVersion)
+			})
+
+			It("should update the metrics", func() {
+				metrics, err := registry.Gather()
+				Expect(err).NotTo(HaveOccurred())
+				Expect(metrics).To(HaveLen(5))
+
+				for _, metric := range metrics {
+					Expect(metric.Metric).To(HaveLen(1))
+					name := pointer.StringDeref(metric.Name, "")
+					if strings.HasSuffix(name, restartCountMetricName) {
+						Expect(*metric.Metric[0].Counter.Value).To(BeNumerically("==", 1))
+						Expect(metric.Metric[0].Label).To(HaveLen(1))
+						Expect(*metric.Metric[0].Label[0].Name).To(Equal(processLabel))
+						Expect(*metric.Metric[0].Label[0].Value).To(Equal("1"))
+						continue
+					}
+					if strings.HasSuffix(name, runningVersionMetricName) {
+						Expect(*metric.Metric[0].Gauge.Value).To(BeNumerically("==", 1))
+						Expect(metric.Metric[0].Label).To(HaveLen(1))
+						Expect(*metric.Metric[0].Label[0].Name).To(Equal(versionLabel))
+						Expect(*metric.Metric[0].Label[0].Value).To(Equal(sevenOneVersion))
+						continue
+					}
+				}
+			})
+
+			When("the same process is restarted with the same version", func() {
+				BeforeEach(func() {
+					monitorMetrics.registerProcessStartup(1, sevenOneVersion)
+				})
+
+				It("should update the metrics", func() {
+					metrics, err := registry.Gather()
+					Expect(err).NotTo(HaveOccurred())
+					Expect(metrics).To(HaveLen(5))
+
+					for _, metric := range metrics {
+						Expect(metric.Metric).To(HaveLen(1))
+						name := pointer.StringDeref(metric.Name, "")
+						if strings.HasSuffix(name, restartCountMetricName) {
+							Expect(*metric.Metric[0].Counter.Value).To(BeNumerically("==", 2))
+							Expect(metric.Metric[0].Label).To(HaveLen(1))
+							Expect(*metric.Metric[0].Label[0].Name).To(Equal(processLabel))
+							Expect(*metric.Metric[0].Label[0].Value).To(Equal("1"))
+							continue
+						}
+						if strings.HasSuffix(name, runningVersionMetricName) {
+							Expect(*metric.Metric[0].Gauge.Value).To(BeNumerically("==", 1))
+							Expect(metric.Metric[0].Label).To(HaveLen(1))
+							Expect(*metric.Metric[0].Label[0].Name).To(Equal(versionLabel))
+							Expect(*metric.Metric[0].Label[0].Value).To(Equal(sevenOneVersion))
+							continue
+						}
+					}
+				})
+			})
+
+			When("the same process is restarted with a different version", func() {
+				BeforeEach(func() {
+					monitorMetrics.registerProcessStartup(1, sevenThreeVersion)
+				})
+
+				It("should update the metrics", func() {
+					metrics, err := registry.Gather()
+					Expect(err).NotTo(HaveOccurred())
+					Expect(metrics).To(HaveLen(5))
+
+					for _, metric := range metrics {
+						name := pointer.StringDeref(metric.Name, "")
+						if strings.HasSuffix(name, runningVersionMetricName) {
+							Expect(metric.Metric).To(HaveLen(2))
+							for _, versionMetric := range metric.Metric {
+								Expect(versionMetric.Label).To(HaveLen(1))
+								Expect(*versionMetric.Label[0].Name).To(Equal(versionLabel))
+								expected := 0
+								if *versionMetric.Label[0].Value == sevenThreeVersion {
+									expected = 1
+								}
+								Expect(*versionMetric.Gauge.Value).To(BeNumerically("==", expected))
+							}
+							continue
+						}
+
+						Expect(metric.Metric).To(HaveLen(1))
+						if strings.HasSuffix(name, restartCountMetricName) {
+							Expect(*metric.Metric[0].Counter.Value).To(BeNumerically("==", 2))
+							Expect(metric.Metric[0].Label).To(HaveLen(1))
+							Expect(*metric.Metric[0].Label[0].Name).To(Equal(processLabel))
+							Expect(*metric.Metric[0].Label[0].Value).To(Equal("1"))
+							continue
+						}
+					}
+				})
+			})
+
+			When("a new process is restarted with a the same version", func() {
+				BeforeEach(func() {
+					monitorMetrics.registerProcessStartup(2, sevenOneVersion)
+				})
+
+				It("should update the metrics", func() {
+					metrics, err := registry.Gather()
+					Expect(err).NotTo(HaveOccurred())
+					Expect(metrics).To(HaveLen(5))
+
+					for _, metric := range metrics {
+						name := pointer.StringDeref(metric.Name, "")
+
+						if strings.HasSuffix(name, restartCountMetricName) {
+							Expect(metric.Metric).To(HaveLen(2))
+							for _, versionMetric := range metric.Metric {
+								Expect(versionMetric.Label).To(HaveLen(1))
+								Expect(*versionMetric.Label[0].Name).To(Equal(processLabel))
+								Expect(*versionMetric.Counter.Value).To(BeNumerically("==", 1))
+							}
+							continue
+						}
+					}
+				})
+			})
+		})
+	})
+})
diff --git a/fdbkubernetesmonitor/monitor.go b/fdbkubernetesmonitor/monitor.go
index d3d1286c2dd..78d7f27fb34 100644
--- a/fdbkubernetesmonitor/monitor.go
+++ b/fdbkubernetesmonitor/monitor.go
@@ -39,6 +39,8 @@ import (
 	"github.com/apple/foundationdb/fdbkubernetesmonitor/api"
 	"github.com/fsnotify/fsnotify"
 	"github.com/go-logr/logr"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/collectors"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
 	"k8s.io/apimachinery/pkg/api/equality"
 	"k8s.io/utils/pointer"
@@ -98,6 +100,9 @@ type Monitor struct {
 
 	// Logger is the logger instance for this monitor.
 	Logger logr.Logger
+
+	// metrics represents the prometheus monitor metrics.
+	metrics *metrics
 }
 
 // StartMonitor starts the monitor loop.
@@ -136,8 +141,15 @@ func StartMonitor(ctx context.Context, logger logr.Logger, configFile string, cu
 		mux.HandleFunc("/debug/pprof/trace", pprof.Trace)
 	}
 
+	reg := prometheus.NewRegistry()
+	// Enable the default go metrics.
+	reg.MustRegister(collectors.NewGoCollector())
+	monitorMetrics := registerMetrics(reg)
+	monitor.metrics = monitorMetrics
+	promHandler := promhttp.HandlerFor(reg, promhttp.HandlerOpts{})
+
 	// Add Prometheus support
-	mux.Handle("/metrics", promhttp.Handler())
+	mux.Handle("/metrics", promHandler)
 	go func() {
 		err := http.ListenAndServe(listenAddr, mux)
 		if err != nil {
@@ -217,6 +229,8 @@ func (monitor *Monitor) acceptConfiguration(configuration *api.ProcessConfigurat
 	monitor.ActiveConfiguration = configuration
 	monitor.ActiveConfigurationBytes = configurationBytes
 	monitor.LastConfigurationTime = time.Now()
+	// Update the prometheus metrics.
+	monitor.metrics.registerConfigurationChange(configuration.Version)
 
 	for processNumber := 1; processNumber <= monitor.ProcessCount; processNumber++ {
 		if monitor.ProcessIDs[processNumber] == 0 {
@@ -297,6 +311,9 @@ func (monitor *Monitor) RunProcess(processNumber int) {
 			continue
 		}
 
+		// Update the prometheus metrics for the process.
+		monitor.metrics.registerProcessStartup(processNumber, monitor.ActiveConfiguration.Version)
+
 		if cmd.Process != nil {
 			pid = cmd.Process.Pid
 		} else {

From 467856031ea338647a8379568da9135aab471f90 Mon Sep 17 00:00:00 2001
From: "Johannes M. Scheuermann" <jscheuermann@apple.com>
Date: Fri, 24 May 2024 15:56:40 +0200
Subject: [PATCH 440/468] Add support in the fdb-kubernetes-monitor to read
 node labels

---
 fdbkubernetesmonitor/api/config.go      |  38 +-
 fdbkubernetesmonitor/api/config_test.go | 562 +++++++++++-------------
 fdbkubernetesmonitor/api/suite_test.go  |  37 ++
 fdbkubernetesmonitor/kubernetes.go      | 138 +++---
 fdbkubernetesmonitor/main.go            |  36 +-
 fdbkubernetesmonitor/monitor.go         |  12 +-
 fdbkubernetesmonitor/suite_test.go      |   2 +-
 7 files changed, 430 insertions(+), 395 deletions(-)
 create mode 100644 fdbkubernetesmonitor/api/suite_test.go

diff --git a/fdbkubernetesmonitor/api/config.go b/fdbkubernetesmonitor/api/config.go
index 07839ccf374..ff4bf9fa958 100644
--- a/fdbkubernetesmonitor/api/config.go
+++ b/fdbkubernetesmonitor/api/config.go
@@ -2,7 +2,7 @@
 //
 // This source file is part of the FoundationDB open source project
 //
-// Copyright 2021 Apple Inc. and the FoundationDB project authors
+// Copyright 2021-2024 Apple Inc. and the FoundationDB project authors
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -21,6 +21,7 @@ package api
 
 import (
 	"fmt"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"net"
 	"os"
 	"strconv"
@@ -57,7 +58,7 @@ type Argument struct {
 	Values []Argument `json:"values,omitempty"`
 
 	// Source provides the name of the environment variable to use for an
-	// Environment type argument.
+	// Environment type argument. This setting is also used for NodeLabelArgumentType.
 	Source string `json:"source,omitempty"`
 
 	// Multiplier provides a multiplier for the process number for ProcessNumber
@@ -93,11 +94,15 @@ const (
 	// IPListArgumentType defines an argument that is a comma-separated list of
 	// IP addresses, provided through an environment variable.
 	IPListArgumentType = "IPList"
+
+	// NodeLabelArgumentType defines an argument that references a label key. The value
+	// of the label will be used as value. The label key must be specified in the Source field.
+	// This ArgumentType requires that the fdb-kubernetes-monitor is able to read the nodes metadata.
+	NodeLabelArgumentType = "NodeLabel"
 )
 
-// GenerateArgument processes an argument and generates its string
-// representation.
-func (argument Argument) GenerateArgument(processNumber int, env map[string]string) (string, error) {
+// GenerateArgument processes an argument and generates its string representation.
+func (argument Argument) GenerateArgument(processNumber int, env map[string]string, node *metav1.PartialObjectMetadata) (string, error) {
 	switch argument.ArgumentType {
 	case "":
 		fallthrough
@@ -106,7 +111,7 @@ func (argument Argument) GenerateArgument(processNumber int, env map[string]stri
 	case ConcatenateArgumentType:
 		concatenated := ""
 		for _, childArgument := range argument.Values {
-			childValue, err := childArgument.GenerateArgument(processNumber, env)
+			childValue, err := childArgument.GenerateArgument(processNumber, env, node)
 			if err != nil {
 				return "", err
 			}
@@ -122,6 +127,17 @@ func (argument Argument) GenerateArgument(processNumber int, env map[string]stri
 		return strconv.Itoa(number), nil
 	case EnvironmentArgumentType, IPListArgumentType:
 		return argument.LookupEnv(env)
+	case NodeLabelArgumentType:
+		if node == nil {
+			return "", fmt.Errorf("no node information present")
+		}
+
+		value, ok := node.Labels[argument.Source]
+		if !ok {
+			return "", fmt.Errorf("label \"%s\" not found", argument.Source)
+		}
+
+		return value, nil
 	default:
 		return "", fmt.Errorf("unsupported argument type %s", argument.ArgumentType)
 	}
@@ -134,9 +150,11 @@ func (argument Argument) LookupEnv(env map[string]string) (string, error) {
 	if env != nil {
 		value, present = env[argument.Source]
 	}
+
 	if !present {
 		value, present = os.LookupEnv(argument.Source)
 	}
+
 	if !present {
 		return "", fmt.Errorf("missing environment variable %s", argument.Source)
 	}
@@ -163,18 +181,18 @@ func (argument Argument) LookupEnv(env map[string]string) (string, error) {
 		}
 		return "", fmt.Errorf("could not find IP with family %d", argument.IPFamily)
 	}
+
 	return value, nil
 }
 
-// GenerateArguments interprets the arguments in the process configuration and
-// generates a command invocation.
-func (configuration *ProcessConfiguration) GenerateArguments(processNumber int, env map[string]string) ([]string, error) {
+// GenerateArguments interprets the arguments in the process configuration and generates a command invocation.
+func (configuration *ProcessConfiguration) GenerateArguments(processNumber int, env map[string]string, node *metav1.PartialObjectMetadata) ([]string, error) {
 	results := make([]string, 0, len(configuration.Arguments)+1)
 	if configuration.BinaryPath != "" {
 		results = append(results, configuration.BinaryPath)
 	}
 	for _, argument := range configuration.Arguments {
-		result, err := argument.GenerateArgument(processNumber, env)
+		result, err := argument.GenerateArgument(processNumber, env, node)
 		if err != nil {
 			return nil, err
 		}
diff --git a/fdbkubernetesmonitor/api/config_test.go b/fdbkubernetesmonitor/api/config_test.go
index 5f80e75f352..c494e3bd13b 100644
--- a/fdbkubernetesmonitor/api/config_test.go
+++ b/fdbkubernetesmonitor/api/config_test.go
@@ -2,7 +2,7 @@
 //
 // This source file is part of the FoundationDB open source project
 //
-// Copyright 2021 Apple Inc. and the FoundationDB project authors
+// Copyright 2021-2024 Apple Inc. and the FoundationDB project authors
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -21,315 +21,279 @@ package api
 
 import (
 	"encoding/json"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"os"
-	"reflect"
-	"testing"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
 )
 
-func loadConfigFromFile(path string) (*ProcessConfiguration, error) {
+func loadConfigFromFile(path string) *ProcessConfiguration {
 	file, err := os.Open(path)
-	if err != nil {
-		return nil, err
-	}
-	defer file.Close()
+	Expect(err).NotTo(HaveOccurred())
+	defer func() {
+		Expect(file.Close()).NotTo(HaveOccurred())
+	}()
 	decoder := json.NewDecoder(file)
 	config := &ProcessConfiguration{}
-	err = decoder.Decode(config)
-	if err != nil {
-		return nil, err
-	}
-	return config, nil
+	Expect(decoder.Decode(config)).NotTo(HaveOccurred())
+
+	return config
 }
 
-func TestGeneratingArgumentsForDefaultConfig(t *testing.T) {
-	config, err := loadConfigFromFile(".testdata/default_config.json")
-	if err != nil {
-		t.Error(err)
-		return
-	}
-
-	arguments, err := config.GenerateArguments(1, map[string]string{
-		"FDB_PUBLIC_IP":   "10.0.0.1",
-		"FDB_POD_IP":      "192.168.0.1",
-		"FDB_ZONE_ID":     "zone1",
-		"FDB_INSTANCE_ID": "storage-1",
+var _ = Describe("Testing FDB Kubernetes monitor API", func() {
+	When("generating arguments for default config", func() {
+		var arguments []string
+
+		BeforeEach(func() {
+			var err error
+			config := loadConfigFromFile(".testdata/default_config.json")
+			arguments, err = config.GenerateArguments(1, map[string]string{
+				"FDB_PUBLIC_IP":   "10.0.0.1",
+				"FDB_POD_IP":      "192.168.0.1",
+				"FDB_ZONE_ID":     "zone1",
+				"FDB_INSTANCE_ID": "storage-1",
+			}, nil)
+			Expect(err).NotTo(HaveOccurred())
+		})
+
+		It("should generate the expected arguments", func() {
+			Expect(arguments).To(HaveExactElements([]string{
+				"--cluster-file", ".testdata/fdb.cluster",
+				"--public-address", "10.0.0.1:4501", "--listen-address", "192.168.0.1:4501",
+				"--datadir", ".testdata/data/1", "--class", "storage",
+				"--locality-zoneid", "zone1", "--locality-instance-id", "storage-1",
+				"--locality-process-id", "storage-1-1",
+			}))
+		})
 	})
-	if err != nil {
-		t.Error(err)
-		return
-	}
-
-	expectedArguments := []string{
-		"--cluster-file", ".testdata/fdb.cluster",
-		"--public-address", "10.0.0.1:4501", "--listen-address", "192.168.0.1:4501",
-		"--datadir", ".testdata/data/1", "--class", "storage",
-		"--locality-zoneid", "zone1", "--locality-instance-id", "storage-1",
-		"--locality-process-id", "storage-1-1",
-	}
-
-	if !reflect.DeepEqual(arguments, expectedArguments) {
-		t.Logf("Expected arguments %v, but got arguments %v", expectedArguments, arguments)
-		t.Fail()
-	}
-
-	config.BinaryPath = "/usr/bin/fdbserver"
-
-	arguments, err = config.GenerateArguments(1, map[string]string{
-		"FDB_PUBLIC_IP":   "10.0.0.1",
-		"FDB_POD_IP":      "192.168.0.1",
-		"FDB_ZONE_ID":     "zone1",
-		"FDB_INSTANCE_ID": "storage-1",
+
+	When("generating arguments for default config with a binary path specified", func() {
+		var arguments []string
+
+		BeforeEach(func() {
+			var err error
+			config := loadConfigFromFile(".testdata/default_config.json")
+			Expect(err).NotTo(HaveOccurred())
+			config.BinaryPath = "/usr/bin/fdbserver"
+
+			arguments, err = config.GenerateArguments(1, map[string]string{
+				"FDB_PUBLIC_IP":   "10.0.0.1",
+				"FDB_POD_IP":      "192.168.0.1",
+				"FDB_ZONE_ID":     "zone1",
+				"FDB_INSTANCE_ID": "storage-1",
+			}, nil)
+			Expect(err).NotTo(HaveOccurred())
+		})
+
+		It("should generate the expected arguments", func() {
+			Expect(arguments).To(HaveExactElements([]string{
+				"/usr/bin/fdbserver",
+				"--cluster-file", ".testdata/fdb.cluster",
+				"--public-address", "10.0.0.1:4501", "--listen-address", "192.168.0.1:4501",
+				"--datadir", ".testdata/data/1", "--class", "storage",
+				"--locality-zoneid", "zone1", "--locality-instance-id", "storage-1",
+				"--locality-process-id", "storage-1-1",
+			}))
+		})
 	})
-	if err != nil {
-		t.Error(err)
-		return
-	}
-
-	expectedArguments = []string{
-		"/usr/bin/fdbserver",
-		"--cluster-file", ".testdata/fdb.cluster",
-		"--public-address", "10.0.0.1:4501", "--listen-address", "192.168.0.1:4501",
-		"--datadir", ".testdata/data/1", "--class", "storage",
-		"--locality-zoneid", "zone1", "--locality-instance-id", "storage-1",
-		"--locality-process-id", "storage-1-1",
-	}
-
-	if !reflect.DeepEqual(arguments, expectedArguments) {
-		t.Logf("Expected arguments %v, but got arguments %v", expectedArguments, arguments)
-		t.Fail()
-	}
-}
 
-func TestGeneratingArgumentForEnvironmentVariable(t *testing.T) {
-	argument := Argument{ArgumentType: EnvironmentArgumentType, Source: "FDB_ZONE_ID"}
-
-	result, err := argument.GenerateArgument(1, map[string]string{"FDB_ZONE_ID": "zone1", "FDB_MACHINE_ID": "machine1"})
-	if err != nil {
-		t.Error(err)
-		return
-	}
-	if result != "zone1" {
-		t.Logf("Expected result zone1, but got result %v", result)
-		t.Fail()
-		return
-	}
-
-	_, err = argument.GenerateArgument(1, map[string]string{"FDB_MACHINE_ID": "machine1"})
-	if err == nil {
-		t.Logf("Expected error result, but did not get an error")
-		t.Fail()
-		return
-	}
-	expectedError := "missing environment variable FDB_ZONE_ID"
-	if err.Error() != expectedError {
-		t.Logf("Expected error %s, but got error %s", expectedError, err)
-		t.Fail()
-		return
-	}
-}
+	When("generating arguments for environment variable", func() {
+		var argument string
+		var err error
+		var env map[string]string
+
+		JustBeforeEach(func() {
+			testArgument := Argument{ArgumentType: EnvironmentArgumentType, Source: "FDB_ZONE_ID"}
+			argument, err = testArgument.GenerateArgument(1, env, nil)
+		})
+
+		When("the env variable is present", func() {
+			BeforeEach(func() {
+				env = map[string]string{"FDB_ZONE_ID": "zone1", "FDB_MACHINE_ID": "machine1"}
+			})
+
+			It("should generate the expected arguments", func() {
+				Expect(err).NotTo(HaveOccurred())
+				Expect(argument).To(Equal("zone1"))
+			})
+		})
+
+		When("the env variable is absent", func() {
+			BeforeEach(func() {
+				env = map[string]string{"FDB_MACHINE_ID": "machine1"}
+			})
+
+			It("should generate the expected arguments", func() {
+				Expect(err).To(HaveOccurred())
+			})
+		})
 
-func TestGeneratingArgumentForIPList(t *testing.T) {
-	argument := Argument{ArgumentType: IPListArgumentType, Source: "FDB_PUBLIC_IP", IPFamily: 4}
-
-	result, err := argument.GenerateArgument(1, map[string]string{"FDB_PUBLIC_IP": "127.0.0.1,::1"})
-	if err != nil {
-		t.Error(err)
-		return
-	}
-	if result != "127.0.0.1" {
-		t.Logf("Expected result 127.0.0.1, but got result %v", result)
-		t.Fail()
-		return
-	}
-
-	result, err = argument.GenerateArgument(1, map[string]string{"FDB_PUBLIC_IP": "::1,127.0.0.1"})
-	if err != nil {
-		t.Error(err)
-		return
-	}
-	if result != "127.0.0.1" {
-		t.Logf("Expected result 127.0.0.1, but got result %v", result)
-		t.Fail()
-		return
-	}
-
-	argument.IPFamily = 6
-
-	result, err = argument.GenerateArgument(1, map[string]string{"FDB_PUBLIC_IP": "127.0.0.1,::1"})
-	if err != nil {
-		t.Error(err)
-		return
-	}
-	if result != "::1" {
-		t.Logf("Expected result ::1, but got result %v", result)
-		t.Fail()
-		return
-	}
-
-	result, err = argument.GenerateArgument(1, map[string]string{"FDB_PUBLIC_IP": "::1,127.0.0.1"})
-	if err != nil {
-		t.Error(err)
-		return
-	}
-	if result != "::1" {
-		t.Logf("Expected result ::1, but got result %v", result)
-		t.Fail()
-		return
-	}
-
-	result, err = argument.GenerateArgument(1, map[string]string{"FDB_PUBLIC_IP": "bad,::1"})
-	if err != nil {
-		t.Error(err)
-		return
-	}
-	if result != "::1" {
-		t.Logf("Expected result ::1, but got result %v", result)
-		t.Fail()
-		return
-	}
-
-	_, err = argument.GenerateArgument(1, map[string]string{"FDB_PUBLIC_IP": "127.0.0.1"})
-	if err == nil {
-		t.Logf("Expected error, but did not get an error")
-		t.Fail()
-		return
-	}
-	expectedError := "could not find IP with family 6"
-	if err.Error() != expectedError {
-		t.Logf("Expected error %s, but got error %s", expectedError, err.Error())
-		t.Fail()
-		return
-	}
-
-	argument.IPFamily = 5
-
-	_, err = argument.GenerateArgument(1, map[string]string{"FDB_PUBLIC_IP": "127.0.0.1"})
-	if err == nil {
-		t.Logf("Expected error, but did not get an error")
-		t.Fail()
-		return
-	}
-	expectedError = "unsupported IP family 5"
-	if err.Error() != expectedError {
-		t.Logf("Expected error %s, but got error %s", expectedError, err.Error())
-		t.Fail()
-		return
-	}
-}
+	})
 
-func TestLookupEnvForEnvironmentVariable(t *testing.T) {
-	argument := Argument{ArgumentType: EnvironmentArgumentType, Source: "FDB_ZONE_ID"}
-
-	result, err := argument.LookupEnv(map[string]string{"FDB_ZONE_ID": "zone1", "FDB_MACHINE_ID": "machine1"})
-	if err != nil {
-		t.Error(err)
-		return
-	}
-	if result != "zone1" {
-		t.Logf("Expected result zone1, but got result %v", result)
-		t.Fail()
-		return
-	}
-
-	_, err = argument.LookupEnv(map[string]string{"FDB_MACHINE_ID": "machine1"})
-	if err == nil {
-		t.Logf("Expected error result, but did not get an error")
-		t.Fail()
-		return
-	}
-	expectedError := "missing environment variable FDB_ZONE_ID"
-	if err.Error() != expectedError {
-		t.Logf("Expected error %s, but got error %s", expectedError, err)
-		t.Fail()
-		return
-	}
-}
+	When("generating arguments for IPList arguments", func() {
+		var argument string
+		var err error
+		var env map[string]string
+		var IPFamily int
+
+		JustBeforeEach(func() {
+			testArgument := Argument{ArgumentType: IPListArgumentType, Source: "FDB_PUBLIC_IP", IPFamily: IPFamily}
+			argument, err = testArgument.GenerateArgument(1, env, nil)
+		})
+
+		When("using IP Family 4", func() {
+			BeforeEach(func() {
+				IPFamily = 4
+			})
+
+			When("the env variable is present and has one address with the right IP family", func() {
+				BeforeEach(func() {
+					env = map[string]string{"FDB_PUBLIC_IP": "127.0.0.1,::1"}
+				})
+
+				It("should generate the expected arguments", func() {
+					Expect(err).NotTo(HaveOccurred())
+					Expect(argument).To(Equal("127.0.0.1"))
+				})
+			})
+
+			When("the env variable is present and has one address with the right IP family with IPv6 first", func() {
+				BeforeEach(func() {
+					env = map[string]string{"FDB_PUBLIC_IP": "::1,127.0.0.1"}
+				})
+
+				It("should generate the expected arguments", func() {
+					Expect(err).NotTo(HaveOccurred())
+					Expect(argument).To(Equal("127.0.0.1"))
+				})
+			})
+
+			When("no IPv4 address is present", func() {
+				BeforeEach(func() {
+					env = map[string]string{"FDB_PUBLIC_IP": "::1"}
+				})
+
+				It("should return an error", func() {
+					Expect(err).To(HaveOccurred())
+					Expect(err.Error()).To(Equal("could not find IP with family 4"))
+					Expect(argument).To(BeEmpty())
+				})
+			})
+		})
+
+		When("using IP Family 6", func() {
+			BeforeEach(func() {
+				IPFamily = 6
+			})
+
+			When("the env variable is present and has one address with the right IP family", func() {
+				BeforeEach(func() {
+					env = map[string]string{"FDB_PUBLIC_IP": "127.0.0.1,::1"}
+				})
+
+				It("should generate the expected arguments", func() {
+					Expect(err).NotTo(HaveOccurred())
+					Expect(argument).To(Equal("::1"))
+				})
+			})
+
+			When("the env variable is present and has one address with the right IP family with IPv6 first", func() {
+				BeforeEach(func() {
+
+					env = map[string]string{"FDB_PUBLIC_IP": "::1,127.0.0.1"}
+				})
+
+				It("should generate the expected arguments", func() {
+					Expect(err).NotTo(HaveOccurred())
+					Expect(argument).To(Equal("::1"))
+				})
+			})
+
+			When("a bad address is present", func() {
+				BeforeEach(func() {
+					env = map[string]string{"FDB_PUBLIC_IP": "bad,::1"}
+				})
+
+				It("should generate the expected arguments", func() {
+					Expect(err).NotTo(HaveOccurred())
+					Expect(argument).To(Equal("::1"))
+				})
+			})
+
+			When("no IPv6 address is present", func() {
+				BeforeEach(func() {
+					env = map[string]string{"FDB_PUBLIC_IP": "127.0.0.1"}
+				})
+
+				It("return an error", func() {
+					Expect(err).To(HaveOccurred())
+					Expect(err.Error()).To(Equal("could not find IP with family 6"))
+					Expect(argument).To(BeEmpty())
+				})
+			})
+		})
+
+		When("using an invalid IP Family", func() {
+			BeforeEach(func() {
+				IPFamily = 5
+			})
+
+			It("should return an error", func() {
+				Expect(err).To(HaveOccurred())
+				Expect(err.Error()).To(Equal("unsupported IP family 5"))
+				Expect(argument).To(BeEmpty())
+			})
+		})
+	})
 
-func TestLookupEnvForIPList(t *testing.T) {
-	argument := Argument{ArgumentType: IPListArgumentType, Source: "FDB_PUBLIC_IP", IPFamily: 4}
-
-	result, err := argument.LookupEnv(map[string]string{"FDB_PUBLIC_IP": "127.0.0.1,::1"})
-	if err != nil {
-		t.Error(err)
-		return
-	}
-	if result != "127.0.0.1" {
-		t.Logf("Expected result 127.0.0.1, but got result %v", result)
-		t.Fail()
-		return
-	}
-
-	result, err = argument.LookupEnv(map[string]string{"FDB_PUBLIC_IP": "::1,127.0.0.1"})
-	if err != nil {
-		t.Error(err)
-		return
-	}
-	if result != "127.0.0.1" {
-		t.Logf("Expected result 127.0.0.1, but got result %v", result)
-		t.Fail()
-		return
-	}
-
-	argument.IPFamily = 6
-
-	result, err = argument.LookupEnv(map[string]string{"FDB_PUBLIC_IP": "127.0.0.1,::1"})
-	if err != nil {
-		t.Error(err)
-		return
-	}
-	if result != "::1" {
-		t.Logf("Expected result ::1, but got result %v", result)
-		t.Fail()
-		return
-	}
-
-	result, err = argument.LookupEnv(map[string]string{"FDB_PUBLIC_IP": "::1,127.0.0.1"})
-	if err != nil {
-		t.Error(err)
-		return
-	}
-	if result != "::1" {
-		t.Logf("Expected result ::1, but got result %v", result)
-		t.Fail()
-		return
-	}
-
-	result, err = argument.LookupEnv(map[string]string{"FDB_PUBLIC_IP": "bad,::1"})
-	if err != nil {
-		t.Error(err)
-		return
-	}
-	if result != "::1" {
-		t.Logf("Expected result ::1, but got result %v", result)
-		t.Fail()
-		return
-	}
-
-	_, err = argument.LookupEnv(map[string]string{"FDB_PUBLIC_IP": "127.0.0.1"})
-	if err == nil {
-		t.Logf("Expected error, but did not get an error")
-		t.Fail()
-		return
-	}
-	expectedError := "could not find IP with family 6"
-	if err.Error() != expectedError {
-		t.Logf("Expected error %s, but got error %s", expectedError, err.Error())
-		t.Fail()
-		return
-	}
-
-	argument.IPFamily = 5
-
-	_, err = argument.LookupEnv(map[string]string{"FDB_PUBLIC_IP": "127.0.0.1"})
-	if err == nil {
-		t.Logf("Expected error, but did not get an error")
-		t.Fail()
-		return
-	}
-	expectedError = "unsupported IP family 5"
-	if err.Error() != expectedError {
-		t.Logf("Expected error %s, but got error %s", expectedError, err.Error())
-		t.Fail()
-		return
-	}
-}
+	When("generating an argument with a node label", func() {
+		var argument string
+		var err error
+		var node *metav1.PartialObjectMetadata
+
+		JustBeforeEach(func() {
+			testArgument := Argument{ArgumentType: NodeLabelArgumentType, Source: "foundationdb.org/zone"}
+			argument, err = testArgument.GenerateArgument(1, nil, node)
+		})
+
+		When("the node is not present", func() {
+			It("should return an error", func() {
+				Expect(err).To(HaveOccurred())
+				Expect(err.Error()).To(Equal("no node information present"))
+				Expect(argument).To(BeEmpty())
+			})
+		})
+
+		When("the node is present but the label is missing", func() {
+			BeforeEach(func() {
+				node = &metav1.PartialObjectMetadata{
+					ObjectMeta: metav1.ObjectMeta{},
+				}
+			})
+
+			It("should return an error", func() {
+				Expect(err).To(HaveOccurred())
+				Expect(err.Error()).To(Equal("label \"foundationdb.org/zone\" not found"))
+				Expect(argument).To(BeEmpty())
+			})
+		})
+
+		When("the node and the label is present", func() {
+			BeforeEach(func() {
+				node = &metav1.PartialObjectMetadata{
+					ObjectMeta: metav1.ObjectMeta{
+						Labels: map[string]string{
+							"foundationdb.org/zone": "awesome",
+						},
+					},
+				}
+			})
+
+			It("should return an error", func() {
+				Expect(err).NotTo(HaveOccurred())
+				Expect(argument).To(Equal("awesome"))
+			})
+		})
+	})
+})
diff --git a/fdbkubernetesmonitor/api/suite_test.go b/fdbkubernetesmonitor/api/suite_test.go
new file mode 100644
index 00000000000..eb6f12252eb
--- /dev/null
+++ b/fdbkubernetesmonitor/api/suite_test.go
@@ -0,0 +1,37 @@
+/*
+ * suite_test.go
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2023-2024 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package api
+
+import (
+	"testing"
+	"time"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+// These tests use Ginkgo (BDD-style Go testing framework). Refer to
+// http://onsi.github.io/ginkgo/ to learn more about Ginkgo.
+func TestAPIs(t *testing.T) {
+	RegisterFailHandler(Fail)
+	SetDefaultEventuallyTimeout(10 * time.Second)
+	RunSpecs(t, "FDB Kubernetes Monitor API")
+}
diff --git a/fdbkubernetesmonitor/kubernetes.go b/fdbkubernetesmonitor/kubernetes.go
index 915f38bd717..fe3dfd9dbd8 100644
--- a/fdbkubernetesmonitor/kubernetes.go
+++ b/fdbkubernetesmonitor/kubernetes.go
@@ -63,8 +63,11 @@ const (
 
 // PodClient is a wrapper around the pod API.
 type PodClient struct {
-	// metadata is the latest metadata that was seen by the fdb-kubernetes-monitor for the according Pod.
-	metadata *metav1.PartialObjectMetadata
+	// podMetadata is the latest metadata that was seen by the fdb-kubernetes-monitor for the according Pod.
+	podMetadata *metav1.PartialObjectMetadata
+
+	// nodeMetadata is the latest metadata that was seen by the fdb-kubernetes-monitor for the according node that hosts the Pod.
+	nodeMetadata *metav1.PartialObjectMetadata
 
 	// TimestampFeed is a channel where the pod client will send updates with
 	// the values from OutdatedConfigMapAnnotation.
@@ -78,7 +81,7 @@ type PodClient struct {
 }
 
 // CreatePodClient creates a new client for working with the pod object.
-func CreatePodClient(ctx context.Context, logger logr.Logger) (*PodClient, error) {
+func CreatePodClient(ctx context.Context, logger logr.Logger, enableNodeWatcher bool) (*PodClient, error) {
 	config, err := rest.InClusterConfig()
 	if err != nil {
 		return nil, err
@@ -100,6 +103,7 @@ func CreatePodClient(ctx context.Context, logger logr.Logger) (*PodClient, error
 
 	namespace := os.Getenv("FDB_POD_NAMESPACE")
 	podName := os.Getenv("FDB_POD_NAME")
+	nodeName := os.Getenv("FDB_NODE_NAME")
 
 	internalCache, err := cache.New(config, cache.Options{
 		Scheme:    scheme,
@@ -109,6 +113,9 @@ func CreatePodClient(ctx context.Context, logger logr.Logger) (*PodClient, error
 			&corev1.Pod{}: {
 				Field: fields.OneTermEqualSelector(metav1.ObjectNameField, podName),
 			},
+			&corev1.Node{}: {
+				Field: fields.OneTermEqualSelector(metav1.ObjectNameField, nodeName),
+			},
 		},
 	})
 	if err != nil {
@@ -122,7 +129,8 @@ func CreatePodClient(ctx context.Context, logger logr.Logger) (*PodClient, error
 	}
 
 	podClient := &PodClient{
-		metadata:      nil,
+		podMetadata:   nil,
+		nodeMetadata:  nil,
 		TimestampFeed: make(chan int64, 10),
 		Logger:        logger,
 	}
@@ -154,14 +162,27 @@ func CreatePodClient(ctx context.Context, logger logr.Logger) (*PodClient, error
 	podClient.Client = controllerClient
 
 	// Fetch the current metadata before returning the PodClient
-	currentMetadata := &metav1.PartialObjectMetadata{}
-	currentMetadata.SetGroupVersionKind(corev1.SchemeGroupVersion.WithKind("Pod"))
-	err = podClient.Client.Get(ctx, client.ObjectKey{Namespace: namespace, Name: podName}, currentMetadata)
+	currentPodMetadata := &metav1.PartialObjectMetadata{}
+	currentPodMetadata.SetGroupVersionKind(corev1.SchemeGroupVersion.WithKind("Pod"))
+	err = podClient.Client.Get(ctx, client.ObjectKey{Namespace: namespace, Name: podName}, currentPodMetadata)
 	if err != nil {
 		return nil, err
 	}
 
-	podClient.metadata = currentMetadata
+	podClient.podMetadata = currentPodMetadata
+
+	// Only if the fdb-kubernetes-monitor should update the node information, add the watcher here by fetching the node
+	// information once during start up.
+	if enableNodeWatcher {
+		currentNodeMetadata := &metav1.PartialObjectMetadata{}
+		currentNodeMetadata.SetGroupVersionKind(corev1.SchemeGroupVersion.WithKind("Node"))
+		err = podClient.Client.Get(ctx, client.ObjectKey{Name: nodeName}, currentNodeMetadata)
+		if err != nil {
+			return nil, err
+		}
+
+		podClient.nodeMetadata = currentNodeMetadata
+	}
 
 	return podClient, nil
 }
@@ -212,8 +233,8 @@ func (podClient *PodClient) updateFdbClusterTimestampAnnotation() error {
 // updateAnnotationsOnPod will update the annotations with the provided annotationChanges. If an annotation exists, it
 // will be updated if the annotation is absent it will be added.
 func (podClient *PodClient) updateAnnotationsOnPod(annotationChanges map[string]string) error {
-	annotations := podClient.metadata.Annotations
-	if len(annotations) == 0 {
+	annotations := podClient.podMetadata.Annotations
+	if len(annotations) > 0 {
 		annotations = map[string]string{}
 	}
 
@@ -227,8 +248,8 @@ func (podClient *PodClient) updateAnnotationsOnPod(annotationChanges map[string]
 			APIVersion: "v1",
 		},
 		ObjectMeta: metav1.ObjectMeta{
-			Namespace:   podClient.metadata.Namespace,
-			Name:        podClient.metadata.Name,
+			Namespace:   podClient.podMetadata.Namespace,
+			Name:        podClient.podMetadata.Name,
 			Annotations: annotations,
 		},
 	}, client.Apply, client.FieldOwner("fdb-kubernetes-monitor"), client.ForceOwnership)
@@ -236,53 +257,57 @@ func (podClient *PodClient) updateAnnotationsOnPod(annotationChanges map[string]
 
 // OnAdd is called when an object is added.
 func (podClient *PodClient) OnAdd(obj interface{}) {
-	pod, ok := obj.(*corev1.Pod)
-	if !ok {
-		return
-	}
-
-	podClient.Logger.Info("Got event for OnAdd", "name", pod.Name, "namespace", pod.Namespace)
-	podClient.metadata = &metav1.PartialObjectMetadata{
-		TypeMeta:   pod.TypeMeta,
-		ObjectMeta: pod.ObjectMeta,
+	switch castedObj := obj.(type) {
+	case *corev1.Pod:
+		podClient.Logger.Info("Got event for OnAdd for Pod resource", "name", castedObj.Name, "namespace", castedObj.Namespace)
+		podClient.podMetadata = &metav1.PartialObjectMetadata{
+			TypeMeta:   castedObj.TypeMeta,
+			ObjectMeta: castedObj.ObjectMeta,
+		}
+	case *corev1.Node:
+		podClient.Logger.Info("Got event for OnAdd Node resource", "name", castedObj.Name)
+		podClient.podMetadata = &metav1.PartialObjectMetadata{
+			TypeMeta:   castedObj.TypeMeta,
+			ObjectMeta: castedObj.ObjectMeta,
+		}
 	}
 }
 
-// OnUpdate is called when an object is modified. Note that oldObj is the
-// last known state of the object-- it is possible that several changes
-// were combined together, so you can't use this to see every single
-// change. OnUpdate is also called when a re-list happens, and it will
+// OnUpdate is also called when a re-list happens, and it will
 // get called even if nothing changed. This is useful for periodically
 // evaluating or syncing something.
 func (podClient *PodClient) OnUpdate(_, newObj interface{}) {
-	pod, ok := newObj.(*corev1.Pod)
-	if !ok {
-		return
-	}
-
-	podClient.Logger.Info("Got event for OnUpdate", "name", pod.Name, "namespace", pod.Namespace, "generation", pod.Generation)
+	switch castedObj := newObj.(type) {
+	case *corev1.Pod:
+		podClient.Logger.Info("Got event for OnUpdate for Pod resource", "name", castedObj.Name, "namespace", castedObj.Namespace, "generation", castedObj.Generation)
+		podClient.podMetadata = &metav1.PartialObjectMetadata{
+			TypeMeta:   castedObj.TypeMeta,
+			ObjectMeta: castedObj.ObjectMeta,
+		}
 
-	podClient.metadata = &metav1.PartialObjectMetadata{
-		TypeMeta:   pod.TypeMeta,
-		ObjectMeta: pod.ObjectMeta,
-	}
+		if podClient.podMetadata.Annotations == nil {
+			return
+		}
 
-	if podClient.metadata == nil {
-		return
-	}
+		annotation := podClient.podMetadata.Annotations[OutdatedConfigMapAnnotation]
+		if annotation == "" {
+			return
+		}
 
-	annotation := podClient.metadata.Annotations[OutdatedConfigMapAnnotation]
-	if annotation == "" {
-		return
-	}
+		timestamp, err := strconv.ParseInt(annotation, 10, 64)
+		if err != nil {
+			podClient.Logger.Error(err, "Error parsing annotation", "key", OutdatedConfigMapAnnotation, "rawAnnotation", annotation)
+			return
+		}
 
-	timestamp, err := strconv.ParseInt(annotation, 10, 64)
-	if err != nil {
-		podClient.Logger.Error(err, "Error parsing annotation", "key", OutdatedConfigMapAnnotation, "rawAnnotation", annotation)
-		return
+		podClient.TimestampFeed <- timestamp
+	case *corev1.Node:
+		podClient.Logger.Info("Got event for OnUpdate Node resource", "name", castedObj.Name)
+		podClient.podMetadata = &metav1.PartialObjectMetadata{
+			TypeMeta:   castedObj.TypeMeta,
+			ObjectMeta: castedObj.ObjectMeta,
+		}
 	}
-
-	podClient.TimestampFeed <- timestamp
 }
 
 // OnDelete will get the final state of the item if it is known, otherwise
@@ -290,15 +315,12 @@ func (podClient *PodClient) OnUpdate(_, newObj interface{}) {
 // happen if the watch is closed and misses the delete event and we don't
 // notice the deletion until the subsequent re-list.
 func (podClient *PodClient) OnDelete(obj interface{}) {
-	pod, ok := obj.(*corev1.Pod)
-	if !ok {
-		return
-	}
-
-	podClient.Logger.Info("Got event for OnDelete", "name", pod.Name, "namespace", pod.Namespace)
-
-	podClient.metadata = &metav1.PartialObjectMetadata{
-		TypeMeta:   pod.TypeMeta,
-		ObjectMeta: pod.ObjectMeta,
+	switch castedObj := obj.(type) {
+	case *corev1.Pod:
+		podClient.Logger.Info("Got event for OnDelete for Pod resource", "name", castedObj.Name, "namespace", castedObj.Namespace)
+		podClient.podMetadata = nil
+	case *corev1.Node:
+		podClient.Logger.Info("Got event for OnDelete Node resource", "name", castedObj.Name)
+		podClient.podMetadata = nil
 	}
 }
diff --git a/fdbkubernetesmonitor/main.go b/fdbkubernetesmonitor/main.go
index e732cbc4409..6c57963e379 100644
--- a/fdbkubernetesmonitor/main.go
+++ b/fdbkubernetesmonitor/main.go
@@ -37,26 +37,19 @@ import (
 )
 
 var (
-	inputDir                string
-	fdbserverPath           string
-	versionFilePath         string
-	sharedBinaryDir         string
-	monitorConfFile         string
-	logPath                 string
-	executionModeString     string
-	outputDir               string
-	mainContainerVersion    string
-	currentContainerVersion string
-	additionalEnvFile       string
-	binaryOutputDirectory   string
-	listenAddress           string
-	copyPrimaryLibrary      string
-	requiredCopyFiles       []string
-	copyFiles               []string
-	copyBinaries            []string
-	copyLibraries           []string
-	processCount            int
-	enablePprof             bool
+	fdbserverPath        string
+	versionFilePath      string
+	sharedBinaryDir      string
+	monitorConfFile      string
+	logPath              string
+	executionModeString  string
+	outputDir            string
+	mainContainerVersion string
+	additionalEnvFile    string
+	listenAddress        string
+	processCount         int
+	enablePprof          bool
+	enableNodeWatcher    bool
 )
 
 type executionMode string
@@ -109,6 +102,7 @@ func main() {
 	pflag.IntVar(&processCount, "process-count", 1, "The number of processes to start")
 	pflag.BoolVar(&enablePprof, "enable-pprof", false, "Enables /debug/pprof endpoints on the listen address")
 	pflag.StringVar(&listenAddress, "listen-address", ":8081", "An address and port to listen on")
+	pflag.BoolVar(&enableNodeWatcher, "enable-node-node", false, "Enables the fdb-kubernetes-monitor to watch the node resource where the current Pod is running. This can be used to read node labels")
 	pflag.Parse()
 
 	logger := zapr.NewLogger(initLogger(logPath))
@@ -132,7 +126,7 @@ func main() {
 			logger.Error(err, "Error loading additional environment")
 			os.Exit(1)
 		}
-		StartMonitor(context.Background(), logger, path.Join(inputDir, monitorConfFile), customEnvironment, processCount, listenAddress, enablePprof, currentContainerVersion)
+		StartMonitor(context.Background(), logger, path.Join(inputDir, monitorConfFile), customEnvironment, processCount, listenAddress, enablePprof, currentContainerVersion, enableNodeWatcher)
 	case executionModeInit:
 		err = CopyFiles(logger, outputDir, copyDetails, requiredCopies)
 		if err != nil {
diff --git a/fdbkubernetesmonitor/monitor.go b/fdbkubernetesmonitor/monitor.go
index 78d7f27fb34..3d7fd37ae60 100644
--- a/fdbkubernetesmonitor/monitor.go
+++ b/fdbkubernetesmonitor/monitor.go
@@ -2,7 +2,7 @@
 //
 // This source file is part of the FoundationDB open source project
 //
-// Copyright 2021 Apple Inc. and the FoundationDB project authors
+// Copyright 2021-2024 Apple Inc. and the FoundationDB project authors
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -106,8 +106,8 @@ type Monitor struct {
 }
 
 // StartMonitor starts the monitor loop.
-func StartMonitor(ctx context.Context, logger logr.Logger, configFile string, customEnvironment map[string]string, processCount int, listenAddr string, enableDebug bool, currentContainerVersion string) {
-	podClient, err := CreatePodClient(ctx, logger)
+func StartMonitor(ctx context.Context, logger logr.Logger, configFile string, customEnvironment map[string]string, processCount int, listenAddr string, enableDebug bool, currentContainerVersion string, enableNodeWatcher bool) {
+	podClient, err := CreatePodClient(ctx, logger, enableNodeWatcher)
 	if err != nil {
 		logger.Error(err, "could not create Pod client")
 		os.Exit(1)
@@ -192,7 +192,7 @@ func (monitor *Monitor) LoadConfiguration() {
 		return
 	}
 
-	_, err = configuration.GenerateArguments(1, monitor.CustomEnvironment)
+	_, err = configuration.GenerateArguments(1, monitor.CustomEnvironment, monitor.PodClient.nodeMetadata)
 	if err != nil {
 		monitor.Logger.Error(err, "Error generating arguments for latest configuration", "configuration", configuration, "binaryPath", configuration.BinaryPath)
 		return
@@ -277,7 +277,7 @@ func (monitor *Monitor) RunProcess(processNumber int) {
 			errorCounter = 0
 		}
 
-		arguments, err := monitor.ActiveConfiguration.GenerateArguments(processNumber, monitor.CustomEnvironment)
+		arguments, err := monitor.ActiveConfiguration.GenerateArguments(processNumber, monitor.CustomEnvironment, monitor.PodClient.nodeMetadata)
 		if err != nil {
 			backoffDuration := getBackoffDuration(errorCounter)
 			logger.Error(err, "Error generating arguments for subprocess", "configuration", monitor.ActiveConfiguration, "errorCounter", errorCounter, "backoffDuration", backoffDuration.String())
@@ -467,7 +467,7 @@ func (monitor *Monitor) Run() {
 			}
 		}
 
-		annotations := monitor.PodClient.metadata.Annotations
+		annotations := monitor.PodClient.podMetadata.Annotations
 		if len(annotations) > 0 {
 			delayValue, ok := annotations[DelayShutdownAnnotation]
 			if ok {
diff --git a/fdbkubernetesmonitor/suite_test.go b/fdbkubernetesmonitor/suite_test.go
index 3883bc7a8f5..97f1d56e587 100644
--- a/fdbkubernetesmonitor/suite_test.go
+++ b/fdbkubernetesmonitor/suite_test.go
@@ -3,7 +3,7 @@
  *
  * This source file is part of the FoundationDB open source project
  *
- * Copyright 2023 Apple Inc. and the FoundationDB project authors
+ * Copyright 2023-2024 Apple Inc. and the FoundationDB project authors
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.

From 186f2f4bd34549441a1c1e0dc31151a9fb1d4e65 Mon Sep 17 00:00:00 2001
From: "Johannes M. Scheuermann" <jscheuermann@apple.com>
Date: Fri, 24 May 2024 17:01:52 +0200
Subject: [PATCH 441/468] Restructure approach to be compatible with operator
 setup

---
 fdbkubernetesmonitor/api/config.go      | 25 ++--------
 fdbkubernetesmonitor/api/config_test.go | 61 ++-----------------------
 fdbkubernetesmonitor/monitor.go         | 46 +++++++++++++++++--
 fdbkubernetesmonitor/monitor_test.go    | 50 +++++++++++++++++++-
 4 files changed, 98 insertions(+), 84 deletions(-)

diff --git a/fdbkubernetesmonitor/api/config.go b/fdbkubernetesmonitor/api/config.go
index ff4bf9fa958..abfa6b14122 100644
--- a/fdbkubernetesmonitor/api/config.go
+++ b/fdbkubernetesmonitor/api/config.go
@@ -21,7 +21,6 @@ package api
 
 import (
 	"fmt"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"net"
 	"os"
 	"strconv"
@@ -94,15 +93,10 @@ const (
 	// IPListArgumentType defines an argument that is a comma-separated list of
 	// IP addresses, provided through an environment variable.
 	IPListArgumentType = "IPList"
-
-	// NodeLabelArgumentType defines an argument that references a label key. The value
-	// of the label will be used as value. The label key must be specified in the Source field.
-	// This ArgumentType requires that the fdb-kubernetes-monitor is able to read the nodes metadata.
-	NodeLabelArgumentType = "NodeLabel"
 )
 
 // GenerateArgument processes an argument and generates its string representation.
-func (argument Argument) GenerateArgument(processNumber int, env map[string]string, node *metav1.PartialObjectMetadata) (string, error) {
+func (argument Argument) GenerateArgument(processNumber int, env map[string]string) (string, error) {
 	switch argument.ArgumentType {
 	case "":
 		fallthrough
@@ -111,7 +105,7 @@ func (argument Argument) GenerateArgument(processNumber int, env map[string]stri
 	case ConcatenateArgumentType:
 		concatenated := ""
 		for _, childArgument := range argument.Values {
-			childValue, err := childArgument.GenerateArgument(processNumber, env, node)
+			childValue, err := childArgument.GenerateArgument(processNumber, env)
 			if err != nil {
 				return "", err
 			}
@@ -127,17 +121,6 @@ func (argument Argument) GenerateArgument(processNumber int, env map[string]stri
 		return strconv.Itoa(number), nil
 	case EnvironmentArgumentType, IPListArgumentType:
 		return argument.LookupEnv(env)
-	case NodeLabelArgumentType:
-		if node == nil {
-			return "", fmt.Errorf("no node information present")
-		}
-
-		value, ok := node.Labels[argument.Source]
-		if !ok {
-			return "", fmt.Errorf("label \"%s\" not found", argument.Source)
-		}
-
-		return value, nil
 	default:
 		return "", fmt.Errorf("unsupported argument type %s", argument.ArgumentType)
 	}
@@ -186,13 +169,13 @@ func (argument Argument) LookupEnv(env map[string]string) (string, error) {
 }
 
 // GenerateArguments interprets the arguments in the process configuration and generates a command invocation.
-func (configuration *ProcessConfiguration) GenerateArguments(processNumber int, env map[string]string, node *metav1.PartialObjectMetadata) ([]string, error) {
+func (configuration *ProcessConfiguration) GenerateArguments(processNumber int, env map[string]string) ([]string, error) {
 	results := make([]string, 0, len(configuration.Arguments)+1)
 	if configuration.BinaryPath != "" {
 		results = append(results, configuration.BinaryPath)
 	}
 	for _, argument := range configuration.Arguments {
-		result, err := argument.GenerateArgument(processNumber, env, node)
+		result, err := argument.GenerateArgument(processNumber, env)
 		if err != nil {
 			return nil, err
 		}
diff --git a/fdbkubernetesmonitor/api/config_test.go b/fdbkubernetesmonitor/api/config_test.go
index c494e3bd13b..a5626d8c8b1 100644
--- a/fdbkubernetesmonitor/api/config_test.go
+++ b/fdbkubernetesmonitor/api/config_test.go
@@ -21,7 +21,6 @@ package api
 
 import (
 	"encoding/json"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"os"
 
 	. "github.com/onsi/ginkgo/v2"
@@ -41,7 +40,7 @@ func loadConfigFromFile(path string) *ProcessConfiguration {
 	return config
 }
 
-var _ = Describe("Testing FDB Kubernetes monitor API", func() {
+var _ = Describe("Testing FDB Kubernetes Monitor API", func() {
 	When("generating arguments for default config", func() {
 		var arguments []string
 
@@ -53,7 +52,7 @@ var _ = Describe("Testing FDB Kubernetes monitor API", func() {
 				"FDB_POD_IP":      "192.168.0.1",
 				"FDB_ZONE_ID":     "zone1",
 				"FDB_INSTANCE_ID": "storage-1",
-			}, nil)
+			})
 			Expect(err).NotTo(HaveOccurred())
 		})
 
@@ -82,7 +81,7 @@ var _ = Describe("Testing FDB Kubernetes monitor API", func() {
 				"FDB_POD_IP":      "192.168.0.1",
 				"FDB_ZONE_ID":     "zone1",
 				"FDB_INSTANCE_ID": "storage-1",
-			}, nil)
+			})
 			Expect(err).NotTo(HaveOccurred())
 		})
 
@@ -105,7 +104,7 @@ var _ = Describe("Testing FDB Kubernetes monitor API", func() {
 
 		JustBeforeEach(func() {
 			testArgument := Argument{ArgumentType: EnvironmentArgumentType, Source: "FDB_ZONE_ID"}
-			argument, err = testArgument.GenerateArgument(1, env, nil)
+			argument, err = testArgument.GenerateArgument(1, env)
 		})
 
 		When("the env variable is present", func() {
@@ -139,7 +138,7 @@ var _ = Describe("Testing FDB Kubernetes monitor API", func() {
 
 		JustBeforeEach(func() {
 			testArgument := Argument{ArgumentType: IPListArgumentType, Source: "FDB_PUBLIC_IP", IPFamily: IPFamily}
-			argument, err = testArgument.GenerateArgument(1, env, nil)
+			argument, err = testArgument.GenerateArgument(1, env)
 		})
 
 		When("using IP Family 4", func() {
@@ -246,54 +245,4 @@ var _ = Describe("Testing FDB Kubernetes monitor API", func() {
 			})
 		})
 	})
-
-	When("generating an argument with a node label", func() {
-		var argument string
-		var err error
-		var node *metav1.PartialObjectMetadata
-
-		JustBeforeEach(func() {
-			testArgument := Argument{ArgumentType: NodeLabelArgumentType, Source: "foundationdb.org/zone"}
-			argument, err = testArgument.GenerateArgument(1, nil, node)
-		})
-
-		When("the node is not present", func() {
-			It("should return an error", func() {
-				Expect(err).To(HaveOccurred())
-				Expect(err.Error()).To(Equal("no node information present"))
-				Expect(argument).To(BeEmpty())
-			})
-		})
-
-		When("the node is present but the label is missing", func() {
-			BeforeEach(func() {
-				node = &metav1.PartialObjectMetadata{
-					ObjectMeta: metav1.ObjectMeta{},
-				}
-			})
-
-			It("should return an error", func() {
-				Expect(err).To(HaveOccurred())
-				Expect(err.Error()).To(Equal("label \"foundationdb.org/zone\" not found"))
-				Expect(argument).To(BeEmpty())
-			})
-		})
-
-		When("the node and the label is present", func() {
-			BeforeEach(func() {
-				node = &metav1.PartialObjectMetadata{
-					ObjectMeta: metav1.ObjectMeta{
-						Labels: map[string]string{
-							"foundationdb.org/zone": "awesome",
-						},
-					},
-				}
-			})
-
-			It("should return an error", func() {
-				Expect(err).NotTo(HaveOccurred())
-				Expect(argument).To(Equal("awesome"))
-			})
-		})
-	})
 })
diff --git a/fdbkubernetesmonitor/monitor.go b/fdbkubernetesmonitor/monitor.go
index 3d7fd37ae60..e050ccc8fe9 100644
--- a/fdbkubernetesmonitor/monitor.go
+++ b/fdbkubernetesmonitor/monitor.go
@@ -32,6 +32,7 @@ import (
 	"os/exec"
 	"os/signal"
 	"path"
+	"strings"
 	"sync"
 	"syscall"
 	"time"
@@ -161,6 +162,37 @@ func StartMonitor(ctx context.Context, logger logr.Logger, configFile string, cu
 	monitor.Run()
 }
 
+// updateCustomEnvironment will add the node labels and their values to the custom environment map. All the generated
+// environment variables will start with NODE_LABEL and "/" and "." will be replaced in the key as "_", e.g. from the
+// label "foundationdb.org/testing = awesome" the env variables NODE_LABEL_FOUNDATIONDB_ORG_TESTING = awesome" will be
+// generated.
+func (monitor *Monitor) updateCustomEnvironmentFromNodeMetadata() {
+	if monitor.PodClient.nodeMetadata == nil {
+		return
+	}
+
+	nodeLabels := monitor.PodClient.nodeMetadata.Labels
+	for key, value := range nodeLabels {
+		sanitizedKey := strings.ReplaceAll(key, "/", "_")
+		sanitizedKey = strings.ReplaceAll(sanitizedKey, ".", "_")
+		envKey := "NODE_LABEL_" + strings.ToUpper(sanitizedKey)
+		currentValue, ok := monitor.CustomEnvironment[envKey]
+		if !ok {
+			monitor.Logger.Info("adding new custom environment variable from node labels", "key", envKey, "value", value)
+			monitor.CustomEnvironment[envKey] = value
+			continue
+		}
+
+		if currentValue == value {
+			continue
+		}
+
+		monitor.Logger.Info("update custom environment variable from node labels", "key", envKey, "newValue", value, "currentValue", currentValue)
+		monitor.CustomEnvironment[envKey] = value
+		continue
+	}
+}
+
 // LoadConfiguration loads the latest configuration from the config file.
 func (monitor *Monitor) LoadConfiguration() {
 	file, err := os.Open(monitor.ConfigFile)
@@ -168,7 +200,10 @@ func (monitor *Monitor) LoadConfiguration() {
 		monitor.Logger.Error(err, "Error reading monitor config file", "monitorConfigPath", monitor.ConfigFile)
 		return
 	}
-	defer file.Close()
+	defer func() {
+		err := file.Close()
+		monitor.Logger.Error(err, "Error could not close file", "monitorConfigPath", monitor.ConfigFile)
+	}()
 	configuration := &api.ProcessConfiguration{}
 	configurationBytes, err := io.ReadAll(file)
 	if err != nil {
@@ -192,7 +227,9 @@ func (monitor *Monitor) LoadConfiguration() {
 		return
 	}
 
-	_, err = configuration.GenerateArguments(1, monitor.CustomEnvironment, monitor.PodClient.nodeMetadata)
+	monitor.updateCustomEnvironmentFromNodeMetadata()
+
+	_, err = configuration.GenerateArguments(1, monitor.CustomEnvironment)
 	if err != nil {
 		monitor.Logger.Error(err, "Error generating arguments for latest configuration", "configuration", configuration, "binaryPath", configuration.BinaryPath)
 		return
@@ -277,7 +314,7 @@ func (monitor *Monitor) RunProcess(processNumber int) {
 			errorCounter = 0
 		}
 
-		arguments, err := monitor.ActiveConfiguration.GenerateArguments(processNumber, monitor.CustomEnvironment, monitor.PodClient.nodeMetadata)
+		arguments, err := monitor.ActiveConfiguration.GenerateArguments(processNumber, monitor.CustomEnvironment)
 		if err != nil {
 			backoffDuration := getBackoffDuration(errorCounter)
 			logger.Error(err, "Error generating arguments for subprocess", "configuration", monitor.ActiveConfiguration, "errorCounter", errorCounter, "backoffDuration", backoffDuration.String())
@@ -375,8 +412,7 @@ func (monitor *Monitor) processRequired(processNumber int) bool {
 	monitor.Mutex.Lock()
 	defer monitor.Mutex.Unlock()
 	logger := monitor.Logger.WithValues("processNumber", processNumber, "area", "processRequired")
-	runProcesses := pointer.BoolDeref(monitor.ActiveConfiguration.RunServers, true)
-	if monitor.ProcessCount < processNumber || !runProcesses {
+	if monitor.ProcessCount < processNumber || !pointer.BoolDeref(monitor.ActiveConfiguration.RunServers, true) {
 		if monitor.ProcessIDs[processNumber] != 0 {
 			logger.Info("Terminating run loop")
 			monitor.ProcessIDs[processNumber] = 0
diff --git a/fdbkubernetesmonitor/monitor_test.go b/fdbkubernetesmonitor/monitor_test.go
index a520c89781f..b65517e34a3 100644
--- a/fdbkubernetesmonitor/monitor_test.go
+++ b/fdbkubernetesmonitor/monitor_test.go
@@ -20,13 +20,59 @@
 package main
 
 import (
-	"time"
-
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"time"
 )
 
 var _ = Describe("Testing FDB Kubernetes Monitor", func() {
+	When("updating the custom environment variables from the node metadata", func() {
+		var monitor *Monitor
+
+		BeforeEach(func() {
+			monitor = &Monitor{
+				Logger: GinkgoLogr,
+				CustomEnvironment: map[string]string{
+					"testing": "testing",
+				},
+				PodClient: &PodClient{},
+			}
+		})
+
+		JustBeforeEach(func() {
+			monitor.updateCustomEnvironmentFromNodeMetadata()
+		})
+
+		When("no node metadata is present", func() {
+			It("shouldn't add new entries", func() {
+				Expect(monitor.CustomEnvironment).To(HaveLen(1))
+				Expect(monitor.CustomEnvironment).To(HaveKeyWithValue("testing", "testing"))
+			})
+		})
+
+		When("node metadata is present", func() {
+			BeforeEach(func() {
+				monitor.PodClient.nodeMetadata = &metav1.PartialObjectMetadata{
+					ObjectMeta: metav1.ObjectMeta{
+						Labels: map[string]string{
+							// Examples are taken from: https://kubernetes.io/docs/reference/labels-annotations-taints/
+							"topology.kubernetes.io/zone": "us-east-1c",
+							"kubernetes.io/hostname":      "ip-172-20-114-199.ec2.internal",
+						},
+					},
+				}
+			})
+
+			It("should add the new entries", func() {
+				Expect(monitor.CustomEnvironment).To(HaveLen(3))
+				Expect(monitor.CustomEnvironment).To(HaveKeyWithValue("testing", "testing"))
+				Expect(monitor.CustomEnvironment).To(HaveKeyWithValue("NODE_LABEL_TOPOLOGY_KUBERNETES_IO_ZONE", "us-east-1c"))
+				Expect(monitor.CustomEnvironment).To(HaveKeyWithValue("NODE_LABEL_KUBERNETES_IO_HOSTNAME", "ip-172-20-114-199.ec2.internal"))
+			})
+		})
+	})
+
 	DescribeTable("when getting the backoff time", func(errorCount int, expected time.Duration) {
 		Expect(getBackoffDuration(errorCount)).To(Equal(expected))
 	},

From e5a4883150981122b3c9d0e5f675992ca2ff47cc Mon Sep 17 00:00:00 2001
From: "Johannes M. Scheuermann" <jscheuermann@apple.com>
Date: Fri, 24 May 2024 17:08:37 +0200
Subject: [PATCH 442/468] Correct name of the flag

---
 fdbkubernetesmonitor/main.go | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/fdbkubernetesmonitor/main.go b/fdbkubernetesmonitor/main.go
index 6c57963e379..1b8205c1c5e 100644
--- a/fdbkubernetesmonitor/main.go
+++ b/fdbkubernetesmonitor/main.go
@@ -49,7 +49,7 @@ var (
 	listenAddress        string
 	processCount         int
 	enablePprof          bool
-	enableNodeWatcher    bool
+	enableNodeWatch      bool
 )
 
 type executionMode string
@@ -102,7 +102,7 @@ func main() {
 	pflag.IntVar(&processCount, "process-count", 1, "The number of processes to start")
 	pflag.BoolVar(&enablePprof, "enable-pprof", false, "Enables /debug/pprof endpoints on the listen address")
 	pflag.StringVar(&listenAddress, "listen-address", ":8081", "An address and port to listen on")
-	pflag.BoolVar(&enableNodeWatcher, "enable-node-node", false, "Enables the fdb-kubernetes-monitor to watch the node resource where the current Pod is running. This can be used to read node labels")
+	pflag.BoolVar(&enableNodeWatch, "enable-node-watch", false, "Enables the fdb-kubernetes-monitor to watch the node resource where the current Pod is running. This can be used to read node labels")
 	pflag.Parse()
 
 	logger := zapr.NewLogger(initLogger(logPath))
@@ -126,7 +126,7 @@ func main() {
 			logger.Error(err, "Error loading additional environment")
 			os.Exit(1)
 		}
-		StartMonitor(context.Background(), logger, path.Join(inputDir, monitorConfFile), customEnvironment, processCount, listenAddress, enablePprof, currentContainerVersion, enableNodeWatcher)
+		StartMonitor(context.Background(), logger, path.Join(inputDir, monitorConfFile), customEnvironment, processCount, listenAddress, enablePprof, currentContainerVersion, enableNodeWatch)
 	case executionModeInit:
 		err = CopyFiles(logger, outputDir, copyDetails, requiredCopies)
 		if err != nil {

From 455157cf2708f738a307278999ba9f56918653b0 Mon Sep 17 00:00:00 2001
From: "Johannes M. Scheuermann" <jscheuermann@apple.com>
Date: Mon, 3 Jun 2024 17:48:53 +0200
Subject: [PATCH 443/468] Add unit tests for PodClient

---
 fdbkubernetesmonitor/go.mod             |   3 +-
 fdbkubernetesmonitor/go.sum             |  14 ++
 fdbkubernetesmonitor/kubernetes.go      |  63 ++++--
 fdbkubernetesmonitor/kubernetes_test.go | 282 ++++++++++++++++++++++++
 fdbkubernetesmonitor/monitor.go         |   2 +-
 5 files changed, 339 insertions(+), 25 deletions(-)
 create mode 100644 fdbkubernetesmonitor/kubernetes_test.go

diff --git a/fdbkubernetesmonitor/go.mod b/fdbkubernetesmonitor/go.mod
index 4e33ab244a5..5b2f7cee23e 100644
--- a/fdbkubernetesmonitor/go.mod
+++ b/fdbkubernetesmonitor/go.mod
@@ -2,7 +2,7 @@
 //
 // This source file is part of the FoundationDB open source project
 //
-// Copyright 2021 Apple Inc. and the FoundationDB project authors
+// Copyright 2021-2024 Apple Inc. and the FoundationDB project authors
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -43,6 +43,7 @@ require (
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/emicklei/go-restful/v3 v3.9.0 // indirect
+	github.com/evanphx/json-patch v4.12.0+incompatible // indirect
 	github.com/evanphx/json-patch/v5 v5.6.0 // indirect
 	github.com/go-openapi/jsonpointer v0.19.5 // indirect
 	github.com/go-openapi/jsonreference v0.20.0 // indirect
diff --git a/fdbkubernetesmonitor/go.sum b/fdbkubernetesmonitor/go.sum
index 0b3cc3df237..56d03a0bc66 100644
--- a/fdbkubernetesmonitor/go.sum
+++ b/fdbkubernetesmonitor/go.sum
@@ -2,6 +2,7 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMT
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
 github.com/benbjohnson/clock v1.3.0 h1:ip6w0uFQkncKQ979AypyG0ER7mqUSBdKLOgAle/AT8A=
+github.com/benbjohnson/clock v1.3.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
@@ -20,6 +21,8 @@ github.com/emicklei/go-restful/v3 v3.9.0 h1:XwGDlfxEnQZzuopoqxwSEllNcCOM9DhhFyhF
 github.com/emicklei/go-restful/v3 v3.9.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc=
 github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
+github.com/evanphx/json-patch v4.12.0+incompatible h1:4onqiflcdA9EOZ4RxV643DvftH5pOlLGNtQ5lPWQu84=
+github.com/evanphx/json-patch v4.12.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
 github.com/evanphx/json-patch/v5 v5.6.0 h1:b91NhWfaz02IuVxO9faSllyAtNXHMPkC5J8sJCLunww=
 github.com/evanphx/json-patch/v5 v5.6.0/go.mod h1:G79N1coSVB93tBe7j6PhzjmR3/2VvlbKOFpnXhI9Bw4=
 github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY=
@@ -43,6 +46,7 @@ github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
 github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
 github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE=
+github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
@@ -73,8 +77,10 @@ github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/
 github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38 h1:yAJXTCF9TqKcTiHJAE8dj7HMvPfh66eeA2JYW7eFpSE=
 github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
 github.com/google/uuid v1.1.2 h1:EVhdT+1Kseyi1/pUmXKaFxYsDNy9RQYkMWRH68J/W7Y=
+github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
 github.com/imdario/mergo v0.3.6 h1:xTNEAn+kxVO7dTZGu0CegyqKZmoWFI0rF8UxjlB2d28=
+github.com/imdario/mergo v0.3.6/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
 github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
 github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
 github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
@@ -85,6 +91,7 @@ github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+o
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
 github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
+github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
@@ -135,12 +142,14 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
 github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
 github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk=
+github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
 go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
 go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ=
 go.uber.org/goleak v1.2.0 h1:xqgm/S+aQvhWFTtR0XK3Jvg7z8kGV8P4X14IzwN3Eqk=
+go.uber.org/goleak v1.2.0/go.mod h1:XJYK+MuIchqpmGmUSAzotztawfKvYLUIgg7guXrwVUo=
 go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU=
 go.uber.org/multierr v1.10.0 h1:S0h4aNzvfcFsC3dRF1jLoaov7oRaKqRGC/pUEJ2yvPQ=
 go.uber.org/multierr v1.10.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
@@ -159,6 +168,7 @@ golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.10.0 h1:lFO9qtOdlre5W1jxS3r/4szv2/6iXxScdzjoBMXNhYk=
+golang.org/x/mod v0.10.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -218,6 +228,7 @@ golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8T
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 gomodules.xyz/jsonpatch/v2 v2.2.0 h1:4pT439QV83L+G9FkcCriY6EkpcK6r6bK+A5FBUMI7qY=
+gomodules.xyz/jsonpatch/v2 v2.2.0/go.mod h1:WXp+iVDkoLQqPudfQ9GBlwB2eZ5DKOnjQZCYdOS8GPY=
 google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
 google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
 google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c=
@@ -246,6 +257,7 @@ gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8
 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
 gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc=
 gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
 gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc=
@@ -263,11 +275,13 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh
 k8s.io/api v0.26.1 h1:f+SWYiPd/GsiWwVRz+NbFyCgvv75Pk9NK6dlkZgpCRQ=
 k8s.io/api v0.26.1/go.mod h1:xd/GBNgR0f707+ATNyPmQ1oyKSgndzXij81FzWGsejg=
 k8s.io/apiextensions-apiserver v0.26.1 h1:cB8h1SRk6e/+i3NOrQgSFij1B2S0Y0wDoNl66bn8RMI=
+k8s.io/apiextensions-apiserver v0.26.1/go.mod h1:AptjOSXDGuE0JICx/Em15PaoO7buLwTs0dGleIHixSM=
 k8s.io/apimachinery v0.26.1 h1:8EZ/eGJL+hY/MYCNwhmDzVqq2lPl3N3Bo8rvweJwXUQ=
 k8s.io/apimachinery v0.26.1/go.mod h1:tnPmbONNJ7ByJNz9+n9kMjNP8ON+1qoAIIC70lztu74=
 k8s.io/client-go v0.26.1 h1:87CXzYJnAMGaa/IDDfRdhTzxk/wzGZ+/HUQpqgVSZXU=
 k8s.io/client-go v0.26.1/go.mod h1:IWNSglg+rQ3OcvDkhY6+QLeasV4OYHDjdqeWkDQZwGE=
 k8s.io/component-base v0.26.1 h1:4ahudpeQXHZL5kko+iDHqLj/FSGAEUnSVO0EBbgDd+4=
+k8s.io/component-base v0.26.1/go.mod h1:VHrLR0b58oC035w6YQiBSbtsf0ThuSwXP+p5dD/kAWU=
 k8s.io/klog/v2 v2.80.1 h1:atnLQ121W371wYYFawwYx1aEY2eUfs4l3J72wtgAwV4=
 k8s.io/klog/v2 v2.80.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0=
 k8s.io/kube-openapi v0.0.0-20221012153701-172d655c2280 h1:+70TFaan3hfJzs+7VK2o+OGxg8HsuBr/5f6tVAjDu6E=
diff --git a/fdbkubernetesmonitor/kubernetes.go b/fdbkubernetesmonitor/kubernetes.go
index fe3dfd9dbd8..571e417423e 100644
--- a/fdbkubernetesmonitor/kubernetes.go
+++ b/fdbkubernetesmonitor/kubernetes.go
@@ -80,17 +80,16 @@ type PodClient struct {
 	client.Client
 }
 
-// CreatePodClient creates a new client for working with the pod object.
-func CreatePodClient(ctx context.Context, logger logr.Logger, enableNodeWatcher bool) (*PodClient, error) {
+func setupCache(namespace string, podName string, nodeName string) (client.WithWatch, cache.Cache, error) {
 	config, err := rest.InClusterConfig()
 	if err != nil {
-		return nil, err
+		return nil, nil, err
 	}
 
 	scheme := runtime.NewScheme()
 	err = clientgoscheme.AddToScheme(scheme)
 	if err != nil {
-		return nil, err
+		return nil, nil, err
 	}
 
 	// Create the new client for writes. This client will also be used to setup the cache.
@@ -98,13 +97,9 @@ func CreatePodClient(ctx context.Context, logger logr.Logger, enableNodeWatcher
 		Scheme: scheme,
 	})
 	if err != nil {
-		return nil, err
+		return nil, nil, err
 	}
 
-	namespace := os.Getenv("FDB_POD_NAMESPACE")
-	podName := os.Getenv("FDB_POD_NAME")
-	nodeName := os.Getenv("FDB_NODE_NAME")
-
 	internalCache, err := cache.New(config, cache.Options{
 		Scheme:    scheme,
 		Mapper:    internalClient.RESTMapper(),
@@ -119,15 +114,19 @@ func CreatePodClient(ctx context.Context, logger logr.Logger, enableNodeWatcher
 		},
 	})
 	if err != nil {
-		return nil, err
+		return nil, nil, err
 	}
 
-	// Fetch the informer for the FoundationDBCluster resource
-	informer, err := internalCache.GetInformer(ctx, &corev1.Pod{})
-	if err != nil {
-		return nil, err
-	}
+	return internalClient, internalCache, nil
+}
 
+// CreatePodClient creates a new client for working with the pod object.
+func CreatePodClient(ctx context.Context, logger logr.Logger, enableNodeWatcher bool, setupCache func(string, string, string) (client.WithWatch, cache.Cache, error)) (*PodClient, error) {
+	namespace := os.Getenv("FDB_POD_NAMESPACE")
+	podName := os.Getenv("FDB_POD_NAME")
+	nodeName := os.Getenv("FDB_NODE_NAME")
+
+	internalClient, internalCache, err := setupCache(namespace, podName, nodeName)
 	podClient := &PodClient{
 		podMetadata:   nil,
 		nodeMetadata:  nil,
@@ -135,8 +134,26 @@ func CreatePodClient(ctx context.Context, logger logr.Logger, enableNodeWatcher
 		Logger:        logger,
 	}
 
-	// Setup an event handler to make sue we get events for new clusters and directly reload.
-	_, err = informer.AddEventHandler(podClient)
+	// Fetch the informer for the Pod resource.
+	podInformer, err := internalCache.GetInformer(ctx, &corev1.Pod{})
+	if err != nil {
+		return nil, err
+	}
+
+	// Setup an event handler to make sure we get events for the Pod and directly reload the information.
+	_, err = podInformer.AddEventHandler(podClient)
+	if err != nil {
+		return nil, err
+	}
+
+	// Fetch the informer for the Pod resource.
+	nodeInformer, err := internalCache.GetInformer(ctx, &corev1.Node{})
+	if err != nil {
+		return nil, err
+	}
+
+	// Setup an event handler to make sure we get events for the node and directly reload the information.
+	_, err = nodeInformer.AddEventHandler(podClient)
 	if err != nil {
 		return nil, err
 	}
@@ -265,8 +282,8 @@ func (podClient *PodClient) OnAdd(obj interface{}) {
 			ObjectMeta: castedObj.ObjectMeta,
 		}
 	case *corev1.Node:
-		podClient.Logger.Info("Got event for OnAdd Node resource", "name", castedObj.Name)
-		podClient.podMetadata = &metav1.PartialObjectMetadata{
+		podClient.Logger.Info("Got event for OnAdd for Node resource", "name", castedObj.Name)
+		podClient.nodeMetadata = &metav1.PartialObjectMetadata{
 			TypeMeta:   castedObj.TypeMeta,
 			ObjectMeta: castedObj.ObjectMeta,
 		}
@@ -302,8 +319,8 @@ func (podClient *PodClient) OnUpdate(_, newObj interface{}) {
 
 		podClient.TimestampFeed <- timestamp
 	case *corev1.Node:
-		podClient.Logger.Info("Got event for OnUpdate Node resource", "name", castedObj.Name)
-		podClient.podMetadata = &metav1.PartialObjectMetadata{
+		podClient.Logger.Info("Got event for OnUpdate for Node resource", "name", castedObj.Name)
+		podClient.nodeMetadata = &metav1.PartialObjectMetadata{
 			TypeMeta:   castedObj.TypeMeta,
 			ObjectMeta: castedObj.ObjectMeta,
 		}
@@ -320,7 +337,7 @@ func (podClient *PodClient) OnDelete(obj interface{}) {
 		podClient.Logger.Info("Got event for OnDelete for Pod resource", "name", castedObj.Name, "namespace", castedObj.Namespace)
 		podClient.podMetadata = nil
 	case *corev1.Node:
-		podClient.Logger.Info("Got event for OnDelete Node resource", "name", castedObj.Name)
-		podClient.podMetadata = nil
+		podClient.Logger.Info("Got event for OnDelete for Node resource", "name", castedObj.Name)
+		podClient.nodeMetadata = nil
 	}
 }
diff --git a/fdbkubernetesmonitor/kubernetes_test.go b/fdbkubernetesmonitor/kubernetes_test.go
new file mode 100644
index 00000000000..3943942c9ef
--- /dev/null
+++ b/fdbkubernetesmonitor/kubernetes_test.go
@@ -0,0 +1,282 @@
+// kubernetes_test.go
+//
+// This source file is part of the FoundationDB open source project
+//
+// Copyright 2023-2024 Apple Inc. and the FoundationDB project authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package main
+
+import (
+	"context"
+	"strconv"
+	"time"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
+	"sigs.k8s.io/controller-runtime/pkg/cache"
+	"sigs.k8s.io/controller-runtime/pkg/cache/informertest"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/client/fake"
+	"sigs.k8s.io/controller-runtime/pkg/controller/controllertest"
+)
+
+var _ = Describe("Testing FDB Pod client", func() {
+	When("updating the custom environment variables from the node metadata", func() {
+		var enableNodeWatcher bool
+		var fakeClient client.WithWatch
+		var podClient *PodClient
+		var namespace, podName, nodeName string
+		var internalCache *informertest.FakeInformers
+
+		BeforeEach(func() {
+			scheme := runtime.NewScheme()
+			Expect(clientgoscheme.AddToScheme(scheme)).NotTo(HaveOccurred())
+			fakeClient = fake.NewClientBuilder().WithScheme(scheme).Build()
+
+			namespace = "fdb-testing"
+			podName = "storage-1"
+			nodeName = "node1"
+
+			GinkgoT().Setenv("FDB_POD_NAMESPACE", namespace)
+			GinkgoT().Setenv("FDB_POD_NAME", podName)
+			GinkgoT().Setenv("FDB_NODE_NAME", nodeName)
+
+			internalCache = &informertest.FakeInformers{}
+			internalCache.Scheme = fakeClient.Scheme()
+		})
+
+		When("the PodClient was started", func() {
+			JustBeforeEach(func() {
+				var err error
+				podClient, err = CreatePodClient(context.Background(), GinkgoLogr, enableNodeWatcher, func(fncNamespace string, fncPodName string, fncNodeName string) (client.WithWatch, cache.Cache, error) {
+					Expect(fncNamespace).To(Equal(namespace))
+					Expect(fncPodName).To(Equal(podName))
+					Expect(fncNodeName).To(Equal(nodeName))
+
+					return fakeClient, internalCache, nil
+				})
+
+				Expect(err).NotTo(HaveOccurred())
+			})
+
+			When("the node watch feature is disabled", func() {
+				BeforeEach(func() {
+					enableNodeWatcher = false
+				})
+
+				It("should have the metadata for the pod but not the node", func() {
+					Expect(podClient.podMetadata).NotTo(BeNil())
+					Expect(podClient.nodeMetadata).To(BeNil())
+				})
+			})
+
+			When("the node watch feature is enabled", func() {
+				BeforeEach(func() {
+					enableNodeWatcher = true
+				})
+
+				It("should have the metadata for the pod and node", func() {
+					Expect(podClient.podMetadata).NotTo(BeNil())
+					Expect(podClient.nodeMetadata).NotTo(BeNil())
+				})
+			})
+		})
+
+		When("the PodClient handles events", func() {
+			BeforeEach(func() {
+				var err error
+				podClient, err = CreatePodClient(context.Background(), GinkgoLogr, enableNodeWatcher, func(fncNamespace string, fncPodName string, fncNodeName string) (client.WithWatch, cache.Cache, error) {
+					Expect(fncNamespace).To(Equal(namespace))
+					Expect(fncPodName).To(Equal(podName))
+					Expect(fncNodeName).To(Equal(nodeName))
+
+					return fakeClient, internalCache, nil
+				})
+
+				Expect(err).NotTo(HaveOccurred())
+			})
+
+			When("events for the pod are received", func() {
+				var fakeInformer *controllertest.FakeInformer
+				var pod *corev1.Pod
+
+				BeforeEach(func() {
+					pod = &corev1.Pod{
+						ObjectMeta: metav1.ObjectMeta{
+							Name:      podName,
+							Namespace: namespace,
+							Labels: map[string]string{
+								"testing": "testing",
+							},
+						},
+					}
+					var err error
+					fakeInformer, err = internalCache.FakeInformerFor(pod)
+					Expect(err).NotTo(HaveOccurred())
+				})
+
+				When("an AddEvent is handled", func() {
+					BeforeEach(func() {
+						fakeInformer.Add(pod)
+					})
+
+					It("should update the pod information", func() {
+						Expect(podClient.podMetadata).NotTo(BeNil())
+						Expect(podClient.podMetadata.Name).To(Equal(podName))
+						Expect(podClient.podMetadata.Namespace).To(Equal(namespace))
+						Expect(podClient.podMetadata.Labels).To(Equal(map[string]string{
+							"testing": "testing",
+						}))
+					})
+				})
+
+				When("an UpdateEvent is handled", func() {
+					BeforeEach(func() {
+						fakeInformer.Update(nil, pod)
+					})
+
+					It("should update the pod information", func() {
+						Expect(podClient.podMetadata).NotTo(BeNil())
+						Expect(podClient.podMetadata.Name).To(Equal(podName))
+						Expect(podClient.podMetadata.Namespace).To(Equal(namespace))
+						Expect(podClient.podMetadata.Labels).To(Equal(map[string]string{
+							"testing": "testing",
+						}))
+
+						Expect(podClient.TimestampFeed).NotTo(Receive())
+					})
+				})
+
+				When("an UpdateEvent is handled that updates the OutdatedConfigMapAnnotation", func() {
+					var timestamp int64
+
+					BeforeEach(func() {
+						timestamp = time.Now().Unix()
+						pod.Annotations = map[string]string{
+							OutdatedConfigMapAnnotation: strconv.FormatInt(timestamp, 10),
+						}
+						fakeInformer.Update(nil, pod)
+					})
+
+					It("should update the pod information and receive an event", func() {
+						Expect(podClient.podMetadata).NotTo(BeNil())
+						Expect(podClient.podMetadata.Name).To(Equal(podName))
+						Expect(podClient.podMetadata.Namespace).To(Equal(namespace))
+						Expect(podClient.podMetadata.Labels).To(Equal(map[string]string{
+							"testing": "testing",
+						}))
+
+						Expect(podClient.TimestampFeed).To(Receive(&timestamp))
+					})
+				})
+
+				When("an UpdateEvent is handled that updates the OutdatedConfigMapAnnotation with a bad value", func() {
+					BeforeEach(func() {
+						pod.Annotations = map[string]string{
+							OutdatedConfigMapAnnotation: "boom!",
+						}
+						fakeInformer.Update(nil, pod)
+					})
+
+					It("should update the Pod information", func() {
+						Expect(podClient.podMetadata).NotTo(BeNil())
+						Expect(podClient.podMetadata.Name).To(Equal(podName))
+						Expect(podClient.podMetadata.Namespace).To(Equal(namespace))
+						Expect(podClient.podMetadata.Labels).To(Equal(map[string]string{
+							"testing": "testing",
+						}))
+
+						Expect(podClient.TimestampFeed).NotTo(Receive())
+					})
+				})
+
+				When("a DeleteEvent is handled", func() {
+					BeforeEach(func() {
+						fakeInformer.Delete(pod)
+					})
+
+					It("should remove the pod information", func() {
+						Expect(podClient.podMetadata).To(BeNil())
+					})
+				})
+			})
+
+			When("events for the node are received", func() {
+				var fakeInformer *controllertest.FakeInformer
+				var node *corev1.Node
+
+				BeforeEach(func() {
+					node = &corev1.Node{
+						ObjectMeta: metav1.ObjectMeta{
+							Name:      podName,
+							Namespace: namespace,
+							Labels: map[string]string{
+								"testing": "testing",
+							},
+						},
+					}
+					var err error
+					fakeInformer, err = internalCache.FakeInformerFor(node)
+					Expect(err).NotTo(HaveOccurred())
+				})
+
+				When("an AddEvent is handled", func() {
+					BeforeEach(func() {
+						fakeInformer.Add(node)
+					})
+
+					It("should update the node information", func() {
+						Expect(podClient.nodeMetadata).NotTo(BeNil())
+						Expect(podClient.nodeMetadata.Name).To(Equal(podName))
+						Expect(podClient.nodeMetadata.Namespace).To(Equal(namespace))
+						Expect(podClient.nodeMetadata.Labels).To(Equal(map[string]string{
+							"testing": "testing",
+						}))
+					})
+				})
+
+				When("an UpdateEvent is handled", func() {
+					BeforeEach(func() {
+						fakeInformer.Update(nil, node)
+					})
+
+					It("should update the node information", func() {
+						Expect(podClient.nodeMetadata).NotTo(BeNil())
+						Expect(podClient.nodeMetadata.Name).To(Equal(podName))
+						Expect(podClient.nodeMetadata.Namespace).To(Equal(namespace))
+						Expect(podClient.nodeMetadata.Labels).To(Equal(map[string]string{
+							"testing": "testing",
+						}))
+					})
+				})
+
+				When("a DeleteEvent is handled", func() {
+					BeforeEach(func() {
+						fakeInformer.Delete(node)
+					})
+
+					It("should remove the node information", func() {
+						Expect(podClient.nodeMetadata).To(BeNil())
+					})
+				})
+			})
+		})
+	})
+})
diff --git a/fdbkubernetesmonitor/monitor.go b/fdbkubernetesmonitor/monitor.go
index e050ccc8fe9..b70352b8576 100644
--- a/fdbkubernetesmonitor/monitor.go
+++ b/fdbkubernetesmonitor/monitor.go
@@ -108,7 +108,7 @@ type Monitor struct {
 
 // StartMonitor starts the monitor loop.
 func StartMonitor(ctx context.Context, logger logr.Logger, configFile string, customEnvironment map[string]string, processCount int, listenAddr string, enableDebug bool, currentContainerVersion string, enableNodeWatcher bool) {
-	podClient, err := CreatePodClient(ctx, logger, enableNodeWatcher)
+	podClient, err := CreatePodClient(ctx, logger, enableNodeWatcher, setupCache)
 	if err != nil {
 		logger.Error(err, "could not create Pod client")
 		os.Exit(1)

From 553dd402223fbd0cc49e599332ec799a992e3a60 Mon Sep 17 00:00:00 2001
From: "Johannes M. Scheuermann" <jscheuermann@apple.com>
Date: Mon, 3 Jun 2024 17:54:50 +0200
Subject: [PATCH 444/468] only setup node informer if node watch feature is
 enabeld

---
 fdbkubernetesmonitor/api/config.go      |  2 +-
 fdbkubernetesmonitor/kubernetes.go      | 21 ++++++++++++---------
 fdbkubernetesmonitor/kubernetes_test.go |  2 ++
 3 files changed, 15 insertions(+), 10 deletions(-)

diff --git a/fdbkubernetesmonitor/api/config.go b/fdbkubernetesmonitor/api/config.go
index abfa6b14122..169cc6524e9 100644
--- a/fdbkubernetesmonitor/api/config.go
+++ b/fdbkubernetesmonitor/api/config.go
@@ -57,7 +57,7 @@ type Argument struct {
 	Values []Argument `json:"values,omitempty"`
 
 	// Source provides the name of the environment variable to use for an
-	// Environment type argument. This setting is also used for NodeLabelArgumentType.
+	// Environment type argument.
 	Source string `json:"source,omitempty"`
 
 	// Multiplier provides a multiplier for the process number for ProcessNumber
diff --git a/fdbkubernetesmonitor/kubernetes.go b/fdbkubernetesmonitor/kubernetes.go
index 571e417423e..068934a31c3 100644
--- a/fdbkubernetesmonitor/kubernetes.go
+++ b/fdbkubernetesmonitor/kubernetes.go
@@ -146,16 +146,19 @@ func CreatePodClient(ctx context.Context, logger logr.Logger, enableNodeWatcher
 		return nil, err
 	}
 
-	// Fetch the informer for the Pod resource.
-	nodeInformer, err := internalCache.GetInformer(ctx, &corev1.Node{})
-	if err != nil {
-		return nil, err
-	}
+	if enableNodeWatcher {
+		var nodeInformer cache.Informer
+		// Fetch the informer for the node resource.
+		nodeInformer, err = internalCache.GetInformer(ctx, &corev1.Node{})
+		if err != nil {
+			return nil, err
+		}
 
-	// Setup an event handler to make sure we get events for the node and directly reload the information.
-	_, err = nodeInformer.AddEventHandler(podClient)
-	if err != nil {
-		return nil, err
+		// Setup an event handler to make sure we get events for the node and directly reload the information.
+		_, err = nodeInformer.AddEventHandler(podClient)
+		if err != nil {
+			return nil, err
+		}
 	}
 
 	// Make sure the internal cache is started.
diff --git a/fdbkubernetesmonitor/kubernetes_test.go b/fdbkubernetesmonitor/kubernetes_test.go
index 3943942c9ef..7727d11f0a7 100644
--- a/fdbkubernetesmonitor/kubernetes_test.go
+++ b/fdbkubernetesmonitor/kubernetes_test.go
@@ -84,6 +84,7 @@ var _ = Describe("Testing FDB Pod client", func() {
 				It("should have the metadata for the pod but not the node", func() {
 					Expect(podClient.podMetadata).NotTo(BeNil())
 					Expect(podClient.nodeMetadata).To(BeNil())
+					Expect(internalCache.InformersByGVK).To(HaveLen(1))
 				})
 			})
 
@@ -95,6 +96,7 @@ var _ = Describe("Testing FDB Pod client", func() {
 				It("should have the metadata for the pod and node", func() {
 					Expect(podClient.podMetadata).NotTo(BeNil())
 					Expect(podClient.nodeMetadata).NotTo(BeNil())
+					Expect(internalCache.InformersByGVK).To(HaveLen(2))
 				})
 			})
 		})

From 958ecd7f973f3484cbd86cbaa20242179cad5006 Mon Sep 17 00:00:00 2001
From: "Johannes M. Scheuermann" <jscheuermann@apple.com>
Date: Mon, 3 Jun 2024 18:15:33 +0200
Subject: [PATCH 445/468] Add additional tests for PodClient

---
 fdbkubernetesmonitor/kubernetes.go      |   2 +-
 fdbkubernetesmonitor/kubernetes_test.go | 470 +++++++++++++++---------
 2 files changed, 290 insertions(+), 182 deletions(-)

diff --git a/fdbkubernetesmonitor/kubernetes.go b/fdbkubernetesmonitor/kubernetes.go
index 068934a31c3..f5f7722d1a8 100644
--- a/fdbkubernetesmonitor/kubernetes.go
+++ b/fdbkubernetesmonitor/kubernetes.go
@@ -254,7 +254,7 @@ func (podClient *PodClient) updateFdbClusterTimestampAnnotation() error {
 // will be updated if the annotation is absent it will be added.
 func (podClient *PodClient) updateAnnotationsOnPod(annotationChanges map[string]string) error {
 	annotations := podClient.podMetadata.Annotations
-	if len(annotations) > 0 {
+	if len(annotations) == 0 {
 		annotations = map[string]string{}
 	}
 
diff --git a/fdbkubernetesmonitor/kubernetes_test.go b/fdbkubernetesmonitor/kubernetes_test.go
index 7727d11f0a7..56e864fd0a7 100644
--- a/fdbkubernetesmonitor/kubernetes_test.go
+++ b/fdbkubernetesmonitor/kubernetes_test.go
@@ -21,6 +21,7 @@ package main
 
 import (
 	"context"
+	"github.com/apple/foundationdb/fdbkubernetesmonitor/api"
 	"strconv"
 	"time"
 
@@ -38,246 +39,353 @@ import (
 )
 
 var _ = Describe("Testing FDB Pod client", func() {
-	When("updating the custom environment variables from the node metadata", func() {
-		var enableNodeWatcher bool
-		var fakeClient client.WithWatch
-		var podClient *PodClient
-		var namespace, podName, nodeName string
-		var internalCache *informertest.FakeInformers
+	var enableNodeWatcher bool
+	var fakeClient client.WithWatch
+	var podClient *PodClient
+	var namespace, podName, nodeName string
+	var internalCache *informertest.FakeInformers
+
+	BeforeEach(func() {
+		scheme := runtime.NewScheme()
+		Expect(clientgoscheme.AddToScheme(scheme)).NotTo(HaveOccurred())
+		fakeClient = fake.NewClientBuilder().WithScheme(scheme).Build()
+
+		namespace = "fdb-testing"
+		podName = "storage-1"
+		nodeName = "node1"
+
+		GinkgoT().Setenv("FDB_POD_NAMESPACE", namespace)
+		GinkgoT().Setenv("FDB_POD_NAME", podName)
+		GinkgoT().Setenv("FDB_NODE_NAME", nodeName)
+
+		internalCache = &informertest.FakeInformers{}
+		internalCache.Scheme = fakeClient.Scheme()
+	})
 
-		BeforeEach(func() {
-			scheme := runtime.NewScheme()
-			Expect(clientgoscheme.AddToScheme(scheme)).NotTo(HaveOccurred())
-			fakeClient = fake.NewClientBuilder().WithScheme(scheme).Build()
+	When("the PodClient was started", func() {
+		JustBeforeEach(func() {
+			var err error
+			podClient, err = CreatePodClient(context.Background(), GinkgoLogr, enableNodeWatcher, func(fncNamespace string, fncPodName string, fncNodeName string) (client.WithWatch, cache.Cache, error) {
+				Expect(fncNamespace).To(Equal(namespace))
+				Expect(fncPodName).To(Equal(podName))
+				Expect(fncNodeName).To(Equal(nodeName))
 
-			namespace = "fdb-testing"
-			podName = "storage-1"
-			nodeName = "node1"
+				return fakeClient, internalCache, nil
+			})
 
-			GinkgoT().Setenv("FDB_POD_NAMESPACE", namespace)
-			GinkgoT().Setenv("FDB_POD_NAME", podName)
-			GinkgoT().Setenv("FDB_NODE_NAME", nodeName)
+			Expect(err).NotTo(HaveOccurred())
+		})
+
+		When("the node watch feature is disabled", func() {
+			BeforeEach(func() {
+				enableNodeWatcher = false
+			})
 
-			internalCache = &informertest.FakeInformers{}
-			internalCache.Scheme = fakeClient.Scheme()
+			It("should have the metadata for the pod but not the node", func() {
+				Expect(podClient.podMetadata).NotTo(BeNil())
+				Expect(podClient.nodeMetadata).To(BeNil())
+				Expect(internalCache.InformersByGVK).To(HaveLen(1))
+			})
 		})
 
-		When("the PodClient was started", func() {
-			JustBeforeEach(func() {
-				var err error
-				podClient, err = CreatePodClient(context.Background(), GinkgoLogr, enableNodeWatcher, func(fncNamespace string, fncPodName string, fncNodeName string) (client.WithWatch, cache.Cache, error) {
-					Expect(fncNamespace).To(Equal(namespace))
-					Expect(fncPodName).To(Equal(podName))
-					Expect(fncNodeName).To(Equal(nodeName))
+		When("the node watch feature is enabled", func() {
+			BeforeEach(func() {
+				enableNodeWatcher = true
+			})
 
-					return fakeClient, internalCache, nil
-				})
+			It("should have the metadata for the pod and node", func() {
+				Expect(podClient.podMetadata).NotTo(BeNil())
+				Expect(podClient.nodeMetadata).NotTo(BeNil())
+				Expect(internalCache.InformersByGVK).To(HaveLen(2))
+			})
+		})
+	})
+
+	When("the PodClient handles events", func() {
+		BeforeEach(func() {
+			var err error
+			podClient, err = CreatePodClient(context.Background(), GinkgoLogr, enableNodeWatcher, func(fncNamespace string, fncPodName string, fncNodeName string) (client.WithWatch, cache.Cache, error) {
+				Expect(fncNamespace).To(Equal(namespace))
+				Expect(fncPodName).To(Equal(podName))
+				Expect(fncNodeName).To(Equal(nodeName))
+
+				return fakeClient, internalCache, nil
+			})
 
+			Expect(err).NotTo(HaveOccurred())
+		})
+
+		When("events for the pod are received", func() {
+			var fakeInformer *controllertest.FakeInformer
+			var pod *corev1.Pod
+
+			BeforeEach(func() {
+				pod = &corev1.Pod{
+					ObjectMeta: metav1.ObjectMeta{
+						Name:      podName,
+						Namespace: namespace,
+						Labels: map[string]string{
+							"testing": "testing",
+						},
+					},
+				}
+				var err error
+				fakeInformer, err = internalCache.FakeInformerFor(pod)
 				Expect(err).NotTo(HaveOccurred())
 			})
 
-			When("the node watch feature is disabled", func() {
+			When("an AddEvent is handled", func() {
 				BeforeEach(func() {
-					enableNodeWatcher = false
+					fakeInformer.Add(pod)
 				})
 
-				It("should have the metadata for the pod but not the node", func() {
+				It("should update the pod information", func() {
 					Expect(podClient.podMetadata).NotTo(BeNil())
-					Expect(podClient.nodeMetadata).To(BeNil())
-					Expect(internalCache.InformersByGVK).To(HaveLen(1))
+					Expect(podClient.podMetadata.Name).To(Equal(podName))
+					Expect(podClient.podMetadata.Namespace).To(Equal(namespace))
+					Expect(podClient.podMetadata.Labels).To(Equal(map[string]string{
+						"testing": "testing",
+					}))
 				})
 			})
 
-			When("the node watch feature is enabled", func() {
+			When("an UpdateEvent is handled", func() {
 				BeforeEach(func() {
-					enableNodeWatcher = true
+					fakeInformer.Update(nil, pod)
 				})
 
-				It("should have the metadata for the pod and node", func() {
+				It("should update the pod information", func() {
 					Expect(podClient.podMetadata).NotTo(BeNil())
-					Expect(podClient.nodeMetadata).NotTo(BeNil())
-					Expect(internalCache.InformersByGVK).To(HaveLen(2))
+					Expect(podClient.podMetadata.Name).To(Equal(podName))
+					Expect(podClient.podMetadata.Namespace).To(Equal(namespace))
+					Expect(podClient.podMetadata.Labels).To(Equal(map[string]string{
+						"testing": "testing",
+					}))
+
+					Expect(podClient.TimestampFeed).NotTo(Receive())
 				})
 			})
-		})
 
-		When("the PodClient handles events", func() {
-			BeforeEach(func() {
-				var err error
-				podClient, err = CreatePodClient(context.Background(), GinkgoLogr, enableNodeWatcher, func(fncNamespace string, fncPodName string, fncNodeName string) (client.WithWatch, cache.Cache, error) {
-					Expect(fncNamespace).To(Equal(namespace))
-					Expect(fncPodName).To(Equal(podName))
-					Expect(fncNodeName).To(Equal(nodeName))
+			When("an UpdateEvent is handled that updates the OutdatedConfigMapAnnotation", func() {
+				var timestamp int64
 
-					return fakeClient, internalCache, nil
+				BeforeEach(func() {
+					timestamp = time.Now().Unix()
+					pod.Annotations = map[string]string{
+						OutdatedConfigMapAnnotation: strconv.FormatInt(timestamp, 10),
+					}
+					fakeInformer.Update(nil, pod)
 				})
 
-				Expect(err).NotTo(HaveOccurred())
-			})
+				It("should update the pod information and receive an event", func() {
+					Expect(podClient.podMetadata).NotTo(BeNil())
+					Expect(podClient.podMetadata.Name).To(Equal(podName))
+					Expect(podClient.podMetadata.Namespace).To(Equal(namespace))
+					Expect(podClient.podMetadata.Labels).To(Equal(map[string]string{
+						"testing": "testing",
+					}))
 
-			When("events for the pod are received", func() {
-				var fakeInformer *controllertest.FakeInformer
-				var pod *corev1.Pod
+					Expect(podClient.TimestampFeed).To(Receive(&timestamp))
+				})
+			})
 
+			When("an UpdateEvent is handled that updates the OutdatedConfigMapAnnotation with a bad value", func() {
 				BeforeEach(func() {
-					pod = &corev1.Pod{
-						ObjectMeta: metav1.ObjectMeta{
-							Name:      podName,
-							Namespace: namespace,
-							Labels: map[string]string{
-								"testing": "testing",
-							},
-						},
+					pod.Annotations = map[string]string{
+						OutdatedConfigMapAnnotation: "boom!",
 					}
-					var err error
-					fakeInformer, err = internalCache.FakeInformerFor(pod)
-					Expect(err).NotTo(HaveOccurred())
+					fakeInformer.Update(nil, pod)
 				})
 
-				When("an AddEvent is handled", func() {
-					BeforeEach(func() {
-						fakeInformer.Add(pod)
-					})
+				It("should update the Pod information", func() {
+					Expect(podClient.podMetadata).NotTo(BeNil())
+					Expect(podClient.podMetadata.Name).To(Equal(podName))
+					Expect(podClient.podMetadata.Namespace).To(Equal(namespace))
+					Expect(podClient.podMetadata.Labels).To(Equal(map[string]string{
+						"testing": "testing",
+					}))
 
-					It("should update the pod information", func() {
-						Expect(podClient.podMetadata).NotTo(BeNil())
-						Expect(podClient.podMetadata.Name).To(Equal(podName))
-						Expect(podClient.podMetadata.Namespace).To(Equal(namespace))
-						Expect(podClient.podMetadata.Labels).To(Equal(map[string]string{
-							"testing": "testing",
-						}))
-					})
+					Expect(podClient.TimestampFeed).NotTo(Receive())
 				})
+			})
 
-				When("an UpdateEvent is handled", func() {
-					BeforeEach(func() {
-						fakeInformer.Update(nil, pod)
-					})
-
-					It("should update the pod information", func() {
-						Expect(podClient.podMetadata).NotTo(BeNil())
-						Expect(podClient.podMetadata.Name).To(Equal(podName))
-						Expect(podClient.podMetadata.Namespace).To(Equal(namespace))
-						Expect(podClient.podMetadata.Labels).To(Equal(map[string]string{
-							"testing": "testing",
-						}))
+			When("a DeleteEvent is handled", func() {
+				BeforeEach(func() {
+					fakeInformer.Delete(pod)
+				})
 
-						Expect(podClient.TimestampFeed).NotTo(Receive())
-					})
+				It("should remove the pod information", func() {
+					Expect(podClient.podMetadata).To(BeNil())
 				})
+			})
+		})
 
-				When("an UpdateEvent is handled that updates the OutdatedConfigMapAnnotation", func() {
-					var timestamp int64
-
-					BeforeEach(func() {
-						timestamp = time.Now().Unix()
-						pod.Annotations = map[string]string{
-							OutdatedConfigMapAnnotation: strconv.FormatInt(timestamp, 10),
-						}
-						fakeInformer.Update(nil, pod)
-					})
-
-					It("should update the pod information and receive an event", func() {
-						Expect(podClient.podMetadata).NotTo(BeNil())
-						Expect(podClient.podMetadata.Name).To(Equal(podName))
-						Expect(podClient.podMetadata.Namespace).To(Equal(namespace))
-						Expect(podClient.podMetadata.Labels).To(Equal(map[string]string{
+		When("events for the node are received", func() {
+			var fakeInformer *controllertest.FakeInformer
+			var node *corev1.Node
+
+			BeforeEach(func() {
+				node = &corev1.Node{
+					ObjectMeta: metav1.ObjectMeta{
+						Name:      podName,
+						Namespace: namespace,
+						Labels: map[string]string{
 							"testing": "testing",
-						}))
+						},
+					},
+				}
+				var err error
+				fakeInformer, err = internalCache.FakeInformerFor(node)
+				Expect(err).NotTo(HaveOccurred())
+			})
 
-						Expect(podClient.TimestampFeed).To(Receive(&timestamp))
-					})
+			When("an AddEvent is handled", func() {
+				BeforeEach(func() {
+					fakeInformer.Add(node)
 				})
 
-				When("an UpdateEvent is handled that updates the OutdatedConfigMapAnnotation with a bad value", func() {
-					BeforeEach(func() {
-						pod.Annotations = map[string]string{
-							OutdatedConfigMapAnnotation: "boom!",
-						}
-						fakeInformer.Update(nil, pod)
-					})
-
-					It("should update the Pod information", func() {
-						Expect(podClient.podMetadata).NotTo(BeNil())
-						Expect(podClient.podMetadata.Name).To(Equal(podName))
-						Expect(podClient.podMetadata.Namespace).To(Equal(namespace))
-						Expect(podClient.podMetadata.Labels).To(Equal(map[string]string{
-							"testing": "testing",
-						}))
+				It("should update the node information", func() {
+					Expect(podClient.nodeMetadata).NotTo(BeNil())
+					Expect(podClient.nodeMetadata.Name).To(Equal(podName))
+					Expect(podClient.nodeMetadata.Namespace).To(Equal(namespace))
+					Expect(podClient.nodeMetadata.Labels).To(Equal(map[string]string{
+						"testing": "testing",
+					}))
+				})
+			})
 
-						Expect(podClient.TimestampFeed).NotTo(Receive())
-					})
+			When("an UpdateEvent is handled", func() {
+				BeforeEach(func() {
+					fakeInformer.Update(nil, node)
 				})
 
-				When("a DeleteEvent is handled", func() {
-					BeforeEach(func() {
-						fakeInformer.Delete(pod)
-					})
+				It("should update the node information", func() {
+					Expect(podClient.nodeMetadata).NotTo(BeNil())
+					Expect(podClient.nodeMetadata.Name).To(Equal(podName))
+					Expect(podClient.nodeMetadata.Namespace).To(Equal(namespace))
+					Expect(podClient.nodeMetadata.Labels).To(Equal(map[string]string{
+						"testing": "testing",
+					}))
+				})
+			})
+
+			When("a DeleteEvent is handled", func() {
+				BeforeEach(func() {
+					fakeInformer.Delete(node)
+				})
 
-					It("should remove the pod information", func() {
-						Expect(podClient.podMetadata).To(BeNil())
-					})
+				It("should remove the node information", func() {
+					Expect(podClient.nodeMetadata).To(BeNil())
 				})
 			})
+		})
+	})
 
-			When("events for the node are received", func() {
-				var fakeInformer *controllertest.FakeInformer
-				var node *corev1.Node
+	When("the PodClient should update the annotations", func() {
+		var monitor *Monitor
 
-				BeforeEach(func() {
-					node = &corev1.Node{
-						ObjectMeta: metav1.ObjectMeta{
-							Name:      podName,
-							Namespace: namespace,
-							Labels: map[string]string{
-								"testing": "testing",
+		JustBeforeEach(func() {
+			var err error
+			podClient, err = CreatePodClient(context.Background(), GinkgoLogr, enableNodeWatcher, func(fncNamespace string, fncPodName string, fncNodeName string) (client.WithWatch, cache.Cache, error) {
+				Expect(fncNamespace).To(Equal(namespace))
+				Expect(fncPodName).To(Equal(podName))
+				Expect(fncNodeName).To(Equal(nodeName))
+
+				return fakeClient, internalCache, nil
+			})
+
+			Expect(err).NotTo(HaveOccurred())
+			// We have to set this value here as we haven't received any update yet and the fake implementation is not
+			// passing through the fake client.
+			podClient.podMetadata = &metav1.PartialObjectMetadata{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      podName,
+					Namespace: namespace,
+				},
+			}
+
+			// Make sure to create the Pod in the fake client otherwise it cannot be patched.
+			Expect(fakeClient.Create(context.Background(), &corev1.Pod{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      podName,
+					Namespace: namespace,
+				},
+			})).NotTo(HaveOccurred())
+
+			// Execute the update annotations.
+			Expect(podClient.UpdateAnnotations(monitor)).NotTo(HaveOccurred())
+		})
+
+		When("no additional env variables are set", func() {
+			BeforeEach(func() {
+				monitor = &Monitor{
+					ActiveConfiguration: &api.ProcessConfiguration{
+						BinaryPath: "/usr/bin",
+					},
+				}
+			})
+
+			It("should update the annotations", func() {
+				pod := &corev1.Pod{}
+				Expect(fakeClient.Get(context.Background(), client.ObjectKey{Namespace: namespace, Name: podName}, pod)).NotTo(HaveOccurred())
+
+				Expect(pod.Annotations).To(HaveKeyWithValue(CurrentConfigurationAnnotation, ""))
+				Expect(pod.Annotations).To(HaveKeyWithValue(EnvironmentAnnotation, "{\"BINARY_DIR\":\"/usr\"}"))
+			})
+		})
+
+		When("one flat additional env variable is set", func() {
+			BeforeEach(func() {
+				GinkgoT().Setenv("TEST", "test-value")
+				monitor = &Monitor{
+					ActiveConfiguration: &api.ProcessConfiguration{
+						BinaryPath: "/usr/bin",
+						Arguments: []api.Argument{
+							{
+								ArgumentType: EnvironmentAnnotation,
+								Source:       "TEST",
 							},
 						},
-					}
-					var err error
-					fakeInformer, err = internalCache.FakeInformerFor(node)
-					Expect(err).NotTo(HaveOccurred())
-				})
-
-				When("an AddEvent is handled", func() {
-					BeforeEach(func() {
-						fakeInformer.Add(node)
-					})
+					},
+				}
+			})
 
-					It("should update the node information", func() {
-						Expect(podClient.nodeMetadata).NotTo(BeNil())
-						Expect(podClient.nodeMetadata.Name).To(Equal(podName))
-						Expect(podClient.nodeMetadata.Namespace).To(Equal(namespace))
-						Expect(podClient.nodeMetadata.Labels).To(Equal(map[string]string{
-							"testing": "testing",
-						}))
-					})
-				})
+			It("should update the annotations", func() {
+				pod := &corev1.Pod{}
+				Expect(fakeClient.Get(context.Background(), client.ObjectKey{Namespace: namespace, Name: podName}, pod)).NotTo(HaveOccurred())
 
-				When("an UpdateEvent is handled", func() {
-					BeforeEach(func() {
-						fakeInformer.Update(nil, node)
-					})
+				Expect(pod.Annotations).To(HaveKeyWithValue(CurrentConfigurationAnnotation, ""))
+				Expect(pod.Annotations).To(HaveKeyWithValue(EnvironmentAnnotation, "{\"BINARY_DIR\":\"/usr\",\"TEST\":\"test-value\"}"))
+			})
+		})
 
-					It("should update the node information", func() {
-						Expect(podClient.nodeMetadata).NotTo(BeNil())
-						Expect(podClient.nodeMetadata.Name).To(Equal(podName))
-						Expect(podClient.nodeMetadata.Namespace).To(Equal(namespace))
-						Expect(podClient.nodeMetadata.Labels).To(Equal(map[string]string{
-							"testing": "testing",
-						}))
-					})
-				})
+		When("one nested flat additional env variable is set", func() {
+			BeforeEach(func() {
+				GinkgoT().Setenv("TEST", "test-value")
+				monitor = &Monitor{
+					ActiveConfiguration: &api.ProcessConfiguration{
+						BinaryPath: "/usr/bin",
+						Arguments: []api.Argument{
+							{
+								ArgumentType: api.ConcatenateArgumentType,
+								Values: []api.Argument{
+									{
+										ArgumentType: api.EnvironmentArgumentType,
+										Source:       "TEST",
+									},
+								},
+							},
+						},
+					},
+				}
+			})
 
-				When("a DeleteEvent is handled", func() {
-					BeforeEach(func() {
-						fakeInformer.Delete(node)
-					})
+			It("should update the annotations", func() {
+				pod := &corev1.Pod{}
+				Expect(fakeClient.Get(context.Background(), client.ObjectKey{Namespace: namespace, Name: podName}, pod)).NotTo(HaveOccurred())
 
-					It("should remove the node information", func() {
-						Expect(podClient.nodeMetadata).To(BeNil())
-					})
-				})
+				Expect(pod.Annotations).To(HaveKeyWithValue(CurrentConfigurationAnnotation, ""))
+				Expect(pod.Annotations).To(HaveKeyWithValue(EnvironmentAnnotation, "{\"BINARY_DIR\":\"/usr\",\"TEST\":\"test-value\"}"))
 			})
 		})
 	})

From e8045cb10a2b1c450c748a855c70c59e70695f49 Mon Sep 17 00:00:00 2001
From: "Johannes M. Scheuermann" <jscheuermann@apple.com>
Date: Thu, 13 Jun 2024 18:29:27 +0200
Subject: [PATCH 446/468] Update format

---
 fdbkubernetesmonitor/kubernetes_test.go | 3 ++-
 fdbkubernetesmonitor/monitor_test.go    | 3 ++-
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/fdbkubernetesmonitor/kubernetes_test.go b/fdbkubernetesmonitor/kubernetes_test.go
index 56e864fd0a7..d33d0881258 100644
--- a/fdbkubernetesmonitor/kubernetes_test.go
+++ b/fdbkubernetesmonitor/kubernetes_test.go
@@ -21,10 +21,11 @@ package main
 
 import (
 	"context"
-	"github.com/apple/foundationdb/fdbkubernetesmonitor/api"
 	"strconv"
 	"time"
 
+	"github.com/apple/foundationdb/fdbkubernetesmonitor/api"
+
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 	corev1 "k8s.io/api/core/v1"
diff --git a/fdbkubernetesmonitor/monitor_test.go b/fdbkubernetesmonitor/monitor_test.go
index b65517e34a3..3f3ef194704 100644
--- a/fdbkubernetesmonitor/monitor_test.go
+++ b/fdbkubernetesmonitor/monitor_test.go
@@ -20,10 +20,11 @@
 package main
 
 import (
+	"time"
+
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"time"
 )
 
 var _ = Describe("Testing FDB Kubernetes Monitor", func() {

From fbb8ae5e61798612f6394eb03a93a43b9a3c8715 Mon Sep 17 00:00:00 2001
From: Hao Fu <77984096+hfu94@users.noreply.github.com>
Date: Wed, 3 Jul 2024 08:03:03 +0800
Subject: [PATCH 447/468] Add client metric test(release-7.1) (#11466)

* Add client metric test

This restarting test start with 7.1 version and setup the sample rate
for transaction log, then it test with 7.3 version and verify transaction
log are still being written.

This test use latency as the sample transaction log metric.

Now only code is added, but it is not run by simulation. It will be run
as part of the restarting test in simulation in 7.3

* add toml

This test should also be run in 7.1 but not as restarting test, to
validate its correctness.

* Consolidate test code

* Fix format issue
---
 fdbserver/CMakeLists.txt                   |   1 +
 fdbserver/workloads/ClientMetric.actor.cpp | 242 +++++++++++++++++++++
 tests/CMakeLists.txt                       |   1 +
 tests/fast/ClientMetric.toml               |  19 ++
 4 files changed, 263 insertions(+)
 create mode 100644 fdbserver/workloads/ClientMetric.actor.cpp
 create mode 100644 tests/fast/ClientMetric.toml

diff --git a/fdbserver/CMakeLists.txt b/fdbserver/CMakeLists.txt
index 31d8c3ed8ea..956ec916e55 100644
--- a/fdbserver/CMakeLists.txt
+++ b/fdbserver/CMakeLists.txt
@@ -199,6 +199,7 @@ set(FDBSERVER_SRCS
   workloads/ChangeConfig.actor.cpp
   workloads/ChangeFeeds.actor.cpp
   workloads/ClearSingleRange.actor.cpp
+  workloads/ClientMetric.actor.cpp
   workloads/ClientTransactionProfileCorrectness.actor.cpp
   workloads/ClogSingleConnection.actor.cpp
   workloads/ClogTlog.actor.cpp
diff --git a/fdbserver/workloads/ClientMetric.actor.cpp b/fdbserver/workloads/ClientMetric.actor.cpp
new file mode 100644
index 00000000000..6a757ea2eb0
--- /dev/null
+++ b/fdbserver/workloads/ClientMetric.actor.cpp
@@ -0,0 +1,242 @@
+/*
+ * ClientMetric.actor.cpp
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2022 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "fdbserver/workloads/workloads.actor.h"
+#include "fdbserver/ServerDBInfo.h"
+#include "fdbclient/GlobalConfig.actor.h"
+#include "fdbclient/ManagementAPI.actor.h"
+#include "fdbclient/RunTransaction.actor.h"
+#include "fdbclient/Tuple.h"
+#include "flow/actorcompiler.h" // has to be last include
+
+static const StringRef sampleTrInfoKey =
+    LiteralStringRef("\xff\x02/fdbClientInfo/client_latency/SSSSSSSSSS/RRRRRRRRRRRRRRRR/NNNNTTTT/XXXX/");
+static const auto versionStampIndex = sampleTrInfoKey.toString().find('S');
+static const int versionStampLength = 10;
+
+static const Key CLIENT_LATENCY_INFO_PREFIX = LiteralStringRef("client_latency/");
+static const Key CLIENT_LATENCY_INFO_CTR_PREFIX = LiteralStringRef("client_latency_counter/");
+
+struct ClientMetricWorkload : TestWorkload {
+	static constexpr auto NAME = "ClientMetric";
+	double samplingProbability;
+	double testDuration;
+	bool toSet;
+	int64_t trInfoSizeLimit;
+	std::vector<Future<Void>> clients;
+
+	ClientMetricWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
+		samplingProbability = getOption(options,
+		                                "samplingProbability"_sr,
+		                                deterministicRandom()->random01()); // rand range 0 - 1
+		toSet = getOption(options, "toSet"_sr, false);
+		trInfoSizeLimit = getOption(options,
+		                            "trInfoSizeLimit"_sr,
+		                            deterministicRandom()->randomInt(100 * 1024, 10 * 1024 * 1024)); // 100 KB - 10 MB
+		testDuration = getOption(options, "testDuration"_sr, 1000.0);
+	}
+
+	std::string description() const override { return "ClientMetric"; }
+
+	static uint64_t getVersionStamp(KeyRef key) {
+		return bigEndian64(
+		    BinaryReader::fromStringRef<int64_t>(key.substr(versionStampIndex, versionStampLength), Unversioned()));
+	}
+
+	Future<Void> setup(Database const& cx) override {
+		if (toSet && this->clientId == 0) {
+			return changeProfilingParameters(cx, trInfoSizeLimit, samplingProbability);
+		}
+		return Void();
+	}
+
+	Future<Void> start(Database const& cx) override {
+		if (this->clientId != 0) {
+			return Void();
+		}
+		return _start(this, cx);
+	}
+
+	ACTOR Future<Void> _start(ClientMetricWorkload* self, Database cx) {
+		try {
+			self->clients.push_back(timeout(self->runner(cx, self), self->testDuration, Void()));
+			wait(waitForAll(self->clients));
+		} catch (Error& e) {
+			TraceEvent("ClientMetricError::_start").error(e);
+		}
+		return Void();
+	}
+
+	ACTOR Future<Void> changeProfilingParameters(Database cx, int64_t sizeLimit, double sampleProbability) {
+		wait(runRYWTransaction(cx, [=](Reference<ReadYourWritesTransaction> tr) -> Future<Void> {
+			Tuple rate = Tuple().appendDouble(sampleProbability);
+			Tuple size = Tuple().append(sizeLimit);
+
+			tr->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES);
+			tr->set(GlobalConfig::prefixedKey(fdbClientInfoTxnSampleRate), rate.pack());
+			tr->set(GlobalConfig::prefixedKey(fdbClientInfoTxnSizeLimit), size.pack());
+			std::cout << "Change globalconfig: sampleRate=" << sampleProbability << " sizeLimit=" << sizeLimit
+			          << std::endl;
+
+			return Void();
+		}));
+		return Void();
+	}
+
+	ACTOR Future<RangeResult> latencyRangeQuery(Database cx, int keysLimit, bool reverse) {
+		state KeySelector begin =
+		    firstGreaterOrEqual(CLIENT_LATENCY_INFO_PREFIX.withPrefix(fdbClientInfoPrefixRange.begin));
+		state KeySelector end = firstGreaterOrEqual(strinc(begin.getKey()));
+		state Reference<ReadYourWritesTransaction> tr = makeReference<ReadYourWritesTransaction>(cx);
+		state RangeResult txInfoEntries;
+		// wait to make sure client metrics are updated
+		wait(delay(CLIENT_KNOBS->CSI_STATUS_DELAY));
+		loop {
+			try {
+				tr->reset();
+				tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
+				tr->setOption(FDBTransactionOptions::LOCK_AWARE);
+				std::string sampleRateStr = "default";
+				std::string sizeLimitStr = "default";
+				const double sampleRateDbl =
+				    cx->globalConfig->get<double>(fdbClientInfoTxnSampleRate, std::numeric_limits<double>::infinity());
+				if (!std::isinf(sampleRateDbl)) {
+					sampleRateStr = std::to_string(sampleRateDbl);
+				}
+				const int64_t sizeLimit = cx->globalConfig->get<int64_t>(fdbClientInfoTxnSizeLimit, -1);
+				if (sizeLimit != -1) {
+					sizeLimitStr = std::to_string(sizeLimit);
+				}
+				std::cout << "Read from globalconfig: sampleRate=" << sampleRateStr << " sizeLimit=" << sizeLimitStr
+				          << std::endl;
+				state RangeResult kvRange = wait(
+				    tr->getRange(begin, end, keysLimit, Snapshot::False, reverse ? Reverse::True : Reverse::False));
+				if (kvRange.empty()) {
+					wait(delay(1.0));
+					std::cout << "WaitingForLatencyMetricToBePresent" << std::endl;
+					TraceEvent("WaitingForLatencyMetricToBePresent").log();
+					continue;
+				}
+				txInfoEntries.arena().dependsOn(kvRange.arena());
+				txInfoEntries.append(txInfoEntries.arena(), kvRange.begin(), kvRange.size());
+				break;
+			} catch (Error& e) {
+				wait(tr->onError(e));
+			}
+		}
+		for (auto& kv : txInfoEntries) {
+			uint64_t vs = getVersionStamp(kv.key);
+			std::cout << "VersionStamp is " << vs << std::endl;
+		}
+		return txInfoEntries;
+	}
+
+	ACTOR Future<Void> writeRandomKeys(Database cx, int total) {
+		state int cnt = 0;
+		state Transaction tr(cx);
+		try {
+			loop {
+				try {
+					wait(delay(0.001));
+					tr.reset();
+					tr.set(Key(deterministicRandom()->randomAlphaNumeric(10)),
+					       Value(Key(deterministicRandom()->randomAlphaNumeric(10))));
+					wait(tr.commit());
+					if (cnt >= total) {
+						break;
+					}
+					++cnt;
+				} catch (Error& e) {
+					wait(tr.onError(e));
+				}
+			}
+		} catch (Error& e) {
+			TraceEvent(SevError, "ClientMetricErrorWhenWriteKeys").error(e);
+			throw;
+		}
+		std::cout << "writeRandomKeys finish, written=" << cnt << std::endl;
+		return Void();
+	}
+
+	ACTOR Future<uint64_t> writeKeysAndGetLatencyVersion(Database cx,
+	                                                     ClientMetricWorkload* self,
+	                                                     int numKeys,
+	                                                     int previousVS) {
+		state int retry = 0;
+		state int max_retry = 10;
+		state int keysLimit = 1;
+		loop {
+			if (retry > max_retry) {
+				// this should not happen, it should succeed after a few retry
+				ASSERT(false);
+			}
+			// write random keys to generate some latency metrics
+			wait(self->writeRandomKeys(cx, numKeys));
+			// get the latest latency metric and parse its version stamp
+			RangeResult r = wait(self->latencyRangeQuery(cx, keysLimit, true));
+			if (r.size() == 0) {
+				// latency metrics might not be present due to transaction batching, retry a few times
+				++retry;
+				continue;
+			}
+			ASSERT(r.size() > 0);
+			// [0] is the latest version, as we have reverse = true
+			KeyRef latest = r[0].key;
+			uint64_t vs = getVersionStamp(latest);
+			ASSERT(vs >= previousVS);
+			if (vs == previousVS) {
+				// it means there is no new latency metrics, retry until we see one
+				++retry;
+				continue;
+			}
+			return vs;
+		}
+	}
+
+	// goal:
+	//      write some random keys, check the latency metric and the latest version stamp vs1
+	//      write some other random keys, check the latency metric and latest version stamp again vs2
+	//      vs2 should be strictly larger than vs1, to verify new latency metrics are added
+	ACTOR Future<Void> runner(Database cx, ClientMetricWorkload* self) {
+		try {
+			state int initialWrites = deterministicRandom()->randomInt(1, 5);
+			state int secondWrites = deterministicRandom()->randomInt(1, 5);
+
+			state uint64_t zeroVS = 0;
+			state uint64_t vs1 = wait(self->writeKeysAndGetLatencyVersion(cx, self, initialWrites, zeroVS));
+			std::cout << "vs1=" << vs1 << std::endl;
+			ASSERT(vs1 > zeroVS);
+
+			state uint64_t vs2 = wait(self->writeKeysAndGetLatencyVersion(cx, self, secondWrites, vs1));
+			std::cout << "vs2=" << vs2 << std::endl;
+			ASSERT(vs2 > vs1);
+
+		} catch (Error& e) {
+			TraceEvent("ClientMetricError").error(e);
+		}
+		return Void();
+	}
+
+	Future<bool> check(Database const& cx) override { return true; }
+
+	void getMetrics(std::vector<PerfMetric>& m) override {}
+};
+
+WorkloadFactory<ClientMetricWorkload> ClientMetricWorkloadFactory("ClientMetric");
diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt
index 3eef8efffe9..497099bf4b9 100644
--- a/tests/CMakeLists.txt
+++ b/tests/CMakeLists.txt
@@ -135,6 +135,7 @@ if(WITH_PYTHON)
   add_fdb_test(TEST_FILES fast/BlobGranuleVerifyAtomicOps.toml)
   add_fdb_test(TEST_FILES fast/BlobGranuleVerifyCycle.toml)
   add_fdb_test(TEST_FILES fast/CacheTest.toml)
+  add_fdb_test(TEST_FILES fast/ClientMetric.toml)
   add_fdb_test(TEST_FILES fast/CloggedSideband.toml)
   add_fdb_test(TEST_FILES fast/ConfigureLocked.toml)
   add_fdb_test(TEST_FILES fast/ConstrainedRandomSelector.toml)
diff --git a/tests/fast/ClientMetric.toml b/tests/fast/ClientMetric.toml
new file mode 100644
index 00000000000..ef4277282e8
--- /dev/null
+++ b/tests/fast/ClientMetric.toml
@@ -0,0 +1,19 @@
+[[test]]
+testTitle='ClientMetricTest'
+clearAfterTest=false
+
+    [[test.workload]]
+    testName='ClientMetric'
+    toSet=true
+    samplingProbability=1.0
+    testDuration=500.0
+    trInfoSizeLimit=177654
+
+    [[test.workload]]
+    testName='RandomClogging'
+    testDuration=30.0
+
+    [[test.workload]]
+    testName='SaveAndKill'
+    restartInfoLocation='simfdb/restartInfo.ini'
+    testDuration=500.0

From 10b98f2bebc125178ee5dc37fbc8e58ccf2f1b42 Mon Sep 17 00:00:00 2001
From: Johannes Scheuermann <johscheuer@users.noreply.github.com>
Date: Fri, 14 Jun 2024 17:44:21 +0200
Subject: [PATCH 448/468] Add a mechanism to allow to specifiy the command line
 flags over env variables (#11462)

---
 fdbkubernetesmonitor/main.go | 35 +++++++++++++++++++++++++++++++++--
 1 file changed, 33 insertions(+), 2 deletions(-)

diff --git a/fdbkubernetesmonitor/main.go b/fdbkubernetesmonitor/main.go
index 1b8205c1c5e..2f79ae44c94 100644
--- a/fdbkubernetesmonitor/main.go
+++ b/fdbkubernetesmonitor/main.go
@@ -79,6 +79,35 @@ func initLogger(logPath string) *zap.Logger {
 	return zap.New(zapcore.NewCore(zapcore.NewJSONEncoder(zap.NewProductionEncoderConfig()), zapcore.AddSync(logWriter), zapcore.InfoLevel))
 }
 
+func convertEnvironmentVariableNameToFlagName(s string) string {
+	return strings.ReplaceAll(strings.ToLower(s), "_", "-")
+}
+
+// parseFlagsAndSetEnvDefaults will parse the provided flags. The following order will be taken:
+// 1. If the flag is provided, take this value
+// 2. If an environment variable with the same name as the flag (just upper case and an underscore instead of a hyphen).
+// 3. Use the default value.
+func parseFlagsAndSetEnvDefaults() error {
+	for _, env := range os.Environ() {
+		envSplit := strings.SplitN(env, "=", 2)
+		envName := envSplit[0]
+		envValue := envSplit[1]
+		flag := pflag.CommandLine.Lookup(convertEnvironmentVariableNameToFlagName(envName))
+		// If no flag with that name is present or the flag is already specified with another value than the default one
+		// ignore the value from the env variable.
+		if flag == nil || flag.Changed {
+			continue
+		}
+
+		if err := flag.Value.Set(envValue); err != nil {
+			return err
+		}
+	}
+	pflag.Parse()
+
+	return nil
+}
+
 func main() {
 	var copyFiles, copyBinaries, copyLibraries, requiredCopyFiles []string
 	var inputDir, copyPrimaryLibrary, binaryOutputDirectory string
@@ -103,10 +132,12 @@ func main() {
 	pflag.BoolVar(&enablePprof, "enable-pprof", false, "Enables /debug/pprof endpoints on the listen address")
 	pflag.StringVar(&listenAddress, "listen-address", ":8081", "An address and port to listen on")
 	pflag.BoolVar(&enableNodeWatch, "enable-node-watch", false, "Enables the fdb-kubernetes-monitor to watch the node resource where the current Pod is running. This can be used to read node labels")
-	pflag.Parse()
+	err := parseFlagsAndSetEnvDefaults()
+	if err != nil {
+		panic(err)
+	}
 
 	logger := zapr.NewLogger(initLogger(logPath))
-
 	versionBytes, err := os.ReadFile(versionFilePath)
 	if err != nil {
 		panic(err)

From 895a0c4db1c475f7a0f52585f391ad1349b08170 Mon Sep 17 00:00:00 2001
From: Nicole Morales <nicole_morales@apple.com>
Date: Wed, 5 Jun 2024 11:51:48 +0100
Subject: [PATCH 449/468] add TLS ability to fdb kubernetes monitor

---
 fdbkubernetesmonitor/main.go    | 11 ++++++++++-
 fdbkubernetesmonitor/monitor.go | 15 +++++++++++++--
 2 files changed, 23 insertions(+), 3 deletions(-)

diff --git a/fdbkubernetesmonitor/main.go b/fdbkubernetesmonitor/main.go
index 2f79ae44c94..bc6bcd230e7 100644
--- a/fdbkubernetesmonitor/main.go
+++ b/fdbkubernetesmonitor/main.go
@@ -47,6 +47,8 @@ var (
 	mainContainerVersion string
 	additionalEnvFile    string
 	listenAddress        string
+	certFile             string
+	keyFile              string
 	processCount         int
 	enablePprof          bool
 	enableNodeWatch      bool
@@ -132,6 +134,8 @@ func main() {
 	pflag.BoolVar(&enablePprof, "enable-pprof", false, "Enables /debug/pprof endpoints on the listen address")
 	pflag.StringVar(&listenAddress, "listen-address", ":8081", "An address and port to listen on")
 	pflag.BoolVar(&enableNodeWatch, "enable-node-watch", false, "Enables the fdb-kubernetes-monitor to watch the node resource where the current Pod is running. This can be used to read node labels")
+	pflag.StringVar(&certFile, "cert-file", "", "The location of a PEM cert for the prometheus HTTP server")
+	pflag.StringVar(&keyFile, "key-file", "", "The location of a PEM key for the prometheus HTTP server")
 	err := parseFlagsAndSetEnvDefaults()
 	if err != nil {
 		panic(err)
@@ -157,7 +161,12 @@ func main() {
 			logger.Error(err, "Error loading additional environment")
 			os.Exit(1)
 		}
-		StartMonitor(context.Background(), logger, path.Join(inputDir, monitorConfFile), customEnvironment, processCount, listenAddress, enablePprof, currentContainerVersion, enableNodeWatch)
+		promConfig := httpConfig{
+			listenAddr: listenAddress,
+			certFile:   certFile,
+			keyFile:    keyFile,
+		}
+		StartMonitor(context.Background(), logger, path.Join(inputDir, monitorConfFile), customEnvironment, processCount, promConfig, enablePprof, currentContainerVersion, enableNodeWatch)
 	case executionModeInit:
 		err = CopyFiles(logger, outputDir, copyDetails, requiredCopies)
 		if err != nil {
diff --git a/fdbkubernetesmonitor/monitor.go b/fdbkubernetesmonitor/monitor.go
index b70352b8576..abba549f3b2 100644
--- a/fdbkubernetesmonitor/monitor.go
+++ b/fdbkubernetesmonitor/monitor.go
@@ -106,8 +106,12 @@ type Monitor struct {
 	metrics *metrics
 }
 
+type httpConfig struct {
+	listenAddr, certFile, keyFile string
+}
+
 // StartMonitor starts the monitor loop.
-func StartMonitor(ctx context.Context, logger logr.Logger, configFile string, customEnvironment map[string]string, processCount int, listenAddr string, enableDebug bool, currentContainerVersion string, enableNodeWatcher bool) {
+func StartMonitor(ctx context.Context, logger logr.Logger, configFile string, customEnvironment map[string]string, processCount int, promConfig httpConfig, enableDebug bool, currentContainerVersion string, enableNodeWatcher bool) {
 	podClient, err := CreatePodClient(ctx, logger, enableNodeWatcher, setupCache)
 	if err != nil {
 		logger.Error(err, "could not create Pod client")
@@ -152,7 +156,14 @@ func StartMonitor(ctx context.Context, logger logr.Logger, configFile string, cu
 	// Add Prometheus support
 	mux.Handle("/metrics", promHandler)
 	go func() {
-		err := http.ListenAndServe(listenAddr, mux)
+		if promConfig.keyFile != "" || promConfig.certFile != "" {
+			err := http.ListenAndServeTLS(promConfig.listenAddr, promConfig.certFile, promConfig.keyFile, mux)
+			if err != nil {
+				logger.Error(err, "could not start HTTPS server")
+				os.Exit(1)
+			}
+		}
+		err := http.ListenAndServe(promConfig.listenAddr, mux)
 		if err != nil {
 			logger.Error(err, "could not start HTTP server")
 			os.Exit(1)

From bb43d9200c1347a33437530d11fcc78d566d9747 Mon Sep 17 00:00:00 2001
From: Nicole Morales <nicole_morales@apple.com>
Date: Fri, 14 Jun 2024 16:39:52 +0100
Subject: [PATCH 450/468] review fixes

---
 fdbkubernetesmonitor/certloader/certloader.go | 56 +++++++++++++++++++
 fdbkubernetesmonitor/main.go                  | 11 ++--
 fdbkubernetesmonitor/monitor.go               | 17 +++++-
 3 files changed, 76 insertions(+), 8 deletions(-)
 create mode 100644 fdbkubernetesmonitor/certloader/certloader.go

diff --git a/fdbkubernetesmonitor/certloader/certloader.go b/fdbkubernetesmonitor/certloader/certloader.go
new file mode 100644
index 00000000000..6cd081a54f5
--- /dev/null
+++ b/fdbkubernetesmonitor/certloader/certloader.go
@@ -0,0 +1,56 @@
+package certloader
+
+import (
+	"crypto/tls"
+	"fmt"
+	"os"
+	"time"
+
+	"github.com/go-logr/logr"
+)
+
+// CertLoader is used to reload certificates if needed.
+type CertLoader struct {
+	// CertFile specifies the path to the x509 certificate.
+	CertFile string
+	// CertFile specifies the path to the x509 private key.
+	KeyFile string
+	// cachedCert will cache the loaded certificate key pair.
+	cachedCert *tls.Certificate
+	// cachedCertModTime is used to track the last modification time of the key file.
+	cachedCertModTime time.Time
+	// logger is the logger for logging.
+	logger logr.Logger
+}
+
+// NewCertLoader creates a new CertLoader.
+func NewCertLoader(logger logr.Logger, certFile string, keyFile string) *CertLoader {
+	return &CertLoader{
+		CertFile: certFile,
+		KeyFile:  keyFile,
+		logger:   logger.WithName("CertLoader"),
+	}
+}
+
+// GetCertificate returns the certificate for the TLS requests and will return the cached certificate if the file has not been changed.
+func (certLoader *CertLoader) GetCertificate(_ *tls.ClientHelloInfo) (*tls.Certificate, error) {
+	stat, err := os.Stat(certLoader.KeyFile)
+	if err != nil {
+		err = fmt.Errorf("failed checking key file: %s modification time: %w", certLoader.KeyFile, err)
+		certLoader.logger.Error(err, "could not load information from key file", "certFile", certLoader.CertFile, "keyFile", certLoader.KeyFile)
+		return nil, err
+	}
+
+	if certLoader.cachedCert == nil || stat.ModTime().After(certLoader.cachedCertModTime) {
+		certLoader.logger.Info("loading new certificates", "certFile", certLoader.CertFile, "keyFile", certLoader.KeyFile, "cachedModificationTime", certLoader.cachedCertModTime.String(), "currentModificationTime", stat.ModTime().String())
+		pair, err := tls.LoadX509KeyPair(certLoader.CertFile, certLoader.KeyFile)
+		if err != nil {
+			return nil, fmt.Errorf("failed loading tls key pair: %w", err)
+		}
+
+		certLoader.cachedCert = &pair
+		certLoader.cachedCertModTime = stat.ModTime()
+	}
+
+	return certLoader.cachedCert, nil
+}
diff --git a/fdbkubernetesmonitor/main.go b/fdbkubernetesmonitor/main.go
index bc6bcd230e7..b1681979089 100644
--- a/fdbkubernetesmonitor/main.go
+++ b/fdbkubernetesmonitor/main.go
@@ -47,8 +47,6 @@ var (
 	mainContainerVersion string
 	additionalEnvFile    string
 	listenAddress        string
-	certFile             string
-	keyFile              string
 	processCount         int
 	enablePprof          bool
 	enableNodeWatch      bool
@@ -112,7 +110,7 @@ func parseFlagsAndSetEnvDefaults() error {
 
 func main() {
 	var copyFiles, copyBinaries, copyLibraries, requiredCopyFiles []string
-	var inputDir, copyPrimaryLibrary, binaryOutputDirectory string
+	var inputDir, copyPrimaryLibrary, binaryOutputDirectory, certPath, keyPath, rootCaPath string
 
 	pflag.StringVar(&executionModeString, "mode", "launcher", "Execution mode. Valid options are launcher, sidecar, and init")
 	pflag.StringVar(&fdbserverPath, "fdbserver-path", "/usr/bin/fdbserver", "Path to the fdbserver binary")
@@ -134,6 +132,9 @@ func main() {
 	pflag.BoolVar(&enablePprof, "enable-pprof", false, "Enables /debug/pprof endpoints on the listen address")
 	pflag.StringVar(&listenAddress, "listen-address", ":8081", "An address and port to listen on")
 	pflag.BoolVar(&enableNodeWatch, "enable-node-watch", false, "Enables the fdb-kubernetes-monitor to watch the node resource where the current Pod is running. This can be used to read node labels")
+	pflag.StringVar(&certPath, "certificate-path", "", "The path of a PEM cert for the prometheus/pprof HTTP server")
+	pflag.StringVar(&keyPath, "certificate-key-path", "", "The path of a PEM key for the prometheus/pprof HTTP server")
+	pflag.Parse()
 	pflag.StringVar(&certFile, "cert-file", "", "The location of a PEM cert for the prometheus HTTP server")
 	pflag.StringVar(&keyFile, "key-file", "", "The location of a PEM key for the prometheus HTTP server")
 	err := parseFlagsAndSetEnvDefaults()
@@ -163,8 +164,8 @@ func main() {
 		}
 		promConfig := httpConfig{
 			listenAddr: listenAddress,
-			certFile:   certFile,
-			keyFile:    keyFile,
+			certPath:   certPath,
+			keyPath:    keyPath,
 		}
 		StartMonitor(context.Background(), logger, path.Join(inputDir, monitorConfFile), customEnvironment, processCount, promConfig, enablePprof, currentContainerVersion, enableNodeWatch)
 	case executionModeInit:
diff --git a/fdbkubernetesmonitor/monitor.go b/fdbkubernetesmonitor/monitor.go
index abba549f3b2..e6d1ed43bc2 100644
--- a/fdbkubernetesmonitor/monitor.go
+++ b/fdbkubernetesmonitor/monitor.go
@@ -22,6 +22,7 @@ package main
 import (
 	"bufio"
 	"context"
+	"crypto/tls"
 	"encoding/json"
 	"errors"
 	"fmt"
@@ -38,6 +39,7 @@ import (
 	"time"
 
 	"github.com/apple/foundationdb/fdbkubernetesmonitor/api"
+	"github.com/apple/foundationdb/fdbkubernetesmonitor/certloader"
 	"github.com/fsnotify/fsnotify"
 	"github.com/go-logr/logr"
 	"github.com/prometheus/client_golang/prometheus"
@@ -107,7 +109,7 @@ type Monitor struct {
 }
 
 type httpConfig struct {
-	listenAddr, certFile, keyFile string
+	listenAddr, certPath, keyPath, rootCaPath string
 }
 
 // StartMonitor starts the monitor loop.
@@ -156,8 +158,17 @@ func StartMonitor(ctx context.Context, logger logr.Logger, configFile string, cu
 	// Add Prometheus support
 	mux.Handle("/metrics", promHandler)
 	go func() {
-		if promConfig.keyFile != "" || promConfig.certFile != "" {
-			err := http.ListenAndServeTLS(promConfig.listenAddr, promConfig.certFile, promConfig.keyFile, mux)
+		if promConfig.keyPath != "" || promConfig.certPath != "" {
+			certLoader := certloader.NewCertLoader(logger, promConfig.certPath, promConfig.keyPath)
+			tlsConfig := &tls.Config{
+				GetCertificate: certLoader.GetCertificate,
+			}
+			server := &http.Server{
+				Addr:      promConfig.listenAddr,
+				Handler:   mux,
+				TLSConfig: tlsConfig,
+			}
+			err = server.ListenAndServeTLS("", "")
 			if err != nil {
 				logger.Error(err, "could not start HTTPS server")
 				os.Exit(1)

From 68b2b6cd5a8925b364416f3f658e47b8736c77e8 Mon Sep 17 00:00:00 2001
From: Nicole Morales <nicole_morales@apple.com>
Date: Fri, 14 Jun 2024 16:59:39 +0100
Subject: [PATCH 451/468] move certloader to internal folder

---
 fdbkubernetesmonitor/{ => internal}/certloader/certloader.go | 0
 fdbkubernetesmonitor/main.go                                 | 5 +----
 fdbkubernetesmonitor/monitor.go                              | 2 +-
 3 files changed, 2 insertions(+), 5 deletions(-)
 rename fdbkubernetesmonitor/{ => internal}/certloader/certloader.go (100%)

diff --git a/fdbkubernetesmonitor/certloader/certloader.go b/fdbkubernetesmonitor/internal/certloader/certloader.go
similarity index 100%
rename from fdbkubernetesmonitor/certloader/certloader.go
rename to fdbkubernetesmonitor/internal/certloader/certloader.go
diff --git a/fdbkubernetesmonitor/main.go b/fdbkubernetesmonitor/main.go
index b1681979089..1eedc95f09f 100644
--- a/fdbkubernetesmonitor/main.go
+++ b/fdbkubernetesmonitor/main.go
@@ -110,7 +110,7 @@ func parseFlagsAndSetEnvDefaults() error {
 
 func main() {
 	var copyFiles, copyBinaries, copyLibraries, requiredCopyFiles []string
-	var inputDir, copyPrimaryLibrary, binaryOutputDirectory, certPath, keyPath, rootCaPath string
+	var inputDir, copyPrimaryLibrary, binaryOutputDirectory, certPath, keyPath string
 
 	pflag.StringVar(&executionModeString, "mode", "launcher", "Execution mode. Valid options are launcher, sidecar, and init")
 	pflag.StringVar(&fdbserverPath, "fdbserver-path", "/usr/bin/fdbserver", "Path to the fdbserver binary")
@@ -134,9 +134,6 @@ func main() {
 	pflag.BoolVar(&enableNodeWatch, "enable-node-watch", false, "Enables the fdb-kubernetes-monitor to watch the node resource where the current Pod is running. This can be used to read node labels")
 	pflag.StringVar(&certPath, "certificate-path", "", "The path of a PEM cert for the prometheus/pprof HTTP server")
 	pflag.StringVar(&keyPath, "certificate-key-path", "", "The path of a PEM key for the prometheus/pprof HTTP server")
-	pflag.Parse()
-	pflag.StringVar(&certFile, "cert-file", "", "The location of a PEM cert for the prometheus HTTP server")
-	pflag.StringVar(&keyFile, "key-file", "", "The location of a PEM key for the prometheus HTTP server")
 	err := parseFlagsAndSetEnvDefaults()
 	if err != nil {
 		panic(err)
diff --git a/fdbkubernetesmonitor/monitor.go b/fdbkubernetesmonitor/monitor.go
index e6d1ed43bc2..63583faa3d3 100644
--- a/fdbkubernetesmonitor/monitor.go
+++ b/fdbkubernetesmonitor/monitor.go
@@ -39,7 +39,7 @@ import (
 	"time"
 
 	"github.com/apple/foundationdb/fdbkubernetesmonitor/api"
-	"github.com/apple/foundationdb/fdbkubernetesmonitor/certloader"
+	"github.com/apple/foundationdb/fdbkubernetesmonitor/internal/certloader"
 	"github.com/fsnotify/fsnotify"
 	"github.com/go-logr/logr"
 	"github.com/prometheus/client_golang/prometheus"

From f72d378f62d16f379231b329bf2beda43124b766 Mon Sep 17 00:00:00 2001
From: Johannes Scheuermann <johscheuer@users.noreply.github.com>
Date: Tue, 18 Jun 2024 11:02:09 +0200
Subject: [PATCH 452/468] fdbkubernetesmonitor: Retry update pod annotations in
 case of an error (#11458)

---
 fdbkubernetesmonitor/kubernetes.go | 60 ++++++++++++++++++++++--------
 fdbkubernetesmonitor/monitor.go    |  2 +-
 2 files changed, 45 insertions(+), 17 deletions(-)

diff --git a/fdbkubernetesmonitor/kubernetes.go b/fdbkubernetesmonitor/kubernetes.go
index f5f7722d1a8..8e81a7c3436 100644
--- a/fdbkubernetesmonitor/kubernetes.go
+++ b/fdbkubernetesmonitor/kubernetes.go
@@ -22,14 +22,18 @@ package main
 import (
 	"context"
 	"encoding/json"
+	"fmt"
 	"os"
 	"path"
 	"strconv"
 	"time"
 
+	"k8s.io/client-go/util/retry"
+
 	"github.com/apple/foundationdb/fdbkubernetesmonitor/api"
 	"github.com/go-logr/logr"
 	corev1 "k8s.io/api/core/v1"
+	k8serrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/fields"
 	"k8s.io/apimachinery/pkg/runtime"
@@ -253,26 +257,50 @@ func (podClient *PodClient) updateFdbClusterTimestampAnnotation() error {
 // updateAnnotationsOnPod will update the annotations with the provided annotationChanges. If an annotation exists, it
 // will be updated if the annotation is absent it will be added.
 func (podClient *PodClient) updateAnnotationsOnPod(annotationChanges map[string]string) error {
-	annotations := podClient.podMetadata.Annotations
-	if len(annotations) == 0 {
-		annotations = map[string]string{}
+	if podClient.podMetadata == nil {
+		return fmt.Errorf("pod client has no metadata present")
 	}
 
-	for key, val := range annotationChanges {
-		annotations[key] = val
+	if !podClient.podMetadata.DeletionTimestamp.IsZero() {
+		return fmt.Errorf("pod is marked for deletion, cannot update annotations")
 	}
 
-	return podClient.Patch(context.Background(), &corev1.Pod{
-		TypeMeta: metav1.TypeMeta{
-			Kind:       "Pod",
-			APIVersion: "v1",
-		},
-		ObjectMeta: metav1.ObjectMeta{
-			Namespace:   podClient.podMetadata.Namespace,
-			Name:        podClient.podMetadata.Name,
-			Annotations: annotations,
-		},
-	}, client.Apply, client.FieldOwner("fdb-kubernetes-monitor"), client.ForceOwnership)
+	// If any error occurs during the update of the annotations, make sure we retry it.
+	return retry.OnError(retry.DefaultRetry, func(err error) bool {
+		podClient.Logger.Error(err, "could not update annotations of pod")
+		// If the pod is marked for deletion, we don't have to retry the patch.
+		if !podClient.podMetadata.DeletionTimestamp.IsZero() {
+			return false
+		}
+
+		// If the resource is not found or the process is forbidden to update the metadata, don't retry it.
+		if k8serrors.IsNotFound(err) || k8serrors.IsForbidden(err) {
+			return false
+		}
+
+		return true
+	}, func() error {
+		annotations := podClient.podMetadata.Annotations
+		if len(annotations) == 0 {
+			annotations = map[string]string{}
+		}
+
+		for key, val := range annotationChanges {
+			annotations[key] = val
+		}
+
+		return podClient.Patch(context.Background(), &corev1.Pod{
+			TypeMeta: metav1.TypeMeta{
+				Kind:       "Pod",
+				APIVersion: "v1",
+			},
+			ObjectMeta: metav1.ObjectMeta{
+				Namespace:   podClient.podMetadata.Namespace,
+				Name:        podClient.podMetadata.Name,
+				Annotations: annotations,
+			},
+		}, client.Apply, client.FieldOwner("fdb-kubernetes-monitor"), client.ForceOwnership)
+	})
 }
 
 // OnAdd is called when an object is added.
diff --git a/fdbkubernetesmonitor/monitor.go b/fdbkubernetesmonitor/monitor.go
index 63583faa3d3..09f897e9071 100644
--- a/fdbkubernetesmonitor/monitor.go
+++ b/fdbkubernetesmonitor/monitor.go
@@ -559,7 +559,7 @@ func (monitor *Monitor) Run() {
 	go func() { monitor.WatchConfiguration(watcher) }()
 
 	// The cluster file will be created and managed by the fdbserver processes, so we have to wait until the fdbserver
-	// processes have been started. Except for the initial cluster creation his file should be present as soon as the
+	// processes have been started. Except for the initial cluster creation this file should be present as soon as the
 	// monitor starts the processes.
 	for {
 		_, err = os.Stat(fdbClusterFilePath)

From 268875440b639bf91be63cdb6560a3a2e5cc397d Mon Sep 17 00:00:00 2001
From: Johannes Scheuermann <johscheuer@users.noreply.github.com>
Date: Mon, 24 Jun 2024 17:01:23 +0200
Subject: [PATCH 453/468] Add support for the isolate process group annotation
 to shutdown fdbserver processes (#11464)

* Add support for the isolate process group annotation to shutdown fdbserver processes
---
 fdbkubernetesmonitor/api/annotations.go  |  47 ++++
 fdbkubernetesmonitor/api/config.go       |  13 +-
 fdbkubernetesmonitor/api/config_test.go  |  23 ++
 fdbkubernetesmonitor/api/suite_test.go   |   2 +-
 fdbkubernetesmonitor/api/version.go      | 185 ++++++++++++
 fdbkubernetesmonitor/api/version_test.go | 122 ++++++++
 fdbkubernetesmonitor/copy.go             |   2 +-
 fdbkubernetesmonitor/copy_test.go        |  14 +-
 fdbkubernetesmonitor/kubernetes.go       |  94 +++----
 fdbkubernetesmonitor/kubernetes_test.go  |  48 ++--
 fdbkubernetesmonitor/main.go             |  20 +-
 fdbkubernetesmonitor/metrics.go          |  16 +-
 fdbkubernetesmonitor/metrics_test.go     |   2 +-
 fdbkubernetesmonitor/monitor.go          | 340 ++++++++++++++---------
 fdbkubernetesmonitor/monitor_test.go     | 119 +++++++-
 fdbkubernetesmonitor/suite_test.go       |   2 +-
 16 files changed, 799 insertions(+), 250 deletions(-)
 create mode 100644 fdbkubernetesmonitor/api/annotations.go
 create mode 100644 fdbkubernetesmonitor/api/version.go
 create mode 100644 fdbkubernetesmonitor/api/version_test.go

diff --git a/fdbkubernetesmonitor/api/annotations.go b/fdbkubernetesmonitor/api/annotations.go
new file mode 100644
index 00000000000..e18aac57f1b
--- /dev/null
+++ b/fdbkubernetesmonitor/api/annotations.go
@@ -0,0 +1,47 @@
+// annotations.go
+//
+// This source file is part of the FoundationDB open source project
+//
+// Copyright 2021-2024 Apple Inc. and the FoundationDB project authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package api
+
+const (
+	// CurrentConfigurationAnnotation is the annotation we use to store the
+	// latest configuration.
+	CurrentConfigurationAnnotation = "foundationdb.org/launcher-current-configuration"
+
+	// EnvironmentAnnotation is the annotation we use to store the environment
+	// variables.
+	EnvironmentAnnotation = "foundationdb.org/launcher-environment"
+
+	// OutdatedConfigMapAnnotation is the annotation we read to get notified of
+	// outdated configuration.
+	OutdatedConfigMapAnnotation = "foundationdb.org/outdated-config-map-seen"
+
+	// DelayShutdownAnnotation defines how long the FDB Kubernetes monitor process should sleep before shutting itself down.
+	// The FDB Kubernetes monitor will always shutdown all fdbserver processes, independent of this setting.
+	// The value of this annotation must be a duration like "60s".
+	DelayShutdownAnnotation = "foundationdb.org/delay-shutdown"
+
+	// ClusterFileChangeDetectedAnnotation is the annotation that will be updated if the fdb.cluster file is updated.
+	ClusterFileChangeDetectedAnnotation = "foundationdb.org/cluster-file-change"
+
+	// IsolateProcessGroupAnnotation is the annotation that defines if the current Pod should be isolated. Isolated
+	// process groups will shutdown the fdbserver instance but keep the Pod and other Kubernetes resources running
+	// for debugging purpose.
+	IsolateProcessGroupAnnotation = "foundationdb.org/isolate-process-group"
+)
diff --git a/fdbkubernetesmonitor/api/config.go b/fdbkubernetesmonitor/api/config.go
index 169cc6524e9..5848d090cdf 100644
--- a/fdbkubernetesmonitor/api/config.go
+++ b/fdbkubernetesmonitor/api/config.go
@@ -25,13 +25,15 @@ import (
 	"os"
 	"strconv"
 	"strings"
+
+	"k8s.io/utils/pointer"
 )
 
 // ProcessConfiguration models the configuration for starting a FoundationDB
 // process.
 type ProcessConfiguration struct {
 	// Version provides the version of FoundationDB the process should run.
-	Version string `json:"version"`
+	Version *Version `json:"version"`
 
 	// RunServers defines whether we should run the server processes.
 	// This defaults to true, but you can set it to false to prevent starting
@@ -183,3 +185,12 @@ func (configuration *ProcessConfiguration) GenerateArguments(processNumber int,
 	}
 	return results, nil
 }
+
+// ShouldRunServers returns true if RunServers is unset or set to true.
+func (configuration *ProcessConfiguration) ShouldRunServers() bool {
+	if configuration == nil {
+		return false
+	}
+
+	return pointer.BoolDeref(configuration.RunServers, true)
+}
diff --git a/fdbkubernetesmonitor/api/config_test.go b/fdbkubernetesmonitor/api/config_test.go
index a5626d8c8b1..1ea417e31c7 100644
--- a/fdbkubernetesmonitor/api/config_test.go
+++ b/fdbkubernetesmonitor/api/config_test.go
@@ -47,6 +47,7 @@ var _ = Describe("Testing FDB Kubernetes Monitor API", func() {
 		BeforeEach(func() {
 			var err error
 			config := loadConfigFromFile(".testdata/default_config.json")
+			Expect(config.Version).To(Equal(&Version{Major: 6, Minor: 3, Patch: 15}))
 			arguments, err = config.GenerateArguments(1, map[string]string{
 				"FDB_PUBLIC_IP":   "10.0.0.1",
 				"FDB_POD_IP":      "192.168.0.1",
@@ -245,4 +246,26 @@ var _ = Describe("Testing FDB Kubernetes Monitor API", func() {
 			})
 		})
 	})
+
+	When("marshalling a process configuration", func() {
+		var out string
+
+		BeforeEach(func() {
+			config := &ProcessConfiguration{
+				Version: &Version{
+					Major: 7,
+					Minor: 1,
+					Patch: 57,
+				},
+			}
+
+			data, err := json.Marshal(config)
+			Expect(err).NotTo(HaveOccurred())
+			out = string(data)
+		})
+
+		It("should parse it correct", func() {
+			Expect(out).To(Equal("{\"version\":\"7.1.57\"}"))
+		})
+	})
 })
diff --git a/fdbkubernetesmonitor/api/suite_test.go b/fdbkubernetesmonitor/api/suite_test.go
index eb6f12252eb..8570c9cb279 100644
--- a/fdbkubernetesmonitor/api/suite_test.go
+++ b/fdbkubernetesmonitor/api/suite_test.go
@@ -3,7 +3,7 @@
  *
  * This source file is part of the FoundationDB open source project
  *
- * Copyright 2023-2024 Apple Inc. and the FoundationDB project authors
+ * Copyright 2021-2024 Apple Inc. and the FoundationDB project authors
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/fdbkubernetesmonitor/api/version.go b/fdbkubernetesmonitor/api/version.go
new file mode 100644
index 00000000000..b44e4bd2cbf
--- /dev/null
+++ b/fdbkubernetesmonitor/api/version.go
@@ -0,0 +1,185 @@
+/*
+ * version.go
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2021-2024 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package api
+
+import (
+	"fmt"
+	"regexp"
+	"strconv"
+	"strings"
+)
+
+// Version represents a version of FoundationDB.
+//
+// This provides convenience methods for checking features available in
+// different versions.
+type Version struct {
+	// Major is the major version
+	Major int
+
+	// Minor is the minor version
+	Minor int
+
+	// Patch is the patch version
+	Patch int
+
+	// ReleaseCandidate is the number from the `-rc\d+` suffix version
+	// of the version if it exists
+	ReleaseCandidate int
+}
+
+// versionRegex describes the format of a FoundationDB version.
+var versionRegex = regexp.MustCompile(`(\d+)\.(\d+)\.(\d+)(-rc(\d+))?`)
+
+// MarshalJSON custom implementation of MarshalJSON
+func (version *Version) MarshalJSON() ([]byte, error) {
+	return []byte(fmt.Sprintf("\"%s\"", version.String())), nil
+}
+
+// UnmarshalJSON custom implementation of UnmarshalJSON
+func (version *Version) UnmarshalJSON(data []byte) error {
+	trimmed := strings.Trim(string(data), "\"")
+	currentVersion, err := ParseFdbVersion(trimmed)
+	if err != nil {
+		return err
+	}
+
+	version.Major = currentVersion.Major
+	version.Minor = currentVersion.Minor
+	version.Patch = currentVersion.Patch
+	version.ReleaseCandidate = currentVersion.ReleaseCandidate
+	return nil
+}
+
+// ParseFdbVersion parses a version from its string representation.
+func ParseFdbVersion(version string) (Version, error) {
+	matches := versionRegex.FindStringSubmatch(version)
+	if matches == nil {
+		return Version{}, fmt.Errorf("could not parse FDB version from %s", version)
+	}
+
+	major, err := strconv.Atoi(matches[1])
+	if err != nil {
+		return Version{}, err
+	}
+
+	minor, err := strconv.Atoi(matches[2])
+	if err != nil {
+		return Version{}, err
+	}
+
+	patch, err := strconv.Atoi(matches[3])
+	if err != nil {
+		return Version{}, err
+	}
+
+	rc, err := strconv.Atoi(matches[5])
+	if err != nil {
+		rc = 0
+	}
+
+	return Version{Major: major, Minor: minor, Patch: patch, ReleaseCandidate: rc}, nil
+}
+
+// String gets the string representation of an FDB version.
+func (version Version) String() string {
+	if version.ReleaseCandidate == 0 {
+		return fmt.Sprintf("%d.%d.%d", version.Major, version.Minor, version.Patch)
+	}
+	return fmt.Sprintf("%d.%d.%d-rc%d", version.Major, version.Minor, version.Patch, version.ReleaseCandidate)
+}
+
+// Compact prints the version in the major.minor format.
+func (version Version) Compact() string {
+	return fmt.Sprintf("%d.%d", version.Major, version.Minor)
+}
+
+// IsAtLeast determines if a version is greater than or equal to another version.
+func (version Version) IsAtLeast(other Version) bool {
+	if version.Major < other.Major {
+		return false
+	}
+	if version.Major > other.Major {
+		return true
+	}
+	if version.Minor < other.Minor {
+		return false
+	}
+	if version.Minor > other.Minor {
+		return true
+	}
+	if version.Patch < other.Patch {
+		return false
+	}
+	if version.Patch > other.Patch {
+		return true
+	}
+	if version.ReleaseCandidate == 0 {
+		return true
+	}
+	if other.ReleaseCandidate == 0 {
+		return false
+	}
+	if version.ReleaseCandidate < other.ReleaseCandidate {
+		return false
+	}
+	if version.ReleaseCandidate > other.ReleaseCandidate {
+		return true
+	}
+	return true
+}
+
+// GetBinaryVersion Returns a version string compatible with the log implemented in the sidecars
+func (version Version) GetBinaryVersion() string {
+	if version.ReleaseCandidate > 0 {
+		return version.String()
+	}
+	return version.Compact()
+}
+
+// IsProtocolCompatible determines whether two versions of FDB are protocol
+// compatible.
+func (version Version) IsProtocolCompatible(other Version) bool {
+	return version.Major == other.Major && version.Minor == other.Minor && version.ReleaseCandidate == other.ReleaseCandidate
+}
+
+// NextMajorVersion returns the next major version of FoundationDB.
+func (version Version) NextMajorVersion() Version {
+	return Version{Major: version.Major + 1, Minor: 0, Patch: 0}
+}
+
+// NextMinorVersion returns the next minor version of FoundationDB.
+func (version Version) NextMinorVersion() Version {
+	return Version{Major: version.Major, Minor: version.Minor + 1, Patch: 0}
+}
+
+// NextPatchVersion returns the next patch version of FoundationDB.
+func (version Version) NextPatchVersion() Version {
+	return Version{Major: version.Major, Minor: version.Minor, Patch: version.Patch + 1}
+}
+
+// Equal checks if two Version are the same.
+func (version Version) Equal(other Version) bool {
+	return version.Major == other.Major &&
+		version.Minor == other.Minor &&
+		version.Patch == other.Patch &&
+		version.ReleaseCandidate == other.ReleaseCandidate
+}
diff --git a/fdbkubernetesmonitor/api/version_test.go b/fdbkubernetesmonitor/api/version_test.go
new file mode 100644
index 00000000000..9062a5a6233
--- /dev/null
+++ b/fdbkubernetesmonitor/api/version_test.go
@@ -0,0 +1,122 @@
+/*
+ * version_test.go
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2021-2024 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package api
+
+import (
+	//"encoding/json"
+	//"os"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("[api] FDBVersion", func() {
+	// TODO test a json marshal and unmarshl!
+
+	When("checking if the protocol and the version are compatible", func() {
+		It("should return the correct compatibility", func() {
+			version := Version{Major: 6, Minor: 2, Patch: 20}
+			Expect(version.IsProtocolCompatible(Version{Major: 6, Minor: 2, Patch: 20})).To(BeTrue())
+			Expect(version.IsProtocolCompatible(Version{Major: 6, Minor: 2, Patch: 22})).To(BeTrue())
+			Expect(version.IsProtocolCompatible(Version{Major: 6, Minor: 3, Patch: 0})).To(BeFalse())
+			Expect(version.IsProtocolCompatible(Version{Major: 6, Minor: 3, Patch: 20})).To(BeFalse())
+			Expect(version.IsProtocolCompatible(Version{Major: 7, Minor: 2, Patch: 20})).To(BeFalse())
+		})
+
+		When("release candidates differ", func() {
+			It("should be incompatible", func() {
+				version := Version{Major: 7, Minor: 0, Patch: 0, ReleaseCandidate: 1}
+				Expect(version.IsProtocolCompatible(Version{Major: 7, Minor: 0, Patch: 0, ReleaseCandidate: 2})).To(BeFalse())
+			})
+		})
+	})
+
+	Context("Using the fdb version", func() {
+		It("should return the fdb version struct", func() {
+			version, err := ParseFdbVersion("6.2.11")
+			Expect(err).NotTo(HaveOccurred())
+			Expect(version).To(Equal(Version{Major: 6, Minor: 2, Patch: 11}))
+
+			version, err = ParseFdbVersion("prerelease-6.2.11")
+			Expect(err).NotTo(HaveOccurred())
+			Expect(version).To(Equal(Version{Major: 6, Minor: 2, Patch: 11}))
+
+			version, err = ParseFdbVersion("test-6.2.11-test")
+			Expect(err).NotTo(HaveOccurred())
+			Expect(version).To(Equal(Version{Major: 6, Minor: 2, Patch: 11, ReleaseCandidate: 0}))
+
+			version, err = ParseFdbVersion("7.0.0")
+			Expect(err).NotTo(HaveOccurred())
+			Expect(version).To(Equal(Version{Major: 7, Minor: 0, Patch: 0, ReleaseCandidate: 0}))
+
+			version, err = ParseFdbVersion("7.0.0-rc1")
+			Expect(err).NotTo(HaveOccurred())
+			Expect(version).To(Equal(Version{Major: 7, Minor: 0, Patch: 0, ReleaseCandidate: 1}))
+
+			version, err = ParseFdbVersion("7.1.0-rc39")
+			Expect(err).NotTo(HaveOccurred())
+			Expect(version).To(Equal(Version{Major: 7, Minor: 1, Patch: 0, ReleaseCandidate: 39}))
+
+			_, err = ParseFdbVersion("6.2")
+			Expect(err).To(HaveOccurred())
+			Expect(err.Error()).To(Equal("could not parse FDB version from 6.2"))
+		})
+
+		It("should format the version correctly", func() {
+			version := Version{Major: 6, Minor: 2, Patch: 11}
+			Expect(version.String()).To(Equal("6.2.11"))
+			version = Version{Major: 6, Minor: 2, Patch: 11, ReleaseCandidate: 0}
+			Expect(version.String()).To(Equal("6.2.11"))
+			version = Version{Major: 6, Minor: 2, Patch: 11, ReleaseCandidate: 1}
+			Expect(version.String()).To(Equal("6.2.11-rc1"))
+		})
+	})
+
+	When("getting the next version of the current FDBVersion", func() {
+		It("should return the correct next version", func() {
+			version := Version{Major: 6, Minor: 2, Patch: 20}
+			Expect(version.NextMajorVersion()).To(Equal(Version{Major: 7, Minor: 0, Patch: 0}))
+			Expect(version.NextMinorVersion()).To(Equal(Version{Major: version.Major, Minor: 3, Patch: 0}))
+			Expect(version.NextPatchVersion()).To(Equal(Version{Major: version.Major, Minor: version.Minor, Patch: 21}))
+		})
+	})
+
+	When("comparing two FDBVersions", func() {
+		It("should return if they are equal", func() {
+			version := Version{Major: 6, Minor: 2, Patch: 20}
+			Expect(version.Equal(version)).To(BeTrue())
+			Expect(version.Equal(Version{Major: 7, Minor: 0, Patch: 0})).To(BeFalse())
+			Expect(version.Equal(Version{Major: 7, Minor: 0, Patch: 0})).To(BeFalse())
+			Expect(version.Equal(Version{Major: 6, Minor: 3, Patch: 20})).To(BeFalse())
+			Expect(version.Equal(Version{Major: 6, Minor: 2, Patch: 21})).To(BeFalse())
+		})
+
+		It("should return correct result for IsAtleast", func() {
+			version := Version{Major: 7, Minor: 1, Patch: 0, ReleaseCandidate: 2}
+			Expect(version.IsAtLeast(Version{Major: 7, Minor: 1, Patch: 0})).To(BeFalse())
+			Expect(version.IsAtLeast(Version{Major: 7, Minor: 1, Patch: 0, ReleaseCandidate: 1})).To(BeTrue())
+			Expect(version.IsAtLeast(Version{Major: 7, Minor: 1, Patch: 0, ReleaseCandidate: 3})).To(BeFalse())
+
+			version = Version{Major: 7, Minor: 1, Patch: 0}
+			Expect(version.IsAtLeast(Version{Major: 7, Minor: 1, Patch: 0, ReleaseCandidate: 1})).To(BeTrue())
+		})
+	})
+})
diff --git a/fdbkubernetesmonitor/copy.go b/fdbkubernetesmonitor/copy.go
index c24d6abce6b..627d425c80b 100644
--- a/fdbkubernetesmonitor/copy.go
+++ b/fdbkubernetesmonitor/copy.go
@@ -81,7 +81,7 @@ func copyFile(logger logr.Logger, inputPath string, outputPath string, required
 }
 
 // CopyFiles copies a list of files into the output directory.
-func CopyFiles(logger logr.Logger, outputDir string, copyDetails map[string]string, requiredCopies map[string]bool) error {
+func copyFiles(logger logr.Logger, outputDir string, copyDetails map[string]string, requiredCopies map[string]bool) error {
 	for inputPath, outputSubpath := range copyDetails {
 		if outputSubpath == "" {
 			outputSubpath = path.Base(inputPath)
diff --git a/fdbkubernetesmonitor/copy_test.go b/fdbkubernetesmonitor/copy_test.go
index 95d18e1f425..a066bbe24c5 100644
--- a/fdbkubernetesmonitor/copy_test.go
+++ b/fdbkubernetesmonitor/copy_test.go
@@ -311,7 +311,7 @@ var _ = Describe("Testing the copy methods", func() {
 					copyDetails, _, err := getCopyDetails("", "", "", nil, binaries, nil, nil, "7.1.43", executionModeInit)
 					Expect(err).NotTo(HaveOccurred())
 					Expect(copyDetails).To(HaveLen(3))
-					Expect(CopyFiles(GinkgoLogr, outputBinaryDir, copyDetails, map[string]bool{})).NotTo(HaveOccurred())
+					Expect(copyFiles(GinkgoLogr, outputBinaryDir, copyDetails, map[string]bool{})).NotTo(HaveOccurred())
 				})
 
 				It("should copy all the files", func() {
@@ -328,7 +328,7 @@ var _ = Describe("Testing the copy methods", func() {
 					copyDetails, _, err := getCopyDetails("", "", "", nil, binaries, nil, nil, "7.1.43", executionModeSidecar)
 					Expect(err).NotTo(HaveOccurred())
 					Expect(copyDetails).To(HaveLen(3))
-					Expect(CopyFiles(GinkgoLogr, outputBinaryDir, copyDetails, map[string]bool{})).NotTo(HaveOccurred())
+					Expect(copyFiles(GinkgoLogr, outputBinaryDir, copyDetails, map[string]bool{})).NotTo(HaveOccurred())
 				})
 
 				It("should copy all the files", func() {
@@ -366,7 +366,7 @@ var _ = Describe("Testing the copy methods", func() {
 				copyDetails, _, err = getCopyDetails("", "", "", nil, nil, libraries, nil, "7.1.43", executionModeInit)
 				Expect(err).NotTo(HaveOccurred())
 				Expect(copyDetails).To(HaveLen(3))
-				Expect(CopyFiles(GinkgoLogr, outputLibraryDir, copyDetails, map[string]bool{})).NotTo(HaveOccurred())
+				Expect(copyFiles(GinkgoLogr, outputLibraryDir, copyDetails, map[string]bool{})).NotTo(HaveOccurred())
 			})
 
 			It("should copy all the files", func() {
@@ -384,7 +384,7 @@ var _ = Describe("Testing the copy methods", func() {
 				copyDetails, _, err = getCopyDetails("", "7.1", "", nil, nil, libraries, nil, "7.1.43", executionModeInit)
 				Expect(err).NotTo(HaveOccurred())
 				Expect(copyDetails).To(HaveLen(3))
-				Expect(CopyFiles(GinkgoLogr, outputLibraryDir, copyDetails, map[string]bool{})).NotTo(HaveOccurred())
+				Expect(copyFiles(GinkgoLogr, outputLibraryDir, copyDetails, map[string]bool{})).NotTo(HaveOccurred())
 			})
 
 			It("should copy all the files", func() {
@@ -413,7 +413,7 @@ var _ = Describe("Testing the copy methods", func() {
 					copyDetails, _, err = getCopyDetails(testInputDir, "", "", []string{"testfile"}, nil, nil, nil, "7.1.43", executionModeInit)
 					Expect(err).NotTo(HaveOccurred())
 					Expect(copyDetails).To(HaveLen(1))
-					Expect(CopyFiles(GinkgoLogr, testOutputDir, copyDetails, map[string]bool{})).NotTo(HaveOccurred())
+					Expect(copyFiles(GinkgoLogr, testOutputDir, copyDetails, map[string]bool{})).NotTo(HaveOccurred())
 				})
 
 				It("should copy the file", func() {
@@ -434,7 +434,7 @@ var _ = Describe("Testing the copy methods", func() {
 
 				When("the file is empty", func() {
 					It("should not copy the file", func() {
-						Expect(CopyFiles(GinkgoLogr, testOutputDir, copyDetails, requiredFiles)).To(HaveOccurred())
+						Expect(copyFiles(GinkgoLogr, testOutputDir, copyDetails, requiredFiles)).To(HaveOccurred())
 						Expect(path.Join(testOutputDir, "testfile")).NotTo(BeAnExistingFile())
 					})
 				})
@@ -445,7 +445,7 @@ var _ = Describe("Testing the copy methods", func() {
 					})
 
 					It("should copy the file", func() {
-						Expect(CopyFiles(GinkgoLogr, testOutputDir, copyDetails, requiredFiles)).NotTo(HaveOccurred())
+						Expect(copyFiles(GinkgoLogr, testOutputDir, copyDetails, requiredFiles)).NotTo(HaveOccurred())
 						Expect(path.Join(testOutputDir, "testfile")).To(BeAnExistingFile())
 					})
 				})
diff --git a/fdbkubernetesmonitor/kubernetes.go b/fdbkubernetesmonitor/kubernetes.go
index 8e81a7c3436..d6c9bb127c2 100644
--- a/fdbkubernetesmonitor/kubernetes.go
+++ b/fdbkubernetesmonitor/kubernetes.go
@@ -43,44 +43,23 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/client"
 )
 
-const (
-	// CurrentConfigurationAnnotation is the annotation we use to store the
-	// latest configuration.
-	CurrentConfigurationAnnotation = "foundationdb.org/launcher-current-configuration"
-
-	// EnvironmentAnnotation is the annotation we use to store the environment
-	// variables.
-	EnvironmentAnnotation = "foundationdb.org/launcher-environment"
-
-	// OutdatedConfigMapAnnotation is the annotation we read to get notified of
-	// outdated configuration.
-	OutdatedConfigMapAnnotation = "foundationdb.org/outdated-config-map-seen"
-
-	// DelayShutdownAnnotation defines how long the FDB Kubernetes monitor process should sleep before shutting itself down.
-	// The FDB Kubernetes monitor will always shutdown all fdbserver processes, independent of this setting.
-	// The value of this annotation must be a duration like "60s".
-	DelayShutdownAnnotation = "foundationdb.org/delay-shutdown"
-
-	// ClusterFileChangeDetectedAnnotation is the annotation that will be updated if the fdb.cluster file is updated.
-	ClusterFileChangeDetectedAnnotation = "foundationdb.org/cluster-file-change"
-)
-
-// PodClient is a wrapper around the pod API.
-type PodClient struct {
+// kubernetesClient is a wrapper around the Kubernetes API.
+type kubernetesClient struct {
 	// podMetadata is the latest metadata that was seen by the fdb-kubernetes-monitor for the according Pod.
 	podMetadata *metav1.PartialObjectMetadata
 
 	// nodeMetadata is the latest metadata that was seen by the fdb-kubernetes-monitor for the according node that hosts the Pod.
 	nodeMetadata *metav1.PartialObjectMetadata
 
-	// TimestampFeed is a channel where the pod client will send updates with
-	// the values from OutdatedConfigMapAnnotation.
+	// TimestampFeed is a channel where the kubernetes client will send updates with
+	// the values from api.OutdatedConfigMapAnnotation. If the api.IsolateProcessGroupAnnotation is changed the current
+	// timestamp will be sent to the channel to force the monitor to change its configuration accordingly.
 	TimestampFeed chan int64
 
 	// Logger is the logger we use for this client.
 	Logger logr.Logger
 
-	// Adds the controller runtime client to the PodClient.
+	// Adds the controller runtime client to the kubernetesClient.
 	client.Client
 }
 
@@ -124,14 +103,14 @@ func setupCache(namespace string, podName string, nodeName string) (client.WithW
 	return internalClient, internalCache, nil
 }
 
-// CreatePodClient creates a new client for working with the pod object.
-func CreatePodClient(ctx context.Context, logger logr.Logger, enableNodeWatcher bool, setupCache func(string, string, string) (client.WithWatch, cache.Cache, error)) (*PodClient, error) {
+// createPodClient creates a new client for working with the pod object.
+func createPodClient(ctx context.Context, logger logr.Logger, enableNodeWatcher bool, setupCache func(string, string, string) (client.WithWatch, cache.Cache, error)) (*kubernetesClient, error) {
 	namespace := os.Getenv("FDB_POD_NAMESPACE")
 	podName := os.Getenv("FDB_POD_NAME")
 	nodeName := os.Getenv("FDB_NODE_NAME")
 
 	internalClient, internalCache, err := setupCache(namespace, podName, nodeName)
-	podClient := &PodClient{
+	podClient := &kubernetesClient{
 		podMetadata:   nil,
 		nodeMetadata:  nil,
 		TimestampFeed: make(chan int64, 10),
@@ -185,7 +164,7 @@ func CreatePodClient(ctx context.Context, logger logr.Logger, enableNodeWatcher
 
 	podClient.Client = controllerClient
 
-	// Fetch the current metadata before returning the PodClient
+	// Fetch the current metadata before returning the kubernetesClient
 	currentPodMetadata := &metav1.PartialObjectMetadata{}
 	currentPodMetadata.SetGroupVersionKind(corev1.SchemeGroupVersion.WithKind("Pod"))
 	err = podClient.Client.Get(ctx, client.ObjectKey{Namespace: namespace, Name: podName}, currentPodMetadata)
@@ -213,9 +192,9 @@ func CreatePodClient(ctx context.Context, logger logr.Logger, enableNodeWatcher
 
 // retrieveEnvironmentVariables extracts the environment variables we have for
 // an argument into a map.
-func retrieveEnvironmentVariables(monitor *Monitor, argument api.Argument, target map[string]string) {
+func retrieveEnvironmentVariables(monitor *monitor, argument api.Argument, target map[string]string) {
 	if argument.Source != "" {
-		value, err := argument.LookupEnv(monitor.CustomEnvironment)
+		value, err := argument.LookupEnv(monitor.customEnvironment)
 		if err == nil {
 			target[argument.Source] = value
 		}
@@ -227,39 +206,44 @@ func retrieveEnvironmentVariables(monitor *Monitor, argument api.Argument, targe
 	}
 }
 
-// UpdateAnnotations updates annotations on the pod after loading new
+// updateAnnotations updates annotations on the pod after loading new
 // configuration.
-func (podClient *PodClient) UpdateAnnotations(monitor *Monitor) error {
+func (podClient *kubernetesClient) updateAnnotations(monitor *monitor) error {
 	environment := make(map[string]string)
-	for _, argument := range monitor.ActiveConfiguration.Arguments {
+	for _, argument := range monitor.activeConfiguration.Arguments {
 		retrieveEnvironmentVariables(monitor, argument, environment)
 	}
-	environment["BINARY_DIR"] = path.Dir(monitor.ActiveConfiguration.BinaryPath)
+	environment["BINARY_DIR"] = path.Dir(monitor.activeConfiguration.BinaryPath)
 	jsonEnvironment, err := json.Marshal(environment)
 	if err != nil {
 		return err
 	}
 
 	return podClient.updateAnnotationsOnPod(map[string]string{
-		CurrentConfigurationAnnotation: string(monitor.ActiveConfigurationBytes),
-		EnvironmentAnnotation:          string(jsonEnvironment),
+		api.CurrentConfigurationAnnotation: string(monitor.activeConfigurationBytes),
+		api.EnvironmentAnnotation:          string(jsonEnvironment),
 	})
 }
 
 // updateFdbClusterTimestampAnnotation updates the ClusterFileChangeDetectedAnnotation annotation on the pod
 // after a change to the fdb.cluster file was detected, e.g. because the coordinators were changed.
-func (podClient *PodClient) updateFdbClusterTimestampAnnotation() error {
+func (podClient *kubernetesClient) updateFdbClusterTimestampAnnotation() error {
 	return podClient.updateAnnotationsOnPod(map[string]string{
-		ClusterFileChangeDetectedAnnotation: strconv.FormatInt(time.Now().Unix(), 10),
+		api.ClusterFileChangeDetectedAnnotation: strconv.FormatInt(time.Now().Unix(), 10),
 	})
 }
 
 // updateAnnotationsOnPod will update the annotations with the provided annotationChanges. If an annotation exists, it
 // will be updated if the annotation is absent it will be added.
-func (podClient *PodClient) updateAnnotationsOnPod(annotationChanges map[string]string) error {
+func (podClient *kubernetesClient) updateAnnotationsOnPod(annotationChanges map[string]string) error {
 	if podClient.podMetadata == nil {
 		return fmt.Errorf("pod client has no metadata present")
 	}
+	
+	annotations := podClient.podMetadata.Annotations
+	if len(annotations) == 0 {
+		annotations = map[string]string{}
+	}
 
 	if !podClient.podMetadata.DeletionTimestamp.IsZero() {
 		return fmt.Errorf("pod is marked for deletion, cannot update annotations")
@@ -304,7 +288,7 @@ func (podClient *PodClient) updateAnnotationsOnPod(annotationChanges map[string]
 }
 
 // OnAdd is called when an object is added.
-func (podClient *PodClient) OnAdd(obj interface{}) {
+func (podClient *kubernetesClient) OnAdd(obj interface{}) {
 	switch castedObj := obj.(type) {
 	case *corev1.Pod:
 		podClient.Logger.Info("Got event for OnAdd for Pod resource", "name", castedObj.Name, "namespace", castedObj.Namespace)
@@ -324,10 +308,15 @@ func (podClient *PodClient) OnAdd(obj interface{}) {
 // OnUpdate is also called when a re-list happens, and it will
 // get called even if nothing changed. This is useful for periodically
 // evaluating or syncing something.
-func (podClient *PodClient) OnUpdate(_, newObj interface{}) {
+func (podClient *kubernetesClient) OnUpdate(_, newObj interface{}) {
 	switch castedObj := newObj.(type) {
 	case *corev1.Pod:
 		podClient.Logger.Info("Got event for OnUpdate for Pod resource", "name", castedObj.Name, "namespace", castedObj.Namespace, "generation", castedObj.Generation)
+		var previousAnnotations map[string]string
+		if podClient.podMetadata != nil {
+			previousAnnotations = podClient.podMetadata.Annotations
+		}
+
 		podClient.podMetadata = &metav1.PartialObjectMetadata{
 			TypeMeta:   castedObj.TypeMeta,
 			ObjectMeta: castedObj.ObjectMeta,
@@ -337,14 +326,25 @@ func (podClient *PodClient) OnUpdate(_, newObj interface{}) {
 			return
 		}
 
-		annotation := podClient.podMetadata.Annotations[OutdatedConfigMapAnnotation]
+		// If the IsolateProcessGroupAnnotation changes force a reload of the configuration to make sure the processes
+		// will be shutdown.
+		previousIsolateProcessGroupAnnotationValue := previousAnnotations[api.IsolateProcessGroupAnnotation]
+		newIsolateProcessGroupAnnotationValue := podClient.podMetadata.Annotations[api.IsolateProcessGroupAnnotation]
+		if previousIsolateProcessGroupAnnotationValue != newIsolateProcessGroupAnnotationValue {
+			podClient.Logger.Info("Got change in isolate process group annotation", "previous", previousIsolateProcessGroupAnnotationValue, "new", newIsolateProcessGroupAnnotationValue)
+			podClient.TimestampFeed <- time.Now().Unix()
+			// In this case we can return as the timestamp feed already has a new value.
+			return
+		}
+
+		annotation := podClient.podMetadata.Annotations[api.OutdatedConfigMapAnnotation]
 		if annotation == "" {
 			return
 		}
 
 		timestamp, err := strconv.ParseInt(annotation, 10, 64)
 		if err != nil {
-			podClient.Logger.Error(err, "Error parsing annotation", "key", OutdatedConfigMapAnnotation, "rawAnnotation", annotation)
+			podClient.Logger.Error(err, "Error parsing annotation", "key", api.OutdatedConfigMapAnnotation, "rawAnnotation", annotation)
 			return
 		}
 
@@ -362,7 +362,7 @@ func (podClient *PodClient) OnUpdate(_, newObj interface{}) {
 // it will get an object of type DeletedFinalStateUnknown. This can
 // happen if the watch is closed and misses the delete event and we don't
 // notice the deletion until the subsequent re-list.
-func (podClient *PodClient) OnDelete(obj interface{}) {
+func (podClient *kubernetesClient) OnDelete(obj interface{}) {
 	switch castedObj := obj.(type) {
 	case *corev1.Pod:
 		podClient.Logger.Info("Got event for OnDelete for Pod resource", "name", castedObj.Name, "namespace", castedObj.Namespace)
diff --git a/fdbkubernetesmonitor/kubernetes_test.go b/fdbkubernetesmonitor/kubernetes_test.go
index d33d0881258..40817a2bb36 100644
--- a/fdbkubernetesmonitor/kubernetes_test.go
+++ b/fdbkubernetesmonitor/kubernetes_test.go
@@ -42,7 +42,7 @@ import (
 var _ = Describe("Testing FDB Pod client", func() {
 	var enableNodeWatcher bool
 	var fakeClient client.WithWatch
-	var podClient *PodClient
+	var podClient *kubernetesClient
 	var namespace, podName, nodeName string
 	var internalCache *informertest.FakeInformers
 
@@ -63,10 +63,10 @@ var _ = Describe("Testing FDB Pod client", func() {
 		internalCache.Scheme = fakeClient.Scheme()
 	})
 
-	When("the PodClient was started", func() {
+	When("the kubernetesClient was started", func() {
 		JustBeforeEach(func() {
 			var err error
-			podClient, err = CreatePodClient(context.Background(), GinkgoLogr, enableNodeWatcher, func(fncNamespace string, fncPodName string, fncNodeName string) (client.WithWatch, cache.Cache, error) {
+			podClient, err = createPodClient(context.Background(), GinkgoLogr, enableNodeWatcher, func(fncNamespace string, fncPodName string, fncNodeName string) (client.WithWatch, cache.Cache, error) {
 				Expect(fncNamespace).To(Equal(namespace))
 				Expect(fncPodName).To(Equal(podName))
 				Expect(fncNodeName).To(Equal(nodeName))
@@ -102,10 +102,10 @@ var _ = Describe("Testing FDB Pod client", func() {
 		})
 	})
 
-	When("the PodClient handles events", func() {
+	When("the kubernetesClient handles events", func() {
 		BeforeEach(func() {
 			var err error
-			podClient, err = CreatePodClient(context.Background(), GinkgoLogr, enableNodeWatcher, func(fncNamespace string, fncPodName string, fncNodeName string) (client.WithWatch, cache.Cache, error) {
+			podClient, err = createPodClient(context.Background(), GinkgoLogr, enableNodeWatcher, func(fncNamespace string, fncPodName string, fncNodeName string) (client.WithWatch, cache.Cache, error) {
 				Expect(fncNamespace).To(Equal(namespace))
 				Expect(fncPodName).To(Equal(podName))
 				Expect(fncNodeName).To(Equal(nodeName))
@@ -173,7 +173,7 @@ var _ = Describe("Testing FDB Pod client", func() {
 				BeforeEach(func() {
 					timestamp = time.Now().Unix()
 					pod.Annotations = map[string]string{
-						OutdatedConfigMapAnnotation: strconv.FormatInt(timestamp, 10),
+						api.OutdatedConfigMapAnnotation: strconv.FormatInt(timestamp, 10),
 					}
 					fakeInformer.Update(nil, pod)
 				})
@@ -193,7 +193,7 @@ var _ = Describe("Testing FDB Pod client", func() {
 			When("an UpdateEvent is handled that updates the OutdatedConfigMapAnnotation with a bad value", func() {
 				BeforeEach(func() {
 					pod.Annotations = map[string]string{
-						OutdatedConfigMapAnnotation: "boom!",
+						api.OutdatedConfigMapAnnotation: "boom!",
 					}
 					fakeInformer.Update(nil, pod)
 				})
@@ -282,12 +282,12 @@ var _ = Describe("Testing FDB Pod client", func() {
 		})
 	})
 
-	When("the PodClient should update the annotations", func() {
-		var monitor *Monitor
+	When("the kubernetesClient should update the annotations", func() {
+		var mon *monitor
 
 		JustBeforeEach(func() {
 			var err error
-			podClient, err = CreatePodClient(context.Background(), GinkgoLogr, enableNodeWatcher, func(fncNamespace string, fncPodName string, fncNodeName string) (client.WithWatch, cache.Cache, error) {
+			podClient, err = createPodClient(context.Background(), GinkgoLogr, enableNodeWatcher, func(fncNamespace string, fncPodName string, fncNodeName string) (client.WithWatch, cache.Cache, error) {
 				Expect(fncNamespace).To(Equal(namespace))
 				Expect(fncPodName).To(Equal(podName))
 				Expect(fncNodeName).To(Equal(nodeName))
@@ -314,13 +314,13 @@ var _ = Describe("Testing FDB Pod client", func() {
 			})).NotTo(HaveOccurred())
 
 			// Execute the update annotations.
-			Expect(podClient.UpdateAnnotations(monitor)).NotTo(HaveOccurred())
+			Expect(podClient.updateAnnotations(mon)).NotTo(HaveOccurred())
 		})
 
 		When("no additional env variables are set", func() {
 			BeforeEach(func() {
-				monitor = &Monitor{
-					ActiveConfiguration: &api.ProcessConfiguration{
+				mon = &monitor{
+					activeConfiguration: &api.ProcessConfiguration{
 						BinaryPath: "/usr/bin",
 					},
 				}
@@ -330,20 +330,20 @@ var _ = Describe("Testing FDB Pod client", func() {
 				pod := &corev1.Pod{}
 				Expect(fakeClient.Get(context.Background(), client.ObjectKey{Namespace: namespace, Name: podName}, pod)).NotTo(HaveOccurred())
 
-				Expect(pod.Annotations).To(HaveKeyWithValue(CurrentConfigurationAnnotation, ""))
-				Expect(pod.Annotations).To(HaveKeyWithValue(EnvironmentAnnotation, "{\"BINARY_DIR\":\"/usr\"}"))
+				Expect(pod.Annotations).To(HaveKeyWithValue(api.CurrentConfigurationAnnotation, ""))
+				Expect(pod.Annotations).To(HaveKeyWithValue(api.EnvironmentAnnotation, "{\"BINARY_DIR\":\"/usr\"}"))
 			})
 		})
 
 		When("one flat additional env variable is set", func() {
 			BeforeEach(func() {
 				GinkgoT().Setenv("TEST", "test-value")
-				monitor = &Monitor{
-					ActiveConfiguration: &api.ProcessConfiguration{
+				mon = &monitor{
+					activeConfiguration: &api.ProcessConfiguration{
 						BinaryPath: "/usr/bin",
 						Arguments: []api.Argument{
 							{
-								ArgumentType: EnvironmentAnnotation,
+								ArgumentType: api.EnvironmentAnnotation,
 								Source:       "TEST",
 							},
 						},
@@ -355,16 +355,16 @@ var _ = Describe("Testing FDB Pod client", func() {
 				pod := &corev1.Pod{}
 				Expect(fakeClient.Get(context.Background(), client.ObjectKey{Namespace: namespace, Name: podName}, pod)).NotTo(HaveOccurred())
 
-				Expect(pod.Annotations).To(HaveKeyWithValue(CurrentConfigurationAnnotation, ""))
-				Expect(pod.Annotations).To(HaveKeyWithValue(EnvironmentAnnotation, "{\"BINARY_DIR\":\"/usr\",\"TEST\":\"test-value\"}"))
+				Expect(pod.Annotations).To(HaveKeyWithValue(api.CurrentConfigurationAnnotation, ""))
+				Expect(pod.Annotations).To(HaveKeyWithValue(api.EnvironmentAnnotation, "{\"BINARY_DIR\":\"/usr\",\"TEST\":\"test-value\"}"))
 			})
 		})
 
 		When("one nested flat additional env variable is set", func() {
 			BeforeEach(func() {
 				GinkgoT().Setenv("TEST", "test-value")
-				monitor = &Monitor{
-					ActiveConfiguration: &api.ProcessConfiguration{
+				mon = &monitor{
+					activeConfiguration: &api.ProcessConfiguration{
 						BinaryPath: "/usr/bin",
 						Arguments: []api.Argument{
 							{
@@ -385,8 +385,8 @@ var _ = Describe("Testing FDB Pod client", func() {
 				pod := &corev1.Pod{}
 				Expect(fakeClient.Get(context.Background(), client.ObjectKey{Namespace: namespace, Name: podName}, pod)).NotTo(HaveOccurred())
 
-				Expect(pod.Annotations).To(HaveKeyWithValue(CurrentConfigurationAnnotation, ""))
-				Expect(pod.Annotations).To(HaveKeyWithValue(EnvironmentAnnotation, "{\"BINARY_DIR\":\"/usr\",\"TEST\":\"test-value\"}"))
+				Expect(pod.Annotations).To(HaveKeyWithValue(api.CurrentConfigurationAnnotation, ""))
+				Expect(pod.Annotations).To(HaveKeyWithValue(api.EnvironmentAnnotation, "{\"BINARY_DIR\":\"/usr\",\"TEST\":\"test-value\"}"))
 			})
 		})
 	})
diff --git a/fdbkubernetesmonitor/main.go b/fdbkubernetesmonitor/main.go
index 1eedc95f09f..c87fa4bd898 100644
--- a/fdbkubernetesmonitor/main.go
+++ b/fdbkubernetesmonitor/main.go
@@ -28,6 +28,8 @@ import (
 	"regexp"
 	"strings"
 
+	"github.com/apple/foundationdb/fdbkubernetesmonitor/api"
+
 	"github.com/go-logr/logr"
 	"github.com/go-logr/zapr"
 	"github.com/spf13/pflag"
@@ -109,7 +111,7 @@ func parseFlagsAndSetEnvDefaults() error {
 }
 
 func main() {
-	var copyFiles, copyBinaries, copyLibraries, requiredCopyFiles []string
+	var filesToCopy, copyBinaries, copyLibraries, requiredCopyFiles []string
 	var inputDir, copyPrimaryLibrary, binaryOutputDirectory, certPath, keyPath string
 
 	pflag.StringVar(&executionModeString, "mode", "launcher", "Execution mode. Valid options are launcher, sidecar, and init")
@@ -118,7 +120,7 @@ func main() {
 	pflag.StringVar(&monitorConfFile, "input-monitor-conf", "config.json", "Name of the file in the input directory that contains the monitor configuration")
 	pflag.StringVar(&logPath, "log-path", "", "Name of a file to send logs to. Logs will be sent to stdout in addition the file you pass in this argument. If this is blank, logs will only by sent to stdout")
 	pflag.StringVar(&outputDir, "output-dir", ".", "Directory to copy files into")
-	pflag.StringArrayVar(&copyFiles, "copy-file", nil, "A list of files to copy")
+	pflag.StringArrayVar(&filesToCopy, "copy-file", nil, "A list of files to copy")
 	pflag.StringArrayVar(&copyBinaries, "copy-binary", nil, "A list of binaries to copy from /usr/bin")
 	pflag.StringVar(&versionFilePath, "version-file", "/var/fdb/version", "Path to a file containing the current FDB version")
 	pflag.StringVar(&sharedBinaryDir, "shared-binary-dir", "/var/fdb/shared-binaries/bin", "A directory containing binaries that are copied from a sidecar process")
@@ -146,7 +148,7 @@ func main() {
 	}
 	currentContainerVersion := strings.TrimSpace(string(versionBytes))
 	mode := executionMode(executionModeString)
-	copyDetails, requiredCopies, err := getCopyDetails(inputDir, copyPrimaryLibrary, binaryOutputDirectory, copyFiles, copyBinaries, copyLibraries, requiredCopyFiles, currentContainerVersion, mode)
+	copyDetails, requiredCopies, err := getCopyDetails(inputDir, copyPrimaryLibrary, binaryOutputDirectory, filesToCopy, copyBinaries, copyLibraries, requiredCopyFiles, currentContainerVersion, mode)
 	if err != nil {
 		logger.Error(err, "Error getting list of files to copy")
 		os.Exit(1)
@@ -164,16 +166,22 @@ func main() {
 			certPath:   certPath,
 			keyPath:    keyPath,
 		}
-		StartMonitor(context.Background(), logger, path.Join(inputDir, monitorConfFile), customEnvironment, processCount, promConfig, enablePprof, currentContainerVersion, enableNodeWatch)
+
+		parsedVersion, err := api.ParseFdbVersion(currentContainerVersion)
+		if err != nil {
+			logger.Error(err, "Error parsing container version", "currentContainerVersion", currentContainerVersion)
+			os.Exit(1)
+		}
+		startMonitor(context.Background(), logger, path.Join(inputDir, monitorConfFile), customEnvironment, processCount, promConfig, enablePprof, parsedVersion, enableNodeWatch)
 	case executionModeInit:
-		err = CopyFiles(logger, outputDir, copyDetails, requiredCopies)
+		err = copyFiles(logger, outputDir, copyDetails, requiredCopies)
 		if err != nil {
 			logger.Error(err, "Error copying files")
 			os.Exit(1)
 		}
 	case executionModeSidecar:
 		if mainContainerVersion != currentContainerVersion {
-			err = CopyFiles(logger, outputDir, copyDetails, requiredCopies)
+			err = copyFiles(logger, outputDir, copyDetails, requiredCopies)
 			if err != nil {
 				logger.Error(err, "Error copying files")
 				os.Exit(1)
diff --git a/fdbkubernetesmonitor/metrics.go b/fdbkubernetesmonitor/metrics.go
index d652c23f958..97a33f1b89b 100644
--- a/fdbkubernetesmonitor/metrics.go
+++ b/fdbkubernetesmonitor/metrics.go
@@ -30,7 +30,7 @@ const (
 	// processLabel represents the process label for the prometheus metrics.
 	processLabel = "process"
 	// namespace is the prometheus namespace for the metrics
-	namespace = "fdbkubernetesmonitor"
+	prometheusNamespace = "fdbkubernetesmonitor"
 	// restartCountMetricName represents the name of the restart metric.
 	restartCountMetricName = "restart_count"
 	// configurationChangeCountMetricName represents the configuration_change_count metric.
@@ -45,7 +45,7 @@ const (
 	desiredVersionMetricName = "desired_version"
 )
 
-// metrics represents the custom prometheus metrics for the Monitor.
+// metrics represents the custom prometheus metrics for the monitor.
 type metrics struct {
 	// restartCount represents the total number of fdbserver process restarts.
 	restartCount *prometheus.CounterVec
@@ -98,32 +98,32 @@ func registerMetrics(reg prometheus.Registerer) *metrics {
 	monitorMetrics := &metrics{
 		restartCount: prometheus.NewCounterVec(
 			prometheus.CounterOpts{
-				Namespace: namespace,
+				Namespace: prometheusNamespace,
 				Name:      restartCountMetricName,
 				Help:      "Number of fdbserver process restarts in total.",
 			}, []string{processLabel}),
 		configurationChangeCount: prometheus.NewCounter(prometheus.CounterOpts{
-			Namespace: namespace,
+			Namespace: prometheusNamespace,
 			Name:      configurationChangeCountMetricName,
 			Help:      "Number of observed configuration changes.",
 		}),
 		lastAppliedConfigurationTimestamp: prometheus.NewGauge(prometheus.GaugeOpts{
-			Namespace: namespace,
+			Namespace: prometheusNamespace,
 			Name:      lastAppliedConfigurationTimestampMetricName,
 			Help:      "Timestamp when the last time the configuration was applied.",
 		}),
 		startTimestamp: prometheus.NewGaugeVec(prometheus.GaugeOpts{
-			Namespace: namespace,
+			Namespace: prometheusNamespace,
 			Name:      startTimestampMetricName,
 			Help:      "Timestamp when the last time the configuration was applied.",
 		}, []string{processLabel}),
 		runningVersion: prometheus.NewGaugeVec(prometheus.GaugeOpts{
-			Namespace: namespace,
+			Namespace: prometheusNamespace,
 			Name:      runningVersionMetricName,
 			Help:      "The current running version of the fdbserver processes started by this monitor.",
 		}, []string{versionLabel}),
 		desiredVersion: prometheus.NewGaugeVec(prometheus.GaugeOpts{
-			Namespace: namespace,
+			Namespace: prometheusNamespace,
 			Name:      desiredVersionMetricName,
 			Help:      "The desired running version of the fdbserver processes started by this monitor.",
 		}, []string{versionLabel}),
diff --git a/fdbkubernetesmonitor/metrics_test.go b/fdbkubernetesmonitor/metrics_test.go
index 8ccb58d2d88..b42c61c79af 100644
--- a/fdbkubernetesmonitor/metrics_test.go
+++ b/fdbkubernetesmonitor/metrics_test.go
@@ -27,7 +27,7 @@ import (
 	"k8s.io/utils/pointer"
 )
 
-var _ = Describe("Testing Monitor metrics", func() {
+var _ = Describe("Testing monitor metrics", func() {
 	When("getting the copy details", func() {
 		sevenOneVersion := "7.1.57"
 		sevenThreeVersion := "7.3.37"
diff --git a/fdbkubernetesmonitor/monitor.go b/fdbkubernetesmonitor/monitor.go
index 09f897e9071..82cb283b5ad 100644
--- a/fdbkubernetesmonitor/monitor.go
+++ b/fdbkubernetesmonitor/monitor.go
@@ -33,6 +33,7 @@ import (
 	"os/exec"
 	"os/signal"
 	"path"
+	"strconv"
 	"strings"
 	"sync"
 	"syscall"
@@ -60,49 +61,49 @@ const (
 	fdbClusterFilePath = "/var/fdb/data/fdb.cluster"
 )
 
-// Monitor provides the main monitor loop
-type Monitor struct {
-	// ConfigFile defines the path to the config file to load.
-	ConfigFile string
+// monitor provides the main monitor loop
+type monitor struct {
+	// configFile defines the path to the config file to load.
+	configFile string
 
-	// CurrentContainerVersion defines the version of the container. This will be the same as the fdbserver version.
-	CurrentContainerVersion string
+	// currentContainerVersion defines the version of the container. This will be the same as the fdbserver version.
+	currentContainerVersion api.Version
 
-	// CustomEnvironment defines the custom environment variables to use when
+	// customEnvironment defines the custom environment variables to use when
 	// interpreting the monitor configuration.
-	CustomEnvironment map[string]string
+	customEnvironment map[string]string
 
-	// ActiveConfiguration defines the active process configuration.
-	ActiveConfiguration *api.ProcessConfiguration
+	// activeConfiguration defines the active process configuration.
+	activeConfiguration *api.ProcessConfiguration
 
-	// ActiveConfigurationBytes defines the source data for the active process
+	// activeConfigurationBytes defines the source data for the active process
 	// configuration.
-	ActiveConfigurationBytes []byte
+	activeConfigurationBytes []byte
 
-	// LastConfigurationTime is the last time we successfully reloaded the
+	// lastConfigurationTime is the last time we successfully reloaded the
 	// configuration file.
-	LastConfigurationTime time.Time
+	lastConfigurationTime time.Time
 
-	// ProcessCount defines how many processes the
-	ProcessCount int
+	// processCount defines how many processes the
+	processCount int
 
-	// ProcessIDs stores the PIDs of the processes that are running. A PID of
+	// processIDs stores the PIDs of the processes that are running. A PID of
 	// zero will indicate that a process does not have a run loop. A PID of -1
 	// will indicate that a process has a run loop but is not currently running
 	// the subprocess.
-	ProcessIDs []int
+	processIDs []int
 
-	// Mutex defines a mutex around working with configuration.
+	// mutex defines a mutex around working with configuration.
 	// This is used to synchronize access to local state like the active
 	// configuration and the process IDs from multiple goroutines.
-	Mutex sync.Mutex
+	mutex sync.Mutex
 
-	// PodClient is a client for posting updates about this pod to
+	// podClient is a client for posting updates about this pod to
 	// Kubernetes.
-	PodClient *PodClient
+	podClient *kubernetesClient
 
-	// Logger is the logger instance for this monitor.
-	Logger logr.Logger
+	// logger is the logger instance for this monitor.
+	logger logr.Logger
 
 	// metrics represents the prometheus monitor metrics.
 	metrics *metrics
@@ -112,25 +113,25 @@ type httpConfig struct {
 	listenAddr, certPath, keyPath, rootCaPath string
 }
 
-// StartMonitor starts the monitor loop.
-func StartMonitor(ctx context.Context, logger logr.Logger, configFile string, customEnvironment map[string]string, processCount int, promConfig httpConfig, enableDebug bool, currentContainerVersion string, enableNodeWatcher bool) {
-	podClient, err := CreatePodClient(ctx, logger, enableNodeWatcher, setupCache)
+// startMonitor starts the monitor loop.
+func startMonitor(ctx context.Context, logger logr.Logger, configFile string, customEnvironment map[string]string, processCount int, promConfig httpConfig, enableDebug bool, currentContainerVersion api.Version, enableNodeWatcher bool) {
+	client, err := createPodClient(ctx, logger, enableNodeWatcher, setupCache)
 	if err != nil {
 		logger.Error(err, "could not create Pod client")
 		os.Exit(1)
 	}
 
-	monitor := &Monitor{
-		ConfigFile:              configFile,
-		PodClient:               podClient,
-		Logger:                  logger,
-		CustomEnvironment:       customEnvironment,
-		ProcessCount:            processCount,
-		ProcessIDs:              make([]int, processCount+1),
-		CurrentContainerVersion: currentContainerVersion,
+	mon := &monitor{
+		configFile:              configFile,
+		podClient:               client,
+		logger:                  logger,
+		customEnvironment:       customEnvironment,
+		processCount:            processCount,
+		processIDs:              make([]int, processCount+1),
+		currentContainerVersion: currentContainerVersion,
 	}
 
-	go func() { monitor.WatchPodTimestamps() }()
+	go func() { mon.watchPodTimestamps() }()
 
 	mux := http.NewServeMux()
 	// Enable pprof endpoints for debugging purposes.
@@ -152,7 +153,7 @@ func StartMonitor(ctx context.Context, logger logr.Logger, configFile string, cu
 	// Enable the default go metrics.
 	reg.MustRegister(collectors.NewGoCollector())
 	monitorMetrics := registerMetrics(reg)
-	monitor.metrics = monitorMetrics
+	mon.metrics = monitorMetrics
 	promHandler := promhttp.HandlerFor(reg, promhttp.HandlerOpts{})
 
 	// Add Prometheus support
@@ -181,27 +182,27 @@ func StartMonitor(ctx context.Context, logger logr.Logger, configFile string, cu
 		}
 	}()
 
-	monitor.Run()
+	mon.run()
 }
 
 // updateCustomEnvironment will add the node labels and their values to the custom environment map. All the generated
 // environment variables will start with NODE_LABEL and "/" and "." will be replaced in the key as "_", e.g. from the
 // label "foundationdb.org/testing = awesome" the env variables NODE_LABEL_FOUNDATIONDB_ORG_TESTING = awesome" will be
 // generated.
-func (monitor *Monitor) updateCustomEnvironmentFromNodeMetadata() {
-	if monitor.PodClient.nodeMetadata == nil {
+func (monitor *monitor) updateCustomEnvironmentFromNodeMetadata() {
+	if monitor.podClient.nodeMetadata == nil {
 		return
 	}
 
-	nodeLabels := monitor.PodClient.nodeMetadata.Labels
+	nodeLabels := monitor.podClient.nodeMetadata.Labels
 	for key, value := range nodeLabels {
 		sanitizedKey := strings.ReplaceAll(key, "/", "_")
 		sanitizedKey = strings.ReplaceAll(sanitizedKey, ".", "_")
 		envKey := "NODE_LABEL_" + strings.ToUpper(sanitizedKey)
-		currentValue, ok := monitor.CustomEnvironment[envKey]
+		currentValue, ok := monitor.customEnvironment[envKey]
 		if !ok {
-			monitor.Logger.Info("adding new custom environment variable from node labels", "key", envKey, "value", value)
-			monitor.CustomEnvironment[envKey] = value
+			monitor.logger.Info("adding new custom environment variable from node labels", "key", envKey, "value", value)
+			monitor.customEnvironment[envKey] = value
 			continue
 		}
 
@@ -209,51 +210,75 @@ func (monitor *Monitor) updateCustomEnvironmentFromNodeMetadata() {
 			continue
 		}
 
-		monitor.Logger.Info("update custom environment variable from node labels", "key", envKey, "newValue", value, "currentValue", currentValue)
-		monitor.CustomEnvironment[envKey] = value
+		monitor.logger.Info("update custom environment variable from node labels", "key", envKey, "newValue", value, "currentValue", currentValue)
+		monitor.customEnvironment[envKey] = value
 		continue
 	}
 }
 
-// LoadConfiguration loads the latest configuration from the config file.
-func (monitor *Monitor) LoadConfiguration() {
-	file, err := os.Open(monitor.ConfigFile)
+// readConfiguration reads the latest configuration from the monitor file.
+func (monitor *monitor) readConfiguration() (*api.ProcessConfiguration, []byte) {
+	file, err := os.Open(monitor.configFile)
 	if err != nil {
-		monitor.Logger.Error(err, "Error reading monitor config file", "monitorConfigPath", monitor.ConfigFile)
-		return
+		monitor.logger.Error(err, "Error reading monitor config file", "monitorConfigPath", monitor.configFile)
+		return nil, nil
 	}
 	defer func() {
 		err := file.Close()
-		monitor.Logger.Error(err, "Error could not close file", "monitorConfigPath", monitor.ConfigFile)
+		monitor.logger.Error(err, "Error could not close file", "monitorConfigPath", monitor.configFile)
 	}()
 	configuration := &api.ProcessConfiguration{}
 	configurationBytes, err := io.ReadAll(file)
 	if err != nil {
-		monitor.Logger.Error(err, "Error reading monitor configuration", "monitorConfigPath", monitor.ConfigFile)
+		monitor.logger.Error(err, "Error reading monitor configuration", "monitorConfigPath", monitor.configFile)
 	}
 	err = json.Unmarshal(configurationBytes, configuration)
 	if err != nil {
-		monitor.Logger.Error(err, "Error parsing monitor configuration", "rawConfiguration", string(configurationBytes))
-		return
+		monitor.logger.Error(err, "Error parsing monitor configuration", "rawConfiguration", string(configurationBytes))
+		return nil, nil
+	}
+
+	if configuration.Version == nil {
+		monitor.logger.Error(err, "Error could not parse configured version", "rawConfiguration", string(configurationBytes))
+		return nil, nil
 	}
 
-	if monitor.CurrentContainerVersion == configuration.Version {
+	// If the versions are protocol compatible don't try to point to another binary path. Otherwise, the processes will
+	// cannot restart when a process crashes during a patch upgrade.
+	if monitor.currentContainerVersion.IsProtocolCompatible(*configuration.Version) {
 		configuration.BinaryPath = fdbserverPath
 	} else {
-		configuration.BinaryPath = path.Join(sharedBinaryDir, configuration.Version, "fdbserver")
+		configuration.BinaryPath = path.Join(sharedBinaryDir, configuration.Version.String(), "fdbserver")
 	}
 
 	err = checkOwnerExecutable(configuration.BinaryPath)
 	if err != nil {
-		monitor.Logger.Error(err, "Error with binary path for latest configuration", "configuration", configuration, "binaryPath", configuration.BinaryPath)
-		return
+		monitor.logger.Error(err, "Error with binary path for latest configuration", "configuration", configuration, "binaryPath", configuration.BinaryPath)
+		return nil, nil
 	}
 
 	monitor.updateCustomEnvironmentFromNodeMetadata()
-
-	_, err = configuration.GenerateArguments(1, monitor.CustomEnvironment)
+	_, err = configuration.GenerateArguments(1, monitor.customEnvironment)
 	if err != nil {
-		monitor.Logger.Error(err, "Error generating arguments for latest configuration", "configuration", configuration, "binaryPath", configuration.BinaryPath)
+		monitor.logger.Error(err, "Error generating arguments for latest configuration", "configuration", configuration, "binaryPath", configuration.BinaryPath)
+		return nil, nil
+	}
+
+	if configuration.ShouldRunServers() {
+		// In case that the process is isolated we don't want to start the servers and we should terminate the running fdbserver
+		// instances.
+		if monitor.processIsIsolated() {
+			configuration.RunServers = pointer.Bool(false)
+		}
+	}
+
+	return configuration, configurationBytes
+}
+
+// loadConfiguration loads the latest configuration from the config file.
+func (monitor *monitor) loadConfiguration() {
+	configuration, configurationBytes := monitor.readConfiguration()
+	if configuration == nil || len(configurationBytes) == 0 {
 		return
 	}
 
@@ -275,33 +300,42 @@ func checkOwnerExecutable(path string) error {
 
 // acceptConfiguration is called when the monitor process parses and accepts
 // a configuration from the local config file.
-func (monitor *Monitor) acceptConfiguration(configuration *api.ProcessConfiguration, configurationBytes []byte) {
-	monitor.Mutex.Lock()
-	defer monitor.Mutex.Unlock()
+func (monitor *monitor) acceptConfiguration(configuration *api.ProcessConfiguration, configurationBytes []byte) {
+	monitor.mutex.Lock()
+	defer monitor.mutex.Unlock()
 
 	// If the configuration hasn't changed ignore those events to prevent noisy logging.
-	if equality.Semantic.DeepEqual(monitor.ActiveConfiguration, configuration) {
+	if equality.Semantic.DeepEqual(monitor.activeConfiguration, configuration) {
 		return
 	}
 
-	monitor.Logger.Info("Received new configuration file", "configuration", configuration)
-	monitor.ActiveConfiguration = configuration
-	monitor.ActiveConfigurationBytes = configurationBytes
-	monitor.LastConfigurationTime = time.Now()
+	monitor.logger.Info("Received new configuration file", "configuration", configuration)
+	monitor.activeConfiguration = configuration
+	monitor.activeConfigurationBytes = configurationBytes
+	monitor.lastConfigurationTime = time.Now()
 	// Update the prometheus metrics.
-	monitor.metrics.registerConfigurationChange(configuration.Version)
+	monitor.metrics.registerConfigurationChange(configuration.Version.String())
 
-	for processNumber := 1; processNumber <= monitor.ProcessCount; processNumber++ {
-		if monitor.ProcessIDs[processNumber] == 0 {
-			monitor.ProcessIDs[processNumber] = -1
+	var hasRunningProcesses bool
+	for processNumber := 1; processNumber <= monitor.processCount; processNumber++ {
+		if monitor.processIDs[processNumber] == 0 {
+			monitor.processIDs[processNumber] = -1
 			tempNumber := processNumber
-			go func() { monitor.RunProcess(tempNumber) }()
+			go func() { monitor.runProcess(tempNumber) }()
+			continue
 		}
+
+		hasRunningProcesses = true
+	}
+
+	// If the monitor has running processes but the processes shouldn't be running, kill them with SIGTERM.
+	if hasRunningProcesses && !monitor.activeConfiguration.ShouldRunServers() {
+		monitor.sendSignalToProcesses(syscall.SIGTERM)
 	}
 
-	err := monitor.PodClient.UpdateAnnotations(monitor)
+	err := monitor.podClient.updateAnnotations(monitor)
 	if err != nil {
-		monitor.Logger.Error(err, "Error updating pod annotations")
+		monitor.logger.Error(err, "Error updating pod annotations")
 	}
 }
 
@@ -315,10 +349,10 @@ func getBackoffDuration(errorCounter int) time.Duration {
 	return timeToBackoff
 }
 
-// RunProcess runs a loop to continually start and watch a process.
-func (monitor *Monitor) RunProcess(processNumber int) {
+// runProcess runs a loop to continually start and watch a process.
+func (monitor *monitor) runProcess(processNumber int) {
 	pid := 0
-	logger := monitor.Logger.WithValues("processNumber", processNumber, "area", "RunProcess")
+	logger := monitor.logger.WithValues("processNumber", processNumber, "area", "runProcess")
 	logger.Info("Starting run loop")
 	startTime := time.Now()
 	// Counts the successive errors that occurred during process start up. Based on the error count the backoff time
@@ -336,10 +370,10 @@ func (monitor *Monitor) RunProcess(processNumber int) {
 			errorCounter = 0
 		}
 
-		arguments, err := monitor.ActiveConfiguration.GenerateArguments(processNumber, monitor.CustomEnvironment)
+		arguments, err := monitor.activeConfiguration.GenerateArguments(processNumber, monitor.customEnvironment)
 		if err != nil {
 			backoffDuration := getBackoffDuration(errorCounter)
-			logger.Error(err, "Error generating arguments for subprocess", "configuration", monitor.ActiveConfiguration, "errorCounter", errorCounter, "backoffDuration", backoffDuration.String())
+			logger.Error(err, "Error generating arguments for subprocess", "configuration", monitor.activeConfiguration, "errorCounter", errorCounter, "backoffDuration", backoffDuration.String())
 			time.Sleep(backoffDuration)
 			errorCounter++
 			continue
@@ -371,7 +405,7 @@ func (monitor *Monitor) RunProcess(processNumber int) {
 		}
 
 		// Update the prometheus metrics for the process.
-		monitor.metrics.registerProcessStartup(processNumber, monitor.ActiveConfiguration.Version)
+		monitor.metrics.registerProcessStartup(processNumber, monitor.activeConfiguration.Version.String())
 
 		if cmd.Process != nil {
 			pid = cmd.Process.Pid
@@ -430,14 +464,14 @@ func (monitor *Monitor) RunProcess(processNumber int) {
 // If the process is no longer desired, this will remove it from the process ID
 // list and return false. If the process is still desired, this will return
 // true.
-func (monitor *Monitor) processRequired(processNumber int) bool {
-	monitor.Mutex.Lock()
-	defer monitor.Mutex.Unlock()
-	logger := monitor.Logger.WithValues("processNumber", processNumber, "area", "processRequired")
-	if monitor.ProcessCount < processNumber || !pointer.BoolDeref(monitor.ActiveConfiguration.RunServers, true) {
-		if monitor.ProcessIDs[processNumber] != 0 {
+func (monitor *monitor) processRequired(processNumber int) bool {
+	monitor.mutex.Lock()
+	defer monitor.mutex.Unlock()
+	logger := monitor.logger.WithValues("processNumber", processNumber, "area", "processRequired")
+	if monitor.processCount < processNumber || !monitor.activeConfiguration.ShouldRunServers() {
+		if monitor.processIDs[processNumber] != 0 {
 			logger.Info("Terminating run loop")
-			monitor.ProcessIDs[processNumber] = 0
+			monitor.processIDs[processNumber] = 0
 		}
 
 		return false
@@ -446,15 +480,39 @@ func (monitor *Monitor) processRequired(processNumber int) bool {
 	return true
 }
 
+// processIsIsolated returns true if the IsolateProcessGroupAnnotation is set to "true".
+func (monitor *monitor) processIsIsolated() bool {
+	if monitor.podClient.podMetadata == nil {
+		return false
+	}
+
+	if monitor.podClient.podMetadata.Annotations == nil {
+		return false
+	}
+
+	val, ok := monitor.podClient.podMetadata.Annotations[api.IsolateProcessGroupAnnotation]
+	if !ok {
+		return false
+	}
+
+	isolated, err := strconv.ParseBool(val)
+	if err != nil {
+		monitor.logger.Error(err, "could not parse the value of the %s annotation", api.IsolateProcessGroupAnnotation)
+		return false
+	}
+
+	return isolated
+}
+
 // updateProcessID records a new Process ID from a newly launched process.
-func (monitor *Monitor) updateProcessID(processNumber int, pid int) {
-	monitor.Mutex.Lock()
-	defer monitor.Mutex.Unlock()
-	monitor.ProcessIDs[processNumber] = pid
+func (monitor *monitor) updateProcessID(processNumber int, pid int) {
+	monitor.mutex.Lock()
+	defer monitor.mutex.Unlock()
+	monitor.processIDs[processNumber] = pid
 }
 
-// WatchConfiguration detects changes to the monitor configuration file.
-func (monitor *Monitor) WatchConfiguration(watcher *fsnotify.Watcher) {
+// watchConfiguration detects changes to the monitor configuration file.
+func (monitor *monitor) watchConfiguration(watcher *fsnotify.Watcher) {
 	for {
 		select {
 		case event, ok := <-watcher.Events:
@@ -462,7 +520,7 @@ func (monitor *Monitor) WatchConfiguration(watcher *fsnotify.Watcher) {
 				return
 			}
 
-			monitor.Logger.Info("Detected event on monitor conf file or cluster file", "event", event)
+			monitor.logger.Info("Detected event on monitor conf file or cluster file", "event", event)
 			if event.Op&fsnotify.Write == fsnotify.Write || event.Op&fsnotify.Create == fsnotify.Create {
 				monitor.handleFileChange(event.Name)
 			} else if event.Op&fsnotify.Remove == fsnotify.Remove {
@@ -476,58 +534,62 @@ func (monitor *Monitor) WatchConfiguration(watcher *fsnotify.Watcher) {
 			if !ok {
 				return
 			}
-			monitor.Logger.Error(err, "Error watching for file system events")
+			monitor.logger.Error(err, "Error watching for file system events")
 		}
 	}
 }
 
 // handleFileChange will perform the required action based on the changed/modified file.
-func (monitor *Monitor) handleFileChange(changedFile string) {
+func (monitor *monitor) handleFileChange(changedFile string) {
 	if changedFile == fdbClusterFilePath {
-		err := monitor.PodClient.updateFdbClusterTimestampAnnotation()
+		err := monitor.podClient.updateFdbClusterTimestampAnnotation()
 		if err != nil {
-			monitor.Logger.Error(err, fmt.Sprintf("could not update %s annotation", ClusterFileChangeDetectedAnnotation))
+			monitor.logger.Error(err, fmt.Sprintf("could not update %s annotation", api.ClusterFileChangeDetectedAnnotation))
 		}
 		return
 	}
 
-	monitor.LoadConfiguration()
+	monitor.loadConfiguration()
 }
 
-// Run runs the monitor loop.
-func (monitor *Monitor) Run() {
+func (monitor *monitor) sendSignalToProcesses(signal os.Signal) {
+	for processNumber, processID := range monitor.processIDs {
+		if processID <= 0 {
+			continue
+		}
+
+		subprocessLogger := monitor.logger.WithValues("processNumber", processNumber, "PID", processID)
+		process, err := os.FindProcess(processID)
+		if err != nil {
+			subprocessLogger.Error(err, "Error finding subprocess")
+			continue
+		}
+		subprocessLogger.Info("Sending signal to subprocess", "signal", signal)
+		err = process.Signal(signal)
+		if err != nil {
+			subprocessLogger.Error(err, "Error signaling subprocess")
+			continue
+		}
+	}
+}
+
+// run runs the monitor loop.
+func (monitor *monitor) run() {
 	done := make(chan bool, 1)
 	signals := make(chan os.Signal, 1)
 	signal.Notify(signals, syscall.SIGINT, syscall.SIGTERM)
 
 	go func() {
 		latestSignal := <-signals
-		monitor.Logger.Info("Received system signal", "signal", latestSignal)
-
-		// Reset the ProcessCount to 0 to make sure the monitor doesn't try to restart the processes.
-		monitor.ProcessCount = 0
-		for processNumber, processID := range monitor.ProcessIDs {
-			if processID <= 0 {
-				continue
-			}
+		monitor.logger.Info("Received system signal", "signal", latestSignal)
 
-			subprocessLogger := monitor.Logger.WithValues("processNumber", processNumber, "PID", processID)
-			process, err := os.FindProcess(processID)
-			if err != nil {
-				subprocessLogger.Error(err, "Error finding subprocess")
-				continue
-			}
-			subprocessLogger.Info("Sending signal to subprocess", "signal", latestSignal)
-			err = process.Signal(latestSignal)
-			if err != nil {
-				subprocessLogger.Error(err, "Error signaling subprocess")
-				continue
-			}
-		}
+		// Reset the processCount to 0 to make sure the monitor doesn't try to restart the processes.
+		monitor.processCount = 0
+		monitor.sendSignalToProcesses(latestSignal)
 
-		annotations := monitor.PodClient.podMetadata.Annotations
+		annotations := monitor.podClient.podMetadata.Annotations
 		if len(annotations) > 0 {
-			delayValue, ok := annotations[DelayShutdownAnnotation]
+			delayValue, ok := annotations[api.DelayShutdownAnnotation]
 			if ok {
 				delay, err := time.ParseDuration(delayValue)
 				if err == nil {
@@ -539,13 +601,13 @@ func (monitor *Monitor) Run() {
 		done <- true
 	}()
 
-	monitor.LoadConfiguration()
+	monitor.loadConfiguration()
 	watcher, err := fsnotify.NewWatcher()
 	if err != nil {
 		panic(err)
 	}
-	monitor.Logger.Info("adding watch for file", "path", path.Base(monitor.ConfigFile))
-	err = watcher.Add(monitor.ConfigFile)
+	monitor.logger.Info("adding watch for file", "path", path.Base(monitor.configFile))
+	err = watcher.Add(monitor.configFile)
 	if err != nil {
 		panic(err)
 	}
@@ -553,10 +615,10 @@ func (monitor *Monitor) Run() {
 	defer func(watcher *fsnotify.Watcher) {
 		err := watcher.Close()
 		if err != nil {
-			monitor.Logger.Error(err, "could not close watcher")
+			monitor.logger.Error(err, "could not close watcher")
 		}
 	}(watcher)
-	go func() { monitor.WatchConfiguration(watcher) }()
+	go func() { monitor.watchConfiguration(watcher) }()
 
 	// The cluster file will be created and managed by the fdbserver processes, so we have to wait until the fdbserver
 	// processes have been started. Except for the initial cluster creation this file should be present as soon as the
@@ -564,12 +626,12 @@ func (monitor *Monitor) Run() {
 	for {
 		_, err = os.Stat(fdbClusterFilePath)
 		if errors.Is(err, os.ErrNotExist) {
-			monitor.Logger.Info("waiting for file to be created", "path", fdbClusterFilePath)
+			monitor.logger.Info("waiting for file to be created", "path", fdbClusterFilePath)
 			time.Sleep(5 * time.Second)
 			continue
 		}
 
-		monitor.Logger.Info("adding watch for file", "path", fdbClusterFilePath)
+		monitor.logger.Info("adding watch for file", "path", fdbClusterFilePath)
 		err = watcher.Add(fdbClusterFilePath)
 		if err != nil {
 			panic(err)
@@ -580,11 +642,11 @@ func (monitor *Monitor) Run() {
 	<-done
 }
 
-// WatchPodTimestamps watches the timestamp feed to reload the configuration.
-func (monitor *Monitor) WatchPodTimestamps() {
-	for timestamp := range monitor.PodClient.TimestampFeed {
-		if timestamp > monitor.LastConfigurationTime.Unix() {
-			monitor.LoadConfiguration()
+// watchPodTimestamps watches the timestamp feed to reload the configuration.
+func (monitor *monitor) watchPodTimestamps() {
+	for timestamp := range monitor.podClient.TimestampFeed {
+		if timestamp > monitor.lastConfigurationTime.Unix() {
+			monitor.loadConfiguration()
 		}
 	}
 }
diff --git a/fdbkubernetesmonitor/monitor_test.go b/fdbkubernetesmonitor/monitor_test.go
index 3f3ef194704..7ee9db84acf 100644
--- a/fdbkubernetesmonitor/monitor_test.go
+++ b/fdbkubernetesmonitor/monitor_test.go
@@ -20,41 +20,46 @@
 package main
 
 import (
+	"os"
+	"path"
 	"time"
 
+	"github.com/apple/foundationdb/fdbkubernetesmonitor/api"
+	"k8s.io/apimachinery/pkg/util/json"
+
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
-var _ = Describe("Testing FDB Kubernetes Monitor", func() {
+var _ = Describe("Testing FDB Kubernetes monitor", func() {
 	When("updating the custom environment variables from the node metadata", func() {
-		var monitor *Monitor
+		var mon *monitor
 
 		BeforeEach(func() {
-			monitor = &Monitor{
-				Logger: GinkgoLogr,
-				CustomEnvironment: map[string]string{
+			mon = &monitor{
+				logger: GinkgoLogr,
+				customEnvironment: map[string]string{
 					"testing": "testing",
 				},
-				PodClient: &PodClient{},
+				podClient: &kubernetesClient{},
 			}
 		})
 
 		JustBeforeEach(func() {
-			monitor.updateCustomEnvironmentFromNodeMetadata()
+			mon.updateCustomEnvironmentFromNodeMetadata()
 		})
 
 		When("no node metadata is present", func() {
 			It("shouldn't add new entries", func() {
-				Expect(monitor.CustomEnvironment).To(HaveLen(1))
-				Expect(monitor.CustomEnvironment).To(HaveKeyWithValue("testing", "testing"))
+				Expect(mon.customEnvironment).To(HaveLen(1))
+				Expect(mon.customEnvironment).To(HaveKeyWithValue("testing", "testing"))
 			})
 		})
 
 		When("node metadata is present", func() {
 			BeforeEach(func() {
-				monitor.PodClient.nodeMetadata = &metav1.PartialObjectMetadata{
+				mon.podClient.nodeMetadata = &metav1.PartialObjectMetadata{
 					ObjectMeta: metav1.ObjectMeta{
 						Labels: map[string]string{
 							// Examples are taken from: https://kubernetes.io/docs/reference/labels-annotations-taints/
@@ -66,10 +71,10 @@ var _ = Describe("Testing FDB Kubernetes Monitor", func() {
 			})
 
 			It("should add the new entries", func() {
-				Expect(monitor.CustomEnvironment).To(HaveLen(3))
-				Expect(monitor.CustomEnvironment).To(HaveKeyWithValue("testing", "testing"))
-				Expect(monitor.CustomEnvironment).To(HaveKeyWithValue("NODE_LABEL_TOPOLOGY_KUBERNETES_IO_ZONE", "us-east-1c"))
-				Expect(monitor.CustomEnvironment).To(HaveKeyWithValue("NODE_LABEL_KUBERNETES_IO_HOSTNAME", "ip-172-20-114-199.ec2.internal"))
+				Expect(mon.customEnvironment).To(HaveLen(3))
+				Expect(mon.customEnvironment).To(HaveKeyWithValue("testing", "testing"))
+				Expect(mon.customEnvironment).To(HaveKeyWithValue("NODE_LABEL_TOPOLOGY_KUBERNETES_IO_ZONE", "us-east-1c"))
+				Expect(mon.customEnvironment).To(HaveKeyWithValue("NODE_LABEL_KUBERNETES_IO_HOSTNAME", "ip-172-20-114-199.ec2.internal"))
 			})
 		})
 	})
@@ -98,4 +103,90 @@ var _ = Describe("Testing FDB Kubernetes Monitor", func() {
 			60*time.Second,
 		),
 	)
+
+	When("reading the configuration file", func() {
+		var mon *monitor
+		var configurationFilePath string
+		var configuration *api.ProcessConfiguration
+		var configurationBytes []byte
+		defaultVersion := api.Version{Major: 7, Minor: 1, Patch: 51}
+
+		BeforeEach(func() {
+			tmpDir := GinkgoT().TempDir()
+			// Set the fdbserverPath to make sure the executable check works.
+			fdbserverPath = path.Join(tmpDir, "fdberver")
+			Expect(os.WriteFile(fdbserverPath, []byte(""), 0700)).NotTo(HaveOccurred())
+			configurationFilePath = path.Join(tmpDir, "config.json")
+			mon = &monitor{
+				logger:                  GinkgoLogr,
+				configFile:              configurationFilePath,
+				customEnvironment:       map[string]string{},
+				podClient:               &kubernetesClient{},
+				currentContainerVersion: defaultVersion,
+			}
+		})
+
+		JustBeforeEach(func() {
+			configuration, configurationBytes = mon.readConfiguration()
+		})
+
+		When("the configuration file is empty", func() {
+			BeforeEach(func() {
+				Expect(os.WriteFile(configurationFilePath, []byte("{}"), 0600)).NotTo(HaveOccurred())
+			})
+
+			It("should return an empty configuration", func() {
+				Expect(configuration).To(BeNil())
+				Expect(configurationBytes).To(BeNil())
+			})
+		})
+
+		When("the configuration is valid", func() {
+			var out []byte
+
+			BeforeEach(func() {
+				config := &api.ProcessConfiguration{
+					Version: &defaultVersion,
+				}
+
+				var err error
+				out, err = json.Marshal(config)
+				Expect(err).NotTo(HaveOccurred())
+				Expect(os.WriteFile(configurationFilePath, out, 0600)).NotTo(HaveOccurred())
+			})
+
+			It("should return the configuration", func() {
+				Expect(configuration).NotTo(BeNil())
+				Expect(configuration.Version).To(Equal(&defaultVersion))
+				Expect(configuration.BinaryPath).To(Equal(fdbserverPath))
+				Expect(configuration.RunServers).To(BeNil())
+				Expect(configuration.Arguments).To(BeEmpty())
+				Expect(configurationBytes).To(Equal(out))
+			})
+
+			When("the pod has the isolate annotation set to true", func() {
+				BeforeEach(func() {
+					mon.podClient = &kubernetesClient{
+						podMetadata: &metav1.PartialObjectMetadata{
+							ObjectMeta: metav1.ObjectMeta{
+								Annotations: map[string]string{
+									api.IsolateProcessGroupAnnotation: "true",
+								},
+							},
+						},
+					}
+				})
+
+				It("should return the configuration with runServers set to false", func() {
+					Expect(configuration).NotTo(BeNil())
+					Expect(configuration.Version).To(Equal(&defaultVersion))
+					Expect(configuration.BinaryPath).To(Equal(fdbserverPath))
+					Expect(configuration.RunServers).NotTo(BeNil())
+					Expect(*configuration.RunServers).To(BeFalse())
+					Expect(configuration.Arguments).To(BeEmpty())
+					Expect(configurationBytes).To(Equal(out))
+				})
+			})
+		})
+	})
 })
diff --git a/fdbkubernetesmonitor/suite_test.go b/fdbkubernetesmonitor/suite_test.go
index 97f1d56e587..c249533690a 100644
--- a/fdbkubernetesmonitor/suite_test.go
+++ b/fdbkubernetesmonitor/suite_test.go
@@ -33,5 +33,5 @@ import (
 func TestAPIs(t *testing.T) {
 	RegisterFailHandler(Fail)
 	SetDefaultEventuallyTimeout(10 * time.Second)
-	RunSpecs(t, "FDB Kubernetes Monitor")
+	RunSpecs(t, "FDB Kubernetes monitor")
 }

From bcb9d35f3dcc993c2079f8bc0f8a5049d5cdb7d2 Mon Sep 17 00:00:00 2001
From: "Johannes M. Scheuermann" <jscheuermann@apple.com>
Date: Thu, 4 Jul 2024 11:28:14 +0200
Subject: [PATCH 454/468] fdb-kubernetes-monitor: Ensure that the annotations
 are updated when the correct configuration is already loaded

---
 fdbkubernetesmonitor/kubernetes.go | 26 ++++++++++++++++----------
 fdbkubernetesmonitor/monitor.go    | 11 ++++++-----
 2 files changed, 22 insertions(+), 15 deletions(-)

diff --git a/fdbkubernetesmonitor/kubernetes.go b/fdbkubernetesmonitor/kubernetes.go
index d6c9bb127c2..2cf3f17c1d3 100644
--- a/fdbkubernetesmonitor/kubernetes.go
+++ b/fdbkubernetesmonitor/kubernetes.go
@@ -239,11 +239,6 @@ func (podClient *kubernetesClient) updateAnnotationsOnPod(annotationChanges map[
 	if podClient.podMetadata == nil {
 		return fmt.Errorf("pod client has no metadata present")
 	}
-	
-	annotations := podClient.podMetadata.Annotations
-	if len(annotations) == 0 {
-		annotations = map[string]string{}
-	}
 
 	if !podClient.podMetadata.DeletionTimestamp.IsZero() {
 		return fmt.Errorf("pod is marked for deletion, cannot update annotations")
@@ -264,13 +259,24 @@ func (podClient *kubernetesClient) updateAnnotationsOnPod(annotationChanges map[
 
 		return true
 	}, func() error {
-		annotations := podClient.podMetadata.Annotations
-		if len(annotations) == 0 {
-			annotations = map[string]string{}
+		currentAnnotations := podClient.podMetadata.Annotations
+		if len(currentAnnotations) == 0 {
+			currentAnnotations = map[string]string{}
 		}
 
+		var hasChanges bool
 		for key, val := range annotationChanges {
-			annotations[key] = val
+			currentValue, present := currentAnnotations[key]
+			if !present || currentValue != val {
+				podClient.Logger.Info("update annotation with new value", "annotation", key, "currentValue", currentValue, "newValue", val, "present", present)
+				currentAnnotations[key] = val
+				hasChanges = true
+			}
+		}
+
+		// If no changes are present, we can skip the patch.
+		if !hasChanges {
+			return nil
 		}
 
 		return podClient.Patch(context.Background(), &corev1.Pod{
@@ -281,7 +287,7 @@ func (podClient *kubernetesClient) updateAnnotationsOnPod(annotationChanges map[
 			ObjectMeta: metav1.ObjectMeta{
 				Namespace:   podClient.podMetadata.Namespace,
 				Name:        podClient.podMetadata.Name,
-				Annotations: annotations,
+				Annotations: currentAnnotations,
 			},
 		}, client.Apply, client.FieldOwner("fdb-kubernetes-monitor"), client.ForceOwnership)
 	})
diff --git a/fdbkubernetesmonitor/monitor.go b/fdbkubernetesmonitor/monitor.go
index 82cb283b5ad..908e6d665b2 100644
--- a/fdbkubernetesmonitor/monitor.go
+++ b/fdbkubernetesmonitor/monitor.go
@@ -283,6 +283,12 @@ func (monitor *monitor) loadConfiguration() {
 	}
 
 	monitor.acceptConfiguration(configuration, configurationBytes)
+	// Always update the annotations if needed to handle cases where the fdb-kubernetes-monitor
+	// has loaded the new configuration but was not able to update the annotations.
+	err := monitor.podClient.updateAnnotations(monitor)
+	if err != nil {
+		monitor.logger.Error(err, "Error updating pod annotations")
+	}
 }
 
 // checkOwnerExecutable validates that a path is a file that exists and is
@@ -332,11 +338,6 @@ func (monitor *monitor) acceptConfiguration(configuration *api.ProcessConfigurat
 	if hasRunningProcesses && !monitor.activeConfiguration.ShouldRunServers() {
 		monitor.sendSignalToProcesses(syscall.SIGTERM)
 	}
-
-	err := monitor.podClient.updateAnnotations(monitor)
-	if err != nil {
-		monitor.logger.Error(err, "Error updating pod annotations")
-	}
 }
 
 // getBackoffDuration returns the backoff duration. The backoff time will increase exponential with a maximum of 60 seconds.

From 6fef3a90853694140056a2ff9ec68e761cc69b9a Mon Sep 17 00:00:00 2001
From: Aaron Molitor <amolitor@apple.com>
Date: Wed, 10 Jul 2024 17:23:35 -0500
Subject: [PATCH 455/468] patch image for building after EOL

---
 packaging/docker/Dockerfile | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/packaging/docker/Dockerfile b/packaging/docker/Dockerfile
index 6cf1e067859..fafbc41f32d 100644
--- a/packaging/docker/Dockerfile
+++ b/packaging/docker/Dockerfile
@@ -18,9 +18,16 @@
 #
 FROM centos:7.9.2009 as base
 
+RUN sed -i s/mirror.centos.org/vault.centos.org/g /etc/yum.repos.d/*.repo && \
+    sed -i s/^#.*baseurl=http/baseurl=http/g /etc/yum.repos.d/*.repo && \
+    sed -i s/^mirrorlist=http/#mirrorlist=http/g /etc/yum.repos.d/*.repo
+
 RUN yum install -y \
     epel-release-7-11 \
     centos-release-scl-2-3.el7.centos && \
+    sed -i s/mirror.centos.org/vault.centos.org/g /etc/yum.repos.d/*.repo && \
+    sed -i s/^#.*baseurl=http/baseurl=http/g /etc/yum.repos.d/*.repo && \
+    sed -i s/^mirrorlist=http/#mirrorlist=http/g /etc/yum.repos.d/*.repo && \
     yum install -y \
     bind-utils-9.11.4-26.P2.el7_9.7 \
     binutils-2.27-44.base.el7_9.1 \

From e2a5bbd6171801b92fd117d5de6809d40ccbdbf0 Mon Sep 17 00:00:00 2001
From: Zhe Wang <zhe.wang@wustl.edu>
Date: Fri, 12 Jul 2024 09:40:48 -0700
Subject: [PATCH 456/468] cherrypick Add consistency-check-urgent-mode to
 tester process class (#11503)

---
 fdbserver/SimulatedCluster.actor.cpp |  3 ++-
 fdbserver/TesterInterface.actor.h    |  3 ++-
 fdbserver/WorkerInterface.actor.h    |  3 ++-
 fdbserver/fdbserver.actor.cpp        | 10 ++++++++--
 fdbserver/tester.actor.cpp           | 13 +++++++++++--
 fdbserver/worker.actor.cpp           | 13 +++++++++----
 6 files changed, 34 insertions(+), 11 deletions(-)

diff --git a/fdbserver/SimulatedCluster.actor.cpp b/fdbserver/SimulatedCluster.actor.cpp
index 6ded84160a4..f1e9b644835 100644
--- a/fdbserver/SimulatedCluster.actor.cpp
+++ b/fdbserver/SimulatedCluster.actor.cpp
@@ -600,7 +600,8 @@ ACTOR Future<ISimulator::KillType> simulatedFDBDRebooter(Reference<IClusterConne
 					                       whitelistBinPaths,
 					                       "",
 					                       {},
-					                       configDBType));
+					                       configDBType,
+					                       false));
 				}
 				if (runBackupAgents != AgentNone) {
 					futures.push_back(runBackup(connRecord));
diff --git a/fdbserver/TesterInterface.actor.h b/fdbserver/TesterInterface.actor.h
index 01dee5181d4..32d519601b9 100644
--- a/fdbserver/TesterInterface.actor.h
+++ b/fdbserver/TesterInterface.actor.h
@@ -120,7 +120,8 @@ struct TesterInterface {
 ACTOR Future<Void> testerServerCore(TesterInterface interf,
                                     Reference<IClusterConnectionRecord> ccr,
                                     Reference<AsyncVar<struct ServerDBInfo> const> serverDBInfo,
-                                    LocalityData locality);
+                                    LocalityData locality,
+                                    bool consistencyCheckUrgentWorkLoadOnly = false);
 
 enum test_location_t { TEST_HERE, TEST_ON_SERVERS, TEST_ON_TESTERS };
 enum test_type_t {
diff --git a/fdbserver/WorkerInterface.actor.h b/fdbserver/WorkerInterface.actor.h
index 44032d805eb..22af286313f 100644
--- a/fdbserver/WorkerInterface.actor.h
+++ b/fdbserver/WorkerInterface.actor.h
@@ -1072,7 +1072,8 @@ ACTOR Future<Void> fdbd(Reference<IClusterConnectionRecord> ccr,
                         std::string whitelistBinPaths,
                         std::string configPath,
                         std::map<std::string, std::string> manualKnobOverrides,
-                        ConfigDBType configDBType);
+                        ConfigDBType configDBType,
+                        bool consistencyCheckUrgentMode);
 
 ACTOR Future<Void> clusterController(Reference<IClusterConnectionRecord> ccr,
                                      Reference<AsyncVar<Optional<ClusterControllerFullInterface>>> currentCC,
diff --git a/fdbserver/fdbserver.actor.cpp b/fdbserver/fdbserver.actor.cpp
index 561f74ea441..1ba54c2611e 100644
--- a/fdbserver/fdbserver.actor.cpp
+++ b/fdbserver/fdbserver.actor.cpp
@@ -107,7 +107,7 @@ enum {
 	OPT_DCID, OPT_MACHINE_CLASS, OPT_BUGGIFY, OPT_VERSION, OPT_BUILD_FLAGS, OPT_CRASHONERROR, OPT_HELP, OPT_NETWORKIMPL, OPT_NOBUFSTDOUT, OPT_BUFSTDOUTERR,
 	OPT_TRACECLOCK, OPT_NUMTESTERS, OPT_DEVHELP, OPT_ROLLSIZE, OPT_MAXLOGS, OPT_MAXLOGSSIZE, OPT_KNOB, OPT_UNITTESTPARAM, OPT_TESTSERVERS, OPT_TEST_ON_SERVERS, OPT_METRICSCONNFILE,
 	OPT_METRICSPREFIX, OPT_LOGGROUP, OPT_LOCALITY, OPT_IO_TRUST_SECONDS, OPT_IO_TRUST_WARN_ONLY, OPT_FILESYSTEM, OPT_PROFILER_RSS_SIZE, OPT_KVFILE,
-	OPT_TRACE_FORMAT, OPT_WHITELIST_BINPATH, OPT_BLOB_CREDENTIAL_FILE, OPT_CONFIG_PATH, OPT_USE_TEST_CONFIG_DB, OPT_FAULT_INJECTION, OPT_PROFILER, OPT_PRINT_SIMTIME, OPT_FLOW_PROCESS_NAME, OPT_FLOW_PROCESS_ENDPOINT
+	OPT_TRACE_FORMAT, OPT_WHITELIST_BINPATH, OPT_BLOB_CREDENTIAL_FILE, OPT_CONFIG_PATH, OPT_USE_TEST_CONFIG_DB, OPT_FAULT_INJECTION, OPT_PROFILER, OPT_PRINT_SIMTIME, OPT_FLOW_PROCESS_NAME, OPT_FLOW_PROCESS_ENDPOINT, OPT_CONSISTENCY_CHECK_URGENT_MODE
 };
 
 CSimpleOpt::SOption g_rgOptions[] = {
@@ -199,6 +199,7 @@ CSimpleOpt::SOption g_rgOptions[] = {
 	{ OPT_PRINT_SIMTIME,         "--print-sim-time",             SO_NONE },
 	{ OPT_FLOW_PROCESS_NAME,     "--process-name",              SO_REQ_SEP },
 	{ OPT_FLOW_PROCESS_ENDPOINT, "--process-endpoint",          SO_REQ_SEP },
+	{ OPT_CONSISTENCY_CHECK_URGENT_MODE, "--consistency-check-urgent-mode", SO_NONE },
 
 #ifndef TLS_DISABLED
 	TLS_OPTION_FLAGS
@@ -1036,6 +1037,7 @@ struct CLIOptions {
 	LocalityData localities;
 	int minTesterCount = 1;
 	bool testOnServers = false;
+	bool consistencyCheckUrgentMode = false;
 
 	TLSConfig tlsConfig = TLSConfig(TLSEndpointType::SERVER);
 	double fileIoTimeout = 0.0;
@@ -1518,6 +1520,9 @@ struct CLIOptions {
 			case OPT_TEST_ON_SERVERS:
 				testOnServers = true;
 				break;
+			case OPT_CONSISTENCY_CHECK_URGENT_MODE:
+				consistencyCheckUrgentMode = true;
+				break;
 			case OPT_METRICSCONNFILE:
 				metricsConnFile = args.OptionArg();
 				break;
@@ -2154,7 +2159,8 @@ int main(int argc, char* argv[]) {
 				                      opts.whitelistBinPaths,
 				                      opts.configPath,
 				                      opts.manualKnobOverrides,
-				                      opts.configDBType));
+				                      opts.configDBType,
+				                      opts.consistencyCheckUrgentMode));
 				actors.push_back(histogramReport());
 				// actors.push_back( recurring( []{}, .001 ) );  // for ASIO latency measurement
 
diff --git a/fdbserver/tester.actor.cpp b/fdbserver/tester.actor.cpp
index 52fd3c4db76..95564759556 100644
--- a/fdbserver/tester.actor.cpp
+++ b/fdbserver/tester.actor.cpp
@@ -707,7 +707,8 @@ ACTOR Future<Void> testerServerWorkload(WorkloadRequest work,
 ACTOR Future<Void> testerServerCore(TesterInterface interf,
                                     Reference<IClusterConnectionRecord> ccr,
                                     Reference<AsyncVar<struct ServerDBInfo> const> dbInfo,
-                                    LocalityData locality) {
+                                    LocalityData locality,
+                                    bool consistencyCheckUrgentWorkLoadOnly) {
 	state PromiseStream<Future<Void>> addWorkload;
 	state Future<Void> workerFatalError = actorCollection(addWorkload.getFuture());
 
@@ -715,7 +716,8 @@ ACTOR Future<Void> testerServerCore(TesterInterface interf,
 	// At any time, we only allow at most 1 consistency checker workload on a server
 	state std::pair<int64_t, Future<Void>> consistencyCheckerUrgentTester = std::make_pair(0, Future<Void>());
 
-	TraceEvent(SevInfo, "StartingTesterServerCore", interf.id());
+	TraceEvent(SevInfo, "StartingTesterServerCore", interf.id())
+	    .detail("ConsistencyCheckUrgentMode", consistencyCheckUrgentWorkLoadOnly);
 	loop choose {
 		when(wait(workerFatalError)) {}
 		when(wait(consistencyCheckerUrgentTester.second.isValid() ? consistencyCheckerUrgentTester.second : Never())) {
@@ -748,6 +750,13 @@ ACTOR Future<Void> testerServerCore(TesterInterface interf,
 				    .detail("ConsistencyCheckerId", consistencyCheckerUrgentTester.first)
 				    .detail("ClientId", work.clientId)
 				    .detail("ClientCount", work.clientCount);
+			} else if (consistencyCheckUrgentWorkLoadOnly) {
+				TraceEvent(SevError, "StartingTesterServerCoreUnexpectedWorkload", interf.id())
+				    .detail("SharedRandomNumber", work.sharedRandomNumber)
+				    .detail("ClientId", work.clientId)
+				    .detail("ClientCount", work.clientCount)
+				    .detail("WorkLoad", work.title);
+				// Drop the workload
 			} else {
 				addWorkload.send(testerServerWorkload(work, ccr, dbInfo, locality, /*isConsistencyCheckUrgent=*/false));
 			}
diff --git a/fdbserver/worker.actor.cpp b/fdbserver/worker.actor.cpp
index c89e5eabda5..228d5431267 100644
--- a/fdbserver/worker.actor.cpp
+++ b/fdbserver/worker.actor.cpp
@@ -1790,7 +1790,8 @@ ACTOR Future<Void> workerServer(Reference<IClusterConnectionRecord> connRecord,
                                 Reference<AsyncVar<ServerDBInfo>> dbInfo,
                                 ConfigBroadcastInterface configBroadcastInterface,
                                 Reference<ConfigNode> configNode,
-                                Reference<LocalConfiguration> localConfig) {
+                                Reference<LocalConfiguration> localConfig,
+                                bool consistencyCheckUrgentMode) {
 	state PromiseStream<ErrorInfo> errors;
 	state Reference<AsyncVar<Optional<DataDistributorInterface>>> ddInterf(
 	    new AsyncVar<Optional<DataDistributorInterface>>());
@@ -1866,7 +1867,9 @@ ACTOR Future<Void> workerServer(Reference<IClusterConnectionRecord> connRecord,
 	errorForwarders.add(loadedPonger(interf.debugPing.getFuture()));
 	errorForwarders.add(waitFailureServer(interf.waitFailure.getFuture()));
 	errorForwarders.add(monitorTraceLogIssues(issues));
-	errorForwarders.add(testerServerCore(interf.testerInterface, connRecord, dbInfo, locality));
+	// If consistencyCheckUrgentMode is set, the tester server drops any workLoad other than consistencyCheckUrgent
+	errorForwarders.add(
+	    testerServerCore(interf.testerInterface, connRecord, dbInfo, locality, consistencyCheckUrgentMode));
 	errorForwarders.add(monitorHighMemory(memoryProfileThreshold));
 
 	filesClosed.add(stopping.getFuture());
@@ -3126,7 +3129,8 @@ ACTOR Future<Void> fdbd(Reference<IClusterConnectionRecord> connRecord,
                         std::string whitelistBinPaths,
                         std::string configPath,
                         std::map<std::string, std::string> manualKnobOverrides,
-                        ConfigDBType configDBType) {
+                        ConfigDBType configDBType,
+                        bool consistencyCheckUrgentMode) {
 	state std::vector<Future<Void>> actors;
 	state Promise<Void> recoveredDiskFiles;
 	state Reference<ConfigNode> configNode;
@@ -3224,7 +3228,8 @@ ACTOR Future<Void> fdbd(Reference<IClusterConnectionRecord> connRecord,
 		                                                 dbInfo,
 		                                                 configBroadcastInterface,
 		                                                 configNode,
-		                                                 localConfig),
+		                                                 localConfig,
+		                                                 consistencyCheckUrgentMode),
 		                                    "WorkerServer",
 		                                    UID(),
 		                                    &normalWorkerErrors()));

From 7aad24e407e6650635d00736b1b58dd182120ad5 Mon Sep 17 00:00:00 2001
From: Hao Fu <77984096+hfu94@users.noreply.github.com>
Date: Tue, 23 Jul 2024 22:20:48 +0800
Subject: [PATCH 457/468] lower the sample rate of client metric in simulation
 (#11524)

---
 fdbserver/workloads/ClientMetric.actor.cpp | 4 ++--
 tests/fast/ClientMetric.toml               | 8 ++------
 2 files changed, 4 insertions(+), 8 deletions(-)

diff --git a/fdbserver/workloads/ClientMetric.actor.cpp b/fdbserver/workloads/ClientMetric.actor.cpp
index 6a757ea2eb0..0a8ea5925df 100644
--- a/fdbserver/workloads/ClientMetric.actor.cpp
+++ b/fdbserver/workloads/ClientMetric.actor.cpp
@@ -216,8 +216,8 @@ struct ClientMetricWorkload : TestWorkload {
 	//      vs2 should be strictly larger than vs1, to verify new latency metrics are added
 	ACTOR Future<Void> runner(Database cx, ClientMetricWorkload* self) {
 		try {
-			state int initialWrites = deterministicRandom()->randomInt(1, 5);
-			state int secondWrites = deterministicRandom()->randomInt(1, 5);
+			state int initialWrites = deterministicRandom()->randomInt(100, 200);
+			state int secondWrites = deterministicRandom()->randomInt(100, 200);
 
 			state uint64_t zeroVS = 0;
 			state uint64_t vs1 = wait(self->writeKeysAndGetLatencyVersion(cx, self, initialWrites, zeroVS));
diff --git a/tests/fast/ClientMetric.toml b/tests/fast/ClientMetric.toml
index ef4277282e8..7dcb1b7ce86 100644
--- a/tests/fast/ClientMetric.toml
+++ b/tests/fast/ClientMetric.toml
@@ -1,17 +1,13 @@
 [[test]]
 testTitle='ClientMetricTest'
 clearAfterTest=false
+runConsistencyCheck=false
 
     [[test.workload]]
     testName='ClientMetric'
     toSet=true
-    samplingProbability=1.0
+    samplingProbability=0.01
     testDuration=500.0
-    trInfoSizeLimit=177654
-
-    [[test.workload]]
-    testName='RandomClogging'
-    testDuration=30.0
 
     [[test.workload]]
     testName='SaveAndKill'

From c9383ec1d7377d8977351dcaa6d2653a5a7a11ab Mon Sep 17 00:00:00 2001
From: FoundationDB CI <foundationdb_ci@apple.com>
Date: Tue, 30 Jul 2024 10:35:29 +0000
Subject: [PATCH 458/468] disable AVX for 7.1.62 release

---
 cmake/ConfigureCompiler.cmake | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cmake/ConfigureCompiler.cmake b/cmake/ConfigureCompiler.cmake
index e52d8665ee8..7e50469ba6a 100644
--- a/cmake/ConfigureCompiler.cmake
+++ b/cmake/ConfigureCompiler.cmake
@@ -253,7 +253,7 @@ else()
       set(USE_AVX512F OFF)
     endif()
   endif()
-  set(USE_AVX ON CACHE BOOL "Enable AVX instructions")
+  set(USE_AVX OFF CACHE BOOL "Enable AVX instructions")
   if (USE_AVX)
     if (CMAKE_HOST_SYSTEM_PROCESSOR MATCHES "^x86")
       add_compile_options(-mavx)

From 1accd8c90b87c4d193b269a9d71434cf9865f04d Mon Sep 17 00:00:00 2001
From: FoundationDB CI <foundationdb_ci@apple.com>
Date: Tue, 30 Jul 2024 10:36:30 +0000
Subject: [PATCH 459/468] enable AVX and update version for 7.1.63 release

---
 CMakeLists.txt                | 2 +-
 cmake/ConfigureCompiler.cmake | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 9296bea0564..87b002602d4 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -23,7 +23,7 @@ else()
 endif()
 
 project(foundationdb
-  VERSION 7.1.62
+  VERSION 7.1.63
   DESCRIPTION "FoundationDB is a scalable, fault-tolerant, ordered key-value store with full ACID transactions."
   HOMEPAGE_URL "http://www.foundationdb.org/"
   LANGUAGES C CXX ASM)
diff --git a/cmake/ConfigureCompiler.cmake b/cmake/ConfigureCompiler.cmake
index 7e50469ba6a..e52d8665ee8 100644
--- a/cmake/ConfigureCompiler.cmake
+++ b/cmake/ConfigureCompiler.cmake
@@ -253,7 +253,7 @@ else()
       set(USE_AVX512F OFF)
     endif()
   endif()
-  set(USE_AVX OFF CACHE BOOL "Enable AVX instructions")
+  set(USE_AVX ON CACHE BOOL "Enable AVX instructions")
   if (USE_AVX)
     if (CMAKE_HOST_SYSTEM_PROCESSOR MATCHES "^x86")
       add_compile_options(-mavx)

From 607c418602d8056172354c81c4d429044d4fbe74 Mon Sep 17 00:00:00 2001
From: Johannes Scheuermann <johscheuer@users.noreply.github.com>
Date: Fri, 2 Aug 2024 12:27:59 +0200
Subject: [PATCH 460/468] [RELEASE-7.1] Change the default base image to
 RockyLinux9 (#11550)

* Change the default base image to RockyLinux9

* Update the sidecar install commands

* Update the python verson used for the sidecar
---
 packaging/docker/Dockerfile                  | 103 +++++++++----------
 packaging/docker/entrypoint.bash             |   4 -
 packaging/docker/kubernetes/test_config.yaml |  14 +--
 3 files changed, 56 insertions(+), 65 deletions(-)

diff --git a/packaging/docker/Dockerfile b/packaging/docker/Dockerfile
index fafbc41f32d..11c0cf49d67 100644
--- a/packaging/docker/Dockerfile
+++ b/packaging/docker/Dockerfile
@@ -16,44 +16,33 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-FROM centos:7.9.2009 as base
-
-RUN sed -i s/mirror.centos.org/vault.centos.org/g /etc/yum.repos.d/*.repo && \
-    sed -i s/^#.*baseurl=http/baseurl=http/g /etc/yum.repos.d/*.repo && \
-    sed -i s/^mirrorlist=http/#mirrorlist=http/g /etc/yum.repos.d/*.repo
-
-RUN yum install -y \
-    epel-release-7-11 \
-    centos-release-scl-2-3.el7.centos && \
-    sed -i s/mirror.centos.org/vault.centos.org/g /etc/yum.repos.d/*.repo && \
-    sed -i s/^#.*baseurl=http/baseurl=http/g /etc/yum.repos.d/*.repo && \
-    sed -i s/^mirrorlist=http/#mirrorlist=http/g /etc/yum.repos.d/*.repo && \
-    yum install -y \
-    bind-utils-9.11.4-26.P2.el7_9.7 \
-    binutils-2.27-44.base.el7_9.1 \
-    curl-7.29.0-59.el7_9.1 \
-    gdb-7.6.1-120.el7 \
-    hostname-3.13-3.el7_7.1 \
-    jq-1.6-2.el7  \
-    less-458-9.el7 \
-    libubsan-7.3.1-5.16.el7 \
-    lsof-4.87-6.el7 \
-    net-tools-2.0-0.25.20131004git.el7 \
-    nmap-ncat-6.40-19.el7 \
-    perf-3.10.0-1160.45.1.el7 \
-    perl-5.16.3-299.el7_9 \
-    procps-ng-3.3.10-28.el7 \
-    strace-4.24-6.el7 \
-    sysstat-10.1.5-19.el7 \
-    tar-1.26-35.el7 \
-    tcpdump-4.9.2-4.el7_7.1 \
-    telnet-0.17-66.el7 \
-    traceroute-2.0.22-2.el7 \
-    unzip-6.0-22.el7_9 \
-    openssl-1.0.2k-24.el7_9.x86_64 \
-    vim-enhanced-7.4.629-8.el7_9 && \
-    yum clean all && \
-    rm -rf /var/cache/yum
+FROM docker.io/rockylinux/rockylinux:9.4-minimal AS base
+
+RUN microdnf install -y \
+    bind-utils \
+    binutils \
+    curl \
+    gdb \
+    hostname \
+    jq \
+    less \
+    libubsan \
+    lsof \
+    net-tools \
+    nmap-ncat \
+    perf \
+    perl \
+    procps-ng \
+    strace \
+    sysstat \
+    tar \
+    tcpdump \
+    telnet \
+    traceroute \
+    unzip \
+    openssl \
+    vim-enhanced && \
+    microdnf clean all
 
 WORKDIR /tmp
 
@@ -66,13 +55,13 @@ RUN curl -Ls https://github.com/krallin/tini/releases/download/v0.19.0/tini-amd6
 
 WORKDIR /
 
-FROM golang:1.22.2-bullseye AS go-build
+FROM golang:1.22.5-bullseye AS go-build
 
 COPY fdbkubernetesmonitor/ /fdbkubernetesmonitor
 WORKDIR /fdbkubernetesmonitor
 RUN go build -o /fdb-kubernetes-monitor *.go
 
-FROM base as foundationdb-base
+FROM base AS foundationdb-base
 
 WORKDIR /tmp
 ARG FDB_VERSION=6.3.22
@@ -123,7 +112,7 @@ RUN curl -Ls $FDB_WEBSITE/$FDB_VERSION/libfdb_c.x86_64.so -o /usr/lib/libfdb_c.s
 RUN rm -rf /tmp/*
 WORKDIR /
 
-FROM foundationdb-base as fdb-kubernetes-monitor
+FROM foundationdb-base AS fdb-kubernetes-monitor
 
 # Install the kubernetes monitor binary
 COPY --from=go-build /fdb-kubernetes-monitor /usr/bin/
@@ -134,14 +123,20 @@ WORKDIR /var/fdb
 VOLUME /var/fdb/data
 ENTRYPOINT ["/usr/bin/fdb-kubernetes-monitor"]
 
-FROM foundationdb-base as foundationdb-kubernetes-sidecar
-
-RUN yum -y install \
-    rh-python38-2.0-4.el7 \
-    yum clean all && \
-    rm -rf /var/cache/yum && \
-    source /opt/rh/rh-python38/enable && \
-    pip3 install watchdog==0.9.0
+FROM foundationdb-base AS foundationdb-kubernetes-sidecar
+
+# We use python3.9 here as the tests with 3.12 resulted in the following error:
+# error:0A000126:SSL routines::unexpected eof while reading. This could be an issue
+# with the installed openssl version. Since the sidecar support will be deprecated in the
+# future it's fine to use python 3.9 here.
+# EOL for python 3.9 is 2025-10: https://devguide.python.org/versions/
+RUN microdnf -y install \
+    python3.9 \
+    python3.9-pip && \
+    microdnf clean all && \
+    update-alternatives --install /usr/bin/python python /usr/bin/python3.9 20 && \
+    update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.9 20 && \
+    python3 -m pip install watchdog==4.0.1
 
 WORKDIR /
 ADD entrypoint.bash sidecar.py /
@@ -152,7 +147,7 @@ VOLUME /var/output-files
 ENV LISTEN_PORT 8080
 ENTRYPOINT ["/usr/bin/tini", "-g", "--", "/entrypoint.bash"]
 
-FROM foundationdb-base as foundationdb
+FROM foundationdb-base AS foundationdb
 
 WORKDIR /tmp
 RUN curl -LsO https://raw.githubusercontent.com/brendangregg/FlameGraph/90533539b75400297092f973163b8a7b067c66d3/stackcollapse-perf.pl && \
@@ -177,11 +172,11 @@ ENV FDB_CLUSTER_FILE_CONTENTS ""
 ENV FDB_PROCESS_CLASS unset
 ENTRYPOINT ["/usr/bin/tini", "-g", "--", "/var/fdb/scripts/fdb.bash"]
 
-FROM foundationdb-base as ycsb
+FROM foundationdb-base AS ycsb
 
-RUN yum -y install \
-    java-11-openjdk-11.0.13.0.8-1.el7_9 && \
-    yum clean all && \
+RUN microdnf -y install \
+    java-11-openjdk java-11-openjdk-devel && \
+    microdnf clean all && \
     rm -rf /var/cache/yum
 
 WORKDIR /tmp
diff --git a/packaging/docker/entrypoint.bash b/packaging/docker/entrypoint.bash
index b9a62cfabdf..165f11bce4b 100755
--- a/packaging/docker/entrypoint.bash
+++ b/packaging/docker/entrypoint.bash
@@ -24,8 +24,4 @@ if [[ -n "$ADDITIONAL_ENV_FILE" ]]; then
   source $ADDITIONAL_ENV_FILE
 fi
 
-if [[ -f "/opt/rh/rh-python38/enable" ]]; then
-    source /opt/rh/rh-python38/enable
-fi
-
 exec /sidecar.py $*
diff --git a/packaging/docker/kubernetes/test_config.yaml b/packaging/docker/kubernetes/test_config.yaml
index 181a49d9889..b0a1d2da615 100644
--- a/packaging/docker/kubernetes/test_config.yaml
+++ b/packaging/docker/kubernetes/test_config.yaml
@@ -18,7 +18,7 @@
 #
 
 # This file provides an example of using a statefulset to launch FDB processes
-# using the foundationdb-kubernetes image.
+# using the fdb-kubernetes-monitor image.
 #
 # This is not a recommended way to run production clusters, but it can be useful
 # to test the image in development.
@@ -43,7 +43,7 @@ spec:
     spec:
       containers:
         - name: foundationdb
-          image: foundationdb/foundationdb-kubernetes:6.3.13-local
+          image: foundationdb/fdb-kubernetes-monitor:7.1.5-local
           imagePullPolicy: IfNotPresent
           args:
             - --input-dir
@@ -91,13 +91,13 @@ spec:
             - name: logs
               mountPath: /var/fdb/logs
         - name: foundationdb-sidecar
-          image: foundationdb/foundationdb-kubernetes:6.3.15-local
+          image: foundationdb/fdb-kubernetes-monitor:7.1.6-local
           imagePullPolicy: IfNotPresent
           args:
             - --mode
             - sidecar
             - --main-container-version
-            - 6.3.13
+            - 7.1.5
             - --output-dir
             - /var/fdb/shared-binaries
             - --copy-binary
@@ -139,7 +139,7 @@ data:
   config.json: |
     {
       "runProcesses": false,
-      "version": "6.3.13",
+      "version": "7.1.5",
       "arguments": [
         {"value": "--cluster-file"},
         {"value": "/var/fdb/data/fdb.cluster"},
@@ -238,8 +238,8 @@ spec:
         - name: dynamic-conf
           emptyDir: {}
       initContainers:
-        - name: foundationdb-kubernetes-init
-          image: foundationdb/foundationdb-kubernetes:6.3.13-local
+        - name: fdb-kubernetes-monitor-init
+          image: foundationdb/fdb-kubernetes-monitor:7.1.5-local
           imagePullPolicy: IfNotPresent
           args:
             - "--mode"

From bdd27f5877f947651c5bfc81c035fc73f4d89817 Mon Sep 17 00:00:00 2001
From: FoundationDB CI <foundationdb_ci@apple.com>
Date: Fri, 2 Aug 2024 15:47:36 +0000
Subject: [PATCH 461/468] update version after 7.1.63 release

---
 CMakeLists.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 87b002602d4..85f345d9277 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -23,7 +23,7 @@ else()
 endif()
 
 project(foundationdb
-  VERSION 7.1.63
+  VERSION 7.1.64
   DESCRIPTION "FoundationDB is a scalable, fault-tolerant, ordered key-value store with full ACID transactions."
   HOMEPAGE_URL "http://www.foundationdb.org/"
   LANGUAGES C CXX ASM)

From 6d49a76a269558d1bd824b1b9645a3c48665d74a Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Mon, 12 Aug 2024 18:00:28 -0700
Subject: [PATCH 462/468] Add more trace events for exclude command

Use PRIORITY_SYSTEM_IMMEDIATE for excludeServersAndLocalities() call.
---
 fdbcli/ExcludeCommand.actor.cpp | 6 ++++++
 fdbcli/Util.actor.cpp           | 2 ++
 2 files changed, 8 insertions(+)

diff --git a/fdbcli/ExcludeCommand.actor.cpp b/fdbcli/ExcludeCommand.actor.cpp
index 8819089f457..6e69e621231 100644
--- a/fdbcli/ExcludeCommand.actor.cpp
+++ b/fdbcli/ExcludeCommand.actor.cpp
@@ -42,6 +42,7 @@ ACTOR Future<bool> excludeServersAndLocalities(Reference<IDatabase> db,
 	state Reference<ITransaction> tr = db->createTransaction();
 	loop {
 		tr->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES);
+		tr->setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
 		try {
 			if (force && servers.size())
 				tr->set(failed ? fdb_cli::failedForceOptionSpecialKey : fdb_cli::excludedForceOptionSpecialKey,
@@ -78,6 +79,7 @@ ACTOR Future<bool> excludeServersAndLocalities(Reference<IDatabase> db,
 				            : "Type `exclude FORCE failed <ADDRESS...>' to exclude without performing safety checks.");
 				return false;
 			}
+			TraceEvent(SevWarn, "ExcludeServersAndLocalitiesError").error(err);
 			wait(safeThreadFutureToFuture(tr->onError(err)));
 		}
 	}
@@ -107,6 +109,7 @@ ACTOR Future<std::vector<std::string>> getExcludedServers(Reference<IDatabase> d
 			}
 			return exclusions;
 		} catch (Error& e) {
+			TraceEvent(SevWarn, "GetExcludedServersError").error(e);
 			wait(safeThreadFutureToFuture(tr->onError(e)));
 		}
 	}
@@ -137,6 +140,7 @@ ACTOR Future<std::vector<std::string>> getExcludedLocalities(Reference<IDatabase
 			}
 			return excludedLocalities;
 		} catch (Error& e) {
+			TraceEvent(SevWarn, "GetExcludedLocalitiesError").error(e);
 			wait(safeThreadFutureToFuture(tr->onError(e)));
 		}
 	}
@@ -186,6 +190,7 @@ ACTOR Future<std::set<NetworkAddress>> checkForExcludingServers(Reference<IDatab
 
 			wait(delayJittered(1.0)); // SOMEDAY: watches!
 		} catch (Error& e) {
+			TraceEvent(SevWarn, "CheckForExcludingServersError").error(e);
 			wait(safeThreadFutureToFuture(tr->onError(e)));
 		}
 	}
@@ -206,6 +211,7 @@ ACTOR Future<Void> checkForCoordinators(Reference<IDatabase> db, std::set<Addres
 			coordinatorList = NetworkAddress::parseList(coordinators.get().toString());
 			break;
 		} catch (Error& e) {
+			TraceEvent(SevWarn, "CheckForCoordinatorsError").error(e);
 			wait(safeThreadFutureToFuture(tr->onError(e)));
 		}
 	}
diff --git a/fdbcli/Util.actor.cpp b/fdbcli/Util.actor.cpp
index 3c21a3f03ee..0c948db2234 100644
--- a/fdbcli/Util.actor.cpp
+++ b/fdbcli/Util.actor.cpp
@@ -155,6 +155,7 @@ ACTOR Future<bool> getWorkers(Reference<IDatabase> db, std::vector<ProcessData>*
 
 			return true;
 		} catch (Error& e) {
+			TraceEvent(SevWarn, "GetWorkersError").error(e);
 			wait(safeThreadFutureToFuture(tr->onError(e)));
 		}
 	}
@@ -181,6 +182,7 @@ ACTOR Future<Void> getStorageServerInterfaces(Reference<IDatabase> db,
 			}
 			return Void();
 		} catch (Error& e) {
+			TraceEvent(SevWarn, "GetStorageServerInterfacesError").error(e);
 			wait(safeThreadFutureToFuture(tr->onError(e)));
 		}
 	}

From 0e93c685b5635b02258dd71ba51fa60406f675bc Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Tue, 13 Aug 2024 15:41:31 -0700
Subject: [PATCH 463/468] Fix a cluster controller crash in status server

The logSystemConfig could be changed after the yield(), thus the iterator can
points to invalid memory.

To reproduce with gcc build at commit 3e90644779:
   -f tests/slow/DDBalanceAndRemoveStatus.toml -s 3134372275 -b off
---
 fdbserver/Status.actor.cpp | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/fdbserver/Status.actor.cpp b/fdbserver/Status.actor.cpp
index 07e2eef1cda..d1a74d267c9 100644
--- a/fdbserver/Status.actor.cpp
+++ b/fdbserver/Status.actor.cpp
@@ -835,9 +835,8 @@ ACTOR static Future<JsonBuilderObject> processStatusFetcher(
 	}
 
 	state std::vector<OldTLogConf>::const_iterator oldTLogIter;
-	for (oldTLogIter = db->get().logSystemConfig.oldTLogs.begin();
-	     oldTLogIter != db->get().logSystemConfig.oldTLogs.end();
-	     ++oldTLogIter) {
+	state std::vector<OldTLogConf> oldTLogs = db->get().logSystemConfig.oldTLogs;
+	for (oldTLogIter = oldTLogs.begin(); oldTLogIter != oldTLogs.end(); ++oldTLogIter) {
 		for (auto& tLogSet : oldTLogIter->tLogs) {
 			for (auto& it : tLogSet.tLogs) {
 				if (it.present()) {

From e70e53896f6ed5014cd3ee0c6b0655b68a42e974 Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyu_zhou@apple.com>
Date: Tue, 10 Sep 2024 13:00:23 -0700
Subject: [PATCH 464/468] Fix fdbbackup crashes [release-7.1] (#11591)

* Capture default database options in fdbbackup in a local variable.

* Add debug logging to help validate the transaction options set in fdbbackup.

* Reformat changes.

* Rename fields in new trace events to match formatting standards.

* Fix references to the wrong DB in the backup actor initialization. (#11588)

* Fix references to the wrong DB in the backup actor initialization.

* Fix variable name.

* Apply clang-format updates.

---------

Co-authored-by: John Brownlee <john_brownlee@apple.com>
---
 fdbbackup/backup.actor.cpp         | 12 ++++++++----
 fdbclient/NativeAPI.actor.cpp      |  1 +
 fdbclient/ReadYourWrites.actor.cpp |  3 +++
 3 files changed, 12 insertions(+), 4 deletions(-)

diff --git a/fdbbackup/backup.actor.cpp b/fdbbackup/backup.actor.cpp
index 89d393cfc51..ae2a15fac3c 100644
--- a/fdbbackup/backup.actor.cpp
+++ b/fdbbackup/backup.actor.cpp
@@ -4103,10 +4103,12 @@ int main(int argc, char* argv[]) {
 				// where the fdbbackup command hangs infinitely. 60 seconds should be more than
 				// enough for all cases to finish and 5 retries should also be good enough for
 				// most cases.
+				int64_t timeout = 60000;
 				db->setOption(FDBDatabaseOptions::TRANSACTION_TIMEOUT,
-				              Optional<StringRef>(StringRef((const uint8_t*)60000, 8)));
+				              Optional<StringRef>(StringRef((const uint8_t*)&timeout, sizeof(timeout))));
+				int64_t retryLimit = 5;
 				db->setOption(FDBDatabaseOptions::TRANSACTION_RETRY_LIMIT,
-				              Optional<StringRef>(StringRef((const uint8_t*)5, 8)));
+				              Optional<StringRef>(StringRef((const uint8_t*)&retryLimit, sizeof(retryLimit))));
 			}
 
 			return result.present();
@@ -4127,10 +4129,12 @@ int main(int argc, char* argv[]) {
 				// where the fdbbackup command hangs infinitely. 60 seconds should be more than
 				// enough for all cases to finish and 5 retries should also be good enough for
 				// most cases.
+				int64_t timeout = 60000;
 				sourceDb->setOption(FDBDatabaseOptions::TRANSACTION_TIMEOUT,
-				                    Optional<StringRef>(StringRef((const uint8_t*)60000, 8)));
+				                    Optional<StringRef>(StringRef((const uint8_t*)&timeout, sizeof(timeout))));
+				int64_t retryLimit = 5;
 				sourceDb->setOption(FDBDatabaseOptions::TRANSACTION_RETRY_LIMIT,
-				                    Optional<StringRef>(StringRef((const uint8_t*)5, 8)));
+				                    Optional<StringRef>(StringRef((const uint8_t*)&retryLimit, sizeof(retryLimit))));
 			}
 
 			return result.present();
diff --git a/fdbclient/NativeAPI.actor.cpp b/fdbclient/NativeAPI.actor.cpp
index 73e34996f5b..486dd1c8ff3 100644
--- a/fdbclient/NativeAPI.actor.cpp
+++ b/fdbclient/NativeAPI.actor.cpp
@@ -2043,6 +2043,7 @@ void DatabaseContext::setOption(FDBDatabaseOptions::Option option, Optional<Stri
 	if (defaultFor >= 0) {
 		ASSERT(FDBTransactionOptions::optionInfo.find((FDBTransactionOptions::Option)defaultFor) !=
 		       FDBTransactionOptions::optionInfo.end());
+		TraceEvent(SevDebug, "DatabaseContextSetPersistentOption").detail("Option", option).detail("Value", value);
 		transactionDefaults.addOption((FDBTransactionOptions::Option)defaultFor, value.castTo<Standalone<StringRef>>());
 	} else {
 		switch (option) {
diff --git a/fdbclient/ReadYourWrites.actor.cpp b/fdbclient/ReadYourWrites.actor.cpp
index 67fb6939534..a2084f2f433 100644
--- a/fdbclient/ReadYourWrites.actor.cpp
+++ b/fdbclient/ReadYourWrites.actor.cpp
@@ -2417,6 +2417,7 @@ void ReadYourWritesTransaction::setOption(FDBTransactionOptions::Option option,
 }
 
 void ReadYourWritesTransaction::setOptionImpl(FDBTransactionOptions::Option option, Optional<StringRef> value) {
+	TraceEvent(SevDebug, "TransactionSetOption").detail("Option", option).detail("Value", value);
 	switch (option) {
 	case FDBTransactionOptions::READ_YOUR_WRITES_DISABLE:
 		validateOptionValueNotPresent(value);
@@ -2454,11 +2455,13 @@ void ReadYourWritesTransaction::setOptionImpl(FDBTransactionOptions::Option opti
 
 	case FDBTransactionOptions::TIMEOUT:
 		options.timeoutInSeconds = extractIntOption(value, 0, std::numeric_limits<int>::max()) / 1000.0;
+		TraceEvent(SevDebug, "TransactionTimeout").detail("TimeoutInSeconds", options.timeoutInSeconds);
 		resetTimeout();
 		break;
 
 	case FDBTransactionOptions::RETRY_LIMIT:
 		options.maxRetries = (int)extractIntOption(value, -1, std::numeric_limits<int>::max());
+		TraceEvent(SevDebug, "TransactionRetryLimit").detail("MaxRetries", options.maxRetries);
 		break;
 
 	case FDBTransactionOptions::DEBUG_RETRY_LOGGING:

From 008354e36fa0b27a73a3e2904d11efa84509b094 Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyu_zhou@apple.com>
Date: Fri, 13 Sep 2024 13:25:21 -0700
Subject: [PATCH 465/468] Fix TracedTooManyLines in simulation (#11659)

---
 fdbclient/ReadYourWrites.actor.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/fdbclient/ReadYourWrites.actor.cpp b/fdbclient/ReadYourWrites.actor.cpp
index a2084f2f433..948ac371ec1 100644
--- a/fdbclient/ReadYourWrites.actor.cpp
+++ b/fdbclient/ReadYourWrites.actor.cpp
@@ -2417,7 +2417,7 @@ void ReadYourWritesTransaction::setOption(FDBTransactionOptions::Option option,
 }
 
 void ReadYourWritesTransaction::setOptionImpl(FDBTransactionOptions::Option option, Optional<StringRef> value) {
-	TraceEvent(SevDebug, "TransactionSetOption").detail("Option", option).detail("Value", value);
+	TraceEvent(SevVerbose, "TransactionSetOption").detail("Option", option).detail("Value", value);
 	switch (option) {
 	case FDBTransactionOptions::READ_YOUR_WRITES_DISABLE:
 		validateOptionValueNotPresent(value);

From 3cf2871ee25a75ead41ea12a444e2070190ca175 Mon Sep 17 00:00:00 2001
From: FoundationDB CI <foundationdb_ci@apple.com>
Date: Thu, 14 Nov 2024 13:42:32 +0000
Subject: [PATCH 466/468] disable AVX for 7.1.64 release

---
 cmake/ConfigureCompiler.cmake | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cmake/ConfigureCompiler.cmake b/cmake/ConfigureCompiler.cmake
index e52d8665ee8..7e50469ba6a 100644
--- a/cmake/ConfigureCompiler.cmake
+++ b/cmake/ConfigureCompiler.cmake
@@ -253,7 +253,7 @@ else()
       set(USE_AVX512F OFF)
     endif()
   endif()
-  set(USE_AVX ON CACHE BOOL "Enable AVX instructions")
+  set(USE_AVX OFF CACHE BOOL "Enable AVX instructions")
   if (USE_AVX)
     if (CMAKE_HOST_SYSTEM_PROCESSOR MATCHES "^x86")
       add_compile_options(-mavx)

From 20349bdb4beea12ff77e23a55588781587a55953 Mon Sep 17 00:00:00 2001
From: FoundationDB CI <foundationdb_ci@apple.com>
Date: Thu, 14 Nov 2024 13:43:35 +0000
Subject: [PATCH 467/468] enable AVX and update version for 7.1.65 release

---
 CMakeLists.txt                | 2 +-
 cmake/ConfigureCompiler.cmake | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 85f345d9277..9e762216e38 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -23,7 +23,7 @@ else()
 endif()
 
 project(foundationdb
-  VERSION 7.1.64
+  VERSION 7.1.65
   DESCRIPTION "FoundationDB is a scalable, fault-tolerant, ordered key-value store with full ACID transactions."
   HOMEPAGE_URL "http://www.foundationdb.org/"
   LANGUAGES C CXX ASM)
diff --git a/cmake/ConfigureCompiler.cmake b/cmake/ConfigureCompiler.cmake
index 7e50469ba6a..e52d8665ee8 100644
--- a/cmake/ConfigureCompiler.cmake
+++ b/cmake/ConfigureCompiler.cmake
@@ -253,7 +253,7 @@ else()
       set(USE_AVX512F OFF)
     endif()
   endif()
-  set(USE_AVX OFF CACHE BOOL "Enable AVX instructions")
+  set(USE_AVX ON CACHE BOOL "Enable AVX instructions")
   if (USE_AVX)
     if (CMAKE_HOST_SYSTEM_PROCESSOR MATCHES "^x86")
       add_compile_options(-mavx)

From e5fe84fc36c71ae870fff85b80e7992f196d8bee Mon Sep 17 00:00:00 2001
From: Mark Shabanov <mshabanov@openintegration.inc>
Date: Thu, 3 Apr 2025 17:15:09 +0300
Subject: [PATCH 468/468] fixed boost download url

---
 cmake/CompileBoost.cmake | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cmake/CompileBoost.cmake b/cmake/CompileBoost.cmake
index 8d93bc8f4db..13bdd6c46ed 100644
--- a/cmake/CompileBoost.cmake
+++ b/cmake/CompileBoost.cmake
@@ -49,7 +49,7 @@ function(compile_boost)
   include(ExternalProject)
   set(BOOST_INSTALL_DIR "${CMAKE_BINARY_DIR}/boost_install")
   ExternalProject_add("${COMPILE_BOOST_TARGET}Project"
-    URL "https://boostorg.jfrog.io/artifactory/main/release/1.78.0/source/boost_1_78_0.tar.bz2"
+    URL "https://archives.boost.io/release/1.78.0/source/boost_1_78_0.tar.bz2"
     URL_HASH SHA256=8681f175d4bdb26c52222665793eef08490d7758529330f98d3b29dd0735bccc
     CONFIGURE_COMMAND ${BOOTSTRAP_COMMAND} ${BOOTSTRAP_ARGS} --with-libraries=${BOOTSTRAP_LIBRARIES} --with-toolset=${BOOST_TOOLSET}
     BUILD_COMMAND ${B2_COMMAND} toolset=${BOOST_TOOLSET} link=static ${COMPILE_BOOST_BUILD_ARGS} --prefix=${BOOST_INSTALL_DIR} ${USER_CONFIG_FLAG} install