Xdb 546 merge main

.clang-tidy

@@ -0,0 +1,21 @@
+---
+Checks: >
+  -*,
+  bugprone-sizeof-expression,
+  bugprone-suspicious-string-compare,
+  bugprone-use-after-move,
+  modernize-use-auto,
+  modernize-use-equals-default,
+  modernize-use-override,
+  modernize-use-using,
+  performance-for-range-copy,
+  readability-container-contains,
+  readability-const-return-type,
+  readability-container-size-empty,
+  readability-delete-null-pointer,
+  readability-duplicate-include,
+  readability-inconsistent-ifelse-braces
+CheckOptions:
+  - key: modernize-use-auto.MinTypeNameLength
+    value: '11'
+...

.github/workflows/stale.yml

@@ -0,0 +1,24 @@
+name: 'Close stale pull requests'
+on:
+  schedule:
+    - cron: '15 20 * * *' # 20:15 UTC
+
+permissions:
+  issues: write
+  pull-requests: write
+
+jobs:
+  stale:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/stale@b5d41d4e1d5dceea10e7104786b73624c18a190f # v10.2.0
+        with:
+          stale-pr-message: >
+            This PR is stale because it has been open 150 days with no activity.
+            Remove stale label or add a comment to prevent automatic closure in 14 days.
+          days-before-pr-stale: 150
+          days-before-close: 14
+          days-before-issue-stale: 5475 # 15 years, basically disabled for issues
+          # throttled to 20 per day
+          operations-per-run: 20
+          ascending: true

.github/workflows/windows-boost-test.yml

@@ -24,7 +24,7 @@ jobs:
         run: |
           vcpkg install boost-filesystem:x64-windows boost-iostreams:x64-windows boost-serialization:x64-windows boost-system:x64-windows boost-program-options:x64-windows boost-url:x64-windows boost-context:x64-windows lz4:x64-windows
         shell: cmd
-
+        
       - name: Configure and Build FoundationDB (without Swift)
         run: |
           mkdir build

.gitignore

@@ -17,6 +17,7 @@ cmake-build-*/
 *.g.S
 *.g.asm
 *.pom
+fdbserver/include/fdbserver/FDBRocksDBVersion.h
 bindings/java/pom*.xml
 bindings/java/src*/main/overview.html
 bindings/java/src*/main/com/apple/foundationdb/NetworkOptions.java
@@ -113,4 +114,5 @@ temp/
 
 # AI assistants
 *.aider*
-*.fdq
+*.fdq
+.claude/

CMakeLists.txt

@@ -314,14 +314,11 @@ if(NOT WIN32)
   if(NOT FOUNDATIONDB_CROSS_COMPILING) # FIXME(swift): make this work when
                                        # x-compiling.
     add_subdirectory(fdbmonitor)
+    add_subdirectory(fdbkubernetesmonitor)
   endif()
 endif()
 add_subdirectory(fdbbackup)
 add_subdirectory(tests)
-if(NOT FOUNDATIONDB_CROSS_COMPILING) # FIXME(swift): make this work when
-                                     # x-compiling.
-  add_subdirectory(flowbench EXCLUDE_FROM_ALL)
-endif()
 if(WITH_PYTHON AND WITH_C_BINDING)
   if(NOT FOUNDATIONDB_CROSS_COMPILING) # FIXME(swift): make this work when
                                        # x-compiling.

README.md

@@ -140,17 +140,18 @@ Building FoundationDB requires at least 8GB of memory. More memory is needed whe
 
 ### macOS
 
-The build under macOS will work the same way as on Linux. [Homebrew](https://brew.sh/) can be used to install the `boost` library and the `ninja` build tool.
+The build under macOS will work the same way as on Linux. [Homebrew](https://brew.sh/) can be used to install the `boost` library and the `ninja` build tool. Be carefull, curent main branch use boost 1.86, do install this version or just let cmake download one. Also, if swift binding is not interest, use -DBUILD_SWIFT_BINDING=OFF.
 
 ```sh
-cmake -G Ninja <FDB_SOURCE_DIR>
+cmake -G Ninja <FDB_SOURCE_DIR> -B <BUILD_DIR>
+cd <BUILD_DIR>
 ninja
 ```
 
 To generate an installable package,
 
 ```sh
-<FDB_SOURCE_DIR>/packaging/osx/buildpkg.sh . <FDB_SOURCE_DIR>
+<FDB_SOURCE_DIR>/packaging/osx/buildpkg.sh <BUILD_DIR> <FDB_SOURCE_DIR>
 ```
 
 ### Windows
@@ -185,6 +186,10 @@ CMake will not produce a `compile_commands.json` by default; you must pass `-DCM
 
 Note that if the building is done inside the `foundationdb/build` Docker image, the resulting paths will still be incorrect and require manual fixing. One will wish to re-run `cmake` with `-DCMAKE_EXPORT_COMPILE_COMMANDS=OFF` to prevent it from reverting the manual changes.
 
+### Code Formatting and Static Analysis
+
+`clang-format` and `clang-tidy` run as part of CI on every pull request. See the [clang-format](https://apple.github.io/foundationdb/clang-format.html) and [clang-tidy](https://apple.github.io/foundationdb/clang-tidy.html) guides for how to run them locally before pushing.
+
 ### Using IDEs
 
 CMake provides built-in support for several popular IDEs. However, most FoundationDB files are written in the `flow` language, which is an extension of the C++ programming language,  for coroutine support (Note that when FoundationDB was being developed, C++20 was not available). The `flow` language will be transpiled into C++ code using `actorcompiler`, while preventing most IDEs from recognizing `flow`-specific syntax.

bindings/c/CMakeLists.txt

@@ -211,7 +211,7 @@ if(NOT WIN32)
   target_link_libraries(fdb_c_client_memory_test PRIVATE fdb_c Threads::Threads)
 
   target_include_directories(fdb_c_api_tester_impl PUBLIC ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR}/foundationdb/ ${CMAKE_SOURCE_DIR}/flow/include ${CMAKE_BINARY_DIR}/flow/include)
-  target_link_libraries(fdb_c_api_tester_impl PRIVATE fdb_cpp toml11_target Threads::Threads fmt::fmt boost_target)
+  target_link_libraries(fdb_c_api_tester_impl PRIVATE fdb_cpp toml11::toml11 Threads::Threads fmt::fmt boost_target)
   if(NOT CMAKE_CXX_COMPILER_ID MATCHES "Clang")
     target_link_libraries(fdb_c_api_tester_impl PRIVATE stdc++fs)
   endif()
@@ -230,6 +230,9 @@ if(NOT WIN32)
   # do not set RPATH for mako
   set_property(TARGET mako PROPERTY SKIP_BUILD_RPATH TRUE)
   target_link_libraries(mako PRIVATE fdb_c fdbclient fmt::fmt Threads::Threads fdb_cpp boost_target rapidjson)
+  if(NOT OPEN_FOR_IDE)
+    strip_debug_symbols(mako)
+  endif()
 
   if(NOT OPEN_FOR_IDE)
     # Make sure that fdb_c.h is compatible with c90

bindings/c/fdb_c.cpp

@@ -86,7 +86,7 @@ extern "C" DLLEXPORT fdb_bool_t fdb_error_predicate(int predicate_test, fdb_erro
 		       code == error_code_commit_proxy_memory_limit_exceeded ||
 		       code == error_code_transaction_throttled_hot_shard || code == error_code_batch_transaction_throttled ||
 		       code == error_code_process_behind || code == error_code_tag_throttled ||
-		       code == error_code_proxy_tag_throttled || code == error_code_transaction_rejected_range_locked;
+		       code == error_code_transaction_rejected_range_locked;
 	}
 	return false;
 }
@@ -222,7 +222,7 @@ class CAPICallback final : public ThreadCallback {
 extern "C" DLLEXPORT fdb_error_t fdb_future_set_callback(FDBFuture* f,
                                                          void (*callbackf)(FDBFuture*, void*),
                                                          void* userdata) {
-	CAPICallback* cb = new CAPICallback(callbackf, f, userdata);
+	auto* cb = new CAPICallback(callbackf, f, userdata);
 	int ignore;
 	CATCH_AND_RETURN(TSAVB(f)->callOrSetAsCallback(cb, ignore, 0););
 }

bindings/c/test/apitester/TesterApiWorkload.cpp

@@ -171,7 +171,7 @@ fdb::KeyRange ApiWorkload::randomNonEmptyKeyRange() {
 }
 
 std::optional<int> ApiWorkload::randomTenant() {
-	if (tenants.size() > 0) {
+	if (!tenants.empty()) {
 		return Random::get().randomInt(0, tenants.size() - 1);
 	} else {
 		return {};
@@ -244,15 +244,15 @@ void ApiWorkload::populateTenantData(TTaskFct cont, std::optional<int> tenantId)
 }
 
 void ApiWorkload::createTenantsIfNecessary(TTaskFct cont) {
-	if (tenants.size() > 0) {
+	if (!tenants.empty()) {
 		ASSERT(false);
 	} else {
 		schedule(cont);
 	}
 }
 
 void ApiWorkload::populateData(TTaskFct cont) {
-	if (tenants.size() > 0) {
+	if (!tenants.empty()) {
 		populateTenantData(cont, std::make_optional(0));
 	} else {
 		populateTenantData(cont, {});

bindings/c/test/apitester/TesterAtomicOpsCorrectnessWorkload.cpp

@@ -37,8 +37,8 @@ class AtomicOpsCorrectnessWorkload : public ApiWorkload {
 	AtomicOpsCorrectnessWorkload(const WorkloadConfig& config) : ApiWorkload(config) {}
 
 private:
-	typedef std::function<uint64_t(uint64_t, uint64_t)> IntAtomicOpFunction;
-	typedef std::function<Value(ValueRef, ValueRef)> AtomicOpFunction;
+	using IntAtomicOpFunction = std::function<uint64_t(uint64_t, uint64_t)>;
+	using AtomicOpFunction = std::function<Value(ValueRef, ValueRef)>;
 
 	enum OpType {
 		OP_ATOMIC_ADD,
@@ -57,7 +57,7 @@ class AtomicOpsCorrectnessWorkload : public ApiWorkload {
 	};
 
 	void randomOperation(TTaskFct cont) override {
-		OpType txType = (OpType)Random::get().randomInt(0, OP_LAST);
+		auto txType = OpType(Random::get().randomInt(0, OP_LAST));
 
 		switch (txType) {
 		case OP_ATOMIC_ADD:

bindings/c/test/apitester/TesterCorrectnessWorkload.cpp

@@ -248,7 +248,7 @@ class ApiCorrectnessWorkload : public ApiWorkload {
 				                      results->size()));
 			    } else {
 				    auto expected_kv = expected.begin();
-				    for (auto actual_kv : *results) {
+				    for (const auto& actual_kv : *results) {
 					    if (actual_kv.key != expected_kv->key || actual_kv.value != expected_kv->value) {
 						    error(fmt::format(
 						        "randomGetRangeOp mismatch. expected key: {} actual key: {} expected value: "
@@ -266,7 +266,7 @@ class ApiCorrectnessWorkload : public ApiWorkload {
 		    getTenant(tenantId));
 	}
 
-	void randomOperation(TTaskFct cont) {
+	void randomOperation(TTaskFct cont) override {
 		std::optional<int> tenantId = randomTenant();
 		OpType txType = (stores[tenantId].size() == 0) ? OP_INSERT : (OpType)Random::get().randomInt(0, OP_LAST);
 

bindings/c/test/apitester/TesterKeyValueStore.cpp

@@ -42,7 +42,7 @@ bool KeyValueStore::exists(fdb::KeyRef key) {
 fdb::Key KeyValueStore::getKey(fdb::KeyRef keyName, bool orEqual, int offset) const {
 	std::unique_lock<std::mutex> lock(mutex);
 	// Begin by getting the start key referenced by the key selector
-	std::map<fdb::Key, fdb::Value>::const_iterator mapItr = store.lower_bound(keyName);
+	auto mapItr = store.lower_bound(keyName);
 
 	// Update the iterator position if necessary based on the value of orEqual
 	int count = 0;
@@ -92,16 +92,16 @@ std::vector<fdb::KeyValue> KeyValueStore::getRange(fdb::KeyRef begin, fdb::KeyRe
 	std::unique_lock<std::mutex> lock(mutex);
 	std::vector<fdb::KeyValue> results;
 	if (!reverse) {
-		std::map<fdb::Key, fdb::Value>::const_iterator mapItr = store.lower_bound(begin);
-
-		for (; mapItr != store.end() && mapItr->first < end && results.size() < limit; mapItr++)
+		for (auto mapItr = store.lower_bound(begin);
+		     mapItr != store.end() && mapItr->first < end && results.size() < limit;
+		     mapItr++)
 			results.push_back(fdb::KeyValue{ mapItr->first, mapItr->second });
 	}
 
 	// Support for reverse getRange queries is supported, but not tested at this time.  This is because reverse range
 	// queries have been disallowed by the database at the API level
 	else {
-		std::map<fdb::Key, fdb::Value>::const_iterator mapItr = store.lower_bound(end);
+		auto mapItr = store.lower_bound(end);
 		if (mapItr == store.begin())
 			return results;
 

bindings/c/test/apitester/TesterTestSpec.cpp

@@ -137,7 +137,7 @@ TestSpec readTomlTestSpec(std::string fileName) {
 
 	// Then parse each test
 	const toml::array& tests = toml::find(conf, "test").as_array();
-	if (tests.size() == 0) {
+	if (tests.empty()) {
 		throw TesterError("Invalid test file. No [test] section found");
 	} else if (tests.size() > 1) {
 		throw TesterError("Invalid test file. More than one [test] section found");

bindings/c/test/apitester/TesterTransactionExecutor.cpp

@@ -97,7 +97,7 @@ class TransactionContextBase : public ITransactionContext {
 		}
 	}
 
-	virtual ~TransactionContextBase() { ASSERT(txState == TxState::DONE); }
+	~TransactionContextBase() override { ASSERT(txState == TxState::DONE); }
 
 	// A state machine:
 	// IN_PROGRESS -> (ON_ERROR -> IN_PROGRESS)* [-> ON_ERROR] -> DONE
@@ -165,7 +165,7 @@ class TransactionContextBase : public ITransactionContext {
 		}
 	}
 
-	virtual void onError(fdb::Error err) override {
+	void onError(fdb::Error err) override {
 		std::unique_lock<std::mutex> lock(mutex);
 		if (txState != TxState::IN_PROGRESS) {
 			// Ignore further errors, if the transaction is in the error handing mode or completed
@@ -438,7 +438,7 @@ class BlockingTransactionContext : public TransactionContextBase {
 		onError(err);
 	}
 
-	virtual void handleOnErrorFuture() override {
+	void handleOnErrorFuture() override {
 		ASSERT(txState == TxState::ON_ERROR);
 
 		auto start = timeNow();
@@ -503,7 +503,7 @@ class AsyncTransactionContext : public TransactionContextBase {
 
 	static void futureReadyCallback(fdb::Future f, void* param) {
 		try {
-			AsyncTransactionContext* txCtx = (AsyncTransactionContext*)param;
+			auto* txCtx = (AsyncTransactionContext*)param;
 			txCtx->onFutureReady(f);
 		} catch (std::exception& err) {
 			fmt::print("Unexpected exception in callback {}\n", err.what());
@@ -550,7 +550,7 @@ class AsyncTransactionContext : public TransactionContextBase {
 		onError(err);
 	}
 
-	virtual void handleOnErrorFuture() override {
+	void handleOnErrorFuture() override {
 		ASSERT(txState == TxState::ON_ERROR);
 
 		onErrorCallTimePoint = timeNow();
@@ -565,7 +565,7 @@ class AsyncTransactionContext : public TransactionContextBase {
 
 	static void onErrorReadyCallback(fdb::Future f, void* param) {
 		try {
-			AsyncTransactionContext* txCtx = (AsyncTransactionContext*)param;
+			auto* txCtx = (AsyncTransactionContext*)param;
 			txCtx->onErrorReady(f);
 		} catch (std::exception& err) {
 			fmt::print("Unexpected exception in callback {}\n", err.what());
@@ -642,7 +642,7 @@ class TransactionExecutorBase : public ITransactionExecutor {
 public:
 	TransactionExecutorBase(const TransactionExecutorOptions& options) : options(options), scheduler(nullptr) {}
 
-	~TransactionExecutorBase() {
+	~TransactionExecutorBase() override {
 		if (tamperClusterFileThread.joinable()) {
 			tamperClusterFileThread.join();
 		}

bindings/c/test/apitester/TesterUtil.h

@@ -136,7 +136,8 @@ KeyRangeArray copyKeyRangeArray(fdb::future_var::KeyRangeRefArray::Type array);
 static_assert(__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__, "Do not support non-little-endian systems");
 
 // Converts a little-endian encoded number into an integral type.
-template <class T, typename = std::enable_if_t<std::is_integral<T>::value>>
+template <class T>
+    requires(std::is_integral_v<T>)
 static T toInteger(fdb::BytesRef value) {
 	ASSERT(value.size() == sizeof(T));
 	T output;
@@ -145,7 +146,8 @@ static T toInteger(fdb::BytesRef value) {
 }
 
 // Converts an integral type to a little-endian encoded byte string.
-template <class T, typename = std::enable_if_t<std::is_integral<T>::value>>
+template <class T>
+    requires(std::is_integral_v<T>)
 static fdb::ByteString toByteString(T value) {
 	fdb::ByteString output(sizeof(T), 0);
 	memcpy(output.data(), (const uint8_t*)&value, sizeof(value));

bindings/c/test/apitester/TesterWorkload.cpp

@@ -204,14 +204,14 @@ void WorkloadManager::run() {
 	std::vector<std::shared_ptr<IWorkload>> initialWorkloads;
 	{
 		std::unique_lock<std::mutex> lock(mutex);
-		for (auto iter : workloads) {
+		for (const auto& iter : workloads) {
 			initialWorkloads.push_back(iter.second.ref);
 		}
 	}
-	for (auto iter : initialWorkloads) {
+	for (const auto& iter : initialWorkloads) {
 		iter->init(this);
 	}
-	for (auto iter : initialWorkloads) {
+	for (const auto& iter : initialWorkloads) {
 		iter->start();
 	}
 	scheduler->join();
@@ -265,7 +265,7 @@ void WorkloadManager::readControlInput(std::string pipeName) {
 	// Open in binary mode and read char-by-char to avoid
 	// any kind of buffering
 	FILE* f = fopen(pipeName.c_str(), "rb");
-	setbuf(f, NULL);
+	setbuf(f, nullptr);
 	std::string line;
 	while (true) {
 		int ch = fgetc(f);
@@ -291,7 +291,7 @@ void WorkloadManager::readControlInput(std::string pipeName) {
 
 void WorkloadManager::schedulePrintStatistics(int timeIntervalMs) {
 	statsTimer = scheduler->scheduleWithDelay(timeIntervalMs, [this, timeIntervalMs]() {
-		for (auto workload : getActiveWorkloads()) {
+		for (const auto& workload : getActiveWorkloads()) {
 			workload->printStats();
 		}
 		this->schedulePrintStatistics(timeIntervalMs);
@@ -301,7 +301,7 @@ void WorkloadManager::schedulePrintStatistics(int timeIntervalMs) {
 std::vector<std::shared_ptr<IWorkload>> WorkloadManager::getActiveWorkloads() {
 	std::unique_lock<std::mutex> lock(mutex);
 	std::vector<std::shared_ptr<IWorkload>> res;
-	for (auto iter : workloads) {
+	for (const auto& iter : workloads) {
 		res.push_back(iter.second.ref);
 	}
 	return res;

bindings/c/test/apitester/fdb_c_api_tester.cpp

@@ -325,7 +325,7 @@ void applyNetworkOptions(TesterOptions& options) {
 		fdb::network::setOption(FDBNetworkOption::FDB_NET_OPTION_RETAIN_CLIENT_LIBRARY_COPIES);
 	}
 
-	for (auto knob : options.testSpec.knobs) {
+	for (const auto& knob : options.testSpec.knobs) {
 		fmt::print(stderr, "Setting knob {}={}\n", knob.first.c_str(), knob.second.c_str());
 		fdb::network::setOption(FDBNetworkOption::FDB_NET_OPTION_KNOB,
 		                        fmt::format("{}={}", knob.first.c_str(), knob.second.c_str()));